diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 793805005de8b..421a239e3b0df 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -116,6 +116,7 @@ /x-pack/plugins/monitoring/ @elastic/infra-monitoring-ui /x-pack/test/functional/apps/monitoring @elastic/infra-monitoring-ui /x-pack/test/api_integration/apis/monitoring @elastic/infra-monitoring-ui +/x-pack/test/api_integration/apis/monitoring_collection @elastic/infra-monitoring-ui # Fleet /fleet_packages.json @elastic/fleet diff --git a/package.json b/package.json index b56a4d1e803aa..a0cab1288d620 100644 --- a/package.json +++ b/package.json @@ -120,6 +120,7 @@ "@emotion/css": "^11.9.0", "@emotion/react": "^11.9.0", "@emotion/serialize": "^1.0.3", + "@grpc/grpc-js": "^1.6.7", "@hapi/accept": "^5.0.2", "@hapi/boom": "^9.1.4", "@hapi/cookie": "^11.0.2", @@ -284,6 +285,13 @@ "@mapbox/mapbox-gl-draw": "1.3.0", "@mapbox/mapbox-gl-rtl-text": "0.2.3", "@mapbox/vector-tile": "1.3.1", + "@opentelemetry/api": "^1.1.0", + "@opentelemetry/api-metrics": "^0.30.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "^0.30.0", + "@opentelemetry/exporter-prometheus": "^0.30.0", + "@opentelemetry/resources": "^1.4.0", + "@opentelemetry/sdk-metrics-base": "^0.30.0", + "@opentelemetry/semantic-conventions": "^1.4.0", "@reduxjs/toolkit": "^1.6.1", "@slack/webhook": "^5.0.4", "@turf/along": "6.0.1", diff --git a/test/common/config.js b/test/common/config.js index 5079d32909ff5..f69afd4e789b5 100644 --- a/test/common/config.js +++ b/test/common/config.js @@ -51,6 +51,8 @@ export default function () { `--server.maxPayload=1679958`, // newsfeed mock service `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'newsfeed')}`, + // otel mock service + `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'otel_metrics')}`, `--newsfeed.service.urlRoot=${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`, `--newsfeed.service.pathTemplate=/api/_newsfeed-FTS-external-service-simulators/kibana/v{VERSION}.json`, // code coverage reporting plugin diff --git a/test/common/fixtures/plugins/otel_metrics/kibana.json b/test/common/fixtures/plugins/otel_metrics/kibana.json new file mode 100644 index 0000000000000..f9cc773c1fe0a --- /dev/null +++ b/test/common/fixtures/plugins/otel_metrics/kibana.json @@ -0,0 +1,15 @@ +{ + "id": "openTelemetryInstrumentedPlugin", + "owner": { + "name": "Stack Monitoring", + "githubTeam": "stack-monitoring-ui" + }, + "version": "1.0.0", + "kibanaVersion": "kibana", + "requiredPlugins": [ + "monitoringCollection" + ], + "optionalPlugins": [], + "server": true, + "ui": false +} \ No newline at end of file diff --git a/test/common/fixtures/plugins/otel_metrics/server/index.ts b/test/common/fixtures/plugins/otel_metrics/server/index.ts new file mode 100644 index 0000000000000..eb5f587592cae --- /dev/null +++ b/test/common/fixtures/plugins/otel_metrics/server/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { OpenTelemetryUsageTest } from './plugin'; + +export const plugin = () => new OpenTelemetryUsageTest(); diff --git a/test/common/fixtures/plugins/otel_metrics/server/monitoring/metrics.ts b/test/common/fixtures/plugins/otel_metrics/server/monitoring/metrics.ts new file mode 100644 index 0000000000000..044cd7bee5441 --- /dev/null +++ b/test/common/fixtures/plugins/otel_metrics/server/monitoring/metrics.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Counter, Meter } from '@opentelemetry/api-metrics'; + +export class Metrics { + requestCounter: Counter; + + constructor(meter: Meter) { + this.requestCounter = meter.createCounter('request_count', { + description: 'Counts total number of requests', + }); + } +} diff --git a/test/common/fixtures/plugins/otel_metrics/server/plugin.ts b/test/common/fixtures/plugins/otel_metrics/server/plugin.ts new file mode 100644 index 0000000000000..65dec472b94f9 --- /dev/null +++ b/test/common/fixtures/plugins/otel_metrics/server/plugin.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { CoreSetup, Plugin } from '@kbn/core/server'; +import { metrics } from '@opentelemetry/api-metrics'; +import { generateOtelMetrics } from './routes'; +import { Metrics } from './monitoring/metrics'; + +export class OpenTelemetryUsageTest implements Plugin { + private metrics: Metrics; + + constructor() { + this.metrics = new Metrics(metrics.getMeter('dummyMetric')); + } + + public setup(core: CoreSetup) { + const router = core.http.createRouter(); + generateOtelMetrics(router, this.metrics); + } + + public start() {} + public stop() {} +} diff --git a/test/common/fixtures/plugins/otel_metrics/server/routes/generate_otel_metrics.ts b/test/common/fixtures/plugins/otel_metrics/server/routes/generate_otel_metrics.ts new file mode 100644 index 0000000000000..6809059ca1472 --- /dev/null +++ b/test/common/fixtures/plugins/otel_metrics/server/routes/generate_otel_metrics.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { IKibanaResponse, IRouter } from '@kbn/core/server'; +import { Metrics } from '../monitoring/metrics'; + +export const generateOtelMetrics = (router: IRouter, metrics: Metrics) => { + router.post( + { + path: '/api/generate_otel_metrics', + validate: {}, + }, + async function (_context, _req, res): Promise> { + metrics.requestCounter.add(1); + return res.ok({}); + } + ); +}; diff --git a/test/common/fixtures/plugins/otel_metrics/server/routes/index.ts b/test/common/fixtures/plugins/otel_metrics/server/routes/index.ts new file mode 100644 index 0000000000000..49ac53bcf5412 --- /dev/null +++ b/test/common/fixtures/plugins/otel_metrics/server/routes/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export * from './generate_otel_metrics'; diff --git a/tsconfig.base.json b/tsconfig.base.json index 9946503830c70..c4a2d579dee9a 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -201,6 +201,8 @@ "@kbn/coverage-fixtures-plugin/*": ["test/common/fixtures/plugins/coverage/*"], "@kbn/newsfeed-fixtures-plugin": ["test/common/fixtures/plugins/newsfeed"], "@kbn/newsfeed-fixtures-plugin/*": ["test/common/fixtures/plugins/newsfeed/*"], + "@kbn/open-telemetry-instrumented-plugin": ["test/common/fixtures/plugins/otel_metrics"], + "@kbn/open-telemetry-instrumented-plugin/*": ["test/common/fixtures/plugins/otel_metrics/*"], "@kbn/kbn-tp-run-pipeline-plugin": ["test/interpreter_functional/plugins/kbn_tp_run_pipeline"], "@kbn/kbn-tp-run-pipeline-plugin/*": ["test/interpreter_functional/plugins/kbn_tp_run_pipeline/*"], "@kbn/app-link-test-plugin": ["test/plugin_functional/plugins/app_link_test"], diff --git a/x-pack/plugins/monitoring_collection/README.md b/x-pack/plugins/monitoring_collection/README.md index 1f2d2984af886..1f9cadf40ee7b 100644 --- a/x-pack/plugins/monitoring_collection/README.md +++ b/x-pack/plugins/monitoring_collection/README.md @@ -2,4 +2,139 @@ ## Plugin -This plugin allows for other plugins to add data to Kibana stack monitoring documents. \ No newline at end of file +This plugin allows for other plugins to add data to Kibana stack monitoring documents. + +## OpenTelemetry Metrics + +### Enable Prometheus endpoint with Elastic Agent Prometheus input + +1. Start [local setup with fleet](../fleet/README.md#running-fleet-server-locally-in-a-container) or a cloud cluster +2. Start Kibana +3. Set up a new agent policy and enroll a new agent in your local machine +4. Install the Prometheus Metrics package + 1. Set **Hosts** with `localhost:5601` + 2. Set **Metrics Path** with `/(BASEPATH)/api/monitoring_collection/v1/prometheus` + 3. Remove the values from **Bearer Token File** and **SSL Certificate Authorities** + 4. Set **Username** and **Password** with `elastic` and `changeme` +5. Add the following configuration to `kibana.dev.yml` + + ```yml + # Enable the prometheus exporter + monitoring_collection.opentelemetry.metrics: + prometheus.enabled: true + ``` + +### Enable OpenTelemetry Metrics API exported as OpenTelemetry Protocol over GRPC + +1. Start [local setup with fleet](../fleet/README.md#running-fleet-server-locally-in-a-container) or a cloud cluster +2. Start Kibana +3. Set up a new agent policy and enroll a new agent in your local machine +4. Install Elastic APM package listening on `localhost:8200` without authentication +5. Add the following configuration to `kibana.dev.yml` + + ```yml + # Enable the OTLP exporter + monitoring_collection.opentelemetry.metrics: + otlp.url: "http://127.0.0.1:8200" + ``` + +You can also provide headers for OTLP endpoints that require authentication: + +```yml +# Enable the OTLP exporter to an authenticated APM endpoint +monitoring_collection.opentelemetry.metrics: + otlp: + url: "https://DEPLOYMENT.apm.REGION.PROVIDER.elastic-cloud.com" + headers: + Authorization: "Bearer SECRET_TOKEN" +``` + +Alternatively, OTLP Exporter can be configured using environment variables `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` and `OTEL_EXPORTER_OTLP_METRICS_HEADERS`. [See OTLP Exporter docs](https://opentelemetry.io/docs/reference/specification/protocol/exporter/) for details. + +It's possible to configure logging for the OTLP integration. If not informed, the default will be `info` + +```yml +monitoring_collection.opentelemetry.metrics: + logLevel: warn | info | debug | warn | none | verbose | all +``` + +For connection-level debug information you can set these variables: + +```bash +export GRPC_NODE_TRACE="xds_client,xds_resolver,cds_balancer,eds_balancer,priority,weighted_target,round_robin,resolving_load_balancer,subchannel,keepalive,dns_resolver,fault_injection,http_filter,csds" +export GRPC_NODE_VERBOSITY=DEBUG +``` + +See the [grpc-node docs](https://github.com/grpc/grpc-node/blob/master/doc/environment_variables.md) for details and other settings. + +### Example of how to instrument the code + +* First, we need to define what metrics we want to instrument with OpenTelemetry + + ```ts + import { Counter, Meter } from '@opentelemetry/api-metrics'; + + export class FooApiMeters { + requestCount: Counter; + + constructor(meter: Meter) { + this.requestCount = meter.createCounter('request_count', { + description: 'Counts total number of requests', + }); + } + } + ``` + + In this example we're using a `Counter` metric, but [OpenTelemetry SDK](https://open-telemetry.github.io/opentelemetry-js/interfaces/_opentelemetry_api_metrics.Meter.html) provides there are other options to record metrics + +* Initialize meter in the plugin setup and pass it to the relevant components that will be instrumented. In this case, we want to instrument `FooApi` routes. + + ```ts + import { IRouter } from '@kbn/core/server'; + import { FooApiMeters } from './foo_api_meters'; + import { metrics } from '@opentelemetry/api-metrics'; + + export class FooApiPlugin implements Plugin { + private metrics: Metrics; + private libs: { router: IRouter, metrics: FooApiMeters}; + + constructor() { + this.metrics = new Metrics(metrics.getMeter('kibana.fooApi')); + } + + public setup(core: CoreSetup) { + const router = core.http.createRouter(); + + this.libs = { + router, + metrics: this.metrics + } + + initMetricsAPIRoute(this.libs); + } + } + ``` + + `monitoring_collection` plugins has to be initialized before the plugin that will be instrumented. If for some reason the instrumentation doesn't record any metrics, make sure `monitoring_collection` is included in the list of `requiredPlugins`. e.g: + + ```json + "requiredPlugins": [ + "monitoringCollection" + ], + ``` + +* Lastly we can use the `metrics` object to instrument the code + + ```ts + export const initMetricsAPIRoute = (libs: { router: IRouter, metrics: FooApiMeters}) => { + router.get( + { + path: '/api/foo', + validate: {}, + }, + async function (_context, _req, res) { + metrics.requestCount.add(1); + return res.ok({}); + } + ); + ``` \ No newline at end of file diff --git a/x-pack/plugins/monitoring_collection/server/config.ts b/x-pack/plugins/monitoring_collection/server/config.ts index 275d2f31e505d..5eda950ebe7f1 100644 --- a/x-pack/plugins/monitoring_collection/server/config.ts +++ b/x-pack/plugins/monitoring_collection/server/config.ts @@ -9,6 +9,19 @@ import { schema, TypeOf } from '@kbn/config-schema'; export const configSchema = schema.object({ enabled: schema.boolean({ defaultValue: true }), + opentelemetry: schema.object({ + metrics: schema.object({ + otlp: schema.object({ + url: schema.maybe(schema.string()), + headers: schema.maybe(schema.recordOf(schema.string(), schema.string())), + exportIntervalMillis: schema.number({ defaultValue: 10000 }), + logLevel: schema.string({ defaultValue: 'info' }), + }), + prometheus: schema.object({ + enabled: schema.boolean({ defaultValue: false }), + }), + }), + }), }); export type MonitoringCollectionConfig = ReturnType; diff --git a/x-pack/plugins/monitoring_collection/server/constants.ts b/x-pack/plugins/monitoring_collection/server/constants.ts index 86231dec6c6c2..92b43a9d80e48 100644 --- a/x-pack/plugins/monitoring_collection/server/constants.ts +++ b/x-pack/plugins/monitoring_collection/server/constants.ts @@ -5,3 +5,5 @@ * 2.0. */ export const TYPE_ALLOWLIST = ['node_rules', 'cluster_rules', 'node_actions', 'cluster_actions']; + +export const MONITORING_COLLECTION_BASE_PATH = '/api/monitoring_collection'; diff --git a/x-pack/plugins/monitoring_collection/server/lib/index.ts b/x-pack/plugins/monitoring_collection/server/lib/index.ts index 0c39a62ab359c..34c1fce763bdc 100644 --- a/x-pack/plugins/monitoring_collection/server/lib/index.ts +++ b/x-pack/plugins/monitoring_collection/server/lib/index.ts @@ -7,3 +7,4 @@ export { getKibanaStats } from './get_kibana_stats'; export { getESClusterUuid } from './get_es_cluster_uuid'; +export { PrometheusExporter } from './prometheus_exporter'; diff --git a/x-pack/plugins/monitoring_collection/server/lib/prometheus_exporter.ts b/x-pack/plugins/monitoring_collection/server/lib/prometheus_exporter.ts new file mode 100644 index 0000000000000..fc4359609bf34 --- /dev/null +++ b/x-pack/plugins/monitoring_collection/server/lib/prometheus_exporter.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AggregationTemporality, MetricReader } from '@opentelemetry/sdk-metrics-base'; +import { + PrometheusExporter as OpenTelemetryPrometheusExporter, + ExporterConfig, + PrometheusSerializer, +} from '@opentelemetry/exporter-prometheus'; +import { KibanaResponseFactory } from '@kbn/core/server'; + +export class PrometheusExporter extends MetricReader { + private readonly prefix?: string; + private readonly appendTimestamp: boolean; + private serializer: PrometheusSerializer; + + constructor(config: ExporterConfig = {}) { + super(); + this.prefix = config.prefix || OpenTelemetryPrometheusExporter.DEFAULT_OPTIONS.prefix; + this.appendTimestamp = + typeof config.appendTimestamp === 'boolean' + ? config.appendTimestamp + : OpenTelemetryPrometheusExporter.DEFAULT_OPTIONS.appendTimestamp; + + this.serializer = new PrometheusSerializer(this.prefix, this.appendTimestamp); + } + + selectAggregationTemporality(): AggregationTemporality { + return AggregationTemporality.CUMULATIVE; + } + + protected onForceFlush(): Promise { + return Promise.resolve(undefined); + } + + protected onShutdown(): Promise { + return Promise.resolve(undefined); + } + + /** + * Responds to incoming message with current state of all metrics. + */ + public async exportMetrics(res: KibanaResponseFactory) { + try { + const collectionResult = await this.collect(); + const { resourceMetrics, errors } = collectionResult; + if (errors.length) { + return res.customError({ + statusCode: 500, + body: `PrometheusExporter: Metrics collection errors ${errors}`, + }); + } + const result = this.serializer.serialize(resourceMetrics); + if (result === '') { + return res.noContent(); + } + return res.ok({ + body: result, + }); + } catch (error) { + return res.customError({ + statusCode: 500, + body: { + message: `PrometheusExporter: Failed to export metrics ${error}`, + }, + }); + } + } +} diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index e1c3a5064a579..1c30a8439cf3c 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -6,10 +6,24 @@ */ import { JsonObject } from '@kbn/utility-types'; -import { CoreSetup, Plugin, PluginInitializerContext, Logger } from '@kbn/core/server'; +import { + CoreSetup, + Plugin, + PluginInitializerContext, + Logger, + ServiceStatus, +} from '@kbn/core/server'; import { MakeSchemaFrom } from '@kbn/usage-collection-plugin/server'; -import { ServiceStatus } from '@kbn/core/server'; -import { registerDynamicRoute } from './routes'; +import { metrics } from '@opentelemetry/api-metrics'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; +import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics-base'; +import { Resource } from '@opentelemetry/resources'; +import { diag, DiagLogger, DiagLogLevel } from '@opentelemetry/api'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; +import * as grpc from '@grpc/grpc-js'; +import { PrometheusExporter } from './lib/prometheus_exporter'; +import { MonitoringCollectionConfig } from './config'; +import { registerDynamicRoute, registerV1PrometheusRoute, PROMETHEUS_PATH } from './routes'; import { TYPE_ALLOWLIST } from './constants'; export interface MonitoringCollectionSetup { @@ -27,12 +41,25 @@ export interface Metric { export class MonitoringCollectionPlugin implements Plugin { private readonly initializerContext: PluginInitializerContext; private readonly logger: Logger; + private readonly config: MonitoringCollectionConfig; + private readonly otlpLogger: DiagLogger; private metrics: Record> = {}; - constructor(initializerContext: PluginInitializerContext) { + private prometheusExporter?: PrometheusExporter; + + constructor(initializerContext: PluginInitializerContext) { this.initializerContext = initializerContext; this.logger = initializerContext.logger.get(); + this.config = initializerContext.config.get(); + + this.otlpLogger = { + debug: (message) => this.logger.debug(message), + error: (message) => this.logger.error(message), + info: (message) => this.logger.info(message), + warn: (message) => this.logger.warn(message), + verbose: (message) => this.logger.trace(message), + }; } async getMetric(type: string) { @@ -46,19 +73,28 @@ export class MonitoringCollectionPlugin implements Plugin; core.status.overall$.subscribe((newStatus) => { status = newStatus; }); + if (this.prometheusExporter) { + registerV1PrometheusRoute({ router, prometheusExporter: this.prometheusExporter }); + } + registerDynamicRoute({ router, config: { kibanaIndex, - kibanaVersion: this.initializerContext.env.packageInfo.version, - server: core.http.getServerInfo(), - uuid: this.initializerContext.env.instanceUuid, + kibanaVersion, + server, + uuid, }, getStatus: () => status, getMetric: async (type: string) => { @@ -85,6 +121,58 @@ export class MonitoringCollectionPlugin implements Plugin { jest.resetAllMocks(); }); -jest.mock('../lib', () => ({ +jest.mock('../../../../lib', () => ({ getESClusterUuid: () => 'clusterA', getKibanaStats: () => ({ name: 'myKibana' }), })); diff --git a/x-pack/plugins/monitoring_collection/server/routes/dynamic_route.ts b/x-pack/plugins/monitoring_collection/server/routes/api/v1/dynamic_route/get_metrics_by_type.ts similarity index 86% rename from x-pack/plugins/monitoring_collection/server/routes/dynamic_route.ts rename to x-pack/plugins/monitoring_collection/server/routes/api/v1/dynamic_route/get_metrics_by_type.ts index 944037dd17a7b..4d18eeb6ec922 100644 --- a/x-pack/plugins/monitoring_collection/server/routes/dynamic_route.ts +++ b/x-pack/plugins/monitoring_collection/server/routes/api/v1/dynamic_route/get_metrics_by_type.ts @@ -7,8 +7,9 @@ import { JsonObject } from '@kbn/utility-types'; import { schema } from '@kbn/config-schema'; import { IRouter, ServiceStatus } from '@kbn/core/server'; -import { getESClusterUuid, getKibanaStats } from '../lib'; -import { MetricResult } from '../plugin'; +import { getESClusterUuid, getKibanaStats } from '../../../../lib'; +import { MetricResult } from '../../../../plugin'; +import { MONITORING_COLLECTION_BASE_PATH } from '../../../../constants'; export function registerDynamicRoute({ router, @@ -34,7 +35,7 @@ export function registerDynamicRoute({ }) { router.get( { - path: `/api/monitoring_collection/{type}`, + path: `${MONITORING_COLLECTION_BASE_PATH}/{type}`, options: { authRequired: true, tags: ['api'], // ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page diff --git a/x-pack/plugins/monitoring_collection/server/routes/api/v1/dynamic_route/index.ts b/x-pack/plugins/monitoring_collection/server/routes/api/v1/dynamic_route/index.ts new file mode 100644 index 0000000000000..973d525b9a77b --- /dev/null +++ b/x-pack/plugins/monitoring_collection/server/routes/api/v1/dynamic_route/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export * from './get_metrics_by_type'; diff --git a/x-pack/plugins/monitoring_collection/server/routes/api/v1/index.ts b/x-pack/plugins/monitoring_collection/server/routes/api/v1/index.ts new file mode 100644 index 0000000000000..e5a70f3f79abc --- /dev/null +++ b/x-pack/plugins/monitoring_collection/server/routes/api/v1/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { registerDynamicRoute } from './dynamic_route'; +export { registerV1PrometheusRoute, PROMETHEUS_PATH } from './prometheus'; diff --git a/x-pack/plugins/monitoring_collection/server/routes/api/v1/prometheus/get_metrics.test.ts b/x-pack/plugins/monitoring_collection/server/routes/api/v1/prometheus/get_metrics.test.ts new file mode 100644 index 0000000000000..b136d982992c4 --- /dev/null +++ b/x-pack/plugins/monitoring_collection/server/routes/api/v1/prometheus/get_metrics.test.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandlerContext } from '@kbn/core/server'; +import { httpServerMock, httpServiceMock } from '@kbn/core/server/mocks'; +import { registerV1PrometheusRoute } from '.'; +import { PrometheusExporter } from '../../../../lib'; + +describe('Prometheus route', () => { + it('forwards the request to the prometheus exporter', async () => { + const router = httpServiceMock.createRouter(); + const prometheusExporter = { + exportMetrics: jest.fn(), + } as Partial as PrometheusExporter; + + registerV1PrometheusRoute({ router, prometheusExporter }); + + const [, handler] = router.get.mock.calls[0]; + + const context = {} as jest.Mocked; + const req = httpServerMock.createKibanaRequest(); + const factory = httpServerMock.createResponseFactory(); + + await handler(context, req, factory); + + expect(prometheusExporter.exportMetrics).toHaveBeenCalledWith(factory); + }); +}); diff --git a/x-pack/plugins/monitoring_collection/server/routes/api/v1/prometheus/get_metrics.ts b/x-pack/plugins/monitoring_collection/server/routes/api/v1/prometheus/get_metrics.ts new file mode 100644 index 0000000000000..6977be155a4fb --- /dev/null +++ b/x-pack/plugins/monitoring_collection/server/routes/api/v1/prometheus/get_metrics.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IRouter } from '@kbn/core/server'; +import { MONITORING_COLLECTION_BASE_PATH } from '../../../../constants'; +import { PrometheusExporter } from '../../../../lib'; + +export const PROMETHEUS_PATH = `${MONITORING_COLLECTION_BASE_PATH}/v1/prometheus`; +export function registerV1PrometheusRoute({ + router, + prometheusExporter, +}: { + router: IRouter; + prometheusExporter: PrometheusExporter; +}) { + router.get( + { + path: PROMETHEUS_PATH, + options: { + authRequired: true, + tags: ['api'], // ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page + }, + validate: {}, + }, + async (_context, _req, res) => { + return prometheusExporter.exportMetrics(res); + } + ); +} diff --git a/x-pack/plugins/monitoring_collection/server/routes/api/v1/prometheus/index.ts b/x-pack/plugins/monitoring_collection/server/routes/api/v1/prometheus/index.ts new file mode 100644 index 0000000000000..5b99f51c94511 --- /dev/null +++ b/x-pack/plugins/monitoring_collection/server/routes/api/v1/prometheus/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export * from './get_metrics'; diff --git a/x-pack/plugins/monitoring_collection/server/routes/index.ts b/x-pack/plugins/monitoring_collection/server/routes/index.ts index eb96ce19f764e..29cd177990593 100644 --- a/x-pack/plugins/monitoring_collection/server/routes/index.ts +++ b/x-pack/plugins/monitoring_collection/server/routes/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export { registerDynamicRoute } from './dynamic_route'; +export { registerV1PrometheusRoute, PROMETHEUS_PATH, registerDynamicRoute } from './api/v1'; diff --git a/x-pack/test/api_integration/apis/index.ts b/x-pack/test/api_integration/apis/index.ts index 6bec2ebe80a13..46b10af2a52b3 100644 --- a/x-pack/test/api_integration/apis/index.ts +++ b/x-pack/test/api_integration/apis/index.ts @@ -36,5 +36,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./watcher')); loadTestFile(require.resolve('./logs_ui')); loadTestFile(require.resolve('./osquery')); + loadTestFile(require.resolve('./monitoring_collection')); }); } diff --git a/x-pack/test/api_integration/apis/monitoring_collection/index.ts b/x-pack/test/api_integration/apis/monitoring_collection/index.ts new file mode 100644 index 0000000000000..e89bd44963c03 --- /dev/null +++ b/x-pack/test/api_integration/apis/monitoring_collection/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Monitoring Collection', function taskManagerSuite() { + loadTestFile(require.resolve('./prometheus')); + }); +} diff --git a/x-pack/test/api_integration/apis/monitoring_collection/prometheus.ts b/x-pack/test/api_integration/apis/monitoring_collection/prometheus.ts new file mode 100644 index 0000000000000..0ac13dda92cb5 --- /dev/null +++ b/x-pack/test/api_integration/apis/monitoring_collection/prometheus.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + + describe('Prometheus endpoint', () => { + it('returns prometheus scraped metrics', async () => { + await supertest.post('/api/generate_otel_metrics').set('kbn-xsrf', 'foo').expect(200); + const response = await supertest.get('/api/monitoring_collection/v1/prometheus').expect(200); + + expect(response.text.replace(/\s+/g, ' ')).to.match( + /^# HELP request_count_total Counts total number of requests # TYPE request_count_total counter request_count_total [0-9]/ + ); + }); + }); +} diff --git a/x-pack/test/api_integration/config.ts b/x-pack/test/api_integration/config.ts index 8cc5fb6f57d42..ca3795e812ee2 100644 --- a/x-pack/test/api_integration/config.ts +++ b/x-pack/test/api_integration/config.ts @@ -37,6 +37,7 @@ export async function getApiIntegrationConfig({ readConfigFile }: FtrConfigProvi '--xpack.ruleRegistry.write.cache.enabled=false', '--xpack.uptime.service.password=test', '--xpack.uptime.service.username=localKibanaIntegrationTestsUser', + '--monitoring_collection.opentelemetry.metrics.prometheus.enabled=true', ], }, esTestCluster: { diff --git a/yarn.lock b/yarn.lock index 47a0d08a5b1c7..7a09d8caa1dfb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1997,6 +1997,25 @@ resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw== +"@grpc/grpc-js@^1.5.9", "@grpc/grpc-js@^1.6.7": + version "1.6.7" + resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.6.7.tgz#4c4fa998ff719fe859ac19fe977fdef097bb99aa" + integrity sha512-eBM03pu9hd3VqDQG+kHahiG1x80RGkkqqRb1Pchcwqej/KkAH95gAvKs6laqaHCycYaPK+TKuNQnOz9UXYA8qw== + dependencies: + "@grpc/proto-loader" "^0.6.4" + "@types/node" ">=12.12.47" + +"@grpc/proto-loader@^0.6.4", "@grpc/proto-loader@^0.6.9": + version "0.6.13" + resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.6.13.tgz#008f989b72a40c60c96cd4088522f09b05ac66bc" + integrity sha512-FjxPYDRTn6Ec3V0arm1FtSpmP6V50wuph2yILpyvTKzjc76oDdoihXqM1DzOW5ubvCC8GivfCnNtfaRE8myJ7g== + dependencies: + "@types/long" "^4.0.1" + lodash.camelcase "^4.3.0" + long "^4.0.0" + protobufjs "^6.11.3" + yargs "^16.2.0" + "@gulp-sourcemaps/identity-map@1.X": version "1.0.2" resolved "https://registry.yarnpkg.com/@gulp-sourcemaps/identity-map/-/identity-map-1.0.2.tgz#1e6fe5d8027b1f285dc0d31762f566bccd73d5a9" @@ -4431,11 +4450,120 @@ "@mattiasbuelens/web-streams-adapter" "~0.1.0" web-streams-polyfill "~3.0.3" -"@opentelemetry/api@^1.1.0": +"@opentelemetry/api-metrics@0.30.0", "@opentelemetry/api-metrics@^0.30.0": + version "0.30.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api-metrics/-/api-metrics-0.30.0.tgz#b5defd10756e81d1c7ce8669ff8a8d2465ba0be8" + integrity sha512-jSb7iiYPY+DSUKIyzfGt0a5K1QGzWY5fSWtUB8Alfi27NhQGHBeuYYC5n9MaBP/HNWw5GpEIhXGEYCF9Pf8IEg== + dependencies: + "@opentelemetry/api" "^1.0.0" + +"@opentelemetry/api@^1.0.0", "@opentelemetry/api@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.1.0.tgz#563539048255bbe1a5f4f586a4a10a1bb737f44a" integrity sha512-hf+3bwuBwtXsugA2ULBc95qxrOqP2pOekLz34BJhcAKawt94vfeNyUKpYc0lZQ/3sCP6LqRa7UAdHA7i5UODzQ== +"@opentelemetry/core@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-1.4.0.tgz#26839ab9e36583a174273a1e1c5b33336c163725" + integrity sha512-faq50VFEdyC7ICAOlhSi+yYZ+peznnGjTJToha9R63i9fVopzpKrkZt7AIdXUmz2+L2OqXrcJs7EIdN/oDyr5w== + dependencies: + "@opentelemetry/semantic-conventions" "1.4.0" + +"@opentelemetry/exporter-metrics-otlp-grpc@^0.30.0": + version "0.30.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-metrics-otlp-grpc/-/exporter-metrics-otlp-grpc-0.30.0.tgz#4117d07b94302ef407dc7625a1b599de308c5476" + integrity sha512-02WEAA3X7A6qveCYISr6mvg8eKl9NeNdZytQiAexzAIItW/ncN3mxmbuf8VVZHNPBe6osisSzxhPpFH3G6Gh+w== + dependencies: + "@grpc/grpc-js" "^1.5.9" + "@grpc/proto-loader" "^0.6.9" + "@opentelemetry/core" "1.4.0" + "@opentelemetry/exporter-metrics-otlp-http" "0.30.0" + "@opentelemetry/otlp-grpc-exporter-base" "0.30.0" + "@opentelemetry/otlp-transformer" "0.30.0" + "@opentelemetry/resources" "1.4.0" + "@opentelemetry/sdk-metrics-base" "0.30.0" + +"@opentelemetry/exporter-metrics-otlp-http@0.30.0": + version "0.30.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.30.0.tgz#9d87e4c3e796e14109ac83e6d4ce5bad215c2a1e" + integrity sha512-2NFR/D9jih1TtEnEyD7oIMR47yb9Kuy5v2x+Fu19vv2gTf1HOhdA+LT4SpkxH+dUixEnDw8n11XBIa/uhNfq3Q== + dependencies: + "@opentelemetry/api-metrics" "0.30.0" + "@opentelemetry/core" "1.4.0" + "@opentelemetry/otlp-exporter-base" "0.30.0" + "@opentelemetry/otlp-transformer" "0.30.0" + "@opentelemetry/resources" "1.4.0" + "@opentelemetry/sdk-metrics-base" "0.30.0" + +"@opentelemetry/exporter-prometheus@^0.30.0": + version "0.30.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.30.0.tgz#f81322d3cb000170e716bc76820600d5649be538" + integrity sha512-y0SXvpzoKR+Tk/UL6F1f7vAcCzqpCDP/cTEa+Z7sX57aEG0HDXLQiLmAgK/BHqcEN5MFQMZ+MDVDsUrvpa6/Jw== + dependencies: + "@opentelemetry/api-metrics" "0.30.0" + "@opentelemetry/core" "1.4.0" + "@opentelemetry/sdk-metrics-base" "0.30.0" + +"@opentelemetry/otlp-exporter-base@0.30.0": + version "0.30.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.30.0.tgz#5f278b3529d38311dbdfc1ebcb764f5e5126e548" + integrity sha512-+dJnj2MSd3tsk+ooEw+0bF+dJs/NjGEVnCB3/FYxnUFaW9cCBbQQyt6X3YQYtYrEx4EEiTlwrW8pUpB1tsup7A== + dependencies: + "@opentelemetry/core" "1.4.0" + +"@opentelemetry/otlp-grpc-exporter-base@0.30.0": + version "0.30.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.30.0.tgz#3fa07667ddf604a028583a2a138b8b4ba8fa9bb0" + integrity sha512-86fuhZ7Z2un3L5Kd7jbH1oEn92v9DD92teErnYRXqYB/qyO61OLxaY6WxH9KOjmbs5CgCdLQ5bvED3wWDe3r7w== + dependencies: + "@grpc/grpc-js" "^1.5.9" + "@grpc/proto-loader" "^0.6.9" + "@opentelemetry/core" "1.4.0" + "@opentelemetry/otlp-exporter-base" "0.30.0" + +"@opentelemetry/otlp-transformer@0.30.0": + version "0.30.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-transformer/-/otlp-transformer-0.30.0.tgz#d81e1ae68dfb31d66cd4ca03ca965cdaa2e2b288" + integrity sha512-BTLXyBPBlCQCG4tXYZjlso4pT+gGpnTjzkFYTPYs52fO5DMWvYHlV8ST/raOIqX7wsamiH2zeqJ9W91017MtdA== + dependencies: + "@opentelemetry/api-metrics" "0.30.0" + "@opentelemetry/core" "1.4.0" + "@opentelemetry/resources" "1.4.0" + "@opentelemetry/sdk-metrics-base" "0.30.0" + "@opentelemetry/sdk-trace-base" "1.4.0" + +"@opentelemetry/resources@1.4.0", "@opentelemetry/resources@^1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-1.4.0.tgz#5e23b0d7976158861059dec17e0ee36a35a5ab85" + integrity sha512-Q3pI5+pCM+Ur7YwK9GbG89UBipwJbfmuzSPAXTw964ZHFzSrz+JAgrETC9rqsUOYdUlj/V7LbRMG5bo72xE0Xw== + dependencies: + "@opentelemetry/core" "1.4.0" + "@opentelemetry/semantic-conventions" "1.4.0" + +"@opentelemetry/sdk-metrics-base@0.30.0", "@opentelemetry/sdk-metrics-base@^0.30.0": + version "0.30.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-metrics-base/-/sdk-metrics-base-0.30.0.tgz#242d9260a89a1ac2bf1e167b3fda758f3883c769" + integrity sha512-3BDg1MYDInDyGvy+bSH8OuCX5nsue7omH6Y2eidCGTTDYRPxDmq9tsRJxnTUepoMAvWX+1sTwZ4JqTFmc1z8Mw== + dependencies: + "@opentelemetry/api-metrics" "0.30.0" + "@opentelemetry/core" "1.4.0" + "@opentelemetry/resources" "1.4.0" + lodash.merge "4.6.2" + +"@opentelemetry/sdk-trace-base@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.4.0.tgz#e54d09c1258cd53d3fe726053ed1cbda9d74f023" + integrity sha512-l7EEjcOgYlKWK0hfxz4Jtkkk2DuGiqBDWmRZf7g2Is9RVneF1IgcrbYZTKGaVfBKA7lPuVtUiQ2qTv3R+dKJrw== + dependencies: + "@opentelemetry/core" "1.4.0" + "@opentelemetry/resources" "1.4.0" + "@opentelemetry/semantic-conventions" "1.4.0" + +"@opentelemetry/semantic-conventions@1.4.0", "@opentelemetry/semantic-conventions@^1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.4.0.tgz#facf2c67d6063b9918d5a5e3fdf25f3a30d547b6" + integrity sha512-Hzl8soGpmyzja9w3kiFFcYJ7n5HNETpplY6cb67KR4QPlxp4FTTresO06qXHgHDhyIInmbLJXuwARjjpsKYGuQ== + "@percy/agent@^0.28.6": version "0.28.6" resolved "https://registry.yarnpkg.com/@percy/agent/-/agent-0.28.6.tgz#b220fab6ddcf63ae4e6c343108ba6955a772ce1c" @@ -7363,6 +7491,11 @@ resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9" integrity sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w== +"@types/long@^4.0.1": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" + integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA== + "@types/lru-cache@^5.1.0": version "5.1.0" resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.0.tgz#57f228f2b80c046b4a1bd5cac031f81f207f4f03" @@ -7530,7 +7663,7 @@ dependencies: "@types/node" "*" -"@types/node@*", "@types/node@12.20.24", "@types/node@16.11.41", "@types/node@>= 8", "@types/node@>=8.9.0", "@types/node@^10.1.0", "@types/node@^14.0.10", "@types/node@^14.14.31": +"@types/node@*", "@types/node@12.20.24", "@types/node@16.11.41", "@types/node@>= 8", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@>=8.9.0", "@types/node@^10.1.0", "@types/node@^14.0.10", "@types/node@^14.14.31": version "16.11.41" resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.41.tgz#88eb485b1bfdb4c224d878b7832239536aa2f813" integrity sha512-mqoYK2TnVjdkGk8qXAVGc/x9nSaTpSrFaGFm43BUH3IdoBV0nta6hYaGmdOvIMlbHJbUEVen3gvwpwovAZKNdQ== @@ -23953,6 +24086,25 @@ protobufjs@6.8.8: "@types/node" "^10.1.0" long "^4.0.0" +protobufjs@^6.11.3: + version "6.11.3" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.11.3.tgz#637a527205a35caa4f3e2a9a4a13ddffe0e7af74" + integrity sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/long" "^4.0.1" + "@types/node" ">=13.7.0" + long "^4.0.0" + protocol-buffers-schema@^3.3.1: version "3.3.2" resolved "https://registry.yarnpkg.com/protocol-buffers-schema/-/protocol-buffers-schema-3.3.2.tgz#00434f608b4e8df54c59e070efeefc37fb4bb859" @@ -30903,7 +31055,7 @@ yargs-unparser@2.0.0: flat "^5.0.2" is-plain-obj "^2.1.0" -yargs@16.2.0: +yargs@16.2.0, yargs@^16.2.0: version "16.2.0" resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66" integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==