[Stack Monitoring] Add OpenTelemetry metrics to Monitoring Collection plugin

.github/CODEOWNERS

@@ -115,6 +115,7 @@
 /x-pack/plugins/monitoring/ @elastic/infra-monitoring-ui
 /x-pack/test/functional/apps/monitoring @elastic/infra-monitoring-ui
 /x-pack/test/api_integration/apis/monitoring @elastic/infra-monitoring-ui
+/x-pack/test/api_integration/apis/monitoring-collection @elastic/infra-monitoring-ui
 
 # Fleet
 /fleet_packages.json @elastic/fleet

package.json

@@ -120,6 +120,7 @@
     "@emotion/css": "^11.9.0",
     "@emotion/react": "^11.9.0",
     "@emotion/serialize": "^1.0.3",
+    "@grpc/grpc-js": "^1.5.9",
     "@hapi/accept": "^5.0.2",
     "@hapi/boom": "^9.1.4",
     "@hapi/cookie": "^11.0.2",
@@ -263,6 +264,12 @@
     "@mapbox/mapbox-gl-draw": "1.3.0",
     "@mapbox/mapbox-gl-rtl-text": "0.2.3",
     "@mapbox/vector-tile": "1.3.1",
+    "@opentelemetry/api-metrics": "0.30.0",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.30.0",
+    "@opentelemetry/exporter-prometheus": "^0.30.0",
+    "@opentelemetry/resources": "^1.3.1",
+    "@opentelemetry/sdk-metrics-base": "^0.29.2",
+    "@opentelemetry/semantic-conventions": "^1.3.1",
     "@reduxjs/toolkit": "^1.6.1",
     "@slack/webhook": "^5.0.4",
     "@turf/along": "6.0.1",

test/common/config.js

@@ -51,6 +51,8 @@ export default function () {
         `--server.maxPayload=1679958`,
         // newsfeed mock service
         `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'newsfeed')}`,
+        // otel mock service
+        `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'otel_metrics')}`,
         `--newsfeed.service.urlRoot=${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`,
         `--newsfeed.service.pathTemplate=/api/_newsfeed-FTS-external-service-simulators/kibana/v{VERSION}.json`,
         // code coverage reporting plugin

test/common/fixtures/plugins/otel_metrics/kibana.json

@@ -0,0 +1,15 @@
+{
+  "id": "openTelemetryInstrumentedPlugin",
+  "owner": {
+    "name": "Stack Monitoring",
+    "githubTeam": "stack-monitoring-ui"
+  },
+  "version": "1.0.0",
+  "kibanaVersion": "kibana",
+  "requiredPlugins": [
+    "monitoringCollection"
+  ],
+  "optionalPlugins": [],
+  "server": true,
+  "ui": false
+}

test/common/fixtures/plugins/otel_metrics/server/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { OpenTelemetryUsageTest } from './plugin';
+
+export const plugin = () => new OpenTelemetryUsageTest();

test/common/fixtures/plugins/otel_metrics/server/monitoring/metrics.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Counter, Meter } from '@opentelemetry/api-metrics';
+
+export class Metrics {
+  ruleExecutions: Counter;
+
+  constructor(meter: Meter) {
+    this.ruleExecutions = meter.createCounter('request_count', {
+      description: 'Counts total number of requests',
+    });
+  }
+}

test/common/fixtures/plugins/otel_metrics/server/plugin.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { CoreSetup, Plugin } from '@kbn/core/server';
+import { metrics } from '@opentelemetry/api-metrics';
+import { generateOtelMetrics } from './routes';
+import { Metrics } from './monitoring/metrics';
+
+export class OpenTelemetryUsageTest implements Plugin {
+  private metrics: Metrics;
+
+  constructor() {
+    this.metrics = new Metrics(metrics.getMeter('dummyMetric'));
+  }
+
+  public setup(core: CoreSetup) {
+    const router = core.http.createRouter();
+    generateOtelMetrics(router, this.metrics);
+  }
+
+  public start() {}
+  public stop() {}
+}

test/common/fixtures/plugins/otel_metrics/server/routes/generate_otel_metrics.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { IKibanaResponse, IRouter } from '@kbn/core/server';
+import { Metrics } from '../monitoring/metrics';
+
+export const generateOtelMetrics = (router: IRouter, metrics: Metrics) => {
+  router.post(
+    {
+      path: '/api/generate_otel_metrics',
+      validate: {},
+    },
+    async function (_context, _req, res): Promise<IKibanaResponse<{}>> {
+      metrics.ruleExecutions.add(1);
+      return res.ok({});
+    }
+  );
+};

test/common/fixtures/plugins/otel_metrics/server/routes/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export * from './generate_otel_metrics';

x-pack/plugins/monitoring_collection/README.md

@@ -2,4 +2,102 @@
 
 ## Plugin
 
-This plugin allows for other plugins to add data to Kibana stack monitoring documents.
+This plugin allows for other plugins to add data to Kibana stack monitoring documents.
+
+## OpenTelemetry Metrics
+
+### Enable Prometheus endpoint with Elastic Agent Prometheus input
+
+1. Start [local setup with fleet](../fleet/README.md#running-fleet-server-locally-in-a-container) or a cloud cluster
+2. Start Kibana
+3. Set up a new agent policy and enroll a new agent in your local machine
+4. Install the Prometheus Metrics package
+  a. Set **Hosts** with `localhost:5601`
+  b. Set **Metrics Path** with `/(BASEPATH)/api/monitoring_collection/v1/prometheus`
+  c. Remove the values from **Bearer Token File** and **SSL Certificate Authorities**
+  d. Set **Username** and **Password** with `elastic` and `changeme`
+5. Add the following configuration to `kibana.dev.yml`
+
+    ```yml
+    # Enable the prometheus exporter
+    monitoring_collection.opentelemetry.metrics:
+      prometheus.enabled: true
+
+    ```
+
+### Enable OpenTelemetry Metrics API exported as OpenTelemetry Protocol
+
+1. Start [local setup with fleet](../fleet/README.md#running-fleet-server-locally-in-a-container) or a cloud cluster
+2. Start Kibana
+3. Set up a new agent policy and enroll a new agent in your local machine
+4. Install Elastic APM package listening on `localhost:8200` without authentication
+5. Add the following configuration to `kibana.dev.yml`
+
+    ```yml
+    # Enable the OTLP exporter
+    monitoring_collection.opentelemetry.metrics:
+      otlp.url: "http://127.0.0.1:8200"
+    ```
+
+### Example of how to instrument the code
+
+* First, we need to define what metrics we want to instrument with OpenTelemetry
+
+  ```ts
+  import { Counter, Meter } from '@opentelemetry/api-metrics';
+
+  export class FooApiMeters {
+    requestCount: Counter;
+
+    constructor(meter: Meter) {
+      this.requestCount = meter.createCounter('request_count', {
+        description: 'Counts total number of requests',
+      });
+    }
+  ```
+
+  In this example we're using a `Counter` metric, but [OpenTelemetry SDK](https://open-telemetry.github.io/opentelemetry-js/interfaces/_opentelemetry_api_metrics.Meter.html) provides there are other options to record metrics
+
+* Initialize meter in the plugin setup and pass it to the relevant components that will be instrumented. In this case, we want to intrument `FooApi` routes
+
+  ```ts
+  import { IRouter } from '@kbn/core/server';
+  import { FooApiMeters } from './foo_api_meters';
+  import { metrics } from '@opentelemetry/api-metrics';
+
+  export class FooApiPlugin implements Plugin {
+    private metrics: Metrics;
+    private libs: { router: IRouter, metrics: FooApiMeters};
+
+    constructor() {
+      this.metrics = new Metrics(metrics.getMeter('kibana.fooApi'));
+    }
+
+    public setup(core: CoreSetup) {
+      const router = core.http.createRouter();
+
+      this.libs = {
+        router,
+        metrics: this.metrics
+      }
+
+      initMetricsAPIRoute(this.libs);
+    }
+  }
+  ```
+
+  * Lastly we can use the `metrics` object to instrument the code
+
+  ```ts
+  export const initMetricsAPIRoute = (libs: { router: IRouter, metrics: FooApiMeters}) => {
+    router.get(
+      {
+        path: '/api/foo',
+        validate: {},
+      },
+      async function (_context, _req, res) {
+        metrics.requestCount.add(1);
+        return res.ok({});
+      }
+    );
+  ```

x-pack/plugins/monitoring_collection/server/config.ts

@@ -9,6 +9,18 @@ import { schema, TypeOf } from '@kbn/config-schema';
 
 export const configSchema = schema.object({
   enabled: schema.boolean({ defaultValue: true }),
+  opentelemetry: schema.object({
+    metrics: schema.object({
+      otlp: schema.object({
+        url: schema.maybe(schema.string()),
+        headers: schema.maybe(schema.recordOf(schema.string(), schema.string())),
+        exportIntervalMillis: schema.number({ defaultValue: 10000 }),
+      }),
+      prometheus: schema.object({
+        enabled: schema.boolean({ defaultValue: false }),
+      }),
+    }),
+  }),
 });
 
 export type MonitoringCollectionConfig = ReturnType<typeof createConfig>;

x-pack/plugins/monitoring_collection/server/constants.ts

@@ -5,3 +5,5 @@
  * 2.0.
  */
 export const TYPE_ALLOWLIST = ['node_rules', 'cluster_rules', 'node_actions', 'cluster_actions'];
+
+export const MONITORING_COLLECTION_BASE_PATH = '/api/monitoring_collection';

x-pack/plugins/monitoring_collection/server/lib/index.ts

@@ -7,3 +7,4 @@
 
 export { getKibanaStats } from './get_kibana_stats';
 export { getESClusterUuid } from './get_es_cluster_uuid';
+export { PrometheusExporter } from './prometheus_exporter';

x-pack/plugins/monitoring_collection/server/lib/prometheus_exporter.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AggregationTemporality, MetricReader } from '@opentelemetry/sdk-metrics-base';
+import {
+  PrometheusExporter as OpenTelemetryPrometheusExporter,
+  ExporterConfig,
+  PrometheusSerializer,
+} from '@opentelemetry/exporter-prometheus';
+import { KibanaResponseFactory } from '@kbn/core/server';
+
+export class PrometheusExporter extends MetricReader {
+  private readonly _prefix?: string;
+  private readonly _appendTimestamp: boolean;
+  private _serializer: PrometheusSerializer;
+
+  constructor(config: ExporterConfig = {}) {
+    super();
+    this._prefix = config.prefix || OpenTelemetryPrometheusExporter.DEFAULT_OPTIONS.prefix;
+    this._appendTimestamp =
+      typeof config.appendTimestamp === 'boolean'
+        ? config.appendTimestamp
+        : OpenTelemetryPrometheusExporter.DEFAULT_OPTIONS.appendTimestamp;
+
+    this._serializer = new PrometheusSerializer(this._prefix, this._appendTimestamp);
+  }
+
+  selectAggregationTemporality(): AggregationTemporality {
+    return AggregationTemporality.CUMULATIVE;
+  }
+
+  protected onForceFlush(): Promise<void> {
+    return Promise.resolve(undefined);
+  }
+
+  protected onShutdown(): Promise<void> {
+    return Promise.resolve(undefined);
+  }
+
+  /**
+   * Responds to incoming message with current state of all metrics.
+   */
+  public async exportMetrics(res: KibanaResponseFactory) {
+    try {
+      const collectionResult = await this.collect();
+      const { resourceMetrics, errors } = collectionResult;
+      if (errors.length) {
+        return res.customError({
+          statusCode: 500,
+          body: `PrometheusExporter: Metrics collection errors ${errors}`,
+        });
+      }
+      const result = this._serializer.serialize(resourceMetrics);
+      if (result === '') {
+        return res.noContent();
+      }
+      return res.ok({
+        body: result,
+      });
+    } catch (error) {
+      return res.customError({
+        statusCode: 500,
+        body: {
+          message: `PrometheusExporter: Failed to export metrics ${error}`,
+        },
+      });
+    }
+  }
+}