[Stack Monitoring] Add OpenTelemetry metrics to Monitoring Collection plugin

.github/CODEOWNERS

@@ -116,6 +116,7 @@
 /x-pack/plugins/monitoring/ @elastic/infra-monitoring-ui
 /x-pack/test/functional/apps/monitoring @elastic/infra-monitoring-ui
 /x-pack/test/api_integration/apis/monitoring @elastic/infra-monitoring-ui
+/x-pack/test/api_integration/apis/monitoring_collection @elastic/infra-monitoring-ui
 
 # Fleet
 /fleet_packages.json @elastic/fleet

package.json

@@ -120,6 +120,7 @@
     "@emotion/css": "^11.9.0",
     "@emotion/react": "^11.9.0",
     "@emotion/serialize": "^1.0.3",
+    "@grpc/grpc-js": "^1.6.7",
     "@hapi/accept": "^5.0.2",
     "@hapi/boom": "^9.1.4",
     "@hapi/cookie": "^11.0.2",
@@ -274,6 +275,13 @@
     "@mapbox/mapbox-gl-draw": "1.3.0",
     "@mapbox/mapbox-gl-rtl-text": "0.2.3",
     "@mapbox/vector-tile": "1.3.1",
+    "@opentelemetry/api": "^1.1.0",
+    "@opentelemetry/api-metrics": "^0.30.0",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.30.0",
+    "@opentelemetry/exporter-prometheus": "^0.30.0",
+    "@opentelemetry/resources": "^1.4.0",
+    "@opentelemetry/sdk-metrics-base": "^0.30.0",
+    "@opentelemetry/semantic-conventions": "^1.4.0",
     "@reduxjs/toolkit": "^1.6.1",
     "@slack/webhook": "^5.0.4",
     "@turf/along": "6.0.1",

test/common/config.js

@@ -51,6 +51,8 @@ export default function () {
         `--server.maxPayload=1679958`,
         // newsfeed mock service
         `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'newsfeed')}`,
+        // otel mock service
+        `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'otel_metrics')}`,
         `--newsfeed.service.urlRoot=${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`,
         `--newsfeed.service.pathTemplate=/api/_newsfeed-FTS-external-service-simulators/kibana/v{VERSION}.json`,
         // code coverage reporting plugin

test/common/fixtures/plugins/otel_metrics/kibana.json

@@ -0,0 +1,15 @@
+{
+  "id": "openTelemetryInstrumentedPlugin",
+  "owner": {
+    "name": "Stack Monitoring",
+    "githubTeam": "stack-monitoring-ui"
+  },
+  "version": "1.0.0",
+  "kibanaVersion": "kibana",
+  "requiredPlugins": [
+    "monitoringCollection"
+  ],
+  "optionalPlugins": [],
+  "server": true,
+  "ui": false
+}

test/common/fixtures/plugins/otel_metrics/server/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { OpenTelemetryUsageTest } from './plugin';
+
+export const plugin = () => new OpenTelemetryUsageTest();

test/common/fixtures/plugins/otel_metrics/server/monitoring/metrics.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Counter, Meter } from '@opentelemetry/api-metrics';
+
+export class Metrics {
+  requestCounter: Counter;
+
+  constructor(meter: Meter) {
+    this.requestCounter = meter.createCounter('request_count', {
+      description: 'Counts total number of requests',
+    });
+  }
+}

test/common/fixtures/plugins/otel_metrics/server/plugin.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { CoreSetup, Plugin } from '@kbn/core/server';
+import { metrics } from '@opentelemetry/api-metrics';
+import { generateOtelMetrics } from './routes';
+import { Metrics } from './monitoring/metrics';
+
+export class OpenTelemetryUsageTest implements Plugin {
+  private metrics: Metrics;
+
+  constructor() {
+    this.metrics = new Metrics(metrics.getMeter('dummyMetric'));
+  }
+
+  public setup(core: CoreSetup) {
+    const router = core.http.createRouter();
+    generateOtelMetrics(router, this.metrics);
+  }
+
+  public start() {}
+  public stop() {}
+}

test/common/fixtures/plugins/otel_metrics/server/routes/generate_otel_metrics.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { IKibanaResponse, IRouter } from '@kbn/core/server';
+import { Metrics } from '../monitoring/metrics';
+
+export const generateOtelMetrics = (router: IRouter, metrics: Metrics) => {
+  router.post(
+    {
+      path: '/api/generate_otel_metrics',
+      validate: {},
+    },
+    async function (_context, _req, res): Promise<IKibanaResponse<{}>> {
+      metrics.requestCounter.add(1);
+      return res.ok({});
+    }
+  );
+};

test/common/fixtures/plugins/otel_metrics/server/routes/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export * from './generate_otel_metrics';

tsconfig.base.json

@@ -201,6 +201,8 @@
       "@kbn/coverage-fixtures-plugin/*": ["test/common/fixtures/plugins/coverage/*"],
       "@kbn/newsfeed-fixtures-plugin": ["test/common/fixtures/plugins/newsfeed"],
       "@kbn/newsfeed-fixtures-plugin/*": ["test/common/fixtures/plugins/newsfeed/*"],
+      "@kbn/open-telemetry-instrumented-plugin": ["test/common/fixtures/plugins/otel_metrics"],
+      "@kbn/open-telemetry-instrumented-plugin/*": ["test/common/fixtures/plugins/otel_metrics/*"],
       "@kbn/kbn-tp-run-pipeline-plugin": ["test/interpreter_functional/plugins/kbn_tp_run_pipeline"],
       "@kbn/kbn-tp-run-pipeline-plugin/*": ["test/interpreter_functional/plugins/kbn_tp_run_pipeline/*"],
       "@kbn/app-link-test-plugin": ["test/plugin_functional/plugins/app_link_test"],

x-pack/plugins/monitoring_collection/README.md

@@ -2,4 +2,139 @@
 
 ## Plugin
 
-This plugin allows for other plugins to add data to Kibana stack monitoring documents.
+This plugin allows for other plugins to add data to Kibana stack monitoring documents.
+
+## OpenTelemetry Metrics
+
+### Enable Prometheus endpoint with Elastic Agent Prometheus input
+
+1. Start [local setup with fleet](../fleet/README.md#running-fleet-server-locally-in-a-container) or a cloud cluster
+2. Start Kibana
+3. Set up a new agent policy and enroll a new agent in your local machine
+4. Install the Prometheus Metrics package
+   1. Set **Hosts** with `localhost:5601`
+   2. Set **Metrics Path** with `/(BASEPATH)/api/monitoring_collection/v1/prometheus`
+   3. Remove the values from **Bearer Token File** and **SSL Certificate Authorities**
+   4. Set **Username** and **Password** with `elastic` and `changeme`
+5. Add the following configuration to `kibana.dev.yml`
+
+    ```yml
+    # Enable the prometheus exporter
+    monitoring_collection.opentelemetry.metrics:
+      prometheus.enabled: true
+    ```
+
+### Enable OpenTelemetry Metrics API exported as OpenTelemetry Protocol over GRPC
+
+1. Start [local setup with fleet](../fleet/README.md#running-fleet-server-locally-in-a-container) or a cloud cluster
+2. Start Kibana
+3. Set up a new agent policy and enroll a new agent in your local machine
+4. Install Elastic APM package listening on `localhost:8200` without authentication
+5. Add the following configuration to `kibana.dev.yml`
+
+    ```yml
+    # Enable the OTLP exporter
+    monitoring_collection.opentelemetry.metrics:
+      otlp.url: "http://127.0.0.1:8200"
+    ```
+
+You can also provide headers for OTLP endpoints that require authentication:
+
+```yml
+# Enable the OTLP exporter to an authenticated APM endpoint
+monitoring_collection.opentelemetry.metrics:
+  otlp:
+    url: "https://DEPLOYMENT.apm.REGION.PROVIDER.elastic-cloud.com"
+    headers:
+      Authorization: "Bearer SECRET_TOKEN"
+```
+
+Alternatively, OTLP Exporter can be configured using environment variables `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`, `OTEL_EXPORTER_OTLP_METRICS_HEADERS`. [See OTLP Exporter docs](https://opentelemetry.io/docs/reference/specification/protocol/exporter/) for details.
+
+It's possible to configure logging for the OTLP integration. If not informed, the default will be `info`
+
+```yml
+monitoring_collection.opentelemetry.metrics:
+  logLevel: warn | info | debug | warn | none | verbose | all
+```
+
+For connection-level debug information you can set these variables:
+
+```bash
+export GRPC_NODE_TRACE="xds_client,xds_resolver,cds_balancer,eds_balancer,priority,weighted_target,round_robin,resolving_load_balancer,subchannel,keepalive,dns_resolver,fault_injection,http_filter,csds"
+export GRPC_NODE_VERBOSITY=DEBUG
+```
+
+See the [grpc-node docs](https://github.com/grpc/grpc-node/blob/master/doc/environment_variables.md) for details and other settings.
+
+### Example of how to instrument the code
+
+* First, we need to define what metrics we want to instrument with OpenTelemetry
+
+  ```ts
+  import { Counter, Meter } from '@opentelemetry/api-metrics';
+
+  export class FooApiMeters {
+    requestCount: Counter;
+
+    constructor(meter: Meter) {
+      this.requestCount = meter.createCounter('request_count', {
+        description: 'Counts total number of requests',
+      });
+    }
+  }
+  ```
+
+  In this example we're using a `Counter` metric, but [OpenTelemetry SDK](https://open-telemetry.github.io/opentelemetry-js/interfaces/_opentelemetry_api_metrics.Meter.html) provides there are other options to record metrics
+
+* Initialize meter in the plugin setup and pass it to the relevant components that will be instrumented. In this case, we want to instrument `FooApi` routes.
+
+  ```ts
+  import { IRouter } from '@kbn/core/server';
+  import { FooApiMeters } from './foo_api_meters';
+  import { metrics } from '@opentelemetry/api-metrics';
+
+  export class FooApiPlugin implements Plugin {
+    private metrics: Metrics;
+    private libs: { router: IRouter, metrics: FooApiMeters};
+
+    constructor() {
+      this.metrics = new Metrics(metrics.getMeter('kibana.fooApi'));
+    }
+
+    public setup(core: CoreSetup) {
+      const router = core.http.createRouter();
+
+      this.libs = {
+        router,
+        metrics: this.metrics
+      }
+
+      initMetricsAPIRoute(this.libs);
+    }
+  }
+  ```
+
+  `monitoring_collection` plugins has to be initialized before the plugin that will be instrumented. If for some reason the instrumentation doesn't record any metrics, make sure `monitoring_collection` is included in the list of `requiredPlugins`. e.g:
+
+  ```json
+  "requiredPlugins": [
+    "monitoringCollection"
+  ],
+  ```
+
+* Lastly we can use the `metrics` object to instrument the code
+
+  ```ts
+  export const initMetricsAPIRoute = (libs: { router: IRouter, metrics: FooApiMeters}) => {
+    router.get(
+      {
+        path: '/api/foo',
+        validate: {},
+      },
+      async function (_context, _req, res) {
+        metrics.requestCount.add(1);
+        return res.ok({});
+      }
+    );
+  ```

x-pack/plugins/monitoring_collection/server/config.ts

@@ -9,6 +9,19 @@ import { schema, TypeOf } from '@kbn/config-schema';
 
 export const configSchema = schema.object({
   enabled: schema.boolean({ defaultValue: true }),
+  opentelemetry: schema.object({
+    metrics: schema.object({
+      otlp: schema.object({
+        url: schema.maybe(schema.string()),
+        headers: schema.maybe(schema.recordOf(schema.string(), schema.string())),
+        exportIntervalMillis: schema.number({ defaultValue: 10000 }),
+        logLevel: schema.string({ defaultValue: 'info' }),
+      }),
+      prometheus: schema.object({
+        enabled: schema.boolean({ defaultValue: false }),
+      }),
+    }),
+  }),
 });
 
 export type MonitoringCollectionConfig = ReturnType<typeof createConfig>;

x-pack/plugins/monitoring_collection/server/constants.ts

@@ -5,3 +5,5 @@
  * 2.0.
  */
 export const TYPE_ALLOWLIST = ['node_rules', 'cluster_rules', 'node_actions', 'cluster_actions'];
+
+export const MONITORING_COLLECTION_BASE_PATH = '/api/monitoring_collection';

x-pack/plugins/monitoring_collection/server/lib/index.ts

@@ -7,3 +7,4 @@
 
 export { getKibanaStats } from './get_kibana_stats';
 export { getESClusterUuid } from './get_es_cluster_uuid';
+export { PrometheusExporter } from './prometheus_exporter';