[AO] Add metric threshold integration test (#157489)

## Summary

Closes #157189

This PR adds a metric threshold integration test. This is the first step
in adding more test coverage for observability rules.

**Steps during the test**
1. Generating fake host data by using a similar implementation as
https://github.com/elastic/high-cardinality-cluster
    - Data is generated for the last 15 mins
- Implementation was simplified only to cover fake hosts and was
converted to typescript
2. Creating an action using an index connector
3. Creating a metric threshold rule containing step number 2 action
4. Checking the status of the rule to be active
5. Checking the triggered action to have the correct parameters
6. Checking the generated alert to have the correct information
7. Clean up

**How to run locally**
- Run server
```
node scripts/functional_tests_server --config x-pack/test/api_integration/apis/metrics_ui/config.ts
```
- Then run the test
```
node scripts/functional_tests__runner --include-pack/test/api_integration/apis/metrics_ui/cometric_threshold_rule.ts --config x-pack/test/api_integration/apis/metrics_ui/config.ts
```

**Reference**
I created elastic/integrations#6168 to find a
better way to generate data and make sure that data matches what
metricbeats generates

---------

Co-authored-by: kibanamachine <[email protected]>

.github/CODEOWNERS

@@ -413,6 +413,7 @@ packages/kbn-import-resolver @elastic/kibana-operations
 x-pack/plugins/index_lifecycle_management @elastic/platform-deployment-management
 x-pack/plugins/index_management @elastic/platform-deployment-management
 test/plugin_functional/plugins/index_patterns @elastic/kibana-app-services
+x-pack/packages/kbn-infra-forge @elastic/actionable-observability
 x-pack/plugins/infra @elastic/infra-monitoring-ui
 x-pack/plugins/ingest_pipelines @elastic/platform-deployment-management
 src/plugins/input_control_vis @elastic/kibana-presentation

package.json

@@ -436,6 +436,7 @@
     "@kbn/index-lifecycle-management-plugin": "link:x-pack/plugins/index_lifecycle_management",
     "@kbn/index-management-plugin": "link:x-pack/plugins/index_management",
     "@kbn/index-patterns-test-plugin": "link:test/plugin_functional/plugins/index_patterns",
+    "@kbn/infra-forge": "link:x-pack/packages/kbn-infra-forge",
     "@kbn/infra-plugin": "link:x-pack/plugins/infra",
     "@kbn/ingest-pipelines-plugin": "link:x-pack/plugins/ingest_pipelines",
     "@kbn/input-control-vis-plugin": "link:src/plugins/input_control_vis",

tsconfig.base.json

@@ -820,6 +820,8 @@
       "@kbn/index-management-plugin/*": ["x-pack/plugins/index_management/*"],
       "@kbn/index-patterns-test-plugin": ["test/plugin_functional/plugins/index_patterns"],
       "@kbn/index-patterns-test-plugin/*": ["test/plugin_functional/plugins/index_patterns/*"],
+      "@kbn/infra-forge": ["x-pack/packages/kbn-infra-forge"],
+      "@kbn/infra-forge/*": ["x-pack/packages/kbn-infra-forge/*"],
       "@kbn/infra-plugin": ["x-pack/plugins/infra"],
       "@kbn/infra-plugin/*": ["x-pack/plugins/infra/*"],
       "@kbn/ingest-pipelines-plugin": ["x-pack/plugins/ingest_pipelines"],

x-pack/packages/kbn-infra-forge/README.md

@@ -0,0 +1,6 @@
+# @kbn/infra-forge
+
+`@kbn/data-forge` is a tool to generate infra data for integration testing.
+
+**Datasets**
+- fake hosts

x-pack/packages/kbn-infra-forge/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { cleanup, generate } from './src/run';

x-pack/packages/kbn-infra-forge/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/packages/kbn-infra-forge'],
+};

x-pack/packages/kbn-infra-forge/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/infra-forge",
+  "owner": "@elastic/actionable-observability"
+}

x-pack/packages/kbn-infra-forge/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/infra-forge",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/index.ts

@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import lodash from 'lodash';
+import type { Moment } from 'moment';
+
+export { template } from './template';
+
+const createGroupIndex = (index: number) => Math.floor(index / 1000) * 1000;
+
+const randomBetween = (start = 0, end = 1, step = 0.1) =>
+  lodash.sample(lodash.range(start, end, step));
+
+let networkDataCount = 0;
+const generateNetworkData = lodash.memoize(() => {
+  networkDataCount += 10000;
+  return networkDataCount;
+});
+
+export const generateEvent = (index: number, timestamp: Moment, interval: number) => {
+  const groupIndex = createGroupIndex(index);
+  return [
+    {
+      '@timestamp': timestamp.toISOString(),
+      tags: [`group-${groupIndex}`, `event-${index}`],
+      host: {
+        name: `host-${index}`,
+        mac: ['00-00-5E-00-53-23', '00-00-5E-00-53-24'],
+        network: {
+          name: `network-${index}`,
+        },
+      },
+      event: {
+        module: 'system',
+        dataset: 'system.cpu',
+      },
+      labels: {
+        groupId: `group-${groupIndex}`,
+        eventId: `event-${index}`,
+      },
+      system: {
+        cpu: {
+          cores: 4,
+          total: {
+            norm: {
+              pct: randomBetween(),
+            },
+          },
+          user: {
+            pct: randomBetween(1, 4),
+          },
+          system: {
+            pct: randomBetween(1, 4),
+          },
+        },
+      },
+      metricset: {
+        period: interval,
+      },
+      container: {
+        id: `container-${index}`,
+        name: 'container-name',
+      },
+    },
+    {
+      '@timestamp': timestamp.toISOString(),
+      host: {
+        name: `host-${index}`,
+        network: {
+          name: `network-${index}`,
+        },
+      },
+      event: {
+        module: 'system',
+        dataset: 'system.network',
+      },
+      labels: {
+        groupId: `group-${groupIndex}`,
+        eventId: `event-${index}`,
+      },
+      system: {
+        network: {
+          name: 'eth0',
+          in: {
+            bytes: generateNetworkData(),
+          },
+          out: {
+            bytes: generateNetworkData(),
+          },
+        },
+      },
+      metricset: {
+        period: interval,
+      },
+      container: {
+        id: `container-${index}`,
+        name: 'container-name',
+      },
+    },
+    {
+      '@timestamp': timestamp.toISOString(),
+      host: {
+        name: `host-${index}`,
+        network: {
+          name: `network-${index}`,
+        },
+      },
+      event: {
+        module: 'system',
+        dataset: 'system.network',
+      },
+      labels: {
+        groupId: `group-${groupIndex}`,
+        eventId: `event-${index}`,
+      },
+      system: {
+        network: {
+          name: 'eth1',
+          in: {
+            bytes: generateNetworkData(),
+          },
+          out: {
+            bytes: generateNetworkData(),
+          },
+        },
+      },
+      metricset: {
+        period: interval,
+      },
+      container: {
+        id: `container-${index}`,
+        name: 'container-name',
+      },
+    },
+  ];
+};

x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/template.ts

@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const template = {
+  order: 1,
+  index_patterns: ['kbn-data-forge*'],
+  settings: {
+    index: {
+      mapping: {
+        total_fields: {
+          limit: '10000',
+        },
+      },
+      number_of_shards: '1',
+      number_of_replicas: '0',
+      query: {
+        default_field: ['message', 'labels.*', 'event.*'],
+      },
+    },
+  },
+  mappings: {
+    dynamic_templates: [
+      {
+        labels: {
+          path_match: 'labels.*',
+          mapping: {
+            type: 'keyword',
+          },
+          match_mapping_type: 'string',
+        },
+      },
+      {
+        strings_as_keyword: {
+          mapping: {
+            ignore_above: 1024,
+            type: 'keyword',
+          },
+          match_mapping_type: 'string',
+        },
+      },
+    ],
+    date_detection: false,
+    properties: {
+      '@timestamp': {
+        type: 'date',
+      },
+      tags: {
+        type: 'keyword',
+      },
+      metricset: {
+        properties: {
+          period: {
+            type: 'long',
+          },
+        },
+      },
+      host: {
+        properties: {
+          name: {
+            type: 'keyword',
+            ignore_above: 256,
+          },
+          network: {
+            properties: {
+              name: {
+                type: 'keyword',
+                ignore_above: 256,
+              },
+            },
+          },
+        },
+      },
+      event: {
+        properties: {
+          dataset: {
+            type: 'keyword',
+            ignore_above: 256,
+          },
+          module: {
+            type: 'keyword',
+            ignore_above: 256,
+          },
+        },
+      },
+      system: {
+        properties: {
+          cpu: {
+            properties: {
+              cores: {
+                type: 'long',
+              },
+              total: {
+                properties: {
+                  norm: {
+                    properties: {
+                      pct: {
+                        scaling_factor: 1000,
+                        type: 'scaled_float',
+                      },
+                    },
+                  },
+                },
+              },
+              user: {
+                properties: {
+                  pct: {
+                    scaling_factor: 1000,
+                    type: 'scaled_float',
+                  },
+                  norm: {
+                    properties: {
+                      pct: {
+                        scaling_factor: 1000,
+                        type: 'scaled_float',
+                      },
+                    },
+                  },
+                },
+              },
+              system: {
+                properties: {
+                  pct: {
+                    scaling_factor: 1000,
+                    type: 'scaled_float',
+                  },
+                },
+              },
+            },
+          },
+          network: {
+            properties: {
+              name: {
+                type: 'keyword',
+                ignore_above: 256,
+              },
+              in: {
+                properties: {
+                  bytes: {
+                    type: 'long',
+                  },
+                },
+              },
+              out: {
+                properties: {
+                  bytes: {
+                    type: 'long',
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+      container: {
+        properties: {
+          id: {
+            type: 'keyword',
+            ignore_above: 256,
+          },
+          name: {
+            type: 'keyword',
+            ignore_above: 256,
+          },
+        },
+      },
+    },
+  },
+  aliases: {
+    'metrics-fake_hosts': {},
+  },
+};