[ML] Alerts as data integration for Anomaly Detection rule type (#166349

)

## Summary

Part of #165958

Replaces usage of the deprecated `alertFactory` with the new alerts
client and adds alerts-as-data integration for Anomaly Detection
alerting rule type.

Alert instances are stored in
`.alerts-ml.anomaly-detection.alerts-default` index and extends the
common `AlertSchema`.

<details>
  <summary>Result mappings</summary>
  
  ```json
{
  ".internal.alerts-ml.anomaly-detection.alerts-default-000001": {
    "mappings": {
      "dynamic": "false",
      "_meta": {
        "namespace": "default",
        "kibana": {
          "version": "8.11.0"
        },
        "managed": true
      },
      "properties": {
        "@timestamp": {
          "type": "date"
        },
        "event": {
          "properties": {
            "action": {
              "type": "keyword"
            },
            "kind": {
              "type": "keyword"
            }
          }
        },
        "kibana": {
          "properties": {
            "alert": {
              "properties": {
                "action_group": {
                  "type": "keyword"
                },
                "anomaly_score": {
                  "type": "double"
                },
                "anomaly_timestamp": {
                  "type": "date"
                },
                "case_ids": {
                  "type": "keyword"
                },
                "duration": {
                  "properties": {
                    "us": {
                      "type": "long"
                    }
                  }
                },
                "end": {
                  "type": "date"
                },
                "flapping": {
                  "type": "boolean"
                },
                "flapping_history": {
                  "type": "boolean"
                },
                "instance": {
                  "properties": {
                    "id": {
                      "type": "keyword"
                    }
                  }
                },
                "is_interim": {
                  "type": "boolean"
                },
                "job_id": {
                  "type": "keyword"
                },
                "last_detected": {
                  "type": "date"
                },
                "maintenance_window_ids": {
                  "type": "keyword"
                },
                "reason": {
                  "type": "keyword"
                },
                "rule": {
                  "properties": {
                    "category": {
                      "type": "keyword"
                    },
                    "consumer": {
                      "type": "keyword"
                    },
                    "execution": {
                      "properties": {
                        "uuid": {
                          "type": "keyword"
                        }
                      }
                    },
                    "name": {
                      "type": "keyword"
                    },
                    "parameters": {
                      "type": "flattened",
                      "ignore_above": 4096
                    },
                    "producer": {
                      "type": "keyword"
                    },
                    "revision": {
                      "type": "long"
                    },
                    "rule_type_id": {
                      "type": "keyword"
                    },
                    "tags": {
                      "type": "keyword"
                    },
                    "uuid": {
                      "type": "keyword"
                    }
                  }
                },
                "start": {
                  "type": "date"
                },
                "status": {
                  "type": "keyword"
                },
                "time_range": {
                  "type": "date_range",
                  "format": "epoch_millis||strict_date_optional_time"
                },
                "top_influencers": {
                  "type": "nested",
                  "dynamic": "false",
                  "properties": {
                    "influencer_field_name": {
                      "type": "keyword"
                    },
                    "influencer_field_value": {
                      "type": "keyword"
                    },
                    "influencer_score": {
                      "type": "double"
                    },
                    "initial_influencer_score": {
                      "type": "double"
                    },
                    "is_interim": {
                      "type": "boolean"
                    },
                    "job_id": {
                      "type": "keyword"
                    },
                    "timestamp": {
                      "type": "date"
                    }
                  }
                },
                "top_records": {
                  "type": "nested",
                  "dynamic": "false",
                  "properties": {
                    "actual": {
                      "type": "double"
                    },
                    "by_field_name": {
                      "type": "keyword"
                    },
                    "by_field_value": {
                      "type": "keyword"
                    },
                    "detector_index": {
                      "type": "integer"
                    },
                    "field_name": {
                      "type": "keyword"
                    },
                    "function": {
                      "type": "keyword"
                    },
                    "initial_record_score": {
                      "type": "double"
                    },
                    "is_interim": {
                      "type": "boolean"
                    },
                    "job_id": {
                      "type": "keyword"
                    },
                    "over_field_name": {
                      "type": "keyword"
                    },
                    "over_field_value": {
                      "type": "keyword"
                    },
                    "partition_field_name": {
                      "type": "keyword"
                    },
                    "partition_field_value": {
                      "type": "keyword"
                    },
                    "record_score": {
                      "type": "double"
                    },
                    "timestamp": {
                      "type": "date"
                    },
                    "typical": {
                      "type": "double"
                    }
                  }
                },
                "url": {
                  "type": "keyword",
                  "index": false,
                  "ignore_above": 2048
                },
                "uuid": {
                  "type": "keyword"
                },
                "workflow_status": {
                  "type": "keyword"
                },
                "workflow_tags": {
                  "type": "keyword"
                }
              }
            },
            "space_ids": {
              "type": "keyword"
            },
            "version": {
              "type": "version"
            }
          }
        },
        "tags": {
          "type": "keyword"
        }
      }
    }
  }
}
  ```
</details>

### Checklist

- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

packages/kbn-alerts-as-data-utils/src/field_maps/types.ts

@@ -34,6 +34,7 @@ export interface EcsMetadata {
   scaling_factor?: number;
   short: string;
   type: string;
+  properties?: Record<string, { type: string }>;
 }
 
 export interface FieldMap {
@@ -50,5 +51,6 @@ export interface FieldMap {
     path?: string;
     scaling_factor?: number;
     dynamic?: boolean | 'strict';
+    properties?: Record<string, { type: string }>;
   };
 }

packages/kbn-alerts-as-data-utils/src/schemas/create_schema_from_field_map.ts

@@ -198,6 +198,7 @@ const generateSchemaLines = ({
         break;
       case 'float':
       case 'integer':
+      case 'double':
         lineWriter.addLine(`${keyToWrite}: ${getSchemaDefinition('schemaNumber', isArray)},`);
         break;
       case 'boolean':

packages/kbn-alerts-as-data-utils/src/schemas/generated/ml_anomaly_detection_schema.ts

@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+// ---------------------------------- WARNING ----------------------------------
+// this file was generated, and should not be edited by hand
+// ---------------------------------- WARNING ----------------------------------
+import * as rt from 'io-ts';
+import { Either } from 'fp-ts/lib/Either';
+import { AlertSchema } from './alert_schema';
+const ISO_DATE_PATTERN = /^d{4}-d{2}-d{2}Td{2}:d{2}:d{2}.d{3}Z$/;
+export const IsoDateString = new rt.Type<string, string, unknown>(
+  'IsoDateString',
+  rt.string.is,
+  (input, context): Either<rt.Errors, string> => {
+    if (typeof input === 'string' && ISO_DATE_PATTERN.test(input)) {
+      return rt.success(input);
+    } else {
+      return rt.failure(input, context);
+    }
+  },
+  rt.identity
+);
+export type IsoDateStringC = typeof IsoDateString;
+export const schemaDate = IsoDateString;
+export const schemaDateArray = rt.array(IsoDateString);
+export const schemaDateRange = rt.partial({
+  gte: schemaDate,
+  lte: schemaDate,
+});
+export const schemaDateRangeArray = rt.array(schemaDateRange);
+export const schemaUnknown = rt.unknown;
+export const schemaUnknownArray = rt.array(rt.unknown);
+export const schemaString = rt.string;
+export const schemaStringArray = rt.array(schemaString);
+export const schemaNumber = rt.number;
+export const schemaNumberArray = rt.array(schemaNumber);
+export const schemaStringOrNumber = rt.union([schemaString, schemaNumber]);
+export const schemaStringOrNumberArray = rt.array(schemaStringOrNumber);
+export const schemaBoolean = rt.boolean;
+export const schemaBooleanArray = rt.array(schemaBoolean);
+const schemaGeoPointCoords = rt.type({
+  type: schemaString,
+  coordinates: schemaNumberArray,
+});
+const schemaGeoPointString = schemaString;
+const schemaGeoPointLatLon = rt.type({
+  lat: schemaNumber,
+  lon: schemaNumber,
+});
+const schemaGeoPointLocation = rt.type({
+  location: schemaNumberArray,
+});
+const schemaGeoPointLocationString = rt.type({
+  location: schemaString,
+});
+export const schemaGeoPoint = rt.union([
+  schemaGeoPointCoords,
+  schemaGeoPointString,
+  schemaGeoPointLatLon,
+  schemaGeoPointLocation,
+  schemaGeoPointLocationString,
+]);
+export const schemaGeoPointArray = rt.array(schemaGeoPoint);
+// prettier-ignore
+const MlAnomalyDetectionAlertRequired = rt.type({
+  kibana: rt.type({
+    alert: rt.type({
+      job_id: schemaString,
+    }),
+  }),
+});
+const MlAnomalyDetectionAlertOptional = rt.partial({
+  kibana: rt.partial({
+    alert: rt.partial({
+      anomaly_score: schemaNumber,
+      anomaly_timestamp: schemaDate,
+      is_interim: schemaBoolean,
+      top_influencers: rt.array(
+        rt.partial({
+          influencer_field_name: schemaString,
+          influencer_field_value: schemaString,
+          influencer_score: schemaNumber,
+          initial_influencer_score: schemaNumber,
+          is_interim: schemaBoolean,
+          job_id: schemaString,
+          timestamp: schemaDate,
+        })
+      ),
+      top_records: rt.array(
+        rt.partial({
+          actual: schemaNumber,
+          by_field_name: schemaString,
+          by_field_value: schemaString,
+          detector_index: schemaNumber,
+          field_name: schemaString,
+          function: schemaString,
+          initial_record_score: schemaNumber,
+          is_interim: schemaBoolean,
+          job_id: schemaString,
+          over_field_name: schemaString,
+          over_field_value: schemaString,
+          partition_field_name: schemaString,
+          partition_field_value: schemaString,
+          record_score: schemaNumber,
+          timestamp: schemaDate,
+          typical: schemaNumber,
+        })
+      ),
+    }),
+  }),
+});
+
+// prettier-ignore
+export const MlAnomalyDetectionAlertSchema = rt.intersection([MlAnomalyDetectionAlertRequired, MlAnomalyDetectionAlertOptional, AlertSchema]);
+// prettier-ignore
+export type MlAnomalyDetectionAlert = rt.TypeOf<typeof MlAnomalyDetectionAlertSchema>;

packages/kbn-alerts-as-data-utils/src/schemas/index.ts

@@ -13,6 +13,7 @@ import type { ObservabilityMetricsAlert } from './generated/observability_metric
 import type { ObservabilitySloAlert } from './generated/observability_slo_schema';
 import type { ObservabilityUptimeAlert } from './generated/observability_uptime_schema';
 import type { SecurityAlert } from './generated/security_schema';
+import type { MlAnomalyDetectionAlert } from './generated/ml_anomaly_detection_schema';
 
 export * from './create_schema_from_field_map';
 
@@ -24,6 +25,7 @@ export type { ObservabilitySloAlert } from './generated/observability_slo_schema
 export type { ObservabilityUptimeAlert } from './generated/observability_uptime_schema';
 export type { SecurityAlert } from './generated/security_schema';
 export type { StackAlert } from './generated/stack_schema';
+export type { MlAnomalyDetectionAlert } from './generated/ml_anomaly_detection_schema';
 
 export type AADAlert =
   | Alert
@@ -32,4 +34,5 @@ export type AADAlert =
   | ObservabilityMetricsAlert
   | ObservabilitySloAlert
   | ObservabilityUptimeAlert
-  | SecurityAlert;
+  | SecurityAlert
+  | MlAnomalyDetectionAlert;

x-pack/plugins/alerting/common/alert_schema/context_to_schema_name.ts

@@ -9,7 +9,7 @@ import { capitalize } from 'lodash';
 
 export const contextToSchemaName = (context: string) => {
   return `${context
-    .split('.')
+    .split(/[.\-]/)
     .map((part: string) => capitalize(part))
     .join('')}Alert`;
 };

x-pack/plugins/ml/common/types/alerts.ts

@@ -44,6 +44,35 @@ interface BaseAnomalyAlertDoc {
   unique_key: string;
 }
 
+export interface TopRecordAADDoc {
+  job_id: string;
+  record_score: number;
+  initial_record_score: number;
+  timestamp: number;
+  is_interim: boolean;
+  function: string;
+  field_name?: string;
+  by_field_name?: string;
+  by_field_value?: string | number;
+  over_field_name?: string;
+  over_field_value?: string | number;
+  partition_field_name?: string;
+  partition_field_value?: string | number;
+  typical: number[];
+  actual: number[];
+  detector_index: number;
+}
+
+export interface TopInfluencerAADDoc {
+  job_id: string;
+  influencer_score: number;
+  initial_influencer_score: number;
+  is_interim: boolean;
+  timestamp: number;
+  influencer_field_name: string;
+  influencer_field_value: string | number;
+}
+
 export interface RecordAnomalyAlertDoc extends BaseAnomalyAlertDoc {
   result_type: typeof ML_ANOMALY_RESULT_TYPE.RECORD;
   function: string;