[ML] AIOps: Log Rate Analysis V2 REST API, replaces references to `te…

…rm` with `item`. (elastic#170274)

This PR uses [conditional
types](https://www.typescriptlang.org/docs/handbook/2/conditional-types.html)
to allow the handling of both multiple API versions within one route handler. The more tricky bit turned out
to be not the updated request body, but the response since it is an
NDJSON stream where some messages were updated. In this case also the
functions that create these messages were updated with conditional types
to be able to create a message that fits the definition of the API
version.

The API integration tests originally had these message identifiers in
the `expected` section of their `testData`. I changed that to use helper
functions that retrieve the expected messages from the stream according
to the expected version. All API integration tests are run on both
versions. The functional tests are run only on the newer version since
the UI is expected to work with version `2` only.

x-pack/packages/ml/agg_utils/src/types.ts

@@ -149,24 +149,37 @@ export interface SignificantTerm extends FieldValuePair {
   unique?: boolean;
 }
 
-/**
- * Represents a data item in a significant term histogram.
- * @interface
- */
-export interface SignificantTermHistogramItem {
+interface SignificantTermHistogramItemBase {
   /** The document count for this item in the overall context. */
   doc_count_overall: number;
 
-  /** The document count for this item in the significant term context. */
-  doc_count_significant_term: number;
-
   /** The numeric key associated with this item. */
   key: number;
 
   /** The string representation of the key. */
   key_as_string: string;
 }
 
+/**
+ * @deprecated since version 2 of internal log rate analysis REST API endpoint
+ */
+interface SignificantTermHistogramItemV1 extends SignificantTermHistogramItemBase {
+  /** The document count for this item in the significant term context. */
+  doc_count_significant_term: number;
+}
+
+interface SignificantTermHistogramItemV2 extends SignificantTermHistogramItemBase {
+  /** The document count for this histogram item in the significant item context. */
+  doc_count_significant_item: number;
+}
+
+/**
+ * Represents a data item in a significant term histogram.
+ */
+export type SignificantTermHistogramItem =
+  | SignificantTermHistogramItemV1
+  | SignificantTermHistogramItemV2;
+
 /**
  * Represents histogram data for a field/value pair.
  * @interface

x-pack/plugins/aiops/common/api/index.ts

@@ -5,27 +5,10 @@
  * 2.0.
  */
 
-import type { HttpSetup } from '@kbn/core/public';
-
-import type {
-  AiopsLogRateAnalysisSchema,
-  AiopsLogRateAnalysisApiAction,
-} from './log_rate_analysis';
-import { streamReducer } from './stream_reducer';
-
 export const AIOPS_API_ENDPOINT = {
   LOG_RATE_ANALYSIS: '/internal/aiops/log_rate_analysis',
   CATEGORIZATION_FIELD_VALIDATION: '/internal/aiops/categorization_field_validation',
 } as const;
 
 type AiopsApiEndpointKeys = keyof typeof AIOPS_API_ENDPOINT;
 export type AiopsApiEndpoint = typeof AIOPS_API_ENDPOINT[AiopsApiEndpointKeys];
-
-export interface AiopsApiLogRateAnalysis {
-  http: HttpSetup;
-  endpoint: AiopsApiEndpoint;
-  apiVersion: string;
-  reducer: typeof streamReducer;
-  body: AiopsLogRateAnalysisSchema;
-  actions: AiopsLogRateAnalysisApiAction;
-}

x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts

@@ -12,10 +12,24 @@ import type {
   SignificantTermGroupHistogram,
 } from '@kbn/ml-agg-utils';
 
+import type { AiopsLogRateAnalysisApiVersion as ApiVersion } from './schema';
+
 export const API_ACTION_NAME = {
+  /** @since API v2 */
+  ADD_SIGNIFICANT_ITEMS: 'add_significant_items',
+  /** @since API v2 */
+  ADD_SIGNIFICANT_ITEMS_HISTOGRAM: 'add_significant_items_histogram',
+  /** @since API v2 */
+  ADD_SIGNIFICANT_ITEMS_GROUP: 'add_significant_items_group',
+  /** @since API v2 */
+  ADD_SIGNIFICANT_ITEMS_GROUP_HISTOGRAM: 'add_significant_items_group_histogram',
+  /** @deprecated since API v2 */
   ADD_SIGNIFICANT_TERMS: 'add_significant_terms',
+  /** @deprecated since API v2 */
   ADD_SIGNIFICANT_TERMS_HISTOGRAM: 'add_significant_terms_histogram',
+  /** @deprecated since API v2 */
   ADD_SIGNIFICANT_TERMS_GROUP: 'add_significant_terms_group',
+  /** @deprecated since API v2 */
   ADD_SIGNIFICANT_TERMS_GROUP_HISTOGRAM: 'add_significant_terms_group_histogram',
   ADD_ERROR: 'add_error',
   PING: 'ping',
@@ -26,60 +40,108 @@ export const API_ACTION_NAME = {
 } as const;
 export type ApiActionName = typeof API_ACTION_NAME[keyof typeof API_ACTION_NAME];
 
-interface ApiActionAddSignificantTerms {
-  type: typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS;
+interface ApiActionAddSignificantTerms<T extends ApiVersion> {
+  type: T extends '1'
+    ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS
+    : T extends '2'
+    ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS
+    : never;
   payload: SignificantTerm[];
 }
 
-export function addSignificantTermsAction(
-  payload: ApiActionAddSignificantTerms['payload']
-): ApiActionAddSignificantTerms {
+export function addSignificantTermsAction<T extends ApiVersion>(
+  payload: ApiActionAddSignificantTerms<T>['payload'],
+  version: T
+): ApiActionAddSignificantTerms<T> {
+  if (version === '1') {
+    return {
+      type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS,
+      payload,
+    } as ApiActionAddSignificantTerms<T>;
+  }
+
   return {
-    type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS,
+    type: API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS,
     payload,
-  };
+  } as ApiActionAddSignificantTerms<T>;
 }
 
-interface ApiActionAddSignificantTermsHistogram {
-  type: typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_HISTOGRAM;
+interface ApiActionAddSignificantTermsHistogram<T extends ApiVersion> {
+  type: T extends '1'
+    ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_HISTOGRAM
+    : T extends '2'
+    ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_HISTOGRAM
+    : never;
   payload: SignificantTermHistogram[];
 }
 
-export function addSignificantTermsHistogramAction(
-  payload: ApiActionAddSignificantTermsHistogram['payload']
-): ApiActionAddSignificantTermsHistogram {
+export function addSignificantTermsHistogramAction<T extends ApiVersion>(
+  payload: ApiActionAddSignificantTermsHistogram<T>['payload'],
+  version: T
+): ApiActionAddSignificantTermsHistogram<T> {
+  if (version === '1') {
+    return {
+      type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_HISTOGRAM,
+      payload,
+    } as ApiActionAddSignificantTermsHistogram<T>;
+  }
+
   return {
-    type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_HISTOGRAM,
+    type: API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_HISTOGRAM,
     payload,
-  };
+  } as ApiActionAddSignificantTermsHistogram<T>;
 }
 
-interface ApiActionAddSignificantTermsGroup {
-  type: typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP;
+interface ApiActionAddSignificantTermsGroup<T extends ApiVersion> {
+  type: T extends '1'
+    ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP
+    : T extends '2'
+    ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_GROUP
+    : never;
   payload: SignificantTermGroup[];
 }
 
-export function addSignificantTermsGroupAction(
-  payload: ApiActionAddSignificantTermsGroup['payload']
-) {
+export function addSignificantTermsGroupAction<T extends ApiVersion>(
+  payload: ApiActionAddSignificantTermsGroup<T>['payload'],
+  version: T
+): ApiActionAddSignificantTermsGroup<T> {
+  if (version === '1') {
+    return {
+      type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP,
+      payload,
+    } as ApiActionAddSignificantTermsGroup<T>;
+  }
+
   return {
-    type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP,
+    type: API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_GROUP,
     payload,
-  };
+  } as ApiActionAddSignificantTermsGroup<T>;
 }
 
-interface ApiActionAddSignificantTermsGroupHistogram {
-  type: typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP_HISTOGRAM;
+interface ApiActionAddSignificantTermsGroupHistogram<T extends ApiVersion> {
+  type: T extends '1'
+    ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP_HISTOGRAM
+    : T extends '2'
+    ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_GROUP_HISTOGRAM
+    : never;
   payload: SignificantTermGroupHistogram[];
 }
 
-export function addSignificantTermsGroupHistogramAction(
-  payload: ApiActionAddSignificantTermsGroupHistogram['payload']
-): ApiActionAddSignificantTermsGroupHistogram {
+export function addSignificantTermsGroupHistogramAction<T extends ApiVersion>(
+  payload: ApiActionAddSignificantTermsGroupHistogram<T>['payload'],
+  version: T
+): ApiActionAddSignificantTermsGroupHistogram<T> {
+  if (version === '1') {
+    return {
+      type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP_HISTOGRAM,
+      payload,
+    } as ApiActionAddSignificantTermsGroupHistogram<T>;
+  }
+
   return {
-    type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP_HISTOGRAM,
+    type: API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_GROUP_HISTOGRAM,
     payload,
-  };
+  } as ApiActionAddSignificantTermsGroupHistogram<T>;
 }
 
 interface ApiActionAddError {
@@ -148,11 +210,11 @@ export function updateLoadingStateAction(
   };
 }
 
-export type AiopsLogRateAnalysisApiAction =
-  | ApiActionAddSignificantTerms
-  | ApiActionAddSignificantTermsGroup
-  | ApiActionAddSignificantTermsHistogram
-  | ApiActionAddSignificantTermsGroupHistogram
+export type AiopsLogRateAnalysisApiAction<T extends ApiVersion> =
+  | ApiActionAddSignificantTerms<T>
+  | ApiActionAddSignificantTermsGroup<T>
+  | ApiActionAddSignificantTermsHistogram<T>
+  | ApiActionAddSignificantTermsGroupHistogram<T>
   | ApiActionAddError
   | ApiActionPing
   | ApiActionResetAll

x-pack/plugins/aiops/common/api/log_rate_analysis/schema.ts

@@ -5,37 +5,17 @@
  * 2.0.
  */
 
-import { schema, TypeOf } from '@kbn/config-schema';
+import type { AiopsLogRateAnalysisSchemaV1 } from './schema_v1';
+import type { AiopsLogRateAnalysisSchemaV2 } from './schema_v2';
 
-export const aiopsLogRateAnalysisSchema = schema.object({
-  start: schema.number(),
-  end: schema.number(),
-  searchQuery: schema.string(),
-  timeFieldName: schema.string(),
-  includeFrozen: schema.maybe(schema.boolean()),
-  grouping: schema.maybe(schema.boolean()),
-  /** Analysis selection time ranges */
-  baselineMin: schema.number(),
-  baselineMax: schema.number(),
-  deviationMin: schema.number(),
-  deviationMax: schema.number(),
-  /** The index to query for log rate analysis */
-  index: schema.string(),
-  /** Settings to override headers derived compression and flush fix */
-  compressResponse: schema.maybe(schema.boolean()),
-  flushFix: schema.maybe(schema.boolean()),
-  /** Overrides to skip steps of the analysis with existing data */
-  overrides: schema.maybe(
-    schema.object({
-      loaded: schema.maybe(schema.number()),
-      remainingFieldCandidates: schema.maybe(schema.arrayOf(schema.string())),
-      // TODO Improve schema
-      significantTerms: schema.maybe(schema.arrayOf(schema.any())),
-      regroupOnly: schema.maybe(schema.boolean()),
-    })
-  ),
-  /** Probability used for the random sampler aggregations */
-  sampleProbability: schema.maybe(schema.number()),
-});
+export type AiopsLogRateAnalysisApiVersion = '1' | '2';
 
-export type AiopsLogRateAnalysisSchema = TypeOf<typeof aiopsLogRateAnalysisSchema>;
+const LATEST_API_VERSION: AiopsLogRateAnalysisApiVersion = '2';
+
+export type AiopsLogRateAnalysisSchema<
+  T extends AiopsLogRateAnalysisApiVersion = typeof LATEST_API_VERSION
+> = T extends '1'
+  ? AiopsLogRateAnalysisSchemaV1
+  : T extends '2'
+  ? AiopsLogRateAnalysisSchemaV2
+  : never;

x-pack/plugins/aiops/common/api/log_rate_analysis/schema_v1.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+export const aiopsLogRateAnalysisSchemaV1 = schema.object({
+  start: schema.number(),
+  end: schema.number(),
+  searchQuery: schema.string(),
+  timeFieldName: schema.string(),
+  includeFrozen: schema.maybe(schema.boolean()),
+  grouping: schema.maybe(schema.boolean()),
+  /** Analysis selection time ranges */
+  baselineMin: schema.number(),
+  baselineMax: schema.number(),
+  deviationMin: schema.number(),
+  deviationMax: schema.number(),
+  /** The index to query for log rate analysis */
+  index: schema.string(),
+  /** Settings to override headers derived compression and flush fix */
+  compressResponse: schema.maybe(schema.boolean()),
+  flushFix: schema.maybe(schema.boolean()),
+  /** Overrides to skip steps of the analysis with existing data */
+  overrides: schema.maybe(
+    schema.object({
+      loaded: schema.maybe(schema.number()),
+      remainingFieldCandidates: schema.maybe(schema.arrayOf(schema.string())),
+      // TODO Improve schema
+      significantTerms: schema.maybe(schema.arrayOf(schema.any())),
+      regroupOnly: schema.maybe(schema.boolean()),
+    })
+  ),
+  /** Probability used for the random sampler aggregations */
+  sampleProbability: schema.maybe(schema.number()),
+});
+
+export type AiopsLogRateAnalysisSchemaV1 = TypeOf<typeof aiopsLogRateAnalysisSchemaV1>;