Merge branch 'main' into 2023_07_17-color_mapping_lens

x-pack/plugins/aiops/common/api/log_categorization/create_categorize_query.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { cloneDeep } from 'lodash';
+
+import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
+
+export function createCategorizeQuery(
+  queryIn: QueryDslQueryContainer,
+  timeField: string,
+  from: number | undefined,
+  to: number | undefined
+) {
+  const query = cloneDeep(queryIn);
+
+  if (query.bool === undefined) {
+    query.bool = {};
+  }
+  if (query.bool.must === undefined) {
+    query.bool.must = [];
+    if (query.match_all !== undefined) {
+      query.bool.must.push({ match_all: query.match_all });
+      delete query.match_all;
+    }
+  }
+  if (query.multi_match !== undefined) {
+    query.bool.should = {
+      multi_match: query.multi_match,
+    };
+    delete query.multi_match;
+  }
+
+  (query.bool.must as QueryDslQueryContainer[]).push({
+    range: {
+      [timeField]: {
+        gte: from,
+        lte: to,
+        format: 'epoch_millis',
+      },
+    },
+  });
+
+  return query;
+}

x-pack/plugins/aiops/common/api/log_categorization/create_category_request.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
+
+import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils';
+
+import { createCategorizeQuery } from './create_categorize_query';
+
+const CATEGORY_LIMIT = 1000;
+const EXAMPLE_LIMIT = 1;
+
+export function createCategoryRequest(
+  index: string,
+  field: string,
+  timeField: string,
+  from: number | undefined,
+  to: number | undefined,
+  queryIn: QueryDslQueryContainer,
+  wrap: ReturnType<typeof createRandomSamplerWrapper>['wrap'],
+  intervalMs?: number
+) {
+  const query = createCategorizeQuery(queryIn, timeField, from, to);
+  const aggs = {
+    categories: {
+      categorize_text: {
+        field,
+        size: CATEGORY_LIMIT,
+      },
+      aggs: {
+        hit: {
+          top_hits: {
+            size: EXAMPLE_LIMIT,
+            sort: [timeField],
+            _source: field,
+          },
+        },
+        ...(intervalMs
+          ? {
+              sparkline: {
+                date_histogram: {
+                  field: timeField,
+                  fixed_interval: `${intervalMs}ms`,
+                },
+              },
+            }
+          : {}),
+      },
+    },
+  };
+
+  return {
+    params: {
+      index,
+      size: 0,
+      body: {
+        query,
+        aggs: wrap(aggs),
+      },
+    },
+  };
+}

x-pack/plugins/aiops/common/api/log_categorization/get_category_query.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Category } from './types';
+
+export const QUERY_MODE = {
+  INCLUDE: 'should',
+  EXCLUDE: 'must_not',
+} as const;
+export type QueryMode = typeof QUERY_MODE[keyof typeof QUERY_MODE];
+
+export const getCategoryQuery = (
+  field: string,
+  categories: Category[],
+  mode: QueryMode = QUERY_MODE.INCLUDE
+) => ({
+  bool: {
+    [mode]: categories.map(({ key: query }) => ({
+      match: {
+        [field]: {
+          auto_generate_synonyms_phrase_query: false,
+          fuzziness: 0,
+          operator: 'and',
+          query,
+        },
+      },
+    })),
+  },
+});

x-pack/plugins/aiops/common/api/log_categorization/process_category_results.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { get } from 'lodash';
+
+import { estypes } from '@elastic/elasticsearch';
+
+import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils';
+
+import type { Category, CategoriesAgg, CatResponse, SparkLinesPerCategory } from './types';
+
+export function processCategoryResults(
+  result: CatResponse,
+  field: string,
+  unwrap: ReturnType<typeof createRandomSamplerWrapper>['unwrap']
+) {
+  const sparkLinesPerCategory: SparkLinesPerCategory = {};
+  const { aggregations } = result.rawResponse;
+  if (aggregations === undefined) {
+    throw new Error('processCategoryResults failed, did not return aggregations.');
+  }
+  const {
+    categories: { buckets },
+  } = unwrap(
+    aggregations as unknown as Record<string, estypes.AggregationsAggregate>
+  ) as CategoriesAgg;
+
+  const categories: Category[] = buckets.map((b) => {
+    sparkLinesPerCategory[b.key] =
+      b.sparkline === undefined
+        ? {}
+        : b.sparkline.buckets.reduce<Record<number, number>>((acc2, cur2) => {
+            acc2[cur2.key] = cur2.doc_count;
+            return acc2;
+          }, {});
+
+    return {
+      key: b.key,
+      count: b.doc_count,
+      examples: b.hit.hits.hits.map((h) => get(h._source, field)),
+    };
+  });
+  return {
+    categories,
+    sparkLinesPerCategory,
+  };
+}

x-pack/plugins/aiops/common/api/log_categorization/types.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { estypes } from '@elastic/elasticsearch';
+
+export interface Category {
+  key: string;
+  count: number;
+  examples: string[];
+  sparkline?: Array<{ doc_count: number; key: number; key_as_string: string }>;
+}
+
+export interface CategoriesAgg {
+  categories: {
+    buckets: Array<{
+      key: string;
+      doc_count: number;
+      hit: { hits: { hits: Array<{ _source: { message: string } }> } };
+      sparkline: {
+        buckets: Array<{ key_as_string: string; key: number; doc_count: number }>;
+      };
+    }>;
+  };
+}
+
+interface CategoriesSampleAgg {
+  sample: CategoriesAgg;
+}
+
+export interface CatResponse {
+  rawResponse: estypes.SearchResponseBody<unknown, CategoriesAgg | CategoriesSampleAgg>;
+}
+
+export type SparkLinesPerCategory = Record<string, Record<number, number>>;

x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx

@@ -6,8 +6,7 @@
  */
 
 import React, { FC, useMemo, useState } from 'react';
-import { i18n } from '@kbn/i18n';
-import type { TimefilterContract } from '@kbn/data-plugin/public';
+
 import {
   useEuiBackgroundColor,
   EuiInMemoryTable,
@@ -19,14 +18,25 @@ import {
   EuiSpacer,
 } from '@elastic/eui';
 
+import { i18n } from '@kbn/i18n';
+import type { TimefilterContract } from '@kbn/data-plugin/public';
 import { DataViewField } from '@kbn/data-views-plugin/common';
 import { Filter } from '@kbn/es-query';
 import { useTableState } from '@kbn/ml-in-memory-table';
-import { useDiscoverLinks, createFilter, QueryMode, QUERY_MODE } from '../use_discover_links';
-import { MiniHistogram } from '../../mini_histogram';
+
+import type {
+  Category,
+  SparkLinesPerCategory,
+} from '../../../../common/api/log_categorization/types';
+
 import { useEuiTheme } from '../../../hooks/use_eui_theme';
 import type { LogCategorizationAppState } from '../../../application/utils/url_state';
-import type { EventRate, Category, SparkLinesPerCategory } from '../use_categorize_request';
+
+import { MiniHistogram } from '../../mini_histogram';
+
+import { useDiscoverLinks, createFilter, QueryMode, QUERY_MODE } from '../use_discover_links';
+import type { EventRate } from '../use_categorize_request';
+
 import { getLabels } from './labels';
 import { TableHeader } from './table_header';
 

x-pack/plugins/aiops/public/components/log_categorization/document_count_chart.tsx

@@ -9,11 +9,15 @@ import React, { FC, useMemo } from 'react';
 
 import { i18n } from '@kbn/i18n';
 import { DocumentCountChart as DocumentCountChartRoot } from '@kbn/aiops-components';
+
+import type { Category, SparkLinesPerCategory } from '../../../common/api/log_categorization/types';
+
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
+import { DocumentCountStats } from '../../get_document_stats';
+
 import { TotalCountHeader } from '../document_count_content/total_count_header';
-import type { Category, SparkLinesPerCategory } from './use_categorize_request';
+
 import type { EventRate } from './use_categorize_request';
-import { DocumentCountStats } from '../../get_document_stats';
 
 interface Props {
   totalCount: number;

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx

@@ -5,10 +5,7 @@
  * 2.0.
  */
 import React, { FC, useState, useEffect, useCallback, useRef, useMemo } from 'react';
-import type { SavedSearch } from '@kbn/saved-search-plugin/public';
-import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
-import { i18n } from '@kbn/i18n';
-import { FormattedMessage } from '@kbn/i18n-react';
+
 import {
   EuiTitle,
   EuiFlyoutHeader,
@@ -18,26 +15,33 @@ import {
   useEuiTheme,
 } from '@elastic/eui';
 
+import type { SavedSearch } from '@kbn/saved-search-plugin/public';
+import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n-react';
 import { buildEmptyFilter, Filter } from '@kbn/es-query';
-
 import { usePageUrlState } from '@kbn/ml-url-state';
 import type { FieldValidationResults } from '@kbn/ml-category-validator';
+
+import type { Category, SparkLinesPerCategory } from '../../../common/api/log_categorization/types';
+
+import {
+  type LogCategorizationPageUrlState,
+  getDefaultLogCategorizationAppState,
+} from '../../application/utils/url_state';
+import { createMergedEsQuery } from '../../application/utils/search_utils';
 import { useData } from '../../hooks/use_data';
 import { useSearch } from '../../hooks/use_search';
+import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
+
 import { useCategorizeRequest } from './use_categorize_request';
-import type { EventRate, Category, SparkLinesPerCategory } from './use_categorize_request';
+import type { EventRate } from './use_categorize_request';
 import { CategoryTable } from './category_table';
-import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 import { InformationText } from './information_text';
-import { createMergedEsQuery } from '../../application/utils/search_utils';
 import { SamplingMenu } from './sampling_menu';
 import { TechnicalPreviewBadge } from './technical_preview_badge';
 import { LoadingCategorization } from './loading_categorization';
 import { useValidateFieldRequest } from './use_validate_category_field';
-import {
-  type LogCategorizationPageUrlState,
-  getDefaultLogCategorizationAppState,
-} from '../../application/utils/url_state';
 import { FieldValidationCallout } from './category_validation_callout';
 
 export interface LogCategorizationPageProps {

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx

@@ -24,9 +24,11 @@ import { Filter, Query } from '@kbn/es-query';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { usePageUrlState, useUrlState } from '@kbn/ml-url-state';
-
 import type { FieldValidationResults } from '@kbn/ml-category-validator';
 import type { SearchQueryLanguage } from '@kbn/ml-query-utils';
+
+import type { Category, SparkLinesPerCategory } from '../../../common/api/log_categorization/types';
+
 import { useDataSource } from '../../hooks/use_data_source';
 import { useData } from '../../hooks/use_data';
 import { useSearch } from '../../hooks/use_search';
@@ -39,7 +41,7 @@ import {
 import { SearchPanel } from '../search_panel';
 import { PageHeader } from '../page_header';
 
-import type { EventRate, Category, SparkLinesPerCategory } from './use_categorize_request';
+import type { EventRate } from './use_categorize_request';
 import { useCategorizeRequest } from './use_categorize_request';
 import { CategoryTable } from './category_table';
 import { DocumentCountChart } from './document_count_chart';