[8.13] [ML] Fixes ES|QL field stats showing skewed numeric distributi…

…on, duplicated examples for string fields, and adds functional tests (#177085) (#178586) # Backport This will backport the following commits from `main` to `8.13`: - [[ML] Fixes ES|QL field stats showing skewed numeric distribution, duplicated examples for string fields, and adds functional tests (#177085)](#177085)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Quynh Nguyen (Quinn) <[email protected]>
elastic · Mar 13, 2024 · 5950c69 · 5950c69
1 parent 1cb17ce
commit 5950c69
Show file tree

Hide file tree

Showing 13 changed files with 412 additions and 22 deletions.
diff --git a/x-pack/plugins/data_visualizer/common/types/field_stats.ts b/x-pack/plugins/data_visualizer/common/types/field_stats.ts
@@ -93,8 +93,8 @@ export interface StringFieldStats {
   fieldName: string;
   isTopValuesSampled: boolean;
   topValues: Bucket[];
-  topValuesSampleSize: number;
-  topValuesSamplerShardSize: number;
+  topValuesSampleSize?: number;
+  topValuesSamplerShardSize?: number;
 }
 
 export interface DateFieldStats {

diff --git a/...ck/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/...ck/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx
@@ -54,10 +54,16 @@ export const TopValues: FC<Props> = ({ stats, fieldFormat, barColor, compressed,
   } = useDataVisualizerKibana();
 
   if (stats === undefined || !stats.topValues) return null;
-  const { topValues, fieldName, sampleCount } = stats;
+  const { topValues: originalTopValues, fieldName, sampleCount } = stats;
 
-  if (topValues?.length === 0) return null;
+  if (originalTopValues?.length === 0) return null;
   const totalDocuments = stats.totalDocuments ?? sampleCount ?? 0;
+
+  const topValues = originalTopValues.map((bucket) => ({
+    ...bucket,
+    percent:
+      typeof bucket.percent === 'number' ? bucket.percent : bucket.doc_count / totalDocuments,
+  }));
   const topValuesOtherCountPercent =
     1 - (topValues ? topValues.reduce((acc, bucket) => acc + bucket.percent, 0) : 0);
   const topValuesOtherCount = Math.floor(topValuesOtherCountPercent * (sampleCount ?? 0));

diff --git a/...izer/public/application/index_data_visualizer/components/search_panel/esql/limit_size.tsx b/...izer/public/application/index_data_visualizer/components/search_panel/esql/limit_size.tsx
@@ -12,34 +12,39 @@ import type { ESQLDefaultLimitSizeOption } from '../../../embeddables/grid_embed
 
 const options = [
   {
+    'data-test-subj': 'dvESQLLimitSize-5000',
     value: '5000',
     text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.limitSizeOptionLabel', {
       defaultMessage: '{limit} rows',
       values: { limit: '5,000' },
     }),
   },
   {
+    'data-test-subj': 'dvESQLLimitSize-10000',
     value: '10000',
     text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.limitSizeOptionLabel', {
       defaultMessage: '{limit} rows',
       values: { limit: '10,000' },
     }),
   },
   {
+    'data-test-subj': 'dvESQLLimitSize-100000',
     value: '100000',
     text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.limitSizeOptionLabel', {
       defaultMessage: '{limit} rows',
       values: { limit: '100,000' },
     }),
   },
   {
+    'data-test-subj': 'dvESQLLimitSize-1000000',
     value: '1000000',
     text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.limitSizeOptionLabel', {
       defaultMessage: '{limit} rows',
       values: { limit: '1,000,000' },
     }),
   },
   {
+    'data-test-subj': 'dvESQLLimitSize-none',
     value: 'none',
     text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.analyzeAll', {
       defaultMessage: 'Analyze all',
@@ -62,6 +67,7 @@ export const ESQLDefaultLimitSizeSelect = ({
 
   return (
     <EuiSelect
+      data-test-subj="dvESQLLimitSizeSelect"
       id={basicSelectId}
       options={options}
       value={limitSize}

diff --git a/...lic/application/index_data_visualizer/search_strategy/esql_requests/get_keyword_fields.ts b/...lic/application/index_data_visualizer/search_strategy/esql_requests/get_keyword_fields.ts
@@ -63,19 +63,14 @@ export const getESQLKeywordFieldStats = async ({
         if (isFulfilled(resp)) {
           const results = resp.value?.rawResponse.values as Array<[BucketCount, BucketTerm]>;
           if (results) {
-            const topValuesSampleSize = results?.reduce((acc: number, row) => acc + row[0], 0);
-
             const terms = results.map((row) => ({
               key: row[1],
               doc_count: row[0],
-              percent: row[0] / topValuesSampleSize,
             }));
 
             return {
               fieldName: field.name,
               topValues: terms,
-              topValuesSampleSize,
-              topValuesSamplerShardSize: topValuesSampleSize,
               isTopValuesSampled: false,
             } as StringFieldStats;
           }

diff --git a/...pplication/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts b/...pplication/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts
@@ -92,7 +92,7 @@ const getESQLNumericFieldStatsInChunk = async ({
           const median = values[startIndex + numericAccessorMap.p50];
 
           const percentiles = values
-            .slice(startIndex + numericAccessorMap.p0, startIndex + numericAccessorMap.p100)
+            .slice(startIndex + numericAccessorMap.p5, startIndex + numericAccessorMap.p100 + 1)
             .map((value: number) => ({ value }));
 
           const distribution = processDistributionData(percentiles, PERCENTILE_SPACING, min);

diff --git a/...c/application/index_data_visualizer/search_strategy/esql_requests/get_text_field_stats.ts b/...c/application/index_data_visualizer/search_strategy/esql_requests/get_text_field_stats.ts
@@ -45,9 +45,9 @@ export const getESQLExampleFieldValues = async ({
 
       if (textFieldsResp) {
         return textFields.map((textField, idx) => {
-          const examples = (textFieldsResp.rawResponse.values as unknown[][]).map(
-            (row) => row[idx]
-          );
+          const examples = [
+            ...new Set((textFieldsResp.rawResponse.values as unknown[][]).map((row) => row[idx])),
+          ];
 
           return {
             fieldName: textField.name,