[ML] Provides hints for empty fields in dropdown options in Anomaly detection & Transform creation wizards, Change point detection view

x-pack/plugins/ml/public/application/components/field_stats_flyout/field_stats_flyout_provider.tsx

@@ -10,15 +10,36 @@ import type { DataView } from '@kbn/data-plugin/common';
 import type { FieldStatsServices } from '@kbn/unified-field-list/src/components/field_stats';
 import type { TimeRange as TimeRangeMs } from '@kbn/ml-date-picker';
 import type { FieldStatsProps } from '@kbn/unified-field-list/src/components/field_stats';
-import { MLFieldStatsFlyoutContext } from './use_field_stats_flytout_context';
+import { useEffect } from 'react';
+import { getProcessedFields } from '@kbn/ml-data-grid';
+import { stringHash } from '@kbn/ml-string-hash';
+import { lastValueFrom } from 'rxjs';
+import { useRef } from 'react';
+import { getMergedSampleDocsForPopulatedFieldsQuery } from './populated_fields/get_merged_populated_fields_query';
+import { useMlKibana } from '../../contexts/kibana';
 import { FieldStatsFlyout } from './field_stats_flyout';
+import { MLFieldStatsFlyoutContext } from './use_field_stats_flytout_context';
+import { PopulatedFieldsCacheManager } from './populated_fields/populated_fields_cache_manager';
 
 export const FieldStatsFlyoutProvider: FC<{
   dataView: DataView;
   fieldStatsServices: FieldStatsServices;
   timeRangeMs?: TimeRangeMs;
   dslQuery?: FieldStatsProps['dslQuery'];
-}> = ({ dataView, fieldStatsServices, timeRangeMs, dslQuery, children }) => {
+  disablePopulatedFields?: boolean;
+}> = ({
+  dataView,
+  fieldStatsServices,
+  timeRangeMs,
+  dslQuery,
+  disablePopulatedFields = false,
+  children,
+}) => {
+  const {
+    services: {
+      data: { search },
+    },
+  } = useMlKibana();
   const [isFieldStatsFlyoutVisible, setFieldStatsIsFlyoutVisible] = useState(false);
   const [fieldName, setFieldName] = useState<string | undefined>();
   const [fieldValue, setFieldValue] = useState<string | number | undefined>();
@@ -27,6 +48,77 @@ export const FieldStatsFlyoutProvider: FC<{
     () => setFieldStatsIsFlyoutVisible(!isFieldStatsFlyoutVisible),
     [isFieldStatsFlyoutVisible]
   );
+  const [manager] = useState(new PopulatedFieldsCacheManager());
+  const [populatedFields, setPopulatedFields] = useState<Set<string> | undefined>();
+  const abortController = useRef(new AbortController());
+
+  useEffect(
+    function fetchSampleDocsEffect() {
+      if (disablePopulatedFields) return;
+
+      const queryAndRunTimeMappings = getMergedSampleDocsForPopulatedFieldsQuery({
+        searchQuery: dslQuery,
+        runtimeFields: dataView.getRuntimeMappings(),
+        datetimeField: dataView.getTimeField()?.name,
+        timeRange: timeRangeMs,
+      });
+      const indexPattern = dataView.getIndexPattern();
+      const esSearchRequestParams = {
+        index: indexPattern,
+        body: {
+          fields: ['*'],
+          _source: false,
+          ...queryAndRunTimeMappings,
+          size: 1000,
+        },
+      };
+      const cacheKey = stringHash(JSON.stringify(esSearchRequestParams)).toString();
+
+      const fetchSampleDocuments = async function () {
+        try {
+          const resp = await lastValueFrom(
+            search.search(
+              {
+                params: esSearchRequestParams,
+              },
+              { abortSignal: abortController.current.signal }
+            )
+          );
+
+          const docs = resp.rawResponse.hits.hits.map((d) => getProcessedFields(d.fields ?? {}));
+
+          // Get all field names for each returned doc and flatten it
+          // to a list of unique field names used across all docs.
+          const fieldsWithData = new Set(docs.map(Object.keys).flat(1));
+          manager.set(cacheKey, fieldsWithData);
+          setPopulatedFields(fieldsWithData);
+        } catch (e) {
+          if (e.name !== 'AbortError') {
+            // eslint-disable-next-line no-console
+            console.error(
+              `An error occurred fetching sample documents to determine populated field stats.
+          \nQuery:\n${JSON.stringify(esSearchRequestParams)}
+          \nError:${e}`
+            );
+          }
+        }
+      };
+
+      const cachedResult = manager.get(cacheKey);
+      if (cachedResult) {
+        return cachedResult;
+      } else {
+        fetchSampleDocuments();
+      }
+
+      return () => {
+        abortController.current.abort();
+        abortController.current = new AbortController();
+      };
+    },
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [JSON.stringify({ dslQuery, dataViewId: dataView.id, timeRangeMs })]
+  );
 
   return (
     <MLFieldStatsFlyoutContext.Provider
@@ -38,6 +130,8 @@ export const FieldStatsFlyoutProvider: FC<{
         fieldName,
         setFieldValue,
         fieldValue,
+        timeRangeMs,
+        populatedFields,
       }}
     >
       <FieldStatsFlyout

x-pack/plugins/ml/public/application/components/field_stats_flyout/field_stats_info_button.tsx

@@ -5,66 +5,92 @@
  * 2.0.
  */
 
-import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiHighlight, EuiToolTip } from '@elastic/eui';
+import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiToolTip, EuiText } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import React from 'react';
 import { FieldIcon } from '@kbn/react-field';
-import { EVENT_RATE_FIELD_ID, type Field } from '@kbn/ml-anomaly-utils';
+import { type Field } from '@kbn/ml-anomaly-utils';
+import { useCurrentThemeVars } from '../../contexts/kibana';
 import { getKbnFieldIconType } from '../../../../common/util/get_field_icon_types';
 
 export type FieldForStats = Pick<Field, 'id' | 'type'>;
 export const FieldStatsInfoButton = ({
   field,
   label,
-  searchValue = '',
   onButtonClick,
+  disabled,
+  isEmpty = false,
+  hideTrigger = false,
 }: {
   field: FieldForStats;
   label: string;
   searchValue?: string;
+  disabled?: boolean;
+  isEmpty?: boolean;
   onButtonClick?: (field: FieldForStats) => void;
+  hideTrigger?: boolean;
 }) => {
+  const themeVars = useCurrentThemeVars();
+  const emptyFieldMessage = isEmpty
+    ? ' ' +
+      i18n.translate('xpack.ml.newJob.wizard.fieldContextPopover.emptyFieldInSampleDocsMsg', {
+        defaultMessage: '(no data found in 1000 sample records)',
+      })
+    : '';
   return (
     <EuiFlexGroup gutterSize="none" alignItems="center">
       <EuiFlexItem grow={false}>
-        <EuiToolTip
-          content={i18n.translate(
-            'xpack.ml.newJob.wizard.fieldContextPopover.inspectFieldStatsTooltip',
-            {
-              defaultMessage: 'Inspect field statistics',
+        {!hideTrigger ? (
+          <EuiToolTip
+            content={
+              i18n.translate(
+                'xpack.ml.newJob.wizard.fieldContextPopover.inspectFieldStatsTooltip',
+                {
+                  defaultMessage: 'Inspect field statistics',
+                }
+              ) + emptyFieldMessage
             }
-          )}
-        >
-          <EuiButtonIcon
-            data-test-subj={`mlInspectFieldStatsButton-${field.id}`}
-            disabled={field.id === EVENT_RATE_FIELD_ID}
-            size="xs"
-            iconType="inspect"
-            onClick={(ev: React.MouseEvent<HTMLButtonElement>) => {
-              if (ev.type === 'click') {
-                ev.currentTarget.focus();
-              }
-              ev.preventDefault();
-              ev.stopPropagation();
+          >
+            <EuiButtonIcon
+              data-test-subj={`mlInspectFieldStatsButton-${field.id}`}
+              disabled={disabled === true}
+              size="xs"
+              iconType="inspect"
+              css={{ color: isEmpty ? themeVars.euiTheme.euiColorDisabled : undefined }}
+              onClick={(ev: React.MouseEvent<HTMLButtonElement>) => {
+                if (ev.type === 'click') {
+                  ev.currentTarget.focus();
+                }
+                ev.preventDefault();
+                ev.stopPropagation();
 
-              if (onButtonClick) {
-                onButtonClick(field);
-              }
-            }}
-            aria-label={i18n.translate(
-              'xpack.ml.newJob.wizard.fieldContextPopover.inspectFieldStatsTooltipArialabel',
-              {
-                defaultMessage: 'Inspect field statistics',
+                if (onButtonClick) {
+                  onButtonClick(field);
+                }
+              }}
+              aria-label={
+                i18n.translate(
+                  'xpack.ml.newJob.wizard.fieldContextPopover.inspectFieldStatsTooltipAriaLabel',
+                  {
+                    defaultMessage: 'Inspect field statistics',
+                  }
+                ) + emptyFieldMessage
               }
-            )}
-          />
-        </EuiToolTip>
+            />
+          </EuiToolTip>
+        ) : null}
       </EuiFlexItem>
-      <EuiFlexItem grow={false} css={{ paddingRight: '4px' }}>
-        <FieldIcon type={getKbnFieldIconType(field.type)} fill="none" />
+      <EuiFlexItem grow={false} css={{ paddingRight: themeVars.euiTheme.euiSizeXS }}>
+        <FieldIcon
+          color={isEmpty ? themeVars.euiTheme.euiColorDisabled : undefined}
+          type={getKbnFieldIconType(field.type)}
+          fill="none"
+        />
       </EuiFlexItem>
       <EuiFlexItem grow={true}>
-        <EuiHighlight search={searchValue}>{label}</EuiHighlight>
+        <EuiText color={isEmpty ? 'subdued' : undefined} size="s">
+          {label}
+        </EuiText>
       </EuiFlexItem>
     </EuiFlexGroup>
   );

x-pack/plugins/ml/public/application/components/field_stats_flyout/populated_fields/get_merged_populated_fields_query.test.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getMergedSampleDocsForPopulatedFieldsQuery } from './get_merged_populated_fields_query';
+
+describe('getMergedSampleDocsForPopulatedFieldsQuery()', () => {
+  it('should wrap the original query in function_score', () => {
+    expect(
+      getMergedSampleDocsForPopulatedFieldsQuery({
+        searchQuery: { match_all: {} },
+        runtimeFields: {},
+      })
+    ).toStrictEqual({
+      query: {
+        function_score: { query: { bool: { must: [{ match_all: {} }] } }, random_score: {} },
+      },
+      runtime_mappings: {},
+    });
+  });
+
+  it('should append the time range to the query if timeRange and datetimeField are provided', () => {
+    expect(
+      getMergedSampleDocsForPopulatedFieldsQuery({
+        searchQuery: {
+          bool: {
+            should: [{ match_phrase: { version: '1' } }],
+            minimum_should_match: 1,
+            filter: [
+              {
+                terms: {
+                  cluster_uuid: '',
+                },
+              },
+            ],
+            must_not: [],
+          },
+        },
+        runtimeFields: {},
+        timeRange: { from: 1613995874349, to: 1614082617000 },
+        datetimeField: '@timestamp',
+      })
+    ).toStrictEqual({
+      query: {
+        function_score: {
+          query: {
+            bool: {
+              filter: [
+                { terms: { cluster_uuid: '' } },
+                {
+                  range: {
+                    '@timestamp': {
+                      format: 'epoch_millis',
+                      gte: 1613995874349,
+                      lte: 1614082617000,
+                    },
+                  },
+                },
+              ],
+              minimum_should_match: 1,
+              must_not: [],
+              should: [{ match_phrase: { version: '1' } }],
+            },
+          },
+          random_score: {},
+        },
+      },
+      runtime_mappings: {},
+    });
+  });
+
+  it('should not append the time range to the query if datetimeField is undefined', () => {
+    expect(
+      getMergedSampleDocsForPopulatedFieldsQuery({
+        searchQuery: {
+          bool: {
+            should: [{ match_phrase: { airline: 'AAL' } }],
+            minimum_should_match: 1,
+            filter: [],
+            must_not: [],
+          },
+        },
+        runtimeFields: {},
+        timeRange: { from: 1613995874349, to: 1614082617000 },
+      })
+    ).toStrictEqual({
+      query: {
+        function_score: {
+          query: {
+            bool: {
+              filter: [],
+              minimum_should_match: 1,
+              must_not: [],
+              should: [{ match_phrase: { airline: 'AAL' } }],
+            },
+          },
+          random_score: {},
+        },
+      },
+      runtime_mappings: {},
+    });
+  });
+});