[ML] Provides hints for empty fields in dropdown options in Anomaly detection & Transform creation wizards, Change point detection view

x-pack/plugins/ml/public/application/components/field_stats_flyout/field_stats_flyout_provider.tsx

@@ -10,15 +10,35 @@ import type { DataView } from '@kbn/data-plugin/common';
 import type { FieldStatsServices } from '@kbn/unified-field-list/src/components/field_stats';
 import type { TimeRange as TimeRangeMs } from '@kbn/ml-date-picker';
 import type { FieldStatsProps } from '@kbn/unified-field-list/src/components/field_stats';
-import { MLFieldStatsFlyoutContext } from './use_field_stats_flytout_context';
+import { useEffect } from 'react';
+import { getProcessedFields } from '@kbn/ml-data-grid';
+import { stringHash } from '@kbn/ml-string-hash';
+import { BehaviorSubject, lastValueFrom } from 'rxjs';
+import { getMergedSampleDocsForPopulatedFieldsQuery } from './populated_fields/get_merged_populated_fields_query';
+import { useMlKibana } from '../../contexts/kibana';
 import { FieldStatsFlyout } from './field_stats_flyout';
+import { MLFieldStatsFlyoutContext } from './use_field_stats_flytout_context';
+import { PopulatedFieldsCacheManager } from './populated_fields/populated_fields_cache_manager';
 
 export const FieldStatsFlyoutProvider: FC<{
   dataView: DataView;
   fieldStatsServices: FieldStatsServices;
   timeRangeMs?: TimeRangeMs;
   dslQuery?: FieldStatsProps['dslQuery'];
-}> = ({ dataView, fieldStatsServices, timeRangeMs, dslQuery, children }) => {
+  disablePopulatedFields?: boolean;
+}> = ({
+  dataView,
+  fieldStatsServices,
+  timeRangeMs,
+  dslQuery,
+  disablePopulatedFields = false,
+  children,
+}) => {
+  const {
+    services: {
+      data: { search },
+    },
+  } = useMlKibana();
   const [isFieldStatsFlyoutVisible, setFieldStatsIsFlyoutVisible] = useState(false);
   const [fieldName, setFieldName] = useState<string | undefined>();
   const [fieldValue, setFieldValue] = useState<string | number | undefined>();
@@ -27,6 +47,68 @@ export const FieldStatsFlyoutProvider: FC<{
     () => setFieldStatsIsFlyoutVisible(!isFieldStatsFlyoutVisible),
     [isFieldStatsFlyoutVisible]
   );
+  const [manager] = useState(new PopulatedFieldsCacheManager());
+  const [populatedFields$] = useState(new BehaviorSubject<Set<string>>(new Set()));
+
+  useEffect(() => {
+    if (disablePopulatedFields) return;
+    const abortController = new AbortController();
+
+    const queryAndRunTimeMappings = getMergedSampleDocsForPopulatedFieldsQuery({
+      searchQuery: dslQuery,
+      runtimeFields: dataView.getRuntimeMappings(),
+      datetimeField: dataView.getTimeField()?.name,
+      timeRange: timeRangeMs,
+    });
+    const indexPattern = dataView.getIndexPattern();
+    const esSearchRequestParams = {
+      index: indexPattern,
+      body: {
+        fields: ['*'],
+        _source: false,
+        ...queryAndRunTimeMappings,
+        size: 1000,
+      },
+    };
+    const cacheKey = stringHash(JSON.stringify(esSearchRequestParams)).toString();
+
+    const fetchSampleDocuments = async function () {
+      try {
+        const resp = await lastValueFrom(
+          search.search(
+            {
+              params: esSearchRequestParams,
+            },
+            { abortSignal: abortController.signal }
+          )
+        );
+
+        const docs = resp.rawResponse.hits.hits.map((d) => getProcessedFields(d.fields ?? {}));
+
+        // Get all field names for each returned doc and flatten it
+        // to a list of unique field names used across all docs.
+        // const allDataViewFields = getFieldsFromKibanaIndexPattern(dataView);
+        const fieldsWithData = new Set(docs.map(Object.keys).flat(1));
+        manager.set(cacheKey, fieldsWithData);
+        populatedFields$.next(fieldsWithData);
+      } catch (e) {
+        // eslint-disable-next-line no-console
+        console.error(
+          `An error occurred fetching sample documents to determine populated field stats.
+          \nQuery:\n${JSON.stringify(esSearchRequestParams)}
+          \nError:${e}`
+        );
+      }
+    };
+
+    const cachedResult = manager.get(cacheKey);
+    if (cachedResult) {
+      return cachedResult;
+    } else {
+      fetchSampleDocuments();
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [JSON.stringify({ query: dslQuery, dataViewId: dataView.id })]);
 
   return (
     <MLFieldStatsFlyoutContext.Provider
@@ -38,6 +120,8 @@ export const FieldStatsFlyoutProvider: FC<{
         fieldName,
         setFieldValue,
         fieldValue,
+        timeRangeMs,
+        populatedFields$,
       }}
     >
       <FieldStatsFlyout

x-pack/plugins/ml/public/application/components/field_stats_flyout/field_stats_info_button.tsx

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiHighlight, EuiToolTip } from '@elastic/eui';
+import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiToolTip, EuiText } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import React from 'react';
 import { FieldIcon } from '@kbn/react-field';
@@ -16,14 +16,18 @@ export type FieldForStats = Pick<Field, 'id' | 'type'>;
 export const FieldStatsInfoButton = ({
   field,
   label,
-  searchValue = '',
   onButtonClick,
+  disabled,
+  isEmpty,
 }: {
   field: FieldForStats;
   label: string;
   searchValue?: string;
+  disabled?: boolean;
+  isEmpty?: boolean;
   onButtonClick?: (field: FieldForStats) => void;
 }) => {
+  const isDisabled = disabled === true || field.id === EVENT_RATE_FIELD_ID;
   return (
     <EuiFlexGroup gutterSize="none" alignItems="center">
       <EuiFlexItem grow={false}>
@@ -37,9 +41,11 @@ export const FieldStatsInfoButton = ({
         >
           <EuiButtonIcon
             data-test-subj={`mlInspectFieldStatsButton-${field.id}`}
-            disabled={field.id === EVENT_RATE_FIELD_ID}
+            // Only disable the button if explicitly disabled
+            disabled={isDisabled}
             size="xs"
             iconType="inspect"
+            css={{ color: isEmpty ? 'gray' : undefined }}
             onClick={(ev: React.MouseEvent<HTMLButtonElement>) => {
               if (ev.type === 'click') {
                 ev.currentTarget.focus();
@@ -61,10 +67,16 @@ export const FieldStatsInfoButton = ({
         </EuiToolTip>
       </EuiFlexItem>
       <EuiFlexItem grow={false} css={{ paddingRight: '4px' }}>
-        <FieldIcon type={getKbnFieldIconType(field.type)} fill="none" />
+        <FieldIcon
+          color={isEmpty ? 'gray' : undefined}
+          type={getKbnFieldIconType(field.type)}
+          fill="none"
+        />
       </EuiFlexItem>
       <EuiFlexItem grow={true}>
-        <EuiHighlight search={searchValue}>{label}</EuiHighlight>
+        <EuiText color={isEmpty ? 'subdued' : undefined} size="s">
+          {label}
+        </EuiText>
       </EuiFlexItem>
     </EuiFlexGroup>
   );

x-pack/plugins/ml/public/application/components/field_stats_flyout/populated_fields/get_merged_populated_fields_query.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { TimeRange as TimeRangeMs } from '@kbn/ml-date-picker';
+import { isPopulatedObject } from '@kbn/ml-is-populated-object';
+import { cloneDeep } from 'lodash';
+import { getDefaultDSLQuery } from '@kbn/ml-query-utils';
+
+export const getMergedSampleDocsForPopulatedFieldsQuery = ({
+  runtimeFields,
+  searchQuery,
+  datetimeField,
+  timeRange,
+}: {
+  runtimeFields: estypes.MappingRuntimeFields;
+  searchQuery?: estypes.QueryDslQueryContainer;
+  datetimeField?: string;
+  timeRange?: TimeRangeMs;
+}): {
+  query: estypes.QueryDslQueryContainer;
+  runtime_mappings?: estypes.MappingRuntimeFields;
+} => {
+  let rangeFilter;
+  if (timeRange && datetimeField !== undefined) {
+    if (isPopulatedObject(timeRange, ['from', 'to']) && timeRange.to > timeRange.from) {
+      rangeFilter = {
+        range: {
+          [datetimeField]: {
+            gte: timeRange.from,
+            lte: timeRange.to,
+            format: 'epoch_millis',
+          },
+        },
+      };
+    }
+  }
+
+  const query = cloneDeep(
+    !searchQuery || isPopulatedObject(searchQuery, ['match_all'])
+      ? getDefaultDSLQuery()
+      : searchQuery
+  );
+
+  if (rangeFilter && isPopulatedObject<string, estypes.QueryDslBoolQuery>(query, ['bool'])) {
+    if (Array.isArray(query.bool.filter)) {
+      query.bool.filter.push(rangeFilter);
+    } else {
+      query.bool.filter = [rangeFilter];
+    }
+  }
+
+  const queryAndRuntimeFields: {
+    query: estypes.QueryDslQueryContainer;
+    runtime_mappings?: estypes.MappingRuntimeFields;
+  } = {
+    query: {
+      function_score: {
+        query,
+        // @ts-expect-error random_score is valid dsl query
+        random_score: {},
+      },
+    },
+  };
+  if (runtimeFields) {
+    queryAndRuntimeFields.runtime_mappings = runtimeFields;
+  }
+  return queryAndRuntimeFields;
+};

x-pack/plugins/ml/public/application/components/field_stats_flyout/populated_fields/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { PopulatedFieldsCacheManager } from './populated_fields_cache_manager';

x-pack/plugins/ml/public/application/components/field_stats_flyout/populated_fields/populated_fields_cache_manager.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+type StringifiedQueryKey = string;
+type UpdatedTimestamp = number;
+
+const DEFAULT_EXPIRATION_MS = 60000;
+export class PopulatedFieldsCacheManager {
+  private _expirationDurationMs = DEFAULT_EXPIRATION_MS; // duration in ms
+
+  private _resultsCache = new Map<StringifiedQueryKey, any>();
+  _lastUpdatedTimestamps = new Map<StringifiedQueryKey, UpdatedTimestamp>();
+
+  constructor(expirationMs = DEFAULT_EXPIRATION_MS) {
+    this._expirationDurationMs = expirationMs;
+  }
+
+  private clearOldCacheIfNeeded() {
+    if (this._resultsCache.size > 10) {
+      this._resultsCache.clear();
+      this._lastUpdatedTimestamps.clear();
+    }
+  }
+
+  private clearExpiredCache(key: StringifiedQueryKey) {
+    // If result is available but past the expiration duration, clear cache
+    const lastUpdatedTs = this._lastUpdatedTimestamps.get(key);
+    const now = Date.now();
+    if (lastUpdatedTs !== undefined && lastUpdatedTs - now > this._expirationDurationMs) {
+      this._resultsCache.delete(key);
+    }
+  }
+
+  public get(key: StringifiedQueryKey) {
+    this.clearExpiredCache(key);
+    return this._resultsCache.get(key);
+  }
+
+  public set(key: StringifiedQueryKey, value: any) {
+    this.clearOldCacheIfNeeded();
+    this._resultsCache.set(key, Date.now());
+    this._resultsCache.set(key, value);
+  }
+}

x-pack/plugins/ml/public/application/components/field_stats_flyout/use_field_stats_flytout_context.ts

@@ -6,6 +6,8 @@
  */
 
 import { createContext, useContext } from 'react';
+import { TimeRange as TimeRangeMs } from '@kbn/ml-date-picker';
+import { BehaviorSubject } from 'rxjs';
 interface MLJobWizardFieldStatsFlyoutProps {
   isFlyoutVisible: boolean;
   setIsFlyoutVisible: (v: boolean) => void;
@@ -14,13 +16,17 @@ interface MLJobWizardFieldStatsFlyoutProps {
   fieldName?: string;
   setFieldValue: (v: string) => void;
   fieldValue?: string | number;
+  timeRangeMs?: TimeRangeMs;
+  populatedFields$: BehaviorSubject<Set<string>>;
 }
 export const MLFieldStatsFlyoutContext = createContext<MLJobWizardFieldStatsFlyoutProps>({
   isFlyoutVisible: false,
   setIsFlyoutVisible: () => {},
   toggleFlyoutVisible: () => {},
   setFieldName: () => {},
   setFieldValue: () => {},
+  timeRangeMs: undefined,
+  populatedFields$: new BehaviorSubject<Set<string>>(new Set()),
 });
 
 export function useFieldStatsFlyoutContext() {

x-pack/plugins/ml/public/application/components/field_stats_flyout/use_field_stats_trigger.tsx

@@ -6,17 +6,18 @@
  */
 
 import React, { ReactNode, useCallback } from 'react';
-import { EuiComboBoxOptionOption } from '@elastic/eui';
+import type { EuiComboBoxOptionOption } from '@elastic/eui';
 import type { Field } from '@kbn/ml-anomaly-utils';
+import useObservable from 'react-use/lib/useObservable';
 import { optionCss } from './eui_combo_box_with_field_stats';
 import { useFieldStatsFlyoutContext } from '.';
 import { FieldForStats, FieldStatsInfoButton } from './field_stats_info_button';
-
 interface Option extends EuiComboBoxOptionOption<string> {
   field: Field;
 }
+
 export const useFieldStatsTrigger = () => {
-  const { setIsFlyoutVisible, setFieldName } = useFieldStatsFlyoutContext();
+  const { setIsFlyoutVisible, setFieldName, populatedFields$ } = useFieldStatsFlyoutContext();
 
   const closeFlyout = useCallback(() => setIsFlyoutVisible(false), [setIsFlyoutVisible]);
 
@@ -29,20 +30,25 @@ export const useFieldStatsTrigger = () => {
     },
     [setFieldName, setIsFlyoutVisible]
   );
+
+  const populatedFields = useObservable(populatedFields$);
+
   const renderOption = useCallback(
     (option: EuiComboBoxOptionOption, searchValue: string): ReactNode => {
       const field = (option as Option).field;
       return option.isGroupLabelOption || !field ? (
         option.label
       ) : (
         <FieldStatsInfoButton
+          isEmpty={!populatedFields?.has(field.id)}
           field={field}
           label={option.label}
           onButtonClick={handleFieldStatsButtonClick}
         />
       );
     },
-    [handleFieldStatsButtonClick]
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [handleFieldStatsButtonClick, populatedFields?.size]
   );
   return {
     renderOption,
@@ -51,5 +57,6 @@ export const useFieldStatsTrigger = () => {
     handleFieldStatsButtonClick,
     closeFlyout,
     optionCss,
+    populatedFields,
   };
 };