diff --git a/src/plugins/discover/public/application/main/components/field_stats_table/field_stats_tab.tsx b/src/plugins/discover/public/application/main/components/field_stats_table/field_stats_tab.tsx index 62c40ef529b7e..ca7c6424015e7 100644 --- a/src/plugins/discover/public/application/main/components/field_stats_table/field_stats_tab.tsx +++ b/src/plugins/discover/public/application/main/components/field_stats_table/field_stats_tab.tsx @@ -15,18 +15,22 @@ import { useAdditionalFieldGroups } from '../../hooks/sidebar/use_additional_fie export const FieldStatisticsTab: React.FC> = React.memo((props) => { const services = useDiscoverServices(); - const querySubscriberResult = useQuerySubscriber({ + const { query, filters } = useQuerySubscriber({ data: services.data, }); const additionalFieldGroups = useAdditionalFieldGroups(); if (!services.dataVisualizer) return null; - return ( ); diff --git a/src/plugins/discover/public/application/main/components/field_stats_table/field_stats_table.tsx b/src/plugins/discover/public/application/main/components/field_stats_table/field_stats_table.tsx index c2a7f30a27117..5829bd42c68c9 100644 --- a/src/plugins/discover/public/application/main/components/field_stats_table/field_stats_table.tsx +++ b/src/plugins/discover/public/application/main/components/field_stats_table/field_stats_table.tsx @@ -10,18 +10,19 @@ import React, { useEffect, useMemo, useCallback } from 'react'; import { METRIC_TYPE } from '@kbn/analytics'; import { EuiFlexItem } from '@elastic/eui'; import { css } from '@emotion/react'; -import useObservable from 'react-use/lib/useObservable'; -import { of, map } from 'rxjs'; +import { of, map, filter } from 'rxjs'; import { BehaviorSubject } from 'rxjs'; +import useObservable from 'react-use/lib/useObservable'; import { convertFieldsToFallbackFields, getAllFallbackFields, getAssociatedSmartFieldsAsString, SmartFieldFallbackTooltip, } from '@kbn/unified-field-list'; -import type { DataVisualizerTableItem } from '@kbn/data-visualizer-plugin/public/application/common/components/stats_table/data_visualizer_stats_table'; +import type { DataVisualizerTableItem } from '@kbn/data-visualizer-plugin/public/application/common/components/stats_table/types'; import { useDiscoverServices } from '../../../../hooks/use_discover_services'; import { FIELD_STATISTICS_LOADED } from './constants'; + import type { NormalSamplingOption, FieldStatisticsTableProps } from './types'; export type { FieldStatisticsTableProps }; @@ -35,9 +36,11 @@ const statsTableCss = css({ }); const fallBacklastReloadRequestTime$ = new BehaviorSubject(0); +const fallbackTotalHits = of(undefined); -export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => { +export const FieldStatisticsTable = React.memo((props: FieldStatisticsTableProps) => { const { + isEsqlMode, dataView, savedSearch, query, @@ -81,10 +84,15 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => { [additionalFieldGroups, allFallbackFields] ); - const totalHits = useObservable(stateContainer?.dataState.data$.totalHits$ ?? of(undefined)); - const totalDocuments = useMemo(() => totalHits?.result, [totalHits]); - const services = useDiscoverServices(); + + // Other apps consuming Discover UI might inject their own proxied data services + // so we need override the kibana context services with the injected proxied services + // to make sure the table use the right service + const overridableServices = useMemo(() => { + return { data: services.data }; + }, [services.data]); + const dataVisualizerService = services.dataVisualizer; // State from Discover we want the embeddable to reflect @@ -95,11 +103,25 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => { const lastReloadRequestTime$ = useMemo(() => { return stateContainer?.dataState?.refetch$ - ? stateContainer?.dataState?.refetch$.pipe(map(() => Date.now())) + ? stateContainer?.dataState?.refetch$.pipe( + map(() => { + return Date.now(); + }) + ) : fallBacklastReloadRequestTime$; }, [stateContainer]); - const lastReloadRequestTime = useObservable(lastReloadRequestTime$, 0); + const totalHitsComplete$ = useMemo(() => { + return stateContainer + ? stateContainer.dataState.data$.totalHits$.pipe( + filter((d) => d.fetchStatus === 'complete'), + map((d) => d?.result) + ) + : fallbackTotalHits; + }, [stateContainer]); + + const totalDocuments = useObservable(totalHitsComplete$); + const lastReloadRequestTime = useObservable(lastReloadRequestTime$); useEffect(() => { // Track should only be called once when component is loaded @@ -119,7 +141,7 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => { const updateState = useCallback( (changes) => { if (changes.showDistributions !== undefined && stateContainer) { - stateContainer.appState.update({ hideAggregatedPreview: !changes.showDistributions }); + stateContainer.appState.update({ hideAggregatedPreview: !changes.showDistributions }, true); } }, [stateContainer] @@ -144,7 +166,9 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => { showPreviewByDefault={showPreviewByDefault} onTableUpdate={updateState} renderFieldName={renderFieldName} + esql={isEsqlMode} + overridableServices={overridableServices} /> ); -}; +}); diff --git a/src/plugins/discover/public/application/main/components/field_stats_table/types.ts b/src/plugins/discover/public/application/main/components/field_stats_table/types.ts index 0ff28f2dcb700..ddd62285d044d 100644 --- a/src/plugins/discover/public/application/main/components/field_stats_table/types.ts +++ b/src/plugins/discover/public/application/main/components/field_stats_table/types.ts @@ -169,4 +169,8 @@ export interface FieldStatisticsTableProps { * Additional field groups (e.g. Smart Fields) */ additionalFieldGroups?: AdditionalFieldGroups; + /** + * If table should query using ES|QL + */ + isEsqlMode?: boolean; } diff --git a/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx index d141b89e453e3..0d94041738f54 100644 --- a/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx @@ -75,6 +75,7 @@ export function DiscoverLayout({ stateContainer }: DiscoverLayoutProps) { history, spaces, observabilityAIAssistant, + dataVisualizer: dataVisualizerService, } = useDiscoverServices(); const pageBackgroundColor = useEuiBackgroundColor('plain'); const globalQueryState = data.query.getState(); @@ -86,12 +87,13 @@ export function DiscoverLayout({ stateContainer }: DiscoverLayoutProps) { state.sort, ]); const isEsqlMode = useIsEsqlMode(); + const viewMode: VIEW_MODE = useAppStateSelector((state) => { - if (state.viewMode === VIEW_MODE.DOCUMENT_LEVEL || state.viewMode === VIEW_MODE.PATTERN_LEVEL) { - return state.viewMode; - } - if (uiSettings.get(SHOW_FIELD_STATISTICS) !== true || isEsqlMode) + const fieldStatsNotAvailable = + !uiSettings.get(SHOW_FIELD_STATISTICS) && !!dataVisualizerService; + if (state.viewMode === VIEW_MODE.AGGREGATED_LEVEL && fieldStatsNotAvailable) { return VIEW_MODE.DOCUMENT_LEVEL; + } return state.viewMode ?? VIEW_MODE.DOCUMENT_LEVEL; }); const [dataView, dataViewLoading] = useInternalStateSelector((state) => [ diff --git a/src/plugins/discover/public/application/main/components/layout/discover_main_content.tsx b/src/plugins/discover/public/application/main/components/layout/discover_main_content.tsx index 7c44ed1deff83..735eae1fa9039 100644 --- a/src/plugins/discover/public/application/main/components/layout/discover_main_content.tsx +++ b/src/plugins/discover/public/application/main/components/layout/discover_main_content.tsx @@ -67,7 +67,7 @@ export const DiscoverMainContent = ({ const setDiscoverViewMode = useCallback( (mode: VIEW_MODE) => { - stateContainer.appState.update({ viewMode: mode }); + stateContainer.appState.update({ viewMode: mode }, true); if (trackUiMetric) { if (mode === VIEW_MODE.AGGREGATED_LEVEL) { @@ -151,6 +151,7 @@ export const DiscoverMainContent = ({ stateContainer={stateContainer} onAddFilter={!isEsqlMode ? onAddFilter : undefined} trackUiMetric={trackUiMetric} + isEsqlMode={isEsqlMode} /> ) : null} diff --git a/src/plugins/discover/public/application/main/hooks/use_esql_mode.test.tsx b/src/plugins/discover/public/application/main/hooks/use_esql_mode.test.tsx index b24bcd3eb42d5..12109ea01a422 100644 --- a/src/plugins/discover/public/application/main/hooks/use_esql_mode.test.tsx +++ b/src/plugins/discover/public/application/main/hooks/use_esql_mode.test.tsx @@ -104,15 +104,12 @@ describe('useEsqlMode', () => { stateContainer.dataState.data$.documents$.next(msgComplete); expect(replaceUrlState).toHaveBeenCalledTimes(0); }); - test('should change viewMode to undefined (default) if it was AGGREGATED_LEVEL', async () => { + test('should not change viewMode to undefined (default) if it was AGGREGATED_LEVEL', async () => { const { replaceUrlState } = renderHookWithContext(false, { viewMode: VIEW_MODE.AGGREGATED_LEVEL, }); - await waitFor(() => expect(replaceUrlState).toHaveBeenCalledTimes(1)); - expect(replaceUrlState).toHaveBeenCalledWith({ - viewMode: undefined, - }); + await waitFor(() => expect(replaceUrlState).toHaveBeenCalledTimes(0)); }); test('should change viewMode to undefined (default) if it was PATTERN_LEVEL', async () => { diff --git a/src/plugins/discover/public/application/main/state_management/discover_saved_search_container.ts b/src/plugins/discover/public/application/main/state_management/discover_saved_search_container.ts index 69ef4b03c742d..0a41f087c75ab 100644 --- a/src/plugins/discover/public/application/main/state_management/discover_saved_search_container.ts +++ b/src/plugins/discover/public/application/main/state_management/discover_saved_search_container.ts @@ -18,6 +18,7 @@ import { } from '@kbn/unified-histogram-plugin/public'; import { SavedObjectSaveOpts } from '@kbn/saved-objects-plugin/public'; import { isEqual, isFunction } from 'lodash'; +import { VIEW_MODE } from '../../../../common/constants'; import { restoreStateFromSavedSearch } from '../../../services/saved_searches/restore_from_saved_search'; import { updateSavedSearch } from './utils/update_saved_search'; import { addLog } from '../../../utils/add_log'; @@ -340,7 +341,12 @@ export function isEqualSavedSearch(savedSearchPrev: SavedSearch, savedSearchNext const prevValue = getSavedSearchFieldForComparison(prevSavedSearch, key); const nextValue = getSavedSearchFieldForComparison(nextSavedSearchWithoutSearchSource, key); - const isSame = isEqual(prevValue, nextValue); + const isSame = + isEqual(prevValue, nextValue) || + // By default, viewMode: undefined is equivalent to documents view + // So they should be treated as same + (key === 'viewMode' && + (prevValue ?? VIEW_MODE.DOCUMENT_LEVEL) === (nextValue ?? VIEW_MODE.DOCUMENT_LEVEL)); if (!isSame) { addLog('[savedSearch] difference between initial and changed version', { diff --git a/src/plugins/discover/public/application/main/state_management/utils/get_state_defaults.test.ts b/src/plugins/discover/public/application/main/state_management/utils/get_state_defaults.test.ts index 28c562d3e7051..86d9ffe99c244 100644 --- a/src/plugins/discover/public/application/main/state_management/utils/get_state_defaults.test.ts +++ b/src/plugins/discover/public/application/main/state_management/utils/get_state_defaults.test.ts @@ -98,14 +98,14 @@ describe('getStateDefaults', () => { }); expect(actualForUndefinedViewMode.viewMode).toBeUndefined(); - const actualForEsqlWithInvalidAggLevelViewMode = getStateDefaults({ + const actualForEsqlWithAggregatedViewMode = getStateDefaults({ services: discoverServiceMock, savedSearch: { ...savedSearchMockWithESQL, viewMode: VIEW_MODE.AGGREGATED_LEVEL, }, }); - expect(actualForEsqlWithInvalidAggLevelViewMode.viewMode).toBe(VIEW_MODE.DOCUMENT_LEVEL); + expect(actualForEsqlWithAggregatedViewMode.viewMode).toBe(VIEW_MODE.AGGREGATED_LEVEL); const actualForEsqlWithInvalidPatternLevelViewMode = getStateDefaults({ services: discoverServiceMock, diff --git a/src/plugins/discover/public/application/main/utils/get_valid_view_mode.test.ts b/src/plugins/discover/public/application/main/utils/get_valid_view_mode.test.ts index ff2d4250b3da8..7d8cd7ed3cc5e 100644 --- a/src/plugins/discover/public/application/main/utils/get_valid_view_mode.test.ts +++ b/src/plugins/discover/public/application/main/utils/get_valid_view_mode.test.ts @@ -60,7 +60,7 @@ describe('getValidViewMode', () => { viewMode: VIEW_MODE.AGGREGATED_LEVEL, isEsqlMode: true, }) - ).toBe(VIEW_MODE.DOCUMENT_LEVEL); + ).toBe(VIEW_MODE.AGGREGATED_LEVEL); expect( getValidViewMode({ diff --git a/src/plugins/discover/public/application/main/utils/get_valid_view_mode.ts b/src/plugins/discover/public/application/main/utils/get_valid_view_mode.ts index 03c3500b7ab2d..96defe6711d95 100644 --- a/src/plugins/discover/public/application/main/utils/get_valid_view_mode.ts +++ b/src/plugins/discover/public/application/main/utils/get_valid_view_mode.ts @@ -20,11 +20,8 @@ export const getValidViewMode = ({ viewMode?: VIEW_MODE; isEsqlMode: boolean; }): VIEW_MODE | undefined => { - if ( - (viewMode === VIEW_MODE.PATTERN_LEVEL || viewMode === VIEW_MODE.AGGREGATED_LEVEL) && - isEsqlMode - ) { - // only this mode is supported for text-based languages + if (viewMode === VIEW_MODE.PATTERN_LEVEL && isEsqlMode) { + // only this mode is supported for ES|QL languages return VIEW_MODE.DOCUMENT_LEVEL; } diff --git a/src/plugins/discover/public/components/view_mode_toggle/view_mode_toggle.test.tsx b/src/plugins/discover/public/components/view_mode_toggle/view_mode_toggle.test.tsx index 4d266af5e7949..08a56af81f4bc 100644 --- a/src/plugins/discover/public/components/view_mode_toggle/view_mode_toggle.test.tsx +++ b/src/plugins/discover/public/components/view_mode_toggle/view_mode_toggle.test.tsx @@ -105,14 +105,14 @@ describe('Document view mode toggle component', () => { expect(findTestSubject(component, 'dscViewModeFieldStatsButton').exists()).toBe(false); }); - it('should not render if ES|QL', async () => { + it('should show document and field stats view if ES|QL', async () => { const component = await mountComponent({ isEsqlMode: true }); - expect(findTestSubject(component, 'dscViewModeToggle').exists()).toBe(false); + expect(findTestSubject(component, 'dscViewModeToggle').exists()).toBe(true); expect(findTestSubject(component, 'discoverQueryTotalHits').exists()).toBe(true); - expect(findTestSubject(component, 'dscViewModeDocumentButton').exists()).toBe(false); + expect(findTestSubject(component, 'dscViewModeDocumentButton').exists()).toBe(true); expect(findTestSubject(component, 'dscViewModePatternAnalysisButton').exists()).toBe(false); - expect(findTestSubject(component, 'dscViewModeFieldStatsButton').exists()).toBe(false); + expect(findTestSubject(component, 'dscViewModeFieldStatsButton').exists()).toBe(true); }); it('should set the view mode to VIEW_MODE.DOCUMENT_LEVEL when dscViewModeDocumentButton is clicked', async () => { diff --git a/src/plugins/discover/public/components/view_mode_toggle/view_mode_toggle.tsx b/src/plugins/discover/public/components/view_mode_toggle/view_mode_toggle.tsx index 11351893b6a26..28eeb9f3661ff 100644 --- a/src/plugins/discover/public/components/view_mode_toggle/view_mode_toggle.tsx +++ b/src/plugins/discover/public/components/view_mode_toggle/view_mode_toggle.tsx @@ -63,7 +63,7 @@ export const DocumentViewModeToggle = ({ useEffect( function checkForPatternAnalysis() { - if (!aiopsService) { + if (!aiopsService || isEsqlMode) { setShowPatternAnalysisTab(false); return; } @@ -76,7 +76,7 @@ export const DocumentViewModeToggle = ({ }) .catch(() => setShowPatternAnalysisTabWrapper(false)); }, - [aiopsService, dataView, setShowPatternAnalysisTabWrapper] + [aiopsService, dataView, isEsqlMode, setShowPatternAnalysisTabWrapper] ); useEffect(() => { @@ -121,7 +121,7 @@ export const DocumentViewModeToggle = ({ )} - {isEsqlMode || (showFieldStatisticsTab === false && showPatternAnalysisTab === false) ? ( + {showFieldStatisticsTab === false && showPatternAnalysisTab === false ? ( ) : ( diff --git a/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx b/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx index 06aeaa42c1376..3a6f9f9c9c8ac 100644 --- a/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx +++ b/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx @@ -664,6 +664,7 @@ export class SavedSearchEmbeddable query={this.input.query} onAddFilter={searchProps.onFilter} searchSessionId={this.input.searchSessionId} + isEsqlMode={isEsqlMode} /> , diff --git a/test/functional/apps/discover/group4/_esql_view.ts b/test/functional/apps/discover/group4/_esql_view.ts index a1cecdbc36d4c..9f60f4991ab4c 100644 --- a/test/functional/apps/discover/group4/_esql_view.ts +++ b/test/functional/apps/discover/group4/_esql_view.ts @@ -79,7 +79,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await testSubjects.exists('showQueryBarMenu')).to.be(false); expect(await testSubjects.exists('addFilter')).to.be(false); - expect(await testSubjects.exists('dscViewModeDocumentButton')).to.be(false); + expect(await testSubjects.exists('dscViewModeDocumentButton')).to.be(true); // when Lens suggests a table, we render an ESQL based histogram expect(await testSubjects.exists('unifiedHistogramChart')).to.be(true); expect(await testSubjects.exists('discoverQueryHits')).to.be(true); diff --git a/test/functional/apps/discover/group6/_view_mode_toggle.ts b/test/functional/apps/discover/group6/_view_mode_toggle.ts index ba964c7532d70..415cb9f1fb85e 100644 --- a/test/functional/apps/discover/group6/_view_mode_toggle.ts +++ b/test/functional/apps/discover/group6/_view_mode_toggle.ts @@ -107,7 +107,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.existOrFail('dscViewModeToggle'); }); - it('should not show view mode toggle for ES|QL searches', async () => { + it('should still show view mode toggle for ES|QL searches', async () => { await testSubjects.click('dscViewModeDocumentButton'); await retry.try(async () => { @@ -119,7 +119,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.discover.selectTextBaseLang(); - await testSubjects.missingOrFail('dscViewModeToggle'); + await testSubjects.existOrFail('dscViewModeToggle'); if (!useLegacyTable) { await testSubjects.existOrFail('unifiedDataTableToolbar'); diff --git a/x-pack/plugins/data_visualizer/common/types/field_request_config.ts b/x-pack/plugins/data_visualizer/common/types/field_request_config.ts index 43ba81eccd784..15db4c1d4832c 100644 --- a/x-pack/plugins/data_visualizer/common/types/field_request_config.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_request_config.ts @@ -81,6 +81,7 @@ export interface FieldVisStats { examples?: Array; timeRangeEarliest?: number; timeRangeLatest?: number; + approximate?: boolean; } export interface DVErrorObject { diff --git a/x-pack/plugins/data_visualizer/common/types/field_stats.ts b/x-pack/plugins/data_visualizer/common/types/field_stats.ts index 2aeea5c4cf033..97a2739f34ae0 100644 --- a/x-pack/plugins/data_visualizer/common/types/field_stats.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_stats.ts @@ -96,6 +96,11 @@ export interface StringFieldStats { sampledValues?: Bucket[]; topValuesSampleSize?: number; topValuesSamplerShardSize?: number; + /** + * Approximate: true for when the terms are from a random subset of the source data + * such that result/count for each term is not deterministic every time + */ + approximate?: boolean; } export interface DateFieldStats { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 75b7d98ec2468..74083dec0e5da 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -35,10 +35,6 @@ import { IndexBasedNumberContentPreview } from './components/field_data_row/numb import { useTableSettings } from './use_table_settings'; import { TopValuesPreview } from './components/field_data_row/top_values_preview'; -import type { - FieldVisConfig, - FileBasedFieldVisConfig, -} from '../../../../../common/types/field_vis_config'; import { isIndexBasedFieldVisConfig } from '../../../../../common/types/field_vis_config'; import { FileBasedNumberContentPreview } from '../field_data_row'; import { BooleanContentPreview } from './components/field_data_row'; @@ -47,12 +43,12 @@ import { DistinctValues } from './components/field_data_row/distinct_values'; import { FieldTypeIcon } from '../field_type_icon'; import './_index.scss'; import type { FieldStatisticTableEmbeddableProps } from '../../../index_data_visualizer/embeddables/grid_embeddable/types'; +import type { DataVisualizerTableItem } from './types'; const FIELD_NAME = 'fieldName'; export type ItemIdToExpandedRowMap = Record; -export type DataVisualizerTableItem = FieldVisConfig | FileBasedFieldVisConfig; interface DataVisualizerTableProps { items: T[]; pageState: DataVisualizerTableState; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts index 6d9f4d5b86d28..b9a0347079e54 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts @@ -5,11 +5,15 @@ * 2.0. */ export type { FieldDataRowProps } from './field_data_row'; -export type { +import type { FieldVisConfig, FileBasedFieldVisConfig, MetricFieldVisStats, } from '../../../../../../common/types/field_vis_config'; + +export type DataVisualizerTableItem = FieldVisConfig | FileBasedFieldVisConfig; + +export type { FieldVisConfig, FileBasedFieldVisConfig, MetricFieldVisStats }; export { isFileBasedFieldVisConfig, isIndexBasedFieldVisConfig, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index ef056c4c12f14..0d7d6b4c480e9 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -43,6 +43,8 @@ interface Props { function getPercentLabel(percent: number): string { if (percent >= 0.1) { return `${roundToDecimalPlace(percent, 1)}%`; + } else if (percent === 0) { + return '0%'; } else { return '< 0.1%'; } @@ -69,7 +71,7 @@ export const TopValues: FC = ({ } = useDataVisualizerKibana(); if (stats === undefined || !stats.topValues) return null; - const { fieldName, sampleCount } = stats; + const { fieldName, sampleCount, approximate } = stats; const originalTopValues = (showSampledValues ? stats.sampledValues : stats.topValues) ?? []; if (originalTopValues?.length === 0) return null; @@ -96,12 +98,28 @@ export const TopValues: FC = ({ /> ); } + /** + * For ES|QL, where are randomly sampling a subset from source data, then query is excuted on top of that data + * So the terms we get might not get the initial count + */ + const method = approximate ? ( + + ) : ( + + ); return totalDocuments > (sampleCount ?? 0) ? ( @@ -115,8 +133,9 @@ export const TopValues: FC = ({ ) : ( @@ -141,6 +160,7 @@ export const TopValues: FC = ({ typeof bucket.percent === 'number' ? bucket.percent : bucket.doc_count / totalDocuments, })); + const shouldShowOtherCount = approximate !== true; const topValuesOtherCountPercent = 1 - (topValues ? topValues.reduce((acc, bucket) => acc + bucket.percent, 0) : 0); const topValuesOtherCount = Math.floor(topValuesOtherCountPercent * (sampleCount ?? 0)); @@ -246,7 +266,7 @@ export const TopValues: FC = ({ ); }) : null} - {topValuesOtherCount > 0 ? ( + {shouldShowOtherCount && topValuesOtherCount > 0 ? ( true; export const IndexDataVisualizerESQL: FC = (dataVisualizerProps) => { const { services } = useDataVisualizerKibana(); const { data } = services; const euiTheme = useCurrentEuiTheme(); - const [query, setQuery] = useState({ esql: '' }); + // Query that has been typed, but has not submitted with cmd + enter + const [localQuery, setLocalQuery] = useState(DEFAULT_ESQL_QUERY); + const [query, setQuery] = useState(DEFAULT_ESQL_QUERY); const [currentDataView, setCurrentDataView] = useState(); const toggleShowEmptyFields = () => { @@ -92,9 +95,6 @@ export const IndexDataVisualizerESQL: FC = (dataVi } }; - // Query that has been typed, but has not submitted with cmd + enter - const [localQuery, setLocalQuery] = useState({ esql: '' }); - const indexPattern = useMemo(() => { let indexPatternFromQuery = ''; if (isESQLQuery(query)) { @@ -105,7 +105,7 @@ export const IndexDataVisualizerESQL: FC = (dataVi return undefined; } return indexPatternFromQuery; - }, [query]); + }, [query?.esql]); useEffect( function updateAdhocDataViewFromQuery() { @@ -169,11 +169,11 @@ export const IndexDataVisualizerESQL: FC = (dataVi metricsStats, timefilter, getItemIdToExpandedRowMap, - onQueryUpdate, + resetData, limitSize, showEmptyFields, fieldsCountStats, - } = useESQLDataVisualizerData(input, dataVisualizerListState, setQuery); + } = useESQLDataVisualizerData(input, dataVisualizerListState); const hasValidTimeField = useMemo( () => currentDataView?.timeFieldName !== undefined, @@ -199,6 +199,15 @@ export const IndexDataVisualizerESQL: FC = (dataVi setLocalQuery(q); } }, []); + const onTextLangQuerySubmit = useCallback( + async (q: AggregateQuery | undefined) => { + if (isESQLQuery(q)) { + resetData(); + setQuery(q); + } + }, + [resetData] + ); return ( = (dataVi false} + onTextLangQuerySubmit={onTextLangQuerySubmit} + expandCodeEditor={expandCodeEditor} isCodeEditorExpanded={true} detectTimestamp={true} hideMinimizeButton={true} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_esql_field_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_esql_field_stats_table.tsx index e5f6c5f3c1835..b4bcd3c0da5a9 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_esql_field_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_esql_field_stats_table.tsx @@ -21,51 +21,53 @@ import { EmbeddableNoResultsEmptyPrompt } from './embeddable_field_stats_no_resu const restorableDefaults = getDefaultESQLDataVisualizerListState(); -const EmbeddableESQLFieldStatsTableWrapper = (props: ESQLDataVisualizerGridEmbeddableState) => { - const { onTableUpdate, ...state } = props; - const [dataVisualizerListState, setDataVisualizerListState] = - useState>(restorableDefaults); +const EmbeddableESQLFieldStatsTableWrapper = React.memo( + (props: ESQLDataVisualizerGridEmbeddableState) => { + const { onTableUpdate } = props; + const [dataVisualizerListState, setDataVisualizerListState] = + useState>(restorableDefaults); - const onTableChange = useCallback( - (update: DataVisualizerTableState) => { - setDataVisualizerListState({ ...dataVisualizerListState, ...update }); - if (onTableUpdate) { - onTableUpdate(update); - } - }, - [dataVisualizerListState, onTableUpdate] - ); + const onTableChange = useCallback( + (update: DataVisualizerTableState) => { + setDataVisualizerListState({ ...dataVisualizerListState, ...update }); + if (onTableUpdate) { + onTableUpdate(update); + } + }, + [dataVisualizerListState, onTableUpdate] + ); - const { - configs, - extendedColumns, - progress, - overallStatsProgress, - setLastRefresh, - getItemIdToExpandedRowMap, - } = useESQLDataVisualizerData(state, dataVisualizerListState); + const { + configs, + extendedColumns, + progress, + overallStatsProgress, + setLastRefresh, + getItemIdToExpandedRowMap, + } = useESQLDataVisualizerData(props, dataVisualizerListState); - useEffect(() => { - setLastRefresh(Date.now()); - }, [state?.lastReloadRequestTime, setLastRefresh]); + useEffect(() => { + setLastRefresh(Date.now()); + }, [props?.lastReloadRequestTime, setLastRefresh]); - if (progress === 100 && configs.length === 0) { - return ; + if (progress === 100 && configs.length === 0) { + return ; + } + return ( + + items={configs} + pageState={dataVisualizerListState} + updatePageState={onTableChange} + getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} + extendedColumns={extendedColumns} + showPreviewByDefault={props?.showPreviewByDefault} + onChange={onTableUpdate} + loading={progress < 100} + overallStatsRunning={overallStatsProgress.isRunning} + /> + ); } - return ( - - items={configs} - pageState={dataVisualizerListState} - updatePageState={onTableChange} - getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} - extendedColumns={extendedColumns} - showPreviewByDefault={state?.showPreviewByDefault} - onChange={onTableUpdate} - loading={progress < 100} - overallStatsRunning={overallStatsProgress.isRunning} - /> - ); -}; +); // exporting as default so it be lazy-loaded // eslint-disable-next-line import/no-default-export export default EmbeddableESQLFieldStatsTableWrapper; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_field_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_field_stats_table.tsx index 96cf9c2f7c8fd..5c40b9e9c94f4 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_field_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_field_stats_table.tsx @@ -23,7 +23,7 @@ const restorableDefaults = getDefaultDataVisualizerListState(); const EmbeddableFieldStatsTableWrapper = ( props: Required ) => { - const { onTableUpdate, onAddFilter, ...state } = props; + const { onTableUpdate, onAddFilter } = props; const [dataVisualizerListState, setDataVisualizerListState] = useState>(restorableDefaults); @@ -46,11 +46,11 @@ const EmbeddableFieldStatsTableWrapper = ( progress, overallStatsProgress, setLastRefresh, - } = useDataVisualizerGridData(state, dataVisualizerListState); + } = useDataVisualizerGridData(props, dataVisualizerListState); useEffect(() => { setLastRefresh(Date.now()); - }, [state?.lastReloadRequestTime, setLastRefresh]); + }, [props?.lastReloadRequestTime, setLastRefresh]); const getItemIdToExpandedRowMap = useCallback( function (itemIds: string[], items: FieldVisConfig[]): ItemIdToExpandedRowMap { @@ -60,17 +60,17 @@ const EmbeddableFieldStatsTableWrapper = ( m[fieldName] = ( ); } return m; }, {} as ItemIdToExpandedRowMap); }, - [state.dataView, searchQueryLanguage, searchString, state.totalDocuments, onAddFilter] + [props.dataView, searchQueryLanguage, searchString, props.totalDocuments, onAddFilter] ); if (progress === 100 && configs.length === 0) { @@ -83,7 +83,7 @@ const EmbeddableFieldStatsTableWrapper = ( updatePageState={onTableChange} getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} extendedColumns={extendedColumns} - showPreviewByDefault={state?.showPreviewByDefault} + showPreviewByDefault={props?.showPreviewByDefault} onChange={onTableUpdate} loading={progress < 100} overallStatsRunning={overallStatsProgress.isRunning} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/field_stats_embeddable_wrapper.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/field_stats_embeddable_wrapper.tsx index c07e748ce2151..4d547635eb504 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/field_stats_embeddable_wrapper.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/field_stats_embeddable_wrapper.tsx @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React from 'react'; +import React, { useMemo } from 'react'; import { EuiEmptyPrompt } from '@elastic/eui'; import type { Required } from 'utility-types'; import { FormattedMessage } from '@kbn/i18n-react'; @@ -28,18 +28,15 @@ const EmbeddableESQLFieldStatsTableWrapper = dynamic( const EmbeddableFieldStatsTableWrapper = dynamic(() => import('./embeddable_field_stats_table')); function isESQLFieldStatisticTableEmbeddableState( - input: unknown + input: FieldStatisticTableEmbeddableProps ): input is ESQLDataVisualizerGridEmbeddableState { return isPopulatedObject(input, ['esql']) && input.esql === true; } function isFieldStatisticTableEmbeddableState( - input: unknown + input: FieldStatisticTableEmbeddableProps ): input is Required { - return ( - isPopulatedObject(input, ['dataView']) && - (!isPopulatedObject(input, ['esql']) || input.esql === false) - ); + return isPopulatedObject(input, ['dataView']) && Boolean(input.esql) === false; } const FieldStatisticsWrapperContent = (props: FieldStatisticTableEmbeddableProps) => { @@ -104,17 +101,54 @@ const FieldStatisticsWrapper = (props: FieldStatisticTableEmbeddableProps) => { unifiedSearch, }; - const kibanaRenderServices = pick(coreStart, 'analytics', 'i18n', 'theme'); - const datePickerDeps: DatePickerDependencies = { - ...pick(services, ['data', 'http', 'notifications', 'theme', 'uiSettings', 'i18n']), - uiSettingsKeys: UI_SETTINGS, - }; + const { overridableServices } = props; + + const kibanaRenderServices = useMemo( + () => pick(coreStart, 'analytics', 'i18n', 'theme'), + // eslint-disable-next-line react-hooks/exhaustive-deps + [] + ); + const servicesWithOverrides = useMemo( + () => ({ ...services, ...(overridableServices ?? {}) }), + // eslint-disable-next-line react-hooks/exhaustive-deps + [] + ); + + const datePickerDeps: DatePickerDependencies = useMemo( + () => ({ + ...pick(servicesWithOverrides, [ + 'data', + 'http', + 'notifications', + 'theme', + 'uiSettings', + 'i18n', + ]), + uiSettingsKeys: UI_SETTINGS, + }), + [servicesWithOverrides] + ); return ( - + - + diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/types.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/types.ts index c42f6dbec0624..a703c012a0575 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/types.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/types.ts @@ -12,13 +12,14 @@ import type { SavedSearch } from '@kbn/saved-search-plugin/public'; import type { BehaviorSubject } from 'rxjs'; import type { DefaultEmbeddableApi } from '@kbn/embeddable-plugin/public'; import type { SerializedTitles } from '@kbn/presentation-publishing'; +import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { DataVisualizerTableState } from '../../../../../common/types'; import type { SamplingOption } from '../../../../../common/types/field_stats'; import type { DATA_VISUALIZER_INDEX_VIEWER } from '../../constants/index_data_visualizer_viewer'; import type { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; import type { DataVisualizerStartDependencies } from '../../../common/types/data_visualizer_plugin'; import type { ESQLQuery } from '../../search_strategy/requests/esql_utils'; -import type { DataVisualizerTableItem } from '../../../common/components/stats_table/data_visualizer_stats_table'; +import type { DataVisualizerTableItem } from '../../../common/components/stats_table/types'; export interface FieldStatisticTableEmbeddableProps { /** @@ -92,6 +93,10 @@ export interface FieldStatisticTableEmbeddableProps { shouldGetSubfields?: boolean; lastReloadRequestTime?: number; onTableUpdate?: (update: Partial) => void; + /** + * Inject Kibana services to override in Kibana provider context + */ + overridableServices?: { data: DataPublicPluginStart }; renderFieldName?: (fieldName: string, item: DataVisualizerTableItem) => JSX.Element; } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_data_visualizer_esql_data.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_data_visualizer_esql_data.tsx index 59638a485a3f0..8513df90d682b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_data_visualizer_esql_data.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_data_visualizer_esql_data.tsx @@ -17,9 +17,9 @@ import useObservable from 'react-use/lib/useObservable'; import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import type { KibanaExecutionContext } from '@kbn/core-execution-context-common'; import { useExecutionContext } from '@kbn/kibana-react-plugin/public'; -import type { AggregateQuery } from '@kbn/es-query'; +import type { AggregateQuery, Query } from '@kbn/es-query'; import { useTimeBuckets } from '@kbn/ml-time-buckets'; -import type { SamplingOption } from '../../../../../common/types/field_stats'; +import { buildEsQuery } from '@kbn/es-query'; import type { FieldVisConfig } from '../../../../../common/types/field_vis_config'; import type { SupportedFieldType } from '../../../../../common/types/job_field_type'; import type { ItemIdToExpandedRowMap } from '../../../common/components/stats_table'; @@ -44,18 +44,14 @@ import type { import { getDefaultPageState } from '../../constants/index_data_visualizer_viewer'; import { DEFAULT_ESQL_LIMIT } from '../../constants/esql_constants'; +type AnyQuery = Query | AggregateQuery; + const defaultSearchQuery = { match_all: {}, }; const FALLBACK_ESQL_QUERY: ESQLQuery = { esql: '' }; -const DEFAULT_SAMPLING_OPTION: SamplingOption = { - mode: 'random_sampling', - seed: '', - probability: 0, -}; const DEFAULT_LIMIT_SIZE = '10000'; - const defaults = getDefaultPageState(); export const getDefaultESQLDataVisualizerListState = ( @@ -82,8 +78,7 @@ export const getDefaultESQLDataVisualizerListState = ( }); export const useESQLDataVisualizerData = ( input: ESQLDataVisualizerGridEmbeddableState, - dataVisualizerListState: ESQLDataVisualizerIndexBasedAppState, - setQuery?: React.Dispatch> + dataVisualizerListState: ESQLDataVisualizerIndexBasedAppState ) => { const [lastRefresh, setLastRefresh] = useState(0); const { services } = useDataVisualizerKibana(); @@ -112,20 +107,31 @@ export const useESQLDataVisualizerData = ( autoRefreshSelector: true, }); - const { currentDataView, query, visibleFieldNames, indexPattern } = useMemo( - () => ({ - currentSavedSearch: input?.savedSearch, - currentDataView: input.dataView, - query: input?.query ?? FALLBACK_ESQL_QUERY, - visibleFieldNames: input?.visibleFieldNames ?? [], - currentFilters: input?.filters, - fieldsToFetch: input?.fieldsToFetch, - /** By default, use random sampling **/ - samplingOption: input?.samplingOption ?? DEFAULT_SAMPLING_OPTION, - indexPattern: input?.indexPattern, - }), - [input] - ); + const { currentDataView, parentQuery, parentFilters, query, visibleFieldNames, indexPattern } = + useMemo(() => { + let q = FALLBACK_ESQL_QUERY; + + if (input?.query && isESQLQuery(input?.query)) q = input.query; + if (input?.savedSearch && isESQLQuery(input.savedSearch.searchSource.getField('query'))) { + q = input.savedSearch.searchSource.getField('query') as ESQLQuery; + } + return { + currentDataView: input.dataView, + query: q ?? FALLBACK_ESQL_QUERY, + // It's possible that in a dashboard setting, we will have additional filters and queries + parentQuery: input?.query, + parentFilters: input?.filters, + visibleFieldNames: input?.visibleFieldNames ?? [], + indexPattern: input?.indexPattern, + }; + }, [ + input.query, + input.savedSearch, + input.dataView, + input?.filters, + input?.visibleFieldNames, + input?.indexPattern, + ]); const restorableDefaults = useMemo( () => getDefaultESQLDataVisualizerListState(dataVisualizerListState), @@ -170,8 +176,25 @@ export const useESQLDataVisualizerData = ( const aggInterval = buckets.getInterval(); - const filter = currentDataView?.timeFieldName - ? ({ + let filter: QueryDslQueryContainer = buildEsQuery( + input.dataView, + (Array.isArray(parentQuery) ? parentQuery : [parentQuery]) as AnyQuery | AnyQuery[], + parentFilters ?? [] + ); + + if (currentDataView?.timeFieldName) { + if (Array.isArray(filter?.bool?.filter)) { + filter.bool!.filter!.push({ + range: { + [currentDataView.timeFieldName]: { + format: 'strict_date_optional_time', + gte: timefilter.getTime().from, + lte: timefilter.getTime().to, + }, + }, + }); + } else { + filter = { bool: { must: [], filter: [ @@ -188,8 +211,9 @@ export const useESQLDataVisualizerData = ( should: [], must_not: [], }, - } as QueryDslQueryContainer) - : undefined; + } as QueryDslQueryContainer; + } + } return { earliest, latest, @@ -211,7 +235,7 @@ export const useESQLDataVisualizerData = ( timefilter, currentDataView?.id, // eslint-disable-next-line react-hooks/exhaustive-deps - JSON.stringify(query), + JSON.stringify({ query, parentQuery, parentFilters }), indexPattern, lastRefresh, limitSize, @@ -591,8 +615,8 @@ export const useESQLDataVisualizerData = ( [totalCount, overallStatsProgress.loaded, fieldStatsProgress.loaded] ); - const onQueryUpdate = useCallback( - async (q?: AggregateQuery) => { + const resetData = useCallback( + (q?: AggregateQuery) => { // When user submits a new query // resets all current requests and other data if (cancelOverallStatsRequest) { @@ -605,11 +629,8 @@ export const useESQLDataVisualizerData = ( setFieldStatFieldsToFetch(undefined); setMetricConfigs(defaults.metricConfigs); setNonMetricConfigs(defaults.nonMetricConfigs); - if (isESQLQuery(q) && setQuery) { - setQuery(q); - } }, - [cancelFieldStatsRequest, cancelOverallStatsRequest, setQuery] + [cancelFieldStatsRequest, cancelOverallStatsRequest] ); return { @@ -628,7 +649,7 @@ export const useESQLDataVisualizerData = ( getItemIdToExpandedRowMap, cancelOverallStatsRequest, cancelFieldStatsRequest, - onQueryUpdate, + resetData, limitSize, showEmptyFields, fieldsCountStats, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts index 06e7737a7d2f8..4634c1991de8b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { ESQL_SEARCH_STRATEGY, KBN_FIELD_TYPES } from '@kbn/data-plugin/common'; +import { ESQL_ASYNC_SEARCH_STRATEGY, KBN_FIELD_TYPES } from '@kbn/data-plugin/common'; import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; import type { AggregateQuery } from '@kbn/es-query'; import { i18n } from '@kbn/i18n'; @@ -16,6 +16,7 @@ import type { ISearchOptions } from '@kbn/search-types'; import type { TimeBucketsInterval } from '@kbn/ml-time-buckets'; import { getESQLWithSafeLimit, appendToESQLQuery } from '@kbn/esql-utils'; import { isDefined } from '@kbn/ml-is-defined'; +import { ESQL_SAFE_LIMIT } from '@kbn/unified-field-list/src/constants'; import { OMIT_FIELDS } from '../../../../../common/constants'; import type { DataStatsFetchProgress, @@ -97,7 +98,10 @@ const getESQLDocumentCountStats = async ( }, }; try { - const esqlResults = await runRequest(request, { ...(searchOptions ?? {}), strategy: 'esql' }); + const esqlResults = await runRequest(request, { + ...(searchOptions ?? {}), + strategy: ESQL_ASYNC_SEARCH_STRATEGY, + }); let totalCount = 0; const _buckets: Record = {}; // @ts-expect-error ES types needs to be updated with columns and values as part of esql response @@ -142,7 +146,10 @@ const getESQLDocumentCountStats = async ( }, }; try { - const esqlResults = await runRequest(request, { ...(searchOptions ?? {}), strategy: 'esql' }); + const esqlResults = await runRequest(request, { + ...(searchOptions ?? {}), + strategy: ESQL_ASYNC_SEARCH_STRATEGY, + }); return { request, documentCountStats: undefined, @@ -266,12 +273,12 @@ export const useESQLOverallStatsData = ( { params: { // Doing this to match with the default limit - query: esqlBaseQuery, + query: getESQLWithSafeLimit(esqlBaseQuery, ESQL_SAFE_LIMIT), ...(filter ? { filter } : {}), dropNullColumns: true, }, }, - { strategy: ESQL_SEARCH_STRATEGY } + { strategy: ESQL_ASYNC_SEARCH_STRATEGY } )) as ESQLResponse | undefined; setQueryHistoryStatus(false); @@ -446,13 +453,13 @@ export const useESQLOverallStatsData = ( setTableData({ exampleDocs }); } } catch (error) { + setQueryHistoryStatus(false); // If error already handled in sub functions, no need to propogate if (error.name !== 'AbortError' && error.handled !== true) { toasts.addError(error, { title: fieldStatsErrorTitle, }); } - setQueryHistoryStatus(false); // Log error to console for better debugging // eslint-disable-next-line no-console console.error(`${fieldStatsErrorTitle}: fetchOverallStats`, error); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 699cfaf787295..6cc039fd08b42 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -109,17 +109,16 @@ export const useDataVisualizerGridData = ( }, [security]); const { currentSavedSearch, currentDataView, currentQuery, currentFilters, samplingOption } = - useMemo( - () => ({ + useMemo(() => { + return { currentSavedSearch: input?.savedSearch, currentDataView: input.dataView, currentQuery: input?.query, currentFilters: input?.filters, /** By default, use random sampling **/ samplingOption: input?.samplingOption ?? DEFAULT_SAMPLING_OPTION, - }), - [input] - ); + }; + }, [input?.savedSearch, input.dataView, input?.query, input?.filters, input?.samplingOption]); const dataViewFields: DataViewField[] = useMemo(() => currentDataView.fields, [currentDataView]); const { visibleFieldNames, fieldsToFetch } = useMemo(() => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_boolean_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_boolean_field_stats.ts index 80941e4ff37fd..a91e42e2f8213 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_boolean_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_boolean_field_stats.ts @@ -7,7 +7,7 @@ import type { UseCancellableSearch } from '@kbn/ml-cancellable-search'; import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; -import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; +import { ESQL_ASYNC_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; import pLimit from 'p-limit'; import { appendToESQLQuery } from '@kbn/esql-utils'; import type { Column } from '../../hooks/esql/use_esql_overall_stats_data'; @@ -57,7 +57,7 @@ export const getESQLBooleanFieldStats = async ({ if (booleanFields.length > 0) { const booleanTopTermsResp = await Promise.allSettled( booleanFields.map(({ request }) => - limiter(() => runRequest(request, { strategy: ESQL_SEARCH_STRATEGY })) + limiter(() => runRequest(request, { strategy: ESQL_ASYNC_SEARCH_STRATEGY })) ) ); if (booleanTopTermsResp) { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_count_and_cardinality.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_count_and_cardinality.ts index 80d3f56c4b907..fc4db29b758c4 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_count_and_cardinality.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_count_and_cardinality.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; +import { ESQL_ASYNC_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; import pLimit from 'p-limit'; import { chunk } from 'lodash'; import { isDefined } from '@kbn/ml-is-defined'; @@ -50,8 +50,7 @@ const getESQLOverallStatsInChunk = async ({ let startIndex = 0; /** Example query: * from {indexPattern} | LIMIT {limitSize} - * | EVAL `ne_{aggregableField}` = MV_MIN({aggregableField}), - * | STATs `{aggregableField}_count` = COUNT(`ne_{aggregableField}`), + * | STATs `{aggregableField}_count` = COUNT(MV_MIN(`{aggregableField}`)), * `{aggregableField}_cardinality` = COUNT_DISTINCT({aggregableField}), * `{nonAggregableField}_count` = COUNT({nonAggregableField}) */ @@ -66,15 +65,11 @@ const getESQLOverallStatsInChunk = async ({ // Ex: for 2 docs, count(fieldName) might return 5 // So we need to do count(EVAL(MV_MIN(fieldName))) instead // to get accurate % of rows where field value exists - evalQuery: `${getSafeESQLName(`ne_${field.name}`)} = MV_MIN(${getSafeESQLName( - `${field.name}` - )})`, - query: `${getSafeESQLName(`${field.name}_count`)} = COUNT(${getSafeESQLName( - `ne_${field.name}` - )}), - ${getSafeESQLName(`${field.name}_cardinality`)} = COUNT_DISTINCT(${getSafeESQLName( + query: `${getSafeESQLName(`${field.name}_count`)} = COUNT(MV_MIN(${getSafeESQLName( field.name - )})`, + )})), ${getSafeESQLName( + `${field.name}_cardinality` + )} = COUNT_DISTINCT(${getSafeESQLName(field.name)})`, }; // +2 for count, and count_dictinct startIndex += 2; @@ -93,17 +88,9 @@ const getESQLOverallStatsInChunk = async ({ } }); - const evalQuery = fieldsToFetch - .map((field) => field.evalQuery) - .filter(isDefined) - .join(','); - let countQuery = fieldsToFetch.length > 0 ? '| STATS ' : ''; countQuery += fieldsToFetch.map((field) => field.query).join(','); - const query = appendToESQLQuery( - esqlBaseQueryWithLimit, - (evalQuery ? ' | EVAL ' + evalQuery : '') + countQuery - ); + const query = appendToESQLQuery(esqlBaseQueryWithLimit, countQuery); const request = { params: { @@ -113,7 +100,7 @@ const getESQLOverallStatsInChunk = async ({ }; try { - const esqlResults = await runRequest(request, { strategy: ESQL_SEARCH_STRATEGY }); + const esqlResults = await runRequest(request, { strategy: ESQL_ASYNC_SEARCH_STRATEGY }); const stats = { aggregatableExistsFields: [] as AggregatableField[], aggregatableNotExistsFields: [] as AggregatableField[], diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_date_field_stats.ts index 4b160d7c18fbb..a7ba5a623e5bf 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_date_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_date_field_stats.ts @@ -7,7 +7,7 @@ import type { UseCancellableSearch } from '@kbn/ml-cancellable-search'; import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; -import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; +import { ESQL_ASYNC_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; import { appendToESQLQuery } from '@kbn/esql-utils'; import type { Column } from '../../hooks/esql/use_esql_overall_stats_data'; import { getSafeESQLName } from '../requests/esql_utils'; @@ -45,7 +45,7 @@ export const getESQLDateFieldStats = async ({ }, }; try { - const dateFieldsResp = await runRequest(request, { strategy: ESQL_SEARCH_STRATEGY }); + const dateFieldsResp = await runRequest(request, { strategy: ESQL_ASYNC_SEARCH_STRATEGY }); if (dateFieldsResp) { return dateFields.map(({ field: dateField }, idx) => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_keyword_fields.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_keyword_fields.ts index 02f9370de7b80..46756a5ef7b4e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_keyword_fields.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_keyword_fields.ts @@ -7,7 +7,7 @@ import type { UseCancellableSearch } from '@kbn/ml-cancellable-search'; import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; -import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; +import { ESQL_ASYNC_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; import pLimit from 'p-limit'; import { appendToESQLQuery } from '@kbn/esql-utils'; import type { Column } from '../../hooks/esql/use_esql_overall_stats_data'; @@ -36,7 +36,7 @@ export const getESQLKeywordFieldStats = async ({ esqlBaseQuery, `| STATS ${getSafeESQLName(`${field.name}_in_records`)} = count(MV_MIN(${getSafeESQLName( field.name - )})), ${getSafeESQLName(`${field.name}_in_values`)} = count(${getSafeESQLName(field.name)}) + )})) BY ${getSafeESQLName(field.name)} | SORT ${getSafeESQLName(`${field.name}_in_records`)} DESC | LIMIT 10` @@ -55,7 +55,7 @@ export const getESQLKeywordFieldStats = async ({ if (keywordFields.length > 0) { const keywordTopTermsResp = await Promise.allSettled( keywordFields.map(({ request }) => - limiter(() => runRequest(request, { strategy: ESQL_SEARCH_STRATEGY })) + limiter(() => runRequest(request, { strategy: ESQL_ASYNC_SEARCH_STRATEGY })) ) ); if (keywordTopTermsResp) { @@ -70,24 +70,19 @@ export const getESQLKeywordFieldStats = async ({ if (results) { const topValuesSampleSize = results.reduce((acc, row) => { - return row[1] + acc; + return row[0] + acc; }, 0); - const sampledValues = results.map((row) => ({ - key: row[2], - doc_count: row[1], - })); - const terms = results.map((row) => ({ - key: row[2], + key: row[1], doc_count: row[0], })); return { fieldName: field.name, topValues: terms, - sampledValues, isTopValuesSampled: true, + approximate: true, topValuesSampleSize, } as StringFieldStats; } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts index c943ed042fe95..2a83954503730 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts @@ -7,7 +7,7 @@ import type { UseCancellableSearch } from '@kbn/ml-cancellable-search'; import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; -import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; +import { ESQL_ASYNC_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; import { appendToESQLQuery } from '@kbn/esql-utils'; import { chunk } from 'lodash'; import pLimit from 'p-limit'; @@ -74,7 +74,7 @@ const getESQLNumericFieldStatsInChunk = async ({ }, }; try { - const fieldStatsResp = await runRequest(request, { strategy: ESQL_SEARCH_STRATEGY }); + const fieldStatsResp = await runRequest(request, { strategy: ESQL_ASYNC_SEARCH_STRATEGY }); if (fieldStatsResp) { const values = fieldStatsResp.rawResponse.values[0]; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/esql_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/esql_utils.ts index fa2182f0bbc6e..05ebd45786021 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/esql_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/esql_utils.ts @@ -23,7 +23,7 @@ export const getSafeESQLName = (str: string) => { }; export function isESQLQuery(arg: unknown): arg is ESQLQuery { - return isPopulatedObject(arg, ['esql']); + return isPopulatedObject(arg, ['esql']) && typeof arg.esql === 'string'; } export const PERCENTS = Array.from( Array(MAX_PERCENT / PERCENTILE_SPACING + 1), diff --git a/x-pack/plugins/data_visualizer/tsconfig.json b/x-pack/plugins/data_visualizer/tsconfig.json index 70d967d2a7128..5a166df52eed2 100644 --- a/x-pack/plugins/data_visualizer/tsconfig.json +++ b/x-pack/plugins/data_visualizer/tsconfig.json @@ -81,7 +81,8 @@ "@kbn/react-kibana-context-theme", "@kbn/presentation-publishing", "@kbn/shared-ux-utility", - "@kbn/search-types" + "@kbn/search-types", + "@kbn/unified-field-list" ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index b5074f2d79934..31c1af10f51b9 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -13113,8 +13113,6 @@ "xpack.dataVisualizer.dataGrid.field.metricDistributionChart.tooltipValueBetweenLabel": "{percent} % des documents ont des valeurs comprises entre {minValFormatted} et {maxValFormatted}", "xpack.dataVisualizer.dataGrid.field.metricDistributionChart.tooltipValueEqualLabel": "{percent} % des documents ont une valeur de {valFormatted}", "xpack.dataVisualizer.dataGrid.field.removeFilterAriaLabel": "Exclure {fieldName} : \"{value}\"", - "xpack.dataVisualizer.dataGrid.field.topValues.calculatedFromSampleRecordsLabel": "Calculé à partir de {sampledDocumentsFormatted} {sampledDocuments, plural, one {exemple d'enregistrement} other {exemples d'enregistrement}}.", - "xpack.dataVisualizer.dataGrid.field.topValues.calculatedFromTotalRecordsLabel": "Calculé à partir de {totalDocumentsFormatted} {totalDocuments, plural, one {enregistrement} other {enregistrements}}.", "xpack.dataVisualizer.dataGrid.fieldExpandedRow.choroplethMapTopValues.calculatedFromSampleRecordsLabel": "Calculé à partir de {sampledDocumentsFormatted} {sampledDocuments, plural, one {exemple d'enregistrement} other {exemples d'enregistrement}}.", "xpack.dataVisualizer.dataGrid.fieldExpandedRow.choroplethMapTopValues.calculatedFromTotalRecordsLabel": "Calculé à partir de {totalDocumentsFormatted} {totalDocuments, plural, one {enregistrement} other {enregistrements}}.", "xpack.dataVisualizer.dataGrid.fieldExpandedRow.numberContent.displayingPercentilesLabel": "Affichage de {minPercent} - {maxPercent} centiles", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index d9ff0f4932c25..c63d94591fe7b 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -13094,8 +13094,6 @@ "xpack.dataVisualizer.dataGrid.field.metricDistributionChart.tooltipValueBetweenLabel": "{percent}% のドキュメントに {minValFormatted} から {maxValFormatted} の間の値があります", "xpack.dataVisualizer.dataGrid.field.metricDistributionChart.tooltipValueEqualLabel": "{percent}% のドキュメントに {valFormatted} の値があります", "xpack.dataVisualizer.dataGrid.field.removeFilterAriaLabel": "{fieldName}の除外：\"{value}\"", - "xpack.dataVisualizer.dataGrid.field.topValues.calculatedFromSampleRecordsLabel": "{sampledDocumentsFormatted}サンプル{sampledDocuments, plural, other {レコード}}から計算されました。", - "xpack.dataVisualizer.dataGrid.field.topValues.calculatedFromTotalRecordsLabel": "{totalDocumentsFormatted} {totalDocuments, plural, other {レコード}}から計算されました。", "xpack.dataVisualizer.dataGrid.fieldExpandedRow.choroplethMapTopValues.calculatedFromSampleRecordsLabel": "{sampledDocumentsFormatted}サンプル{sampledDocuments, plural, other {レコード}}から計算されました。", "xpack.dataVisualizer.dataGrid.fieldExpandedRow.choroplethMapTopValues.calculatedFromTotalRecordsLabel": "{totalDocumentsFormatted} {totalDocuments, plural, other {レコード}}から計算されました。", "xpack.dataVisualizer.dataGrid.fieldExpandedRow.numberContent.displayingPercentilesLabel": "{minPercent} - {maxPercent} パーセンタイルを表示中", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 5ddb8e643a359..4b7fcc5ca9c60 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -13119,8 +13119,6 @@ "xpack.dataVisualizer.dataGrid.field.metricDistributionChart.tooltipValueBetweenLabel": "{percent}% 的文档具有介于 {minValFormatted} 和 {maxValFormatted} 之间的值", "xpack.dataVisualizer.dataGrid.field.metricDistributionChart.tooltipValueEqualLabel": "{percent}% 的文档的值为 {valFormatted}", "xpack.dataVisualizer.dataGrid.field.removeFilterAriaLabel": "筛除 {fieldName}：“{value}”", - "xpack.dataVisualizer.dataGrid.field.topValues.calculatedFromSampleRecordsLabel": "基于 {sampledDocumentsFormatted} 个样例{sampledDocuments, plural, other {记录}}计算。", - "xpack.dataVisualizer.dataGrid.field.topValues.calculatedFromTotalRecordsLabel": "基于 {totalDocumentsFormatted} 个样例{totalDocuments, plural, other {记录}}计算。", "xpack.dataVisualizer.dataGrid.fieldExpandedRow.choroplethMapTopValues.calculatedFromSampleRecordsLabel": "基于 {sampledDocumentsFormatted} 个样例{sampledDocuments, plural, other {记录}}计算。", "xpack.dataVisualizer.dataGrid.fieldExpandedRow.choroplethMapTopValues.calculatedFromTotalRecordsLabel": "基于 {totalDocumentsFormatted} 个样例{totalDocuments, plural, other {记录}}计算。", "xpack.dataVisualizer.dataGrid.fieldExpandedRow.numberContent.displayingPercentilesLabel": "正在显示 {minPercent} - {maxPercent} 百分位数", diff --git a/x-pack/test/functional/apps/ml/data_visualizer/esql_data_visualizer.ts b/x-pack/test/functional/apps/ml/data_visualizer/esql_data_visualizer.ts index b4eebbc48b231..f6fe276ac33b7 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/esql_data_visualizer.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/esql_data_visualizer.ts @@ -91,7 +91,7 @@ const esqlFarequoteData = { existsInDocs: true, aggregatable: true, loading: false, - exampleCount: 11, + exampleCount: 10, docCountFormatted: '86,274 (100%)', viewableInLens: false, }, @@ -175,7 +175,7 @@ const esqlSampleLogData: TestData = { aggregatable: true, loading: false, docCountFormatted: '143 (100%)', - exampleCount: 11, + exampleCount: 10, viewableInLens: false, }, ], diff --git a/x-pack/test_serverless/functional/test_suites/common/discover/esql/_esql_view.ts b/x-pack/test_serverless/functional/test_suites/common/discover/esql/_esql_view.ts index 8a0fec8355571..6ce8021024366 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover/esql/_esql_view.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover/esql/_esql_view.ts @@ -79,7 +79,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.missingOrFail('showQueryBarMenu'); await testSubjects.missingOrFail('addFilter'); - await testSubjects.missingOrFail('dscViewModeDocumentButton'); + await testSubjects.existOrFail('dscViewModeToggle'); + await testSubjects.existOrFail('dscViewModeDocumentButton'); // when Lens suggests a table, we render an ESQL based histogram await testSubjects.existOrFail('unifiedHistogramChart'); await testSubjects.existOrFail('discoverQueryHits');