diff --git a/packages/kbn-react-field/src/field_icon/__snapshots__/field_icon.test.tsx.snap b/packages/kbn-react-field/src/field_icon/__snapshots__/field_icon.test.tsx.snap index f6870a5209c1e..0e9ae4ee2aaaa 100644 --- a/packages/kbn-react-field/src/field_icon/__snapshots__/field_icon.test.tsx.snap +++ b/packages/kbn-react-field/src/field_icon/__snapshots__/field_icon.test.tsx.snap @@ -95,6 +95,16 @@ exports[`FieldIcon renders known field types geo_shape is rendered 1`] = ` /> `; +exports[`FieldIcon renders known field types histogram is rendered 1`] = ` + +`; + exports[`FieldIcon renders known field types ip is rendered 1`] = ` `; +exports[`FieldIcon renders known field types keyword is rendered 1`] = ` + +`; + exports[`FieldIcon renders known field types murmur3 is rendered 1`] = ` `; +exports[`FieldIcon renders known field types text is rendered 1`] = ` + +`; + exports[`FieldIcon renders with className if provided 1`] = ` > = { murmur3: { iconType: 'tokenFile' }, number: { iconType: 'tokenNumber' }, number_range: { iconType: 'tokenNumber' }, + histogram: { iconType: 'tokenHistogram' }, _source: { iconType: 'editorCodeBlock', color: 'gray' }, string: { iconType: 'tokenString' }, + text: { iconType: 'tokenString' }, + keyword: { iconType: 'tokenKeyword' }, nested: { iconType: 'tokenNested' }, }; diff --git a/src/plugins/discover/public/application/components/field_stats_table/constants.ts b/src/plugins/discover/public/application/components/field_stats_table/constants.ts new file mode 100644 index 0000000000000..bf1a36da59ecf --- /dev/null +++ b/src/plugins/discover/public/application/components/field_stats_table/constants.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** Telemetry related to field statistics table **/ +export const FIELD_STATISTICS_LOADED = 'field_statistics_loaded'; +export const FIELD_STATISTICS_VIEW_CLICK = 'field_statistics_view_click'; +export const DOCUMENTS_VIEW_CLICK = 'documents_view_click'; diff --git a/src/plugins/discover/public/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx similarity index 78% rename from src/plugins/discover/public/components/data_visualizer_grid/data_visualizer_grid.tsx rename to src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx index 511aa90f5f4a4..5061ab0ba3746 100644 --- a/src/plugins/discover/public/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx @@ -7,18 +7,21 @@ */ import React, { useEffect, useMemo, useRef, useState } from 'react'; -import { Filter } from '@kbn/es-query'; -import { IndexPatternField, IndexPattern, DataView, Query } from '../../../../data/common'; -import { DiscoverServices } from '../../build_services'; +import type { Filter } from '@kbn/es-query'; +import { METRIC_TYPE, UiCounterMetricType } from '@kbn/analytics'; +import { IndexPatternField, IndexPattern, DataView, Query } from '../../../../../data/common'; +import type { DiscoverServices } from '../../../build_services'; import { EmbeddableInput, EmbeddableOutput, ErrorEmbeddable, IEmbeddable, isErrorEmbeddable, -} from '../../../../embeddable/public'; -import { SavedSearch } from '../../services/saved_searches'; -import { GetStateReturn } from '../../application/main/services/discover_state'; +} from '../../../../../embeddable/public'; +import { FIELD_STATISTICS_LOADED } from './constants'; +import type { SavedSearch } from '../../../services/saved_searches'; +import type { GetStateReturn } from '../../main/services/discover_state'; +import { DataRefetch$ } from '../../main/utils/use_saved_search'; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { indexPattern: IndexPattern; @@ -36,7 +39,7 @@ export interface DataVisualizerGridEmbeddableOutput extends EmbeddableOutput { showDistributions?: boolean; } -export interface DiscoverDataVisualizerGridProps { +export interface FieldStatisticsTableProps { /** * Determines which columns are displayed */ @@ -69,14 +72,24 @@ export interface DiscoverDataVisualizerGridProps { * Filters query to update the table content */ filters?: Filter[]; + /** + * State container with persisted settings + */ stateContainer?: GetStateReturn; /** * Callback to add a filter to filter bar */ onAddFilter?: (field: IndexPatternField | string, value: string, type: '+' | '-') => void; + /** + * Metric tracking function + * @param metricType + * @param eventName + */ + trackUiMetric?: (metricType: UiCounterMetricType, eventName: string | string[]) => void; + savedSearchRefetch$?: DataRefetch$; } -export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { +export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => { const { services, indexPattern, @@ -86,9 +99,10 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp filters, stateContainer, onAddFilter, + trackUiMetric, + savedSearchRefetch$, } = props; const { uiSettings } = services; - const [embeddable, setEmbeddable] = useState< | ErrorEmbeddable | IEmbeddable @@ -109,10 +123,16 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp } }); + const refetch = savedSearchRefetch$?.subscribe(() => { + if (embeddable && !isErrorEmbeddable(embeddable)) { + embeddable.updateInput({ lastReloadRequestTime: Date.now() }); + } + }); return () => { sub?.unsubscribe(); + refetch?.unsubscribe(); }; - }, [embeddable, stateContainer]); + }, [embeddable, stateContainer, savedSearchRefetch$]); useEffect(() => { if (embeddable && !isErrorEmbeddable(embeddable)) { @@ -135,17 +155,11 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp embeddable.updateInput({ showPreviewByDefault, }); + embeddable.reload(); } }, [showPreviewByDefault, uiSettings, embeddable]); - useEffect(() => { - return () => { - // Clean up embeddable upon unmounting - embeddable?.destroy(); - }; - }, [embeddable]); - useEffect(() => { let unmounted = false; const loadEmbeddable = async () => { @@ -181,8 +195,15 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp useEffect(() => { if (embeddableRoot.current && embeddable) { embeddable.render(embeddableRoot.current); + + trackUiMetric?.(METRIC_TYPE.LOADED, FIELD_STATISTICS_LOADED); } - }, [embeddable, embeddableRoot, uiSettings]); + + return () => { + // Clean up embeddable upon unmounting + embeddable?.destroy(); + }; + }, [embeddable, embeddableRoot, uiSettings, trackUiMetric]); return (
- { stateContainer.setAppState({ viewMode: mode }); + + if (trackUiMetric) { + if (mode === VIEW_MODE.AGGREGATED_LEVEL) { + trackUiMetric(METRIC_TYPE.CLICK, FIELD_STATISTICS_VIEW_CLICK); + } else { + trackUiMetric(METRIC_TYPE.CLICK, DOCUMENTS_VIEW_CLICK); + } + } }, - [stateContainer] + [trackUiMetric, stateContainer] ); const fetchCounter = useRef(0); @@ -315,7 +327,7 @@ export function DiscoverLayout({ stateContainer={stateContainer} /> ) : ( - )} diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_field.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_field.tsx index 678eddcdf02c0..6864a1c5c2d4a 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_field.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_field.tsx @@ -59,9 +59,11 @@ const FieldInfoIcon: React.FC = memo(() => ( )); -const DiscoverFieldTypeIcon: React.FC<{ field: IndexPatternField }> = memo(({ field }) => ( - -)); +const DiscoverFieldTypeIcon: React.FC<{ field: IndexPatternField }> = memo(({ field }) => { + // If it's a string type, we want to distinguish between keyword and text + const tempType = field.type === 'string' && field.esTypes ? field.esTypes[0] : field.type; + return ; +}); const FieldName: React.FC<{ field: IndexPatternField }> = memo(({ field }) => { const title = diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx index b2b5c8a056995..9dd7ef19ffc07 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { each, cloneDeep } from 'lodash'; +import { cloneDeep, each } from 'lodash'; import { ReactWrapper } from 'enzyme'; import { findTestSubject } from '@elastic/eui/lib/test'; // @ts-expect-error diff --git a/src/plugins/discover/public/application/main/components/sidebar/lib/get_field_type_name.ts b/src/plugins/discover/public/application/main/components/sidebar/lib/get_field_type_name.ts index e2d4c2f7ddcf2..f68395593bd8b 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/lib/get_field_type_name.ts +++ b/src/plugins/discover/public/application/main/components/sidebar/lib/get_field_type_name.ts @@ -51,6 +51,15 @@ export function getFieldTypeName(type: string) { return i18n.translate('discover.fieldNameIcons.stringFieldAriaLabel', { defaultMessage: 'String field', }); + case 'text': + return i18n.translate('discover.fieldNameIcons.textFieldAriaLabel', { + defaultMessage: 'Text field', + }); + case 'keyword': + return i18n.translate('discover.fieldNameIcons.keywordFieldAriaLabel', { + defaultMessage: 'Keyword field', + }); + case 'nested': return i18n.translate('discover.fieldNameIcons.nestedFieldAriaLabel', { defaultMessage: 'Nested field', diff --git a/src/plugins/discover/public/application/main/services/discover_state.ts b/src/plugins/discover/public/application/main/services/discover_state.ts index 66e013e8f20ea..0b855a27cc74e 100644 --- a/src/plugins/discover/public/application/main/services/discover_state.ts +++ b/src/plugins/discover/public/application/main/services/discover_state.ts @@ -411,5 +411,7 @@ function createUrlGeneratorState({ } : undefined, useHash: false, + viewMode: appState.viewMode, + hideAggregatedPreview: appState.hideAggregatedPreview, }; } diff --git a/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx b/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx index 914b9f25d29ae..c04e6515cfbe1 100644 --- a/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx +++ b/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx @@ -45,9 +45,9 @@ import { DiscoverGridSettings } from '../components/discover_grid/types'; import { DocTableProps } from '../components/doc_table/doc_table_wrapper'; import { getDefaultSort } from '../components/doc_table'; import { SortOrder } from '../components/doc_table/components/table_header/helpers'; -import { updateSearchSource } from './utils/update_search_source'; import { VIEW_MODE } from '../components/view_mode_toggle'; -import { FieldStatsTableEmbeddable } from '../components/data_visualizer_grid/field_stats_table_embeddable'; +import { updateSearchSource } from './utils/update_search_source'; +import { FieldStatsTableSavedSearchEmbeddable } from '../application/components/field_stats_table'; export type SearchProps = Partial & Partial & { @@ -391,7 +391,7 @@ export class SavedSearchEmbeddable Array.isArray(searchProps.columns) ) { ReactDOM.render( - ; @@ -102,6 +111,8 @@ export class DiscoverAppLocatorDefinition implements LocatorDefinition esFilters.isFilterPinned(f)); if (refreshInterval) queryState.refreshInterval = refreshInterval; + if (viewMode) appState.viewMode = viewMode; + if (hideAggregatedPreview) appState.hideAggregatedPreview = hideAggregatedPreview; let path = `#/${savedSearchPath}`; path = setStateToKbnUrl('_g', queryState, { useHash }, path); diff --git a/src/plugins/discover/public/url_generator.ts b/src/plugins/discover/public/url_generator.ts index 7cc729fd7f7e5..32e89691574df 100644 --- a/src/plugins/discover/public/url_generator.ts +++ b/src/plugins/discover/public/url_generator.ts @@ -10,6 +10,7 @@ import type { UrlGeneratorsDefinition } from '../../share/public'; import type { TimeRange, Filter, Query, QueryState, RefreshInterval } from '../../data/public'; import { esFilters } from '../../data/public'; import { setStateToKbnUrl } from '../../kibana_utils/public'; +import { VIEW_MODE } from './components/view_mode_toggle'; export const DISCOVER_APP_URL_GENERATOR = 'DISCOVER_APP_URL_GENERATOR'; @@ -75,6 +76,8 @@ export interface DiscoverUrlGeneratorState { * id of the used saved query */ savedQuery?: string; + viewMode?: VIEW_MODE; + hideAggregatedPreview?: boolean; } interface Params { @@ -104,6 +107,8 @@ export class DiscoverUrlGenerator savedQuery, sort, interval, + viewMode, + hideAggregatedPreview, }: DiscoverUrlGeneratorState): Promise => { const savedSearchPath = savedSearchId ? `view/${encodeURIComponent(savedSearchId)}` : ''; const appState: { @@ -114,6 +119,8 @@ export class DiscoverUrlGenerator interval?: string; sort?: string[][]; savedQuery?: string; + viewMode?: VIEW_MODE; + hideAggregatedPreview?: boolean; } = {}; const queryState: QueryState = {}; @@ -130,6 +137,8 @@ export class DiscoverUrlGenerator if (filters && filters.length) queryState.filters = filters?.filter((f) => esFilters.isFilterPinned(f)); if (refreshInterval) queryState.refreshInterval = refreshInterval; + if (viewMode) appState.viewMode = viewMode; + if (hideAggregatedPreview) appState.hideAggregatedPreview = hideAggregatedPreview; let url = `${this.params.appBasePath}#/${savedSearchPath}`; url = setStateToKbnUrl('_g', queryState, { useHash }, url); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/time_buckets.d.ts b/x-pack/plugins/data_visualizer/common/services/time_buckets.d.ts similarity index 96% rename from x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/time_buckets.d.ts rename to x-pack/plugins/data_visualizer/common/services/time_buckets.d.ts index 9a5410918a099..62a3187be47dc 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/time_buckets.d.ts +++ b/x-pack/plugins/data_visualizer/common/services/time_buckets.d.ts @@ -31,7 +31,7 @@ export declare class TimeBuckets { public setMaxBars(maxBars: number): void; public setInterval(interval: string): void; public setBounds(bounds: TimeRangeBounds): void; - public getBounds(): { min: any; max: any }; + public getBounds(): { min: Moment; max: Moment }; public getInterval(): TimeBucketsInterval; public getScaledDateFormat(): string; } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/time_buckets.js b/x-pack/plugins/data_visualizer/common/services/time_buckets.js similarity index 98% rename from x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/time_buckets.js rename to x-pack/plugins/data_visualizer/common/services/time_buckets.js index 5d54b6c936fb2..49de535ee6c26 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/time_buckets.js +++ b/x-pack/plugins/data_visualizer/common/services/time_buckets.js @@ -5,12 +5,12 @@ * 2.0. */ -import { FIELD_FORMAT_IDS } from '../../../../../../../src/plugins/field_formats/common'; -import { UI_SETTINGS } from '../../../../../../../src/plugins/data/common'; +import { FIELD_FORMAT_IDS } from '../../../../../src/plugins/field_formats/common'; +import { UI_SETTINGS } from '../../../../../src/plugins/data/common'; import { ary, assign, isPlainObject, isString, sortBy } from 'lodash'; import moment from 'moment'; import dateMath from '@elastic/datemath'; -import { parseInterval } from '../../common/util/parse_interval'; +import { parseInterval } from '../utils/parse_interval'; const { duration: d } = moment; diff --git a/x-pack/plugins/data_visualizer/common/types/field_request_config.ts b/x-pack/plugins/data_visualizer/common/types/field_request_config.ts index 36e8fe14b7002..f0ea7079bf750 100644 --- a/x-pack/plugins/data_visualizer/common/types/field_request_config.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_request_config.ts @@ -14,7 +14,7 @@ export interface Percentile { } export interface FieldRequestConfig { - fieldName?: string; + fieldName: string; type: JobFieldType; cardinality: number; } @@ -29,6 +29,7 @@ export interface DocumentCounts { } export interface FieldVisStats { + error?: Error; cardinality?: number; count?: number; sampleCount?: number; @@ -58,3 +59,10 @@ export interface FieldVisStats { timeRangeEarliest?: number; timeRangeLatest?: number; } + +export interface DVErrorObject { + causedBy?: string; + message: string; + statusCode?: number; + fullError?: Error; +} diff --git a/x-pack/plugins/data_visualizer/server/types/chart_data.ts b/x-pack/plugins/data_visualizer/common/types/field_stats.ts similarity index 50% rename from x-pack/plugins/data_visualizer/server/types/chart_data.ts rename to x-pack/plugins/data_visualizer/common/types/field_stats.ts index 99c23cf88b5ba..8932a0641cbe6 100644 --- a/x-pack/plugins/data_visualizer/server/types/chart_data.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_stats.ts @@ -5,6 +5,12 @@ * 2.0. */ +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { Query } from '@kbn/es-query'; +import { isPopulatedObject } from '../utils/object_utils'; +import { IKibanaSearchResponse } from '../../../../../src/plugins/data/common'; +import { TimeBucketsInterval } from '../services/time_buckets'; + export interface FieldData { fieldName: string; existsInDocs: boolean; @@ -19,6 +25,12 @@ export interface Field { fieldName: string; type: string; cardinality: number; + safeFieldName: string; +} + +// @todo: check +export function isValidField(arg: unknown): arg is Field { + return isPopulatedObject(arg, ['fieldName', 'type']) && typeof arg.fieldName === 'string'; } export interface HistogramField { @@ -27,19 +39,25 @@ export interface HistogramField { } export interface Distribution { - percentiles: any[]; + percentiles: Array<{ value?: number; percent: number; minValue: number; maxValue: number }>; minPercentile: number; maxPercentile: number; } -export interface Aggs { - [key: string]: any; -} - export interface Bucket { doc_count: number; } +export interface FieldStatsError { + fieldName?: string; + fields?: Field[]; + error: Error; +} + +export const isIKibanaSearchResponse = (arg: unknown): arg is IKibanaSearchResponse => { + return isPopulatedObject(arg, ['rawResponse']); +}; + export interface NumericFieldStats { fieldName: string; count: number; @@ -78,15 +96,15 @@ export interface BooleanFieldStats { } export interface DocumentCountStats { - documentCounts: { - interval: number; - buckets: { [key: string]: number }; - }; + interval: number; + buckets: { [key: string]: number }; + timeRangeEarliest: number; + timeRangeLatest: number; } export interface FieldExamples { fieldName: string; - examples: any[]; + examples: unknown[]; } export interface NumericColumnStats { @@ -97,10 +115,7 @@ export interface NumericColumnStats { export type NumericColumnStatsMap = Record; export interface AggHistogram { - histogram: { - field: string; - interval: number; - }; + histogram: estypes.AggregationsHistogramAggregation; } export interface AggTerms { @@ -142,17 +157,8 @@ export interface UnsupportedChartData { type: 'unsupported'; } -export interface FieldAggCardinality { - field: string; - percent?: any; -} - -export interface ScriptAggCardinality { - script: any; -} - export interface AggCardinality { - cardinality: FieldAggCardinality | ScriptAggCardinality; + cardinality: estypes.AggregationsCardinalityAggregation; } export type ChartRequestAgg = AggHistogram | AggCardinality | AggTerms; @@ -166,3 +172,77 @@ export type BatchStats = | DateFieldStats | DocumentCountStats | FieldExamples; + +export type FieldStats = + | NumericFieldStats + | StringFieldStats + | BooleanFieldStats + | DateFieldStats + | FieldExamples + | FieldStatsError; + +export function isValidFieldStats(arg: unknown): arg is FieldStats { + return isPopulatedObject(arg, ['fieldName', 'type', 'count']); +} + +export interface FieldStatsCommonRequestParams { + index: string; + samplerShardSize: number; + timeFieldName?: string; + earliestMs?: number | undefined; + latestMs?: number | undefined; + runtimeFieldMap?: estypes.MappingRuntimeFields; + intervalMs?: number; + query: estypes.QueryDslQueryContainer; + maxExamples?: number; +} + +export interface OverallStatsSearchStrategyParams { + sessionId?: string; + earliest?: number; + latest?: number; + aggInterval: TimeBucketsInterval; + intervalMs?: number; + searchQuery: Query['query']; + samplerShardSize: number; + index: string; + timeFieldName?: string; + runtimeFieldMap?: estypes.MappingRuntimeFields; + aggregatableFields: string[]; + nonAggregatableFields: string[]; +} + +export interface FieldStatsSearchStrategyReturnBase { + progress: DataStatsFetchProgress; + fieldStats: Map | undefined; + startFetch: () => void; + cancelFetch: () => void; +} + +export interface DataStatsFetchProgress { + error?: Error; + isRunning: boolean; + loaded: number; + total: number; +} + +export interface FieldData { + fieldName: string; + existsInDocs: boolean; + stats?: { + sampleCount?: number; + count?: number; + cardinality?: number; + }; +} + +export interface Field { + fieldName: string; + type: string; + cardinality: number; + safeFieldName: string; +} + +export interface Aggs { + [key: string]: estypes.AggregationsAggregationContainer; +} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_vis_config.ts b/x-pack/plugins/data_visualizer/common/types/field_vis_config.ts similarity index 92% rename from x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_vis_config.ts rename to x-pack/plugins/data_visualizer/common/types/field_vis_config.ts index eeb9fe12692fd..dcd7da74b85ef 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_vis_config.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_vis_config.ts @@ -5,8 +5,7 @@ * 2.0. */ -import type { Percentile, JobFieldType, FieldVisStats } from '../../../../../../common/types'; - +import type { Percentile, JobFieldType, FieldVisStats } from './index'; export interface MetricFieldVisStats { avg?: number; distribution?: { @@ -23,7 +22,7 @@ export interface MetricFieldVisStats { // which display the field information. export interface FieldVisConfig { type: JobFieldType; - fieldName?: string; + fieldName: string; displayName?: string; existsInDocs: boolean; aggregatable: boolean; diff --git a/x-pack/plugins/data_visualizer/common/types/index.ts b/x-pack/plugins/data_visualizer/common/types/index.ts index 1153b45e1cce2..381f7a556b18d 100644 --- a/x-pack/plugins/data_visualizer/common/types/index.ts +++ b/x-pack/plugins/data_visualizer/common/types/index.ts @@ -15,7 +15,6 @@ export type { FieldVisStats, Percentile, } from './field_request_config'; -export type InputData = any[]; export interface DataVisualizerTableState { pageSize: number; diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/parse_interval.ts b/x-pack/plugins/data_visualizer/common/utils/parse_interval.ts similarity index 100% rename from x-pack/plugins/data_visualizer/public/application/common/util/parse_interval.ts rename to x-pack/plugins/data_visualizer/common/utils/parse_interval.ts diff --git a/x-pack/plugins/data_visualizer/common/utils/query_utils.ts b/x-pack/plugins/data_visualizer/common/utils/query_utils.ts index 2aa4cd063d1b1..dc21bbcae96c3 100644 --- a/x-pack/plugins/data_visualizer/common/utils/query_utils.ts +++ b/x-pack/plugins/data_visualizer/common/utils/query_utils.ts @@ -6,6 +6,8 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { Query } from '@kbn/es-query'; + /* * Contains utility functions for building and processing queries. */ @@ -16,8 +18,8 @@ export function buildBaseFilterCriteria( timeFieldName?: string, earliestMs?: number, latestMs?: number, - query?: object -) { + query?: Query['query'] +): estypes.QueryDslQueryContainer[] { const filterCriteria = []; if (timeFieldName && earliestMs && latestMs) { filterCriteria.push({ @@ -31,7 +33,7 @@ export function buildBaseFilterCriteria( }); } - if (query) { + if (query && typeof query === 'object') { filterCriteria.push(query); } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx index d49dbdc7cb446..832e18a12369f 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx @@ -7,30 +7,25 @@ import React, { FC } from 'react'; import { DocumentCountChart, DocumentCountChartPoint } from './document_count_chart'; -import { FieldVisConfig, FileBasedFieldVisConfig } from '../stats_table/types'; import { TotalCountHeader } from './total_count_header'; +import { DocumentCountStats } from '../../../../../common/types/field_stats'; export interface Props { - config?: FieldVisConfig | FileBasedFieldVisConfig; + documentCountStats?: DocumentCountStats; totalCount: number; } -export const DocumentCountContent: FC = ({ config, totalCount }) => { - if (config?.stats === undefined) { +export const DocumentCountContent: FC = ({ documentCountStats, totalCount }) => { + if (documentCountStats === undefined) { return totalCount !== undefined ? : null; } - const { documentCounts, timeRangeEarliest, timeRangeLatest } = config.stats; - if ( - documentCounts === undefined || - timeRangeEarliest === undefined || - timeRangeLatest === undefined - ) - return null; + const { timeRangeEarliest, timeRangeLatest } = documentCountStats; + if (timeRangeEarliest === undefined || timeRangeLatest === undefined) return null; let chartPoints: DocumentCountChartPoint[] = []; - if (documentCounts.buckets !== undefined) { - const buckets: Record = documentCounts?.buckets; + if (documentCountStats.buckets !== undefined) { + const buckets: Record = documentCountStats?.buckets; chartPoints = Object.entries(buckets).map(([time, value]) => ({ time: +time, value })); } @@ -41,7 +36,7 @@ export const DocumentCountContent: FC = ({ config, totalCount }) => { chartPoints={chartPoints} timeRangeEarliest={timeRangeEarliest} timeRangeLatest={timeRangeLatest} - interval={documentCounts.interval} + interval={documentCountStats.interval} /> ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/file_based_expanded_row.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/file_based_expanded_row.tsx index 8a9f9a25c16fa..7ba1615e22b43 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/file_based_expanded_row.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/file_based_expanded_row.tsx @@ -17,7 +17,7 @@ import { } from '../stats_table/components/field_data_expanded_row'; import { GeoPointContent } from './geo_point_content/geo_point_content'; import { JOB_FIELD_TYPES } from '../../../../../common'; -import type { FileBasedFieldVisConfig } from '../stats_table/types/field_vis_config'; +import type { FileBasedFieldVisConfig } from '../../../../../common/types/field_vis_config'; export const FileBasedDataVisualizerExpandedRow = ({ item }: { item: FileBasedFieldVisConfig }) => { const config = item; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx index 79af35f1c8005..b87da2b3da789 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx @@ -23,6 +23,7 @@ import { IndexPattern } from '../../../../../../../../src/plugins/data/common'; import { CombinedQuery } from '../../../index_data_visualizer/types/combined_query'; import { LoadingIndicator } from '../loading_indicator'; import { IndexPatternField } from '../../../../../../../../src/plugins/data/common'; +import { ErrorMessageContent } from '../stats_table/components/field_data_expanded_row/error_message'; export const IndexBasedDataVisualizerExpandedRow = ({ item, @@ -46,6 +47,10 @@ export const IndexBasedDataVisualizerExpandedRow = ({ return ; } + if (config.stats?.error) { + return ; + } + switch (type) { case JOB_FIELD_TYPES.NUMBER: return ; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_names_filter/field_names_filter.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_names_filter/field_names_filter.tsx index 88b4cd406b33c..58e9b9b5740dc 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_names_filter/field_names_filter.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_names_filter/field_names_filter.tsx @@ -11,7 +11,7 @@ import { MultiSelectPicker } from '../multi_select_picker'; import type { FileBasedFieldVisConfig, FileBasedUnknownFieldVisConfig, -} from '../stats_table/types/field_vis_config'; +} from '../../../../../common/types/field_vis_config'; interface Props { fields: Array; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/__snapshots__/field_type_icon.test.tsx.snap b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/__snapshots__/field_type_icon.test.tsx.snap index 398dc5dad2dc7..af4464cbc6b4e 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/__snapshots__/field_type_icon.test.tsx.snap +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/__snapshots__/field_type_icon.test.tsx.snap @@ -3,15 +3,13 @@ exports[`FieldTypeIcon render component when type matches a field type 1`] = ` - `; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.test.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.test.tsx index b6a5ff3e5dbed..0c036dd6c6d76 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.test.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.test.tsx @@ -14,7 +14,7 @@ import { JOB_FIELD_TYPES } from '../../../../../common'; describe('FieldTypeIcon', () => { test(`render component when type matches a field type`, () => { const typeIconComponent = shallow( - + ); expect(typeIconComponent).toMatchSnapshot(); }); @@ -24,7 +24,7 @@ describe('FieldTypeIcon', () => { jest.useFakeTimers(); const typeIconComponent = mount( - + ); expect(typeIconComponent.find('EuiToolTip').children()).toHaveLength(1); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx index 9d803e3d4a80c..2a9767ccd62b1 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx @@ -6,103 +6,32 @@ */ import React, { FC } from 'react'; -import { EuiToken, EuiToolTip } from '@elastic/eui'; +import { EuiToolTip } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { getJobTypeAriaLabel } from '../../util/field_types_utils'; +import { FieldIcon } from '@kbn/react-field/field_icon'; +import { getJobTypeLabel } from '../../util/field_types_utils'; import type { JobFieldType } from '../../../../../common'; import './_index.scss'; interface FieldTypeIconProps { tooltipEnabled: boolean; type: JobFieldType; - needsAria: boolean; } -interface FieldTypeIconContainerProps { - ariaLabel: string | null; - iconType: string; - color?: string; - needsAria: boolean; - [key: string]: any; -} - -// defaultIcon => a unknown datatype -const defaultIcon = { iconType: 'questionInCircle', color: 'gray' }; - -// Extended & modified version of src/plugins/kibana_react/public/field_icon/field_icon.tsx -export const typeToEuiIconMap: Record = { - boolean: { iconType: 'tokenBoolean' }, - // icon for a data view mapping conflict in discover - conflict: { iconType: 'alert', color: 'euiColorVis9' }, - date: { iconType: 'tokenDate' }, - date_range: { iconType: 'tokenDate' }, - geo_point: { iconType: 'tokenGeo' }, - geo_shape: { iconType: 'tokenGeo' }, - ip: { iconType: 'tokenIP' }, - ip_range: { iconType: 'tokenIP' }, - // is a plugin's data type https://www.elastic.co/guide/en/elasticsearch/plugins/current/mapper-murmur3-usage.html - murmur3: { iconType: 'tokenFile' }, - number: { iconType: 'tokenNumber' }, - number_range: { iconType: 'tokenNumber' }, - histogram: { iconType: 'tokenHistogram' }, - _source: { iconType: 'editorCodeBlock', color: 'gray' }, - string: { iconType: 'tokenString' }, - text: { iconType: 'tokenString' }, - keyword: { iconType: 'tokenString' }, - nested: { iconType: 'tokenNested' }, -}; - -export const FieldTypeIcon: FC = ({ - tooltipEnabled = false, - type, - needsAria = true, -}) => { - const ariaLabel = getJobTypeAriaLabel(type); - const token = typeToEuiIconMap[type] || defaultIcon; - const containerProps = { ...token, ariaLabel, needsAria }; - +export const FieldTypeIcon: FC = ({ tooltipEnabled = false, type }) => { + const label = + getJobTypeLabel(type) ?? + i18n.translate('xpack.dataVisualizer.fieldTypeIcon.fieldTypeTooltip', { + defaultMessage: '{type} type', + values: { type }, + }); if (tooltipEnabled === true) { return ( - - + + ); } - return ; -}; - -// If the tooltip is used, it will apply its events to its first inner child. -// To pass on its properties we apply `rest` to the outer `span` element. -const FieldTypeIconContainer: FC = ({ - ariaLabel, - iconType, - color, - needsAria, - ...rest -}) => { - const wrapperProps: { className: string; 'aria-label'?: string } = { - className: 'field-type-icon', - }; - if (needsAria && ariaLabel) { - wrapperProps['aria-label'] = ariaLabel; - } - return ( - - ); + return ; }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx index 97dc2077d5931..0fa860bc6f55e 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx @@ -12,7 +12,7 @@ import { MultiSelectPicker, Option } from '../multi_select_picker'; import type { FileBasedFieldVisConfig, FileBasedUnknownFieldVisConfig, -} from '../stats_table/types/field_vis_config'; +} from '../../../../../common/types/field_vis_config'; import { FieldTypeIcon } from '../field_type_icon'; import { jobTypeLabels } from '../../util/field_types_utils'; @@ -50,7 +50,7 @@ export const DataVisualizerFieldTypesFilter: FC = ({ {label} {type && ( - + )} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx index b57072eed2944..1173ede84e631 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx @@ -10,7 +10,7 @@ import { EuiFlexGroup, EuiSpacer } from '@elastic/eui'; import type { FindFileStructureResponse } from '../../../../../../file_upload/common'; import type { DataVisualizerTableState } from '../../../../../common'; import { DataVisualizerTable, ItemIdToExpandedRowMap } from '../stats_table'; -import type { FileBasedFieldVisConfig } from '../stats_table/types/field_vis_config'; +import type { FileBasedFieldVisConfig } from '../../../../../common/types/field_vis_config'; import { FileBasedDataVisualizerExpandedRow } from '../expanded_row'; import { DataVisualizerFieldNamesFilter } from '../field_names_filter'; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/filter_fields.ts b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/filter_fields.ts index 6c164233bdbc1..9f1ea4af22537 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/filter_fields.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/filter_fields.ts @@ -9,7 +9,7 @@ import { JOB_FIELD_TYPES } from '../../../../../common'; import type { FileBasedFieldVisConfig, FileBasedUnknownFieldVisConfig, -} from '../stats_table/types/field_vis_config'; +} from '../../../../../common/types/field_vis_config'; export function filterFields( fields: Array, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/error_message.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/error_message.tsx new file mode 100644 index 0000000000000..1d4a685457e25 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/error_message.tsx @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiText } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import React from 'react'; +import { DVErrorObject } from '../../../../../index_data_visualizer/utils/error_utils'; + +export const ErrorMessageContent = ({ + fieldName, + error, +}: { + fieldName: string; + error: DVErrorObject; +}) => { + return ( + + + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx index a5db86e0c30a0..d32a8a6dfb907 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx @@ -21,12 +21,14 @@ export const IpContent: FC = ({ config, onAddFilter }) => { return ( - + {stats && ( + + )} ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx index 6f946fc1025ed..4fc73f0831dfc 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx @@ -30,8 +30,9 @@ export const TextContent: FC = ({ config }) => { {numExamples > 0 && } {numExamples === 0 && ( - + { expect(getLegendText(validUnsupportedChartData, 20)).toBe('Chart not supported.'); }); it('should return the chart legend text for empty datasets', () => { - expect(getLegendText(validNumericChartData, 20)).toBe('0 documents contain field.'); + expect(getLegendText(validNumericChartData, 20)).toBe(''); }); it('should return the chart legend text for boolean chart types', () => { const { getByText } = render( @@ -186,7 +186,7 @@ describe('useColumnChart()', () => { ); expect(result.current.data).toStrictEqual([]); - expect(result.current.legendText).toBe('0 documents contain field.'); + expect(result.current.legendText).toBe(''); expect(result.current.xScaleType).toBe('linear'); }); }); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.tsx index 60e1595c64ece..827e4a7f44857 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.tsx @@ -83,9 +83,7 @@ export const getLegendText = (chartData: ChartData, maxChartColumns: number): Le } if (chartData.data.length === 0) { - return i18n.translate('xpack.dataVisualizer.dataGridChart.notEnoughData', { - defaultMessage: `0 documents contain field.`, - }); + return ''; } if (chartData.type === 'boolean') { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 4e1c03aa987bd..976afc464a672 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -33,12 +33,13 @@ import { FieldVisConfig, FileBasedFieldVisConfig, isIndexBasedFieldVisConfig, -} from './types/field_vis_config'; +} from '../../../../../common/types/field_vis_config'; import { FileBasedNumberContentPreview } from '../field_data_row'; import { BooleanContentPreview } from './components/field_data_row'; import { calculateTableColumnsDimensions } from './utils'; import { DistinctValues } from './components/field_data_row/distinct_values'; import { FieldTypeIcon } from '../field_type_icon'; +import './_index.scss'; const FIELD_NAME = 'fieldName'; @@ -54,6 +55,7 @@ interface DataVisualizerTableProps { showPreviewByDefault?: boolean; /** Callback to receive any updates when table or page state is changed **/ onChange?: (update: Partial) => void; + loading?: boolean; } export const DataVisualizerTable = ({ @@ -64,6 +66,7 @@ export const DataVisualizerTable = ({ extendedColumns, showPreviewByDefault, onChange, + loading, }: DataVisualizerTableProps) => { const [expandedRowItemIds, setExpandedRowItemIds] = useState([]); const [expandAll, setExpandAll] = useState(false); @@ -180,7 +183,7 @@ export const DataVisualizerTable = ({ defaultMessage: 'Type', }), render: (fieldType: JobFieldType) => { - return ; + return ; }, width: dimensions.type, sortable: true, @@ -322,6 +325,13 @@ export const DataVisualizerTable = ({ {(resizeRef) => (
+ message={ + loading + ? i18n.translate('xpack.dataVisualizer.dataGrid.searchingMessage', { + defaultMessage: 'Searching', + }) + : undefined + } className={'dvTable'} items={items} itemId={FIELD_NAME} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_data_row.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_data_row.ts index 94b704764c93b..3d7678c7b60a5 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_data_row.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_data_row.ts @@ -5,8 +5,11 @@ * 2.0. */ -import type { FieldVisConfig, FileBasedFieldVisConfig } from './field_vis_config'; import { IndexPatternField } from '../../../../../../../../../src/plugins/data/common'; +import { + FieldVisConfig, + FileBasedFieldVisConfig, +} from '../../../../../../common/types/field_vis_config'; export interface FieldDataRowProps { config: FieldVisConfig | FileBasedFieldVisConfig; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts index 00f8ac0c74eb9..6d9f4d5b86d28 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts @@ -4,11 +4,13 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - export type { FieldDataRowProps } from './field_data_row'; export type { FieldVisConfig, FileBasedFieldVisConfig, MetricFieldVisStats, -} from './field_vis_config'; -export { isFileBasedFieldVisConfig, isIndexBasedFieldVisConfig } from './field_vis_config'; +} from '../../../../../../common/types/field_vis_config'; +export { + isFileBasedFieldVisConfig, + isIndexBasedFieldVisConfig, +} from '../../../../../../common/types/field_vis_config'; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index e2793512e23df..c9b4137a0106d 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -43,7 +43,7 @@ function getPercentLabel(docCount: number, topValuesSampleSize: number): string } export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed, onAddFilter }) => { - if (stats === undefined) return null; + if (stats === undefined || !stats.topValues) return null; const { topValues, topValuesSampleSize, @@ -81,11 +81,11 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed, size="xs" label={kibanaFieldFormat(value.key, fieldFormat)} className={classNames('eui-textTruncate', 'topValuesValueLabelContainer')} - valueText={ + valueText={`${value.doc_count}${ progressBarMax !== undefined - ? getPercentLabel(value.doc_count, progressBarMax) - : undefined - } + ? ` (${getPercentLabel(value.doc_count, progressBarMax)})` + : '' + }`} /> {fieldName !== undefined && value.key !== undefined && onAddFilter !== undefined ? ( diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.test.ts b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.test.ts index 5c0867c7a0745..710ba12313f17 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.test.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.test.ts @@ -6,24 +6,23 @@ */ import { JOB_FIELD_TYPES } from '../../../../common'; -import { getJobTypeAriaLabel, jobTypeAriaLabels } from './field_types_utils'; +import { getJobTypeLabel, jobTypeLabels } from './field_types_utils'; describe('field type utils', () => { - describe('getJobTypeAriaLabel: Getting a field type aria label by passing what it is stored in constants', () => { + describe('getJobTypeLabel: Getting a field type aria label by passing what it is stored in constants', () => { test('should returns all JOB_FIELD_TYPES labels exactly as it is for each correct value', () => { const keys = Object.keys(JOB_FIELD_TYPES); const receivedLabels: Record = {}; - const testStorage = jobTypeAriaLabels; - keys.forEach((constant) => { - receivedLabels[constant] = getJobTypeAriaLabel( - JOB_FIELD_TYPES[constant as keyof typeof JOB_FIELD_TYPES] - ); + const testStorage = jobTypeLabels; + keys.forEach((key) => { + const constant = key as keyof typeof JOB_FIELD_TYPES; + receivedLabels[JOB_FIELD_TYPES[constant]] = getJobTypeLabel(JOB_FIELD_TYPES[constant]); }); expect(receivedLabels).toEqual(testStorage); }); test('should returns NULL as JOB_FIELD_TYPES does not contain such a keyword', () => { - expect(getJobTypeAriaLabel('JOB_FIELD_TYPES')).toBe(null); + expect(getJobTypeLabel('JOB_FIELD_TYPES')).toBe(null); }); }); }); diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts index 3e459cd2b079b..1fda7140dbab2 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts @@ -10,40 +10,8 @@ import { JOB_FIELD_TYPES } from '../../../../common'; import type { IndexPatternField } from '../../../../../../../src/plugins/data/common'; import { KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/common'; -export const jobTypeAriaLabels = { - BOOLEAN: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.booleanTypeAriaLabel', { - defaultMessage: 'boolean type', - }), - DATE: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.dateTypeAriaLabel', { - defaultMessage: 'date type', - }), - GEO_POINT: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.geoPointTypeAriaLabel', { - defaultMessage: '{geoPointParam} type', - values: { - geoPointParam: 'geo point', - }, - }), - GEO_SHAPE: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.geoShapeTypeAriaLabel', { - defaultMessage: 'geo shape type', - }), - IP: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.ipTypeAriaLabel', { - defaultMessage: 'ip type', - }), - KEYWORD: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.keywordTypeAriaLabel', { - defaultMessage: 'keyword type', - }), - NUMBER: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.numberTypeAriaLabel', { - defaultMessage: 'number type', - }), - HISTOGRAM: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.histogramTypeAriaLabel', { - defaultMessage: 'histogram type', - }), - TEXT: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.textTypeAriaLabel', { - defaultMessage: 'text type', - }), - UNKNOWN: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.unknownTypeAriaLabel', { - defaultMessage: 'unknown type', - }), +export const getJobTypeLabel = (type: string) => { + return type in jobTypeLabels ? jobTypeLabels[type as keyof typeof jobTypeLabels] : null; }; export const jobTypeLabels = { @@ -88,16 +56,6 @@ export const jobTypeLabels = { }), }; -export const getJobTypeAriaLabel = (type: string) => { - const requestedFieldType = Object.keys(JOB_FIELD_TYPES).find( - (k) => JOB_FIELD_TYPES[k as keyof typeof JOB_FIELD_TYPES] === type - ); - if (requestedFieldType === undefined) { - return null; - } - return jobTypeAriaLabels[requestedFieldType as keyof typeof jobTypeAriaLabels]; -}; - // convert kibana types to ML Job types // this is needed because kibana types only have string and not text and keyword. // and we can't use ES_FIELD_TYPES because it has no NUMBER type diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/parse_interval.test.ts b/x-pack/plugins/data_visualizer/public/application/common/util/parse_interval.test.ts index a1608960a91bc..c259f82d12bfb 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/util/parse_interval.test.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/util/parse_interval.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { parseInterval } from './parse_interval'; +import { parseInterval } from '../../../../common/utils/parse_interval'; describe('ML parse interval util', () => { test('should correctly parse an interval containing a valid unit and value', () => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/full_time_range_selector/full_time_range_selector.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/full_time_range_selector/full_time_range_selector.tsx index e5bd7a0d6f526..ebddd5527f5a2 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/full_time_range_selector/full_time_range_selector.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/full_time_range_selector/full_time_range_selector.tsx @@ -6,7 +6,6 @@ */ import React, { FC } from 'react'; - import { FormattedMessage } from '@kbn/i18n/react'; import { Query, IndexPattern, TimefilterContract } from 'src/plugins/data/public'; import { EuiButton } from '@elastic/eui'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index cdf4b718a93b7..f528d8378bcd2 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -6,7 +6,6 @@ */ import React, { FC, Fragment, useEffect, useMemo, useState, useCallback, useRef } from 'react'; -import { merge } from 'rxjs'; import { EuiFlexGroup, EuiFlexItem, @@ -16,6 +15,7 @@ import { EuiPageContentHeader, EuiPageContentHeaderSection, EuiPanel, + EuiProgress, EuiSpacer, EuiTitle, } from '@elastic/eui'; @@ -24,12 +24,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { Required } from 'utility-types'; import { i18n } from '@kbn/i18n'; import { Filter } from '@kbn/es-query'; -import { - KBN_FIELD_TYPES, - UI_SETTINGS, - Query, - generateFilters, -} from '../../../../../../../../src/plugins/data/public'; +import { Query, generateFilters } from '../../../../../../../../src/plugins/data/public'; import { FullTimeRangeSelector } from '../full_time_range_selector'; import { usePageUrlState, useUrlState } from '../../../common/util/url_state'; import { @@ -37,39 +32,29 @@ import { ItemIdToExpandedRowMap, } from '../../../common/components/stats_table'; import { FieldVisConfig } from '../../../common/components/stats_table/types'; -import type { - MetricFieldsStats, - TotalFieldsStats, -} from '../../../common/components/stats_table/components/field_count_stats'; +import type { TotalFieldsStats } from '../../../common/components/stats_table/components/field_count_stats'; import { OverallStats } from '../../types/overall_stats'; import { getActions } from '../../../common/components/field_data_row/action_menu'; import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; import { DATA_VISUALIZER_INDEX_VIEWER } from '../../constants/index_data_visualizer_viewer'; import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../../types/combined_query'; -import { - FieldRequestConfig, - JobFieldType, - SavedSearchSavedObject, -} from '../../../../../common/types'; +import { JobFieldType, SavedSearchSavedObject } from '../../../../../common/types'; import { useDataVisualizerKibana } from '../../../kibana_context'; import { FieldCountPanel } from '../../../common/components/field_count_panel'; import { DocumentCountContent } from '../../../common/components/document_count_content'; -import { DataLoader } from '../../data_loader/data_loader'; -import { JOB_FIELD_TYPES, OMIT_FIELDS } from '../../../../../common'; -import { useTimefilter } from '../../hooks/use_time_filter'; +import { OMIT_FIELDS } from '../../../../../common'; import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; import { SearchPanel } from '../search_panel'; import { ActionsPanel } from '../actions_panel'; import { DatePickerWrapper } from '../../../common/components/date_picker_wrapper'; -import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; import { HelpMenu } from '../../../common/components/help_menu'; -import { TimeBuckets } from '../../services/time_buckets'; -import { createMergedEsQuery, getEsQueryFromSavedSearch } from '../../utils/saved_search_utils'; +import { createMergedEsQuery } from '../../utils/saved_search_utils'; import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; import { ResultLink } from '../../../common/components/results_links'; -import { extractErrorProperties } from '../../utils/error_utils'; import { IndexPatternField, IndexPattern } from '../../../../../../../../src/plugins/data/common'; +import { useDataVisualizerGridData } from '../../hooks/use_data_visualizer_grid_data'; +import { DataVisualizerGridInput } from '../../embeddables/grid_embeddable/grid_embeddable'; import './_index.scss'; interface DataVisualizerPageState { @@ -155,61 +140,14 @@ export const IndexDataVisualizerView: FC = (dataVi } }, [dataVisualizerProps?.currentSavedSearch]); - useEffect(() => { - return () => { - // When navigating away from the data view - // Reset all previously set filters - // to make sure new page doesn't have unrelated filters - data.query.filterManager.removeAll(); - }; - }, [currentIndexPattern.id, data.query.filterManager]); - - const getTimeBuckets = useCallback(() => { - return new TimeBuckets({ - [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), - [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), - dateFormat: uiSettings.get('dateFormat'), - 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), - }); - }, [uiSettings]); - - const timefilter = useTimefilter({ - timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, - autoRefreshSelector: true, - }); - - const dataLoader = useMemo( - () => new DataLoader(currentIndexPattern, toasts), - [currentIndexPattern, toasts] - ); - - useEffect(() => { - if (globalState?.time !== undefined) { - timefilter.setTime({ - from: globalState.time.from, - to: globalState.time.to, - }); - setLastRefresh(Date.now()); - } - }, [globalState, timefilter]); - - useEffect(() => { - if (globalState?.refreshInterval !== undefined) { - timefilter.setRefreshInterval(globalState.refreshInterval); - setLastRefresh(Date.now()); - } - }, [globalState, timefilter]); - - const [lastRefresh, setLastRefresh] = useState(0); - useEffect(() => { if (!currentIndexPattern.isTimeBased()) { toasts.addWarning({ title: i18n.translate( - 'xpack.dataVisualizer.index.dataViewNotBasedOnTimeSeriesNotificationTitle', + 'xpack.dataVisualizer.index.indexPatternNotBasedOnTimeSeriesNotificationTitle', { - defaultMessage: 'The data view {dataViewTitle} is not based on a time series', - values: { dataViewTitle: currentIndexPattern.title }, + defaultMessage: 'The index pattern {indexPatternTitle} is not based on a time series', + values: { indexPatternTitle: currentIndexPattern.title }, } ), text: i18n.translate( @@ -225,7 +163,7 @@ export const IndexDataVisualizerView: FC = (dataVi const indexPatternFields: IndexPatternField[] = currentIndexPattern.fields; const fieldTypes = useMemo(() => { - // Obtain the list of non metric field types which appear in the data view. + // Obtain the list of non metric field types which appear in the index pattern. const indexedFieldTypes: JobFieldType[] = []; indexPatternFields.forEach((field) => { if (!OMIT_FIELDS.includes(field.name) && field.scripted !== true) { @@ -238,35 +176,6 @@ export const IndexDataVisualizerView: FC = (dataVi return indexedFieldTypes.sort(); }, [indexPatternFields]); - const defaults = getDefaultPageState(); - - const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - const searchData = getEsQueryFromSavedSearch({ - indexPattern: currentIndexPattern, - uiSettings, - savedSearch: currentSavedSearch, - filterManager: data.query.filterManager, - }); - - if (searchData === undefined || dataVisualizerListState.searchString !== '') { - if (dataVisualizerListState.filters) { - data.query.filterManager.setFilters(dataVisualizerListState.filters); - } - return { - searchQuery: dataVisualizerListState.searchQuery, - searchString: dataVisualizerListState.searchString, - searchQueryLanguage: dataVisualizerListState.searchQueryLanguage, - }; - } else { - return { - searchQuery: searchData.searchQuery, - searchString: searchData.searchString, - searchQueryLanguage: searchData.queryLanguage, - }; - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState, data.query]); - const setSearchParams = useCallback( (searchParams: { searchQuery: Query['query']; @@ -275,7 +184,7 @@ export const IndexDataVisualizerView: FC = (dataVi filters: Filter[]; }) => { // When the user loads saved search and then clear or modify the query - // we should remove the saved search and replace it with the data view id + // we should remove the saved search and replace it with the index pattern id if (currentSavedSearch !== null) { setCurrentSavedSearch(null); } @@ -318,15 +227,58 @@ export const IndexDataVisualizerView: FC = (dataVi }); }; - const [overallStats, setOverallStats] = useState(defaults.overallStats); + const input: DataVisualizerGridInput = useMemo(() => { + return { + indexPattern: currentIndexPattern, + savedSearch: currentSavedSearch, + visibleFieldNames, + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentIndexPattern.id, currentSavedSearch?.id, visibleFieldNames]); + + const { + configs, + searchQueryLanguage, + searchString, + overallStats, + searchQuery, + documentCountStats, + metricsStats, + timefilter, + setLastRefresh, + progress, + } = useDataVisualizerGridData(input, dataVisualizerListState, setGlobalState); + + useEffect(() => { + return () => { + // When navigating away from the index pattern + // Reset all previously set filters + // to make sure new page doesn't have unrelated filters + data.query.filterManager.removeAll(); + }; + }, [currentIndexPattern.id, data.query.filterManager]); + + useEffect(() => { + // Force refresh on index pattern change + setLastRefresh(Date.now()); + }, [currentIndexPattern.id, setLastRefresh]); - const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); - const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); - const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); - const [metricsStats, setMetricsStats] = useState(); + useEffect(() => { + if (globalState?.time !== undefined) { + timefilter.setTime({ + from: globalState.time.from, + to: globalState.time.to, + }); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [JSON.stringify(globalState?.time), timefilter]); - const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); - const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); + useEffect(() => { + if (globalState?.refreshInterval !== undefined) { + timefilter.setRefreshInterval(globalState.refreshInterval); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [JSON.stringify(globalState?.refreshInterval), timefilter]); const onAddFilter = useCallback( (field: IndexPatternField | string, values: string, operation: '+' | '-') => { @@ -374,422 +326,8 @@ export const IndexDataVisualizerView: FC = (dataVi ] ); - useEffect(() => { - const timeUpdateSubscription = merge( - timefilter.getTimeUpdate$(), - dataVisualizerRefresh$ - ).subscribe(() => { - setGlobalState({ - time: timefilter.getTime(), - refreshInterval: timefilter.getRefreshInterval(), - }); - setLastRefresh(Date.now()); - }); - return () => { - timeUpdateSubscription.unsubscribe(); - }; - }); - - useEffect(() => { - loadOverallStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchQuery, samplerShardSize, lastRefresh]); - - useEffect(() => { - createMetricCards(); - createNonMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [overallStats, showEmptyFields]); - - useEffect(() => { - loadMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricConfigs]); - - useEffect(() => { - loadNonMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricConfigs]); - - useEffect(() => { - createMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricsLoaded]); - - useEffect(() => { - createNonMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricsLoaded]); - - async function loadOverallStats() { - const tf = timefilter as any; - let earliest; - let latest; - - const activeBounds = tf.getActiveBounds(); - - if (currentIndexPattern.timeFieldName !== undefined && activeBounds === undefined) { - return; - } - - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = activeBounds.min.valueOf(); - latest = activeBounds.max.valueOf(); - } - - try { - const allStats = await dataLoader.loadOverallData( - searchQuery, - samplerShardSize, - earliest, - latest - ); - // Because load overall stats perform queries in batches - // there could be multiple errors - if (Array.isArray(allStats.errors) && allStats.errors.length > 0) { - allStats.errors.forEach((err: any) => { - dataLoader.displayError(extractErrorProperties(err)); - }); - } - setOverallStats(allStats); - } catch (err) { - dataLoader.displayError(err.body ?? err); - } - } - - async function loadMetricFieldStats() { - // Only request data for fields that exist in documents. - if (metricConfigs.length === 0) { - return; - } - - const configsToLoad = metricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - // Obtain the interval to use for date histogram aggregations - // (such as the document count chart). Aim for 75 bars. - const buckets = getTimeBuckets(); - - const tf = timefilter as any; - let earliest: number | undefined; - let latest: number | undefined; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - const bounds = tf.getActiveBounds(); - const BAR_TARGET = 75; - buckets.setInterval('auto'); - buckets.setBounds(bounds); - buckets.setBarTarget(BAR_TARGET); - const aggInterval = buckets.getInterval(); - - try { - const metricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existMetricFields, - aggInterval.asMilliseconds() - ); - - // Add the metric stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - metricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - configWithStats.loading = false; - configs.push(configWithStats); - } else { - // Document count card. - configWithStats.stats = metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === undefined - ); - - if (configWithStats.stats !== undefined) { - // Add earliest / latest of timefilter for setting x axis domain. - configWithStats.stats.timeRangeEarliest = earliest; - configWithStats.stats.timeRangeLatest = latest; - } - setDocumentCountStats(configWithStats); - } - }); - - setMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - async function loadNonMetricFieldStats() { - // Only request data for fields that exist in documents. - if (nonMetricConfigs.length === 0) { - return; - } - - const configsToLoad = nonMetricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existNonMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - const tf = timefilter as any; - let earliest; - let latest; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - try { - const nonMetricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existNonMetricFields - ); - - // Add the field stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - nonMetricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...nonMetricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - } - configWithStats.loading = false; - configs.push(configWithStats); - }); - - setNonMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - const createMetricCards = useCallback(() => { - const configs: FieldVisConfig[] = []; - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; - - const allMetricFields = indexPatternFields.filter((f) => { - return ( - f.type === KBN_FIELD_TYPES.NUMBER && - f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true - ); - }); - const metricExistsFields = allMetricFields.filter((f) => { - return aggregatableExistsFields.find((existsF) => { - return existsF.fieldName === f.spec.name; - }); - }); - - // Add a config for 'document count', identified by no field name if indexpattern is time based. - if (currentIndexPattern.timeFieldName !== undefined) { - configs.push({ - type: JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - loading: true, - aggregatable: true, - }); - } - - if (metricsLoaded === false) { - setMetricsLoaded(true); - return; - } - - let aggregatableFields: any[] = overallStats.aggregatableExistsFields; - if (allMetricFields.length !== metricExistsFields.length && metricsLoaded === true) { - aggregatableFields = aggregatableFields.concat(overallStats.aggregatableNotExistsFields); - } - - const metricFieldsToShow = - metricsLoaded === true && showEmptyFields === true ? allMetricFields : metricExistsFields; - - metricFieldsToShow.forEach((field) => { - const fieldData = aggregatableFields.find((f) => { - return f.fieldName === field.spec.name; - }); - - const metricConfig: FieldVisConfig = { - ...(fieldData ? fieldData : {}), - fieldFormat: currentIndexPattern.getFormatterForField(field), - type: JOB_FIELD_TYPES.NUMBER, - loading: true, - aggregatable: true, - deletable: field.runtimeField !== undefined, - }; - if (field.displayName !== metricConfig.fieldName) { - metricConfig.displayName = field.displayName; - } - - configs.push(metricConfig); - }); - - setMetricsStats({ - totalMetricFieldsCount: allMetricFields.length, - visibleMetricsCount: metricFieldsToShow.length, - }); - setMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - metricsLoaded, - overallStats, - showEmptyFields, - ]); - - const createNonMetricCards = useCallback(() => { - const allNonMetricFields = indexPatternFields.filter((f) => { - return ( - f.type !== KBN_FIELD_TYPES.NUMBER && - f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true - ); - }); - // Obtain the list of all non-metric fields which appear in documents - // (aggregatable or not aggregatable). - const populatedNonMetricFields: any[] = []; // Kibana data view non metric fields. - let nonMetricFieldData: any[] = []; // Basic non metric field data loaded from requesting overall stats. - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; - const nonAggregatableExistsFields: any[] = overallStats.nonAggregatableExistsFields || []; - - allNonMetricFields.forEach((f) => { - const checkAggregatableField = aggregatableExistsFields.find( - (existsField) => existsField.fieldName === f.spec.name - ); - - if (checkAggregatableField !== undefined) { - populatedNonMetricFields.push(f); - nonMetricFieldData.push(checkAggregatableField); - } else { - const checkNonAggregatableField = nonAggregatableExistsFields.find( - (existsField) => existsField.fieldName === f.spec.name - ); - - if (checkNonAggregatableField !== undefined) { - populatedNonMetricFields.push(f); - nonMetricFieldData.push(checkNonAggregatableField); - } - } - }); - - if (nonMetricsLoaded === false) { - setNonMetricsLoaded(true); - return; - } - - if (allNonMetricFields.length !== nonMetricFieldData.length && showEmptyFields === true) { - // Combine the field data obtained from Elasticsearch into a single array. - nonMetricFieldData = nonMetricFieldData.concat( - overallStats.aggregatableNotExistsFields, - overallStats.nonAggregatableNotExistsFields - ); - } - - const nonMetricFieldsToShow = showEmptyFields ? allNonMetricFields : populatedNonMetricFields; - - const configs: FieldVisConfig[] = []; - - nonMetricFieldsToShow.forEach((field) => { - const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); - - const nonMetricConfig = { - ...(fieldData ? fieldData : {}), - fieldFormat: currentIndexPattern.getFormatterForField(field), - aggregatable: field.aggregatable, - scripted: field.scripted, - loading: fieldData?.existsInDocs, - deletable: field.runtimeField !== undefined, - }; - - // Map the field type from the Kibana data view to the field type - // used in the data visualizer. - const dataVisualizerType = kbnTypeToJobType(field); - if (dataVisualizerType !== undefined) { - nonMetricConfig.type = dataVisualizerType; - } else { - // Add a flag to indicate that this is one of the 'other' Kibana - // field types that do not yet have a specific card type. - nonMetricConfig.type = field.type; - nonMetricConfig.isUnsupportedType = true; - } - - if (field.displayName !== nonMetricConfig.fieldName) { - nonMetricConfig.displayName = field.displayName; - } - - configs.push(nonMetricConfig); - }); - - setNonMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - nonMetricsLoaded, - overallStats, - showEmptyFields, - ]); - const wizardPanelWidth = '280px'; - const configs = useMemo(() => { - let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; - if (visibleFieldTypes && visibleFieldTypes.length > 0) { - combinedConfigs = combinedConfigs.filter( - (config) => visibleFieldTypes.findIndex((field) => field === config.type) > -1 - ); - } - if (visibleFieldNames && visibleFieldNames.length > 0) { - combinedConfigs = combinedConfigs.filter( - (config) => visibleFieldNames.findIndex((field) => field === config.fieldName) > -1 - ); - } - - return combinedConfigs; - }, [nonMetricConfigs, metricConfigs, visibleFieldTypes, visibleFieldNames]); - const fieldsCountStats: TotalFieldsStats | undefined = useMemo(() => { let _visibleFieldsCount = 0; let _totalFieldsCount = 0; @@ -923,7 +461,7 @@ export const IndexDataVisualizerView: FC = (dataVi {overallStats?.totalCount !== undefined && ( @@ -953,12 +491,14 @@ export const IndexDataVisualizerView: FC = (dataVi metricsStats={metricsStats} /> + items={configs} pageState={dataVisualizerListState} updatePageState={setDataVisualizerListState} getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} extendedColumns={extendedColumns} + loading={progress < 100} /> diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/field_type_filter.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/field_type_filter.tsx index 7e86425c0a891..ee54683b08435 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/field_type_filter.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/field_type_filter.tsx @@ -29,7 +29,7 @@ export const DataVisualizerFieldTypeFilter: FC<{ {label} {indexedFieldName && ( - + )} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx index f55114ca36d78..25ed13121fc34 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx @@ -22,6 +22,7 @@ import { SearchQueryLanguage } from '../../types/combined_query'; import { useDataVisualizerKibana } from '../../../kibana_context'; import './_index.scss'; import { createMergedEsQuery } from '../../utils/saved_search_utils'; +import { OverallStats } from '../../types/overall_stats'; interface Props { indexPattern: IndexPattern; searchString: Query['query']; @@ -29,7 +30,7 @@ interface Props { searchQueryLanguage: SearchQueryLanguage; samplerShardSize: number; setSamplerShardSize(s: number): void; - overallStats: any; + overallStats: OverallStats; indexedFieldTypes: JobFieldType[]; setVisibleFieldTypes(q: string[]): void; visibleFieldTypes: string[]; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts deleted file mode 100644 index e0a2852a57b29..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// Maximum number of examples to obtain for text type fields. -import { CoreSetup } from 'kibana/public'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { i18n } from '@kbn/i18n'; -import { IndexPattern } from '../../../../../../../src/plugins/data/common'; -import { NON_AGGREGATABLE_FIELD_TYPES, OMIT_FIELDS } from '../../../../common/constants'; -import { FieldRequestConfig } from '../../../../common/types'; -import { getVisualizerFieldStats, getVisualizerOverallStats } from '../services/visualizer_stats'; - -type IndexPatternTitle = string; -type SavedSearchQuery = Record | null | undefined; - -const MAX_EXAMPLES_DEFAULT: number = 10; - -export class DataLoader { - private _indexPattern: IndexPattern; - private _runtimeMappings: estypes.MappingRuntimeFields; - private _indexPatternTitle: IndexPatternTitle = ''; - private _maxExamples: number = MAX_EXAMPLES_DEFAULT; - private _toastNotifications: CoreSetup['notifications']['toasts']; - - constructor( - indexPattern: IndexPattern, - toastNotifications: CoreSetup['notifications']['toasts'] - ) { - this._indexPattern = indexPattern; - this._runtimeMappings = this._indexPattern.getComputedFields() - .runtimeFields as estypes.MappingRuntimeFields; - this._indexPatternTitle = indexPattern.title; - this._toastNotifications = toastNotifications; - } - - async loadOverallData( - query: string | SavedSearchQuery, - samplerShardSize: number, - earliest: number | undefined, - latest: number | undefined - ): Promise { - const aggregatableFields: string[] = []; - const nonAggregatableFields: string[] = []; - this._indexPattern.fields.forEach((field) => { - const fieldName = field.displayName !== undefined ? field.displayName : field.name; - if (this.isDisplayField(fieldName) === true) { - if (field.aggregatable === true && !NON_AGGREGATABLE_FIELD_TYPES.has(field.type)) { - aggregatableFields.push(field.name); - } else { - nonAggregatableFields.push(field.name); - } - } - }); - - // Need to find: - // 1. List of aggregatable fields that do exist in docs - // 2. List of aggregatable fields that do not exist in docs - // 3. List of non-aggregatable fields that do exist in docs. - // 4. List of non-aggregatable fields that do not exist in docs. - const stats = await getVisualizerOverallStats({ - indexPatternTitle: this._indexPatternTitle, - query, - timeFieldName: this._indexPattern.timeFieldName, - samplerShardSize, - earliest, - latest, - aggregatableFields, - nonAggregatableFields, - runtimeMappings: this._runtimeMappings, - }); - - return stats; - } - - async loadFieldStats( - query: string | SavedSearchQuery, - samplerShardSize: number, - earliest: number | undefined, - latest: number | undefined, - fields: FieldRequestConfig[], - interval?: number - ): Promise { - const stats = await getVisualizerFieldStats({ - indexPatternTitle: this._indexPatternTitle, - query, - timeFieldName: this._indexPattern.timeFieldName, - earliest, - latest, - samplerShardSize, - interval, - fields, - maxExamples: this._maxExamples, - runtimeMappings: this._runtimeMappings, - }); - - return stats; - } - - displayError(err: any) { - if (err.statusCode === 500) { - this._toastNotifications.addError(err, { - title: i18n.translate('xpack.dataVisualizer.index.dataLoader.internalServerErrorMessage', { - defaultMessage: - 'Error loading data in index {index}. {message}. ' + - 'The request may have timed out. Try using a smaller sample size or narrowing the time range.', - values: { - index: this._indexPattern.title, - message: err.error ?? err.message, - }, - }), - }); - } else { - this._toastNotifications.addError(err, { - title: i18n.translate('xpack.dataVisualizer.index.errorLoadingDataMessage', { - defaultMessage: 'Error loading data in index {index}. {message}.', - values: { - index: this._indexPattern.title, - message: err.error ?? err.message, - }, - }), - }); - } - } - - public set maxExamples(max: number) { - this._maxExamples = max; - } - - public get maxExamples(): number { - return this._maxExamples; - } - - // Returns whether the field with the specified name should be displayed, - // as certain fields such as _id and _source should be omitted from the view. - public isDisplayField(fieldName: string): boolean { - return !OMIT_FIELDS.includes(fieldName); - } -} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index f59225b1c019f..0391d5ae5d5d5 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -8,7 +8,7 @@ import { Observable, Subject } from 'rxjs'; import { CoreStart } from 'kibana/public'; import ReactDOM from 'react-dom'; -import React, { Suspense, useCallback, useState } from 'react'; +import React, { Suspense, useCallback, useEffect, useState } from 'react'; import useObservable from 'react-use/lib/useObservable'; import { EuiEmptyPrompt, EuiIcon, EuiSpacer, EuiText } from '@elastic/eui'; import { Filter } from '@kbn/es-query'; @@ -36,15 +36,15 @@ import { } from '../../../common/components/stats_table'; import { FieldVisConfig } from '../../../common/components/stats_table/types'; import { getDefaultDataVisualizerListState } from '../../components/index_data_visualizer_view/index_data_visualizer_view'; -import { DataVisualizerTableState } from '../../../../../common'; +import { DataVisualizerTableState, SavedSearchSavedObject } from '../../../../../common'; import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; -import { useDataVisualizerGridData } from './use_data_visualizer_grid_data'; +import { useDataVisualizerGridData } from '../../hooks/use_data_visualizer_grid_data'; export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerStartDependencies]; -export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { +export interface DataVisualizerGridInput { indexPattern: IndexPattern; - savedSearch?: SavedSearch; + savedSearch?: SavedSearch | SavedSearchSavedObject | null; query?: Query; visibleFieldNames?: string[]; filters?: Filter[]; @@ -54,6 +54,7 @@ export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { */ onAddFilter?: (field: IndexPatternField | string, value: string, type: '+' | '-') => void; } +export type DataVisualizerGridEmbeddableInput = EmbeddableInput & DataVisualizerGridInput; export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; export type IDataVisualizerGridEmbeddable = typeof DataVisualizerGridEmbeddable; @@ -79,8 +80,13 @@ export const EmbeddableWrapper = ({ }, [dataVisualizerListState, onOutputChange] ); - const { configs, searchQueryLanguage, searchString, extendedColumns, loaded } = + const { configs, searchQueryLanguage, searchString, extendedColumns, progress, setLastRefresh } = useDataVisualizerGridData(input, dataVisualizerListState); + + useEffect(() => { + setLastRefresh(Date.now()); + }, [input?.lastReloadRequestTime, setLastRefresh]); + const getItemIdToExpandedRowMap = useCallback( function (itemIds: string[], items: FieldVisConfig[]): ItemIdToExpandedRowMap { return itemIds.reduce((m: ItemIdToExpandedRowMap, fieldName: string) => { @@ -101,13 +107,7 @@ export const EmbeddableWrapper = ({ [input, searchQueryLanguage, searchString] ); - if ( - loaded && - (configs.length === 0 || - // FIXME: Configs might have a placeholder document count stats field - // This will be removed in the future - (configs.length === 1 && configs[0].fieldName === undefined)) - ) { + if (progress === 100 && configs.length === 0) { return (
); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts similarity index 52% rename from x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts rename to x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index fc0fc7a2134b4..e6e7a96e0329f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -10,39 +10,54 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { merge } from 'rxjs'; import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; import { i18n } from '@kbn/i18n'; -import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; -import { useDataVisualizerKibana } from '../../../kibana_context'; -import { getEsQueryFromSavedSearch } from '../../utils/saved_search_utils'; -import { MetricFieldsStats } from '../../../common/components/stats_table/components/field_count_stats'; -import { DataLoader } from '../../data_loader/data_loader'; -import { useTimefilter } from '../../hooks/use_time_filter'; -import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; -import { TimeBuckets } from '../../services/time_buckets'; +import { DataVisualizerIndexBasedAppState } from '../types/index_data_visualizer_state'; +import { useDataVisualizerKibana } from '../../kibana_context'; +import { getEsQueryFromSavedSearch } from '../utils/saved_search_utils'; +import { MetricFieldsStats } from '../../common/components/stats_table/components/field_count_stats'; +import { useTimefilter } from './use_time_filter'; +import { dataVisualizerRefresh$ } from '../services/timefilter_refresh_service'; +import { TimeBuckets } from '../../../../common/services/time_buckets'; import { DataViewField, KBN_FIELD_TYPES, UI_SETTINGS, -} from '../../../../../../../../src/plugins/data/common'; -import { extractErrorProperties } from '../../utils/error_utils'; -import { FieldVisConfig } from '../../../common/components/stats_table/types'; -import { FieldRequestConfig, JOB_FIELD_TYPES } from '../../../../../common'; -import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; -import { getActions } from '../../../common/components/field_data_row/action_menu'; -import { DataVisualizerGridEmbeddableInput } from './grid_embeddable'; -import { getDefaultPageState } from '../../components/index_data_visualizer_view/index_data_visualizer_view'; +} from '../../../../../../../src/plugins/data/common'; +import { FieldVisConfig } from '../../common/components/stats_table/types'; +import { + FieldRequestConfig, + JOB_FIELD_TYPES, + JobFieldType, + NON_AGGREGATABLE_FIELD_TYPES, + OMIT_FIELDS, +} from '../../../../common'; +import { kbnTypeToJobType } from '../../common/util/field_types_utils'; +import { getActions } from '../../common/components/field_data_row/action_menu'; +import { DataVisualizerGridInput } from '../embeddables/grid_embeddable/grid_embeddable'; +import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; +import { useFieldStatsSearchStrategy } from './use_field_stats'; +import { useOverallStats } from './use_overall_stats'; +import { OverallStatsSearchStrategyParams } from '../../../../common/types/field_stats'; +import { Dictionary } from '../../common/util/url_state'; +import { AggregatableField, NonAggregatableField } from '../types/overall_stats'; const defaults = getDefaultPageState(); +function isDisplayField(fieldName: string): boolean { + return !OMIT_FIELDS.includes(fieldName); +} + export const useDataVisualizerGridData = ( - input: DataVisualizerGridEmbeddableInput, - dataVisualizerListState: Required + input: DataVisualizerGridInput, + dataVisualizerListState: Required, + onUpdate?: (params: Dictionary) => void ) => { const { services } = useDataVisualizerKibana(); - const { notifications, uiSettings } = services; - const { toasts } = notifications; + const { uiSettings, data } = services; const { samplerShardSize, visibleFieldTypes, showEmptyFields } = dataVisualizerListState; + const dataVisualizerListStateRef = useRef(dataVisualizerListState); const [lastRefresh, setLastRefresh] = useState(0); + const [searchSessionId, setSearchSessionId] = useState(); const { currentSavedSearch, @@ -61,6 +76,7 @@ export const useDataVisualizerGridData = ( [input] ); + /** Prepare required params to pass to search strategy **/ const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { const searchData = getEsQueryFromSavedSearch({ indexPattern: currentIndexPattern, @@ -68,9 +84,13 @@ export const useDataVisualizerGridData = ( savedSearch: currentSavedSearch, query: currentQuery, filters: currentFilters, + filterManager: data.query.filterManager, }); if (searchData === undefined || dataVisualizerListState.searchString !== '') { + if (dataVisualizerListState.filters) { + data.query.filterManager.setFilters(dataVisualizerListState.filters); + } return { searchQuery: dataVisualizerListState.searchQuery, searchString: dataVisualizerListState.searchString, @@ -85,16 +105,40 @@ export const useDataVisualizerGridData = ( } // eslint-disable-next-line react-hooks/exhaustive-deps }, [ - currentSavedSearch, - currentIndexPattern, - dataVisualizerListState, - currentQuery, - currentFilters, + currentSavedSearch?.id, + currentIndexPattern.id, + dataVisualizerListState.searchString, + dataVisualizerListState.searchQueryLanguage, + // eslint-disable-next-line react-hooks/exhaustive-deps + JSON.stringify({ + searchQuery: dataVisualizerListState.searchQuery, + currentQuery, + currentFilters, + }), + lastRefresh, ]); - const [overallStats, setOverallStats] = useState(defaults.overallStats); + useEffect(() => { + const currentSearchSessionId = data.search?.session?.getSessionId(); + if (currentSearchSessionId !== undefined) { + setSearchSessionId(currentSearchSessionId); + } + }, [data]); + + const _timeBuckets = useMemo(() => { + return new TimeBuckets({ + [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), + [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), + dateFormat: uiSettings.get('dateFormat'), + 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), + }); + }, [uiSettings]); + + const timefilter = useTimefilter({ + timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, + autoRefreshSelector: true, + }); - const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); const [metricsStats, setMetricsStats] = useState(); @@ -102,21 +146,134 @@ export const useDataVisualizerGridData = ( const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); - const dataLoader = useMemo( - () => new DataLoader(currentIndexPattern, toasts), - [currentIndexPattern, toasts] + /** Search strategy **/ + const fieldStatsRequest: OverallStatsSearchStrategyParams | undefined = useMemo( + () => { + // Obtain the interval to use for date histogram aggregations + // (such as the document count chart). Aim for 75 bars. + const buckets = _timeBuckets; + + const tf = timefilter; + + if (!buckets || !tf || !currentIndexPattern) return; + + const activeBounds = tf.getActiveBounds(); + + let earliest: number | undefined; + let latest: number | undefined; + if (activeBounds !== undefined && currentIndexPattern.timeFieldName !== undefined) { + earliest = activeBounds.min?.valueOf(); + latest = activeBounds.max?.valueOf(); + } + + const bounds = tf.getActiveBounds(); + const BAR_TARGET = 75; + buckets.setInterval('auto'); + + if (bounds) { + buckets.setBounds(bounds); + buckets.setBarTarget(BAR_TARGET); + } + + const aggInterval = buckets.getInterval(); + + const aggregatableFields: string[] = []; + const nonAggregatableFields: string[] = []; + currentIndexPattern.fields.forEach((field) => { + const fieldName = field.displayName !== undefined ? field.displayName : field.name; + if (!OMIT_FIELDS.includes(fieldName)) { + if (field.aggregatable === true && !NON_AGGREGATABLE_FIELD_TYPES.has(field.type)) { + aggregatableFields.push(field.name); + } else { + nonAggregatableFields.push(field.name); + } + } + }); + return { + earliest, + latest, + aggInterval, + intervalMs: aggInterval?.asMilliseconds(), + searchQuery, + samplerShardSize, + sessionId: searchSessionId, + index: currentIndexPattern.title, + timeFieldName: currentIndexPattern.timeFieldName, + runtimeFieldMap: currentIndexPattern.getComputedFields().runtimeFields, + aggregatableFields, + nonAggregatableFields, + }; + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [ + _timeBuckets, + timefilter, + currentIndexPattern.id, + // eslint-disable-next-line react-hooks/exhaustive-deps + JSON.stringify(searchQuery), + samplerShardSize, + searchSessionId, + lastRefresh, + ] + ); + + const { overallStats, progress: overallStatsProgress } = useOverallStats( + fieldStatsRequest, + lastRefresh ); - const timefilter = useTimefilter({ - timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, - autoRefreshSelector: true, - }); + const configsWithoutStats = useMemo(() => { + if (overallStatsProgress.loaded < 100) return; + const existMetricFields = metricConfigs + .map((config) => { + if (config.existsInDocs === false) return; + return { + fieldName: config.fieldName, + type: config.type, + cardinality: config.stats?.cardinality ?? 0, + }; + }) + .filter((c) => c !== undefined) as FieldRequestConfig[]; + + // Pass the field name, type and cardinality in the request. + // Top values will be obtained on a sample if cardinality > 100000. + const existNonMetricFields: FieldRequestConfig[] = nonMetricConfigs + .map((config) => { + if (config.existsInDocs === false) return; + return { + fieldName: config.fieldName, + type: config.type, + cardinality: config.stats?.cardinality ?? 0, + }; + }) + .filter((c) => c !== undefined) as FieldRequestConfig[]; + + return { metricConfigs: existMetricFields, nonMetricConfigs: existNonMetricFields }; + }, [metricConfigs, nonMetricConfigs, overallStatsProgress.loaded]); + + const strategyResponse = useFieldStatsSearchStrategy( + fieldStatsRequest, + configsWithoutStats, + dataVisualizerListStateRef.current + ); + + const combinedProgress = useMemo( + () => overallStatsProgress.loaded * 0.2 + strategyResponse.progress.loaded * 0.8, + [overallStatsProgress.loaded, strategyResponse.progress.loaded] + ); useEffect(() => { const timeUpdateSubscription = merge( timefilter.getTimeUpdate$(), + timefilter.getAutoRefreshFetch$(), dataVisualizerRefresh$ ).subscribe(() => { + if (onUpdate) { + onUpdate({ + time: timefilter.getTime(), + refreshInterval: timefilter.getRefreshInterval(), + }); + } setLastRefresh(Date.now()); }); return () => { @@ -124,65 +281,21 @@ export const useDataVisualizerGridData = ( }; }); - const getTimeBuckets = useCallback(() => { - return new TimeBuckets({ - [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), - [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), - dateFormat: uiSettings.get('dateFormat'), - 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), - }); - }, [uiSettings]); - const indexPatternFields: DataViewField[] = useMemo( () => currentIndexPattern.fields, [currentIndexPattern] ); - async function loadOverallStats() { - const tf = timefilter as any; - let earliest; - let latest; - - const activeBounds = tf.getActiveBounds(); - - if (currentIndexPattern.timeFieldName !== undefined && activeBounds === undefined) { - return; - } - - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = activeBounds.min.valueOf(); - latest = activeBounds.max.valueOf(); - } - - try { - const allStats = await dataLoader.loadOverallData( - searchQuery, - samplerShardSize, - earliest, - latest - ); - // Because load overall stats perform queries in batches - // there could be multiple errors - if (Array.isArray(allStats.errors) && allStats.errors.length > 0) { - allStats.errors.forEach((err: any) => { - dataLoader.displayError(extractErrorProperties(err)); - }); - } - setOverallStats(allStats); - } catch (err) { - dataLoader.displayError(err.body ?? err); - } - } - const createMetricCards = useCallback(() => { const configs: FieldVisConfig[] = []; - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; + const aggregatableExistsFields: AggregatableField[] = + overallStats.aggregatableExistsFields || []; const allMetricFields = indexPatternFields.filter((f) => { return ( f.type === KBN_FIELD_TYPES.NUMBER && f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true + isDisplayField(f.displayName) === true ); }); const metricExistsFields = allMetricFields.filter((f) => { @@ -191,22 +304,12 @@ export const useDataVisualizerGridData = ( }); }); - // Add a config for 'document count', identified by no field name if indexpattern is time based. - if (currentIndexPattern.timeFieldName !== undefined) { - configs.push({ - type: JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - loading: true, - aggregatable: true, - }); - } - if (metricsLoaded === false) { setMetricsLoaded(true); return; } - let aggregatableFields: any[] = overallStats.aggregatableExistsFields; + let aggregatableFields: AggregatableField[] = overallStats.aggregatableExistsFields; if (allMetricFields.length !== metricExistsFields.length && metricsLoaded === true) { aggregatableFields = aggregatableFields.concat(overallStats.aggregatableNotExistsFields); } @@ -218,9 +321,10 @@ export const useDataVisualizerGridData = ( const fieldData = aggregatableFields.find((f) => { return f.fieldName === field.spec.name; }); + if (!fieldData) return; const metricConfig: FieldVisConfig = { - ...(fieldData ? fieldData : {}), + ...fieldData, fieldFormat: currentIndexPattern.getFormatterForField(field), type: JOB_FIELD_TYPES.NUMBER, loading: true, @@ -239,29 +343,24 @@ export const useDataVisualizerGridData = ( visibleMetricsCount: metricFieldsToShow.length, }); setMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - metricsLoaded, - overallStats, - showEmptyFields, - ]); + }, [currentIndexPattern, indexPatternFields, metricsLoaded, overallStats, showEmptyFields]); const createNonMetricCards = useCallback(() => { const allNonMetricFields = indexPatternFields.filter((f) => { return ( f.type !== KBN_FIELD_TYPES.NUMBER && f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true + isDisplayField(f.displayName) === true ); }); // Obtain the list of all non-metric fields which appear in documents // (aggregatable or not aggregatable). - const populatedNonMetricFields: any[] = []; // Kibana index pattern non metric fields. - let nonMetricFieldData: any[] = []; // Basic non metric field data loaded from requesting overall stats. - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; - const nonAggregatableExistsFields: any[] = overallStats.nonAggregatableExistsFields || []; + const populatedNonMetricFields: DataViewField[] = []; // Kibana index pattern non metric fields. + let nonMetricFieldData: Array = []; // Basic non metric field data loaded from requesting overall stats. + const aggregatableExistsFields: AggregatableField[] = + overallStats.aggregatableExistsFields || []; + const nonAggregatableExistsFields: NonAggregatableField[] = + overallStats.nonAggregatableExistsFields || []; allNonMetricFields.forEach((f) => { const checkAggregatableField = aggregatableExistsFields.find( @@ -303,12 +402,11 @@ export const useDataVisualizerGridData = ( nonMetricFieldsToShow.forEach((field) => { const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); - const nonMetricConfig = { + const nonMetricConfig: Partial = { ...(fieldData ? fieldData : {}), fieldFormat: currentIndexPattern.getFormatterForField(field), aggregatable: field.aggregatable, - scripted: field.scripted, - loading: fieldData?.existsInDocs, + loading: fieldData?.existsInDocs ?? true, deletable: field.runtimeField !== undefined, }; @@ -320,7 +418,7 @@ export const useDataVisualizerGridData = ( } else { // Add a flag to indicate that this is one of the 'other' Kibana // field types that do not yet have a specific card type. - nonMetricConfig.type = field.type; + nonMetricConfig.type = field.type as JobFieldType; nonMetricConfig.isUnsupportedType = true; } @@ -328,171 +426,11 @@ export const useDataVisualizerGridData = ( nonMetricConfig.displayName = field.displayName; } - configs.push(nonMetricConfig); + configs.push(nonMetricConfig as FieldVisConfig); }); setNonMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - nonMetricsLoaded, - overallStats, - showEmptyFields, - ]); - - async function loadMetricFieldStats() { - // Only request data for fields that exist in documents. - if (metricConfigs.length === 0) { - return; - } - - const configsToLoad = metricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - // Obtain the interval to use for date histogram aggregations - // (such as the document count chart). Aim for 75 bars. - const buckets = getTimeBuckets(); - - const tf = timefilter as any; - let earliest: number | undefined; - let latest: number | undefined; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - const bounds = tf.getActiveBounds(); - const BAR_TARGET = 75; - buckets.setInterval('auto'); - buckets.setBounds(bounds); - buckets.setBarTarget(BAR_TARGET); - const aggInterval = buckets.getInterval(); - - try { - const metricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existMetricFields, - aggInterval.asMilliseconds() - ); - - // Add the metric stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - metricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - configWithStats.loading = false; - configs.push(configWithStats); - } else { - // Document count card. - configWithStats.stats = metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === undefined - ); - - if (configWithStats.stats !== undefined) { - // Add earliest / latest of timefilter for setting x axis domain. - configWithStats.stats.timeRangeEarliest = earliest; - configWithStats.stats.timeRangeLatest = latest; - } - setDocumentCountStats(configWithStats); - } - }); - - setMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - async function loadNonMetricFieldStats() { - // Only request data for fields that exist in documents. - if (nonMetricConfigs.length === 0) { - return; - } - - const configsToLoad = nonMetricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existNonMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - const tf = timefilter as any; - let earliest; - let latest; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - try { - const nonMetricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existNonMetricFields - ); - - // Add the field stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - nonMetricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...nonMetricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - } - configWithStats.loading = false; - configs.push(configWithStats); - }); - - setNonMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - useEffect(() => { - loadOverallStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchQuery, samplerShardSize, lastRefresh]); + }, [currentIndexPattern, indexPatternFields, nonMetricsLoaded, overallStats, showEmptyFields]); useEffect(() => { createMetricCards(); @@ -500,27 +438,8 @@ export const useDataVisualizerGridData = ( // eslint-disable-next-line react-hooks/exhaustive-deps }, [overallStats, showEmptyFields]); - useEffect(() => { - loadMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricConfigs]); - - useEffect(() => { - loadNonMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricConfigs]); - - useEffect(() => { - createMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricsLoaded]); - - useEffect(() => { - createNonMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricsLoaded]); - const configs = useMemo(() => { + const fieldStats = strategyResponse.fieldStats; let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; if (visibleFieldTypes && visibleFieldTypes.length > 0) { combinedConfigs = combinedConfigs.filter( @@ -533,8 +452,27 @@ export const useDataVisualizerGridData = ( ); } + if (fieldStats) { + combinedConfigs = combinedConfigs.map((c) => { + const loadedFullStats = fieldStats.get(c.fieldName) ?? {}; + return loadedFullStats + ? { + ...c, + loading: false, + stats: { ...c.stats, ...loadedFullStats }, + } + : c; + }); + } + return combinedConfigs; - }, [nonMetricConfigs, metricConfigs, visibleFieldTypes, visibleFieldNames]); + }, [ + nonMetricConfigs, + metricConfigs, + visibleFieldTypes, + visibleFieldNames, + strategyResponse.fieldStats, + ]); // Some actions open up fly-out or popup // This variable is used to keep track of them and clean up when unmounting @@ -575,13 +513,16 @@ export const useDataVisualizerGridData = ( }, [input.indexPattern, services, searchQueryLanguage, searchString]); return { + progress: combinedProgress, configs, searchQueryLanguage, searchString, searchQuery, extendedColumns, - documentCountStats, + documentCountStats: overallStats.documentCountStats, metricsStats, - loaded: metricsLoaded && nonMetricsLoaded, + overallStats, + timefilter, + setLastRefresh, }; }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts new file mode 100644 index 0000000000000..64654d56db05b --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -0,0 +1,287 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useEffect, useReducer, useRef, useState } from 'react'; +import { combineLatest, Observable, Subject, Subscription } from 'rxjs'; +import { i18n } from '@kbn/i18n'; +import { last, cloneDeep } from 'lodash'; +import { switchMap } from 'rxjs/operators'; +import type { + DataStatsFetchProgress, + FieldStatsSearchStrategyReturnBase, + OverallStatsSearchStrategyParams, + FieldStatsCommonRequestParams, + Field, +} from '../../../../common/types/field_stats'; +import { useDataVisualizerKibana } from '../../kibana_context'; +import type { FieldRequestConfig } from '../../../../common'; +import type { DataVisualizerIndexBasedAppState } from '../types/index_data_visualizer_state'; +import { + buildBaseFilterCriteria, + getSafeAggregationName, +} from '../../../../common/utils/query_utils'; +import type { FieldStats, FieldStatsError } from '../../../../common/types/field_stats'; +import { getInitialProgress, getReducer } from '../progress_utils'; +import { MAX_EXAMPLES_DEFAULT } from '../search_strategy/requests/constants'; +import type { ISearchOptions } from '../../../../../../../src/plugins/data/common'; +import { getFieldsStats } from '../search_strategy/requests/get_fields_stats'; +interface FieldStatsParams { + metricConfigs: FieldRequestConfig[]; + nonMetricConfigs: FieldRequestConfig[]; +} + +const createBatchedRequests = (fields: Field[], maxBatchSize = 10) => { + // Batch up fields by type, getting stats for multiple fields at a time. + const batches: Field[][] = []; + const batchedFields: { [key: string]: Field[][] } = {}; + + fields.forEach((field) => { + const fieldType = field.type; + if (batchedFields[fieldType] === undefined) { + batchedFields[fieldType] = [[]]; + } + let lastArray: Field[] = last(batchedFields[fieldType]) as Field[]; + if (lastArray.length === maxBatchSize) { + lastArray = []; + batchedFields[fieldType].push(lastArray); + } + lastArray.push(field); + }); + + Object.values(batchedFields).forEach((lists) => { + batches.push(...lists); + }); + return batches; +}; + +export function useFieldStatsSearchStrategy( + searchStrategyParams: OverallStatsSearchStrategyParams | undefined, + fieldStatsParams: FieldStatsParams | undefined, + initialDataVisualizerListState: DataVisualizerIndexBasedAppState +): FieldStatsSearchStrategyReturnBase { + const { + services: { + data, + notifications: { toasts }, + }, + } = useDataVisualizerKibana(); + + const [fieldStats, setFieldStats] = useState>(); + const [fetchState, setFetchState] = useReducer( + getReducer(), + getInitialProgress() + ); + + const abortCtrl = useRef(new AbortController()); + const searchSubscription$ = useRef(); + const retries$ = useRef(); + + const startFetch = useCallback(() => { + searchSubscription$.current?.unsubscribe(); + retries$.current?.unsubscribe(); + + abortCtrl.current.abort(); + abortCtrl.current = new AbortController(); + setFetchState({ + ...getInitialProgress(), + error: undefined, + }); + setFieldStats(undefined); + + if ( + !searchStrategyParams || + !fieldStatsParams || + (fieldStatsParams.metricConfigs.length === 0 && + fieldStatsParams.nonMetricConfigs.length === 0) + ) { + setFetchState({ + loaded: 100, + isRunning: false, + }); + + return; + } + + const { sortField, sortDirection } = initialDataVisualizerListState; + /** + * Sort the list of fields by the initial sort field and sort direction + * Then divide into chunks by the initial page size + */ + + let sortedConfigs = [...fieldStatsParams.metricConfigs, ...fieldStatsParams.nonMetricConfigs]; + + if (sortField === 'fieldName' || sortField === 'type') { + sortedConfigs = sortedConfigs.sort((a, b) => a[sortField].localeCompare(b[sortField])); + } + if (sortDirection === 'desc') { + sortedConfigs = sortedConfigs.reverse(); + } + + const filterCriteria = buildBaseFilterCriteria( + searchStrategyParams.timeFieldName, + searchStrategyParams.earliest, + searchStrategyParams.latest, + searchStrategyParams.searchQuery + ); + + const params: FieldStatsCommonRequestParams = { + index: searchStrategyParams.index, + samplerShardSize: searchStrategyParams.samplerShardSize, + timeFieldName: searchStrategyParams.timeFieldName, + earliestMs: searchStrategyParams.earliest, + latestMs: searchStrategyParams.latest, + runtimeFieldMap: searchStrategyParams.runtimeFieldMap, + intervalMs: searchStrategyParams.intervalMs, + query: { + bool: { + filter: filterCriteria, + }, + }, + maxExamples: MAX_EXAMPLES_DEFAULT, + }; + const searchOptions: ISearchOptions = { + abortSignal: abortCtrl.current.signal, + sessionId: searchStrategyParams?.sessionId, + }; + + const batches = createBatchedRequests( + sortedConfigs.map((config, idx) => ({ + fieldName: config.fieldName, + type: config.type, + cardinality: config.cardinality, + safeFieldName: getSafeAggregationName(config.fieldName, idx), + })), + 10 + ); + + const statsMap$ = new Subject(); + const fieldsToRetry$ = new Subject(); + + const fieldStatsSub = combineLatest( + batches + .map((batch) => getFieldsStats(data.search, params, batch, searchOptions)) + .filter((obs) => obs !== undefined) as Array> + ); + const onError = (error: any) => { + toasts.addError(error, { + title: i18n.translate('xpack.dataVisualizer.index.errorFetchingFieldStatisticsMessage', { + defaultMessage: 'Error fetching field statistics', + }), + }); + setFetchState({ + isRunning: false, + error, + }); + }; + + const onComplete = () => { + setFetchState({ + isRunning: false, + }); + }; + + // First, attempt to fetch field stats in batches of 10 + searchSubscription$.current = fieldStatsSub.subscribe({ + next: (resp) => { + if (resp) { + const statsMap = new Map(); + const failedFields: Field[] = []; + resp.forEach((batchResponse) => { + if (Array.isArray(batchResponse)) { + batchResponse.forEach((f) => { + if (f.fieldName !== undefined) { + statsMap.set(f.fieldName, f); + } + }); + } else { + // If an error occurred during batch + // retry each field in the failed batch individually + failedFields.push(...(batchResponse.fields ?? [])); + } + }); + + setFetchState({ + loaded: (statsMap.size / sortedConfigs.length) * 100, + isRunning: true, + }); + + setFieldStats(statsMap); + + if (failedFields.length > 0) { + statsMap$.next(statsMap); + fieldsToRetry$.next(failedFields); + } + } + }, + error: onError, + complete: onComplete, + }); + + // If any of batches failed, retry each of the failed field at least one time individually + retries$.current = combineLatest([ + statsMap$, + fieldsToRetry$.pipe( + switchMap((failedFields) => { + return combineLatest( + failedFields + .map((failedField) => + getFieldsStats(data.search, params, [failedField], searchOptions) + ) + .filter((obs) => obs !== undefined) + ); + }) + ), + ]).subscribe({ + next: (resp) => { + const statsMap = cloneDeep(resp[0]) as Map; + const fieldBatches = resp[1]; + + if (Array.isArray(fieldBatches)) { + fieldBatches.forEach((f) => { + if (Array.isArray(f) && f.length === 1) { + statsMap.set(f[0].fieldName, f[0]); + } + }); + setFieldStats(statsMap); + setFetchState({ + loaded: (statsMap.size / sortedConfigs.length) * 100, + isRunning: true, + }); + } + }, + error: onError, + complete: onComplete, + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [data.search, toasts, fieldStatsParams, initialDataVisualizerListState]); + + const cancelFetch = useCallback(() => { + searchSubscription$.current?.unsubscribe(); + searchSubscription$.current = undefined; + + retries$.current?.unsubscribe(); + retries$.current = undefined; + + abortCtrl.current.abort(); + setFetchState({ + isRunning: false, + }); + }, []); + + // auto-update + useEffect(() => { + startFetch(); + return cancelFetch; + }, [startFetch, cancelFetch]); + + return { + progress: fetchState, + fieldStats, + startFetch, + cancelFetch, + }; +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts new file mode 100644 index 0000000000000..92a95bfacea42 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -0,0 +1,267 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useEffect, useState, useRef, useMemo, useReducer } from 'react'; +import { forkJoin, of, Subscription } from 'rxjs'; +import { switchMap } from 'rxjs/operators'; +import { i18n } from '@kbn/i18n'; +import type { ToastsStart } from 'kibana/public'; +import { chunk } from 'lodash'; +import { useDataVisualizerKibana } from '../../kibana_context'; +import { + AggregatableFieldOverallStats, + checkAggregatableFieldsExistRequest, + checkNonAggregatableFieldExistsRequest, + processAggregatableFieldsExistResponse, + processNonAggregatableFieldsExistResponse, +} from '../search_strategy/requests/overall_stats'; +import type { + IKibanaSearchRequest, + IKibanaSearchResponse, + ISearchOptions, +} from '../../../../../../../src/plugins/data/common'; +import type { OverallStats } from '../types/overall_stats'; +import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; +import { extractErrorProperties } from '../utils/error_utils'; +import type { + DataStatsFetchProgress, + OverallStatsSearchStrategyParams, +} from '../../../../common/types/field_stats'; +import { + getDocumentCountStatsRequest, + processDocumentCountStats, +} from '../search_strategy/requests/get_document_stats'; +import { getInitialProgress, getReducer } from '../progress_utils'; + +function displayError(toastNotifications: ToastsStart, indexPattern: string, err: any) { + if (err.statusCode === 500) { + toastNotifications.addError(err, { + title: i18n.translate('xpack.dataVisualizer.index.dataLoader.internalServerErrorMessage', { + defaultMessage: + 'Error loading data in index {index}. {message}. ' + + 'The request may have timed out. Try using a smaller sample size or narrowing the time range.', + values: { + index: indexPattern, + message: err.error ?? err.message, + }, + }), + }); + } else { + toastNotifications.addError(err, { + title: i18n.translate('xpack.dataVisualizer.index.errorLoadingDataMessage', { + defaultMessage: 'Error loading data in index {index}. {message}.', + values: { + index: indexPattern, + message: err.error ?? err.message, + }, + }), + }); + } +} + +export function useOverallStats( + searchStrategyParams: TParams | undefined, + lastRefresh: number +): { + progress: DataStatsFetchProgress; + overallStats: OverallStats; +} { + const { + services: { + data, + notifications: { toasts }, + }, + } = useDataVisualizerKibana(); + + const [stats, setOverallStats] = useState(getDefaultPageState().overallStats); + const [fetchState, setFetchState] = useReducer( + getReducer(), + getInitialProgress() + ); + + const abortCtrl = useRef(new AbortController()); + const searchSubscription$ = useRef(); + + const startFetch = useCallback(() => { + searchSubscription$.current?.unsubscribe(); + abortCtrl.current.abort(); + abortCtrl.current = new AbortController(); + + if (!searchStrategyParams || lastRefresh === 0) return; + + setFetchState({ + ...getInitialProgress(), + error: undefined, + }); + + const { + aggregatableFields, + nonAggregatableFields, + index, + searchQuery, + timeFieldName, + earliest, + latest, + intervalMs, + runtimeFieldMap, + samplerShardSize, + } = searchStrategyParams; + + const searchOptions: ISearchOptions = { + abortSignal: abortCtrl.current.signal, + sessionId: searchStrategyParams?.sessionId, + }; + const nonAggregatableOverallStats$ = + nonAggregatableFields.length > 0 + ? forkJoin( + nonAggregatableFields.map((fieldName: string) => + data.search + .search( + { + params: checkNonAggregatableFieldExistsRequest( + index, + searchQuery, + fieldName, + timeFieldName, + earliest, + latest, + runtimeFieldMap + ), + }, + searchOptions + ) + .pipe( + switchMap((resp) => { + return of({ + ...resp, + rawResponse: { ...resp.rawResponse, fieldName }, + } as IKibanaSearchResponse); + }) + ) + ) + ) + : of(undefined); + + // Have to divide into smaller requests to avoid 413 payload too large + const aggregatableFieldsChunks = chunk(aggregatableFields, 30); + + const aggregatableOverallStats$ = forkJoin( + aggregatableFields.length > 0 + ? aggregatableFieldsChunks.map((aggregatableFieldsChunk) => + data.search + .search( + { + params: checkAggregatableFieldsExistRequest( + index, + searchQuery, + aggregatableFieldsChunk, + samplerShardSize, + timeFieldName, + earliest, + latest, + undefined, + runtimeFieldMap + ), + }, + searchOptions + ) + .pipe( + switchMap((resp) => { + return of({ + ...resp, + aggregatableFields: aggregatableFieldsChunk, + } as AggregatableFieldOverallStats); + }) + ) + ) + : of(undefined) + ); + + const documentCountStats$ = + timeFieldName !== undefined && intervalMs !== undefined && intervalMs > 0 + ? data.search.search( + { + params: getDocumentCountStatsRequest(searchStrategyParams), + }, + searchOptions + ) + : of(undefined); + const sub = forkJoin({ + documentCountStatsResp: documentCountStats$, + nonAggregatableOverallStatsResp: nonAggregatableOverallStats$, + aggregatableOverallStatsResp: aggregatableOverallStats$, + }).pipe( + switchMap( + ({ + documentCountStatsResp, + nonAggregatableOverallStatsResp, + aggregatableOverallStatsResp, + }) => { + const aggregatableOverallStats = processAggregatableFieldsExistResponse( + aggregatableOverallStatsResp, + aggregatableFields, + samplerShardSize + ); + const nonAggregatableOverallStats = processNonAggregatableFieldsExistResponse( + nonAggregatableOverallStatsResp, + nonAggregatableFields + ); + + return of({ + documentCountStats: processDocumentCountStats( + documentCountStatsResp?.rawResponse, + searchStrategyParams + ), + ...nonAggregatableOverallStats, + ...aggregatableOverallStats, + }); + } + ) + ); + + searchSubscription$.current = sub.subscribe({ + next: (overallStats) => { + if (overallStats) { + setOverallStats(overallStats); + } + }, + error: (error) => { + displayError(toasts, searchStrategyParams.index, extractErrorProperties(error)); + setFetchState({ + isRunning: false, + error, + }); + }, + complete: () => { + setFetchState({ + loaded: 100, + isRunning: false, + }); + }, + }); + }, [data.search, searchStrategyParams, toasts, lastRefresh]); + + const cancelFetch = useCallback(() => { + searchSubscription$.current?.unsubscribe(); + searchSubscription$.current = undefined; + abortCtrl.current.abort(); + }, []); + + // auto-update + useEffect(() => { + startFetch(); + return cancelFetch; + }, [startFetch, cancelFetch]); + + return useMemo( + () => ({ + progress: fetchState, + overallStats: stats, + }), + [stats, fetchState] + ); +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/progress_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/progress_utils.ts new file mode 100644 index 0000000000000..f329fe47e75b0 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/progress_utils.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataStatsFetchProgress } from '../../../common/types/field_stats'; + +export const getInitialProgress = (): DataStatsFetchProgress => ({ + isRunning: false, + loaded: 0, + total: 100, +}); + +export const getReducer = + () => + (prev: T, update: Partial): T => ({ + ...prev, + ...update, + }); diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/constants.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/constants.ts similarity index 81% rename from x-pack/plugins/data_visualizer/server/models/data_visualizer/constants.ts rename to x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/constants.ts index 91bd394aee797..6da11fd850acc 100644 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/constants.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/constants.ts @@ -11,3 +11,7 @@ export const AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE = 200; export const FIELDS_REQUEST_BATCH_SIZE = 10; export const MAX_CHART_COLUMNS = 20; + +export const MAX_EXAMPLES_DEFAULT = 10; +export const MAX_PERCENT = 100; +export const PERCENTILE_SPACING = 5; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts new file mode 100644 index 0000000000000..b5359915ef63e --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { get } from 'lodash'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { Observable, of } from 'rxjs'; +import { catchError, map } from 'rxjs/operators'; +import { + buildSamplerAggregation, + getSamplerAggregationsResponsePath, +} from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { + Field, + BooleanFieldStats, + Aggs, + FieldStatsCommonRequestParams, +} from '../../../../../common/types/field_stats'; +import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; +import type { + IKibanaSearchRequest, + IKibanaSearchResponse, + ISearchOptions, + ISearchStart, +} from '../../../../../../../../src/plugins/data/public'; +import { extractErrorProperties } from '../../utils/error_utils'; + +export const getBooleanFieldsStatsRequest = ( + params: FieldStatsCommonRequestParams, + fields: Field[] +) => { + const { index, query, runtimeFieldMap, samplerShardSize } = params; + + const size = 0; + const aggs: Aggs = {}; + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; + aggs[`${safeFieldName}_value_count`] = { + filter: { exists: { field: field.fieldName } }, + }; + aggs[`${safeFieldName}_values`] = { + terms: { + field: field.fieldName, + size: 2, + }, + }; + }); + const searchBody = { + query, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchBooleanFieldsStats = ( + dataSearch: ISearchStart, + params: FieldStatsCommonRequestParams, + fields: Field[], + options: ISearchOptions +): Observable => { + const { samplerShardSize } = params; + const request: estypes.SearchRequest = getBooleanFieldsStatsRequest(params, fields); + return dataSearch + .search({ params: request }, options) + .pipe( + catchError((e) => + of({ + fields, + error: extractErrorProperties(e), + } as FieldStatsError) + ), + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; + + const aggregations = resp.rawResponse.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + + const batchStats: BooleanFieldStats[] = fields.map((field, i) => { + const safeFieldName = field.fieldName; + const stats: BooleanFieldStats = { + fieldName: field.fieldName, + count: get(aggregations, [...aggsPath, `${safeFieldName}_value_count`, 'doc_count'], 0), + trueCount: 0, + falseCount: 0, + }; + + const valueBuckets: Array<{ [key: string]: number }> = get( + aggregations, + [...aggsPath, `${safeFieldName}_values`, 'buckets'], + [] + ); + valueBuckets.forEach((bucket) => { + stats[`${bucket.key_as_string}Count`] = bucket.doc_count; + }); + return stats; + }); + + return batchStats; + }) + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts new file mode 100644 index 0000000000000..07bdc8c14301c --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { get } from 'lodash'; +import { Observable, of } from 'rxjs'; +import { catchError, map } from 'rxjs/operators'; +import { + buildSamplerAggregation, + getSamplerAggregationsResponsePath, +} from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats'; +import type { Field, DateFieldStats, Aggs } from '../../../../../common/types/field_stats'; +import type { + IKibanaSearchRequest, + IKibanaSearchResponse, + ISearchOptions, + ISearchStart, +} from '../../../../../../../../src/plugins/data/public'; +import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; +import { extractErrorProperties } from '../../utils/error_utils'; + +export const getDateFieldsStatsRequest = ( + params: FieldStatsCommonRequestParams, + fields: Field[] +) => { + const { index, query, runtimeFieldMap, samplerShardSize } = params; + + const size = 0; + + const aggs: Aggs = {}; + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; + aggs[`${safeFieldName}_field_stats`] = { + filter: { exists: { field: field.fieldName } }, + aggs: { + actual_stats: { + stats: { field: field.fieldName }, + }, + }, + }; + }); + + const searchBody = { + query, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchDateFieldsStats = ( + dataSearch: ISearchStart, + params: FieldStatsCommonRequestParams, + fields: Field[], + options: ISearchOptions +): Observable => { + const { samplerShardSize } = params; + + const request: estypes.SearchRequest = getDateFieldsStatsRequest(params, fields); + return dataSearch + .search({ params: request }, options) + .pipe( + catchError((e) => + of({ + fields, + error: extractErrorProperties(e), + } as FieldStatsError) + ), + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; + const aggregations = resp.rawResponse.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + + const batchStats: DateFieldStats[] = fields.map((field, i) => { + const safeFieldName = field.safeFieldName; + const docCount = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], + 0 + ); + const fieldStatsResp = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], + {} + ); + return { + fieldName: field.fieldName, + count: docCount, + earliest: get(fieldStatsResp, 'min', 0), + latest: get(fieldStatsResp, 'max', 0), + } as DateFieldStats; + }); + return batchStats; + }) + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts new file mode 100644 index 0000000000000..cdd69f5d3a369 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { each, get } from 'lodash'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { buildBaseFilterCriteria } from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { + DocumentCountStats, + OverallStatsSearchStrategyParams, +} from '../../../../../common/types/field_stats'; + +export const getDocumentCountStatsRequest = (params: OverallStatsSearchStrategyParams) => { + const { + index, + timeFieldName, + earliest: earliestMs, + latest: latestMs, + runtimeFieldMap, + searchQuery, + intervalMs, + } = params; + + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, searchQuery); + + // Don't use the sampler aggregation as this can lead to some potentially + // confusing date histogram results depending on the date range of data amongst shards. + + const aggs = { + eventRate: { + date_histogram: { + field: timeFieldName, + fixed_interval: `${intervalMs}ms`, + min_doc_count: 1, + }, + }, + }; + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs, + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + return { + index, + size, + body: searchBody, + }; +}; + +export const processDocumentCountStats = ( + body: estypes.SearchResponse | undefined, + params: OverallStatsSearchStrategyParams +): DocumentCountStats | undefined => { + if ( + !body || + params.intervalMs === undefined || + params.earliest === undefined || + params.latest === undefined + ) { + return undefined; + } + const buckets: { [key: string]: number } = {}; + const dataByTimeBucket: Array<{ key: string; doc_count: number }> = get( + body, + ['aggregations', 'eventRate', 'buckets'], + [] + ); + each(dataByTimeBucket, (dataForTime) => { + const time = dataForTime.key; + buckets[time] = dataForTime.doc_count; + }); + + return { + interval: params.intervalMs, + buckets, + timeRangeEarliest: params.earliest, + timeRangeLatest: params.latest, + }; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts new file mode 100644 index 0000000000000..618e47bb97e1d --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { get } from 'lodash'; +import { combineLatest, of } from 'rxjs'; +import { catchError, map } from 'rxjs/operators'; +import { buildBaseFilterCriteria } from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { + Field, + FieldExamples, + FieldStatsCommonRequestParams, +} from '../../../../../common/types/field_stats'; +import type { + IKibanaSearchRequest, + IKibanaSearchResponse, + ISearchOptions, + ISearchStart, +} from '../../../../../../../../src/plugins/data/public'; +import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; +import { extractErrorProperties } from '../../utils/error_utils'; +import { MAX_EXAMPLES_DEFAULT } from './constants'; + +export const getFieldExamplesRequest = (params: FieldStatsCommonRequestParams, field: Field) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, maxExamples } = + params; + + // Request at least 100 docs so that we have a chance of obtaining + // 'maxExamples' of the field. + const size = Math.max(100, maxExamples ?? MAX_EXAMPLES_DEFAULT); + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + // Use an exists filter to return examples of the field. + if (Array.isArray(filterCriteria)) { + filterCriteria.push({ + exists: { field: field.fieldName }, + }); + } + + const searchBody = { + fields: [field.fieldName], + _source: false, + query: { + bool: { + filter: filterCriteria, + }, + }, + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchFieldsExamples = ( + dataSearch: ISearchStart, + params: FieldStatsCommonRequestParams, + fields: Field[], + options: ISearchOptions +) => { + const { maxExamples } = params; + return combineLatest( + fields.map((field) => { + const request: estypes.SearchRequest = getFieldExamplesRequest(params, field); + + return dataSearch + .search({ params: request }, options) + .pipe( + catchError((e) => + of({ + fieldName: field.fieldName, + fields, + error: extractErrorProperties(e), + } as FieldStatsError) + ), + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; + const body = resp.rawResponse; + const stats = { + fieldName: field.fieldName, + examples: [] as unknown[], + } as FieldExamples; + + if (body.hits.total > 0) { + const hits = body.hits.hits; + for (let i = 0; i < hits.length; i++) { + // Use lodash get() to support field names containing dots. + const doc: object[] | undefined = get(hits[i].fields, field.fieldName); + // the results from fields query is always an array + if (Array.isArray(doc) && doc.length > 0) { + const example = doc[0]; + if (example !== undefined && stats.examples.indexOf(example) === -1) { + stats.examples.push(example); + if (stats.examples.length === maxExamples) { + break; + } + } + } + } + } + + return stats; + }) + ); + }) + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts new file mode 100644 index 0000000000000..aa19aa9fbb495 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Observable } from 'rxjs'; +import type { FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats'; +import type { FieldStatsError } from '../../../../../common/types/field_stats'; +import type { ISearchOptions } from '../../../../../../../../src/plugins/data/common'; +import { ISearchStart } from '../../../../../../../../src/plugins/data/public'; +import type { FieldStats } from '../../../../../common/types/field_stats'; +import { JOB_FIELD_TYPES } from '../../../../../common'; +import { fetchDateFieldsStats } from './get_date_field_stats'; +import { fetchBooleanFieldsStats } from './get_boolean_field_stats'; +import { fetchFieldsExamples } from './get_field_examples'; +import { fetchNumericFieldsStats } from './get_numeric_field_stats'; +import { fetchStringFieldsStats } from './get_string_field_stats'; + +export const getFieldsStats = ( + dataSearch: ISearchStart, + params: FieldStatsCommonRequestParams, + fields: Array<{ + fieldName: string; + type: string; + cardinality: number; + safeFieldName: string; + }>, + options: ISearchOptions +): Observable | undefined => { + const fieldType = fields[0].type; + switch (fieldType) { + case JOB_FIELD_TYPES.NUMBER: + return fetchNumericFieldsStats(dataSearch, params, fields, options); + case JOB_FIELD_TYPES.KEYWORD: + case JOB_FIELD_TYPES.IP: + return fetchStringFieldsStats(dataSearch, params, fields, options); + case JOB_FIELD_TYPES.DATE: + return fetchDateFieldsStats(dataSearch, params, fields, options); + case JOB_FIELD_TYPES.BOOLEAN: + return fetchBooleanFieldsStats(dataSearch, params, fields, options); + case JOB_FIELD_TYPES.TEXT: + return fetchFieldsExamples(dataSearch, params, fields, options); + default: + // Use an exists filter on the the field name to get + // examples of the field, so cannot batch up. + return fetchFieldsExamples(dataSearch, params, fields, options); + } +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts new file mode 100644 index 0000000000000..89ae7598b30fd --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts @@ -0,0 +1,207 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { find, get } from 'lodash'; +import { catchError, map } from 'rxjs/operators'; +import { Observable, of } from 'rxjs'; +import { AggregationsTermsAggregation } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { + MAX_PERCENT, + PERCENTILE_SPACING, + SAMPLER_TOP_TERMS_SHARD_SIZE, + SAMPLER_TOP_TERMS_THRESHOLD, +} from './constants'; +import { + buildSamplerAggregation, + getSamplerAggregationsResponsePath, +} from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { Aggs, FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats'; +import type { + Field, + NumericFieldStats, + Bucket, + FieldStatsError, +} from '../../../../../common/types/field_stats'; +import { processDistributionData } from '../../utils/process_distribution_data'; +import { + IKibanaSearchRequest, + IKibanaSearchResponse, + ISearchOptions, +} from '../../../../../../../../src/plugins/data/common'; +import type { ISearchStart } from '../../../../../../../../src/plugins/data/public'; +import { extractErrorProperties } from '../../utils/error_utils'; +import { isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; + +export const getNumericFieldsStatsRequest = ( + params: FieldStatsCommonRequestParams, + fields: Field[] +) => { + const { index, query, runtimeFieldMap, samplerShardSize } = params; + + const size = 0; + + // Build the percents parameter which defines the percentiles to query + // for the metric distribution data. + // Use a fixed percentile spacing of 5%. + let count = 0; + const percents = Array.from( + Array(MAX_PERCENT / PERCENTILE_SPACING), + () => (count += PERCENTILE_SPACING) + ); + + const aggs: Aggs = {}; + + fields.forEach((field, i) => { + const { safeFieldName } = field; + + aggs[`${safeFieldName}_field_stats`] = { + filter: { exists: { field: field.fieldName } }, + aggs: { + actual_stats: { + stats: { field: field.fieldName }, + }, + }, + }; + aggs[`${safeFieldName}_percentiles`] = { + percentiles: { + field: field.fieldName, + percents, + keyed: false, + }, + }; + + const top = { + terms: { + field: field.fieldName, + size: 10, + order: { + _count: 'desc', + }, + } as AggregationsTermsAggregation, + }; + + // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation + // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + aggs[`${safeFieldName}_top`] = { + sampler: { + shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, + }, + aggs: { + top, + }, + }; + } else { + aggs[`${safeFieldName}_top`] = top; + } + }); + + const searchBody = { + query, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchNumericFieldsStats = ( + dataSearch: ISearchStart, + params: FieldStatsCommonRequestParams, + fields: Field[], + options: ISearchOptions +): Observable => { + const { samplerShardSize } = params; + const request: estypes.SearchRequest = getNumericFieldsStatsRequest(params, fields); + + return dataSearch + .search({ params: request }, options) + .pipe( + catchError((e) => { + // @todo: kick off another requests individually + return of({ + fields, + error: extractErrorProperties(e), + } as FieldStatsError); + }), + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; + + const aggregations = resp.rawResponse.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + + const batchStats: NumericFieldStats[] = []; + + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; + const docCount = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], + 0 + ); + const fieldStatsResp = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], + {} + ); + + const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + topAggsPath.push('top'); + } + + const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); + + const stats: NumericFieldStats = { + fieldName: field.fieldName, + count: docCount, + min: get(fieldStatsResp, 'min', 0), + max: get(fieldStatsResp, 'max', 0), + avg: get(fieldStatsResp, 'avg', 0), + isTopValuesSampled: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, + topValues, + topValuesSampleSize: topValues.reduce( + (acc, curr) => acc + curr.doc_count, + get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) + ), + topValuesSamplerShardSize: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? SAMPLER_TOP_TERMS_SHARD_SIZE + : samplerShardSize, + }; + + if (stats.count > 0) { + const percentiles = get( + aggregations, + [...aggsPath, `${safeFieldName}_percentiles`, 'values'], + [] + ); + const medianPercentile: { value: number; key: number } | undefined = find(percentiles, { + key: 50, + }); + stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; + stats.distribution = processDistributionData( + percentiles, + PERCENTILE_SPACING, + stats.min + ); + } + + batchStats.push(stats); + }); + + return batchStats; + }) + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts new file mode 100644 index 0000000000000..024464c1947c8 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { get } from 'lodash'; +import { Observable, of } from 'rxjs'; +import { catchError, map } from 'rxjs/operators'; +import { AggregationsTermsAggregation } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { SAMPLER_TOP_TERMS_SHARD_SIZE, SAMPLER_TOP_TERMS_THRESHOLD } from './constants'; +import { + buildSamplerAggregation, + getSamplerAggregationsResponsePath, +} from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { + Aggs, + Bucket, + Field, + FieldStatsCommonRequestParams, + StringFieldStats, +} from '../../../../../common/types/field_stats'; +import type { + IKibanaSearchRequest, + IKibanaSearchResponse, + ISearchOptions, + ISearchStart, +} from '../../../../../../../../src/plugins/data/public'; +import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; +import { extractErrorProperties } from '../../utils/error_utils'; + +export const getStringFieldStatsRequest = ( + params: FieldStatsCommonRequestParams, + fields: Field[] +) => { + const { index, query, runtimeFieldMap, samplerShardSize } = params; + + const size = 0; + + const aggs: Aggs = {}; + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; + const top = { + terms: { + field: field.fieldName, + size: 10, + order: { + _count: 'desc', + }, + } as AggregationsTermsAggregation, + }; + + // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation + // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + aggs[`${safeFieldName}_top`] = { + sampler: { + shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, + }, + aggs: { + top, + }, + }; + } else { + aggs[`${safeFieldName}_top`] = top; + } + }); + + const searchBody = { + query, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchStringFieldsStats = ( + dataSearch: ISearchStart, + params: FieldStatsCommonRequestParams, + fields: Field[], + options: ISearchOptions +): Observable => { + const { samplerShardSize } = params; + const request: estypes.SearchRequest = getStringFieldStatsRequest(params, fields); + + return dataSearch + .search({ params: request }, options) + .pipe( + catchError((e) => + of({ + fields, + error: extractErrorProperties(e), + } as FieldStatsError) + ), + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; + const aggregations = resp.rawResponse.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const batchStats: StringFieldStats[] = []; + + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; + + const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + topAggsPath.push('top'); + } + + const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); + + const stats = { + fieldName: field.fieldName, + isTopValuesSampled: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, + topValues, + topValuesSampleSize: topValues.reduce( + (acc, curr) => acc + curr.doc_count, + get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) + ), + topValuesSamplerShardSize: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? SAMPLER_TOP_TERMS_SHARD_SIZE + : samplerShardSize, + }; + + batchStats.push(stats); + }); + + return batchStats; + }) + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts new file mode 100644 index 0000000000000..fb392cc17b05b --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts @@ -0,0 +1,227 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { get } from 'lodash'; +import { Query } from '@kbn/es-query'; +import { + buildBaseFilterCriteria, + buildSamplerAggregation, + getSafeAggregationName, + getSamplerAggregationsResponsePath, +} from '../../../../../common/utils/query_utils'; +import { getDatafeedAggregations } from '../../../../../common/utils/datafeed_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import { IKibanaSearchResponse } from '../../../../../../../../src/plugins/data/common'; +import { AggregatableField, NonAggregatableField } from '../../types/overall_stats'; +import { AggCardinality, Aggs } from '../../../../../common/types/field_stats'; + +export const checkAggregatableFieldsExistRequest = ( + indexPatternTitle: string, + query: Query['query'], + aggregatableFields: string[], + samplerShardSize: number, + timeFieldName: string | undefined, + earliestMs?: number, + latestMs?: number, + datafeedConfig?: estypes.MlDatafeed, + runtimeMappings?: estypes.MappingRuntimeFields +): estypes.SearchRequest => { + const index = indexPatternTitle; + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + const datafeedAggregations = getDatafeedAggregations(datafeedConfig); + + // Value count aggregation faster way of checking if field exists than using + // filter aggregation with exists query. + const aggs: Aggs = datafeedAggregations !== undefined ? { ...datafeedAggregations } : {}; + + // Combine runtime fields from the data view as well as the datafeed + const combinedRuntimeMappings: estypes.MappingRuntimeFields = { + ...(isPopulatedObject(runtimeMappings) ? runtimeMappings : {}), + ...(isPopulatedObject(datafeedConfig) && isPopulatedObject(datafeedConfig.runtime_mappings) + ? datafeedConfig.runtime_mappings + : {}), + }; + + aggregatableFields.forEach((field, i) => { + const safeFieldName = getSafeAggregationName(field, i); + aggs[`${safeFieldName}_count`] = { + filter: { exists: { field } }, + }; + + let cardinalityField: AggCardinality; + if (datafeedConfig?.script_fields?.hasOwnProperty(field)) { + cardinalityField = aggs[`${safeFieldName}_cardinality`] = { + cardinality: { script: datafeedConfig?.script_fields[field].script }, + }; + } else { + cardinalityField = { + cardinality: { field }, + }; + } + aggs[`${safeFieldName}_cardinality`] = cardinalityField; + }); + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + ...(isPopulatedObject(aggs) ? { aggs: buildSamplerAggregation(aggs, samplerShardSize) } : {}), + ...(isPopulatedObject(combinedRuntimeMappings) + ? { runtime_mappings: combinedRuntimeMappings } + : {}), + }; + + return { + index, + track_total_hits: true, + size, + body: searchBody, + }; +}; + +export interface AggregatableFieldOverallStats extends IKibanaSearchResponse { + aggregatableFields: string[]; +} +export const processAggregatableFieldsExistResponse = ( + responses: AggregatableFieldOverallStats[] | undefined, + aggregatableFields: string[], + samplerShardSize: number, + datafeedConfig?: estypes.MlDatafeed +) => { + const stats = { + totalCount: 0, + aggregatableExistsFields: [] as AggregatableField[], + aggregatableNotExistsFields: [] as AggregatableField[], + }; + + if (!responses || aggregatableFields.length === 0) return stats; + + responses.forEach(({ rawResponse: body, aggregatableFields: aggregatableFieldsChunk }) => { + const aggregations = body.aggregations; + const totalCount = (body.hits.total as estypes.SearchTotalHits).value ?? body.hits.total; + stats.totalCount = totalCount as number; + + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const sampleCount = + samplerShardSize > 0 ? get(aggregations, ['sample', 'doc_count'], 0) : totalCount; + aggregatableFieldsChunk.forEach((field, i) => { + const safeFieldName = getSafeAggregationName(field, i); + const count = get(aggregations, [...aggsPath, `${safeFieldName}_count`, 'doc_count'], 0); + if (count > 0) { + const cardinality = get( + aggregations, + [...aggsPath, `${safeFieldName}_cardinality`, 'value'], + 0 + ); + stats.aggregatableExistsFields.push({ + fieldName: field, + existsInDocs: true, + stats: { + sampleCount, + count, + cardinality, + }, + }); + } else { + if ( + datafeedConfig?.script_fields?.hasOwnProperty(field) || + datafeedConfig?.runtime_mappings?.hasOwnProperty(field) + ) { + const cardinality = get( + aggregations, + [...aggsPath, `${safeFieldName}_cardinality`, 'value'], + 0 + ); + stats.aggregatableExistsFields.push({ + fieldName: field, + existsInDocs: true, + stats: { + sampleCount, + count, + cardinality, + }, + }); + } else { + stats.aggregatableNotExistsFields.push({ + fieldName: field, + existsInDocs: false, + stats: {}, + }); + } + } + }); + }); + + return stats as { + totalCount: number; + aggregatableExistsFields: AggregatableField[]; + aggregatableNotExistsFields: AggregatableField[]; + }; +}; + +export const checkNonAggregatableFieldExistsRequest = ( + indexPatternTitle: string, + query: Query['query'], + field: string, + timeFieldName: string | undefined, + earliestMs: number | undefined, + latestMs: number | undefined, + runtimeMappings?: estypes.MappingRuntimeFields +): estypes.SearchRequest => { + const index = indexPatternTitle; + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), + }; + if (Array.isArray(filterCriteria)) { + filterCriteria.push({ exists: { field } }); + } + + return { + index, + size, + body: searchBody, + }; +}; + +export const processNonAggregatableFieldsExistResponse = ( + results: IKibanaSearchResponse[] | undefined, + nonAggregatableFields: string[] +) => { + const stats = { + nonAggregatableExistsFields: [] as NonAggregatableField[], + nonAggregatableNotExistsFields: [] as NonAggregatableField[], + }; + + if (!results || nonAggregatableFields.length === 0) return stats; + + nonAggregatableFields.forEach((fieldName) => { + const foundField = results.find((r) => r.rawResponse.fieldName === fieldName); + const existsInDocs = foundField !== undefined && foundField.rawResponse.hits.total > 0; + const fieldData: NonAggregatableField = { + fieldName, + existsInDocs, + }; + if (existsInDocs === true) { + stats.nonAggregatableExistsFields.push(fieldData); + } else { + stats.nonAggregatableNotExistsFields.push(fieldData); + } + }); + return stats; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/visualizer_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/visualizer_stats.ts deleted file mode 100644 index 3653936f3d12e..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/visualizer_stats.ts +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { lazyLoadModules } from '../../../lazy_load_bundle'; -import type { DocumentCounts, FieldRequestConfig, FieldVisStats } from '../../../../common/types'; -import { OverallStats } from '../types/overall_stats'; - -export function basePath() { - return '/internal/data_visualizer'; -} - -export async function getVisualizerOverallStats({ - indexPatternTitle, - query, - timeFieldName, - earliest, - latest, - samplerShardSize, - aggregatableFields, - nonAggregatableFields, - runtimeMappings, -}: { - indexPatternTitle: string; - query: any; - timeFieldName?: string; - earliest?: number; - latest?: number; - samplerShardSize?: number; - aggregatableFields: string[]; - nonAggregatableFields: string[]; - runtimeMappings?: estypes.MappingRuntimeFields; -}) { - const body = JSON.stringify({ - query, - timeFieldName, - earliest, - latest, - samplerShardSize, - aggregatableFields, - nonAggregatableFields, - runtimeMappings, - }); - - const fileUploadModules = await lazyLoadModules(); - return await fileUploadModules.getHttp().fetch({ - path: `${basePath()}/get_overall_stats/${indexPatternTitle}`, - method: 'POST', - body, - }); -} - -export async function getVisualizerFieldStats({ - indexPatternTitle, - query, - timeFieldName, - earliest, - latest, - samplerShardSize, - interval, - fields, - maxExamples, - runtimeMappings, -}: { - indexPatternTitle: string; - query: any; - timeFieldName?: string; - earliest?: number; - latest?: number; - samplerShardSize?: number; - interval?: number; - fields?: FieldRequestConfig[]; - maxExamples?: number; - runtimeMappings?: estypes.MappingRuntimeFields; -}) { - const body = JSON.stringify({ - query, - timeFieldName, - earliest, - latest, - samplerShardSize, - interval, - fields, - maxExamples, - runtimeMappings, - }); - - const fileUploadModules = await lazyLoadModules(); - return await fileUploadModules.getHttp().fetch<[DocumentCounts, FieldVisStats]>({ - path: `${basePath()}/get_field_stats/${indexPatternTitle}`, - method: 'POST', - body, - }); -} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts index 734a47d7f01b0..13590505a5d1a 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { Query } from '@kbn/es-query'; + export const SEARCH_QUERY_LANGUAGE = { KUERY: 'kuery', LUCENE: 'lucene', @@ -13,7 +15,7 @@ export const SEARCH_QUERY_LANGUAGE = { export type SearchQueryLanguage = typeof SEARCH_QUERY_LANGUAGE[keyof typeof SEARCH_QUERY_LANGUAGE]; export interface CombinedQuery { - searchString: string | { [key: string]: any }; + searchString: Query['query']; searchQueryLanguage: string; } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts index 2672dc69ac29a..84a6142f012da 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { DocumentCountStats } from '../../../../common/types/field_stats'; + export interface AggregatableField { fieldName: string; stats: { @@ -19,8 +21,9 @@ export type NonAggregatableField = Omit; export interface OverallStats { totalCount: number; + documentCountStats?: DocumentCountStats; aggregatableExistsFields: AggregatableField[]; - aggregatableNotExistsFields: NonAggregatableField[]; - nonAggregatableExistsFields: AggregatableField[]; + aggregatableNotExistsFields: AggregatableField[]; + nonAggregatableExistsFields: NonAggregatableField[]; nonAggregatableNotExistsFields: NonAggregatableField[]; } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/error_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/error_utils.ts index 9bb36496a149e..58c4c820c28d7 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/error_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/error_utils.ts @@ -85,7 +85,7 @@ export function isDVResponseError(error: any): error is DVResponseError { } export function isBoomError(error: any): error is Boom.Boom { - return error.isBoom === true; + return error?.isBoom === true; } export function isWrappedError(error: any): error is WrappedError { diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts similarity index 95% rename from x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts rename to x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts index 4e40c2baaf701..46719c06e2264 100644 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts @@ -6,7 +6,7 @@ */ import { last } from 'lodash'; -import { Distribution } from '../../types'; +import type { Distribution } from '../../../../common/types/field_stats'; export const processDistributionData = ( percentiles: Array<{ value: number }>, @@ -49,7 +49,7 @@ export const processDistributionData = ( // Add in 0-5 and 95-100% if they don't add more // than 25% to the value range at either end. - const lastValue: number = (last(percentileBuckets) as any).value; + const lastValue: number = (last(percentileBuckets) as { value: number }).value; const maxDiff = 0.25 * (lastValue - lowerBound); if (lowerBound - dataMin < maxDiff) { percentileBuckets.splice(0, 0, percentiles[0]); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts index ad3229676b31b..586f636a088e1 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts @@ -75,7 +75,7 @@ const kqlSavedSearch: SavedSearch = { title: 'farequote_filter_and_kuery', description: '', columns: ['_source'], - // @ts-expect-error We don't need the full object here + // @ts-expect-error kibanaSavedObjectMeta: { searchSourceJSON: '{"highlightAll":true,"version":true,"query":{"query":"responsetime > 49","language":"kuery"},"filter":[{"meta":{"index":"90a978e0-1c80-11ec-b1d7-f7e5cf21b9e0","negate":false,"disabled":false,"alias":null,"type":"phrase","key":"airline","value":"ASA","params":{"query":"ASA","type":"phrase"}},"query":{"match":{"airline":{"query":"ASA","type":"phrase"}}},"$state":{"store":"appState"}}],"indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index"}', diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index 1401b1038b8f2..5ebdbcff0b26e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -15,6 +15,7 @@ import { Query, Filter, } from '@kbn/es-query'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { isSavedSearchSavedObject, SavedSearchSavedObject } from '../../../../common/types'; import { IndexPattern, SearchSource } from '../../../../../../../src/plugins/data/common'; import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; @@ -43,7 +44,7 @@ export function getDefaultQuery() { export function getQueryFromSavedSearchObject(savedSearch: SavedSearchSavedObject | SavedSearch) { const search = isSavedSearchSavedObject(savedSearch) ? savedSearch?.attributes?.kibanaSavedObjectMeta - : // @ts-expect-error kibanaSavedObjectMeta does exist + : // @ts-ignore savedSearch?.kibanaSavedObjectMeta; const parsed = @@ -76,7 +77,7 @@ export function createMergedEsQuery( indexPattern?: IndexPattern, uiSettings?: IUiSettingsClient ) { - let combinedQuery: any = getDefaultQuery(); + let combinedQuery: QueryDslQueryContainer = getDefaultQuery(); if (query && query.language === SEARCH_QUERY_LANGUAGE.KUERY) { const ast = fromKueryExpression(query.query); @@ -86,12 +87,12 @@ export function createMergedEsQuery( if (combinedQuery.bool !== undefined) { const filterQuery = buildQueryFromFilters(filters, indexPattern); - if (Array.isArray(combinedQuery.bool.filter) === false) { + if (!Array.isArray(combinedQuery.bool.filter)) { combinedQuery.bool.filter = combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; } - if (Array.isArray(combinedQuery.bool.must_not) === false) { + if (!Array.isArray(combinedQuery.bool.must_not)) { combinedQuery.bool.must_not = combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; } @@ -145,8 +146,20 @@ export function getEsQueryFromSavedSearch({ savedSearch.searchSource.getParent() !== undefined && userQuery ) { + // Flattened query from search source may contain a clause that narrows the time range + // which might interfere with global time pickers so we need to remove + const savedQuery = + cloneDeep(savedSearch.searchSource.getSearchRequestBody()?.query) ?? getDefaultQuery(); + const timeField = savedSearch.searchSource.getField('index')?.timeFieldName; + + if (Array.isArray(savedQuery.bool.filter) && timeField !== undefined) { + savedQuery.bool.filter = savedQuery.bool.filter.filter( + (c: QueryDslQueryContainer) => + !(c.hasOwnProperty('range') && c.range?.hasOwnProperty(timeField)) + ); + } return { - searchQuery: savedSearch.searchSource.getSearchRequestBody()?.query ?? getDefaultQuery(), + searchQuery: savedQuery, searchString: userQuery.query, queryLanguage: userQuery.language as SearchQueryLanguage, }; diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/check_fields_exist.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/check_fields_exist.ts deleted file mode 100644 index 24b4deeecdddd..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/check_fields_exist.ts +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { get } from 'lodash'; -import { IScopedClusterClient } from 'kibana/server'; -import { AggCardinality, Aggs, FieldData } from '../../types'; -import { - buildBaseFilterCriteria, - buildSamplerAggregation, - getSafeAggregationName, - getSamplerAggregationsResponsePath, -} from '../../../common/utils/query_utils'; -import { getDatafeedAggregations } from '../../../common/utils/datafeed_utils'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; - -export const checkAggregatableFieldsExist = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - aggregatableFields: string[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs?: number, - latestMs?: number, - datafeedConfig?: estypes.MlDatafeed, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - const datafeedAggregations = getDatafeedAggregations(datafeedConfig); - - // Value count aggregation faster way of checking if field exists than using - // filter aggregation with exists query. - const aggs: Aggs = datafeedAggregations !== undefined ? { ...datafeedAggregations } : {}; - - // Combine runtime fields from the data view as well as the datafeed - const combinedRuntimeMappings: estypes.MappingRuntimeFields = { - ...(isPopulatedObject(runtimeMappings) ? runtimeMappings : {}), - ...(isPopulatedObject(datafeedConfig) && isPopulatedObject(datafeedConfig.runtime_mappings) - ? datafeedConfig.runtime_mappings - : {}), - }; - - aggregatableFields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field, i); - aggs[`${safeFieldName}_count`] = { - filter: { exists: { field } }, - }; - - let cardinalityField: AggCardinality; - if (datafeedConfig?.script_fields?.hasOwnProperty(field)) { - cardinalityField = aggs[`${safeFieldName}_cardinality`] = { - cardinality: { script: datafeedConfig?.script_fields[field].script }, - }; - } else { - cardinalityField = { - cardinality: { field }, - }; - } - aggs[`${safeFieldName}_cardinality`] = cardinalityField; - }); - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - ...(isPopulatedObject(aggs) ? { aggs: buildSamplerAggregation(aggs, samplerShardSize) } : {}), - ...(isPopulatedObject(combinedRuntimeMappings) - ? { runtime_mappings: combinedRuntimeMappings } - : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - track_total_hits: true, - size, - body: searchBody, - }); - - const aggregations = body.aggregations; - // @ts-expect-error incorrect search response type - const totalCount = body.hits.total.value; - const stats = { - totalCount, - aggregatableExistsFields: [] as FieldData[], - aggregatableNotExistsFields: [] as FieldData[], - }; - - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const sampleCount = - samplerShardSize > 0 ? get(aggregations, ['sample', 'doc_count'], 0) : totalCount; - aggregatableFields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field, i); - const count = get(aggregations, [...aggsPath, `${safeFieldName}_count`, 'doc_count'], 0); - if (count > 0) { - const cardinality = get( - aggregations, - [...aggsPath, `${safeFieldName}_cardinality`, 'value'], - 0 - ); - stats.aggregatableExistsFields.push({ - fieldName: field, - existsInDocs: true, - stats: { - sampleCount, - count, - cardinality, - }, - }); - } else { - if ( - datafeedConfig?.script_fields?.hasOwnProperty(field) || - datafeedConfig?.runtime_mappings?.hasOwnProperty(field) - ) { - const cardinality = get( - aggregations, - [...aggsPath, `${safeFieldName}_cardinality`, 'value'], - 0 - ); - stats.aggregatableExistsFields.push({ - fieldName: field, - existsInDocs: true, - stats: { - sampleCount, - count, - cardinality, - }, - }); - } else { - stats.aggregatableNotExistsFields.push({ - fieldName: field, - existsInDocs: false, - }); - } - } - }); - - return stats; -}; - -export const checkNonAggregatableFieldExists = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - field: string, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - filterCriteria.push({ exists: { field } }); - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - // @ts-expect-error incorrect search response type - return body.hits.total.value > 0; -}; diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/data_visualizer.ts deleted file mode 100644 index 42e7f93cc8789..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/data_visualizer.ts +++ /dev/null @@ -1,489 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IScopedClusterClient } from 'kibana/server'; -import { each, last } from 'lodash'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { JOB_FIELD_TYPES } from '../../../common'; -import type { - BatchStats, - FieldData, - HistogramField, - Field, - DocumentCountStats, - FieldExamples, -} from '../../types'; -import { getHistogramsForFields } from './get_histogram_for_fields'; -import { - checkAggregatableFieldsExist, - checkNonAggregatableFieldExists, -} from './check_fields_exist'; -import { AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE, FIELDS_REQUEST_BATCH_SIZE } from './constants'; -import { getFieldExamples } from './get_field_examples'; -import { - getBooleanFieldsStats, - getDateFieldsStats, - getDocumentCountStats, - getNumericFieldsStats, - getStringFieldsStats, -} from './get_fields_stats'; -import { wrapError } from '../../utils/error_wrapper'; - -export class DataVisualizer { - private _client: IScopedClusterClient; - - constructor(client: IScopedClusterClient) { - this._client = client; - } - - // Obtains overall stats on the fields in the supplied data view, returning an object - // containing the total document count, and four arrays showing which of the supplied - // aggregatable and non-aggregatable fields do or do not exist in documents. - // Sampling will be used if supplied samplerShardSize > 0. - async getOverallStats( - indexPatternTitle: string, - query: object, - aggregatableFields: string[], - nonAggregatableFields: string[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - const stats = { - totalCount: 0, - aggregatableExistsFields: [] as FieldData[], - aggregatableNotExistsFields: [] as FieldData[], - nonAggregatableExistsFields: [] as FieldData[], - nonAggregatableNotExistsFields: [] as FieldData[], - errors: [] as any[], - }; - - // To avoid checking for the existence of too many aggregatable fields in one request, - // split the check into multiple batches (max 200 fields per request). - const batches: string[][] = [[]]; - each(aggregatableFields, (field) => { - let lastArray: string[] = last(batches) as string[]; - if (lastArray.length === AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE) { - lastArray = []; - batches.push(lastArray); - } - lastArray.push(field); - }); - - await Promise.all( - batches.map(async (fields) => { - try { - const batchStats = await this.checkAggregatableFieldsExist( - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - undefined, - runtimeMappings - ); - - // Total count will be returned with each batch of fields. Just overwrite. - stats.totalCount = batchStats.totalCount; - - // Add to the lists of fields which do and do not exist. - stats.aggregatableExistsFields.push(...batchStats.aggregatableExistsFields); - stats.aggregatableNotExistsFields.push(...batchStats.aggregatableNotExistsFields); - } catch (e) { - // If index not found, no need to proceed with other batches - if (e.statusCode === 404) { - throw e; - } - stats.errors.push(wrapError(e)); - } - }) - ); - - await Promise.all( - nonAggregatableFields.map(async (field) => { - try { - const existsInDocs = await this.checkNonAggregatableFieldExists( - indexPatternTitle, - query, - field, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - - const fieldData: FieldData = { - fieldName: field, - existsInDocs, - stats: {}, - }; - - if (existsInDocs === true) { - stats.nonAggregatableExistsFields.push(fieldData); - } else { - stats.nonAggregatableNotExistsFields.push(fieldData); - } - } catch (e) { - stats.errors.push(wrapError(e)); - } - }) - ); - - return stats; - } - - // Obtains binned histograms for supplied list of fields. The statistics for each field in the - // returned array depend on the type of the field (keyword, number, date etc). - // Sampling will be used if supplied samplerShardSize > 0. - async getHistogramsForFields( - indexPatternTitle: string, - query: any, - fields: HistogramField[], - samplerShardSize: number, - runtimeMappings?: estypes.MappingRuntimeFields - ): Promise { - return await getHistogramsForFields( - this._client, - indexPatternTitle, - query, - fields, - samplerShardSize, - runtimeMappings - ); - } - - // Obtains statistics for supplied list of fields. The statistics for each field in the - // returned array depend on the type of the field (keyword, number, date etc). - // Sampling will be used if supplied samplerShardSize > 0. - async getStatsForFields( - indexPatternTitle: string, - query: any, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - intervalMs: number | undefined, - maxExamples: number, - runtimeMappings: estypes.MappingRuntimeFields - ): Promise { - // Batch up fields by type, getting stats for multiple fields at a time. - const batches: Field[][] = []; - const batchedFields: { [key: string]: Field[][] } = {}; - each(fields, (field) => { - if (field.fieldName === undefined) { - // undefined fieldName is used for a document count request. - // getDocumentCountStats requires timeField - don't add to batched requests if not defined - if (timeFieldName !== undefined) { - batches.push([field]); - } - } else { - const fieldType = field.type; - if (batchedFields[fieldType] === undefined) { - batchedFields[fieldType] = [[]]; - } - let lastArray: Field[] = last(batchedFields[fieldType]) as Field[]; - if (lastArray.length === FIELDS_REQUEST_BATCH_SIZE) { - lastArray = []; - batchedFields[fieldType].push(lastArray); - } - lastArray.push(field); - } - }); - - each(batchedFields, (lists) => { - batches.push(...lists); - }); - - let results: BatchStats[] = []; - await Promise.all( - batches.map(async (batch) => { - let batchStats: BatchStats[] = []; - const first = batch[0]; - switch (first.type) { - case JOB_FIELD_TYPES.NUMBER: - // undefined fieldName is used for a document count request. - if (first.fieldName !== undefined) { - batchStats = await this.getNumericFieldsStats( - indexPatternTitle, - query, - batch, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - } else { - // Will only ever be one document count card, - // so no value in batching up the single request. - if (intervalMs !== undefined) { - const stats = await this.getDocumentCountStats( - indexPatternTitle, - query, - timeFieldName, - earliestMs, - latestMs, - intervalMs, - runtimeMappings - ); - batchStats.push(stats); - } - } - break; - case JOB_FIELD_TYPES.KEYWORD: - case JOB_FIELD_TYPES.IP: - batchStats = await this.getStringFieldsStats( - indexPatternTitle, - query, - batch, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - break; - case JOB_FIELD_TYPES.DATE: - batchStats = await this.getDateFieldsStats( - indexPatternTitle, - query, - batch, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - break; - case JOB_FIELD_TYPES.BOOLEAN: - batchStats = await this.getBooleanFieldsStats( - indexPatternTitle, - query, - batch, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - break; - case JOB_FIELD_TYPES.TEXT: - default: - // Use an exists filter on the the field name to get - // examples of the field, so cannot batch up. - await Promise.all( - batch.map(async (field) => { - const stats = await this.getFieldExamples( - indexPatternTitle, - query, - field.fieldName, - timeFieldName, - earliestMs, - latestMs, - maxExamples, - runtimeMappings - ); - batchStats.push(stats); - }) - ); - break; - } - - results = [...results, ...batchStats]; - }) - ); - - return results; - } - - async checkAggregatableFieldsExist( - indexPatternTitle: string, - query: any, - aggregatableFields: string[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs?: number, - latestMs?: number, - datafeedConfig?: estypes.MlDatafeed, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - return await checkAggregatableFieldsExist( - this._client, - indexPatternTitle, - query, - aggregatableFields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - datafeedConfig, - runtimeMappings - ); - } - - async checkNonAggregatableFieldExists( - indexPatternTitle: string, - query: any, - field: string, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - return await checkNonAggregatableFieldExists( - this._client, - indexPatternTitle, - query, - field, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - } - - async getDocumentCountStats( - indexPatternTitle: string, - query: any, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - intervalMs: number, - runtimeMappings: estypes.MappingRuntimeFields - ): Promise { - return await getDocumentCountStats( - this._client, - indexPatternTitle, - query, - timeFieldName, - earliestMs, - latestMs, - intervalMs, - runtimeMappings - ); - } - - async getNumericFieldsStats( - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - return await getNumericFieldsStats( - this._client, - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - } - - async getStringFieldsStats( - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - return await getStringFieldsStats( - this._client, - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - } - - async getDateFieldsStats( - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - return await getDateFieldsStats( - this._client, - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - } - - async getBooleanFieldsStats( - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - return await getBooleanFieldsStats( - this._client, - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - } - - async getFieldExamples( - indexPatternTitle: string, - query: any, - field: string, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - maxExamples: number, - runtimeMappings?: estypes.MappingRuntimeFields - ): Promise { - return await getFieldExamples( - this._client, - indexPatternTitle, - query, - field, - timeFieldName, - earliestMs, - latestMs, - maxExamples, - runtimeMappings - ); - } -} diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_field_examples.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_field_examples.ts deleted file mode 100644 index 78adfb9e81b95..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_field_examples.ts +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { get } from 'lodash'; -import { IScopedClusterClient } from 'kibana/server'; -import { buildBaseFilterCriteria } from '../../../common/utils/query_utils'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; -import { FieldExamples } from '../../types/chart_data'; - -export const getFieldExamples = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - field: string, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - maxExamples: number, - runtimeMappings?: estypes.MappingRuntimeFields -): Promise => { - const { asCurrentUser } = client; - - const index = indexPatternTitle; - - // Request at least 100 docs so that we have a chance of obtaining - // 'maxExamples' of the field. - const size = Math.max(100, maxExamples); - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - // Use an exists filter to return examples of the field. - filterCriteria.push({ - exists: { field }, - }); - - const searchBody = { - fields: [field], - _source: false, - query: { - bool: { - filter: filterCriteria, - }, - }, - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - const stats = { - fieldName: field, - examples: [] as any[], - }; - // @ts-expect-error incorrect search response type - if (body.hits.total.value > 0) { - const hits = body.hits.hits; - for (let i = 0; i < hits.length; i++) { - // Use lodash get() to support field names containing dots. - const doc: object[] | undefined = get(hits[i].fields, field); - // the results from fields query is always an array - if (Array.isArray(doc) && doc.length > 0) { - const example = doc[0]; - if (example !== undefined && stats.examples.indexOf(example) === -1) { - stats.examples.push(example); - if (stats.examples.length === maxExamples) { - break; - } - } - } - } - } - - return stats; -}; diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts deleted file mode 100644 index da93719e9ed93..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts +++ /dev/null @@ -1,478 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { each, find, get } from 'lodash'; -import { IScopedClusterClient } from 'kibana/server'; -import { - Aggs, - BooleanFieldStats, - Bucket, - DateFieldStats, - DocumentCountStats, - Field, - NumericFieldStats, - StringFieldStats, -} from '../../types'; -import { - buildBaseFilterCriteria, - buildSamplerAggregation, - getSafeAggregationName, - getSamplerAggregationsResponsePath, -} from '../../../common/utils/query_utils'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; -import { processDistributionData } from './process_distribution_data'; -import { SAMPLER_TOP_TERMS_SHARD_SIZE, SAMPLER_TOP_TERMS_THRESHOLD } from './constants'; - -export const getDocumentCountStats = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - intervalMs: number, - runtimeMappings: estypes.MappingRuntimeFields -): Promise => { - const { asCurrentUser } = client; - - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - // Don't use the sampler aggregation as this can lead to some potentially - // confusing date histogram results depending on the date range of data amongst shards. - - const aggs = { - eventRate: { - date_histogram: { - field: timeFieldName, - fixed_interval: `${intervalMs}ms`, - min_doc_count: 1, - }, - }, - }; - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs, - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - - const buckets: { [key: string]: number } = {}; - const dataByTimeBucket: Array<{ key: string; doc_count: number }> = get( - body, - ['aggregations', 'eventRate', 'buckets'], - [] - ); - each(dataByTimeBucket, (dataForTime) => { - const time = dataForTime.key; - buckets[time] = dataForTime.doc_count; - }); - - return { - documentCounts: { - interval: intervalMs, - buckets, - }, - }; -}; - -export const getNumericFieldsStats = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - // Build the percents parameter which defines the percentiles to query - // for the metric distribution data. - // Use a fixed percentile spacing of 5%. - const MAX_PERCENT = 100; - const PERCENTILE_SPACING = 5; - let count = 0; - const percents = Array.from( - Array(MAX_PERCENT / PERCENTILE_SPACING), - () => (count += PERCENTILE_SPACING) - ); - - const aggs: { [key: string]: any } = {}; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - aggs[`${safeFieldName}_field_stats`] = { - filter: { exists: { field: field.fieldName } }, - aggs: { - actual_stats: { - stats: { field: field.fieldName }, - }, - }, - }; - aggs[`${safeFieldName}_percentiles`] = { - percentiles: { - field: field.fieldName, - percents, - keyed: false, - }, - }; - - const top = { - terms: { - field: field.fieldName, - size: 10, - order: { - _count: 'desc', - }, - }, - }; - - // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation - // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - aggs[`${safeFieldName}_top`] = { - sampler: { - shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, - }, - aggs: { - top, - }, - }; - } else { - aggs[`${safeFieldName}_top`] = top; - } - }); - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const batchStats: NumericFieldStats[] = []; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - const docCount = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], - 0 - ); - const fieldStatsResp = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], - {} - ); - - const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } - - const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); - - const stats: NumericFieldStats = { - fieldName: field.fieldName, - count: docCount, - min: get(fieldStatsResp, 'min', 0), - max: get(fieldStatsResp, 'max', 0), - avg: get(fieldStatsResp, 'avg', 0), - isTopValuesSampled: field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, - topValues, - topValuesSampleSize: topValues.reduce( - (acc, curr) => acc + curr.doc_count, - get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) - ), - topValuesSamplerShardSize: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? SAMPLER_TOP_TERMS_SHARD_SIZE - : samplerShardSize, - }; - - if (stats.count > 0) { - const percentiles = get( - aggregations, - [...aggsPath, `${safeFieldName}_percentiles`, 'values'], - [] - ); - const medianPercentile: { value: number; key: number } | undefined = find(percentiles, { - key: 50, - }); - stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; - stats.distribution = processDistributionData(percentiles, PERCENTILE_SPACING, stats.min); - } - - batchStats.push(stats); - }); - - return batchStats; -}; - -export const getStringFieldsStats = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - const aggs: Aggs = {}; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - const top = { - terms: { - field: field.fieldName, - size: 10, - order: { - _count: 'desc', - }, - }, - }; - - // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation - // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - aggs[`${safeFieldName}_top`] = { - sampler: { - shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, - }, - aggs: { - top, - }, - }; - } else { - aggs[`${safeFieldName}_top`] = top; - } - }); - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const batchStats: StringFieldStats[] = []; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - - const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } - - const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); - - const stats = { - fieldName: field.fieldName, - isTopValuesSampled: field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, - topValues, - topValuesSampleSize: topValues.reduce( - (acc, curr) => acc + curr.doc_count, - get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) - ), - topValuesSamplerShardSize: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? SAMPLER_TOP_TERMS_SHARD_SIZE - : samplerShardSize, - }; - - batchStats.push(stats); - }); - - return batchStats; -}; - -export const getDateFieldsStats = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - const aggs: Aggs = {}; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - aggs[`${safeFieldName}_field_stats`] = { - filter: { exists: { field: field.fieldName } }, - aggs: { - actual_stats: { - stats: { field: field.fieldName }, - }, - }, - }; - }); - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const batchStats: DateFieldStats[] = []; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - const docCount = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], - 0 - ); - const fieldStatsResp = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], - {} - ); - batchStats.push({ - fieldName: field.fieldName, - count: docCount, - earliest: get(fieldStatsResp, 'min', 0), - latest: get(fieldStatsResp, 'max', 0), - }); - }); - - return batchStats; -}; - -export const getBooleanFieldsStats = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - const aggs: Aggs = {}; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - aggs[`${safeFieldName}_value_count`] = { - filter: { exists: { field: field.fieldName } }, - }; - aggs[`${safeFieldName}_values`] = { - terms: { - field: field.fieldName, - size: 2, - }, - }; - }); - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const batchStats: BooleanFieldStats[] = []; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - const stats: BooleanFieldStats = { - fieldName: field.fieldName, - count: get(aggregations, [...aggsPath, `${safeFieldName}_value_count`, 'doc_count'], 0), - trueCount: 0, - falseCount: 0, - }; - - const valueBuckets: Array<{ [key: string]: number }> = get( - aggregations, - [...aggsPath, `${safeFieldName}_values`, 'buckets'], - [] - ); - valueBuckets.forEach((bucket) => { - stats[`${bucket.key_as_string}Count`] = bucket.doc_count; - }); - - batchStats.push(stats); - }); - - return batchStats; -}; diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_histogram_for_fields.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_histogram_for_fields.ts deleted file mode 100644 index 1cbf40a22b056..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_histogram_for_fields.ts +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IScopedClusterClient } from 'kibana/server'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { get } from 'lodash'; -import { ChartData, ChartRequestAgg, HistogramField, NumericColumnStatsMap } from '../../types'; -import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/common'; -import { stringHash } from '../../../common/utils/string_utils'; -import { - buildSamplerAggregation, - getSamplerAggregationsResponsePath, -} from '../../../common/utils/query_utils'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; -import { MAX_CHART_COLUMNS } from './constants'; - -export const getAggIntervals = async ( - { asCurrentUser }: IScopedClusterClient, - indexPatternTitle: string, - query: any, - fields: HistogramField[], - samplerShardSize: number, - runtimeMappings?: estypes.MappingRuntimeFields -): Promise => { - const numericColumns = fields.filter((field) => { - return field.type === KBN_FIELD_TYPES.NUMBER || field.type === KBN_FIELD_TYPES.DATE; - }); - - if (numericColumns.length === 0) { - return {}; - } - - const minMaxAggs = numericColumns.reduce((aggs, c) => { - const id = stringHash(c.fieldName); - aggs[id] = { - stats: { - field: c.fieldName, - }, - }; - return aggs; - }, {} as Record); - - const { body } = await asCurrentUser.search({ - index: indexPatternTitle, - size: 0, - body: { - query, - aggs: buildSamplerAggregation(minMaxAggs, samplerShardSize), - size: 0, - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }, - }); - - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const aggregations = aggsPath.length > 0 ? get(body.aggregations, aggsPath) : body.aggregations; - - return Object.keys(aggregations).reduce((p, aggName) => { - const stats = [aggregations[aggName].min, aggregations[aggName].max]; - if (!stats.includes(null)) { - const delta = aggregations[aggName].max - aggregations[aggName].min; - - let aggInterval = 1; - - if (delta > MAX_CHART_COLUMNS || delta <= 1) { - aggInterval = delta / (MAX_CHART_COLUMNS - 1); - } - - p[aggName] = { interval: aggInterval, min: stats[0], max: stats[1] }; - } - - return p; - }, {} as NumericColumnStatsMap); -}; - -export const getHistogramsForFields = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - fields: HistogramField[], - samplerShardSize: number, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - const aggIntervals = await getAggIntervals( - client, - indexPatternTitle, - query, - fields, - samplerShardSize, - runtimeMappings - ); - - const chartDataAggs = fields.reduce((aggs, field) => { - const fieldName = field.fieldName; - const fieldType = field.type; - const id = stringHash(fieldName); - if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { - if (aggIntervals[id] !== undefined) { - aggs[`${id}_histogram`] = { - histogram: { - field: fieldName, - interval: aggIntervals[id].interval !== 0 ? aggIntervals[id].interval : 1, - }, - }; - } - } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { - if (fieldType === KBN_FIELD_TYPES.STRING) { - aggs[`${id}_cardinality`] = { - cardinality: { - field: fieldName, - }, - }; - } - aggs[`${id}_terms`] = { - terms: { - field: fieldName, - size: MAX_CHART_COLUMNS, - }, - }; - } - return aggs; - }, {} as Record); - - if (Object.keys(chartDataAggs).length === 0) { - return []; - } - - const { body } = await asCurrentUser.search({ - index: indexPatternTitle, - size: 0, - body: { - query, - aggs: buildSamplerAggregation(chartDataAggs, samplerShardSize), - size: 0, - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }, - }); - - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const aggregations = aggsPath.length > 0 ? get(body.aggregations, aggsPath) : body.aggregations; - - const chartsData: ChartData[] = fields.map((field): ChartData => { - const fieldName = field.fieldName; - const fieldType = field.type; - const id = stringHash(field.fieldName); - - if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { - if (aggIntervals[id] === undefined) { - return { - type: 'numeric', - data: [], - interval: 0, - stats: [0, 0], - id: fieldName, - }; - } - - return { - data: aggregations[`${id}_histogram`].buckets, - interval: aggIntervals[id].interval, - stats: [aggIntervals[id].min, aggIntervals[id].max], - type: 'numeric', - id: fieldName, - }; - } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { - return { - type: fieldType === KBN_FIELD_TYPES.STRING ? 'ordinal' : 'boolean', - cardinality: - fieldType === KBN_FIELD_TYPES.STRING ? aggregations[`${id}_cardinality`].value : 2, - data: aggregations[`${id}_terms`].buckets, - id: fieldName, - }; - } - - return { - type: 'unsupported', - id: fieldName, - }; - }); - - return chartsData; -}; diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/index.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/index.ts deleted file mode 100644 index a29957b159b7e..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './data_visualizer'; diff --git a/x-pack/plugins/data_visualizer/server/plugin.ts b/x-pack/plugins/data_visualizer/server/plugin.ts index e2e0637ef8f3f..9ef6ca5ae6a69 100644 --- a/x-pack/plugins/data_visualizer/server/plugin.ts +++ b/x-pack/plugins/data_visualizer/server/plugin.ts @@ -7,15 +7,12 @@ import { CoreSetup, CoreStart, Plugin } from 'src/core/server'; import { StartDeps, SetupDeps } from './types'; -import { dataVisualizerRoutes } from './routes'; import { registerWithCustomIntegrations } from './register_custom_integration'; export class DataVisualizerPlugin implements Plugin { constructor() {} setup(coreSetup: CoreSetup, plugins: SetupDeps) { - dataVisualizerRoutes(coreSetup); - // home-plugin required if (plugins.home && plugins.customIntegrations) { registerWithCustomIntegrations(plugins.customIntegrations); diff --git a/x-pack/plugins/data_visualizer/server/routes/index.ts b/x-pack/plugins/data_visualizer/server/routes/index.ts deleted file mode 100644 index 892f6cbd77361..0000000000000 --- a/x-pack/plugins/data_visualizer/server/routes/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { dataVisualizerRoutes } from './routes'; diff --git a/x-pack/plugins/data_visualizer/server/routes/routes.ts b/x-pack/plugins/data_visualizer/server/routes/routes.ts deleted file mode 100644 index 1ec2eaa242c1c..0000000000000 --- a/x-pack/plugins/data_visualizer/server/routes/routes.ts +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { CoreSetup, IScopedClusterClient } from 'kibana/server'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { - dataVisualizerFieldHistogramsSchema, - dataVisualizerFieldStatsSchema, - dataVisualizerOverallStatsSchema, - dataViewTitleSchema, -} from './schemas'; -import type { Field, StartDeps, HistogramField } from '../types'; -import { DataVisualizer } from '../models/data_visualizer'; -import { wrapError } from '../utils/error_wrapper'; - -function getOverallStats( - client: IScopedClusterClient, - indexPatternTitle: string, - query: object, - aggregatableFields: string[], - nonAggregatableFields: string[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings: estypes.MappingRuntimeFields -) { - const dv = new DataVisualizer(client); - return dv.getOverallStats( - indexPatternTitle, - query, - aggregatableFields, - nonAggregatableFields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); -} - -function getStatsForFields( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - interval: number | undefined, - maxExamples: number, - runtimeMappings: estypes.MappingRuntimeFields -) { - const dv = new DataVisualizer(client); - return dv.getStatsForFields( - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - interval, - maxExamples, - runtimeMappings - ); -} - -function getHistogramsForFields( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - fields: HistogramField[], - samplerShardSize: number, - runtimeMappings: estypes.MappingRuntimeFields -) { - const dv = new DataVisualizer(client); - return dv.getHistogramsForFields( - indexPatternTitle, - query, - fields, - samplerShardSize, - runtimeMappings - ); -} -/** - * Routes for the index data visualizer. - */ -export function dataVisualizerRoutes(coreSetup: CoreSetup) { - const router = coreSetup.http.createRouter(); - - /** - * @apiGroup DataVisualizer - * - * @api {post} /internal/data_visualizer/get_field_histograms/:dataViewTitle Get histograms for fields - * @apiName GetHistogramsForFields - * @apiDescription Returns the histograms on a list fields in the specified data view. - * - * @apiSchema (params) dataViewTitleSchema - * @apiSchema (body) dataVisualizerFieldHistogramsSchema - * - * @apiSuccess {Object} fieldName histograms by field, keyed on the name of the field. - */ - router.post( - { - path: '/internal/data_visualizer/get_field_histograms/{dataViewTitle}', - validate: { - params: dataViewTitleSchema, - body: dataVisualizerFieldHistogramsSchema, - }, - }, - async (context, request, response) => { - try { - const { - params: { dataViewTitle }, - body: { query, fields, samplerShardSize, runtimeMappings }, - } = request; - - const results = await getHistogramsForFields( - context.core.elasticsearch.client, - dataViewTitle, - query, - fields, - samplerShardSize, - runtimeMappings - ); - - return response.ok({ - body: results, - }); - } catch (e) { - return response.customError(wrapError(e)); - } - } - ); - - /** - * @apiGroup DataVisualizer - * - * @api {post} /internal/data_visualizer/get_field_stats/:dataViewTitle Get stats for fields - * @apiName GetStatsForFields - * @apiDescription Returns the stats on individual fields in the specified data view. - * - * @apiSchema (params) dataViewTitleSchema - * @apiSchema (body) dataVisualizerFieldStatsSchema - * - * @apiSuccess {Object} fieldName stats by field, keyed on the name of the field. - */ - router.post( - { - path: '/internal/data_visualizer/get_field_stats/{dataViewTitle}', - validate: { - params: dataViewTitleSchema, - body: dataVisualizerFieldStatsSchema, - }, - }, - async (context, request, response) => { - try { - const { - params: { dataViewTitle }, - body: { - query, - fields, - samplerShardSize, - timeFieldName, - earliest, - latest, - interval, - maxExamples, - runtimeMappings, - }, - } = request; - const results = await getStatsForFields( - context.core.elasticsearch.client, - dataViewTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliest, - latest, - interval, - maxExamples, - runtimeMappings - ); - - return response.ok({ - body: results, - }); - } catch (e) { - return response.customError(wrapError(e)); - } - } - ); - - /** - * @apiGroup DataVisualizer - * - * @api {post} /internal/data_visualizer/get_overall_stats/:dataViewTitle Get overall stats - * @apiName GetOverallStats - * @apiDescription Returns the top level overall stats for the specified data view. - * - * @apiSchema (params) dataViewTitleSchema - * @apiSchema (body) dataVisualizerOverallStatsSchema - * - * @apiSuccess {number} totalCount total count of documents. - * @apiSuccess {Object} aggregatableExistsFields stats on aggregatable fields that exist in documents. - * @apiSuccess {Object} aggregatableNotExistsFields stats on aggregatable fields that do not exist in documents. - * @apiSuccess {Object} nonAggregatableExistsFields stats on non-aggregatable fields that exist in documents. - * @apiSuccess {Object} nonAggregatableNotExistsFields stats on non-aggregatable fields that do not exist in documents. - */ - router.post( - { - path: '/internal/data_visualizer/get_overall_stats/{dataViewTitle}', - validate: { - params: dataViewTitleSchema, - body: dataVisualizerOverallStatsSchema, - }, - }, - async (context, request, response) => { - try { - const { - params: { dataViewTitle }, - body: { - query, - aggregatableFields, - nonAggregatableFields, - samplerShardSize, - timeFieldName, - earliest, - latest, - runtimeMappings, - }, - } = request; - - const results = await getOverallStats( - context.core.elasticsearch.client, - dataViewTitle, - query, - aggregatableFields, - nonAggregatableFields, - samplerShardSize, - timeFieldName, - earliest, - latest, - runtimeMappings - ); - - return response.ok({ - body: results, - }); - } catch (e) { - return response.customError(wrapError(e)); - } - } - ); -} diff --git a/x-pack/plugins/data_visualizer/server/routes/schemas/index.ts b/x-pack/plugins/data_visualizer/server/routes/schemas/index.ts deleted file mode 100644 index 156336feef29e..0000000000000 --- a/x-pack/plugins/data_visualizer/server/routes/schemas/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './index_data_visualizer_schemas'; diff --git a/x-pack/plugins/data_visualizer/server/routes/schemas/index_data_visualizer_schemas.ts b/x-pack/plugins/data_visualizer/server/routes/schemas/index_data_visualizer_schemas.ts deleted file mode 100644 index 3b5797622734f..0000000000000 --- a/x-pack/plugins/data_visualizer/server/routes/schemas/index_data_visualizer_schemas.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { schema } from '@kbn/config-schema'; -import { isRuntimeField } from '../../../common/utils/runtime_field_utils'; - -export const runtimeMappingsSchema = schema.object( - {}, - { - unknowns: 'allow', - validate: (v: object) => { - if (Object.values(v).some((o) => !isRuntimeField(o))) { - return 'Invalid runtime field'; - } - }, - } -); - -export const dataViewTitleSchema = schema.object({ - /** Title of the data view for which to return stats. */ - dataViewTitle: schema.string(), -}); - -export const dataVisualizerFieldHistogramsSchema = schema.object({ - /** Query to match documents in the index. */ - query: schema.any(), - /** The fields to return histogram data. */ - fields: schema.arrayOf(schema.any()), - /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ - samplerShardSize: schema.number(), - /** Optional search time runtime fields */ - runtimeMappings: runtimeMappingsSchema, -}); - -export const dataVisualizerFieldStatsSchema = schema.object({ - /** Query to match documents in the index. */ - query: schema.any(), - fields: schema.arrayOf(schema.any()), - /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ - samplerShardSize: schema.number(), - /** Name of the time field in the index (optional). */ - timeFieldName: schema.maybe(schema.string()), - /** Earliest timestamp for search, as epoch ms (optional). */ - earliest: schema.maybe(schema.number()), - /** Latest timestamp for search, as epoch ms (optional). */ - latest: schema.maybe(schema.number()), - /** Aggregation interval, in milliseconds, to use for obtaining document counts over time (optional). */ - interval: schema.maybe(schema.number()), - /** Maximum number of examples to return for text type fields. */ - maxExamples: schema.number(), - /** Optional search time runtime fields */ - runtimeMappings: runtimeMappingsSchema, -}); - -export const dataVisualizerOverallStatsSchema = schema.object({ - /** Query to match documents in the index. */ - query: schema.any(), - /** Names of aggregatable fields for which to return stats. */ - aggregatableFields: schema.arrayOf(schema.string()), - /** Names of non-aggregatable fields for which to return stats. */ - nonAggregatableFields: schema.arrayOf(schema.string()), - /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ - samplerShardSize: schema.number(), - /** Name of the time field in the index (optional). */ - timeFieldName: schema.maybe(schema.string()), - /** Earliest timestamp for search, as epoch ms (optional). */ - earliest: schema.maybe(schema.number()), - /** Latest timestamp for search, as epoch ms (optional). */ - latest: schema.maybe(schema.number()), - /** Optional search time runtime fields */ - runtimeMappings: runtimeMappingsSchema, -}); diff --git a/x-pack/plugins/data_visualizer/server/types/deps.ts b/x-pack/plugins/data_visualizer/server/types/deps.ts index 1f6dba0592f6f..8ee8c75abe543 100644 --- a/x-pack/plugins/data_visualizer/server/types/deps.ts +++ b/x-pack/plugins/data_visualizer/server/types/deps.ts @@ -9,12 +9,18 @@ import type { SecurityPluginStart } from '../../../security/server'; import type { UsageCollectionSetup } from '../../../../../src/plugins/usage_collection/server'; import { CustomIntegrationsPluginSetup } from '../../../../../src/plugins/custom_integrations/server'; import { HomeServerPluginSetup } from '../../../../../src/plugins/home/server'; +import { + PluginSetup as DataPluginSetup, + PluginStart as DataPluginStart, +} from '../../../../../src/plugins/data/server'; export interface StartDeps { security?: SecurityPluginStart; + data: DataPluginStart; } export interface SetupDeps { usageCollection: UsageCollectionSetup; customIntegrations?: CustomIntegrationsPluginSetup; home?: HomeServerPluginSetup; + data: DataPluginSetup; } diff --git a/x-pack/plugins/data_visualizer/server/types/index.ts b/x-pack/plugins/data_visualizer/server/types/index.ts index e0379b514de32..2fc0fb2a6173b 100644 --- a/x-pack/plugins/data_visualizer/server/types/index.ts +++ b/x-pack/plugins/data_visualizer/server/types/index.ts @@ -5,4 +5,3 @@ * 2.0. */ export * from './deps'; -export * from './chart_data'; diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 5b4fe28ca5147..38bb3eb523d24 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -8972,22 +8972,13 @@ "xpack.dataVisualizer.dataGrid.showDistributionsTooltip": "分布を表示", "xpack.dataVisualizer.dataGrid.typeColumnName": "型", "xpack.dataVisualizer.dataGridChart.histogramNotAvailable": "グラフはサポートされていません。", - "xpack.dataVisualizer.dataGridChart.notEnoughData": "0個のドキュメントにフィールドが含まれます。", "xpack.dataVisualizer.dataGridChart.topCategoriesLegend": "上位 {maxChartColumns}/{cardinality} カテゴリ", "xpack.dataVisualizer.description": "CSV、NDJSON、またはログファイルをインポートします。", "xpack.dataVisualizer.fieldNameSelect": "フィールド名", "xpack.dataVisualizer.fieldStats.maxTitle": "最高", "xpack.dataVisualizer.fieldStats.medianTitle": "中間", "xpack.dataVisualizer.fieldStats.minTitle": "分", - "xpack.dataVisualizer.fieldTypeIcon.booleanTypeAriaLabel": "ブールタイプ", - "xpack.dataVisualizer.fieldTypeIcon.dateTypeAriaLabel": "日付タイプ", "xpack.dataVisualizer.fieldTypeIcon.fieldTypeTooltip": "{type} タイプ", - "xpack.dataVisualizer.fieldTypeIcon.geoPointTypeAriaLabel": "{geoPointParam} タイプ", - "xpack.dataVisualizer.fieldTypeIcon.ipTypeAriaLabel": "IP タイプ", - "xpack.dataVisualizer.fieldTypeIcon.keywordTypeAriaLabel": "キーワードタイプ", - "xpack.dataVisualizer.fieldTypeIcon.numberTypeAriaLabel": "数字タイプ", - "xpack.dataVisualizer.fieldTypeIcon.textTypeAriaLabel": "テキストタイプ", - "xpack.dataVisualizer.fieldTypeIcon.unknownTypeAriaLabel": "不明なタイプ", "xpack.dataVisualizer.fieldTypeSelect": "フィールド型", "xpack.dataVisualizer.file.aboutPanel.analyzingDataTitle": "データを分析中", "xpack.dataVisualizer.file.aboutPanel.selectOrDragAndDropFileDescription": "ファイルを選択するかドラッグ & ドロップしてください", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index f166dc1d92c59..d5a2172e24104 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -9055,7 +9055,6 @@ "xpack.dataVisualizer.dataGrid.showDistributionsTooltip": "显示分布", "xpack.dataVisualizer.dataGrid.typeColumnName": "类型", "xpack.dataVisualizer.dataGridChart.histogramNotAvailable": "不支持图表。", - "xpack.dataVisualizer.dataGridChart.notEnoughData": "0 个文档包含字段。", "xpack.dataVisualizer.dataGridChart.singleCategoryLegend": "{cardinality, plural, other {# 个类别}}", "xpack.dataVisualizer.dataGridChart.topCategoriesLegend": "{cardinality} 个类别中的排名前 {maxChartColumns} 个", "xpack.dataVisualizer.description": "导入您自己的 CSV、NDJSON 或日志文件。", @@ -9063,15 +9062,7 @@ "xpack.dataVisualizer.fieldStats.maxTitle": "最大值", "xpack.dataVisualizer.fieldStats.medianTitle": "中值", "xpack.dataVisualizer.fieldStats.minTitle": "最小值", - "xpack.dataVisualizer.fieldTypeIcon.booleanTypeAriaLabel": "布尔类型", - "xpack.dataVisualizer.fieldTypeIcon.dateTypeAriaLabel": "日期类型", "xpack.dataVisualizer.fieldTypeIcon.fieldTypeTooltip": "{type} 类型", - "xpack.dataVisualizer.fieldTypeIcon.geoPointTypeAriaLabel": "{geoPointParam} 类型", - "xpack.dataVisualizer.fieldTypeIcon.ipTypeAriaLabel": "IP 类型", - "xpack.dataVisualizer.fieldTypeIcon.keywordTypeAriaLabel": "关键字类型", - "xpack.dataVisualizer.fieldTypeIcon.numberTypeAriaLabel": "数字类型", - "xpack.dataVisualizer.fieldTypeIcon.textTypeAriaLabel": "文本类型", - "xpack.dataVisualizer.fieldTypeIcon.unknownTypeAriaLabel": "未知类型", "xpack.dataVisualizer.fieldTypeSelect": "字段类型", "xpack.dataVisualizer.file.aboutPanel.analyzingDataTitle": "正在分析数据", "xpack.dataVisualizer.file.aboutPanel.selectOrDragAndDropFileDescription": "选择或拖放文件", diff --git a/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts b/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts deleted file mode 100644 index 488df74b46968..0000000000000 --- a/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; - -import { FtrProviderContext } from '../../../ftr_provider_context'; -import { USER } from '../../../../functional/services/ml/security_common'; -import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api'; - -export default ({ getService }: FtrProviderContext) => { - const esArchiver = getService('esArchiver'); - const supertest = getService('supertestWithoutAuth'); - const ml = getService('ml'); - - const fieldHistogramsTestData = { - testTitle: 'returns histogram data for fields', - index: 'ft_farequote', - user: USER.ML_POWERUSER, - requestBody: { - query: { bool: { should: [{ match_phrase: { airline: 'JZA' } }], minimum_should_match: 1 } }, - fields: [ - { fieldName: '@timestamp', type: 'date' }, - { fieldName: 'airline', type: 'string' }, - { fieldName: 'responsetime', type: 'number' }, - ], - samplerShardSize: -1, // No sampling, as otherwise counts could vary on each run. - }, - expected: { - responseCode: 200, - responseBody: [ - { - dataLength: 20, - type: 'numeric', - id: '@timestamp', - }, - { type: 'ordinal', dataLength: 1, id: 'airline' }, - { - dataLength: 20, - type: 'numeric', - id: 'responsetime', - }, - ], - }, - }; - - const errorTestData = { - testTitle: 'returns error for index which does not exist', - index: 'ft_farequote_not_exists', - user: USER.ML_POWERUSER, - requestBody: { - query: { bool: { must: [{ match_all: {} }] } }, - fields: [{ fieldName: 'responsetime', type: 'number' }], - samplerShardSize: -1, - }, - expected: { - responseCode: 404, - responseBody: { - statusCode: 404, - error: 'Not Found', - message: 'index_not_found_exception', - }, - }, - }; - - async function runGetFieldHistogramsRequest( - index: string, - user: USER, - requestBody: object, - expectedResponsecode: number - ): Promise { - const { body } = await supertest - .post(`/api/ml/data_visualizer/get_field_histograms/${index}`) - .auth(user, ml.securityCommon.getPasswordForUser(user)) - .set(COMMON_REQUEST_HEADERS) - .send(requestBody) - .expect(expectedResponsecode); - - return body; - } - - describe('get_field_histograms', function () { - before(async () => { - await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); - await ml.testResources.setKibanaTimeZoneToUTC(); - }); - - it(`${fieldHistogramsTestData.testTitle}`, async () => { - const body = await runGetFieldHistogramsRequest( - fieldHistogramsTestData.index, - fieldHistogramsTestData.user, - fieldHistogramsTestData.requestBody, - fieldHistogramsTestData.expected.responseCode - ); - - const expected = fieldHistogramsTestData.expected; - - const actual = body.map((b: any) => ({ - dataLength: b.data.length, - type: b.type, - id: b.id, - })); - expect(actual).to.eql(expected.responseBody); - }); - - it(`${errorTestData.testTitle}`, async () => { - const body = await runGetFieldHistogramsRequest( - errorTestData.index, - errorTestData.user, - errorTestData.requestBody, - errorTestData.expected.responseCode - ); - - expect(body.error).to.eql(errorTestData.expected.responseBody.error); - expect(body.message).to.contain(errorTestData.expected.responseBody.message); - }); - }); -}; diff --git a/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_stats.ts b/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_stats.ts deleted file mode 100644 index 65fd1c1ff0c85..0000000000000 --- a/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_stats.ts +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { sortBy } from 'lodash'; - -import { FtrProviderContext } from '../../../ftr_provider_context'; -import { USER } from '../../../../functional/services/ml/security_common'; -import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api'; - -export default ({ getService }: FtrProviderContext) => { - const esArchiver = getService('esArchiver'); - const supertest = getService('supertestWithoutAuth'); - const ml = getService('ml'); - - const metricFieldsTestData = { - testTitle: 'returns stats for metric fields over all time', - index: 'ft_farequote', - user: USER.ML_POWERUSER, - requestBody: { - query: { - bool: { - must: { - term: { airline: 'JZA' }, // Only use one airline to ensure no sampling. - }, - }, - }, - fields: [ - { type: 'number', cardinality: 0 }, - { fieldName: 'responsetime', type: 'number', cardinality: 4249 }, - ], - samplerShardSize: -1, // No sampling, as otherwise counts could vary on each run. - timeFieldName: '@timestamp', - interval: 86400000, - maxExamples: 10, - }, - expected: { - responseCode: 200, - responseBody: [ - { - documentCounts: { - interval: 86400000, - buckets: { - '1454803200000': 846, - '1454889600000': 846, - '1454976000000': 859, - '1455062400000': 851, - '1455148800000': 858, - }, - }, - }, - { - // Cannot verify median and percentiles responses as the ES percentiles agg is non-deterministic. - fieldName: 'responsetime', - count: 4260, - min: 963.4293212890625, - max: 1042.13525390625, - avg: 1000.0378077547315, - isTopValuesSampled: false, - topValues: [ - { key: 980.0411987304688, doc_count: 2 }, - { key: 989.278076171875, doc_count: 2 }, - { key: 989.763916015625, doc_count: 2 }, - { key: 991.290771484375, doc_count: 2 }, - { key: 992.0765991210938, doc_count: 2 }, - { key: 993.8115844726562, doc_count: 2 }, - { key: 993.8973999023438, doc_count: 2 }, - { key: 994.0230102539062, doc_count: 2 }, - { key: 994.364990234375, doc_count: 2 }, - { key: 994.916015625, doc_count: 2 }, - ], - topValuesSampleSize: 4260, - topValuesSamplerShardSize: -1, - }, - ], - }, - }; - - const nonMetricFieldsTestData = { - testTitle: 'returns stats for non-metric fields specifying query and time range', - index: 'ft_farequote', - user: USER.ML_POWERUSER, - requestBody: { - query: { - bool: { - must: { - term: { airline: 'AAL' }, - }, - }, - }, - fields: [ - { fieldName: '@timestamp', type: 'date', cardinality: 4751 }, - { fieldName: '@version.keyword', type: 'keyword', cardinality: 1 }, - { fieldName: 'airline', type: 'keyword', cardinality: 19 }, - { fieldName: 'type', type: 'text', cardinality: 0 }, - { fieldName: 'type.keyword', type: 'keyword', cardinality: 1 }, - ], - samplerShardSize: -1, // No sampling, as otherwise counts would vary on each run. - timeFieldName: '@timestamp', - earliest: 1454889600000, // February 8, 2016 12:00:00 AM GMT - latest: 1454976000000, // February 9, 2016 12:00:00 AM GMT - maxExamples: 10, - }, - expected: { - responseCode: 200, - responseBody: [ - { fieldName: '@timestamp', count: 1733, earliest: 1454889602000, latest: 1454975948000 }, - { - fieldName: '@version.keyword', - isTopValuesSampled: false, - topValues: [{ key: '1', doc_count: 1733 }], - topValuesSampleSize: 1733, - topValuesSamplerShardSize: -1, - }, - { - fieldName: 'airline', - isTopValuesSampled: false, - topValues: [{ key: 'AAL', doc_count: 1733 }], - topValuesSampleSize: 1733, - topValuesSamplerShardSize: -1, - }, - { - fieldName: 'type.keyword', - isTopValuesSampled: false, - topValues: [{ key: 'farequote', doc_count: 1733 }], - topValuesSampleSize: 1733, - topValuesSamplerShardSize: -1, - }, - { fieldName: 'type', examples: ['farequote'] }, - ], - }, - }; - - const errorTestData = { - testTitle: 'returns error for index which does not exist', - index: 'ft_farequote_not_exists', - user: USER.ML_POWERUSER, - requestBody: { - query: { bool: { must: [{ match_all: {} }] } }, - fields: [ - { type: 'number', cardinality: 0 }, - { fieldName: 'responsetime', type: 'number', cardinality: 4249 }, - ], - samplerShardSize: -1, // No sampling, as otherwise counts could vary on each run. - timeFieldName: '@timestamp', - interval: 86400000, - maxExamples: 10, - }, - expected: { - responseCode: 404, - responseBody: { - statusCode: 404, - error: 'Not Found', - message: 'index_not_found_exception', - }, - }, - }; - - async function runGetFieldStatsRequest( - index: string, - user: USER, - requestBody: object, - expectedResponsecode: number - ): Promise { - const { body } = await supertest - .post(`/internal/data_visualizer/get_field_stats/${index}`) - .auth(user, ml.securityCommon.getPasswordForUser(user)) - .set(COMMON_REQUEST_HEADERS) - .send(requestBody) - .expect(expectedResponsecode); - - return body; - } - - // Move these tests to file_data_visualizer plugin - describe('get_field_stats', function () { - before(async () => { - await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); - await ml.testResources.setKibanaTimeZoneToUTC(); - }); - - it(`${metricFieldsTestData.testTitle}`, async () => { - const body = await runGetFieldStatsRequest( - metricFieldsTestData.index, - metricFieldsTestData.user, - metricFieldsTestData.requestBody, - metricFieldsTestData.expected.responseCode - ); - - // Cannot verify median and percentiles responses as the ES percentiles agg is non-deterministic. - const expected = metricFieldsTestData.expected; - expect(body).to.have.length(expected.responseBody.length); - - const actualDocCounts = body[0]; - const expectedDocCounts = expected.responseBody[0]; - expect(actualDocCounts).to.eql(expectedDocCounts); - - const actualFieldData = { ...body[1] }; - delete actualFieldData.median; - delete actualFieldData.distribution; - - expect(actualFieldData).to.eql(expected.responseBody[1]); - }); - - it(`${nonMetricFieldsTestData.testTitle}`, async () => { - const body = await runGetFieldStatsRequest( - nonMetricFieldsTestData.index, - nonMetricFieldsTestData.user, - nonMetricFieldsTestData.requestBody, - nonMetricFieldsTestData.expected.responseCode - ); - - const expectedRspFields = sortBy(nonMetricFieldsTestData.expected.responseBody, 'fieldName'); - const actualRspFields = sortBy(body, 'fieldName'); - expect(actualRspFields).to.eql(expectedRspFields); - }); - - it(`${errorTestData.testTitle}`, async () => { - const body = await runGetFieldStatsRequest( - errorTestData.index, - errorTestData.user, - errorTestData.requestBody, - errorTestData.expected.responseCode - ); - - expect(body.error).to.eql(errorTestData.expected.responseBody.error); - expect(body.message).to.contain(errorTestData.expected.responseBody.message); - }); - }); -}; diff --git a/x-pack/test/api_integration/apis/ml/data_visualizer/get_overall_stats.ts b/x-pack/test/api_integration/apis/ml/data_visualizer/get_overall_stats.ts deleted file mode 100644 index 7987875a75519..0000000000000 --- a/x-pack/test/api_integration/apis/ml/data_visualizer/get_overall_stats.ts +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; - -import { FtrProviderContext } from '../../../ftr_provider_context'; -import { USER } from '../../../../functional/services/ml/security_common'; -import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api'; - -export default ({ getService }: FtrProviderContext) => { - const esArchiver = getService('esArchiver'); - const supertest = getService('supertestWithoutAuth'); - const ml = getService('ml'); - - const testDataList = [ - { - testTitle: 'returns stats over all time', - index: 'ft_farequote', - user: USER.ML_POWERUSER, - requestBody: { - query: { bool: { must: [{ match_all: {} }] } }, - aggregatableFields: ['@timestamp', 'airline', 'responsetime', 'sourcetype'], - nonAggregatableFields: ['type'], - samplerShardSize: -1, // No sampling, as otherwise counts would vary on each run. - timeFieldName: '@timestamp', - }, - expected: { - responseCode: 200, - responseBody: { - totalCount: 86274, - aggregatableExistsFields: [ - { - fieldName: '@timestamp', - existsInDocs: true, - stats: { sampleCount: 86274, count: 86274, cardinality: 78580 }, - }, - { - fieldName: 'airline', - existsInDocs: true, - stats: { sampleCount: 86274, count: 86274, cardinality: 19 }, - }, - { - fieldName: 'responsetime', - existsInDocs: true, - stats: { sampleCount: 86274, count: 86274, cardinality: 83346 }, - }, - ], - aggregatableNotExistsFields: [{ fieldName: 'sourcetype', existsInDocs: false }], - nonAggregatableExistsFields: [{ fieldName: 'type', existsInDocs: true, stats: {} }], - nonAggregatableNotExistsFields: [], - errors: [], - }, - }, - }, - { - testTitle: 'returns stats when specifying query and time range', - index: 'ft_farequote', - user: USER.ML_POWERUSER, - requestBody: { - query: { - bool: { - must: { - term: { airline: 'AAL' }, - }, - }, - }, - aggregatableFields: ['@timestamp', 'airline', 'responsetime', 'sourcetype'], - nonAggregatableFields: ['type'], - samplerShardSize: -1, // No sampling, as otherwise counts would vary on each run. - timeFieldName: '@timestamp', - earliest: 1454889600000, // February 8, 2016 12:00:00 AM GMT - latest: 1454976000000, // February 9, 2016 12:00:00 AM GMT - }, - expected: { - responseCode: 200, - responseBody: { - totalCount: 1733, - aggregatableExistsFields: [ - { - fieldName: '@timestamp', - existsInDocs: true, - stats: { sampleCount: 1733, count: 1733, cardinality: 1713 }, - }, - { - fieldName: 'airline', - existsInDocs: true, - stats: { sampleCount: 1733, count: 1733, cardinality: 1 }, - }, - { - fieldName: 'responsetime', - existsInDocs: true, - stats: { sampleCount: 1733, count: 1733, cardinality: 1730 }, - }, - ], - aggregatableNotExistsFields: [{ fieldName: 'sourcetype', existsInDocs: false }], - nonAggregatableExistsFields: [{ fieldName: 'type', existsInDocs: true, stats: {} }], - nonAggregatableNotExistsFields: [], - errors: [], - }, - }, - }, - { - testTitle: 'returns error for index which does not exist', - index: 'ft_farequote_not_exist', - user: USER.ML_POWERUSER, - requestBody: { - query: { bool: { must: [{ match_all: {} }] } }, - aggregatableFields: ['@timestamp', 'airline', 'responsetime', 'sourcetype'], - nonAggregatableFields: ['@version', 'type'], - samplerShardSize: 1000, - timeFieldName: '@timestamp', - }, - expected: { - responseCode: 404, - responseBody: { - statusCode: 404, - error: 'Not Found', - message: 'index_not_found_exception', - }, - }, - }, - ]; - - // Move these tests to file_data_visualizer plugin - describe('get_overall_stats', function () { - before(async () => { - await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); - await ml.testResources.setKibanaTimeZoneToUTC(); - }); - - for (const testData of testDataList) { - it(`${testData.testTitle}`, async () => { - const { body } = await supertest - .post(`/internal/data_visualizer/get_overall_stats/${testData.index}`) - .auth(testData.user, ml.securityCommon.getPasswordForUser(testData.user)) - .set(COMMON_REQUEST_HEADERS) - .send(testData.requestBody) - .expect(testData.expected.responseCode); - - if (body.error === undefined) { - expect(body).to.eql(testData.expected.responseBody); - } else { - expect(body.error).to.eql(testData.expected.responseBody.error); - expect(body.message).to.contain(testData.expected.responseBody.message); - } - }); - } - }); -}; diff --git a/x-pack/test/api_integration/apis/ml/data_visualizer/index.ts b/x-pack/test/api_integration/apis/ml/data_visualizer/index.ts deleted file mode 100644 index 3865247979a4a..0000000000000 --- a/x-pack/test/api_integration/apis/ml/data_visualizer/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ loadTestFile }: FtrProviderContext) { - describe('data visualizer', function () { - loadTestFile(require.resolve('./get_field_stats')); - loadTestFile(require.resolve('./get_overall_stats')); - }); -} diff --git a/x-pack/test/api_integration/apis/ml/index.ts b/x-pack/test/api_integration/apis/ml/index.ts index 06910e8fac67e..f276cbe5355ca 100644 --- a/x-pack/test/api_integration/apis/ml/index.ts +++ b/x-pack/test/api_integration/apis/ml/index.ts @@ -74,7 +74,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./calendars')); loadTestFile(require.resolve('./datafeeds')); loadTestFile(require.resolve('./data_frame_analytics')); - loadTestFile(require.resolve('./data_visualizer')); loadTestFile(require.resolve('./fields_service')); loadTestFile(require.resolve('./filters')); loadTestFile(require.resolve('./indices')); diff --git a/x-pack/test/api_integration_basic/apis/index.ts b/x-pack/test/api_integration_basic/apis/index.ts index 27869095bd792..9490d4c277675 100644 --- a/x-pack/test/api_integration_basic/apis/index.ts +++ b/x-pack/test/api_integration_basic/apis/index.ts @@ -11,7 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { describe('apis', function () { this.tags('ciGroup11'); - loadTestFile(require.resolve('./ml')); loadTestFile(require.resolve('./transform')); loadTestFile(require.resolve('./security_solution')); }); diff --git a/x-pack/test/api_integration_basic/apis/ml/data_visualizer/index.ts b/x-pack/test/api_integration_basic/apis/ml/data_visualizer/index.ts deleted file mode 100644 index 85b462a01760b..0000000000000 --- a/x-pack/test/api_integration_basic/apis/ml/data_visualizer/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ loadTestFile }: FtrProviderContext) { - describe('data visualizer', function () { - // The data visualizer APIs should work the same as with a trial license - loadTestFile(require.resolve('../../../../api_integration/apis/ml/data_visualizer')); - }); -} diff --git a/x-pack/test/api_integration_basic/apis/ml/index.ts b/x-pack/test/api_integration_basic/apis/ml/index.ts deleted file mode 100644 index 5ca70103f41eb..0000000000000 --- a/x-pack/test/api_integration_basic/apis/ml/index.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../ftr_provider_context'; - -export default function ({ getService, loadTestFile }: FtrProviderContext) { - const esArchiver = getService('esArchiver'); - const ml = getService('ml'); - - describe('machine learning basic license', function () { - this.tags(['mlqa']); - - before(async () => { - await ml.securityCommon.createMlRoles(); - await ml.securityCommon.createMlUsers(); - }); - - after(async () => { - await ml.securityCommon.cleanMlUsers(); - await ml.securityCommon.cleanMlRoles(); - - await ml.testResources.deleteIndexPatternByTitle('ft_farequote'); - - await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote'); - - await ml.testResources.resetKibanaTimeZone(); - }); - - loadTestFile(require.resolve('./data_visualizer')); - }); -} diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_index_pattern_management.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_index_pattern_management.ts index c5461e3bb9c21..69a1edb403369 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_index_pattern_management.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_index_pattern_management.ts @@ -7,23 +7,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; import { ML_JOB_FIELD_TYPES } from '../../../../../plugins/ml/common/constants/field_types'; -import { FieldVisConfig } from '../../../../../plugins/data_visualizer/public/application/common/components/stats_table/types'; - -interface MetricFieldVisConfig extends FieldVisConfig { - statsMaxDecimalPlaces: number; - docCountFormatted: string; - topValuesCount: number; - viewableInLens: boolean; - hasActionMenu: boolean; -} - -interface NonMetricFieldVisConfig extends FieldVisConfig { - docCountFormatted: string; - exampleCount: number; - viewableInLens: boolean; - hasActionMenu: boolean; -} - +import { MetricFieldVisConfig, NonMetricFieldVisConfig } from './types'; interface TestData { suiteTitle: string; sourceIndexOrSavedSearch: string; diff --git a/x-pack/test/functional/apps/ml/data_visualizer/types.ts b/x-pack/test/functional/apps/ml/data_visualizer/types.ts index dc38bc31f568b..a5a016289b461 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/types.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/types.ts @@ -5,20 +5,24 @@ * 2.0. */ -import { FieldVisConfig } from '../../../../../plugins/data_visualizer/public/application/common/components/stats_table/types'; +import type { FieldVisConfig } from '../../../../../plugins/data_visualizer/public/application/common/components/stats_table/types'; export interface MetricFieldVisConfig extends FieldVisConfig { + fieldName: string; statsMaxDecimalPlaces: number; docCountFormatted: string; topValuesCount: number; viewableInLens: boolean; + hasActionMenu?: boolean; } export interface NonMetricFieldVisConfig extends FieldVisConfig { + fieldName: string; docCountFormatted: string; exampleCount: number; exampleContent?: string[]; viewableInLens: boolean; + hasActionMenu?: boolean; } export interface TestData {