From c7a35e618d76d71222ccf06387e1c8d24caf2ad8 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 19 Jul 2021 14:31:51 -0500 Subject: [PATCH 001/188] [ML] Initial embed --- .../components/layout/discover_layout.tsx | 20 ++ .../components/sidebar/discover_sidebar.tsx | 1 + .../data_visualizer_grid.tsx | 185 ++++++++++++++++++ .../components/data_visualizer_grid/index.ts | 9 + src/plugins/discover/public/build_services.ts | 3 + .../index_data_visualizer_view.tsx | 21 ++ .../embeddables/grid_embeddable/constants.ts | 8 + .../embeddable_loading_fallback.tsx | 20 ++ .../grid_embeddable/grid_embeddable.tsx | 96 +++++++++ .../grid_embeddable_factory.tsx | 99 ++++++++++ .../embeddables/grid_embeddable/index.ts | 0 .../embeddables/index.ts | 20 ++ .../plugins/data_visualizer/public/plugin.ts | 7 +- .../routes/new_job/index_or_search.tsx | 3 + 14 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx create mode 100644 src/plugins/discover/public/application/components/data_visualizer_grid/index.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/constants.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_loading_fallback.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index a10674323e5cb..f38c137b5441a 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -51,10 +51,13 @@ import { DiscoverUninitialized } from '../uninitialized/uninitialized'; import { SavedSearchDataMessage } from '../../services/use_saved_search'; import { useDataGridColumns } from '../../../../helpers/use_data_grid_columns'; import { FetchStatus } from '../../../../types'; +import { DiscoverDataVisualizerGrid } from '../../../../components/data_visualizer_grid'; const DocTableLegacyMemoized = React.memo(DocTableLegacy); const SidebarMemoized = React.memo(DiscoverSidebarResponsive); const DataGridMemoized = React.memo(DiscoverGrid); +const DataVisualizerGridMemoized = React.memo(DiscoverDataVisualizerGrid); + const TopNavMemoized = React.memo(DiscoverTopNav); const DiscoverChartMemoized = React.memo(DiscoverChart); @@ -337,6 +340,23 @@ export function DiscoverLayout({ defaultMessage="Documents" /> + {isLegacy && rows && rows.length && ( void; + /** + * Function to add a filter in the grid cell or document flyout + */ + onFilter: DocViewFilterFn; + /** + * Function used in the grid header and flyout to remove a column + * @param column + */ + onRemoveColumn: (column: string) => void; + /** + * Function triggered when a column is resized by the user + */ + onResize?: (colSettings: { columnId: string; width: number }) => void; + /** + * Function to set all columns + */ + onSetColumns: (columns: string[]) => void; + /** + * function to change sorting of the documents, skipped when isSortEnabled is set to false + */ + onSort?: (sort: string[][]) => void; + /** + * Array of documents provided by Elasticsearch + */ + rows?: ElasticSearchHit[]; + /** + * The max size of the documents returned by Elasticsearch + */ + sampleSize: number; + /** + * Function to set the expanded document, which is displayed in a flyout + */ + setExpandedDoc: (doc: ElasticSearchHit | undefined) => void; + /** + * Grid display settings persisted in Elasticsearch (e.g. column width) + */ + settings?: any; + /** + * Saved search description + */ + searchDescription?: string; + /** + * Saved search title + */ + searchTitle?: string; + /** + * Discover plugin services + */ + services: DiscoverServices; + /** + * Determines whether the time columns should be displayed (legacy settings) + */ + showTimeCol: boolean; + /** + * Manage user sorting control + */ + isSortEnabled?: boolean; + /** + * Current sort setting + */ + sort: SortPairArr[]; + /** + * How the data is fetched + */ + useNewFieldsApi: boolean; + /** + * Manage pagination control + */ + isPaginationEnabled?: boolean; + /** + * List of used control columns (available: 'openDetails', 'select') + */ + controlColumnIds?: string[]; +} + +export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { + return ; +}); + +export const DiscoverDataVisualizerGrid = ({ + ariaLabelledBy, + columns, + indexPattern, + isLoading, + expandedDoc, + onAddColumn, + onFilter, + onRemoveColumn, + onResize, + onSetColumns, + onSort, + rows, + sampleSize, + searchDescription, + searchTitle, + services, + setExpandedDoc, + settings, + showTimeCol, + sort, + useNewFieldsApi, + isSortEnabled = true, + isPaginationEnabled = true, + controlColumnIds = ['openDetails', 'select'], + className, +}: DiscoverDataVisualizerGridProps) => { + const [embeddable, setEmbeddable] = useState(); + const embeddableRoot: React.RefObject = useRef(null); + + useEffect(() => { + const loadEmbeddable = async () => { + if (services?.embeddable) { + const factory = services.embeddable.getEmbeddableFactory('data_visualizer_grid'); + if (factory) { + const test = await factory.create({ id: 'test' }); + setEmbeddable(test); + } + } + }; + loadEmbeddable(); + }, [services?.embeddable]); + + // We can only render after embeddable has already initialized + useEffect(() => { + if (embeddableRoot.current && embeddable) { + embeddable.render(embeddableRoot.current); + } + }, [embeddable, embeddableRoot]); + + return ( + <> +
+ Hello embeddable + + ); +}; diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/index.ts b/src/plugins/discover/public/application/components/data_visualizer_grid/index.ts new file mode 100644 index 0000000000000..dc85495a7c2ec --- /dev/null +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { DiscoverDataVisualizerGrid } from './data_visualizer_grid'; diff --git a/src/plugins/discover/public/build_services.ts b/src/plugins/discover/public/build_services.ts index b42bf6a81742c..5052d1755c921 100644 --- a/src/plugins/discover/public/build_services.ts +++ b/src/plugins/discover/public/build_services.ts @@ -36,6 +36,7 @@ import { KibanaLegacyStart } from '../../kibana_legacy/public'; import { UrlForwardingStart } from '../../url_forwarding/public'; import { NavigationPublicPluginStart } from '../../navigation/public'; import { IndexPatternFieldEditorStart } from '../../index_pattern_field_editor/public'; +import { EmbeddableStart } from '../../embeddable/public'; export interface DiscoverServices { addBasePath: (path: string) => string; @@ -44,6 +45,7 @@ export interface DiscoverServices { core: CoreStart; data: DataPublicPluginStart; docLinks: DocLinksStart; + embeddable: EmbeddableStart; history: () => History; theme: ChartsPluginStart['theme']; filterManager: FilterManager; @@ -84,6 +86,7 @@ export async function buildServices( core, data: plugins.data, docLinks: core.docLinks, + embeddable: plugins.embeddable, theme: plugins.charts.theme, filterManager: plugins.data.query.filterManager, getEmbeddableInjector, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index f45c4a89d006c..f96eff6068cec 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -800,6 +800,27 @@ export const IndexDataVisualizerView: FC = (dataVi const helpLink = docLinks.links.ml.guide; + if (true) { + return ( + <> + + + + + items={configs} + pageState={dataVisualizerListState} + updatePageState={setDataVisualizerListState} + getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} + extendedColumns={extendedColumns} + /> + + ); + } return ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/constants.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/constants.ts new file mode 100644 index 0000000000000..26004db8fd529 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE = 'data_visualizer_grid'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_loading_fallback.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_loading_fallback.tsx new file mode 100644 index 0000000000000..01644efd6652c --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_loading_fallback.tsx @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { EuiLoadingSpinner, EuiSpacer, EuiText } from '@elastic/eui'; + +export const EmbeddableLoading = () => { + return ( + + + + + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx new file mode 100644 index 0000000000000..381d9cf493e57 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Subject } from 'rxjs'; +import { CoreStart } from 'kibana/public'; +import ReactDOM from 'react-dom'; +import React, { Suspense } from 'react'; +import { + Embeddable, + EmbeddableInput, + EmbeddableOutput, + IContainer, +} from '../../../../../../../../src/plugins/embeddable/public'; +import { KibanaContextProvider } from '../../../../../../../../src/plugins/kibana_react/public'; +import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; +import { EmbeddableLoading } from './embeddable_loading_fallback'; +import { IndexDataVisualizerView } from '../../components/index_data_visualizer_view'; +import { DataVisualizerUrlStateContextProvider } from '../../index_data_visualizer'; +import { DataVisualizerPluginStart } from '../../../../plugin'; + +export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerPluginStart]; +export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { + id: string; +} +export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; + +export type IDataVisualizerGridEmbeddable = typeof DataVisualizerGridEmbeddable; + +export class DataVisualizerGridEmbeddable extends Embeddable< + DataVisualizerGridEmbeddableInput, + DataVisualizerGridEmbeddableOutput +> { + private node?: HTMLElement; + private reload$ = new Subject(); + public readonly type: string = DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE; + + constructor( + initialInput: DataVisualizerGridEmbeddableInput, + public services: DataVisualizerGridEmbeddableServices, + parent?: IContainer + ) { + super(initialInput, {}, parent); + this.initializeOutput(initialInput); + } + + private async initializeOutput(initialInput: DataVisualizerGridEmbeddableInput) { + try { + // do something + } catch (e) { + // Unable to find and load index pattern but we can ignore the error + // as we only load it to support the filter & query bar + // the visualizations should still work correctly + } + } + + public render(node: HTMLElement) { + super.render(node); + this.node = node; + + const I18nContext = this.services[0].i18n.Context; + + ReactDOM.render( + + + }> + Hello World + {/* */} + + + , + node + ); + } + + public destroy() { + super.destroy(); + if (this.node) { + ReactDOM.unmountComponentAtNode(this.node); + } + } + + public reload() { + this.reload$.next(); + } + + public supportedTriggers() { + return []; + } +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx new file mode 100644 index 0000000000000..8eb38fd29dbc1 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// import { StartServicesAccessor } from 'kibana/public'; +import { i18n } from '@kbn/i18n'; +import { StartServicesAccessor } from 'kibana/public'; +import { + EmbeddableFactoryDefinition, + IContainer, +} from '../../../../../../../../src/plugins/embeddable/public'; +import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; +import { + DataVisualizerGridEmbeddableInput, + DataVisualizerGridEmbeddableServices, +} from './grid_embeddable'; +import { DataVisualizerPluginStart, DataVisualizerStartDependencies } from '../../../../plugin'; + +export class DataVisualizerGridEmbeddableFactory + implements EmbeddableFactoryDefinition { + public readonly type = DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE; + + public readonly grouping = [ + { + id: 'data_visualizer_grid', + getDisplayName: () => 'Data Visualizer Grid', + }, + ]; + + constructor( + private getStartServices: StartServicesAccessor< + DataVisualizerStartDependencies, + DataVisualizerPluginStart + > + ) {} + + public async isEditable() { + return false; + } + + public getDisplayName() { + return i18n.translate('xpack.dataVisualizer.index.components.grid.displayName', { + defaultMessage: 'Data Visualizer Grid', + }); + } + + public getDescription() { + return i18n.translate('xpack.dataVisualizer.index.components.grid.description', { + defaultMessage: 'Visualize data', + }); + } + + public async getExplicitInput(): Promise> { + // const [coreStart] = await this.getServices(); + // + // try { + // const { resolveEmbeddableDataVisualizerGridUserInput } = await import( + // './anomaly_charts_setup_flyout' + // ); + // return await resolveEmbeddableDataVisualizerGridUserInput(coreStart); + // } catch (e) { + // return Promise.reject(); + // } + } + + private async getServices(): Promise { + const [coreStart, pluginsStart] = await this.getStartServices(); + console.log('getServices', pluginsStart); + + // + // const { AnomalyDetectorService } = await import( + // '../../application/services/anomaly_detector_service' + // ); + // const { mlApiServicesProvider } = await import('../../application/services/ml_api_service'); + // const { mlResultsServiceProvider } = await import('../../application/services/results_service'); + // + // const httpService = new HttpService(coreStart.http); + // const anomalyDetectorService = new AnomalyDetectorService(httpService); + // const mlApiServices = mlApiServicesProvider(httpService); + // const mlResultsService = mlResultsServiceProvider(mlApiServices); + // + // const anomalyExplorerService = new AnomalyExplorerChartsService( + // pluginsStart.data.query.timefilter.timefilter, + // mlApiServices, + // mlResultsService + // ); + // + return [coreStart, pluginsStart]; + } + + public async create(initialInput: DataVisualizerGridEmbeddableInput, parent?: IContainer) { + const [coreStart, pluginsStart] = await this.getServices(); + const { DataVisualizerGridEmbeddable } = await import('./grid_embeddable'); + return new DataVisualizerGridEmbeddable(initialInput, [coreStart, pluginsStart], parent); + } +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts new file mode 100644 index 0000000000000..ef10f5bccf202 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup } from 'kibana/public'; +import { EmbeddableSetup } from '../../../../../../../src/plugins/embeddable/public'; +import { DataVisualizerGridEmbeddableFactory } from './grid_embeddable/grid_embeddable_factory'; + +export function registerEmbeddables(embeddable: EmbeddableSetup, core: CoreSetup) { + const dataVisualizerGridEmbeddableFactory = new DataVisualizerGridEmbeddableFactory( + core.getStartServices + ); + embeddable.registerEmbeddableFactory( + dataVisualizerGridEmbeddableFactory.type, + dataVisualizerGridEmbeddableFactory + ); +} diff --git a/x-pack/plugins/data_visualizer/public/plugin.ts b/x-pack/plugins/data_visualizer/public/plugin.ts index 4b71b08e9cf27..e4313172e037c 100644 --- a/x-pack/plugins/data_visualizer/public/plugin.ts +++ b/x-pack/plugins/data_visualizer/public/plugin.ts @@ -6,7 +6,7 @@ */ import { CoreSetup, CoreStart } from 'kibana/public'; -import type { EmbeddableStart } from '../../../../src/plugins/embeddable/public'; +import type { EmbeddableSetup, EmbeddableStart } from '../../../../src/plugins/embeddable/public'; import type { SharePluginStart } from '../../../../src/plugins/share/public'; import { Plugin } from '../../../../src/core/public'; @@ -21,9 +21,11 @@ import type { IndexPatternFieldEditorStart } from '../../../../src/plugins/index import { getFileDataVisualizerComponent, getIndexDataVisualizerComponent } from './api'; import { getMaxBytesFormatted } from './application/common/util/get_max_bytes'; import { registerHomeAddData, registerHomeFeatureCatalogue } from './register_home'; +import { registerEmbeddables } from './application/index_data_visualizer/embeddables'; export interface DataVisualizerSetupDependencies { home?: HomePublicPluginSetup; + embeddable: EmbeddableSetup; } export interface DataVisualizerStartDependencies { data: DataPublicPluginStart; @@ -52,6 +54,9 @@ export class DataVisualizerPlugin registerHomeAddData(plugins.home); registerHomeFeatureCatalogue(plugins.home); } + if (plugins.embeddable) { + registerEmbeddables(plugins.embeddable, core); + } } public start(core: CoreStart, plugins: DataVisualizerStartDependencies) { diff --git a/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx b/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx index 5d903bd865911..5420fbbfbec12 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx @@ -81,8 +81,11 @@ const PageWrapper: FC = ({ nextStepPath, deps, mode }) = services: { http: { basePath }, application: { navigateToUrl }, + embeddable: embeddablePlugin, }, } = useMlKibana(); + + console.log('embeddablePlugin', embeddablePlugin); const { redirectToMlAccessDeniedPage } = deps; const redirectToJobsManagementPage = useCreateAndNavigateToMlLink( ML_PAGES.ANOMALY_DETECTION_JOBS_MANAGE From 5b48b4157a38b181e80424eee8ee75b7f29a707f Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 20 Jul 2021 09:16:34 -0500 Subject: [PATCH 002/188] [ML] Initial embed props --- .../components/layout/discover_layout.tsx | 13 +- .../data_visualizer_grid.tsx | 157 +++++------------- .../index_data_visualizer_view.tsx | 21 --- .../grid_embeddable/grid_embeddable.tsx | 38 +++-- .../grid_embeddable_factory.tsx | 33 ---- .../index_data_visualizer.tsx | 2 +- 6 files changed, 72 insertions(+), 192 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index f38c137b5441a..ea5f259314eca 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -341,21 +341,14 @@ export function DiscoverLayout({ /> {isLegacy && rows && rows.length && ( void; - /** - * Function to add a filter in the grid cell or document flyout - */ - onFilter: DocViewFilterFn; - /** - * Function used in the grid header and flyout to remove a column - * @param column - */ - onRemoveColumn: (column: string) => void; - /** - * Function triggered when a column is resized by the user - */ - onResize?: (colSettings: { columnId: string; width: number }) => void; - /** - * Function to set all columns - */ - onSetColumns: (columns: string[]) => void; - /** - * function to change sorting of the documents, skipped when isSortEnabled is set to false - */ - onSort?: (sort: string[][]) => void; - /** - * Array of documents provided by Elasticsearch - */ - rows?: ElasticSearchHit[]; /** * The max size of the documents returned by Elasticsearch */ sampleSize: number; - /** - * Function to set the expanded document, which is displayed in a flyout - */ - setExpandedDoc: (doc: ElasticSearchHit | undefined) => void; - /** - * Grid display settings persisted in Elasticsearch (e.g. column width) - */ - settings?: any; /** * Saved search description */ @@ -92,77 +43,56 @@ export interface DiscoverDataVisualizerGridProps { * Discover plugin services */ services: DiscoverServices; - /** - * Determines whether the time columns should be displayed (legacy settings) - */ - showTimeCol: boolean; - /** - * Manage user sorting control - */ - isSortEnabled?: boolean; - /** - * Current sort setting - */ - sort: SortPairArr[]; - /** - * How the data is fetched - */ - useNewFieldsApi: boolean; - /** - * Manage pagination control - */ - isPaginationEnabled?: boolean; - /** - * List of used control columns (available: 'openDetails', 'select') - */ - controlColumnIds?: string[]; + savedSearch?: SavedSearch; + query?: Query; } export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { return ; }); -export const DiscoverDataVisualizerGrid = ({ - ariaLabelledBy, - columns, - indexPattern, - isLoading, - expandedDoc, - onAddColumn, - onFilter, - onRemoveColumn, - onResize, - onSetColumns, - onSort, - rows, - sampleSize, - searchDescription, - searchTitle, - services, - setExpandedDoc, - settings, - showTimeCol, - sort, - useNewFieldsApi, - isSortEnabled = true, - isPaginationEnabled = true, - controlColumnIds = ['openDetails', 'select'], - className, -}: DiscoverDataVisualizerGridProps) => { - const [embeddable, setEmbeddable] = useState(); +export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { + const { services, indexPattern, savedSearch, query } = props; + const [embeddable, setEmbeddable] = useState< + | ErrorEmbeddable + | IEmbeddable + | undefined + >(); const embeddableRoot: React.RefObject = useRef(null); + useEffect(() => { + if (embeddable && !isErrorEmbeddable(embeddable)) { + // Update embeddable whenever one of the important input changes + embeddable.updateInput({ indexPattern, savedSearch, query }); + embeddable.reload(); + } + }, [embeddable, indexPattern, savedSearch, query]); + + useEffect(() => { + return () => { + // Clean up embeddable upon unmounting + if (embeddable) { + embeddable.destroy(); + } + }; + }, [embeddable]); + useEffect(() => { const loadEmbeddable = async () => { if (services?.embeddable) { - const factory = services.embeddable.getEmbeddableFactory('data_visualizer_grid'); + const factory = services.embeddable.getEmbeddableFactory< + DataVisualizerGridEmbeddableInput, + DataVisualizerGridEmbeddableOutput + >('data_visualizer_grid'); if (factory) { - const test = await factory.create({ id: 'test' }); + // Initialize embeddable with information available at mount + const test = await factory.create({ id: 'test', indexPattern, savedSearch, query }); setEmbeddable(test); } } }; loadEmbeddable(); + // eslint-disable-next-line react-hooks/exhaustive-deps }, [services?.embeddable]); // We can only render after embeddable has already initialized @@ -174,12 +104,7 @@ export const DiscoverDataVisualizerGrid = ({ return ( <> -
- Hello embeddable +
); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index f96eff6068cec..f45c4a89d006c 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -800,27 +800,6 @@ export const IndexDataVisualizerView: FC = (dataVi const helpLink = docLinks.links.ml.guide; - if (true) { - return ( - <> - - - - - items={configs} - pageState={dataVisualizerListState} - updatePageState={setDataVisualizerListState} - getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} - extendedColumns={extendedColumns} - /> - - ); - } return ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 381d9cf493e57..f647c51d8f3e8 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -5,10 +5,11 @@ * 2.0. */ -import { Subject } from 'rxjs'; +import { Observable, Subject } from 'rxjs'; import { CoreStart } from 'kibana/public'; import ReactDOM from 'react-dom'; import React, { Suspense } from 'react'; +import useObservable from 'react-use/lib/useObservable'; import { Embeddable, EmbeddableInput, @@ -18,18 +19,31 @@ import { import { KibanaContextProvider } from '../../../../../../../../src/plugins/kibana_react/public'; import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; import { EmbeddableLoading } from './embeddable_loading_fallback'; -import { IndexDataVisualizerView } from '../../components/index_data_visualizer_view'; -import { DataVisualizerUrlStateContextProvider } from '../../index_data_visualizer'; -import { DataVisualizerPluginStart } from '../../../../plugin'; - -export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerPluginStart]; +import { DataVisualizerStartDependencies } from '../../../../plugin'; +import { IndexPattern, Query } from '../../../../../../../../src/plugins/data/common'; +import { SavedSearch } from '../../../../../../../../src/plugins/discover/public'; +export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerStartDependencies]; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { - id: string; + indexPattern?: IndexPattern; + savedSearch?: SavedSearch; + query?: Query; } export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; export type IDataVisualizerGridEmbeddable = typeof DataVisualizerGridEmbeddable; +export const IndexDataVisualizerViewWrapper = (props: { + id: string; + embeddableContext: InstanceType; + embeddableInput: Readonly>; + // refresh: Observable; +}) => { + const { embeddableInput } = props; + + const data = useObservable(embeddableInput); + + return
Hello world 2
; +}; export class DataVisualizerGridEmbeddable extends Embeddable< DataVisualizerGridEmbeddableInput, DataVisualizerGridEmbeddableOutput @@ -68,10 +82,12 @@ export class DataVisualizerGridEmbeddable extends Embeddable< }> Hello World - {/* */} + , diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx index 8eb38fd29dbc1..1a08e1e16ac18 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx @@ -53,41 +53,8 @@ export class DataVisualizerGridEmbeddableFactory }); } - public async getExplicitInput(): Promise> { - // const [coreStart] = await this.getServices(); - // - // try { - // const { resolveEmbeddableDataVisualizerGridUserInput } = await import( - // './anomaly_charts_setup_flyout' - // ); - // return await resolveEmbeddableDataVisualizerGridUserInput(coreStart); - // } catch (e) { - // return Promise.reject(); - // } - } - private async getServices(): Promise { const [coreStart, pluginsStart] = await this.getStartServices(); - console.log('getServices', pluginsStart); - - // - // const { AnomalyDetectorService } = await import( - // '../../application/services/anomaly_detector_service' - // ); - // const { mlApiServicesProvider } = await import('../../application/services/ml_api_service'); - // const { mlResultsServiceProvider } = await import('../../application/services/results_service'); - // - // const httpService = new HttpService(coreStart.http); - // const anomalyDetectorService = new AnomalyDetectorService(httpService); - // const mlApiServices = mlApiServicesProvider(httpService); - // const mlResultsService = mlResultsServiceProvider(mlApiServices); - // - // const anomalyExplorerService = new AnomalyExplorerChartsService( - // pluginsStart.data.query.timefilter.timefilter, - // mlApiServices, - // mlResultsService - // ); - // return [coreStart, pluginsStart]; } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx index 8e0230a9bc6f9..25b382c835c84 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx @@ -51,6 +51,7 @@ export const DataVisualizerUrlStateContextProvider: FC( undefined @@ -58,7 +59,6 @@ export const DataVisualizerUrlStateContextProvider: FC | null>( null ); - const { search: searchString } = useLocation(); useEffect(() => { const prevSearchString = searchString; From a0801999cebec9c1f91ec09d298022354e6c12ab Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 20 Jul 2021 14:45:38 -0500 Subject: [PATCH 003/188] [ML] Add top nav link to data viz --- .../top_nav/discover_topnav.test.tsx | 12 +- .../top_nav/get_top_nav_links.test.ts | 7 + .../components/top_nav/get_top_nav_links.ts | 62 +++++++++ .../index_data_visualizer/locator/index.ts | 8 ++ .../locator/locator.test.ts | 111 +++++++++++++++ .../index_data_visualizer/locator/locator.ts | 130 ++++++++++++++++++ .../plugins/data_visualizer/public/plugin.ts | 13 ++ .../ml/public/locator/ml_locator.test.ts | 2 +- 8 files changed, 342 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/index.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.test.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx index 687532cd94f08..1a68bc4948c3a 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx @@ -41,13 +41,21 @@ describe('Discover topnav component', () => { const props = getProps(true); const component = shallowWithIntl(); const topMenuConfig = component.props().config.map((obj: TopNavMenuData) => obj.id); - expect(topMenuConfig).toEqual(['options', 'new', 'save', 'open', 'share', 'inspect']); + expect(topMenuConfig).toEqual([ + 'options', + 'new', + 'save', + 'open', + 'share', + 'inspect', + 'dataVisualizer', + ]); }); test('generated config of TopNavMenu config is correct when no discover save permissions are assigned', () => { const props = getProps(false); const component = shallowWithIntl(); const topMenuConfig = component.props().config.map((obj: TopNavMenuData) => obj.id); - expect(topMenuConfig).toEqual(['options', 'new', 'open', 'share', 'inspect']); + expect(topMenuConfig).toEqual(['options', 'new', 'open', 'share', 'inspect', 'dataVisualizer']); }); }); diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts index 6a6fb8a44a5cf..fae1c54b07211 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts @@ -80,6 +80,13 @@ test('getTopNavLinks result', () => { "run": [Function], "testId": "openInspectorButton", }, + Object { + "description": "Open in Data visualizer", + "id": "dataVisualizer", + "label": "Data visualizer", + "run": [Function], + "testId": "dataVisualizerTopNavButton", + }, ] `); }); diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts index f19b30cda5f8a..e44d1d5e3213f 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts @@ -8,6 +8,12 @@ import { i18n } from '@kbn/i18n'; import moment from 'moment'; +import { + fromKueryExpression, + toElasticsearchQuery, + luceneStringToDsl, + decorateQuery, +} from '@kbn/es-query'; import { showOpenSearchPanel } from './show_open_search_panel'; import { getSharingData, showPublicUrlSwitch } from '../../utils/get_sharing_data'; import { unhashUrl } from '../../../../../../../kibana_utils/public'; @@ -17,6 +23,8 @@ import { onSaveSearch } from './on_save_search'; import { GetStateReturn } from '../../services/discover_state'; import { IndexPattern, ISearchSource } from '../../../../../kibana_services'; import { openOptionsPopover } from './open_options_popover'; +import { RefreshInterval, UI_SETTINGS } from '../../../../../../../data/public'; +import { SerializableState } from '../../../../../../../kibana_utils/common'; /** * Helper function to build the top nav links @@ -149,6 +157,59 @@ export const getTopNavLinks = ({ }, }; + const dataVisualizer = { + id: 'dataVisualizer', + label: i18n.translate('discover.localMenu.dataVisualizerTitle', { + defaultMessage: 'Data visualizer', + }), + description: i18n.translate('discover.localMenu.dataVisualizerDescription', { + defaultMessage: 'Open in Data visualizer', + }), + testId: 'dataVisualizerTopNavButton', + run: async () => { + if (!services?.share?.url.locators) { + return; + } + const dvUrlGenerator = services.share.url.locators.get('DATA_VISUALIZER_APP_LOCATOR'); + const extractedQuery = state.appStateContainer.getState().query; + const timeRange = services.timefilter.getTime(); + const refreshInterval = services.timefilter.getRefreshInterval() as RefreshInterval & + SerializableState; + + const params: SerializableState = { + indexPatternId: indexPattern.id, + savedSearchId: savedSearch.id, + timeRange, + refreshInterval, + }; + if (extractedQuery) { + const queryLanguage = extractedQuery.language; + const qryString = extractedQuery.query; + let qry; + if (queryLanguage === 'kuery') { + const ast = fromKueryExpression(qryString); + qry = toElasticsearchQuery(ast, indexPattern); + } else { + qry = luceneStringToDsl(qryString); + decorateQuery(qry, services.uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS)); + } + + params.query = { + searchQuery: qry as SerializableState, + searchString: qryString, + searchQueryLanguage: queryLanguage, + }; + } + + const url = await dvUrlGenerator?.getUrl(params, { absolute: true }); + + // We want to open the Data visualizer in a new tab + if (url !== undefined) { + window.open(url, '_blank'); + } + }, + }; + return [ ...(services.capabilities.advancedSettings.save ? [options] : []), newSearch, @@ -156,5 +217,6 @@ export const getTopNavLinks = ({ openSearch, shareSearch, inspectSearch, + dataVisualizer, ]; }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/index.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/index.ts new file mode 100644 index 0000000000000..fb3e0100bbf75 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './locator'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.test.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.test.ts new file mode 100644 index 0000000000000..5fdbcea12006c --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.test.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndexDataVisualizerLocatorDefinition } from './locator'; + +describe('Index data visualizer locator', () => { + const definition = new IndexDataVisualizerLocatorDefinition(); + + it('should generate valid URL for the Index Data Visualizer Viewer page with global settings', async () => { + const location = await definition.getLocation({ + indexPatternId: '3da93760-e0af-11ea-9ad3-3bcfc330e42a', + timeRange: { + from: 'now-30m', + to: 'now', + }, + refreshInterval: { pause: false, value: 300 }, + }); + + expect(location).toMatchObject({ + app: 'ml', + path: + '/jobs/new_job/datavisualizer?index=3da93760-e0af-11ea-9ad3-3bcfc330e42a&_a=(DATA_VISUALIZER_INDEX_VIEWER:())&_g=(refreshInterval:(pause:!f,value:300),time:(from:now-30m,to:now))', + state: {}, + }); + }); + + it('should prioritize savedSearchId even when index pattern id is available', async () => { + const location = await definition.getLocation({ + indexPatternId: '3da93760-e0af-11ea-9ad3-3bcfc330e42a', + savedSearchId: '45014020-dffa-11eb-b120-a105fbbe93b3', + }); + + expect(location).toMatchObject({ + app: 'ml', + path: + '/jobs/new_job/datavisualizer?savedSearchId=45014020-dffa-11eb-b120-a105fbbe93b3&_a=(DATA_VISUALIZER_INDEX_VIEWER:())&_g=()', + state: {}, + }); + }); + + it('should generate valid URL with field names and field types', async () => { + const location = await definition.getLocation({ + indexPatternId: '3da93760-e0af-11ea-9ad3-3bcfc330e42a', + visibleFieldNames: ['@timestamp', 'responsetime'], + visibleFieldTypes: ['number'], + }); + + expect(location).toMatchObject({ + app: 'ml', + path: + "/jobs/new_job/datavisualizer?_a=(DATA_VISUALIZER_INDEX_VIEWER:(visibleFieldNames:!('@timestamp',responsetime),visibleFieldTypes:!(number)))&_g=()", + state: {}, + }); + }); + + it('should generate valid URL with KQL query', async () => { + const location = await definition.getLocation({ + indexPatternId: '3da93760-e0af-11ea-9ad3-3bcfc330e42a', + query: { + searchQuery: { + bool: { + should: [ + { + match: { + region: 'ap-northwest-1', + }, + }, + ], + minimum_should_match: 1, + }, + }, + searchString: 'region : ap-northwest-1', + searchQueryLanguage: 'kuery', + }, + }); + + expect(location).toMatchObject({ + app: 'ml', + path: + "/jobs/new_job/datavisualizer?index=3da93760-e0af-11ea-9ad3-3bcfc330e42a&_a=(DATA_VISUALIZER_INDEX_VIEWER:(searchQuery:(bool:(minimum_should_match:1,should:!((match:(region:ap-northwest-1))))),searchQueryLanguage:kuery,searchString:'region : ap-northwest-1'))&_g=()", + state: {}, + }); + }); + + it('should generate valid URL with Lucene query', async () => { + const location = await definition.getLocation({ + indexPatternId: '3da93760-e0af-11ea-9ad3-3bcfc330e42a', + query: { + searchQuery: { + query_string: { + query: 'region: ap-northwest-1', + analyze_wildcard: true, + }, + }, + searchString: 'region : ap-northwest-1', + searchQueryLanguage: 'lucene', + }, + }); + + expect(location).toMatchObject({ + app: 'ml', + path: + "/jobs/new_job/datavisualizer?index=3da93760-e0af-11ea-9ad3-3bcfc330e42a&_a=(DATA_VISUALIZER_INDEX_VIEWER:(searchQuery:(query_string:(analyze_wildcard:!t,query:'region: ap-northwest-1')),searchQueryLanguage:lucene,searchString:'region : ap-northwest-1'))&_g=()", + state: {}, + }); + }); +}); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts new file mode 100644 index 0000000000000..e64fbb8e02ec1 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// @ts-ignore +import { encode } from 'rison-node'; +import { stringify } from 'query-string'; +import { SerializableState } from '../../../../../../../src/plugins/kibana_utils/common'; +import { RefreshInterval, TimeRange } from '../../../../../../../src/plugins/data/common'; +import { LocatorDefinition, LocatorPublic } from '../../../../../../../src/plugins/share/common'; +import { QueryState } from '../../../../../../../src/plugins/data/public'; +import { Dictionary, isRisonSerializationRequired } from '../../common/util/url_state'; +import { SearchQueryLanguage } from '../types/combined_query'; + +export const DATA_VISUALIZER_APP_LOCATOR = 'DATA_VISUALIZER_APP_LOCATOR'; + +export interface IndexDataVisualizerLocatorParams extends SerializableState { + /** + * Optionally set saved search ID. + */ + savedSearchId?: string; + + /** + * Optionally set index pattern ID. + */ + indexPatternId?: string; + + /** + * Optionally set the time range in the time picker. + */ + timeRange?: TimeRange; + + /** + * Optionally set the refresh interval. + */ + refreshInterval?: RefreshInterval & SerializableState; + + /** + * Optionally set a query. + */ + query?: { + searchQuery: SerializableState; + searchString: string | SerializableState; + searchQueryLanguage: SearchQueryLanguage; + }; + + /** + * If not given, will use the uiSettings configuration for `storeInSessionStorage`. useHash determines + * whether to hash the data in the url to avoid url length issues. + */ + useHash?: boolean; + /** + * Optionally set visible field names. + */ + visibleFieldNames?: string[]; + /** + * Optionally set visible field types. + */ + visibleFieldTypes?: string[]; +} + +export type IndexDataVisualizerLocator = LocatorPublic; + +export class IndexDataVisualizerLocatorDefinition + implements LocatorDefinition { + public readonly id = DATA_VISUALIZER_APP_LOCATOR; + + constructor() {} + + public readonly getLocation = async (params: IndexDataVisualizerLocatorParams) => { + const { + indexPatternId, + query, + refreshInterval, + savedSearchId, + timeRange, + visibleFieldNames, + visibleFieldTypes, + } = params; + + const appState: { + searchQuery?: { [key: string]: any }; + searchQueryLanguage?: string; + searchString?: string | SerializableState; + visibleFieldNames?: string[]; + visibleFieldTypes?: string[]; + } = {}; + const queryState: QueryState = {}; + + if (query) { + appState.searchQuery = query.searchQuery; + appState.searchString = query.searchString; + appState.searchQueryLanguage = query.searchQueryLanguage; + } + if (visibleFieldNames) appState.visibleFieldNames = visibleFieldNames; + if (visibleFieldTypes) appState.visibleFieldTypes = visibleFieldTypes; + + if (timeRange) queryState.time = timeRange; + if (refreshInterval) queryState.refreshInterval = refreshInterval; + + const urlState: Dictionary = { + ...(savedSearchId ? { savedSearchId } : { index: indexPatternId }), + _a: { DATA_VISUALIZER_INDEX_VIEWER: appState }, + _g: queryState, + }; + + const parsedQueryString: Dictionary = {}; + Object.keys(urlState).forEach((a) => { + if (isRisonSerializationRequired(a)) { + parsedQueryString[a] = encode(urlState[a]); + } else { + parsedQueryString[a] = urlState[a]; + } + }); + const newLocationSearchString = stringify(parsedQueryString, { + sort: false, + encode: false, + }); + + const path = `/jobs/new_job/datavisualizer?${newLocationSearchString}`; + return { + app: 'ml', + path, + state: {}, + }; + }; +} diff --git a/x-pack/plugins/data_visualizer/public/plugin.ts b/x-pack/plugins/data_visualizer/public/plugin.ts index 4b71b08e9cf27..565013fc013be 100644 --- a/x-pack/plugins/data_visualizer/public/plugin.ts +++ b/x-pack/plugins/data_visualizer/public/plugin.ts @@ -21,8 +21,14 @@ import type { IndexPatternFieldEditorStart } from '../../../../src/plugins/index import { getFileDataVisualizerComponent, getIndexDataVisualizerComponent } from './api'; import { getMaxBytesFormatted } from './application/common/util/get_max_bytes'; import { registerHomeAddData, registerHomeFeatureCatalogue } from './register_home'; +import { + IndexDataVisualizerLocator, + IndexDataVisualizerLocatorDefinition, +} from './application/index_data_visualizer/locator'; +import { SharePluginSetup } from '../../../../src/plugins/share/public'; export interface DataVisualizerSetupDependencies { + share?: SharePluginSetup; home?: HomePublicPluginSetup; } export interface DataVisualizerStartDependencies { @@ -47,11 +53,17 @@ export class DataVisualizerPlugin DataVisualizerSetupDependencies, DataVisualizerStartDependencies > { + private locator?: IndexDataVisualizerLocator; + public setup(core: CoreSetup, plugins: DataVisualizerSetupDependencies) { if (plugins.home) { registerHomeAddData(plugins.home); registerHomeFeatureCatalogue(plugins.home); } + + if (plugins.share) { + this.locator = plugins.share.url.locators.create(new IndexDataVisualizerLocatorDefinition()); + } } public start(core: CoreStart, plugins: DataVisualizerStartDependencies) { @@ -60,6 +72,7 @@ export class DataVisualizerPlugin getFileDataVisualizerComponent, getIndexDataVisualizerComponent, getMaxBytesFormatted, + locator: this.locator, }; } } diff --git a/x-pack/plugins/ml/public/locator/ml_locator.test.ts b/x-pack/plugins/ml/public/locator/ml_locator.test.ts index 97bb6cb2966f2..c033e00dea70f 100644 --- a/x-pack/plugins/ml/public/locator/ml_locator.test.ts +++ b/x-pack/plugins/ml/public/locator/ml_locator.test.ts @@ -9,7 +9,7 @@ import { MlLocatorDefinition } from './ml_locator'; import { ML_PAGES } from '../../common/constants/locator'; import { ANALYSIS_CONFIG_TYPE } from '../../common/constants/data_frame_analytics'; -describe('MlUrlGenerator', () => { +describe('ML locator', () => { const definition = new MlLocatorDefinition(); describe('AnomalyDetection', () => { From c570d425dfc20a2cf8b1191a6562040c93349bef Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 21 Jul 2021 15:47:30 -0500 Subject: [PATCH 004/188] Add visible fields --- .../main/components/layout/discover_layout.tsx | 1 + .../main/components/top_nav/discover_topnav.tsx | 14 +++++++++++++- .../main/components/top_nav/get_top_nav_links.ts | 6 ++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index a10674323e5cb..954e17fd78dac 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -231,6 +231,7 @@ export function DiscoverLayout({ services={services} stateContainer={stateContainer} updateQuery={onUpdateQuery} + columns={columns} />

diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx index 9afda73401084..6d973a6f83ea9 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx @@ -21,6 +21,7 @@ export type DiscoverTopNavProps = Pick< savedQuery?: string; updateQuery: (payload: { dateRange: TimeRange; query?: Query }, isUpdate?: boolean) => void; stateContainer: GetStateReturn; + columns: string[]; }; export const DiscoverTopNav = ({ @@ -34,6 +35,7 @@ export const DiscoverTopNav = ({ navigateTo, savedSearch, services, + columns, }: DiscoverTopNavProps) => { const showDatePicker = useMemo(() => indexPattern.isTimeBased(), [indexPattern]); const { TopNavMenu } = services.navigation.ui; @@ -47,8 +49,18 @@ export const DiscoverTopNav = ({ state: stateContainer, onOpenInspector, searchSource, + columns, }), - [indexPattern, navigateTo, onOpenInspector, searchSource, stateContainer, savedSearch, services] + [ + columns, + indexPattern, + navigateTo, + onOpenInspector, + searchSource, + stateContainer, + savedSearch, + services, + ] ); const updateSavedQueryId = (newSavedQueryId: string | undefined) => { diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts index e44d1d5e3213f..d2a72dbcdcdb3 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts @@ -37,6 +37,7 @@ export const getTopNavLinks = ({ state, onOpenInspector, searchSource, + columns, }: { indexPattern: IndexPattern; navigateTo: (url: string) => void; @@ -45,6 +46,7 @@ export const getTopNavLinks = ({ state: GetStateReturn; onOpenInspector: () => void; searchSource: ISearchSource; + columns: string[]; }) => { const options = { id: 'options', @@ -182,6 +184,10 @@ export const getTopNavLinks = ({ timeRange, refreshInterval, }; + + if (columns) { + params.visibleFieldNames = columns; + } if (extractedQuery) { const queryLanguage = extractedQuery.language; const qryString = extractedQuery.query; From b2dd8cbedaf1a8de0c2a48fae2a7bacd326ff96e Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 22 Jul 2021 13:33:12 -0500 Subject: [PATCH 005/188] Add add data service to register links --- .../components/top_nav/discover_topnav.tsx | 62 ++++++++++-- .../components/top_nav/get_top_nav_links.ts | 78 ++------------- .../apps/main/services/add_top_nav_links.ts | 44 +++++++++ src/plugins/discover/public/build_services.ts | 6 +- src/plugins/discover/public/plugin.tsx | 12 ++- .../services/discover_top_nav_link.ts | 96 +++++++++++++++++++ .../plugins/data_visualizer/public/plugin.ts | 11 +++ 7 files changed, 231 insertions(+), 78 deletions(-) create mode 100644 src/plugins/discover/public/application/apps/main/services/add_top_nav_links.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx index 6d973a6f83ea9..1c7a17cce747c 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx @@ -5,9 +5,9 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import React, { useMemo } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { DiscoverLayoutProps } from '../layout/types'; -import { getTopNavLinks } from './get_top_nav_links'; +import { getTopNavLinks, TopNavLink } from './get_top_nav_links'; import { Query, TimeRange } from '../../../../../../../data/common/query'; import { getHeaderActionMenuMounter } from '../../../../../kibana_services'; import { GetStateReturn } from '../../services/discover_state'; @@ -37,11 +37,60 @@ export const DiscoverTopNav = ({ services, columns, }: DiscoverTopNavProps) => { + const [registeredTopNavLinks, setRegisteredTopNavLinks] = useState([]); const showDatePicker = useMemo(() => indexPattern.isTimeBased(), [indexPattern]); const { TopNavMenu } = services.navigation.ui; + + useEffect(() => { + let unmounted = false; + + const loadRegisteredTopNavLinks = async () => { + const callbacks = services.addDataService.getTopNavLinks(); + const params = { + indexPattern, + onOpenInspector, + query, + savedQuery, + stateContainer, + updateQuery, + searchSource, + navigateTo, + savedSearch, + services, + columns, + }; + const links = await Promise.all( + callbacks.map((cb) => { + return cb(params); + }) + ); + if (!unmounted) { + setRegisteredTopNavLinks(links); + } + }; + loadRegisteredTopNavLinks(); + + return () => { + unmounted = true; + }; + }, [ + services?.addDataService, + indexPattern, + onOpenInspector, + query, + savedQuery, + stateContainer, + updateQuery, + searchSource, + navigateTo, + savedSearch, + services, + columns, + ]); + const topNavMenu = useMemo( - () => - getTopNavLinks({ + () => [ + ...getTopNavLinks({ indexPattern, navigateTo, savedSearch, @@ -49,10 +98,10 @@ export const DiscoverTopNav = ({ state: stateContainer, onOpenInspector, searchSource, - columns, }), + ...registeredTopNavLinks, + ], [ - columns, indexPattern, navigateTo, onOpenInspector, @@ -60,6 +109,7 @@ export const DiscoverTopNav = ({ stateContainer, savedSearch, services, + registeredTopNavLinks, ] ); diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts index d2a72dbcdcdb3..7e7f2f9ee057b 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts @@ -8,12 +8,6 @@ import { i18n } from '@kbn/i18n'; import moment from 'moment'; -import { - fromKueryExpression, - toElasticsearchQuery, - luceneStringToDsl, - decorateQuery, -} from '@kbn/es-query'; import { showOpenSearchPanel } from './show_open_search_panel'; import { getSharingData, showPublicUrlSwitch } from '../../utils/get_sharing_data'; import { unhashUrl } from '../../../../../../../kibana_utils/public'; @@ -23,8 +17,14 @@ import { onSaveSearch } from './on_save_search'; import { GetStateReturn } from '../../services/discover_state'; import { IndexPattern, ISearchSource } from '../../../../../kibana_services'; import { openOptionsPopover } from './open_options_popover'; -import { RefreshInterval, UI_SETTINGS } from '../../../../../../../data/public'; -import { SerializableState } from '../../../../../../../kibana_utils/common'; + +export interface TopNavLink { + id: string; + label: string; + description: string; + run: (anchorElement: HTMLElement) => void; + testId: string; +} /** * Helper function to build the top nav links @@ -37,7 +37,6 @@ export const getTopNavLinks = ({ state, onOpenInspector, searchSource, - columns, }: { indexPattern: IndexPattern; navigateTo: (url: string) => void; @@ -46,8 +45,7 @@ export const getTopNavLinks = ({ state: GetStateReturn; onOpenInspector: () => void; searchSource: ISearchSource; - columns: string[]; -}) => { +}): TopNavLink[] => { const options = { id: 'options', label: i18n.translate('discover.localMenu.localMenu.optionsTitle', { @@ -159,63 +157,6 @@ export const getTopNavLinks = ({ }, }; - const dataVisualizer = { - id: 'dataVisualizer', - label: i18n.translate('discover.localMenu.dataVisualizerTitle', { - defaultMessage: 'Data visualizer', - }), - description: i18n.translate('discover.localMenu.dataVisualizerDescription', { - defaultMessage: 'Open in Data visualizer', - }), - testId: 'dataVisualizerTopNavButton', - run: async () => { - if (!services?.share?.url.locators) { - return; - } - const dvUrlGenerator = services.share.url.locators.get('DATA_VISUALIZER_APP_LOCATOR'); - const extractedQuery = state.appStateContainer.getState().query; - const timeRange = services.timefilter.getTime(); - const refreshInterval = services.timefilter.getRefreshInterval() as RefreshInterval & - SerializableState; - - const params: SerializableState = { - indexPatternId: indexPattern.id, - savedSearchId: savedSearch.id, - timeRange, - refreshInterval, - }; - - if (columns) { - params.visibleFieldNames = columns; - } - if (extractedQuery) { - const queryLanguage = extractedQuery.language; - const qryString = extractedQuery.query; - let qry; - if (queryLanguage === 'kuery') { - const ast = fromKueryExpression(qryString); - qry = toElasticsearchQuery(ast, indexPattern); - } else { - qry = luceneStringToDsl(qryString); - decorateQuery(qry, services.uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS)); - } - - params.query = { - searchQuery: qry as SerializableState, - searchString: qryString, - searchQueryLanguage: queryLanguage, - }; - } - - const url = await dvUrlGenerator?.getUrl(params, { absolute: true }); - - // We want to open the Data visualizer in a new tab - if (url !== undefined) { - window.open(url, '_blank'); - } - }, - }; - return [ ...(services.capabilities.advancedSettings.save ? [options] : []), newSearch, @@ -223,6 +164,5 @@ export const getTopNavLinks = ({ openSearch, shareSearch, inspectSearch, - dataVisualizer, ]; }; diff --git a/src/plugins/discover/public/application/apps/main/services/add_top_nav_links.ts b/src/plugins/discover/public/application/apps/main/services/add_top_nav_links.ts new file mode 100644 index 0000000000000..7ab9d8bdb56eb --- /dev/null +++ b/src/plugins/discover/public/application/apps/main/services/add_top_nav_links.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { DiscoverTopNavProps } from '../components/top_nav/discover_topnav'; +import type { TopNavMenuData } from '../../../../../../navigation/public'; + +/** + * Async callback function that generates a TopNavMenuData object + * given the arguments provided + */ +export type AddNavLinkCallback = (params: DiscoverTopNavProps) => Promise; + +/** + * Service for other plugins to register additional links or actions + * within Discover (e.g. top navigation bar) + */ +export class AddDataService { + private TopNavMenuDatas: Record = {}; + + public setup() { + return { + /** + * Registers an async callback function that will return a valid top nav link action + */ + registerTopNavLinks: (id: string, callbackFn: AddNavLinkCallback) => { + if (this.TopNavMenuDatas[id]) { + throw new Error(`link ${id} already exists`); + } + this.TopNavMenuDatas[id] = callbackFn; + }, + }; + } + + public getTopNavMenuDatas() { + return Object.values(this.TopNavMenuDatas); + } +} + +export type AddDataServiceSetup = ReturnType; diff --git a/src/plugins/discover/public/build_services.ts b/src/plugins/discover/public/build_services.ts index b42bf6a81742c..69e433a4e491b 100644 --- a/src/plugins/discover/public/build_services.ts +++ b/src/plugins/discover/public/build_services.ts @@ -36,9 +36,11 @@ import { KibanaLegacyStart } from '../../kibana_legacy/public'; import { UrlForwardingStart } from '../../url_forwarding/public'; import { NavigationPublicPluginStart } from '../../navigation/public'; import { IndexPatternFieldEditorStart } from '../../index_pattern_field_editor/public'; +import { AddDataService } from './application/apps/main/services/add_top_nav_links'; export interface DiscoverServices { addBasePath: (path: string) => string; + addDataService: AddDataService; capabilities: Capabilities; chrome: ChromeStart; core: CoreStart; @@ -68,7 +70,8 @@ export async function buildServices( core: CoreStart, plugins: DiscoverStartPlugins, context: PluginInitializerContext, - getEmbeddableInjector: () => Promise + getEmbeddableInjector: () => Promise, + addDataService: AddDataService ): Promise { const services = { savedObjectsClient: core.savedObjects.client, @@ -78,6 +81,7 @@ export async function buildServices( const { usageCollection } = plugins; return { + addDataService, addBasePath: core.http.basePath.prepend, capabilities: core.application.capabilities, chrome: core.chrome, diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index 1e8a5cdac95ef..7ba9a1b76dbd6 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -64,6 +64,10 @@ import { UsageCollectionSetup } from '../../usage_collection/public'; import { replaceUrlHashQuery } from '../../kibana_utils/public/'; import { IndexPatternFieldEditorStart } from '../../../plugins/index_pattern_field_editor/public'; import { SourceViewer } from './application/components/source_viewer/source_viewer'; +import { + AddDataService, + AddDataServiceSetup, +} from './application/apps/main/services/add_top_nav_links'; declare module '../../share/public' { export interface UrlGeneratorStateMapping { @@ -115,6 +119,8 @@ export interface DiscoverSetup { * ``` */ readonly locator: undefined | DiscoverAppLocator; + // @todo: add documentation + addData: AddDataServiceSetup; } export interface DiscoverStart { @@ -207,7 +213,7 @@ export class DiscoverPlugin private stopUrlTracking: (() => void) | undefined = undefined; private servicesInitialized: boolean = false; private innerAngularInitialized: boolean = false; - + private readonly addDataService = new AddDataService(); /** * @deprecated */ @@ -389,6 +395,7 @@ export class DiscoverPlugin addDocView: this.docViewsRegistry.addDocView.bind(this.docViewsRegistry), }, locator: this.locator, + addData: { ...this.addDataService.setup() }, }; } @@ -424,7 +431,8 @@ export class DiscoverPlugin core, plugins, this.initializerContext, - this.getEmbeddableInjector + this.getEmbeddableInjector, + this.addDataService ); setServices(services); this.servicesInitialized = true; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts new file mode 100644 index 0000000000000..7a768d469a19c --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { i18n } from '@kbn/i18n'; +import { + fromKueryExpression, + toElasticsearchQuery, + luceneStringToDsl, + decorateQuery, +} from '@kbn/es-query'; +import { + RefreshInterval, + SerializableState, + UI_SETTINGS, +} from '../../../../../../../src/plugins/data/common'; +import { IndexDataVisualizerLocator } from '../locator'; +import { isPopulatedObject } from '../../../../common/utils/object_utils'; +import { DiscoverSetup } from '../../../../../../../src/plugins/discover/public'; + +export const DISCOVER_DV_TOP_NAV_LINK_ID = 'indexDataVisualizer'; + +export class DiscoverNavLinkRegistrar { + private readonly locator?: IndexDataVisualizerLocator; + public readonly id = DISCOVER_DV_TOP_NAV_LINK_ID; + + constructor(locator: IndexDataVisualizerLocator) { + this.locator = locator; + } + + registerDiscoverTopNavLink: Parameters< + DiscoverSetup['addData']['registerTopNavLinks'] + >[1] = async (args) => { + if (!this.locator) { + throw Error('IndexDataVisualizerLocator not available'); + } + if (!isPopulatedObject(args)) { + throw Error('Invalid arguments'); + } + + return { + id: DISCOVER_DV_TOP_NAV_LINK_ID, + label: i18n.translate('discover.localMenu.dataVisualizerTitle', { + defaultMessage: 'Data visualizer', + }), + description: i18n.translate('discover.localMenu.dataVisualizerDescription', { + defaultMessage: 'Open in Data visualizer', + }), + testId: 'dataVisualizerTopNavButton', + run: async () => { + const { stateContainer, services, indexPattern, savedSearch, columns } = args; + const extractedQuery = stateContainer.appStateContainer.getState().query; + const timeRange = services.timefilter.getTime(); + const refreshInterval = services.timefilter.getRefreshInterval() as RefreshInterval & + SerializableState; + const params: SerializableState = { + indexPatternId: indexPattern.id, + savedSearchId: savedSearch.id, + timeRange, + refreshInterval, + }; + + if (columns) { + params.visibleFieldNames = columns; + } + if (extractedQuery) { + const queryLanguage = extractedQuery.language; + const qryString = extractedQuery.query; + let qry; + if (queryLanguage === 'kuery') { + const ast = fromKueryExpression(qryString); + qry = toElasticsearchQuery(ast, indexPattern); + } else { + qry = luceneStringToDsl(qryString); + decorateQuery(qry, services.uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS)); + } + + params.query = { + searchQuery: qry as SerializableState, + searchString: qryString, + searchQueryLanguage: queryLanguage, + }; + } + + const url = await this.locator?.getUrl(params, { absolute: true }); + + // We want to open the Data visualizer in a new tab + if (url !== undefined) { + window.open(url, '_blank'); + } + }, + }; + }; +} diff --git a/x-pack/plugins/data_visualizer/public/plugin.ts b/x-pack/plugins/data_visualizer/public/plugin.ts index 565013fc013be..a001657ae48d8 100644 --- a/x-pack/plugins/data_visualizer/public/plugin.ts +++ b/x-pack/plugins/data_visualizer/public/plugin.ts @@ -26,10 +26,13 @@ import { IndexDataVisualizerLocatorDefinition, } from './application/index_data_visualizer/locator'; import { SharePluginSetup } from '../../../../src/plugins/share/public'; +import { DiscoverSetup } from '../../../../src/plugins/discover/public'; +import { DiscoverNavLinkRegistrar } from './application/index_data_visualizer/services/discover_top_nav_link'; export interface DataVisualizerSetupDependencies { share?: SharePluginSetup; home?: HomePublicPluginSetup; + discover?: DiscoverSetup; } export interface DataVisualizerStartDependencies { data: DataPublicPluginStart; @@ -64,6 +67,14 @@ export class DataVisualizerPlugin if (plugins.share) { this.locator = plugins.share.url.locators.create(new IndexDataVisualizerLocatorDefinition()); } + + if (plugins.discover?.addData && this.locator) { + const discoverNavLinkRegistrar = new DiscoverNavLinkRegistrar(this.locator); + plugins.discover.addData.registerTopNavLinks( + discoverNavLinkRegistrar.id, + discoverNavLinkRegistrar.registerDiscoverTopNavLink + ); + } } public start(core: CoreStart, plugins: DataVisualizerStartDependencies) { From 28d54d0a81c94f2de53a5e28f1b750d83e78faf5 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 22 Jul 2021 13:51:50 -0500 Subject: [PATCH 006/188] Renames, refactor, use constants --- .../apps/main/components/top_nav/discover_topnav.tsx | 7 ++++--- .../apps/main/components/top_nav/get_top_nav_links.ts | 11 ++--------- .../{add_top_nav_links.ts => add_top_nav_data.ts} | 8 ++++---- src/plugins/discover/public/build_services.ts | 6 +++--- src/plugins/discover/public/plugin.tsx | 10 +++++----- .../services/discover_top_nav_link.ts | 7 ++++--- x-pack/plugins/data_visualizer/public/plugin.ts | 2 +- 7 files changed, 23 insertions(+), 28 deletions(-) rename src/plugins/discover/public/application/apps/main/services/{add_top_nav_links.ts => add_top_nav_data.ts} (84%) diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx index 1c7a17cce747c..13733561aa2b1 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx @@ -7,10 +7,11 @@ */ import React, { useEffect, useMemo, useState } from 'react'; import { DiscoverLayoutProps } from '../layout/types'; -import { getTopNavLinks, TopNavLink } from './get_top_nav_links'; +import { getTopNavLinks } from './get_top_nav_links'; import { Query, TimeRange } from '../../../../../../../data/common/query'; import { getHeaderActionMenuMounter } from '../../../../../kibana_services'; import { GetStateReturn } from '../../services/discover_state'; +import { TopNavMenuData } from '../../../../../../../navigation/public'; export type DiscoverTopNavProps = Pick< DiscoverLayoutProps, @@ -37,7 +38,7 @@ export const DiscoverTopNav = ({ services, columns, }: DiscoverTopNavProps) => { - const [registeredTopNavLinks, setRegisteredTopNavLinks] = useState([]); + const [registeredTopNavLinks, setRegisteredTopNavLinks] = useState([]); const showDatePicker = useMemo(() => indexPattern.isTimeBased(), [indexPattern]); const { TopNavMenu } = services.navigation.ui; @@ -45,7 +46,7 @@ export const DiscoverTopNav = ({ let unmounted = false; const loadRegisteredTopNavLinks = async () => { - const callbacks = services.addDataService.getTopNavLinks(); + const callbacks = services.addDataService.getTopNavLinkGetters(); const params = { indexPattern, onOpenInspector, diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts index 7e7f2f9ee057b..148d40155da6d 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts @@ -17,14 +17,7 @@ import { onSaveSearch } from './on_save_search'; import { GetStateReturn } from '../../services/discover_state'; import { IndexPattern, ISearchSource } from '../../../../../kibana_services'; import { openOptionsPopover } from './open_options_popover'; - -export interface TopNavLink { - id: string; - label: string; - description: string; - run: (anchorElement: HTMLElement) => void; - testId: string; -} +import type { TopNavMenuData } from '../../../../../../../navigation/public'; /** * Helper function to build the top nav links @@ -45,7 +38,7 @@ export const getTopNavLinks = ({ state: GetStateReturn; onOpenInspector: () => void; searchSource: ISearchSource; -}): TopNavLink[] => { +}): TopNavMenuData[] => { const options = { id: 'options', label: i18n.translate('discover.localMenu.localMenu.optionsTitle', { diff --git a/src/plugins/discover/public/application/apps/main/services/add_top_nav_links.ts b/src/plugins/discover/public/application/apps/main/services/add_top_nav_data.ts similarity index 84% rename from src/plugins/discover/public/application/apps/main/services/add_top_nav_links.ts rename to src/plugins/discover/public/application/apps/main/services/add_top_nav_data.ts index 7ab9d8bdb56eb..768c6227fb2da 100644 --- a/src/plugins/discover/public/application/apps/main/services/add_top_nav_links.ts +++ b/src/plugins/discover/public/application/apps/main/services/add_top_nav_data.ts @@ -19,7 +19,7 @@ export type AddNavLinkCallback = (params: DiscoverTopNavProps) => Promise = {}; public setup() { @@ -27,7 +27,7 @@ export class AddDataService { /** * Registers an async callback function that will return a valid top nav link action */ - registerTopNavLinks: (id: string, callbackFn: AddNavLinkCallback) => { + registerTopNavLinkGetter: (id: string, callbackFn: AddNavLinkCallback) => { if (this.TopNavMenuDatas[id]) { throw new Error(`link ${id} already exists`); } @@ -36,9 +36,9 @@ export class AddDataService { }; } - public getTopNavMenuDatas() { + public getTopNavLinkGetters() { return Object.values(this.TopNavMenuDatas); } } -export type AddDataServiceSetup = ReturnType; +export type AddTopNavDataServiceSetup = ReturnType; diff --git a/src/plugins/discover/public/build_services.ts b/src/plugins/discover/public/build_services.ts index 69e433a4e491b..f79bc6ca387bc 100644 --- a/src/plugins/discover/public/build_services.ts +++ b/src/plugins/discover/public/build_services.ts @@ -36,11 +36,11 @@ import { KibanaLegacyStart } from '../../kibana_legacy/public'; import { UrlForwardingStart } from '../../url_forwarding/public'; import { NavigationPublicPluginStart } from '../../navigation/public'; import { IndexPatternFieldEditorStart } from '../../index_pattern_field_editor/public'; -import { AddDataService } from './application/apps/main/services/add_top_nav_links'; +import { AddTopNavDataService } from './application/apps/main/services/add_top_nav_links'; export interface DiscoverServices { addBasePath: (path: string) => string; - addDataService: AddDataService; + addDataService: AddTopNavDataService; capabilities: Capabilities; chrome: ChromeStart; core: CoreStart; @@ -71,7 +71,7 @@ export async function buildServices( plugins: DiscoverStartPlugins, context: PluginInitializerContext, getEmbeddableInjector: () => Promise, - addDataService: AddDataService + addDataService: AddTopNavDataService ): Promise { const services = { savedObjectsClient: core.savedObjects.client, diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index 7ba9a1b76dbd6..d6e7eb946263b 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -65,9 +65,9 @@ import { replaceUrlHashQuery } from '../../kibana_utils/public/'; import { IndexPatternFieldEditorStart } from '../../../plugins/index_pattern_field_editor/public'; import { SourceViewer } from './application/components/source_viewer/source_viewer'; import { - AddDataService, - AddDataServiceSetup, -} from './application/apps/main/services/add_top_nav_links'; + AddTopNavDataService, + AddTopNavDataServiceSetup, +} from './application/apps/main/services/add_top_nav_data'; declare module '../../share/public' { export interface UrlGeneratorStateMapping { @@ -120,7 +120,7 @@ export interface DiscoverSetup { */ readonly locator: undefined | DiscoverAppLocator; // @todo: add documentation - addData: AddDataServiceSetup; + addData: AddTopNavDataServiceSetup; } export interface DiscoverStart { @@ -213,7 +213,7 @@ export class DiscoverPlugin private stopUrlTracking: (() => void) | undefined = undefined; private servicesInitialized: boolean = false; private innerAngularInitialized: boolean = false; - private readonly addDataService = new AddDataService(); + private readonly addDataService = new AddTopNavDataService(); /** * @deprecated */ diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts index 7a768d469a19c..22e86aecffc27 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts @@ -18,7 +18,8 @@ import { } from '../../../../../../../src/plugins/data/common'; import { IndexDataVisualizerLocator } from '../locator'; import { isPopulatedObject } from '../../../../common/utils/object_utils'; -import { DiscoverSetup } from '../../../../../../../src/plugins/discover/public'; +import type { DiscoverSetup } from '../../../../../../../src/plugins/discover/public'; +import { SEARCH_QUERY_LANGUAGE } from '../types/combined_query'; export const DISCOVER_DV_TOP_NAV_LINK_ID = 'indexDataVisualizer'; @@ -31,7 +32,7 @@ export class DiscoverNavLinkRegistrar { } registerDiscoverTopNavLink: Parameters< - DiscoverSetup['addData']['registerTopNavLinks'] + DiscoverSetup['addData']['registerTopNavLinkGetter'] >[1] = async (args) => { if (!this.locator) { throw Error('IndexDataVisualizerLocator not available'); @@ -69,7 +70,7 @@ export class DiscoverNavLinkRegistrar { const queryLanguage = extractedQuery.language; const qryString = extractedQuery.query; let qry; - if (queryLanguage === 'kuery') { + if (queryLanguage === SEARCH_QUERY_LANGUAGE.KUERY) { const ast = fromKueryExpression(qryString); qry = toElasticsearchQuery(ast, indexPattern); } else { diff --git a/x-pack/plugins/data_visualizer/public/plugin.ts b/x-pack/plugins/data_visualizer/public/plugin.ts index a001657ae48d8..b3bb538aaef6c 100644 --- a/x-pack/plugins/data_visualizer/public/plugin.ts +++ b/x-pack/plugins/data_visualizer/public/plugin.ts @@ -70,7 +70,7 @@ export class DataVisualizerPlugin if (plugins.discover?.addData && this.locator) { const discoverNavLinkRegistrar = new DiscoverNavLinkRegistrar(this.locator); - plugins.discover.addData.registerTopNavLinks( + plugins.discover.addData.registerTopNavLinkGetter( discoverNavLinkRegistrar.id, discoverNavLinkRegistrar.registerDiscoverTopNavLink ); From 4128150892ea027863978d709e8e0f544e5e978c Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 22 Jul 2021 14:05:42 -0500 Subject: [PATCH 007/188] Renames, refactor, use constants --- .../top_nav/discover_topnav.test.tsx | 1 + .../components/top_nav/discover_topnav.tsx | 46 ++++++++++--------- .../main/services/add_top_nav_data.mock.ts | 31 +++++++++++++ src/plugins/discover/public/build_services.ts | 8 ++-- src/plugins/discover/public/mocks.ts | 2 + src/plugins/discover/public/plugin.tsx | 8 ++-- .../services/discover_top_nav_link.ts | 2 +- .../plugins/data_visualizer/public/plugin.ts | 4 +- 8 files changed, 69 insertions(+), 33 deletions(-) create mode 100644 src/plugins/discover/public/application/apps/main/services/add_top_nav_data.mock.ts diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx index 1a68bc4948c3a..e3715132e4ed9 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx @@ -33,6 +33,7 @@ function getProps(savePermissions = true): DiscoverTopNavProps { updateQuery: jest.fn(), onOpenInspector: jest.fn(), searchSource: {} as ISearchSource, + columns: [], }; } diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx index 13733561aa2b1..3b11c8975ed18 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx @@ -46,27 +46,29 @@ export const DiscoverTopNav = ({ let unmounted = false; const loadRegisteredTopNavLinks = async () => { - const callbacks = services.addDataService.getTopNavLinkGetters(); - const params = { - indexPattern, - onOpenInspector, - query, - savedQuery, - stateContainer, - updateQuery, - searchSource, - navigateTo, - savedSearch, - services, - columns, - }; - const links = await Promise.all( - callbacks.map((cb) => { - return cb(params); - }) - ); - if (!unmounted) { - setRegisteredTopNavLinks(links); + if (services?.addTopNavDataService) { + const callbacks = services.addTopNavDataService.getTopNavLinkGetters(); + const params = { + indexPattern, + onOpenInspector, + query, + savedQuery, + stateContainer, + updateQuery, + searchSource, + navigateTo, + savedSearch, + services, + columns, + }; + const links = await Promise.all( + callbacks.map((cb) => { + return cb(params); + }) + ); + if (!unmounted) { + setRegisteredTopNavLinks(links); + } } }; loadRegisteredTopNavLinks(); @@ -75,7 +77,7 @@ export const DiscoverTopNav = ({ unmounted = true; }; }, [ - services?.addDataService, + services?.addTopNavDataService, indexPattern, onOpenInspector, query, diff --git a/src/plugins/discover/public/application/apps/main/services/add_top_nav_data.mock.ts b/src/plugins/discover/public/application/apps/main/services/add_top_nav_data.mock.ts new file mode 100644 index 0000000000000..4f65eb17aff70 --- /dev/null +++ b/src/plugins/discover/public/application/apps/main/services/add_top_nav_data.mock.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { AddTopNavDataService, AddTopNavDataServiceSetup } from './add_top_nav_data'; +import type { PublicMethodsOf } from '@kbn/utility-types'; + +const createSetupMock = (): jest.Mocked => { + const setup = { + registerTopNavLinkGetter: jest.fn(), + }; + return setup; +}; + +const createMock = (): jest.Mocked> => { + const service = { + setup: jest.fn(), + getTopNavLinkGetters: jest.fn(() => []), + }; + service.setup.mockImplementation(createSetupMock); + return service; +}; + +export const addTopNavDataServiceMock = { + createSetup: createSetupMock, + create: createMock, +}; diff --git a/src/plugins/discover/public/build_services.ts b/src/plugins/discover/public/build_services.ts index f79bc6ca387bc..f7c4a31e3456a 100644 --- a/src/plugins/discover/public/build_services.ts +++ b/src/plugins/discover/public/build_services.ts @@ -36,11 +36,11 @@ import { KibanaLegacyStart } from '../../kibana_legacy/public'; import { UrlForwardingStart } from '../../url_forwarding/public'; import { NavigationPublicPluginStart } from '../../navigation/public'; import { IndexPatternFieldEditorStart } from '../../index_pattern_field_editor/public'; -import { AddTopNavDataService } from './application/apps/main/services/add_top_nav_links'; +import type { AddTopNavDataService } from './application/apps/main/services/add_top_nav_data'; export interface DiscoverServices { addBasePath: (path: string) => string; - addDataService: AddTopNavDataService; + addTopNavDataService: AddTopNavDataService; capabilities: Capabilities; chrome: ChromeStart; core: CoreStart; @@ -71,7 +71,7 @@ export async function buildServices( plugins: DiscoverStartPlugins, context: PluginInitializerContext, getEmbeddableInjector: () => Promise, - addDataService: AddTopNavDataService + addTopNavDataService: AddTopNavDataService ): Promise { const services = { savedObjectsClient: core.savedObjects.client, @@ -81,7 +81,7 @@ export async function buildServices( const { usageCollection } = plugins; return { - addDataService, + addTopNavDataService, addBasePath: core.http.basePath.prepend, capabilities: core.application.capabilities, chrome: core.chrome, diff --git a/src/plugins/discover/public/mocks.ts b/src/plugins/discover/public/mocks.ts index e2000e422f227..8704c4e71972b 100644 --- a/src/plugins/discover/public/mocks.ts +++ b/src/plugins/discover/public/mocks.ts @@ -7,6 +7,7 @@ */ import { DiscoverSetup, DiscoverStart } from '.'; +import { addTopNavDataServiceMock } from './application/apps/main/services/add_top_nav_data.mock'; export type Setup = jest.Mocked; export type Start = jest.Mocked; @@ -26,6 +27,7 @@ const createSetupContract = (): Setup => { telemetry: jest.fn(), migrations: {}, }, + addTopNavData: addTopNavDataServiceMock.createSetup(), }; return setupContract; }; diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index d6e7eb946263b..f5d761d4a95e9 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -120,7 +120,7 @@ export interface DiscoverSetup { */ readonly locator: undefined | DiscoverAppLocator; // @todo: add documentation - addData: AddTopNavDataServiceSetup; + addTopNavData: AddTopNavDataServiceSetup; } export interface DiscoverStart { @@ -213,7 +213,7 @@ export class DiscoverPlugin private stopUrlTracking: (() => void) | undefined = undefined; private servicesInitialized: boolean = false; private innerAngularInitialized: boolean = false; - private readonly addDataService = new AddTopNavDataService(); + private readonly addTopNavDataService = new AddTopNavDataService(); /** * @deprecated */ @@ -395,7 +395,7 @@ export class DiscoverPlugin addDocView: this.docViewsRegistry.addDocView.bind(this.docViewsRegistry), }, locator: this.locator, - addData: { ...this.addDataService.setup() }, + addTopNavData: { ...this.addTopNavDataService.setup() }, }; } @@ -432,7 +432,7 @@ export class DiscoverPlugin plugins, this.initializerContext, this.getEmbeddableInjector, - this.addDataService + this.addTopNavDataService ); setServices(services); this.servicesInitialized = true; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts index 22e86aecffc27..282df11ade670 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts @@ -32,7 +32,7 @@ export class DiscoverNavLinkRegistrar { } registerDiscoverTopNavLink: Parameters< - DiscoverSetup['addData']['registerTopNavLinkGetter'] + DiscoverSetup['addTopNavData']['registerTopNavLinkGetter'] >[1] = async (args) => { if (!this.locator) { throw Error('IndexDataVisualizerLocator not available'); diff --git a/x-pack/plugins/data_visualizer/public/plugin.ts b/x-pack/plugins/data_visualizer/public/plugin.ts index b3bb538aaef6c..f9b799138ad19 100644 --- a/x-pack/plugins/data_visualizer/public/plugin.ts +++ b/x-pack/plugins/data_visualizer/public/plugin.ts @@ -68,9 +68,9 @@ export class DataVisualizerPlugin this.locator = plugins.share.url.locators.create(new IndexDataVisualizerLocatorDefinition()); } - if (plugins.discover?.addData && this.locator) { + if (plugins.discover?.addTopNavData && this.locator) { const discoverNavLinkRegistrar = new DiscoverNavLinkRegistrar(this.locator); - plugins.discover.addData.registerTopNavLinkGetter( + plugins.discover.addTopNavData.registerTopNavLinkGetter( discoverNavLinkRegistrar.id, discoverNavLinkRegistrar.registerDiscoverTopNavLink ); From 9fbffbfd234951ed3b213829cf8eb06c35f89095 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Sun, 25 Jul 2021 17:48:26 -0500 Subject: [PATCH 008/188] Update tests and mocks --- .../top_nav/discover_topnav.test.tsx | 33 +++++++++---------- .../components/top_nav/discover_topnav.tsx | 6 ++-- .../top_nav/get_top_nav_links.test.ts | 7 ---- src/plugins/discover/public/build_services.ts | 6 ++-- 4 files changed, 21 insertions(+), 31 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx index e3715132e4ed9..447fe41459c17 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx @@ -16,6 +16,7 @@ import { ISearchSource, Query } from '../../../../../../../data/common'; import { GetStateReturn } from '../../services/discover_state'; import { setHeaderActionMenuMounter } from '../../../../../kibana_services'; import { discoverServiceMock } from '../../../../../__mocks__/services'; +import { act } from 'react-dom/test-utils'; setHeaderActionMenuMounter(jest.fn()); @@ -38,25 +39,21 @@ function getProps(savePermissions = true): DiscoverTopNavProps { } describe('Discover topnav component', () => { - test('generated config of TopNavMenu config is correct when discover save permissions are assigned', () => { - const props = getProps(true); - const component = shallowWithIntl(); - const topMenuConfig = component.props().config.map((obj: TopNavMenuData) => obj.id); - expect(topMenuConfig).toEqual([ - 'options', - 'new', - 'save', - 'open', - 'share', - 'inspect', - 'dataVisualizer', - ]); + test('generated config of TopNavMenu config is correct when discover save permissions are assigned', async () => { + await act(async () => { + const props = getProps(true); + const component = shallowWithIntl(); + const topMenuConfig = component.props().config.map((obj: TopNavMenuData) => obj.id); + expect(topMenuConfig).toEqual(['options', 'new', 'save', 'open', 'share', 'inspect']); + }); }); - test('generated config of TopNavMenu config is correct when no discover save permissions are assigned', () => { - const props = getProps(false); - const component = shallowWithIntl(); - const topMenuConfig = component.props().config.map((obj: TopNavMenuData) => obj.id); - expect(topMenuConfig).toEqual(['options', 'new', 'open', 'share', 'inspect', 'dataVisualizer']); + test('generated config of TopNavMenu config is correct when no discover save permissions are assigned', async () => { + await act(async () => { + const props = getProps(false); + const component = shallowWithIntl(); + const topMenuConfig = component.props().config.map((obj: TopNavMenuData) => obj.id); + expect(topMenuConfig).toEqual(['options', 'new', 'open', 'share', 'inspect']); + }); }); }); diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx index 3b11c8975ed18..2b8034e05d84a 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx @@ -46,8 +46,8 @@ export const DiscoverTopNav = ({ let unmounted = false; const loadRegisteredTopNavLinks = async () => { - if (services?.addTopNavDataService) { - const callbacks = services.addTopNavDataService.getTopNavLinkGetters(); + if (services?.addTopNavData) { + const callbacks = services.addTopNavData.getTopNavLinkGetters(); const params = { indexPattern, onOpenInspector, @@ -77,7 +77,7 @@ export const DiscoverTopNav = ({ unmounted = true; }; }, [ - services?.addTopNavDataService, + services?.addTopNavData, indexPattern, onOpenInspector, query, diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts index fae1c54b07211..6a6fb8a44a5cf 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts @@ -80,13 +80,6 @@ test('getTopNavLinks result', () => { "run": [Function], "testId": "openInspectorButton", }, - Object { - "description": "Open in Data visualizer", - "id": "dataVisualizer", - "label": "Data visualizer", - "run": [Function], - "testId": "dataVisualizerTopNavButton", - }, ] `); }); diff --git a/src/plugins/discover/public/build_services.ts b/src/plugins/discover/public/build_services.ts index f7c4a31e3456a..a44dbf01d9602 100644 --- a/src/plugins/discover/public/build_services.ts +++ b/src/plugins/discover/public/build_services.ts @@ -40,7 +40,7 @@ import type { AddTopNavDataService } from './application/apps/main/services/add_ export interface DiscoverServices { addBasePath: (path: string) => string; - addTopNavDataService: AddTopNavDataService; + addTopNavData: AddTopNavDataService; capabilities: Capabilities; chrome: ChromeStart; core: CoreStart; @@ -71,7 +71,7 @@ export async function buildServices( plugins: DiscoverStartPlugins, context: PluginInitializerContext, getEmbeddableInjector: () => Promise, - addTopNavDataService: AddTopNavDataService + addTopNavData: AddTopNavDataService ): Promise { const services = { savedObjectsClient: core.savedObjects.client, @@ -81,8 +81,8 @@ export async function buildServices( const { usageCollection } = plugins; return { - addTopNavDataService, addBasePath: core.http.basePath.prepend, + addTopNavData, capabilities: core.application.capabilities, chrome: core.chrome, core, From eaa2563d27b2afd18eac99182092e42d8b95de76 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 26 Jul 2021 15:27:58 -0500 Subject: [PATCH 009/188] Embeddable --- .../components/layout/discover_layout.tsx | 91 +-- .../data_visualizer_grid.tsx | 21 +- .../data_visualizer/common/types/index.ts | 5 + .../index_data_visualizer_view.tsx | 5 +- .../grid_embeddable/grid_embeddable.tsx | 653 +++++++++++++++++- .../utils/saved_search_utils.ts | 36 +- 6 files changed, 741 insertions(+), 70 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 698928ce13141..9c4b37bb43340 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -83,6 +83,7 @@ export function DiscoverLayout({ state, stateContainer, }: DiscoverLayoutProps) { + console.log('state', services.timefilter.getBounds()); const { trackUiMetric, capabilities, indexPatterns, data, uiSettings, filterManager } = services; const sampleSize = useMemo(() => uiSettings.get(SAMPLE_SIZE_SETTING), [uiSettings]); @@ -362,51 +363,51 @@ export function DiscoverLayout({ query={state.query} columns={columns} /> - {isLegacy && rows && rows.length && ( - - )} - {!isLegacy && rows && rows.length && ( -
- -
- )} + {/* {isLegacy && rows && rows.length && (*/} + {/* */} + {/* )}*/} + {/* {!isLegacy && rows && rows.length && (*/} + {/*
*/} + {/* */} + {/*
*/} + {/* )}*/} diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index e83fdcf3bc46e..37c0658dfe8e0 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import React, { useEffect, useRef, useState } from 'react'; +import React, { useEffect, useMemo, useRef, useState } from 'react'; import { EuiDataGrid, EuiDataGridProps } from '@elastic/eui'; import { IndexPattern, Query } from '../../../../../data/common'; import { DiscoverServices } from '../../../build_services'; @@ -60,13 +60,19 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp >(); const embeddableRoot: React.RefObject = useRef(null); + const timeBounds = useMemo(() => services.timefilter.getBounds(), [services.timefilter]); useEffect(() => { if (embeddable && !isErrorEmbeddable(embeddable)) { // Update embeddable whenever one of the important input changes - embeddable.updateInput({ indexPattern, savedSearch, query }); + embeddable.updateInput({ + indexPattern, + savedSearch, + query, + timeBounds, + }); embeddable.reload(); } - }, [embeddable, indexPattern, savedSearch, query]); + }, [embeddable, indexPattern, savedSearch, query, timeBounds]); useEffect(() => { return () => { @@ -86,8 +92,13 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp >('data_visualizer_grid'); if (factory) { // Initialize embeddable with information available at mount - const test = await factory.create({ id: 'test', indexPattern, savedSearch, query }); - setEmbeddable(test); + const initializedEmbeddable = await factory.create({ + id: 'test', + indexPattern, + savedSearch, + query, + }); + setEmbeddable(initializedEmbeddable); } } }; diff --git a/x-pack/plugins/data_visualizer/common/types/index.ts b/x-pack/plugins/data_visualizer/common/types/index.ts index 8b51142e19129..6ab7e6cccbe74 100644 --- a/x-pack/plugins/data_visualizer/common/types/index.ts +++ b/x-pack/plugins/data_visualizer/common/types/index.ts @@ -6,6 +6,7 @@ */ import type { SimpleSavedObject } from 'kibana/public'; +import { isPopulatedObject } from '../utils/object_utils'; export type { JobFieldType } from './job_field_type'; export type { FieldRequestConfig, @@ -27,3 +28,7 @@ export interface DataVisualizerTableState { } export type SavedSearchSavedObject = SimpleSavedObject; + +export function isSavedSearchSavedObject(arg: unknown): arg is SavedSearchSavedObject { + return isPopulatedObject(arg, ['id', 'type']); +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index f45c4a89d006c..16121a06e2d1e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -70,7 +70,7 @@ import { DataVisualizerIndexPatternManagement } from '../index_pattern_managemen import { ResultLink } from '../../../common/components/results_links'; import { extractErrorProperties } from '../../utils/error_utils'; -interface DataVisualizerPageState { +export interface DataVisualizerPageState { overallStats: OverallStats; metricConfigs: FieldVisConfig[]; totalMetricFieldCount: number; @@ -85,7 +85,7 @@ const defaultSearchQuery = { match_all: {}, }; -function getDefaultPageState(): DataVisualizerPageState { +export function getDefaultPageState(): DataVisualizerPageState { return { overallStats: { totalCount: 0, @@ -140,6 +140,7 @@ export const IndexDataVisualizerView: FC = (dataVi const [currentSavedSearch, setCurrentSavedSearch] = useState( dataVisualizerProps.currentSavedSearch ); + console.log('--currentSavedSearch--', currentSavedSearch); const { currentIndexPattern, additionalLinks } = dataVisualizerProps; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index f647c51d8f3e8..1e57ef7ed2f33 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -8,8 +8,25 @@ import { Observable, Subject } from 'rxjs'; import { CoreStart } from 'kibana/public'; import ReactDOM from 'react-dom'; -import React, { Suspense } from 'react'; +import React, { + Dispatch, + SetStateAction, + Suspense, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; import useObservable from 'react-use/lib/useObservable'; +import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + fromKueryExpression, + toElasticsearchQuery, + luceneStringToDsl, + decorateQuery, +} from '@kbn/es-query'; import { Embeddable, EmbeddableInput, @@ -20,11 +37,39 @@ import { KibanaContextProvider } from '../../../../../../../../src/plugins/kiban import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; import { EmbeddableLoading } from './embeddable_loading_fallback'; import { DataVisualizerStartDependencies } from '../../../../plugin'; -import { IndexPattern, Query } from '../../../../../../../../src/plugins/data/common'; +import { + IndexPattern, + IndexPatternField, + KBN_FIELD_TYPES, + Query, + UI_SETTINGS, +} from '../../../../../../../../src/plugins/data/common'; import { SavedSearch } from '../../../../../../../../src/plugins/discover/public'; +import { + DataVisualizerTable, + ItemIdToExpandedRowMap, +} from '../../../common/components/stats_table'; +import { FieldVisConfig } from '../../../common/components/stats_table/types'; +import { MetricFieldsStats } from '../../../common/components/stats_table/components/field_count_stats'; +import { + getDefaultDataVisualizerListState, + getDefaultPageState, +} from '../../components/index_data_visualizer_view/index_data_visualizer_view'; + +import { extractSearchData } from '../../utils/saved_search_utils'; +import { useDataVisualizerKibana } from '../../../kibana_context'; +import { extractErrorProperties } from '../../utils/error_utils'; +import { DataLoader } from '../../data_loader/data_loader'; +import { useTimefilter } from '../../hooks/use_time_filter'; +import { FieldRequestConfig, JOB_FIELD_TYPES } from '../../../../../common'; +import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; +import { TimeBuckets } from '../../services/time_buckets'; +import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; +import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; +import { getActions } from '../../../common/components/field_data_row/action_menu'; export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerStartDependencies]; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { - indexPattern?: IndexPattern; + indexPattern: IndexPattern; savedSearch?: SavedSearch; query?: Query; } @@ -32,6 +77,600 @@ export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; export type IDataVisualizerGridEmbeddable = typeof DataVisualizerGridEmbeddable; +const restorableDefaults = getDefaultDataVisualizerListState(); +const defaults = getDefaultPageState(); + +const useDataVisualizerGridData = ( + input: DataVisualizerGridEmbeddableInput, + dataVisualizerListState: DataVisualizerIndexBasedAppState, + setDataVisualizerListState: Dispatch> +) => { + const { services } = useDataVisualizerKibana(); + const { notifications, uiSettings } = services; + const { toasts } = notifications; + const { + samplerShardSize, + visibleFieldTypes, + visibleFieldNames, + showEmptyFields, + } = dataVisualizerListState; + + const { currentSavedSearch, currentIndexPattern, currentQuery } = useMemo( + () => ({ + currentSavedSearch: input?.savedSearch, + currentIndexPattern: input.indexPattern, + currentQuery: input.query, + }), + [input] + ); + console.log('--currentSavedSearch--', currentSavedSearch); + console.log('--currentIndexPattern--', currentIndexPattern); + console.log('--currentIndexPattern--', input.query); + + const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { + // Prioritize current query in the search bar first + // because a saved search could have been previously loaded + // but the input has been changed + if (currentQuery) { + const queryLanguage = currentQuery.language; + const qryString = currentQuery.query; + let qry; + if (queryLanguage === 'kuery') { + const ast = fromKueryExpression(qryString); + qry = toElasticsearchQuery(ast, currentIndexPattern); + } else { + qry = luceneStringToDsl(qryString); + decorateQuery(qry, services.uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS)); + } + return { + searchQuery: qry, + searchString: qryString, + searchQueryLanguage: queryLanguage, + }; + } + + // Then process query from saved search + const searchData = extractSearchData( + currentSavedSearch, + currentIndexPattern, + uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS) + ); + + if (searchData === undefined || dataVisualizerListState.searchString !== '') { + return { + searchQuery: dataVisualizerListState.searchQuery, + searchString: dataVisualizerListState.searchString, + searchQueryLanguage: dataVisualizerListState.searchQueryLanguage, + }; + } + return { + searchQuery: searchData.searchQuery, + searchString: searchData.searchString, + searchQueryLanguage: searchData.queryLanguage, + }; + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState, currentQuery]); + + const [overallStats, setOverallStats] = useState(defaults.overallStats); + + const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); + const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); + const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); + const [metricsStats, setMetricsStats] = useState(); + + const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); + const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); + + const dataLoader = useMemo(() => new DataLoader(currentIndexPattern, toasts), [ + currentIndexPattern, + toasts, + ]); + + const timefilter = useTimefilter({ + timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, + autoRefreshSelector: true, + }); + + const getTimeBuckets = useCallback(() => { + return new TimeBuckets({ + [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), + [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), + dateFormat: uiSettings.get('dateFormat'), + 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), + }); + }, [uiSettings]); + + const indexPatternFields: IndexPatternField[] = useMemo(() => currentIndexPattern.fields, [ + currentIndexPattern, + ]); + + async function loadOverallStats() { + const tf = timefilter as any; + let earliest; + let latest; + + const activeBounds = tf.getActiveBounds(); + + if (currentIndexPattern.timeFieldName !== undefined && activeBounds === undefined) { + return; + } + + if (currentIndexPattern.timeFieldName !== undefined) { + earliest = activeBounds.min.valueOf(); + latest = activeBounds.max.valueOf(); + } + + try { + const allStats = await dataLoader.loadOverallData( + searchQuery, + samplerShardSize, + earliest, + latest + ); + // Because load overall stats perform queries in batches + // there could be multiple errors + if (Array.isArray(allStats.errors) && allStats.errors.length > 0) { + allStats.errors.forEach((err: any) => { + dataLoader.displayError(extractErrorProperties(err)); + }); + } + setOverallStats(allStats); + } catch (err) { + dataLoader.displayError(err.body ?? err); + } + } + + const createMetricCards = useCallback(() => { + const configs: FieldVisConfig[] = []; + const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; + + const allMetricFields = indexPatternFields.filter((f) => { + return ( + f.type === KBN_FIELD_TYPES.NUMBER && + f.displayName !== undefined && + dataLoader.isDisplayField(f.displayName) === true + ); + }); + const metricExistsFields = allMetricFields.filter((f) => { + return aggregatableExistsFields.find((existsF) => { + return existsF.fieldName === f.spec.name; + }); + }); + + // Add a config for 'document count', identified by no field name if indexpattern is time based. + if (currentIndexPattern.timeFieldName !== undefined) { + configs.push({ + type: JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + loading: true, + aggregatable: true, + }); + } + + if (metricsLoaded === false) { + setMetricsLoaded(true); + return; + } + + let aggregatableFields: any[] = overallStats.aggregatableExistsFields; + if (allMetricFields.length !== metricExistsFields.length && metricsLoaded === true) { + aggregatableFields = aggregatableFields.concat(overallStats.aggregatableNotExistsFields); + } + + const metricFieldsToShow = + metricsLoaded === true && showEmptyFields === true ? allMetricFields : metricExistsFields; + + metricFieldsToShow.forEach((field) => { + const fieldData = aggregatableFields.find((f) => { + return f.fieldName === field.spec.name; + }); + + const metricConfig: FieldVisConfig = { + ...(fieldData ? fieldData : {}), + fieldFormat: currentIndexPattern.getFormatterForField(field), + type: JOB_FIELD_TYPES.NUMBER, + loading: true, + aggregatable: true, + deletable: field.runtimeField !== undefined, + }; + if (field.displayName !== metricConfig.fieldName) { + metricConfig.displayName = field.displayName; + } + + configs.push(metricConfig); + }); + + setMetricsStats({ + totalMetricFieldsCount: allMetricFields.length, + visibleMetricsCount: metricFieldsToShow.length, + }); + setMetricConfigs(configs); + }, [ + currentIndexPattern, + dataLoader, + indexPatternFields, + metricsLoaded, + overallStats, + showEmptyFields, + ]); + + const createNonMetricCards = useCallback(() => { + const allNonMetricFields = indexPatternFields.filter((f) => { + return ( + f.type !== KBN_FIELD_TYPES.NUMBER && + f.displayName !== undefined && + dataLoader.isDisplayField(f.displayName) === true + ); + }); + // Obtain the list of all non-metric fields which appear in documents + // (aggregatable or not aggregatable). + const populatedNonMetricFields: any[] = []; // Kibana index pattern non metric fields. + let nonMetricFieldData: any[] = []; // Basic non metric field data loaded from requesting overall stats. + const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; + const nonAggregatableExistsFields: any[] = overallStats.nonAggregatableExistsFields || []; + + allNonMetricFields.forEach((f) => { + const checkAggregatableField = aggregatableExistsFields.find( + (existsField) => existsField.fieldName === f.spec.name + ); + + if (checkAggregatableField !== undefined) { + populatedNonMetricFields.push(f); + nonMetricFieldData.push(checkAggregatableField); + } else { + const checkNonAggregatableField = nonAggregatableExistsFields.find( + (existsField) => existsField.fieldName === f.spec.name + ); + + if (checkNonAggregatableField !== undefined) { + populatedNonMetricFields.push(f); + nonMetricFieldData.push(checkNonAggregatableField); + } + } + }); + + if (nonMetricsLoaded === false) { + setNonMetricsLoaded(true); + return; + } + + if (allNonMetricFields.length !== nonMetricFieldData.length && showEmptyFields === true) { + // Combine the field data obtained from Elasticsearch into a single array. + nonMetricFieldData = nonMetricFieldData.concat( + overallStats.aggregatableNotExistsFields, + overallStats.nonAggregatableNotExistsFields + ); + } + + const nonMetricFieldsToShow = showEmptyFields ? allNonMetricFields : populatedNonMetricFields; + + const configs: FieldVisConfig[] = []; + + nonMetricFieldsToShow.forEach((field) => { + const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); + + const nonMetricConfig = { + ...fieldData, + fieldFormat: currentIndexPattern.getFormatterForField(field), + aggregatable: field.aggregatable, + scripted: field.scripted, + loading: fieldData.existsInDocs, + deletable: field.runtimeField !== undefined, + }; + + // Map the field type from the Kibana index pattern to the field type + // used in the data visualizer. + const dataVisualizerType = kbnTypeToJobType(field); + if (dataVisualizerType !== undefined) { + nonMetricConfig.type = dataVisualizerType; + } else { + // Add a flag to indicate that this is one of the 'other' Kibana + // field types that do not yet have a specific card type. + nonMetricConfig.type = field.type; + nonMetricConfig.isUnsupportedType = true; + } + + if (field.displayName !== nonMetricConfig.fieldName) { + nonMetricConfig.displayName = field.displayName; + } + + configs.push(nonMetricConfig); + }); + + setNonMetricConfigs(configs); + }, [ + currentIndexPattern, + dataLoader, + indexPatternFields, + nonMetricsLoaded, + overallStats, + showEmptyFields, + ]); + + async function loadMetricFieldStats() { + // Only request data for fields that exist in documents. + if (metricConfigs.length === 0) { + return; + } + + const configsToLoad = metricConfigs.filter( + (config) => config.existsInDocs === true && config.loading === true + ); + if (configsToLoad.length === 0) { + return; + } + + // Pass the field name, type and cardinality in the request. + // Top values will be obtained on a sample if cardinality > 100000. + const existMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { + const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; + if (config.stats !== undefined && config.stats.cardinality !== undefined) { + props.cardinality = config.stats.cardinality; + } + return props; + }); + + // Obtain the interval to use for date histogram aggregations + // (such as the document count chart). Aim for 75 bars. + const buckets = getTimeBuckets(); + + const tf = timefilter as any; + let earliest: number | undefined; + let latest: number | undefined; + if (currentIndexPattern.timeFieldName !== undefined) { + earliest = tf.getActiveBounds().min.valueOf(); + latest = tf.getActiveBounds().max.valueOf(); + } + + const bounds = tf.getActiveBounds(); + const BAR_TARGET = 75; + buckets.setInterval('auto'); + buckets.setBounds(bounds); + buckets.setBarTarget(BAR_TARGET); + const aggInterval = buckets.getInterval(); + + try { + const metricFieldStats = await dataLoader.loadFieldStats( + searchQuery, + samplerShardSize, + earliest, + latest, + existMetricFields, + aggInterval.asMilliseconds() + ); + + // Add the metric stats to the existing stats in the corresponding config. + const configs: FieldVisConfig[] = []; + metricConfigs.forEach((config) => { + const configWithStats = { ...config }; + if (config.fieldName !== undefined) { + configWithStats.stats = { + ...configWithStats.stats, + ...metricFieldStats.find( + (fieldStats: any) => fieldStats.fieldName === config.fieldName + ), + }; + configWithStats.loading = false; + configs.push(configWithStats); + } else { + // Document count card. + configWithStats.stats = metricFieldStats.find( + (fieldStats: any) => fieldStats.fieldName === undefined + ); + + if (configWithStats.stats !== undefined) { + // Add earliest / latest of timefilter for setting x axis domain. + configWithStats.stats.timeRangeEarliest = earliest; + configWithStats.stats.timeRangeLatest = latest; + } + setDocumentCountStats(configWithStats); + } + }); + + setMetricConfigs(configs); + } catch (err) { + dataLoader.displayError(err); + } + } + + async function loadNonMetricFieldStats() { + // Only request data for fields that exist in documents. + if (nonMetricConfigs.length === 0) { + return; + } + + const configsToLoad = nonMetricConfigs.filter( + (config) => config.existsInDocs === true && config.loading === true + ); + if (configsToLoad.length === 0) { + return; + } + + // Pass the field name, type and cardinality in the request. + // Top values will be obtained on a sample if cardinality > 100000. + const existNonMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { + const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; + if (config.stats !== undefined && config.stats.cardinality !== undefined) { + props.cardinality = config.stats.cardinality; + } + return props; + }); + + const tf = timefilter as any; + let earliest; + let latest; + if (currentIndexPattern.timeFieldName !== undefined) { + earliest = tf.getActiveBounds().min.valueOf(); + latest = tf.getActiveBounds().max.valueOf(); + } + + try { + const nonMetricFieldStats = await dataLoader.loadFieldStats( + searchQuery, + samplerShardSize, + earliest, + latest, + existNonMetricFields + ); + + // Add the field stats to the existing stats in the corresponding config. + const configs: FieldVisConfig[] = []; + nonMetricConfigs.forEach((config) => { + const configWithStats = { ...config }; + if (config.fieldName !== undefined) { + configWithStats.stats = { + ...configWithStats.stats, + ...nonMetricFieldStats.find( + (fieldStats: any) => fieldStats.fieldName === config.fieldName + ), + }; + } + configWithStats.loading = false; + configs.push(configWithStats); + }); + + setNonMetricConfigs(configs); + } catch (err) { + dataLoader.displayError(err); + } + } + + useEffect(() => { + loadOverallStats(); + // @todo: add back lastRefresh as dep + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [searchQuery, samplerShardSize]); + + useEffect(() => { + createMetricCards(); + createNonMetricCards(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [overallStats, showEmptyFields]); + + useEffect(() => { + loadMetricFieldStats(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [metricConfigs]); + + useEffect(() => { + loadNonMetricFieldStats(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [nonMetricConfigs]); + + useEffect(() => { + createMetricCards(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [metricsLoaded]); + + useEffect(() => { + createNonMetricCards(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [nonMetricsLoaded]); + + const configs = useMemo(() => { + let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; + if (visibleFieldTypes && visibleFieldTypes.length > 0) { + combinedConfigs = combinedConfigs.filter( + (config) => visibleFieldTypes.findIndex((field) => field === config.type) > -1 + ); + } + if (visibleFieldNames && visibleFieldNames.length > 0) { + combinedConfigs = combinedConfigs.filter( + (config) => visibleFieldNames.findIndex((field) => field === config.fieldName) > -1 + ); + } + + return combinedConfigs; + }, [nonMetricConfigs, metricConfigs, visibleFieldTypes, visibleFieldNames]); + + // Some actions open up fly-out or popup + // This variable is used to keep track of them and clean up when unmounting + const actionFlyoutRef = useRef<() => void | undefined>(); + useEffect(() => { + const ref = actionFlyoutRef; + return () => { + // Clean up any of the flyout/editor opened from the actions + if (ref.current) { + ref.current(); + } + }; + }, []); + + // Inject custom action column for the index based visualizer + // Hide the column completely if no access to any of the plugins + const extendedColumns = useMemo(() => { + const actions = getActions( + input.indexPattern, + services, + { + searchQueryLanguage, + searchString, + }, + actionFlyoutRef + ); + if (!Array.isArray(actions) || actions.length < 1) return; + + const actionColumn: EuiTableActionsColumnType = { + name: ( + + ), + actions, + width: '100px', + }; + + return [actionColumn]; + }, [input.indexPattern, services, searchQueryLanguage, searchString]); + + return { configs, searchQueryLanguage, searchString, searchQuery, extendedColumns }; +}; + +export const Test = ({ input }: { input: DataVisualizerGridEmbeddableInput }) => { + const [ + dataVisualizerListState, + setDataVisualizerListState, + ] = useState(restorableDefaults); + + const { configs, searchQueryLanguage, searchString, extendedColumns } = useDataVisualizerGridData( + input, + dataVisualizerListState, + setDataVisualizerListState + ); + const getItemIdToExpandedRowMap = useCallback( + function (itemIds: string[], items: FieldVisConfig[]): ItemIdToExpandedRowMap { + return itemIds.reduce((m: ItemIdToExpandedRowMap, fieldName: string) => { + const item = items.find((fieldVisConfig) => fieldVisConfig.fieldName === fieldName); + if (item !== undefined) { + m[fieldName] = ( + + ); + } + return m; + }, {} as ItemIdToExpandedRowMap); + }, + [input, searchQueryLanguage, searchString] + ); + + return ( + + items={configs} + pageState={dataVisualizerListState} + updatePageState={setDataVisualizerListState} + getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} + extendedColumns={extendedColumns} + /> + ); + + return
; +}; + export const IndexDataVisualizerViewWrapper = (props: { id: string; embeddableContext: InstanceType; @@ -40,7 +679,12 @@ export const IndexDataVisualizerViewWrapper = (props: { }) => { const { embeddableInput } = props; - const data = useObservable(embeddableInput); + const input = useObservable(embeddableInput); + if (input && input.indexPattern) { + return ; + } else { + return
; + } return
Hello world 2
; }; @@ -81,7 +725,6 @@ export class DataVisualizerGridEmbeddable extends Embeddable< }> - Hello World | string ) { - if (!savedSearch) { - return undefined; - } + if (!savedSearch || !currentIndexPattern) return; + + const query = getQueryFromSavedSearch(savedSearch); + if (!query) return; - const { query: extractedQuery } = getQueryFromSavedSearch(savedSearch); + const extractedQuery = query.query; const queryLanguage = extractedQuery.language as SearchQueryLanguage; const qryString = extractedQuery.query; let qry; From df58defd118808b690ee0a563f43989c341da7c2 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 26 Jul 2021 17:09:48 -0500 Subject: [PATCH 010/188] Update hook to update upon time udpate --- .../data_visualizer_grid.tsx | 17 +++----- .../grid_embeddable/grid_embeddable.tsx | 43 +++++++++++-------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index 37c0658dfe8e0..8b5b56eda3578 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import React, { useEffect, useMemo, useRef, useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import { EuiDataGrid, EuiDataGridProps } from '@elastic/eui'; import { IndexPattern, Query } from '../../../../../data/common'; import { DiscoverServices } from '../../../build_services'; @@ -52,7 +52,7 @@ export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { }); export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { - const { services, indexPattern, savedSearch, query } = props; + const { services, indexPattern, savedSearch, query, columns } = props; const [embeddable, setEmbeddable] = useState< | ErrorEmbeddable | IEmbeddable @@ -60,7 +60,6 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp >(); const embeddableRoot: React.RefObject = useRef(null); - const timeBounds = useMemo(() => services.timefilter.getBounds(), [services.timefilter]); useEffect(() => { if (embeddable && !isErrorEmbeddable(embeddable)) { // Update embeddable whenever one of the important input changes @@ -68,11 +67,11 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp indexPattern, savedSearch, query, - timeBounds, + visibleFieldNames: columns, }); embeddable.reload(); } - }, [embeddable, indexPattern, savedSearch, query, timeBounds]); + }, [embeddable, indexPattern, savedSearch, query, columns]); useEffect(() => { return () => { @@ -93,7 +92,7 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp if (factory) { // Initialize embeddable with information available at mount const initializedEmbeddable = await factory.create({ - id: 'test', + id: 'discover_data_visualizer_grid', indexPattern, savedSearch, query, @@ -113,9 +112,5 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp } }, [embeddable, embeddableRoot]); - return ( - <> -
- - ); + return
; }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 1e57ef7ed2f33..debd09c152ba0 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import { Observable, Subject } from 'rxjs'; +import { merge, Observable, Subject } from 'rxjs'; import { CoreStart } from 'kibana/public'; import ReactDOM from 'react-dom'; import React, { @@ -67,11 +67,13 @@ import { TimeBuckets } from '../../services/time_buckets'; import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; import { getActions } from '../../../common/components/field_data_row/action_menu'; +import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerStartDependencies]; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { indexPattern: IndexPattern; savedSearch?: SavedSearch; query?: Query; + visibleFieldNames?: string[]; } export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; @@ -88,24 +90,20 @@ const useDataVisualizerGridData = ( const { services } = useDataVisualizerKibana(); const { notifications, uiSettings } = services; const { toasts } = notifications; - const { - samplerShardSize, - visibleFieldTypes, - visibleFieldNames, - showEmptyFields, - } = dataVisualizerListState; + // @todo: consolidate dataVisualizerListState and input + const { samplerShardSize, visibleFieldTypes, showEmptyFields } = dataVisualizerListState; + + const [lastRefresh, setLastRefresh] = useState(0); - const { currentSavedSearch, currentIndexPattern, currentQuery } = useMemo( + const { currentSavedSearch, currentIndexPattern, currentQuery, visibleFieldNames } = useMemo( () => ({ currentSavedSearch: input?.savedSearch, currentIndexPattern: input.indexPattern, - currentQuery: input.query, + currentQuery: input?.query, + visibleFieldNames: input?.visibleFieldNames ?? [], }), [input] ); - console.log('--currentSavedSearch--', currentSavedSearch); - console.log('--currentIndexPattern--', currentIndexPattern); - console.log('--currentIndexPattern--', input.query); const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { // Prioritize current query in the search bar first @@ -172,6 +170,18 @@ const useDataVisualizerGridData = ( autoRefreshSelector: true, }); + useEffect(() => { + const timeUpdateSubscription = merge( + timefilter.getTimeUpdate$(), + dataVisualizerRefresh$ + ).subscribe(() => { + setLastRefresh(Date.now()); + }); + return () => { + timeUpdateSubscription.unsubscribe(); + }; + }); + const getTimeBuckets = useCallback(() => { return new TimeBuckets({ [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), @@ -538,9 +548,8 @@ const useDataVisualizerGridData = ( useEffect(() => { loadOverallStats(); - // @todo: add back lastRefresh as dep // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchQuery, samplerShardSize]); + }, [searchQuery, samplerShardSize, lastRefresh]); useEffect(() => { createMetricCards(); @@ -628,7 +637,7 @@ const useDataVisualizerGridData = ( return { configs, searchQueryLanguage, searchString, searchQuery, extendedColumns }; }; -export const Test = ({ input }: { input: DataVisualizerGridEmbeddableInput }) => { +export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddableInput }) => { const [ dataVisualizerListState, setDataVisualizerListState, @@ -681,12 +690,10 @@ export const IndexDataVisualizerViewWrapper = (props: { const input = useObservable(embeddableInput); if (input && input.indexPattern) { - return ; + return ; } else { return
; } - - return
Hello world 2
; }; export class DataVisualizerGridEmbeddable extends Embeddable< DataVisualizerGridEmbeddableInput, From e9e18b7ce0236aedd71776818e40633552b070b8 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 15:08:24 -0500 Subject: [PATCH 011/188] Add filter support to query --- .../index_data_visualizer_view.tsx | 8 +- .../utils/saved_search_utils.ts | 114 +++++++----------- 2 files changed, 46 insertions(+), 76 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index f45c4a89d006c..67d2ef0472a87 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -65,7 +65,7 @@ import { DatePickerWrapper } from '../../../common/components/date_picker_wrappe import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; import { HelpMenu } from '../../../common/components/help_menu'; import { TimeBuckets } from '../../services/time_buckets'; -import { extractSearchData } from '../../utils/saved_search_utils'; +import { createSearchItems, extractSearchData } from '../../utils/saved_search_utils'; import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; import { ResultLink } from '../../../common/components/results_links'; import { extractErrorProperties } from '../../utils/error_utils'; @@ -224,11 +224,7 @@ export const IndexDataVisualizerView: FC = (dataVi const defaults = getDefaultPageState(); const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - const searchData = extractSearchData( - currentSavedSearch, - currentIndexPattern, - uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS) - ); + const searchData = extractSearchData(currentSavedSearch, currentIndexPattern, uiSettings); if (searchData === undefined || dataVisualizerListState.searchString !== '') { return { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index a04795ceb9d7d..b467ed750d266 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -7,10 +7,17 @@ import { cloneDeep } from 'lodash'; import { IUiSettingsClient } from 'kibana/public'; +import { + fromKueryExpression, + toElasticsearchQuery, + buildQueryFromFilters, + buildEsQuery, + Query, +} from '@kbn/es-query'; import { SavedSearchSavedObject } from '../../../../common/types'; import { IndexPattern } from '../../../../../../../src/plugins/data/common/index_patterns/index_patterns'; -import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; -import { esKuery, esQuery, Query } from '../../../../../../../src/plugins/data/public'; +import { SEARCH_QUERY_LANGUAGE } from '../types/combined_query'; +import { getEsQueryConfig } from '../../../../../../../src/plugins/data/common'; export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject) { const search = savedSearch.attributes.kibanaSavedObjectMeta as { searchSourceJSON: string }; @@ -25,28 +32,49 @@ export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject) { */ export function extractSearchData( savedSearch: SavedSearchSavedObject | null, - currentIndexPattern: IndexPattern, - queryStringOptions: Record | string + indexPattern: IndexPattern, + uiSettings: IUiSettingsClient ) { if (!savedSearch) { return undefined; } - const { query: extractedQuery } = getQueryFromSavedSearch(savedSearch); - const queryLanguage = extractedQuery.language as SearchQueryLanguage; - const qryString = extractedQuery.query; - let qry; - if (queryLanguage === SEARCH_QUERY_LANGUAGE.KUERY) { - const ast = esKuery.fromKueryExpression(qryString); - qry = esKuery.toElasticsearchQuery(ast, currentIndexPattern); + const data = getQueryFromSavedSearch(savedSearch); + let combinedQuery: any = getDefaultDatafeedQuery(); + + const query = data.query; + const filter = data.filter; + + const filters = Array.isArray(filter) ? filter : []; + + if (query.language === SEARCH_QUERY_LANGUAGE.KUERY) { + const ast = fromKueryExpression(query.query); + if (query.query !== '') { + combinedQuery = toElasticsearchQuery(ast, indexPattern); + } + const filterQuery = buildQueryFromFilters(filters, indexPattern); + + if (Array.isArray(combinedQuery.bool.filter) === false) { + combinedQuery.bool.filter = + combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; + } + + if (Array.isArray(combinedQuery.bool.must_not) === false) { + combinedQuery.bool.must_not = + combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; + } + + combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; + combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; } else { - qry = esQuery.luceneStringToDsl(qryString); - esQuery.decorateQuery(qry, queryStringOptions); + const esQueryConfigs = getEsQueryConfig(uiSettings); + combinedQuery = buildEsQuery(indexPattern, [query], filters, esQueryConfigs); } + return { - searchQuery: qry, - searchString: qryString, - queryLanguage, + searchQuery: combinedQuery, + searchString: data.query.query, + queryLanguage: data.query.language, }; } @@ -63,57 +91,3 @@ const DEFAULT_QUERY = { export function getDefaultDatafeedQuery() { return cloneDeep(DEFAULT_QUERY); } - -export function createSearchItems( - kibanaConfig: IUiSettingsClient, - indexPattern: IndexPattern | undefined, - savedSearch: SavedSearchSavedObject | null -) { - // query is only used by the data visualizer as it needs - // a lucene query_string. - // Using a blank query will cause match_all:{} to be used - // when passed through luceneStringToDsl - let query: Query = { - query: '', - language: 'lucene', - }; - - let combinedQuery: any = getDefaultDatafeedQuery(); - if (savedSearch !== null) { - const data = getQueryFromSavedSearch(savedSearch); - - query = data.query; - const filter = data.filter; - - const filters = Array.isArray(filter) ? filter : []; - - if (query.language === SEARCH_QUERY_LANGUAGE.KUERY) { - const ast = esKuery.fromKueryExpression(query.query); - if (query.query !== '') { - combinedQuery = esKuery.toElasticsearchQuery(ast, indexPattern); - } - const filterQuery = esQuery.buildQueryFromFilters(filters, indexPattern); - - if (Array.isArray(combinedQuery.bool.filter) === false) { - combinedQuery.bool.filter = - combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; - } - - if (Array.isArray(combinedQuery.bool.must_not) === false) { - combinedQuery.bool.must_not = - combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; - } - - combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; - combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; - } else { - const esQueryConfigs = esQuery.getEsQueryConfig(kibanaConfig); - combinedQuery = esQuery.buildEsQuery(indexPattern, [query], filters, esQueryConfigs); - } - } - - return { - query, - combinedQuery, - }; -} From 60aaa39f9c7fc92fa15a66a4fcee092d1e99a742 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 15:24:57 -0500 Subject: [PATCH 012/188] Refactor filter utilities --- .../index_data_visualizer_view.tsx | 2 +- .../services/discover_top_nav_link.ts | 30 +++------- .../utils/saved_search_utils.ts | 59 ++++++++++++------- 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 67d2ef0472a87..143503c2c3683 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -65,7 +65,7 @@ import { DatePickerWrapper } from '../../../common/components/date_picker_wrappe import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; import { HelpMenu } from '../../../common/components/help_menu'; import { TimeBuckets } from '../../services/time_buckets'; -import { createSearchItems, extractSearchData } from '../../utils/saved_search_utils'; +import { extractSearchData } from '../../utils/saved_search_utils'; import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; import { ResultLink } from '../../../common/components/results_links'; import { extractErrorProperties } from '../../utils/error_utils'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts index 282df11ade670..a5a3cd9893b54 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts @@ -5,21 +5,11 @@ * 2.0. */ import { i18n } from '@kbn/i18n'; -import { - fromKueryExpression, - toElasticsearchQuery, - luceneStringToDsl, - decorateQuery, -} from '@kbn/es-query'; -import { - RefreshInterval, - SerializableState, - UI_SETTINGS, -} from '../../../../../../../src/plugins/data/common'; +import { RefreshInterval, SerializableState } from '../../../../../../../src/plugins/data/common'; import { IndexDataVisualizerLocator } from '../locator'; import { isPopulatedObject } from '../../../../common/utils/object_utils'; import type { DiscoverSetup } from '../../../../../../../src/plugins/discover/public'; -import { SEARCH_QUERY_LANGUAGE } from '../types/combined_query'; +import { createCombinedQuery } from '../utils/saved_search_utils'; export const DISCOVER_DV_TOP_NAV_LINK_ID = 'indexDataVisualizer'; @@ -52,7 +42,9 @@ export class DiscoverNavLinkRegistrar { testId: 'dataVisualizerTopNavButton', run: async () => { const { stateContainer, services, indexPattern, savedSearch, columns } = args; - const extractedQuery = stateContainer.appStateContainer.getState().query; + const state = stateContainer.appStateContainer.getState(); + const { query: extractedQuery, filters } = state; + const timeRange = services.timefilter.getTime(); const refreshInterval = services.timefilter.getRefreshInterval() as RefreshInterval & SerializableState; @@ -66,20 +58,14 @@ export class DiscoverNavLinkRegistrar { if (columns) { params.visibleFieldNames = columns; } + if (extractedQuery) { const queryLanguage = extractedQuery.language; const qryString = extractedQuery.query; - let qry; - if (queryLanguage === SEARCH_QUERY_LANGUAGE.KUERY) { - const ast = fromKueryExpression(qryString); - qry = toElasticsearchQuery(ast, indexPattern); - } else { - qry = luceneStringToDsl(qryString); - decorateQuery(qry, services.uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS)); - } + const combinedQuery = createCombinedQuery(extractedQuery, filters ?? []); params.query = { - searchQuery: qry as SerializableState, + searchQuery: combinedQuery as SerializableState, searchString: qryString, searchQueryLanguage: queryLanguage, }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index b467ed750d266..c3af526e37dcc 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -13,40 +13,29 @@ import { buildQueryFromFilters, buildEsQuery, Query, + Filter, } from '@kbn/es-query'; import { SavedSearchSavedObject } from '../../../../common/types'; import { IndexPattern } from '../../../../../../../src/plugins/data/common/index_patterns/index_patterns'; -import { SEARCH_QUERY_LANGUAGE } from '../types/combined_query'; +import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; import { getEsQueryConfig } from '../../../../../../../src/plugins/data/common'; export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject) { const search = savedSearch.attributes.kibanaSavedObjectMeta as { searchSourceJSON: string }; return JSON.parse(search.searchSourceJSON) as { query: Query; - filter: any[]; + filter: Filter[]; }; } -/** - * Extract query data from the saved search object. - */ -export function extractSearchData( - savedSearch: SavedSearchSavedObject | null, - indexPattern: IndexPattern, - uiSettings: IUiSettingsClient +export function createCombinedQuery( + query: Query, + filters: Filter[], + indexPattern?: IndexPattern, + uiSettings?: IUiSettingsClient ) { - if (!savedSearch) { - return undefined; - } - - const data = getQueryFromSavedSearch(savedSearch); let combinedQuery: any = getDefaultDatafeedQuery(); - const query = data.query; - const filter = data.filter; - - const filters = Array.isArray(filter) ? filter : []; - if (query.language === SEARCH_QUERY_LANGUAGE.KUERY) { const ast = fromKueryExpression(query.query); if (query.query !== '') { @@ -67,14 +56,40 @@ export function extractSearchData( combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; } else { - const esQueryConfigs = getEsQueryConfig(uiSettings); - combinedQuery = buildEsQuery(indexPattern, [query], filters, esQueryConfigs); + combinedQuery = buildEsQuery( + indexPattern, + [query], + filters, + uiSettings ? getEsQueryConfig(uiSettings) : undefined + ); } + return combinedQuery; +} + +/** + * Extract query data from the saved search object. + */ +export function extractSearchData( + savedSearch: SavedSearchSavedObject | null, + indexPattern: IndexPattern, + uiSettings: IUiSettingsClient +) { + if (!savedSearch) { + return undefined; + } + + const data = getQueryFromSavedSearch(savedSearch); + const combinedQuery = createCombinedQuery( + data.query, + Array.isArray(data.filter) ? data.filter : [], + indexPattern, + uiSettings + ); return { searchQuery: combinedQuery, searchString: data.query.query, - queryLanguage: data.query.language, + queryLanguage: data.query.language as SearchQueryLanguage, }; } From e65edcf2884d85d94ac6a88bd61a42d050975b5f Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 16:07:42 -0500 Subject: [PATCH 013/188] Add filter support for embeddable --- .../grid_embeddable/grid_embeddable.tsx | 54 ++++--------------- 1 file changed, 10 insertions(+), 44 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index debd09c152ba0..22d9d89d16c76 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -21,12 +21,6 @@ import React, { import useObservable from 'react-use/lib/useObservable'; import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; import { FormattedMessage } from '@kbn/i18n/react'; -import { - fromKueryExpression, - toElasticsearchQuery, - luceneStringToDsl, - decorateQuery, -} from '@kbn/es-query'; import { Embeddable, EmbeddableInput, @@ -84,8 +78,7 @@ const defaults = getDefaultPageState(); const useDataVisualizerGridData = ( input: DataVisualizerGridEmbeddableInput, - dataVisualizerListState: DataVisualizerIndexBasedAppState, - setDataVisualizerListState: Dispatch> + dataVisualizerListState: DataVisualizerIndexBasedAppState ) => { const { services } = useDataVisualizerKibana(); const { notifications, uiSettings } = services; @@ -106,33 +99,7 @@ const useDataVisualizerGridData = ( ); const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - // Prioritize current query in the search bar first - // because a saved search could have been previously loaded - // but the input has been changed - if (currentQuery) { - const queryLanguage = currentQuery.language; - const qryString = currentQuery.query; - let qry; - if (queryLanguage === 'kuery') { - const ast = fromKueryExpression(qryString); - qry = toElasticsearchQuery(ast, currentIndexPattern); - } else { - qry = luceneStringToDsl(qryString); - decorateQuery(qry, services.uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS)); - } - return { - searchQuery: qry, - searchString: qryString, - searchQueryLanguage: queryLanguage, - }; - } - - // Then process query from saved search - const searchData = extractSearchData( - currentSavedSearch, - currentIndexPattern, - uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS) - ); + const searchData = extractSearchData(currentSavedSearch, currentIndexPattern, uiSettings); if (searchData === undefined || dataVisualizerListState.searchString !== '') { return { @@ -140,15 +107,15 @@ const useDataVisualizerGridData = ( searchString: dataVisualizerListState.searchString, searchQueryLanguage: dataVisualizerListState.searchQueryLanguage, }; + } else { + return { + searchQuery: searchData.searchQuery, + searchString: searchData.searchString, + searchQueryLanguage: searchData.queryLanguage, + }; } - return { - searchQuery: searchData.searchQuery, - searchString: searchData.searchString, - searchQueryLanguage: searchData.queryLanguage, - }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState, currentQuery]); + }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState]); const [overallStats, setOverallStats] = useState(defaults.overallStats); @@ -645,8 +612,7 @@ export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddable const { configs, searchQueryLanguage, searchString, extendedColumns } = useDataVisualizerGridData( input, - dataVisualizerListState, - setDataVisualizerListState + dataVisualizerListState ); const getItemIdToExpandedRowMap = useCallback( function (itemIds: string[], items: FieldVisConfig[]): ItemIdToExpandedRowMap { From 19eaa696eca4a7c108cefedbddb796a374f865a9 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 16:14:59 -0500 Subject: [PATCH 014/188] Fix saved search data undefined --- .../index_data_visualizer/utils/saved_search_utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index 6567fb18ea203..d65348811535b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -87,6 +87,8 @@ export function extractSearchData( } const data = getQueryFromSavedSearch(savedSearch); + + if (!data) return; const combinedQuery = createCombinedQuery( data.query, Array.isArray(data.filter) ? data.filter : [], From 9215bcc245876c68b8713427c157db054b9c387d Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 16:48:05 -0500 Subject: [PATCH 015/188] Prototype aggregated view/document view switcher --- .../main/components/chart/discover_chart.tsx | 7 ++ .../components/layout/discover_layout.tsx | 90 +++++++++++-------- .../top_nav/open_options_popover.tsx | 23 +++++ .../data_visualizer_grid.tsx | 9 +- 4 files changed, 89 insertions(+), 40 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx index f1967d5b10b3e..55b4feec0c354 100644 --- a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx +++ b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx @@ -18,6 +18,7 @@ import { AppState, GetStateReturn } from '../../services/discover_state'; import { TimechartBucketInterval } from '../timechart_header/timechart_header'; import { Chart as IChart } from './point_series'; import { DiscoverHistogram } from './histogram'; +import { DocumentViewOption } from '../top_nav/open_options_popover'; const TimechartHeaderMemoized = React.memo(TimechartHeader); const DiscoverHistogramMemoized = React.memo(DiscoverHistogram); @@ -33,6 +34,8 @@ export function DiscoverChart({ state, stateContainer, timefield, + viewId, + setViewId, }: { config: IUiSettingsClient; data: DataPublicPluginStart; @@ -46,6 +49,8 @@ export function DiscoverChart({ state: AppState; stateContainer: GetStateReturn; timefield?: string; + viewId: string; + setViewId: (viewId: string) => void; }) { const chartRef = useRef<{ element: HTMLElement | null; moveFocus: boolean }>({ element: null, @@ -98,6 +103,7 @@ export function DiscoverChart({ onResetQuery={resetQuery} /> + {!state.hideChart && ( )} + {!state.hideChart && chartData && ( diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 98bf8e795f79f..a136e9c13e723 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -88,6 +88,8 @@ export function DiscoverLayout({ const sampleSize = useMemo(() => uiSettings.get(SAMPLE_SIZE_SETTING), [uiSettings]); const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); + const [viewId, setViewId] = useState(`discoverViewOptionDocument`); + const scrollableDesktop = useRef(null); const collapseIcon = useRef(null); @@ -336,6 +338,8 @@ export function DiscoverLayout({ savedSearch={savedSearch} stateContainer={stateContainer} timefield={timeField} + viewId={viewId} + setViewId={setViewId} /> @@ -353,61 +357,69 @@ export function DiscoverLayout({ defaultMessage="Documents" />

- - {isLegacy && rows && rows.length && ( - )} - {!isLegacy && rows && rows.length && ( -
- -
- )} + )} + {viewId === 'discoverViewOptionDocument' && + !isLegacy && + rows && + rows.length && ( +
+ +
+ )} diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx index 6e90c702c2bfd..38d6b1468aab7 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx @@ -20,6 +20,7 @@ import { EuiHorizontalRule, EuiButtonEmpty, EuiTextAlign, + EuiButtonGroup, } from '@elastic/eui'; import './open_options_popover.scss'; import { DOC_TABLE_LEGACY } from '../../../../../../common'; @@ -33,6 +34,28 @@ interface OptionsPopoverProps { anchorElement: HTMLElement; } +const toggleButtons = [ + { + id: `discoverViewOptionDocument`, + label: 'Document view', + }, + { + id: `discoverViewOptionAggregated`, + label: 'Aggregated view', + }, +]; + +export const DocumentViewOption = ({ viewId, setViewId }) => { + return ( + setViewId(id)} + /> + ); +}; export function OptionsPopover(props: OptionsPopoverProps) { const { core: { uiSettings }, diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index 8b5b56eda3578..34fd62e2bc182 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -112,5 +112,12 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp } }, [embeddable, embeddableRoot]); - return
; + return ( +
+ ); }; From 76f9ad697a1ba5fed4687ad0992c2bd95925847f Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 17:12:03 -0500 Subject: [PATCH 016/188] Prototype flyout --- .../components/layout/discover_layout.tsx | 1 + .../top_nav/data_visualizer_panel.tsx | 67 +++++++++++++++++++ .../components/top_nav/get_top_nav_links.ts | 22 ++++++ .../top_nav/open_options_popover.tsx | 10 ++- .../top_nav/show_data_visualizer_panel.tsx | 58 ++++++++++++++++ 5 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 src/plugins/discover/public/application/apps/main/components/top_nav/data_visualizer_panel.tsx create mode 100644 src/plugins/discover/public/application/apps/main/components/top_nav/show_data_visualizer_panel.tsx diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index a136e9c13e723..2566f36c860f4 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -88,6 +88,7 @@ export function DiscoverLayout({ const sampleSize = useMemo(() => uiSettings.get(SAMPLE_SIZE_SETTING), [uiSettings]); const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); + // remember in storage const [viewId, setViewId] = useState(`discoverViewOptionDocument`); const scrollableDesktop = useRef(null); diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/data_visualizer_panel.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/data_visualizer_panel.tsx new file mode 100644 index 0000000000000..6695694259cfb --- /dev/null +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/data_visualizer_panel.tsx @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { EuiFlyout, EuiFlyoutBody, EuiFlyoutFooter, EuiFlyoutHeader, EuiTitle } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import React, { useMemo } from 'react'; +import { IndexPattern } from '../../../../../kibana_services'; +import { SavedSearch } from '../../../../../saved_searches'; +import { DiscoverServices } from '../../../../../build_services'; +import { GetStateReturn } from '../../services/discover_state'; +import { DiscoverDataVisualizerGrid } from '../../../../components/data_visualizer_grid'; +import { SAMPLE_SIZE_SETTING } from '../../../../../../common'; + +const DataVisualizerGridPanelMemoized = React.memo(DiscoverDataVisualizerGrid); + +interface DataVisualizerPanelProps { + onClose: () => void; + indexPattern: IndexPattern; + savedSearch: SavedSearch; + services: DiscoverServices; + state: GetStateReturn; +} + +export function DataVisualizerPanel({ + onClose, + savedSearch, + services, + indexPattern, + state, +}: DataVisualizerPanelProps) { + const { uiSettings } = services; + const sampleSize = useMemo(() => uiSettings.get(SAMPLE_SIZE_SETTING), [uiSettings]); + const appState = state.appStateContainer.getState(); + + return ( + + + +

+ +

+ + + + + + + + ); +} diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts index 148d40155da6d..e694d4956a78c 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts @@ -18,6 +18,7 @@ import { GetStateReturn } from '../../services/discover_state'; import { IndexPattern, ISearchSource } from '../../../../../kibana_services'; import { openOptionsPopover } from './open_options_popover'; import type { TopNavMenuData } from '../../../../../../../navigation/public'; +import { showDataVisualizerPanel } from './show_data_visualizer_panel'; /** * Helper function to build the top nav links @@ -150,6 +151,26 @@ export const getTopNavLinks = ({ }, }; + const openDataVisualizerFlyout = { + id: 'inspect', + label: i18n.translate('discover.localMenu.openDataVisualizerFlyoutTitle', { + defaultMessage: 'Open Data visualizer flyout', + }), + description: i18n.translate('discover.localMenu.openDataVisualizerFlyoutDescription', { + defaultMessage: 'Open Data visualizer flyout', + }), + testId: 'openDataVisualizerFlyout', + run: () => { + showDataVisualizerPanel({ + I18nContext: services.core.i18n.Context, + indexPattern, + savedSearch, + services, + state, + }); + }, + }; + return [ ...(services.capabilities.advancedSettings.save ? [options] : []), newSearch, @@ -157,5 +178,6 @@ export const getTopNavLinks = ({ openSearch, shareSearch, inspectSearch, + openDataVisualizerFlyout, ]; }; diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx index 38d6b1468aab7..689149c52072d 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx @@ -45,14 +45,20 @@ const toggleButtons = [ }, ]; -export const DocumentViewOption = ({ viewId, setViewId }) => { +export const DocumentViewOption = ({ + viewId, + setViewId, +}: { + viewId: string; + setViewId: (id: string) => void; +}) => { return ( setViewId(id)} + onChange={(id: string) => setViewId(id)} /> ); }; diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/show_data_visualizer_panel.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/show_data_visualizer_panel.tsx new file mode 100644 index 0000000000000..b5ffafcfbde89 --- /dev/null +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/show_data_visualizer_panel.tsx @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { I18nStart } from 'kibana/public'; +import ReactDOM from 'react-dom'; +import React from 'react'; +import { DataVisualizerPanel } from './data_visualizer_panel'; +import { IndexPattern } from '../../../../../../../data/common'; +import { SavedSearch } from '../../../../../saved_searches'; +import { DiscoverServices } from '../../../../../build_services'; +import { GetStateReturn } from '../../services/discover_state'; + +let isOpen = false; + +export function showDataVisualizerPanel({ + I18nContext, + indexPattern, + savedSearch, + services, + state, +}: { + I18nContext: I18nStart['Context']; + indexPattern: IndexPattern; + savedSearch: SavedSearch; + services: DiscoverServices; + state: GetStateReturn; +}) { + if (isOpen) { + return; + } + + isOpen = true; + const container = document.createElement('div'); + const onClose = () => { + ReactDOM.unmountComponentAtNode(container); + document.body.removeChild(container); + isOpen = false; + }; + + document.body.appendChild(container); + const element = ( + + + + ); + ReactDOM.render(element, container); +} From 0e4bacb35e789afd7fff086869ea435d51b65b74 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 17:23:37 -0500 Subject: [PATCH 017/188] Prototype save document view option in storage --- .../apps/main/components/layout/discover_layout.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 2566f36c860f4..3971d49b9c0be 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -68,6 +68,8 @@ interface DiscoverLayoutFetchState extends SavedSearchDataMessage { rows: ElasticSearchHit[]; } +let storageViewPreference = 'discoverViewOptionDocument'; + export function DiscoverLayout({ indexPattern, indexPatternList, @@ -89,7 +91,13 @@ export function DiscoverLayout({ const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); // remember in storage - const [viewId, setViewId] = useState(`discoverViewOptionDocument`); + const [viewId, setViewId] = useState(storageViewPreference); + + const changeViewId = (option: string) => { + // @todo: temp hack to replace storage + storageViewPreference = option; + setViewId(option); + }; const scrollableDesktop = useRef(null); const collapseIcon = useRef(null); @@ -340,7 +348,7 @@ export function DiscoverLayout({ stateContainer={stateContainer} timefield={timeField} viewId={viewId} - setViewId={setViewId} + setViewId={changeViewId} /> From c40ec0ae9158debc65f088650aa66f27c2898008 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 28 Jul 2021 11:56:45 -0500 Subject: [PATCH 018/188] Fix filter and query conflict with saved search --- .../components/layout/discover_layout.tsx | 1 + .../data_visualizer_grid.tsx | 8 +- .../index_data_visualizer_view.tsx | 6 +- .../grid_embeddable/grid_embeddable.tsx | 48 ++++++++---- .../utils/saved_search_utils.ts | 76 +++++++++++++------ 5 files changed, 98 insertions(+), 41 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 3971d49b9c0be..8ebaea9e17d92 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -375,6 +375,7 @@ export function DiscoverLayout({ searchTitle={savedSearch.lastSavedTitle} sampleSize={sampleSize} query={state.query} + filters={state.filters} columns={columns} /> )} diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index 34fd62e2bc182..fa4c734db9c01 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -8,6 +8,7 @@ import React, { useEffect, useRef, useState } from 'react'; import { EuiDataGrid, EuiDataGridProps } from '@elastic/eui'; +import { Filter } from '@kbn/es-query'; import { IndexPattern, Query } from '../../../../../data/common'; import { DiscoverServices } from '../../../build_services'; import { ErrorEmbeddable, IEmbeddable, isErrorEmbeddable } from '../../../../../embeddable/public'; @@ -17,7 +18,6 @@ import type { DataVisualizerGridEmbeddableOutput, // eslint-disable-next-line @kbn/eslint/no-restricted-paths } from '../../../../../../../x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable'; - export interface DiscoverDataVisualizerGridProps { /** * Determines which columns are displayed @@ -45,6 +45,7 @@ export interface DiscoverDataVisualizerGridProps { services: DiscoverServices; savedSearch?: SavedSearch; query?: Query; + filters?: Filter[]; } export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { @@ -52,7 +53,7 @@ export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { }); export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { - const { services, indexPattern, savedSearch, query, columns } = props; + const { services, indexPattern, savedSearch, query, columns, filters } = props; const [embeddable, setEmbeddable] = useState< | ErrorEmbeddable | IEmbeddable @@ -67,11 +68,12 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp indexPattern, savedSearch, query, + filters, visibleFieldNames: columns, }); embeddable.reload(); } - }, [embeddable, indexPattern, savedSearch, query, columns]); + }, [embeddable, indexPattern, savedSearch, query, columns, filters]); useEffect(() => { return () => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index ea800293a0937..2650b97435dbc 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -224,7 +224,11 @@ export const IndexDataVisualizerView: FC = (dataVi const defaults = getDefaultPageState(); const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - const searchData = extractSearchData(currentSavedSearch, currentIndexPattern, uiSettings); + const searchData = extractSearchData({ + indexPattern: currentIndexPattern, + uiSettings, + savedSearch: currentSavedSearch, + }); if (searchData === undefined || dataVisualizerListState.searchString !== '') { return { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 22d9d89d16c76..b3514bff5444f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -8,19 +8,11 @@ import { merge, Observable, Subject } from 'rxjs'; import { CoreStart } from 'kibana/public'; import ReactDOM from 'react-dom'; -import React, { - Dispatch, - SetStateAction, - Suspense, - useCallback, - useEffect, - useMemo, - useRef, - useState, -} from 'react'; +import React, { Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import useObservable from 'react-use/lib/useObservable'; import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; import { FormattedMessage } from '@kbn/i18n/react'; +import { Filter } from '@kbn/es-query'; import { Embeddable, EmbeddableInput, @@ -68,6 +60,7 @@ export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { savedSearch?: SavedSearch; query?: Query; visibleFieldNames?: string[]; + filters?: Filter[]; } export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; @@ -88,18 +81,31 @@ const useDataVisualizerGridData = ( const [lastRefresh, setLastRefresh] = useState(0); - const { currentSavedSearch, currentIndexPattern, currentQuery, visibleFieldNames } = useMemo( + const { + currentSavedSearch, + currentIndexPattern, + currentQuery, + currentFilters, + visibleFieldNames, + } = useMemo( () => ({ currentSavedSearch: input?.savedSearch, currentIndexPattern: input.indexPattern, currentQuery: input?.query, visibleFieldNames: input?.visibleFieldNames ?? [], + currentFilters: input?.filters, }), [input] ); const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - const searchData = extractSearchData(currentSavedSearch, currentIndexPattern, uiSettings); + const searchData = extractSearchData({ + indexPattern: currentIndexPattern, + uiSettings, + savedSearch: currentSavedSearch, + query: currentQuery, + filters: currentFilters, + }); if (searchData === undefined || dataVisualizerListState.searchString !== '') { return { @@ -115,7 +121,13 @@ const useDataVisualizerGridData = ( }; } // eslint-disable-next-line react-hooks/exhaustive-deps - }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState]); + }, [ + currentSavedSearch, + currentIndexPattern, + dataVisualizerListState, + currentQuery, + currentFilters, + ]); const [overallStats, setOverallStats] = useState(defaults.overallStats); @@ -601,7 +613,15 @@ const useDataVisualizerGridData = ( return [actionColumn]; }, [input.indexPattern, services, searchQueryLanguage, searchString]); - return { configs, searchQueryLanguage, searchString, searchQuery, extendedColumns }; + return { + configs, + searchQueryLanguage, + searchString, + searchQuery, + extendedColumns, + documentCountStats, + metricsStats, + }; }; export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddableInput }) => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index d65348811535b..3b63f08ed1c3b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -75,32 +75,62 @@ export function createCombinedQuery( } /** - * Extract query data from the saved search object. + * Extract query data from the saved search object + * and merge with query data and filters */ -export function extractSearchData( - savedSearch: SavedSearchSavedObject | null, - indexPattern: IndexPattern, - uiSettings: IUiSettingsClient -) { - if (!savedSearch) { - return undefined; +export function extractSearchData({ + indexPattern, + uiSettings, + savedSearch, + query, + filters, +}: { + indexPattern: IndexPattern; + uiSettings: IUiSettingsClient; + savedSearch: SavedSearchSavedObject | SavedSearch | null | undefined; + query?: Query; + filters?: Filter[]; +}) { + if (!indexPattern || !savedSearch) return; + + const savedSearchData = getQueryFromSavedSearch(savedSearch); + const userQuery = query; + const userFilters = filters; + + // If no saved search available, use user's query and filters + if (!savedSearchData && userQuery) { + const combinedQuery = createCombinedQuery( + userQuery, + Array.isArray(userFilters) ? userFilters : [], + indexPattern, + uiSettings + ); + + return { + searchQuery: combinedQuery, + searchString: userQuery.query, + queryLanguage: userQuery.language as SearchQueryLanguage, + }; } - const data = getQueryFromSavedSearch(savedSearch); - - if (!data) return; - const combinedQuery = createCombinedQuery( - data.query, - Array.isArray(data.filter) ? data.filter : [], - indexPattern, - uiSettings - ); - - return { - searchQuery: combinedQuery, - searchString: data.query.query, - queryLanguage: data.query.language as SearchQueryLanguage, - }; + // If saved search available, merge saved search with latest user query or filters differ from extracted saved search data + if (savedSearchData) { + const currentQuery = userQuery ?? savedSearchData?.query; + const currentFilters = userFilters ?? savedSearchData?.filter; + + const combinedQuery = createCombinedQuery( + currentQuery, + Array.isArray(currentFilters) ? currentFilters : [], + indexPattern, + uiSettings + ); + + return { + searchQuery: combinedQuery, + searchString: currentQuery.query, + queryLanguage: currentQuery.language as SearchQueryLanguage, + }; + } } const DEFAULT_QUERY = { From 8f88ffdd87e848908c3b24d6d7d0120e2cb7eb3b Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 28 Jul 2021 12:15:14 -0500 Subject: [PATCH 019/188] Minor styling edits --- .../apps/main/components/layout/discover_layout.tsx | 3 ++- .../apps/main/components/top_nav/open_options_popover.tsx | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 8ebaea9e17d92..3822cfef2d86a 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -355,7 +355,8 @@ export function DiscoverLayout({
{ return ( Date: Mon, 19 Jul 2021 14:31:51 -0500 Subject: [PATCH 020/188] [ML] Initial embed --- .../components/layout/discover_layout.tsx | 20 ++ .../components/sidebar/discover_sidebar.tsx | 1 + .../data_visualizer_grid.tsx | 185 ++++++++++++++++++ .../components/data_visualizer_grid/index.ts | 9 + src/plugins/discover/public/build_services.ts | 3 + .../index_data_visualizer_view.tsx | 21 ++ .../embeddables/grid_embeddable/constants.ts | 8 + .../embeddable_loading_fallback.tsx | 20 ++ .../grid_embeddable/grid_embeddable.tsx | 96 +++++++++ .../grid_embeddable_factory.tsx | 99 ++++++++++ .../embeddables/grid_embeddable/index.ts | 0 .../embeddables/index.ts | 20 ++ .../plugins/data_visualizer/public/plugin.ts | 7 +- .../routes/new_job/index_or_search.tsx | 3 + 14 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx create mode 100644 src/plugins/discover/public/application/components/data_visualizer_grid/index.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/constants.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_loading_fallback.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 8cc3be810a7ee..51eac4baf74ee 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -51,10 +51,13 @@ import { DiscoverUninitialized } from '../uninitialized/uninitialized'; import { SavedSearchDataMessage } from '../../services/use_saved_search'; import { useDataGridColumns } from '../../../../helpers/use_data_grid_columns'; import { FetchStatus } from '../../../../types'; +import { DiscoverDataVisualizerGrid } from '../../../../components/data_visualizer_grid'; const DocTableLegacyMemoized = React.memo(DocTableLegacy); const SidebarMemoized = React.memo(DiscoverSidebarResponsive); const DataGridMemoized = React.memo(DiscoverGrid); +const DataVisualizerGridMemoized = React.memo(DiscoverDataVisualizerGrid); + const TopNavMemoized = React.memo(DiscoverTopNav); const DiscoverChartMemoized = React.memo(DiscoverChart); @@ -349,6 +352,23 @@ export function DiscoverLayout({ defaultMessage="Documents" /> + {isLegacy && rows && rows.length && ( void; + /** + * Function to add a filter in the grid cell or document flyout + */ + onFilter: DocViewFilterFn; + /** + * Function used in the grid header and flyout to remove a column + * @param column + */ + onRemoveColumn: (column: string) => void; + /** + * Function triggered when a column is resized by the user + */ + onResize?: (colSettings: { columnId: string; width: number }) => void; + /** + * Function to set all columns + */ + onSetColumns: (columns: string[]) => void; + /** + * function to change sorting of the documents, skipped when isSortEnabled is set to false + */ + onSort?: (sort: string[][]) => void; + /** + * Array of documents provided by Elasticsearch + */ + rows?: ElasticSearchHit[]; + /** + * The max size of the documents returned by Elasticsearch + */ + sampleSize: number; + /** + * Function to set the expanded document, which is displayed in a flyout + */ + setExpandedDoc: (doc: ElasticSearchHit | undefined) => void; + /** + * Grid display settings persisted in Elasticsearch (e.g. column width) + */ + settings?: any; + /** + * Saved search description + */ + searchDescription?: string; + /** + * Saved search title + */ + searchTitle?: string; + /** + * Discover plugin services + */ + services: DiscoverServices; + /** + * Determines whether the time columns should be displayed (legacy settings) + */ + showTimeCol: boolean; + /** + * Manage user sorting control + */ + isSortEnabled?: boolean; + /** + * Current sort setting + */ + sort: SortPairArr[]; + /** + * How the data is fetched + */ + useNewFieldsApi: boolean; + /** + * Manage pagination control + */ + isPaginationEnabled?: boolean; + /** + * List of used control columns (available: 'openDetails', 'select') + */ + controlColumnIds?: string[]; +} + +export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { + return ; +}); + +export const DiscoverDataVisualizerGrid = ({ + ariaLabelledBy, + columns, + indexPattern, + isLoading, + expandedDoc, + onAddColumn, + onFilter, + onRemoveColumn, + onResize, + onSetColumns, + onSort, + rows, + sampleSize, + searchDescription, + searchTitle, + services, + setExpandedDoc, + settings, + showTimeCol, + sort, + useNewFieldsApi, + isSortEnabled = true, + isPaginationEnabled = true, + controlColumnIds = ['openDetails', 'select'], + className, +}: DiscoverDataVisualizerGridProps) => { + const [embeddable, setEmbeddable] = useState(); + const embeddableRoot: React.RefObject = useRef(null); + + useEffect(() => { + const loadEmbeddable = async () => { + if (services?.embeddable) { + const factory = services.embeddable.getEmbeddableFactory('data_visualizer_grid'); + if (factory) { + const test = await factory.create({ id: 'test' }); + setEmbeddable(test); + } + } + }; + loadEmbeddable(); + }, [services?.embeddable]); + + // We can only render after embeddable has already initialized + useEffect(() => { + if (embeddableRoot.current && embeddable) { + embeddable.render(embeddableRoot.current); + } + }, [embeddable, embeddableRoot]); + + return ( + <> +
+ Hello embeddable + + ); +}; diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/index.ts b/src/plugins/discover/public/application/components/data_visualizer_grid/index.ts new file mode 100644 index 0000000000000..dc85495a7c2ec --- /dev/null +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { DiscoverDataVisualizerGrid } from './data_visualizer_grid'; diff --git a/src/plugins/discover/public/build_services.ts b/src/plugins/discover/public/build_services.ts index b42bf6a81742c..5052d1755c921 100644 --- a/src/plugins/discover/public/build_services.ts +++ b/src/plugins/discover/public/build_services.ts @@ -36,6 +36,7 @@ import { KibanaLegacyStart } from '../../kibana_legacy/public'; import { UrlForwardingStart } from '../../url_forwarding/public'; import { NavigationPublicPluginStart } from '../../navigation/public'; import { IndexPatternFieldEditorStart } from '../../index_pattern_field_editor/public'; +import { EmbeddableStart } from '../../embeddable/public'; export interface DiscoverServices { addBasePath: (path: string) => string; @@ -44,6 +45,7 @@ export interface DiscoverServices { core: CoreStart; data: DataPublicPluginStart; docLinks: DocLinksStart; + embeddable: EmbeddableStart; history: () => History; theme: ChartsPluginStart['theme']; filterManager: FilterManager; @@ -84,6 +86,7 @@ export async function buildServices( core, data: plugins.data, docLinks: core.docLinks, + embeddable: plugins.embeddable, theme: plugins.charts.theme, filterManager: plugins.data.query.filterManager, getEmbeddableInjector, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index f45c4a89d006c..f96eff6068cec 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -800,6 +800,27 @@ export const IndexDataVisualizerView: FC = (dataVi const helpLink = docLinks.links.ml.guide; + if (true) { + return ( + <> + + + + + items={configs} + pageState={dataVisualizerListState} + updatePageState={setDataVisualizerListState} + getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} + extendedColumns={extendedColumns} + /> + + ); + } return ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/constants.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/constants.ts new file mode 100644 index 0000000000000..26004db8fd529 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE = 'data_visualizer_grid'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_loading_fallback.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_loading_fallback.tsx new file mode 100644 index 0000000000000..01644efd6652c --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_loading_fallback.tsx @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { EuiLoadingSpinner, EuiSpacer, EuiText } from '@elastic/eui'; + +export const EmbeddableLoading = () => { + return ( + + + + + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx new file mode 100644 index 0000000000000..381d9cf493e57 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Subject } from 'rxjs'; +import { CoreStart } from 'kibana/public'; +import ReactDOM from 'react-dom'; +import React, { Suspense } from 'react'; +import { + Embeddable, + EmbeddableInput, + EmbeddableOutput, + IContainer, +} from '../../../../../../../../src/plugins/embeddable/public'; +import { KibanaContextProvider } from '../../../../../../../../src/plugins/kibana_react/public'; +import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; +import { EmbeddableLoading } from './embeddable_loading_fallback'; +import { IndexDataVisualizerView } from '../../components/index_data_visualizer_view'; +import { DataVisualizerUrlStateContextProvider } from '../../index_data_visualizer'; +import { DataVisualizerPluginStart } from '../../../../plugin'; + +export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerPluginStart]; +export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { + id: string; +} +export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; + +export type IDataVisualizerGridEmbeddable = typeof DataVisualizerGridEmbeddable; + +export class DataVisualizerGridEmbeddable extends Embeddable< + DataVisualizerGridEmbeddableInput, + DataVisualizerGridEmbeddableOutput +> { + private node?: HTMLElement; + private reload$ = new Subject(); + public readonly type: string = DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE; + + constructor( + initialInput: DataVisualizerGridEmbeddableInput, + public services: DataVisualizerGridEmbeddableServices, + parent?: IContainer + ) { + super(initialInput, {}, parent); + this.initializeOutput(initialInput); + } + + private async initializeOutput(initialInput: DataVisualizerGridEmbeddableInput) { + try { + // do something + } catch (e) { + // Unable to find and load index pattern but we can ignore the error + // as we only load it to support the filter & query bar + // the visualizations should still work correctly + } + } + + public render(node: HTMLElement) { + super.render(node); + this.node = node; + + const I18nContext = this.services[0].i18n.Context; + + ReactDOM.render( + + + }> + Hello World + {/* */} + + + , + node + ); + } + + public destroy() { + super.destroy(); + if (this.node) { + ReactDOM.unmountComponentAtNode(this.node); + } + } + + public reload() { + this.reload$.next(); + } + + public supportedTriggers() { + return []; + } +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx new file mode 100644 index 0000000000000..8eb38fd29dbc1 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// import { StartServicesAccessor } from 'kibana/public'; +import { i18n } from '@kbn/i18n'; +import { StartServicesAccessor } from 'kibana/public'; +import { + EmbeddableFactoryDefinition, + IContainer, +} from '../../../../../../../../src/plugins/embeddable/public'; +import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; +import { + DataVisualizerGridEmbeddableInput, + DataVisualizerGridEmbeddableServices, +} from './grid_embeddable'; +import { DataVisualizerPluginStart, DataVisualizerStartDependencies } from '../../../../plugin'; + +export class DataVisualizerGridEmbeddableFactory + implements EmbeddableFactoryDefinition { + public readonly type = DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE; + + public readonly grouping = [ + { + id: 'data_visualizer_grid', + getDisplayName: () => 'Data Visualizer Grid', + }, + ]; + + constructor( + private getStartServices: StartServicesAccessor< + DataVisualizerStartDependencies, + DataVisualizerPluginStart + > + ) {} + + public async isEditable() { + return false; + } + + public getDisplayName() { + return i18n.translate('xpack.dataVisualizer.index.components.grid.displayName', { + defaultMessage: 'Data Visualizer Grid', + }); + } + + public getDescription() { + return i18n.translate('xpack.dataVisualizer.index.components.grid.description', { + defaultMessage: 'Visualize data', + }); + } + + public async getExplicitInput(): Promise> { + // const [coreStart] = await this.getServices(); + // + // try { + // const { resolveEmbeddableDataVisualizerGridUserInput } = await import( + // './anomaly_charts_setup_flyout' + // ); + // return await resolveEmbeddableDataVisualizerGridUserInput(coreStart); + // } catch (e) { + // return Promise.reject(); + // } + } + + private async getServices(): Promise { + const [coreStart, pluginsStart] = await this.getStartServices(); + console.log('getServices', pluginsStart); + + // + // const { AnomalyDetectorService } = await import( + // '../../application/services/anomaly_detector_service' + // ); + // const { mlApiServicesProvider } = await import('../../application/services/ml_api_service'); + // const { mlResultsServiceProvider } = await import('../../application/services/results_service'); + // + // const httpService = new HttpService(coreStart.http); + // const anomalyDetectorService = new AnomalyDetectorService(httpService); + // const mlApiServices = mlApiServicesProvider(httpService); + // const mlResultsService = mlResultsServiceProvider(mlApiServices); + // + // const anomalyExplorerService = new AnomalyExplorerChartsService( + // pluginsStart.data.query.timefilter.timefilter, + // mlApiServices, + // mlResultsService + // ); + // + return [coreStart, pluginsStart]; + } + + public async create(initialInput: DataVisualizerGridEmbeddableInput, parent?: IContainer) { + const [coreStart, pluginsStart] = await this.getServices(); + const { DataVisualizerGridEmbeddable } = await import('./grid_embeddable'); + return new DataVisualizerGridEmbeddable(initialInput, [coreStart, pluginsStart], parent); + } +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts new file mode 100644 index 0000000000000..ef10f5bccf202 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup } from 'kibana/public'; +import { EmbeddableSetup } from '../../../../../../../src/plugins/embeddable/public'; +import { DataVisualizerGridEmbeddableFactory } from './grid_embeddable/grid_embeddable_factory'; + +export function registerEmbeddables(embeddable: EmbeddableSetup, core: CoreSetup) { + const dataVisualizerGridEmbeddableFactory = new DataVisualizerGridEmbeddableFactory( + core.getStartServices + ); + embeddable.registerEmbeddableFactory( + dataVisualizerGridEmbeddableFactory.type, + dataVisualizerGridEmbeddableFactory + ); +} diff --git a/x-pack/plugins/data_visualizer/public/plugin.ts b/x-pack/plugins/data_visualizer/public/plugin.ts index 4b71b08e9cf27..e4313172e037c 100644 --- a/x-pack/plugins/data_visualizer/public/plugin.ts +++ b/x-pack/plugins/data_visualizer/public/plugin.ts @@ -6,7 +6,7 @@ */ import { CoreSetup, CoreStart } from 'kibana/public'; -import type { EmbeddableStart } from '../../../../src/plugins/embeddable/public'; +import type { EmbeddableSetup, EmbeddableStart } from '../../../../src/plugins/embeddable/public'; import type { SharePluginStart } from '../../../../src/plugins/share/public'; import { Plugin } from '../../../../src/core/public'; @@ -21,9 +21,11 @@ import type { IndexPatternFieldEditorStart } from '../../../../src/plugins/index import { getFileDataVisualizerComponent, getIndexDataVisualizerComponent } from './api'; import { getMaxBytesFormatted } from './application/common/util/get_max_bytes'; import { registerHomeAddData, registerHomeFeatureCatalogue } from './register_home'; +import { registerEmbeddables } from './application/index_data_visualizer/embeddables'; export interface DataVisualizerSetupDependencies { home?: HomePublicPluginSetup; + embeddable: EmbeddableSetup; } export interface DataVisualizerStartDependencies { data: DataPublicPluginStart; @@ -52,6 +54,9 @@ export class DataVisualizerPlugin registerHomeAddData(plugins.home); registerHomeFeatureCatalogue(plugins.home); } + if (plugins.embeddable) { + registerEmbeddables(plugins.embeddable, core); + } } public start(core: CoreStart, plugins: DataVisualizerStartDependencies) { diff --git a/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx b/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx index 5d903bd865911..5420fbbfbec12 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx @@ -81,8 +81,11 @@ const PageWrapper: FC = ({ nextStepPath, deps, mode }) = services: { http: { basePath }, application: { navigateToUrl }, + embeddable: embeddablePlugin, }, } = useMlKibana(); + + console.log('embeddablePlugin', embeddablePlugin); const { redirectToMlAccessDeniedPage } = deps; const redirectToJobsManagementPage = useCreateAndNavigateToMlLink( ML_PAGES.ANOMALY_DETECTION_JOBS_MANAGE From dedbbec1f9fb584474bcfaf719874df2c30750c3 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 20 Jul 2021 09:16:34 -0500 Subject: [PATCH 021/188] [ML] Initial embed props --- .../components/layout/discover_layout.tsx | 13 +- .../data_visualizer_grid.tsx | 157 +++++------------- .../index_data_visualizer_view.tsx | 21 --- .../grid_embeddable/grid_embeddable.tsx | 38 +++-- .../grid_embeddable_factory.tsx | 33 ---- .../index_data_visualizer.tsx | 2 +- 6 files changed, 72 insertions(+), 192 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 51eac4baf74ee..698928ce13141 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -353,21 +353,14 @@ export function DiscoverLayout({ /> {isLegacy && rows && rows.length && ( void; - /** - * Function to add a filter in the grid cell or document flyout - */ - onFilter: DocViewFilterFn; - /** - * Function used in the grid header and flyout to remove a column - * @param column - */ - onRemoveColumn: (column: string) => void; - /** - * Function triggered when a column is resized by the user - */ - onResize?: (colSettings: { columnId: string; width: number }) => void; - /** - * Function to set all columns - */ - onSetColumns: (columns: string[]) => void; - /** - * function to change sorting of the documents, skipped when isSortEnabled is set to false - */ - onSort?: (sort: string[][]) => void; - /** - * Array of documents provided by Elasticsearch - */ - rows?: ElasticSearchHit[]; /** * The max size of the documents returned by Elasticsearch */ sampleSize: number; - /** - * Function to set the expanded document, which is displayed in a flyout - */ - setExpandedDoc: (doc: ElasticSearchHit | undefined) => void; - /** - * Grid display settings persisted in Elasticsearch (e.g. column width) - */ - settings?: any; /** * Saved search description */ @@ -92,77 +43,56 @@ export interface DiscoverDataVisualizerGridProps { * Discover plugin services */ services: DiscoverServices; - /** - * Determines whether the time columns should be displayed (legacy settings) - */ - showTimeCol: boolean; - /** - * Manage user sorting control - */ - isSortEnabled?: boolean; - /** - * Current sort setting - */ - sort: SortPairArr[]; - /** - * How the data is fetched - */ - useNewFieldsApi: boolean; - /** - * Manage pagination control - */ - isPaginationEnabled?: boolean; - /** - * List of used control columns (available: 'openDetails', 'select') - */ - controlColumnIds?: string[]; + savedSearch?: SavedSearch; + query?: Query; } export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { return ; }); -export const DiscoverDataVisualizerGrid = ({ - ariaLabelledBy, - columns, - indexPattern, - isLoading, - expandedDoc, - onAddColumn, - onFilter, - onRemoveColumn, - onResize, - onSetColumns, - onSort, - rows, - sampleSize, - searchDescription, - searchTitle, - services, - setExpandedDoc, - settings, - showTimeCol, - sort, - useNewFieldsApi, - isSortEnabled = true, - isPaginationEnabled = true, - controlColumnIds = ['openDetails', 'select'], - className, -}: DiscoverDataVisualizerGridProps) => { - const [embeddable, setEmbeddable] = useState(); +export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { + const { services, indexPattern, savedSearch, query } = props; + const [embeddable, setEmbeddable] = useState< + | ErrorEmbeddable + | IEmbeddable + | undefined + >(); const embeddableRoot: React.RefObject = useRef(null); + useEffect(() => { + if (embeddable && !isErrorEmbeddable(embeddable)) { + // Update embeddable whenever one of the important input changes + embeddable.updateInput({ indexPattern, savedSearch, query }); + embeddable.reload(); + } + }, [embeddable, indexPattern, savedSearch, query]); + + useEffect(() => { + return () => { + // Clean up embeddable upon unmounting + if (embeddable) { + embeddable.destroy(); + } + }; + }, [embeddable]); + useEffect(() => { const loadEmbeddable = async () => { if (services?.embeddable) { - const factory = services.embeddable.getEmbeddableFactory('data_visualizer_grid'); + const factory = services.embeddable.getEmbeddableFactory< + DataVisualizerGridEmbeddableInput, + DataVisualizerGridEmbeddableOutput + >('data_visualizer_grid'); if (factory) { - const test = await factory.create({ id: 'test' }); + // Initialize embeddable with information available at mount + const test = await factory.create({ id: 'test', indexPattern, savedSearch, query }); setEmbeddable(test); } } }; loadEmbeddable(); + // eslint-disable-next-line react-hooks/exhaustive-deps }, [services?.embeddable]); // We can only render after embeddable has already initialized @@ -174,12 +104,7 @@ export const DiscoverDataVisualizerGrid = ({ return ( <> -
- Hello embeddable +
); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index f96eff6068cec..f45c4a89d006c 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -800,27 +800,6 @@ export const IndexDataVisualizerView: FC = (dataVi const helpLink = docLinks.links.ml.guide; - if (true) { - return ( - <> - - - - - items={configs} - pageState={dataVisualizerListState} - updatePageState={setDataVisualizerListState} - getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} - extendedColumns={extendedColumns} - /> - - ); - } return ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 381d9cf493e57..f647c51d8f3e8 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -5,10 +5,11 @@ * 2.0. */ -import { Subject } from 'rxjs'; +import { Observable, Subject } from 'rxjs'; import { CoreStart } from 'kibana/public'; import ReactDOM from 'react-dom'; import React, { Suspense } from 'react'; +import useObservable from 'react-use/lib/useObservable'; import { Embeddable, EmbeddableInput, @@ -18,18 +19,31 @@ import { import { KibanaContextProvider } from '../../../../../../../../src/plugins/kibana_react/public'; import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; import { EmbeddableLoading } from './embeddable_loading_fallback'; -import { IndexDataVisualizerView } from '../../components/index_data_visualizer_view'; -import { DataVisualizerUrlStateContextProvider } from '../../index_data_visualizer'; -import { DataVisualizerPluginStart } from '../../../../plugin'; - -export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerPluginStart]; +import { DataVisualizerStartDependencies } from '../../../../plugin'; +import { IndexPattern, Query } from '../../../../../../../../src/plugins/data/common'; +import { SavedSearch } from '../../../../../../../../src/plugins/discover/public'; +export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerStartDependencies]; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { - id: string; + indexPattern?: IndexPattern; + savedSearch?: SavedSearch; + query?: Query; } export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; export type IDataVisualizerGridEmbeddable = typeof DataVisualizerGridEmbeddable; +export const IndexDataVisualizerViewWrapper = (props: { + id: string; + embeddableContext: InstanceType; + embeddableInput: Readonly>; + // refresh: Observable; +}) => { + const { embeddableInput } = props; + + const data = useObservable(embeddableInput); + + return
Hello world 2
; +}; export class DataVisualizerGridEmbeddable extends Embeddable< DataVisualizerGridEmbeddableInput, DataVisualizerGridEmbeddableOutput @@ -68,10 +82,12 @@ export class DataVisualizerGridEmbeddable extends Embeddable< }> Hello World - {/* */} + , diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx index 8eb38fd29dbc1..1a08e1e16ac18 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx @@ -53,41 +53,8 @@ export class DataVisualizerGridEmbeddableFactory }); } - public async getExplicitInput(): Promise> { - // const [coreStart] = await this.getServices(); - // - // try { - // const { resolveEmbeddableDataVisualizerGridUserInput } = await import( - // './anomaly_charts_setup_flyout' - // ); - // return await resolveEmbeddableDataVisualizerGridUserInput(coreStart); - // } catch (e) { - // return Promise.reject(); - // } - } - private async getServices(): Promise { const [coreStart, pluginsStart] = await this.getStartServices(); - console.log('getServices', pluginsStart); - - // - // const { AnomalyDetectorService } = await import( - // '../../application/services/anomaly_detector_service' - // ); - // const { mlApiServicesProvider } = await import('../../application/services/ml_api_service'); - // const { mlResultsServiceProvider } = await import('../../application/services/results_service'); - // - // const httpService = new HttpService(coreStart.http); - // const anomalyDetectorService = new AnomalyDetectorService(httpService); - // const mlApiServices = mlApiServicesProvider(httpService); - // const mlResultsService = mlResultsServiceProvider(mlApiServices); - // - // const anomalyExplorerService = new AnomalyExplorerChartsService( - // pluginsStart.data.query.timefilter.timefilter, - // mlApiServices, - // mlResultsService - // ); - // return [coreStart, pluginsStart]; } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx index 2835588625a6e..50c76c7155cb4 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx @@ -51,6 +51,7 @@ export const DataVisualizerUrlStateContextProvider: FC( undefined @@ -58,7 +59,6 @@ export const DataVisualizerUrlStateContextProvider: FC | null>( null ); - const { search: searchString } = useLocation(); useEffect(() => { const prevSearchString = searchString; From f5a563f9209c9b0b0bc74da246f6c2afc9766361 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 20 Jul 2021 14:45:38 -0500 Subject: [PATCH 022/188] Add embeddable 1 --- .../top_nav/discover_topnav.test.tsx | 12 +- .../top_nav/get_top_nav_links.test.ts | 7 + .../components/top_nav/get_top_nav_links.ts | 62 +++++++++ .../index_data_visualizer/locator/index.ts | 8 ++ .../locator/locator.test.ts | 111 +++++++++++++++ .../index_data_visualizer/locator/locator.ts | 130 ++++++++++++++++++ .../ml/public/locator/ml_locator.test.ts | 2 +- 7 files changed, 329 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/index.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.test.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx index 687532cd94f08..1a68bc4948c3a 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx @@ -41,13 +41,21 @@ describe('Discover topnav component', () => { const props = getProps(true); const component = shallowWithIntl(); const topMenuConfig = component.props().config.map((obj: TopNavMenuData) => obj.id); - expect(topMenuConfig).toEqual(['options', 'new', 'save', 'open', 'share', 'inspect']); + expect(topMenuConfig).toEqual([ + 'options', + 'new', + 'save', + 'open', + 'share', + 'inspect', + 'dataVisualizer', + ]); }); test('generated config of TopNavMenu config is correct when no discover save permissions are assigned', () => { const props = getProps(false); const component = shallowWithIntl(); const topMenuConfig = component.props().config.map((obj: TopNavMenuData) => obj.id); - expect(topMenuConfig).toEqual(['options', 'new', 'open', 'share', 'inspect']); + expect(topMenuConfig).toEqual(['options', 'new', 'open', 'share', 'inspect', 'dataVisualizer']); }); }); diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts index 6a6fb8a44a5cf..fae1c54b07211 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts @@ -80,6 +80,13 @@ test('getTopNavLinks result', () => { "run": [Function], "testId": "openInspectorButton", }, + Object { + "description": "Open in Data visualizer", + "id": "dataVisualizer", + "label": "Data visualizer", + "run": [Function], + "testId": "dataVisualizerTopNavButton", + }, ] `); }); diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts index f19b30cda5f8a..e44d1d5e3213f 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts @@ -8,6 +8,12 @@ import { i18n } from '@kbn/i18n'; import moment from 'moment'; +import { + fromKueryExpression, + toElasticsearchQuery, + luceneStringToDsl, + decorateQuery, +} from '@kbn/es-query'; import { showOpenSearchPanel } from './show_open_search_panel'; import { getSharingData, showPublicUrlSwitch } from '../../utils/get_sharing_data'; import { unhashUrl } from '../../../../../../../kibana_utils/public'; @@ -17,6 +23,8 @@ import { onSaveSearch } from './on_save_search'; import { GetStateReturn } from '../../services/discover_state'; import { IndexPattern, ISearchSource } from '../../../../../kibana_services'; import { openOptionsPopover } from './open_options_popover'; +import { RefreshInterval, UI_SETTINGS } from '../../../../../../../data/public'; +import { SerializableState } from '../../../../../../../kibana_utils/common'; /** * Helper function to build the top nav links @@ -149,6 +157,59 @@ export const getTopNavLinks = ({ }, }; + const dataVisualizer = { + id: 'dataVisualizer', + label: i18n.translate('discover.localMenu.dataVisualizerTitle', { + defaultMessage: 'Data visualizer', + }), + description: i18n.translate('discover.localMenu.dataVisualizerDescription', { + defaultMessage: 'Open in Data visualizer', + }), + testId: 'dataVisualizerTopNavButton', + run: async () => { + if (!services?.share?.url.locators) { + return; + } + const dvUrlGenerator = services.share.url.locators.get('DATA_VISUALIZER_APP_LOCATOR'); + const extractedQuery = state.appStateContainer.getState().query; + const timeRange = services.timefilter.getTime(); + const refreshInterval = services.timefilter.getRefreshInterval() as RefreshInterval & + SerializableState; + + const params: SerializableState = { + indexPatternId: indexPattern.id, + savedSearchId: savedSearch.id, + timeRange, + refreshInterval, + }; + if (extractedQuery) { + const queryLanguage = extractedQuery.language; + const qryString = extractedQuery.query; + let qry; + if (queryLanguage === 'kuery') { + const ast = fromKueryExpression(qryString); + qry = toElasticsearchQuery(ast, indexPattern); + } else { + qry = luceneStringToDsl(qryString); + decorateQuery(qry, services.uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS)); + } + + params.query = { + searchQuery: qry as SerializableState, + searchString: qryString, + searchQueryLanguage: queryLanguage, + }; + } + + const url = await dvUrlGenerator?.getUrl(params, { absolute: true }); + + // We want to open the Data visualizer in a new tab + if (url !== undefined) { + window.open(url, '_blank'); + } + }, + }; + return [ ...(services.capabilities.advancedSettings.save ? [options] : []), newSearch, @@ -156,5 +217,6 @@ export const getTopNavLinks = ({ openSearch, shareSearch, inspectSearch, + dataVisualizer, ]; }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/index.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/index.ts new file mode 100644 index 0000000000000..fb3e0100bbf75 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './locator'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.test.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.test.ts new file mode 100644 index 0000000000000..5fdbcea12006c --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.test.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndexDataVisualizerLocatorDefinition } from './locator'; + +describe('Index data visualizer locator', () => { + const definition = new IndexDataVisualizerLocatorDefinition(); + + it('should generate valid URL for the Index Data Visualizer Viewer page with global settings', async () => { + const location = await definition.getLocation({ + indexPatternId: '3da93760-e0af-11ea-9ad3-3bcfc330e42a', + timeRange: { + from: 'now-30m', + to: 'now', + }, + refreshInterval: { pause: false, value: 300 }, + }); + + expect(location).toMatchObject({ + app: 'ml', + path: + '/jobs/new_job/datavisualizer?index=3da93760-e0af-11ea-9ad3-3bcfc330e42a&_a=(DATA_VISUALIZER_INDEX_VIEWER:())&_g=(refreshInterval:(pause:!f,value:300),time:(from:now-30m,to:now))', + state: {}, + }); + }); + + it('should prioritize savedSearchId even when index pattern id is available', async () => { + const location = await definition.getLocation({ + indexPatternId: '3da93760-e0af-11ea-9ad3-3bcfc330e42a', + savedSearchId: '45014020-dffa-11eb-b120-a105fbbe93b3', + }); + + expect(location).toMatchObject({ + app: 'ml', + path: + '/jobs/new_job/datavisualizer?savedSearchId=45014020-dffa-11eb-b120-a105fbbe93b3&_a=(DATA_VISUALIZER_INDEX_VIEWER:())&_g=()', + state: {}, + }); + }); + + it('should generate valid URL with field names and field types', async () => { + const location = await definition.getLocation({ + indexPatternId: '3da93760-e0af-11ea-9ad3-3bcfc330e42a', + visibleFieldNames: ['@timestamp', 'responsetime'], + visibleFieldTypes: ['number'], + }); + + expect(location).toMatchObject({ + app: 'ml', + path: + "/jobs/new_job/datavisualizer?_a=(DATA_VISUALIZER_INDEX_VIEWER:(visibleFieldNames:!('@timestamp',responsetime),visibleFieldTypes:!(number)))&_g=()", + state: {}, + }); + }); + + it('should generate valid URL with KQL query', async () => { + const location = await definition.getLocation({ + indexPatternId: '3da93760-e0af-11ea-9ad3-3bcfc330e42a', + query: { + searchQuery: { + bool: { + should: [ + { + match: { + region: 'ap-northwest-1', + }, + }, + ], + minimum_should_match: 1, + }, + }, + searchString: 'region : ap-northwest-1', + searchQueryLanguage: 'kuery', + }, + }); + + expect(location).toMatchObject({ + app: 'ml', + path: + "/jobs/new_job/datavisualizer?index=3da93760-e0af-11ea-9ad3-3bcfc330e42a&_a=(DATA_VISUALIZER_INDEX_VIEWER:(searchQuery:(bool:(minimum_should_match:1,should:!((match:(region:ap-northwest-1))))),searchQueryLanguage:kuery,searchString:'region : ap-northwest-1'))&_g=()", + state: {}, + }); + }); + + it('should generate valid URL with Lucene query', async () => { + const location = await definition.getLocation({ + indexPatternId: '3da93760-e0af-11ea-9ad3-3bcfc330e42a', + query: { + searchQuery: { + query_string: { + query: 'region: ap-northwest-1', + analyze_wildcard: true, + }, + }, + searchString: 'region : ap-northwest-1', + searchQueryLanguage: 'lucene', + }, + }); + + expect(location).toMatchObject({ + app: 'ml', + path: + "/jobs/new_job/datavisualizer?index=3da93760-e0af-11ea-9ad3-3bcfc330e42a&_a=(DATA_VISUALIZER_INDEX_VIEWER:(searchQuery:(query_string:(analyze_wildcard:!t,query:'region: ap-northwest-1')),searchQueryLanguage:lucene,searchString:'region : ap-northwest-1'))&_g=()", + state: {}, + }); + }); +}); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts new file mode 100644 index 0000000000000..e64fbb8e02ec1 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// @ts-ignore +import { encode } from 'rison-node'; +import { stringify } from 'query-string'; +import { SerializableState } from '../../../../../../../src/plugins/kibana_utils/common'; +import { RefreshInterval, TimeRange } from '../../../../../../../src/plugins/data/common'; +import { LocatorDefinition, LocatorPublic } from '../../../../../../../src/plugins/share/common'; +import { QueryState } from '../../../../../../../src/plugins/data/public'; +import { Dictionary, isRisonSerializationRequired } from '../../common/util/url_state'; +import { SearchQueryLanguage } from '../types/combined_query'; + +export const DATA_VISUALIZER_APP_LOCATOR = 'DATA_VISUALIZER_APP_LOCATOR'; + +export interface IndexDataVisualizerLocatorParams extends SerializableState { + /** + * Optionally set saved search ID. + */ + savedSearchId?: string; + + /** + * Optionally set index pattern ID. + */ + indexPatternId?: string; + + /** + * Optionally set the time range in the time picker. + */ + timeRange?: TimeRange; + + /** + * Optionally set the refresh interval. + */ + refreshInterval?: RefreshInterval & SerializableState; + + /** + * Optionally set a query. + */ + query?: { + searchQuery: SerializableState; + searchString: string | SerializableState; + searchQueryLanguage: SearchQueryLanguage; + }; + + /** + * If not given, will use the uiSettings configuration for `storeInSessionStorage`. useHash determines + * whether to hash the data in the url to avoid url length issues. + */ + useHash?: boolean; + /** + * Optionally set visible field names. + */ + visibleFieldNames?: string[]; + /** + * Optionally set visible field types. + */ + visibleFieldTypes?: string[]; +} + +export type IndexDataVisualizerLocator = LocatorPublic; + +export class IndexDataVisualizerLocatorDefinition + implements LocatorDefinition { + public readonly id = DATA_VISUALIZER_APP_LOCATOR; + + constructor() {} + + public readonly getLocation = async (params: IndexDataVisualizerLocatorParams) => { + const { + indexPatternId, + query, + refreshInterval, + savedSearchId, + timeRange, + visibleFieldNames, + visibleFieldTypes, + } = params; + + const appState: { + searchQuery?: { [key: string]: any }; + searchQueryLanguage?: string; + searchString?: string | SerializableState; + visibleFieldNames?: string[]; + visibleFieldTypes?: string[]; + } = {}; + const queryState: QueryState = {}; + + if (query) { + appState.searchQuery = query.searchQuery; + appState.searchString = query.searchString; + appState.searchQueryLanguage = query.searchQueryLanguage; + } + if (visibleFieldNames) appState.visibleFieldNames = visibleFieldNames; + if (visibleFieldTypes) appState.visibleFieldTypes = visibleFieldTypes; + + if (timeRange) queryState.time = timeRange; + if (refreshInterval) queryState.refreshInterval = refreshInterval; + + const urlState: Dictionary = { + ...(savedSearchId ? { savedSearchId } : { index: indexPatternId }), + _a: { DATA_VISUALIZER_INDEX_VIEWER: appState }, + _g: queryState, + }; + + const parsedQueryString: Dictionary = {}; + Object.keys(urlState).forEach((a) => { + if (isRisonSerializationRequired(a)) { + parsedQueryString[a] = encode(urlState[a]); + } else { + parsedQueryString[a] = urlState[a]; + } + }); + const newLocationSearchString = stringify(parsedQueryString, { + sort: false, + encode: false, + }); + + const path = `/jobs/new_job/datavisualizer?${newLocationSearchString}`; + return { + app: 'ml', + path, + state: {}, + }; + }; +} diff --git a/x-pack/plugins/ml/public/locator/ml_locator.test.ts b/x-pack/plugins/ml/public/locator/ml_locator.test.ts index 97bb6cb2966f2..c033e00dea70f 100644 --- a/x-pack/plugins/ml/public/locator/ml_locator.test.ts +++ b/x-pack/plugins/ml/public/locator/ml_locator.test.ts @@ -9,7 +9,7 @@ import { MlLocatorDefinition } from './ml_locator'; import { ML_PAGES } from '../../common/constants/locator'; import { ANALYSIS_CONFIG_TYPE } from '../../common/constants/data_frame_analytics'; -describe('MlUrlGenerator', () => { +describe('ML locator', () => { const definition = new MlLocatorDefinition(); describe('AnomalyDetection', () => { From 215a9833dee117def9c13eeb73710fafbf044692 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 21 Jul 2021 15:47:30 -0500 Subject: [PATCH 023/188] Add visible fields --- .../main/components/layout/discover_layout.tsx | 1 + .../main/components/top_nav/discover_topnav.tsx | 14 +++++++++++++- .../main/components/top_nav/get_top_nav_links.ts | 6 ++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 698928ce13141..98bf8e795f79f 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -241,6 +241,7 @@ export function DiscoverLayout({ services={services} stateContainer={stateContainer} updateQuery={onUpdateQuery} + columns={columns} />

diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx index 9afda73401084..6d973a6f83ea9 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx @@ -21,6 +21,7 @@ export type DiscoverTopNavProps = Pick< savedQuery?: string; updateQuery: (payload: { dateRange: TimeRange; query?: Query }, isUpdate?: boolean) => void; stateContainer: GetStateReturn; + columns: string[]; }; export const DiscoverTopNav = ({ @@ -34,6 +35,7 @@ export const DiscoverTopNav = ({ navigateTo, savedSearch, services, + columns, }: DiscoverTopNavProps) => { const showDatePicker = useMemo(() => indexPattern.isTimeBased(), [indexPattern]); const { TopNavMenu } = services.navigation.ui; @@ -47,8 +49,18 @@ export const DiscoverTopNav = ({ state: stateContainer, onOpenInspector, searchSource, + columns, }), - [indexPattern, navigateTo, onOpenInspector, searchSource, stateContainer, savedSearch, services] + [ + columns, + indexPattern, + navigateTo, + onOpenInspector, + searchSource, + stateContainer, + savedSearch, + services, + ] ); const updateSavedQueryId = (newSavedQueryId: string | undefined) => { diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts index e44d1d5e3213f..d2a72dbcdcdb3 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts @@ -37,6 +37,7 @@ export const getTopNavLinks = ({ state, onOpenInspector, searchSource, + columns, }: { indexPattern: IndexPattern; navigateTo: (url: string) => void; @@ -45,6 +46,7 @@ export const getTopNavLinks = ({ state: GetStateReturn; onOpenInspector: () => void; searchSource: ISearchSource; + columns: string[]; }) => { const options = { id: 'options', @@ -182,6 +184,10 @@ export const getTopNavLinks = ({ timeRange, refreshInterval, }; + + if (columns) { + params.visibleFieldNames = columns; + } if (extractedQuery) { const queryLanguage = extractedQuery.language; const qryString = extractedQuery.query; From e76781cee2b6ff9ca37603569abc0a48ad49b0c4 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 22 Jul 2021 13:33:12 -0500 Subject: [PATCH 024/188] Embeddable 2 --- .../components/top_nav/discover_topnav.tsx | 62 ++++++++++-- .../components/top_nav/get_top_nav_links.ts | 78 ++------------- .../apps/main/services/add_top_nav_links.ts | 44 +++++++++ src/plugins/discover/public/plugin.tsx | 12 ++- .../services/discover_top_nav_link.ts | 96 +++++++++++++++++++ 5 files changed, 215 insertions(+), 77 deletions(-) create mode 100644 src/plugins/discover/public/application/apps/main/services/add_top_nav_links.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx index 6d973a6f83ea9..1c7a17cce747c 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx @@ -5,9 +5,9 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import React, { useMemo } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { DiscoverLayoutProps } from '../layout/types'; -import { getTopNavLinks } from './get_top_nav_links'; +import { getTopNavLinks, TopNavLink } from './get_top_nav_links'; import { Query, TimeRange } from '../../../../../../../data/common/query'; import { getHeaderActionMenuMounter } from '../../../../../kibana_services'; import { GetStateReturn } from '../../services/discover_state'; @@ -37,11 +37,60 @@ export const DiscoverTopNav = ({ services, columns, }: DiscoverTopNavProps) => { + const [registeredTopNavLinks, setRegisteredTopNavLinks] = useState([]); const showDatePicker = useMemo(() => indexPattern.isTimeBased(), [indexPattern]); const { TopNavMenu } = services.navigation.ui; + + useEffect(() => { + let unmounted = false; + + const loadRegisteredTopNavLinks = async () => { + const callbacks = services.addDataService.getTopNavLinks(); + const params = { + indexPattern, + onOpenInspector, + query, + savedQuery, + stateContainer, + updateQuery, + searchSource, + navigateTo, + savedSearch, + services, + columns, + }; + const links = await Promise.all( + callbacks.map((cb) => { + return cb(params); + }) + ); + if (!unmounted) { + setRegisteredTopNavLinks(links); + } + }; + loadRegisteredTopNavLinks(); + + return () => { + unmounted = true; + }; + }, [ + services?.addDataService, + indexPattern, + onOpenInspector, + query, + savedQuery, + stateContainer, + updateQuery, + searchSource, + navigateTo, + savedSearch, + services, + columns, + ]); + const topNavMenu = useMemo( - () => - getTopNavLinks({ + () => [ + ...getTopNavLinks({ indexPattern, navigateTo, savedSearch, @@ -49,10 +98,10 @@ export const DiscoverTopNav = ({ state: stateContainer, onOpenInspector, searchSource, - columns, }), + ...registeredTopNavLinks, + ], [ - columns, indexPattern, navigateTo, onOpenInspector, @@ -60,6 +109,7 @@ export const DiscoverTopNav = ({ stateContainer, savedSearch, services, + registeredTopNavLinks, ] ); diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts index d2a72dbcdcdb3..7e7f2f9ee057b 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts @@ -8,12 +8,6 @@ import { i18n } from '@kbn/i18n'; import moment from 'moment'; -import { - fromKueryExpression, - toElasticsearchQuery, - luceneStringToDsl, - decorateQuery, -} from '@kbn/es-query'; import { showOpenSearchPanel } from './show_open_search_panel'; import { getSharingData, showPublicUrlSwitch } from '../../utils/get_sharing_data'; import { unhashUrl } from '../../../../../../../kibana_utils/public'; @@ -23,8 +17,14 @@ import { onSaveSearch } from './on_save_search'; import { GetStateReturn } from '../../services/discover_state'; import { IndexPattern, ISearchSource } from '../../../../../kibana_services'; import { openOptionsPopover } from './open_options_popover'; -import { RefreshInterval, UI_SETTINGS } from '../../../../../../../data/public'; -import { SerializableState } from '../../../../../../../kibana_utils/common'; + +export interface TopNavLink { + id: string; + label: string; + description: string; + run: (anchorElement: HTMLElement) => void; + testId: string; +} /** * Helper function to build the top nav links @@ -37,7 +37,6 @@ export const getTopNavLinks = ({ state, onOpenInspector, searchSource, - columns, }: { indexPattern: IndexPattern; navigateTo: (url: string) => void; @@ -46,8 +45,7 @@ export const getTopNavLinks = ({ state: GetStateReturn; onOpenInspector: () => void; searchSource: ISearchSource; - columns: string[]; -}) => { +}): TopNavLink[] => { const options = { id: 'options', label: i18n.translate('discover.localMenu.localMenu.optionsTitle', { @@ -159,63 +157,6 @@ export const getTopNavLinks = ({ }, }; - const dataVisualizer = { - id: 'dataVisualizer', - label: i18n.translate('discover.localMenu.dataVisualizerTitle', { - defaultMessage: 'Data visualizer', - }), - description: i18n.translate('discover.localMenu.dataVisualizerDescription', { - defaultMessage: 'Open in Data visualizer', - }), - testId: 'dataVisualizerTopNavButton', - run: async () => { - if (!services?.share?.url.locators) { - return; - } - const dvUrlGenerator = services.share.url.locators.get('DATA_VISUALIZER_APP_LOCATOR'); - const extractedQuery = state.appStateContainer.getState().query; - const timeRange = services.timefilter.getTime(); - const refreshInterval = services.timefilter.getRefreshInterval() as RefreshInterval & - SerializableState; - - const params: SerializableState = { - indexPatternId: indexPattern.id, - savedSearchId: savedSearch.id, - timeRange, - refreshInterval, - }; - - if (columns) { - params.visibleFieldNames = columns; - } - if (extractedQuery) { - const queryLanguage = extractedQuery.language; - const qryString = extractedQuery.query; - let qry; - if (queryLanguage === 'kuery') { - const ast = fromKueryExpression(qryString); - qry = toElasticsearchQuery(ast, indexPattern); - } else { - qry = luceneStringToDsl(qryString); - decorateQuery(qry, services.uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS)); - } - - params.query = { - searchQuery: qry as SerializableState, - searchString: qryString, - searchQueryLanguage: queryLanguage, - }; - } - - const url = await dvUrlGenerator?.getUrl(params, { absolute: true }); - - // We want to open the Data visualizer in a new tab - if (url !== undefined) { - window.open(url, '_blank'); - } - }, - }; - return [ ...(services.capabilities.advancedSettings.save ? [options] : []), newSearch, @@ -223,6 +164,5 @@ export const getTopNavLinks = ({ openSearch, shareSearch, inspectSearch, - dataVisualizer, ]; }; diff --git a/src/plugins/discover/public/application/apps/main/services/add_top_nav_links.ts b/src/plugins/discover/public/application/apps/main/services/add_top_nav_links.ts new file mode 100644 index 0000000000000..7ab9d8bdb56eb --- /dev/null +++ b/src/plugins/discover/public/application/apps/main/services/add_top_nav_links.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { DiscoverTopNavProps } from '../components/top_nav/discover_topnav'; +import type { TopNavMenuData } from '../../../../../../navigation/public'; + +/** + * Async callback function that generates a TopNavMenuData object + * given the arguments provided + */ +export type AddNavLinkCallback = (params: DiscoverTopNavProps) => Promise; + +/** + * Service for other plugins to register additional links or actions + * within Discover (e.g. top navigation bar) + */ +export class AddDataService { + private TopNavMenuDatas: Record = {}; + + public setup() { + return { + /** + * Registers an async callback function that will return a valid top nav link action + */ + registerTopNavLinks: (id: string, callbackFn: AddNavLinkCallback) => { + if (this.TopNavMenuDatas[id]) { + throw new Error(`link ${id} already exists`); + } + this.TopNavMenuDatas[id] = callbackFn; + }, + }; + } + + public getTopNavMenuDatas() { + return Object.values(this.TopNavMenuDatas); + } +} + +export type AddDataServiceSetup = ReturnType; diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index 64f3c73eb4f7e..078eb6e888a04 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -64,6 +64,10 @@ import { UsageCollectionSetup } from '../../usage_collection/public'; import { replaceUrlHashQuery } from '../../kibana_utils/public/'; import { IndexPatternFieldEditorStart } from '../../../plugins/index_pattern_field_editor/public'; import { SourceViewer } from './application/components/source_viewer/source_viewer'; +import { + AddDataService, + AddDataServiceSetup, +} from './application/apps/main/services/add_top_nav_links'; declare module '../../share/public' { export interface UrlGeneratorStateMapping { @@ -115,6 +119,8 @@ export interface DiscoverSetup { * ``` */ readonly locator: undefined | DiscoverAppLocator; + // @todo: add documentation + addData: AddDataServiceSetup; } export interface DiscoverStart { @@ -207,7 +213,7 @@ export class DiscoverPlugin private stopUrlTracking: (() => void) | undefined = undefined; private servicesInitialized: boolean = false; private innerAngularInitialized: boolean = false; - + private readonly addDataService = new AddDataService(); /** * @deprecated */ @@ -389,6 +395,7 @@ export class DiscoverPlugin addDocView: this.docViewsRegistry.addDocView.bind(this.docViewsRegistry), }, locator: this.locator, + addData: { ...this.addDataService.setup() }, }; } @@ -424,7 +431,8 @@ export class DiscoverPlugin core, plugins, this.initializerContext, - this.getEmbeddableInjector + this.getEmbeddableInjector, + this.addDataService ); setServices(services); this.servicesInitialized = true; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts new file mode 100644 index 0000000000000..7a768d469a19c --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { i18n } from '@kbn/i18n'; +import { + fromKueryExpression, + toElasticsearchQuery, + luceneStringToDsl, + decorateQuery, +} from '@kbn/es-query'; +import { + RefreshInterval, + SerializableState, + UI_SETTINGS, +} from '../../../../../../../src/plugins/data/common'; +import { IndexDataVisualizerLocator } from '../locator'; +import { isPopulatedObject } from '../../../../common/utils/object_utils'; +import { DiscoverSetup } from '../../../../../../../src/plugins/discover/public'; + +export const DISCOVER_DV_TOP_NAV_LINK_ID = 'indexDataVisualizer'; + +export class DiscoverNavLinkRegistrar { + private readonly locator?: IndexDataVisualizerLocator; + public readonly id = DISCOVER_DV_TOP_NAV_LINK_ID; + + constructor(locator: IndexDataVisualizerLocator) { + this.locator = locator; + } + + registerDiscoverTopNavLink: Parameters< + DiscoverSetup['addData']['registerTopNavLinks'] + >[1] = async (args) => { + if (!this.locator) { + throw Error('IndexDataVisualizerLocator not available'); + } + if (!isPopulatedObject(args)) { + throw Error('Invalid arguments'); + } + + return { + id: DISCOVER_DV_TOP_NAV_LINK_ID, + label: i18n.translate('discover.localMenu.dataVisualizerTitle', { + defaultMessage: 'Data visualizer', + }), + description: i18n.translate('discover.localMenu.dataVisualizerDescription', { + defaultMessage: 'Open in Data visualizer', + }), + testId: 'dataVisualizerTopNavButton', + run: async () => { + const { stateContainer, services, indexPattern, savedSearch, columns } = args; + const extractedQuery = stateContainer.appStateContainer.getState().query; + const timeRange = services.timefilter.getTime(); + const refreshInterval = services.timefilter.getRefreshInterval() as RefreshInterval & + SerializableState; + const params: SerializableState = { + indexPatternId: indexPattern.id, + savedSearchId: savedSearch.id, + timeRange, + refreshInterval, + }; + + if (columns) { + params.visibleFieldNames = columns; + } + if (extractedQuery) { + const queryLanguage = extractedQuery.language; + const qryString = extractedQuery.query; + let qry; + if (queryLanguage === 'kuery') { + const ast = fromKueryExpression(qryString); + qry = toElasticsearchQuery(ast, indexPattern); + } else { + qry = luceneStringToDsl(qryString); + decorateQuery(qry, services.uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS)); + } + + params.query = { + searchQuery: qry as SerializableState, + searchString: qryString, + searchQueryLanguage: queryLanguage, + }; + } + + const url = await this.locator?.getUrl(params, { absolute: true }); + + // We want to open the Data visualizer in a new tab + if (url !== undefined) { + window.open(url, '_blank'); + } + }, + }; + }; +} From eb827fa7015eb8f58d728f285f396b9cd2c66502 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 15:08:24 -0500 Subject: [PATCH 025/188] Add filter support to query --- .../index_data_visualizer_view.tsx | 8 +- .../utils/saved_search_utils.ts | 114 +++++++----------- 2 files changed, 46 insertions(+), 76 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index f45c4a89d006c..67d2ef0472a87 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -65,7 +65,7 @@ import { DatePickerWrapper } from '../../../common/components/date_picker_wrappe import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; import { HelpMenu } from '../../../common/components/help_menu'; import { TimeBuckets } from '../../services/time_buckets'; -import { extractSearchData } from '../../utils/saved_search_utils'; +import { createSearchItems, extractSearchData } from '../../utils/saved_search_utils'; import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; import { ResultLink } from '../../../common/components/results_links'; import { extractErrorProperties } from '../../utils/error_utils'; @@ -224,11 +224,7 @@ export const IndexDataVisualizerView: FC = (dataVi const defaults = getDefaultPageState(); const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - const searchData = extractSearchData( - currentSavedSearch, - currentIndexPattern, - uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS) - ); + const searchData = extractSearchData(currentSavedSearch, currentIndexPattern, uiSettings); if (searchData === undefined || dataVisualizerListState.searchString !== '') { return { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index a04795ceb9d7d..b467ed750d266 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -7,10 +7,17 @@ import { cloneDeep } from 'lodash'; import { IUiSettingsClient } from 'kibana/public'; +import { + fromKueryExpression, + toElasticsearchQuery, + buildQueryFromFilters, + buildEsQuery, + Query, +} from '@kbn/es-query'; import { SavedSearchSavedObject } from '../../../../common/types'; import { IndexPattern } from '../../../../../../../src/plugins/data/common/index_patterns/index_patterns'; -import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; -import { esKuery, esQuery, Query } from '../../../../../../../src/plugins/data/public'; +import { SEARCH_QUERY_LANGUAGE } from '../types/combined_query'; +import { getEsQueryConfig } from '../../../../../../../src/plugins/data/common'; export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject) { const search = savedSearch.attributes.kibanaSavedObjectMeta as { searchSourceJSON: string }; @@ -25,28 +32,49 @@ export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject) { */ export function extractSearchData( savedSearch: SavedSearchSavedObject | null, - currentIndexPattern: IndexPattern, - queryStringOptions: Record | string + indexPattern: IndexPattern, + uiSettings: IUiSettingsClient ) { if (!savedSearch) { return undefined; } - const { query: extractedQuery } = getQueryFromSavedSearch(savedSearch); - const queryLanguage = extractedQuery.language as SearchQueryLanguage; - const qryString = extractedQuery.query; - let qry; - if (queryLanguage === SEARCH_QUERY_LANGUAGE.KUERY) { - const ast = esKuery.fromKueryExpression(qryString); - qry = esKuery.toElasticsearchQuery(ast, currentIndexPattern); + const data = getQueryFromSavedSearch(savedSearch); + let combinedQuery: any = getDefaultDatafeedQuery(); + + const query = data.query; + const filter = data.filter; + + const filters = Array.isArray(filter) ? filter : []; + + if (query.language === SEARCH_QUERY_LANGUAGE.KUERY) { + const ast = fromKueryExpression(query.query); + if (query.query !== '') { + combinedQuery = toElasticsearchQuery(ast, indexPattern); + } + const filterQuery = buildQueryFromFilters(filters, indexPattern); + + if (Array.isArray(combinedQuery.bool.filter) === false) { + combinedQuery.bool.filter = + combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; + } + + if (Array.isArray(combinedQuery.bool.must_not) === false) { + combinedQuery.bool.must_not = + combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; + } + + combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; + combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; } else { - qry = esQuery.luceneStringToDsl(qryString); - esQuery.decorateQuery(qry, queryStringOptions); + const esQueryConfigs = getEsQueryConfig(uiSettings); + combinedQuery = buildEsQuery(indexPattern, [query], filters, esQueryConfigs); } + return { - searchQuery: qry, - searchString: qryString, - queryLanguage, + searchQuery: combinedQuery, + searchString: data.query.query, + queryLanguage: data.query.language, }; } @@ -63,57 +91,3 @@ const DEFAULT_QUERY = { export function getDefaultDatafeedQuery() { return cloneDeep(DEFAULT_QUERY); } - -export function createSearchItems( - kibanaConfig: IUiSettingsClient, - indexPattern: IndexPattern | undefined, - savedSearch: SavedSearchSavedObject | null -) { - // query is only used by the data visualizer as it needs - // a lucene query_string. - // Using a blank query will cause match_all:{} to be used - // when passed through luceneStringToDsl - let query: Query = { - query: '', - language: 'lucene', - }; - - let combinedQuery: any = getDefaultDatafeedQuery(); - if (savedSearch !== null) { - const data = getQueryFromSavedSearch(savedSearch); - - query = data.query; - const filter = data.filter; - - const filters = Array.isArray(filter) ? filter : []; - - if (query.language === SEARCH_QUERY_LANGUAGE.KUERY) { - const ast = esKuery.fromKueryExpression(query.query); - if (query.query !== '') { - combinedQuery = esKuery.toElasticsearchQuery(ast, indexPattern); - } - const filterQuery = esQuery.buildQueryFromFilters(filters, indexPattern); - - if (Array.isArray(combinedQuery.bool.filter) === false) { - combinedQuery.bool.filter = - combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; - } - - if (Array.isArray(combinedQuery.bool.must_not) === false) { - combinedQuery.bool.must_not = - combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; - } - - combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; - combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; - } else { - const esQueryConfigs = esQuery.getEsQueryConfig(kibanaConfig); - combinedQuery = esQuery.buildEsQuery(indexPattern, [query], filters, esQueryConfigs); - } - } - - return { - query, - combinedQuery, - }; -} From b39328dc8e069e00d776a4096b602ba84799a5be Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 15:24:57 -0500 Subject: [PATCH 026/188] Refactor filter utilities --- .../index_data_visualizer_view.tsx | 2 +- .../services/discover_top_nav_link.ts | 23 +++----- .../utils/saved_search_utils.ts | 59 ++++++++++++------- 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 67d2ef0472a87..143503c2c3683 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -65,7 +65,7 @@ import { DatePickerWrapper } from '../../../common/components/date_picker_wrappe import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; import { HelpMenu } from '../../../common/components/help_menu'; import { TimeBuckets } from '../../services/time_buckets'; -import { createSearchItems, extractSearchData } from '../../utils/saved_search_utils'; +import { extractSearchData } from '../../utils/saved_search_utils'; import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; import { ResultLink } from '../../../common/components/results_links'; import { extractErrorProperties } from '../../utils/error_utils'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts index 7a768d469a19c..95f2944804bb2 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts @@ -5,20 +5,11 @@ * 2.0. */ import { i18n } from '@kbn/i18n'; -import { - fromKueryExpression, - toElasticsearchQuery, - luceneStringToDsl, - decorateQuery, -} from '@kbn/es-query'; -import { - RefreshInterval, - SerializableState, - UI_SETTINGS, -} from '../../../../../../../src/plugins/data/common'; +import { RefreshInterval, SerializableState } from '../../../../../../../src/plugins/data/common'; import { IndexDataVisualizerLocator } from '../locator'; import { isPopulatedObject } from '../../../../common/utils/object_utils'; -import { DiscoverSetup } from '../../../../../../../src/plugins/discover/public'; +import type { DiscoverSetup } from '../../../../../../../src/plugins/discover/public'; +import { createCombinedQuery } from '../utils/saved_search_utils'; export const DISCOVER_DV_TOP_NAV_LINK_ID = 'indexDataVisualizer'; @@ -51,7 +42,9 @@ export class DiscoverNavLinkRegistrar { testId: 'dataVisualizerTopNavButton', run: async () => { const { stateContainer, services, indexPattern, savedSearch, columns } = args; - const extractedQuery = stateContainer.appStateContainer.getState().query; + const state = stateContainer.appStateContainer.getState(); + const { query: extractedQuery, filters } = state; + const timeRange = services.timefilter.getTime(); const refreshInterval = services.timefilter.getRefreshInterval() as RefreshInterval & SerializableState; @@ -65,9 +58,11 @@ export class DiscoverNavLinkRegistrar { if (columns) { params.visibleFieldNames = columns; } + if (extractedQuery) { const queryLanguage = extractedQuery.language; const qryString = extractedQuery.query; + const combinedQuery = createCombinedQuery(extractedQuery, filters ?? []); let qry; if (queryLanguage === 'kuery') { const ast = fromKueryExpression(qryString); @@ -78,7 +73,7 @@ export class DiscoverNavLinkRegistrar { } params.query = { - searchQuery: qry as SerializableState, + searchQuery: combinedQuery as SerializableState, searchString: qryString, searchQueryLanguage: queryLanguage, }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index b467ed750d266..c3af526e37dcc 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -13,40 +13,29 @@ import { buildQueryFromFilters, buildEsQuery, Query, + Filter, } from '@kbn/es-query'; import { SavedSearchSavedObject } from '../../../../common/types'; import { IndexPattern } from '../../../../../../../src/plugins/data/common/index_patterns/index_patterns'; -import { SEARCH_QUERY_LANGUAGE } from '../types/combined_query'; +import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; import { getEsQueryConfig } from '../../../../../../../src/plugins/data/common'; export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject) { const search = savedSearch.attributes.kibanaSavedObjectMeta as { searchSourceJSON: string }; return JSON.parse(search.searchSourceJSON) as { query: Query; - filter: any[]; + filter: Filter[]; }; } -/** - * Extract query data from the saved search object. - */ -export function extractSearchData( - savedSearch: SavedSearchSavedObject | null, - indexPattern: IndexPattern, - uiSettings: IUiSettingsClient +export function createCombinedQuery( + query: Query, + filters: Filter[], + indexPattern?: IndexPattern, + uiSettings?: IUiSettingsClient ) { - if (!savedSearch) { - return undefined; - } - - const data = getQueryFromSavedSearch(savedSearch); let combinedQuery: any = getDefaultDatafeedQuery(); - const query = data.query; - const filter = data.filter; - - const filters = Array.isArray(filter) ? filter : []; - if (query.language === SEARCH_QUERY_LANGUAGE.KUERY) { const ast = fromKueryExpression(query.query); if (query.query !== '') { @@ -67,14 +56,40 @@ export function extractSearchData( combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; } else { - const esQueryConfigs = getEsQueryConfig(uiSettings); - combinedQuery = buildEsQuery(indexPattern, [query], filters, esQueryConfigs); + combinedQuery = buildEsQuery( + indexPattern, + [query], + filters, + uiSettings ? getEsQueryConfig(uiSettings) : undefined + ); } + return combinedQuery; +} + +/** + * Extract query data from the saved search object. + */ +export function extractSearchData( + savedSearch: SavedSearchSavedObject | null, + indexPattern: IndexPattern, + uiSettings: IUiSettingsClient +) { + if (!savedSearch) { + return undefined; + } + + const data = getQueryFromSavedSearch(savedSearch); + const combinedQuery = createCombinedQuery( + data.query, + Array.isArray(data.filter) ? data.filter : [], + indexPattern, + uiSettings + ); return { searchQuery: combinedQuery, searchString: data.query.query, - queryLanguage: data.query.language, + queryLanguage: data.query.language as SearchQueryLanguage, }; } From 51b43bdcbe78554220962c54a4fb50c1aeeff180 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 16:07:42 -0500 Subject: [PATCH 027/188] Add filter support for embeddable --- .../grid_embeddable/grid_embeddable.tsx | 632 +++++++++++++++++- 1 file changed, 624 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index f647c51d8f3e8..22d9d89d16c76 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -5,11 +5,22 @@ * 2.0. */ -import { Observable, Subject } from 'rxjs'; +import { merge, Observable, Subject } from 'rxjs'; import { CoreStart } from 'kibana/public'; import ReactDOM from 'react-dom'; -import React, { Suspense } from 'react'; +import React, { + Dispatch, + SetStateAction, + Suspense, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; import useObservable from 'react-use/lib/useObservable'; +import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; +import { FormattedMessage } from '@kbn/i18n/react'; import { Embeddable, EmbeddableInput, @@ -20,18 +31,621 @@ import { KibanaContextProvider } from '../../../../../../../../src/plugins/kiban import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; import { EmbeddableLoading } from './embeddable_loading_fallback'; import { DataVisualizerStartDependencies } from '../../../../plugin'; -import { IndexPattern, Query } from '../../../../../../../../src/plugins/data/common'; +import { + IndexPattern, + IndexPatternField, + KBN_FIELD_TYPES, + Query, + UI_SETTINGS, +} from '../../../../../../../../src/plugins/data/common'; import { SavedSearch } from '../../../../../../../../src/plugins/discover/public'; +import { + DataVisualizerTable, + ItemIdToExpandedRowMap, +} from '../../../common/components/stats_table'; +import { FieldVisConfig } from '../../../common/components/stats_table/types'; +import { MetricFieldsStats } from '../../../common/components/stats_table/components/field_count_stats'; +import { + getDefaultDataVisualizerListState, + getDefaultPageState, +} from '../../components/index_data_visualizer_view/index_data_visualizer_view'; + +import { extractSearchData } from '../../utils/saved_search_utils'; +import { useDataVisualizerKibana } from '../../../kibana_context'; +import { extractErrorProperties } from '../../utils/error_utils'; +import { DataLoader } from '../../data_loader/data_loader'; +import { useTimefilter } from '../../hooks/use_time_filter'; +import { FieldRequestConfig, JOB_FIELD_TYPES } from '../../../../../common'; +import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; +import { TimeBuckets } from '../../services/time_buckets'; +import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; +import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; +import { getActions } from '../../../common/components/field_data_row/action_menu'; +import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerStartDependencies]; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { - indexPattern?: IndexPattern; + indexPattern: IndexPattern; savedSearch?: SavedSearch; query?: Query; + visibleFieldNames?: string[]; } export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; export type IDataVisualizerGridEmbeddable = typeof DataVisualizerGridEmbeddable; +const restorableDefaults = getDefaultDataVisualizerListState(); +const defaults = getDefaultPageState(); + +const useDataVisualizerGridData = ( + input: DataVisualizerGridEmbeddableInput, + dataVisualizerListState: DataVisualizerIndexBasedAppState +) => { + const { services } = useDataVisualizerKibana(); + const { notifications, uiSettings } = services; + const { toasts } = notifications; + // @todo: consolidate dataVisualizerListState and input + const { samplerShardSize, visibleFieldTypes, showEmptyFields } = dataVisualizerListState; + + const [lastRefresh, setLastRefresh] = useState(0); + + const { currentSavedSearch, currentIndexPattern, currentQuery, visibleFieldNames } = useMemo( + () => ({ + currentSavedSearch: input?.savedSearch, + currentIndexPattern: input.indexPattern, + currentQuery: input?.query, + visibleFieldNames: input?.visibleFieldNames ?? [], + }), + [input] + ); + + const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { + const searchData = extractSearchData(currentSavedSearch, currentIndexPattern, uiSettings); + + if (searchData === undefined || dataVisualizerListState.searchString !== '') { + return { + searchQuery: dataVisualizerListState.searchQuery, + searchString: dataVisualizerListState.searchString, + searchQueryLanguage: dataVisualizerListState.searchQueryLanguage, + }; + } else { + return { + searchQuery: searchData.searchQuery, + searchString: searchData.searchString, + searchQueryLanguage: searchData.queryLanguage, + }; + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState]); + + const [overallStats, setOverallStats] = useState(defaults.overallStats); + + const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); + const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); + const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); + const [metricsStats, setMetricsStats] = useState(); + + const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); + const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); + + const dataLoader = useMemo(() => new DataLoader(currentIndexPattern, toasts), [ + currentIndexPattern, + toasts, + ]); + + const timefilter = useTimefilter({ + timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, + autoRefreshSelector: true, + }); + + useEffect(() => { + const timeUpdateSubscription = merge( + timefilter.getTimeUpdate$(), + dataVisualizerRefresh$ + ).subscribe(() => { + setLastRefresh(Date.now()); + }); + return () => { + timeUpdateSubscription.unsubscribe(); + }; + }); + + const getTimeBuckets = useCallback(() => { + return new TimeBuckets({ + [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), + [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), + dateFormat: uiSettings.get('dateFormat'), + 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), + }); + }, [uiSettings]); + + const indexPatternFields: IndexPatternField[] = useMemo(() => currentIndexPattern.fields, [ + currentIndexPattern, + ]); + + async function loadOverallStats() { + const tf = timefilter as any; + let earliest; + let latest; + + const activeBounds = tf.getActiveBounds(); + + if (currentIndexPattern.timeFieldName !== undefined && activeBounds === undefined) { + return; + } + + if (currentIndexPattern.timeFieldName !== undefined) { + earliest = activeBounds.min.valueOf(); + latest = activeBounds.max.valueOf(); + } + + try { + const allStats = await dataLoader.loadOverallData( + searchQuery, + samplerShardSize, + earliest, + latest + ); + // Because load overall stats perform queries in batches + // there could be multiple errors + if (Array.isArray(allStats.errors) && allStats.errors.length > 0) { + allStats.errors.forEach((err: any) => { + dataLoader.displayError(extractErrorProperties(err)); + }); + } + setOverallStats(allStats); + } catch (err) { + dataLoader.displayError(err.body ?? err); + } + } + + const createMetricCards = useCallback(() => { + const configs: FieldVisConfig[] = []; + const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; + + const allMetricFields = indexPatternFields.filter((f) => { + return ( + f.type === KBN_FIELD_TYPES.NUMBER && + f.displayName !== undefined && + dataLoader.isDisplayField(f.displayName) === true + ); + }); + const metricExistsFields = allMetricFields.filter((f) => { + return aggregatableExistsFields.find((existsF) => { + return existsF.fieldName === f.spec.name; + }); + }); + + // Add a config for 'document count', identified by no field name if indexpattern is time based. + if (currentIndexPattern.timeFieldName !== undefined) { + configs.push({ + type: JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + loading: true, + aggregatable: true, + }); + } + + if (metricsLoaded === false) { + setMetricsLoaded(true); + return; + } + + let aggregatableFields: any[] = overallStats.aggregatableExistsFields; + if (allMetricFields.length !== metricExistsFields.length && metricsLoaded === true) { + aggregatableFields = aggregatableFields.concat(overallStats.aggregatableNotExistsFields); + } + + const metricFieldsToShow = + metricsLoaded === true && showEmptyFields === true ? allMetricFields : metricExistsFields; + + metricFieldsToShow.forEach((field) => { + const fieldData = aggregatableFields.find((f) => { + return f.fieldName === field.spec.name; + }); + + const metricConfig: FieldVisConfig = { + ...(fieldData ? fieldData : {}), + fieldFormat: currentIndexPattern.getFormatterForField(field), + type: JOB_FIELD_TYPES.NUMBER, + loading: true, + aggregatable: true, + deletable: field.runtimeField !== undefined, + }; + if (field.displayName !== metricConfig.fieldName) { + metricConfig.displayName = field.displayName; + } + + configs.push(metricConfig); + }); + + setMetricsStats({ + totalMetricFieldsCount: allMetricFields.length, + visibleMetricsCount: metricFieldsToShow.length, + }); + setMetricConfigs(configs); + }, [ + currentIndexPattern, + dataLoader, + indexPatternFields, + metricsLoaded, + overallStats, + showEmptyFields, + ]); + + const createNonMetricCards = useCallback(() => { + const allNonMetricFields = indexPatternFields.filter((f) => { + return ( + f.type !== KBN_FIELD_TYPES.NUMBER && + f.displayName !== undefined && + dataLoader.isDisplayField(f.displayName) === true + ); + }); + // Obtain the list of all non-metric fields which appear in documents + // (aggregatable or not aggregatable). + const populatedNonMetricFields: any[] = []; // Kibana index pattern non metric fields. + let nonMetricFieldData: any[] = []; // Basic non metric field data loaded from requesting overall stats. + const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; + const nonAggregatableExistsFields: any[] = overallStats.nonAggregatableExistsFields || []; + + allNonMetricFields.forEach((f) => { + const checkAggregatableField = aggregatableExistsFields.find( + (existsField) => existsField.fieldName === f.spec.name + ); + + if (checkAggregatableField !== undefined) { + populatedNonMetricFields.push(f); + nonMetricFieldData.push(checkAggregatableField); + } else { + const checkNonAggregatableField = nonAggregatableExistsFields.find( + (existsField) => existsField.fieldName === f.spec.name + ); + + if (checkNonAggregatableField !== undefined) { + populatedNonMetricFields.push(f); + nonMetricFieldData.push(checkNonAggregatableField); + } + } + }); + + if (nonMetricsLoaded === false) { + setNonMetricsLoaded(true); + return; + } + + if (allNonMetricFields.length !== nonMetricFieldData.length && showEmptyFields === true) { + // Combine the field data obtained from Elasticsearch into a single array. + nonMetricFieldData = nonMetricFieldData.concat( + overallStats.aggregatableNotExistsFields, + overallStats.nonAggregatableNotExistsFields + ); + } + + const nonMetricFieldsToShow = showEmptyFields ? allNonMetricFields : populatedNonMetricFields; + + const configs: FieldVisConfig[] = []; + + nonMetricFieldsToShow.forEach((field) => { + const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); + + const nonMetricConfig = { + ...fieldData, + fieldFormat: currentIndexPattern.getFormatterForField(field), + aggregatable: field.aggregatable, + scripted: field.scripted, + loading: fieldData.existsInDocs, + deletable: field.runtimeField !== undefined, + }; + + // Map the field type from the Kibana index pattern to the field type + // used in the data visualizer. + const dataVisualizerType = kbnTypeToJobType(field); + if (dataVisualizerType !== undefined) { + nonMetricConfig.type = dataVisualizerType; + } else { + // Add a flag to indicate that this is one of the 'other' Kibana + // field types that do not yet have a specific card type. + nonMetricConfig.type = field.type; + nonMetricConfig.isUnsupportedType = true; + } + + if (field.displayName !== nonMetricConfig.fieldName) { + nonMetricConfig.displayName = field.displayName; + } + + configs.push(nonMetricConfig); + }); + + setNonMetricConfigs(configs); + }, [ + currentIndexPattern, + dataLoader, + indexPatternFields, + nonMetricsLoaded, + overallStats, + showEmptyFields, + ]); + + async function loadMetricFieldStats() { + // Only request data for fields that exist in documents. + if (metricConfigs.length === 0) { + return; + } + + const configsToLoad = metricConfigs.filter( + (config) => config.existsInDocs === true && config.loading === true + ); + if (configsToLoad.length === 0) { + return; + } + + // Pass the field name, type and cardinality in the request. + // Top values will be obtained on a sample if cardinality > 100000. + const existMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { + const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; + if (config.stats !== undefined && config.stats.cardinality !== undefined) { + props.cardinality = config.stats.cardinality; + } + return props; + }); + + // Obtain the interval to use for date histogram aggregations + // (such as the document count chart). Aim for 75 bars. + const buckets = getTimeBuckets(); + + const tf = timefilter as any; + let earliest: number | undefined; + let latest: number | undefined; + if (currentIndexPattern.timeFieldName !== undefined) { + earliest = tf.getActiveBounds().min.valueOf(); + latest = tf.getActiveBounds().max.valueOf(); + } + + const bounds = tf.getActiveBounds(); + const BAR_TARGET = 75; + buckets.setInterval('auto'); + buckets.setBounds(bounds); + buckets.setBarTarget(BAR_TARGET); + const aggInterval = buckets.getInterval(); + + try { + const metricFieldStats = await dataLoader.loadFieldStats( + searchQuery, + samplerShardSize, + earliest, + latest, + existMetricFields, + aggInterval.asMilliseconds() + ); + + // Add the metric stats to the existing stats in the corresponding config. + const configs: FieldVisConfig[] = []; + metricConfigs.forEach((config) => { + const configWithStats = { ...config }; + if (config.fieldName !== undefined) { + configWithStats.stats = { + ...configWithStats.stats, + ...metricFieldStats.find( + (fieldStats: any) => fieldStats.fieldName === config.fieldName + ), + }; + configWithStats.loading = false; + configs.push(configWithStats); + } else { + // Document count card. + configWithStats.stats = metricFieldStats.find( + (fieldStats: any) => fieldStats.fieldName === undefined + ); + + if (configWithStats.stats !== undefined) { + // Add earliest / latest of timefilter for setting x axis domain. + configWithStats.stats.timeRangeEarliest = earliest; + configWithStats.stats.timeRangeLatest = latest; + } + setDocumentCountStats(configWithStats); + } + }); + + setMetricConfigs(configs); + } catch (err) { + dataLoader.displayError(err); + } + } + + async function loadNonMetricFieldStats() { + // Only request data for fields that exist in documents. + if (nonMetricConfigs.length === 0) { + return; + } + + const configsToLoad = nonMetricConfigs.filter( + (config) => config.existsInDocs === true && config.loading === true + ); + if (configsToLoad.length === 0) { + return; + } + + // Pass the field name, type and cardinality in the request. + // Top values will be obtained on a sample if cardinality > 100000. + const existNonMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { + const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; + if (config.stats !== undefined && config.stats.cardinality !== undefined) { + props.cardinality = config.stats.cardinality; + } + return props; + }); + + const tf = timefilter as any; + let earliest; + let latest; + if (currentIndexPattern.timeFieldName !== undefined) { + earliest = tf.getActiveBounds().min.valueOf(); + latest = tf.getActiveBounds().max.valueOf(); + } + + try { + const nonMetricFieldStats = await dataLoader.loadFieldStats( + searchQuery, + samplerShardSize, + earliest, + latest, + existNonMetricFields + ); + + // Add the field stats to the existing stats in the corresponding config. + const configs: FieldVisConfig[] = []; + nonMetricConfigs.forEach((config) => { + const configWithStats = { ...config }; + if (config.fieldName !== undefined) { + configWithStats.stats = { + ...configWithStats.stats, + ...nonMetricFieldStats.find( + (fieldStats: any) => fieldStats.fieldName === config.fieldName + ), + }; + } + configWithStats.loading = false; + configs.push(configWithStats); + }); + + setNonMetricConfigs(configs); + } catch (err) { + dataLoader.displayError(err); + } + } + + useEffect(() => { + loadOverallStats(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [searchQuery, samplerShardSize, lastRefresh]); + + useEffect(() => { + createMetricCards(); + createNonMetricCards(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [overallStats, showEmptyFields]); + + useEffect(() => { + loadMetricFieldStats(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [metricConfigs]); + + useEffect(() => { + loadNonMetricFieldStats(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [nonMetricConfigs]); + + useEffect(() => { + createMetricCards(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [metricsLoaded]); + + useEffect(() => { + createNonMetricCards(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [nonMetricsLoaded]); + + const configs = useMemo(() => { + let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; + if (visibleFieldTypes && visibleFieldTypes.length > 0) { + combinedConfigs = combinedConfigs.filter( + (config) => visibleFieldTypes.findIndex((field) => field === config.type) > -1 + ); + } + if (visibleFieldNames && visibleFieldNames.length > 0) { + combinedConfigs = combinedConfigs.filter( + (config) => visibleFieldNames.findIndex((field) => field === config.fieldName) > -1 + ); + } + + return combinedConfigs; + }, [nonMetricConfigs, metricConfigs, visibleFieldTypes, visibleFieldNames]); + + // Some actions open up fly-out or popup + // This variable is used to keep track of them and clean up when unmounting + const actionFlyoutRef = useRef<() => void | undefined>(); + useEffect(() => { + const ref = actionFlyoutRef; + return () => { + // Clean up any of the flyout/editor opened from the actions + if (ref.current) { + ref.current(); + } + }; + }, []); + + // Inject custom action column for the index based visualizer + // Hide the column completely if no access to any of the plugins + const extendedColumns = useMemo(() => { + const actions = getActions( + input.indexPattern, + services, + { + searchQueryLanguage, + searchString, + }, + actionFlyoutRef + ); + if (!Array.isArray(actions) || actions.length < 1) return; + + const actionColumn: EuiTableActionsColumnType = { + name: ( + + ), + actions, + width: '100px', + }; + + return [actionColumn]; + }, [input.indexPattern, services, searchQueryLanguage, searchString]); + + return { configs, searchQueryLanguage, searchString, searchQuery, extendedColumns }; +}; + +export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddableInput }) => { + const [ + dataVisualizerListState, + setDataVisualizerListState, + ] = useState(restorableDefaults); + + const { configs, searchQueryLanguage, searchString, extendedColumns } = useDataVisualizerGridData( + input, + dataVisualizerListState + ); + const getItemIdToExpandedRowMap = useCallback( + function (itemIds: string[], items: FieldVisConfig[]): ItemIdToExpandedRowMap { + return itemIds.reduce((m: ItemIdToExpandedRowMap, fieldName: string) => { + const item = items.find((fieldVisConfig) => fieldVisConfig.fieldName === fieldName); + if (item !== undefined) { + m[fieldName] = ( + + ); + } + return m; + }, {} as ItemIdToExpandedRowMap); + }, + [input, searchQueryLanguage, searchString] + ); + + return ( + + items={configs} + pageState={dataVisualizerListState} + updatePageState={setDataVisualizerListState} + getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} + extendedColumns={extendedColumns} + /> + ); + + return
; +}; + export const IndexDataVisualizerViewWrapper = (props: { id: string; embeddableContext: InstanceType; @@ -40,9 +654,12 @@ export const IndexDataVisualizerViewWrapper = (props: { }) => { const { embeddableInput } = props; - const data = useObservable(embeddableInput); - - return
Hello world 2
; + const input = useObservable(embeddableInput); + if (input && input.indexPattern) { + return ; + } else { + return
; + } }; export class DataVisualizerGridEmbeddable extends Embeddable< DataVisualizerGridEmbeddableInput, @@ -81,7 +698,6 @@ export class DataVisualizerGridEmbeddable extends Embeddable< }> - Hello World Date: Tue, 27 Jul 2021 16:14:59 -0500 Subject: [PATCH 028/188] Fix saved search data undefined --- .../index_data_visualizer/utils/saved_search_utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index c3af526e37dcc..daee21102b752 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -79,6 +79,8 @@ export function extractSearchData( } const data = getQueryFromSavedSearch(savedSearch); + + if (!data) return; const combinedQuery = createCombinedQuery( data.query, Array.isArray(data.filter) ? data.filter : [], From 7f26d968ff48371c0443a1b32302e126a074cca0 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 16:48:05 -0500 Subject: [PATCH 029/188] Prototype aggregated view/document view switcher --- .../main/components/chart/discover_chart.tsx | 7 ++ .../components/layout/discover_layout.tsx | 90 +++++++++++-------- .../top_nav/open_options_popover.tsx | 23 +++++ .../data_visualizer_grid.tsx | 26 +++--- 4 files changed, 97 insertions(+), 49 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx index f1967d5b10b3e..55b4feec0c354 100644 --- a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx +++ b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx @@ -18,6 +18,7 @@ import { AppState, GetStateReturn } from '../../services/discover_state'; import { TimechartBucketInterval } from '../timechart_header/timechart_header'; import { Chart as IChart } from './point_series'; import { DiscoverHistogram } from './histogram'; +import { DocumentViewOption } from '../top_nav/open_options_popover'; const TimechartHeaderMemoized = React.memo(TimechartHeader); const DiscoverHistogramMemoized = React.memo(DiscoverHistogram); @@ -33,6 +34,8 @@ export function DiscoverChart({ state, stateContainer, timefield, + viewId, + setViewId, }: { config: IUiSettingsClient; data: DataPublicPluginStart; @@ -46,6 +49,8 @@ export function DiscoverChart({ state: AppState; stateContainer: GetStateReturn; timefield?: string; + viewId: string; + setViewId: (viewId: string) => void; }) { const chartRef = useRef<{ element: HTMLElement | null; moveFocus: boolean }>({ element: null, @@ -98,6 +103,7 @@ export function DiscoverChart({ onResetQuery={resetQuery} /> + {!state.hideChart && ( )} + {!state.hideChart && chartData && ( diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 98bf8e795f79f..a136e9c13e723 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -88,6 +88,8 @@ export function DiscoverLayout({ const sampleSize = useMemo(() => uiSettings.get(SAMPLE_SIZE_SETTING), [uiSettings]); const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); + const [viewId, setViewId] = useState(`discoverViewOptionDocument`); + const scrollableDesktop = useRef(null); const collapseIcon = useRef(null); @@ -336,6 +338,8 @@ export function DiscoverLayout({ savedSearch={savedSearch} stateContainer={stateContainer} timefield={timeField} + viewId={viewId} + setViewId={setViewId} /> @@ -353,61 +357,69 @@ export function DiscoverLayout({ defaultMessage="Documents" />

- - {isLegacy && rows && rows.length && ( - )} - {!isLegacy && rows && rows.length && ( -
- -
- )} + )} + {viewId === 'discoverViewOptionDocument' && + !isLegacy && + rows && + rows.length && ( +
+ +
+ )}
diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx index 6e90c702c2bfd..38d6b1468aab7 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx @@ -20,6 +20,7 @@ import { EuiHorizontalRule, EuiButtonEmpty, EuiTextAlign, + EuiButtonGroup, } from '@elastic/eui'; import './open_options_popover.scss'; import { DOC_TABLE_LEGACY } from '../../../../../../common'; @@ -33,6 +34,28 @@ interface OptionsPopoverProps { anchorElement: HTMLElement; } +const toggleButtons = [ + { + id: `discoverViewOptionDocument`, + label: 'Document view', + }, + { + id: `discoverViewOptionAggregated`, + label: 'Aggregated view', + }, +]; + +export const DocumentViewOption = ({ viewId, setViewId }) => { + return ( + setViewId(id)} + /> + ); +}; export function OptionsPopover(props: OptionsPopoverProps) { const { core: { uiSettings }, diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index e83fdcf3bc46e..8b5b56eda3578 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -52,7 +52,7 @@ export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { }); export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { - const { services, indexPattern, savedSearch, query } = props; + const { services, indexPattern, savedSearch, query, columns } = props; const [embeddable, setEmbeddable] = useState< | ErrorEmbeddable | IEmbeddable @@ -63,10 +63,15 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp useEffect(() => { if (embeddable && !isErrorEmbeddable(embeddable)) { // Update embeddable whenever one of the important input changes - embeddable.updateInput({ indexPattern, savedSearch, query }); + embeddable.updateInput({ + indexPattern, + savedSearch, + query, + visibleFieldNames: columns, + }); embeddable.reload(); } - }, [embeddable, indexPattern, savedSearch, query]); + }, [embeddable, indexPattern, savedSearch, query, columns]); useEffect(() => { return () => { @@ -86,8 +91,13 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp >('data_visualizer_grid'); if (factory) { // Initialize embeddable with information available at mount - const test = await factory.create({ id: 'test', indexPattern, savedSearch, query }); - setEmbeddable(test); + const initializedEmbeddable = await factory.create({ + id: 'discover_data_visualizer_grid', + indexPattern, + savedSearch, + query, + }); + setEmbeddable(initializedEmbeddable); } } }; @@ -102,9 +112,5 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp } }, [embeddable, embeddableRoot]); - return ( - <> -
- - ); + return
; }; From ad499395dfcfbdb7728a716f68322332add89227 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 17:12:03 -0500 Subject: [PATCH 030/188] Prototype flyout --- .../components/layout/discover_layout.tsx | 1 + .../top_nav/data_visualizer_panel.tsx | 67 +++++++++++++++++++ .../components/top_nav/get_top_nav_links.ts | 11 +-- .../top_nav/open_options_popover.tsx | 10 ++- .../top_nav/show_data_visualizer_panel.tsx | 58 ++++++++++++++++ 5 files changed, 136 insertions(+), 11 deletions(-) create mode 100644 src/plugins/discover/public/application/apps/main/components/top_nav/data_visualizer_panel.tsx create mode 100644 src/plugins/discover/public/application/apps/main/components/top_nav/show_data_visualizer_panel.tsx diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index a136e9c13e723..2566f36c860f4 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -88,6 +88,7 @@ export function DiscoverLayout({ const sampleSize = useMemo(() => uiSettings.get(SAMPLE_SIZE_SETTING), [uiSettings]); const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); + // remember in storage const [viewId, setViewId] = useState(`discoverViewOptionDocument`); const scrollableDesktop = useRef(null); diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/data_visualizer_panel.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/data_visualizer_panel.tsx new file mode 100644 index 0000000000000..6695694259cfb --- /dev/null +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/data_visualizer_panel.tsx @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { EuiFlyout, EuiFlyoutBody, EuiFlyoutFooter, EuiFlyoutHeader, EuiTitle } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import React, { useMemo } from 'react'; +import { IndexPattern } from '../../../../../kibana_services'; +import { SavedSearch } from '../../../../../saved_searches'; +import { DiscoverServices } from '../../../../../build_services'; +import { GetStateReturn } from '../../services/discover_state'; +import { DiscoverDataVisualizerGrid } from '../../../../components/data_visualizer_grid'; +import { SAMPLE_SIZE_SETTING } from '../../../../../../common'; + +const DataVisualizerGridPanelMemoized = React.memo(DiscoverDataVisualizerGrid); + +interface DataVisualizerPanelProps { + onClose: () => void; + indexPattern: IndexPattern; + savedSearch: SavedSearch; + services: DiscoverServices; + state: GetStateReturn; +} + +export function DataVisualizerPanel({ + onClose, + savedSearch, + services, + indexPattern, + state, +}: DataVisualizerPanelProps) { + const { uiSettings } = services; + const sampleSize = useMemo(() => uiSettings.get(SAMPLE_SIZE_SETTING), [uiSettings]); + const appState = state.appStateContainer.getState(); + + return ( + + + +

+ +

+ + + + + + + + ); +} diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts index 7e7f2f9ee057b..148d40155da6d 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.ts @@ -17,14 +17,7 @@ import { onSaveSearch } from './on_save_search'; import { GetStateReturn } from '../../services/discover_state'; import { IndexPattern, ISearchSource } from '../../../../../kibana_services'; import { openOptionsPopover } from './open_options_popover'; - -export interface TopNavLink { - id: string; - label: string; - description: string; - run: (anchorElement: HTMLElement) => void; - testId: string; -} +import type { TopNavMenuData } from '../../../../../../../navigation/public'; /** * Helper function to build the top nav links @@ -45,7 +38,7 @@ export const getTopNavLinks = ({ state: GetStateReturn; onOpenInspector: () => void; searchSource: ISearchSource; -}): TopNavLink[] => { +}): TopNavMenuData[] => { const options = { id: 'options', label: i18n.translate('discover.localMenu.localMenu.optionsTitle', { diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx index 38d6b1468aab7..689149c52072d 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx @@ -45,14 +45,20 @@ const toggleButtons = [ }, ]; -export const DocumentViewOption = ({ viewId, setViewId }) => { +export const DocumentViewOption = ({ + viewId, + setViewId, +}: { + viewId: string; + setViewId: (id: string) => void; +}) => { return ( setViewId(id)} + onChange={(id: string) => setViewId(id)} /> ); }; diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/show_data_visualizer_panel.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/show_data_visualizer_panel.tsx new file mode 100644 index 0000000000000..b5ffafcfbde89 --- /dev/null +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/show_data_visualizer_panel.tsx @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { I18nStart } from 'kibana/public'; +import ReactDOM from 'react-dom'; +import React from 'react'; +import { DataVisualizerPanel } from './data_visualizer_panel'; +import { IndexPattern } from '../../../../../../../data/common'; +import { SavedSearch } from '../../../../../saved_searches'; +import { DiscoverServices } from '../../../../../build_services'; +import { GetStateReturn } from '../../services/discover_state'; + +let isOpen = false; + +export function showDataVisualizerPanel({ + I18nContext, + indexPattern, + savedSearch, + services, + state, +}: { + I18nContext: I18nStart['Context']; + indexPattern: IndexPattern; + savedSearch: SavedSearch; + services: DiscoverServices; + state: GetStateReturn; +}) { + if (isOpen) { + return; + } + + isOpen = true; + const container = document.createElement('div'); + const onClose = () => { + ReactDOM.unmountComponentAtNode(container); + document.body.removeChild(container); + isOpen = false; + }; + + document.body.appendChild(container); + const element = ( + + + + ); + ReactDOM.render(element, container); +} From 25ac180b657c9ab8c21f8f59f11f5ca1dfb2c6b5 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Jul 2021 17:23:37 -0500 Subject: [PATCH 031/188] Prototype save document view option in storage --- .../apps/main/components/layout/discover_layout.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 2566f36c860f4..3971d49b9c0be 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -68,6 +68,8 @@ interface DiscoverLayoutFetchState extends SavedSearchDataMessage { rows: ElasticSearchHit[]; } +let storageViewPreference = 'discoverViewOptionDocument'; + export function DiscoverLayout({ indexPattern, indexPatternList, @@ -89,7 +91,13 @@ export function DiscoverLayout({ const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); // remember in storage - const [viewId, setViewId] = useState(`discoverViewOptionDocument`); + const [viewId, setViewId] = useState(storageViewPreference); + + const changeViewId = (option: string) => { + // @todo: temp hack to replace storage + storageViewPreference = option; + setViewId(option); + }; const scrollableDesktop = useRef(null); const collapseIcon = useRef(null); @@ -340,7 +348,7 @@ export function DiscoverLayout({ stateContainer={stateContainer} timefield={timeField} viewId={viewId} - setViewId={setViewId} + setViewId={changeViewId} /> From 534c70bd281260d291272e4f0a34299f7a9d24f5 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 28 Jul 2021 11:56:45 -0500 Subject: [PATCH 032/188] Fix filter and query conflict with saved search --- .../components/layout/discover_layout.tsx | 1 + .../data_visualizer_grid.tsx | 8 ++- .../index_data_visualizer_view.tsx | 6 +- .../grid_embeddable/grid_embeddable.tsx | 48 +++++++++---- .../utils/saved_search_utils.ts | 72 +++++++++++++------ 5 files changed, 96 insertions(+), 39 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 3971d49b9c0be..8ebaea9e17d92 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -375,6 +375,7 @@ export function DiscoverLayout({ searchTitle={savedSearch.lastSavedTitle} sampleSize={sampleSize} query={state.query} + filters={state.filters} columns={columns} /> )} diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index 8b5b56eda3578..bd6e0df4a67b1 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -8,6 +8,7 @@ import React, { useEffect, useRef, useState } from 'react'; import { EuiDataGrid, EuiDataGridProps } from '@elastic/eui'; +import { Filter } from '@kbn/es-query'; import { IndexPattern, Query } from '../../../../../data/common'; import { DiscoverServices } from '../../../build_services'; import { ErrorEmbeddable, IEmbeddable, isErrorEmbeddable } from '../../../../../embeddable/public'; @@ -17,7 +18,6 @@ import type { DataVisualizerGridEmbeddableOutput, // eslint-disable-next-line @kbn/eslint/no-restricted-paths } from '../../../../../../../x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable'; - export interface DiscoverDataVisualizerGridProps { /** * Determines which columns are displayed @@ -45,6 +45,7 @@ export interface DiscoverDataVisualizerGridProps { services: DiscoverServices; savedSearch?: SavedSearch; query?: Query; + filters?: Filter[]; } export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { @@ -52,7 +53,7 @@ export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { }); export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { - const { services, indexPattern, savedSearch, query, columns } = props; + const { services, indexPattern, savedSearch, query, columns, filters } = props; const [embeddable, setEmbeddable] = useState< | ErrorEmbeddable | IEmbeddable @@ -67,11 +68,12 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp indexPattern, savedSearch, query, + filters, visibleFieldNames: columns, }); embeddable.reload(); } - }, [embeddable, indexPattern, savedSearch, query, columns]); + }, [embeddable, indexPattern, savedSearch, query, columns, filters]); useEffect(() => { return () => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 143503c2c3683..e2554a1b91a43 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -224,7 +224,11 @@ export const IndexDataVisualizerView: FC = (dataVi const defaults = getDefaultPageState(); const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - const searchData = extractSearchData(currentSavedSearch, currentIndexPattern, uiSettings); + const searchData = extractSearchData({ + indexPattern: currentIndexPattern, + uiSettings, + savedSearch: currentSavedSearch, + }); if (searchData === undefined || dataVisualizerListState.searchString !== '') { return { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 22d9d89d16c76..b3514bff5444f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -8,19 +8,11 @@ import { merge, Observable, Subject } from 'rxjs'; import { CoreStart } from 'kibana/public'; import ReactDOM from 'react-dom'; -import React, { - Dispatch, - SetStateAction, - Suspense, - useCallback, - useEffect, - useMemo, - useRef, - useState, -} from 'react'; +import React, { Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import useObservable from 'react-use/lib/useObservable'; import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; import { FormattedMessage } from '@kbn/i18n/react'; +import { Filter } from '@kbn/es-query'; import { Embeddable, EmbeddableInput, @@ -68,6 +60,7 @@ export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { savedSearch?: SavedSearch; query?: Query; visibleFieldNames?: string[]; + filters?: Filter[]; } export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; @@ -88,18 +81,31 @@ const useDataVisualizerGridData = ( const [lastRefresh, setLastRefresh] = useState(0); - const { currentSavedSearch, currentIndexPattern, currentQuery, visibleFieldNames } = useMemo( + const { + currentSavedSearch, + currentIndexPattern, + currentQuery, + currentFilters, + visibleFieldNames, + } = useMemo( () => ({ currentSavedSearch: input?.savedSearch, currentIndexPattern: input.indexPattern, currentQuery: input?.query, visibleFieldNames: input?.visibleFieldNames ?? [], + currentFilters: input?.filters, }), [input] ); const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - const searchData = extractSearchData(currentSavedSearch, currentIndexPattern, uiSettings); + const searchData = extractSearchData({ + indexPattern: currentIndexPattern, + uiSettings, + savedSearch: currentSavedSearch, + query: currentQuery, + filters: currentFilters, + }); if (searchData === undefined || dataVisualizerListState.searchString !== '') { return { @@ -115,7 +121,13 @@ const useDataVisualizerGridData = ( }; } // eslint-disable-next-line react-hooks/exhaustive-deps - }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState]); + }, [ + currentSavedSearch, + currentIndexPattern, + dataVisualizerListState, + currentQuery, + currentFilters, + ]); const [overallStats, setOverallStats] = useState(defaults.overallStats); @@ -601,7 +613,15 @@ const useDataVisualizerGridData = ( return [actionColumn]; }, [input.indexPattern, services, searchQueryLanguage, searchString]); - return { configs, searchQueryLanguage, searchString, searchQuery, extendedColumns }; + return { + configs, + searchQueryLanguage, + searchString, + searchQuery, + extendedColumns, + documentCountStats, + metricsStats, + }; }; export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddableInput }) => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index daee21102b752..69d4cb97a7588 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -67,32 +67,62 @@ export function createCombinedQuery( } /** - * Extract query data from the saved search object. + * Extract query data from the saved search object + * and merge with query data and filters */ -export function extractSearchData( - savedSearch: SavedSearchSavedObject | null, - indexPattern: IndexPattern, - uiSettings: IUiSettingsClient -) { - if (!savedSearch) { - return undefined; +export function extractSearchData({ + indexPattern, + uiSettings, + savedSearch, + query, + filters, +}: { + indexPattern: IndexPattern; + uiSettings: IUiSettingsClient; + savedSearch: SavedSearchSavedObject | SavedSearch | null | undefined; + query?: Query; + filters?: Filter[]; +}) { + if (!indexPattern || !savedSearch) return; + + const savedSearchData = getQueryFromSavedSearch(savedSearch); + const userQuery = query; + const userFilters = filters; + + // If no saved search available, use user's query and filters + if (!savedSearchData && userQuery) { + const combinedQuery = createCombinedQuery( + userQuery, + Array.isArray(userFilters) ? userFilters : [], + indexPattern, + uiSettings + ); + + return { + searchQuery: combinedQuery, + searchString: userQuery.query, + queryLanguage: userQuery.language as SearchQueryLanguage, + }; } - const data = getQueryFromSavedSearch(savedSearch); + // If saved search available, merge saved search with latest user query or filters differ from extracted saved search data + if (savedSearchData) { + const currentQuery = userQuery ?? savedSearchData?.query; + const currentFilters = userFilters ?? savedSearchData?.filter; - if (!data) return; - const combinedQuery = createCombinedQuery( - data.query, - Array.isArray(data.filter) ? data.filter : [], - indexPattern, - uiSettings - ); + const combinedQuery = createCombinedQuery( + currentQuery, + Array.isArray(currentFilters) ? currentFilters : [], + indexPattern, + uiSettings + ); - return { - searchQuery: combinedQuery, - searchString: data.query.query, - queryLanguage: data.query.language as SearchQueryLanguage, - }; + return { + searchQuery: combinedQuery, + searchString: currentQuery.query, + queryLanguage: currentQuery.language as SearchQueryLanguage, + }; + } } const DEFAULT_QUERY = { From c09314ec30adafab0e8434fed2c490626362f4b4 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 28 Jul 2021 12:15:14 -0500 Subject: [PATCH 033/188] Minor styling edits --- .../apps/main/components/layout/discover_layout.tsx | 3 ++- .../apps/main/components/top_nav/open_options_popover.tsx | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 8ebaea9e17d92..3822cfef2d86a 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -355,7 +355,8 @@ export function DiscoverLayout({
{ return ( Date: Thu, 29 Jul 2021 10:04:48 -0500 Subject: [PATCH 034/188] Fix missing code after conflicts --- .../data_visualizer/common/types/index.ts | 5 +++++ .../index_data_visualizer_view.tsx | 2 +- .../utils/saved_search_utils.ts | 22 +++++++++++++------ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/data_visualizer/common/types/index.ts b/x-pack/plugins/data_visualizer/common/types/index.ts index 8b51142e19129..6ab7e6cccbe74 100644 --- a/x-pack/plugins/data_visualizer/common/types/index.ts +++ b/x-pack/plugins/data_visualizer/common/types/index.ts @@ -6,6 +6,7 @@ */ import type { SimpleSavedObject } from 'kibana/public'; +import { isPopulatedObject } from '../utils/object_utils'; export type { JobFieldType } from './job_field_type'; export type { FieldRequestConfig, @@ -27,3 +28,7 @@ export interface DataVisualizerTableState { } export type SavedSearchSavedObject = SimpleSavedObject; + +export function isSavedSearchSavedObject(arg: unknown): arg is SavedSearchSavedObject { + return isPopulatedObject(arg, ['id', 'type']); +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index e2554a1b91a43..3783d396bc9a5 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -85,7 +85,7 @@ const defaultSearchQuery = { match_all: {}, }; -function getDefaultPageState(): DataVisualizerPageState { +export function getDefaultPageState(): DataVisualizerPageState { return { overallStats: { totalCount: 0, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index 69d4cb97a7588..f3437df12ed62 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -15,17 +15,25 @@ import { Query, Filter, } from '@kbn/es-query'; -import { SavedSearchSavedObject } from '../../../../common/types'; +import { isSavedSearchSavedObject, SavedSearchSavedObject } from '../../../../common/types'; import { IndexPattern } from '../../../../../../../src/plugins/data/common/index_patterns/index_patterns'; import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; +import { SavedSearch } from '../../../../../../../src/plugins/discover/public'; import { getEsQueryConfig } from '../../../../../../../src/plugins/data/common'; -export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject) { - const search = savedSearch.attributes.kibanaSavedObjectMeta as { searchSourceJSON: string }; - return JSON.parse(search.searchSourceJSON) as { - query: Query; - filter: Filter[]; - }; +export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject | SavedSearch) { + // @todo: add type assertion for savedsearch + const search = isSavedSearchSavedObject(savedSearch) + ? savedSearch?.attributes?.kibanaSavedObjectMeta + : // @ts-expect-error kibanaSavedObjectMeta does exist + savedSearch?.kibanaSavedObjectMeta; + + return typeof search?.searchSourceJSON === 'string' + ? (JSON.parse(search.searchSourceJSON) as { + query: Query; + filter: any[]; + }) + : undefined; } export function createCombinedQuery( From a9169d7c3488e433c6aa7ef6263f7acf220762d8 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 29 Jul 2021 10:09:12 -0500 Subject: [PATCH 035/188] Remove dv locator and flyout --- .../components/layout/discover_layout.tsx | 1 - .../components/top_nav/discover_topnav.tsx | 66 +------------- .../top_nav/open_search_panel.test.tsx | 26 ------ .../top_nav/show_data_visualizer_panel.tsx | 58 ------------ src/plugins/discover/public/plugin.tsx | 11 +-- .../services/discover_top_nav_link.ts | 91 ------------------- 6 files changed, 4 insertions(+), 249 deletions(-) delete mode 100644 src/plugins/discover/public/application/apps/main/components/top_nav/open_search_panel.test.tsx delete mode 100644 src/plugins/discover/public/application/apps/main/components/top_nav/show_data_visualizer_panel.tsx delete mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 3822cfef2d86a..977dd28951255 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -252,7 +252,6 @@ export function DiscoverLayout({ services={services} stateContainer={stateContainer} updateQuery={onUpdateQuery} - columns={columns} />

diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx index 1c7a17cce747c..584a22838f74b 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.tsx @@ -5,9 +5,9 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import React, { useEffect, useMemo, useState } from 'react'; +import React, { useMemo } from 'react'; import { DiscoverLayoutProps } from '../layout/types'; -import { getTopNavLinks, TopNavLink } from './get_top_nav_links'; +import { getTopNavLinks } from './get_top_nav_links'; import { Query, TimeRange } from '../../../../../../../data/common/query'; import { getHeaderActionMenuMounter } from '../../../../../kibana_services'; import { GetStateReturn } from '../../services/discover_state'; @@ -21,7 +21,6 @@ export type DiscoverTopNavProps = Pick< savedQuery?: string; updateQuery: (payload: { dateRange: TimeRange; query?: Query }, isUpdate?: boolean) => void; stateContainer: GetStateReturn; - columns: string[]; }; export const DiscoverTopNav = ({ @@ -35,59 +34,10 @@ export const DiscoverTopNav = ({ navigateTo, savedSearch, services, - columns, }: DiscoverTopNavProps) => { - const [registeredTopNavLinks, setRegisteredTopNavLinks] = useState([]); const showDatePicker = useMemo(() => indexPattern.isTimeBased(), [indexPattern]); const { TopNavMenu } = services.navigation.ui; - useEffect(() => { - let unmounted = false; - - const loadRegisteredTopNavLinks = async () => { - const callbacks = services.addDataService.getTopNavLinks(); - const params = { - indexPattern, - onOpenInspector, - query, - savedQuery, - stateContainer, - updateQuery, - searchSource, - navigateTo, - savedSearch, - services, - columns, - }; - const links = await Promise.all( - callbacks.map((cb) => { - return cb(params); - }) - ); - if (!unmounted) { - setRegisteredTopNavLinks(links); - } - }; - loadRegisteredTopNavLinks(); - - return () => { - unmounted = true; - }; - }, [ - services?.addDataService, - indexPattern, - onOpenInspector, - query, - savedQuery, - stateContainer, - updateQuery, - searchSource, - navigateTo, - savedSearch, - services, - columns, - ]); - const topNavMenu = useMemo( () => [ ...getTopNavLinks({ @@ -99,18 +49,8 @@ export const DiscoverTopNav = ({ onOpenInspector, searchSource, }), - ...registeredTopNavLinks, ], - [ - indexPattern, - navigateTo, - onOpenInspector, - searchSource, - stateContainer, - savedSearch, - services, - registeredTopNavLinks, - ] + [indexPattern, navigateTo, onOpenInspector, searchSource, stateContainer, savedSearch, services] ); const updateSavedQueryId = (newSavedQueryId: string | undefined) => { diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/open_search_panel.test.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/open_search_panel.test.tsx deleted file mode 100644 index b34abab728eb1..0000000000000 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/open_search_panel.test.tsx +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import React from 'react'; -import { shallow } from 'enzyme'; - -jest.mock('../../../../../kibana_services', () => { - return { - getServices: () => ({ - core: { uiSettings: {}, savedObjects: {} }, - addBasePath: (path: string) => path, - }), - }; -}); - -import { OpenSearchPanel } from './open_search_panel'; - -test('render', () => { - const component = shallow(); - expect(component).toMatchSnapshot(); -}); diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/show_data_visualizer_panel.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/show_data_visualizer_panel.tsx deleted file mode 100644 index b5ffafcfbde89..0000000000000 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/show_data_visualizer_panel.tsx +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { I18nStart } from 'kibana/public'; -import ReactDOM from 'react-dom'; -import React from 'react'; -import { DataVisualizerPanel } from './data_visualizer_panel'; -import { IndexPattern } from '../../../../../../../data/common'; -import { SavedSearch } from '../../../../../saved_searches'; -import { DiscoverServices } from '../../../../../build_services'; -import { GetStateReturn } from '../../services/discover_state'; - -let isOpen = false; - -export function showDataVisualizerPanel({ - I18nContext, - indexPattern, - savedSearch, - services, - state, -}: { - I18nContext: I18nStart['Context']; - indexPattern: IndexPattern; - savedSearch: SavedSearch; - services: DiscoverServices; - state: GetStateReturn; -}) { - if (isOpen) { - return; - } - - isOpen = true; - const container = document.createElement('div'); - const onClose = () => { - ReactDOM.unmountComponentAtNode(container); - document.body.removeChild(container); - isOpen = false; - }; - - document.body.appendChild(container); - const element = ( - - - - ); - ReactDOM.render(element, container); -} diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index 078eb6e888a04..f23cac5440ed2 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -64,10 +64,6 @@ import { UsageCollectionSetup } from '../../usage_collection/public'; import { replaceUrlHashQuery } from '../../kibana_utils/public/'; import { IndexPatternFieldEditorStart } from '../../../plugins/index_pattern_field_editor/public'; import { SourceViewer } from './application/components/source_viewer/source_viewer'; -import { - AddDataService, - AddDataServiceSetup, -} from './application/apps/main/services/add_top_nav_links'; declare module '../../share/public' { export interface UrlGeneratorStateMapping { @@ -119,8 +115,6 @@ export interface DiscoverSetup { * ``` */ readonly locator: undefined | DiscoverAppLocator; - // @todo: add documentation - addData: AddDataServiceSetup; } export interface DiscoverStart { @@ -213,7 +207,6 @@ export class DiscoverPlugin private stopUrlTracking: (() => void) | undefined = undefined; private servicesInitialized: boolean = false; private innerAngularInitialized: boolean = false; - private readonly addDataService = new AddDataService(); /** * @deprecated */ @@ -395,7 +388,6 @@ export class DiscoverPlugin addDocView: this.docViewsRegistry.addDocView.bind(this.docViewsRegistry), }, locator: this.locator, - addData: { ...this.addDataService.setup() }, }; } @@ -431,8 +423,7 @@ export class DiscoverPlugin core, plugins, this.initializerContext, - this.getEmbeddableInjector, - this.addDataService + this.getEmbeddableInjector ); setServices(services); this.servicesInitialized = true; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts deleted file mode 100644 index 95f2944804bb2..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/discover_top_nav_link.ts +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { i18n } from '@kbn/i18n'; -import { RefreshInterval, SerializableState } from '../../../../../../../src/plugins/data/common'; -import { IndexDataVisualizerLocator } from '../locator'; -import { isPopulatedObject } from '../../../../common/utils/object_utils'; -import type { DiscoverSetup } from '../../../../../../../src/plugins/discover/public'; -import { createCombinedQuery } from '../utils/saved_search_utils'; - -export const DISCOVER_DV_TOP_NAV_LINK_ID = 'indexDataVisualizer'; - -export class DiscoverNavLinkRegistrar { - private readonly locator?: IndexDataVisualizerLocator; - public readonly id = DISCOVER_DV_TOP_NAV_LINK_ID; - - constructor(locator: IndexDataVisualizerLocator) { - this.locator = locator; - } - - registerDiscoverTopNavLink: Parameters< - DiscoverSetup['addData']['registerTopNavLinks'] - >[1] = async (args) => { - if (!this.locator) { - throw Error('IndexDataVisualizerLocator not available'); - } - if (!isPopulatedObject(args)) { - throw Error('Invalid arguments'); - } - - return { - id: DISCOVER_DV_TOP_NAV_LINK_ID, - label: i18n.translate('discover.localMenu.dataVisualizerTitle', { - defaultMessage: 'Data visualizer', - }), - description: i18n.translate('discover.localMenu.dataVisualizerDescription', { - defaultMessage: 'Open in Data visualizer', - }), - testId: 'dataVisualizerTopNavButton', - run: async () => { - const { stateContainer, services, indexPattern, savedSearch, columns } = args; - const state = stateContainer.appStateContainer.getState(); - const { query: extractedQuery, filters } = state; - - const timeRange = services.timefilter.getTime(); - const refreshInterval = services.timefilter.getRefreshInterval() as RefreshInterval & - SerializableState; - const params: SerializableState = { - indexPatternId: indexPattern.id, - savedSearchId: savedSearch.id, - timeRange, - refreshInterval, - }; - - if (columns) { - params.visibleFieldNames = columns; - } - - if (extractedQuery) { - const queryLanguage = extractedQuery.language; - const qryString = extractedQuery.query; - const combinedQuery = createCombinedQuery(extractedQuery, filters ?? []); - let qry; - if (queryLanguage === 'kuery') { - const ast = fromKueryExpression(qryString); - qry = toElasticsearchQuery(ast, indexPattern); - } else { - qry = luceneStringToDsl(qryString); - decorateQuery(qry, services.uiSettings.get(UI_SETTINGS.QUERY_STRING_OPTIONS)); - } - - params.query = { - searchQuery: combinedQuery as SerializableState, - searchString: qryString, - searchQueryLanguage: queryLanguage, - }; - } - - const url = await this.locator?.getUrl(params, { absolute: true }); - - // We want to open the Data visualizer in a new tab - if (url !== undefined) { - window.open(url, '_blank'); - } - }, - }; - }; -} From 545221b3b44eff925966ab4c81f4b3ac143bc4b6 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 29 Jul 2021 10:21:46 -0500 Subject: [PATCH 036/188] Make types happy --- .../open_search_panel.test.tsx.snap | 69 ------------------- .../top_nav/discover_topnav.test.tsx | 12 +--- .../top_nav/get_top_nav_links.test.ts | 7 -- .../grid_embeddable/grid_embeddable.tsx | 35 ++++------ 4 files changed, 17 insertions(+), 106 deletions(-) delete mode 100644 src/plugins/discover/public/application/apps/main/components/top_nav/__snapshots__/open_search_panel.test.tsx.snap diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/__snapshots__/open_search_panel.test.tsx.snap b/src/plugins/discover/public/application/apps/main/components/top_nav/__snapshots__/open_search_panel.test.tsx.snap deleted file mode 100644 index 2c2674b158bfc..0000000000000 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/__snapshots__/open_search_panel.test.tsx.snap +++ /dev/null @@ -1,69 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`render 1`] = ` - - - -

- -

- - - - - } - onChoose={[Function]} - savedObjectMetaData={ - Array [ - Object { - "getIconForSavedObject": [Function], - "name": "Saved search", - "type": "search", - }, - ] - } - savedObjects={Object {}} - uiSettings={Object {}} - /> - - - - - - - - - - - -`; diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx index 1a68bc4948c3a..687532cd94f08 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/discover_topnav.test.tsx @@ -41,21 +41,13 @@ describe('Discover topnav component', () => { const props = getProps(true); const component = shallowWithIntl(); const topMenuConfig = component.props().config.map((obj: TopNavMenuData) => obj.id); - expect(topMenuConfig).toEqual([ - 'options', - 'new', - 'save', - 'open', - 'share', - 'inspect', - 'dataVisualizer', - ]); + expect(topMenuConfig).toEqual(['options', 'new', 'save', 'open', 'share', 'inspect']); }); test('generated config of TopNavMenu config is correct when no discover save permissions are assigned', () => { const props = getProps(false); const component = shallowWithIntl(); const topMenuConfig = component.props().config.map((obj: TopNavMenuData) => obj.id); - expect(topMenuConfig).toEqual(['options', 'new', 'open', 'share', 'inspect', 'dataVisualizer']); + expect(topMenuConfig).toEqual(['options', 'new', 'open', 'share', 'inspect']); }); }); diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts index fae1c54b07211..6a6fb8a44a5cf 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/get_top_nav_links.test.ts @@ -80,13 +80,6 @@ test('getTopNavLinks result', () => { "run": [Function], "testId": "openInspectorButton", }, - Object { - "description": "Open in Data visualizer", - "id": "dataVisualizer", - "label": "Data visualizer", - "run": [Function], - "testId": "dataVisualizerTopNavButton", - }, ] `); }); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index b3514bff5444f..f7b30492361f6 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -13,6 +13,7 @@ import useObservable from 'react-use/lib/useObservable'; import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; import { FormattedMessage } from '@kbn/i18n/react'; import { Filter } from '@kbn/es-query'; +import { Required } from 'utility-types'; import { Embeddable, EmbeddableInput, @@ -47,7 +48,11 @@ import { useDataVisualizerKibana } from '../../../kibana_context'; import { extractErrorProperties } from '../../utils/error_utils'; import { DataLoader } from '../../data_loader/data_loader'; import { useTimefilter } from '../../hooks/use_time_filter'; -import { FieldRequestConfig, JOB_FIELD_TYPES } from '../../../../../common'; +import { + DataVisualizerTableState, + FieldRequestConfig, + JOB_FIELD_TYPES, +} from '../../../../../common'; import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; import { TimeBuckets } from '../../services/time_buckets'; import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; @@ -69,9 +74,10 @@ export type IDataVisualizerGridEmbeddable = typeof DataVisualizerGridEmbeddable; const restorableDefaults = getDefaultDataVisualizerListState(); const defaults = getDefaultPageState(); +// @todo: consolidate this hook with the view const useDataVisualizerGridData = ( input: DataVisualizerGridEmbeddableInput, - dataVisualizerListState: DataVisualizerIndexBasedAppState + dataVisualizerListState: Required ) => { const { services } = useDataVisualizerKibana(); const { notifications, uiSettings } = services; @@ -625,11 +631,13 @@ const useDataVisualizerGridData = ( }; export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddableInput }) => { - const [ - dataVisualizerListState, - setDataVisualizerListState, - ] = useState(restorableDefaults); + const [dataVisualizerListState, setDataVisualizerListState] = useState< + Required + >(restorableDefaults); + const onTableChange = (update: DataVisualizerTableState) => { + setDataVisualizerListState({ ...dataVisualizerListState, ...update }); + }; const { configs, searchQueryLanguage, searchString, extendedColumns } = useDataVisualizerGridData( input, dataVisualizerListState @@ -657,7 +665,7 @@ export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddable items={configs} pageState={dataVisualizerListState} - updatePageState={setDataVisualizerListState} + updatePageState={onTableChange} getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} extendedColumns={extendedColumns} /> @@ -670,7 +678,6 @@ export const IndexDataVisualizerViewWrapper = (props: { id: string; embeddableContext: InstanceType; embeddableInput: Readonly>; - // refresh: Observable; }) => { const { embeddableInput } = props; @@ -695,17 +702,6 @@ export class DataVisualizerGridEmbeddable extends Embeddable< parent?: IContainer ) { super(initialInput, {}, parent); - this.initializeOutput(initialInput); - } - - private async initializeOutput(initialInput: DataVisualizerGridEmbeddableInput) { - try { - // do something - } catch (e) { - // Unable to find and load index pattern but we can ignore the error - // as we only load it to support the filter & query bar - // the visualizations should still work correctly - } } public render(node: HTMLElement) { @@ -722,7 +718,6 @@ export class DataVisualizerGridEmbeddable extends Embeddable< id={this.input.id} embeddableContext={this} embeddableInput={this.getInput$()} - // refresh={this.reload$.asObservable()} /> From 5fe3c5f96e004c0a33cc2c6d8a7cc5f0d823f40d Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 29 Jul 2021 10:31:04 -0500 Subject: [PATCH 037/188] Fix types --- .../index_data_visualizer_view.tsx | 5 ++++- .../embeddables/grid_embeddable/grid_embeddable.tsx | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 3783d396bc9a5..aaef6b27e287d 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -103,7 +103,9 @@ export function getDefaultPageState(): DataVisualizerPageState { documentCountStats: undefined, }; } -export const getDefaultDataVisualizerListState = (): Required => ({ +export const getDefaultDataVisualizerListState = ( + overrides?: Partial +): Required => ({ pageIndex: 0, pageSize: 10, sortField: 'fieldName', @@ -117,6 +119,7 @@ export const getDefaultDataVisualizerListState = (): Required From 212869ab2356518daef36c966fe5d9cd22a182fd Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 29 Jul 2021 15:02:35 -0500 Subject: [PATCH 038/188] Rename toggle option --- .../main/components/chart/discover_chart.tsx | 15 ++++--- .../components/layout/discover_layout.tsx | 16 +++---- .../top_nav/open_options_popover.tsx | 32 -------------- .../components/view_mode_toggle/constants.ts | 12 +++++ .../main/components/view_mode_toggle/index.ts | 10 +++++ .../view_mode_toggle/view_mode_toggle.tsx | 44 +++++++++++++++++++ 6 files changed, 83 insertions(+), 46 deletions(-) create mode 100644 src/plugins/discover/public/application/apps/main/components/view_mode_toggle/constants.ts create mode 100644 src/plugins/discover/public/application/apps/main/components/view_mode_toggle/index.ts create mode 100644 src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx diff --git a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx index 55b4feec0c354..1718812b6cd25 100644 --- a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx +++ b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx @@ -18,7 +18,7 @@ import { AppState, GetStateReturn } from '../../services/discover_state'; import { TimechartBucketInterval } from '../timechart_header/timechart_header'; import { Chart as IChart } from './point_series'; import { DiscoverHistogram } from './histogram'; -import { DocumentViewOption } from '../top_nav/open_options_popover'; +import { DocumentViewModeToggle } from '../view_mode_toggle'; const TimechartHeaderMemoized = React.memo(TimechartHeader); const DiscoverHistogramMemoized = React.memo(DiscoverHistogram); @@ -34,8 +34,8 @@ export function DiscoverChart({ state, stateContainer, timefield, - viewId, - setViewId, + discoverViewMode, + setDiscoverViewMode, }: { config: IUiSettingsClient; data: DataPublicPluginStart; @@ -49,8 +49,8 @@ export function DiscoverChart({ state: AppState; stateContainer: GetStateReturn; timefield?: string; - viewId: string; - setViewId: (viewId: string) => void; + discoverViewMode: string; + setDiscoverViewMode: (discoverViewMode: string) => void; }) { const chartRef = useRef<{ element: HTMLElement | null; moveFocus: boolean }>({ element: null, @@ -134,7 +134,10 @@ export function DiscoverChart({ )} - + {!state.hideChart && chartData && ( diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 977dd28951255..efa6a8cb06bdb 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -52,6 +52,7 @@ import { SavedSearchDataMessage } from '../../services/use_saved_search'; import { useDataGridColumns } from '../../../../helpers/use_data_grid_columns'; import { FetchStatus } from '../../../../types'; import { DiscoverDataVisualizerGrid } from '../../../../components/data_visualizer_grid'; +import { DISCOVER_VIEW_MODES } from '../view_mode_toggle'; const DocTableLegacyMemoized = React.memo(DocTableLegacy); const SidebarMemoized = React.memo(DiscoverSidebarResponsive); @@ -90,13 +91,12 @@ export function DiscoverLayout({ const sampleSize = useMemo(() => uiSettings.get(SAMPLE_SIZE_SETTING), [uiSettings]); const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); - // remember in storage - const [viewId, setViewId] = useState(storageViewPreference); + const [discoverViewMode, setDiscoverViewMode] = useState(storageViewPreference); const changeViewId = (option: string) => { // @todo: temp hack to replace storage storageViewPreference = option; - setViewId(option); + setDiscoverViewMode(option); }; const scrollableDesktop = useRef(null); @@ -346,8 +346,8 @@ export function DiscoverLayout({ savedSearch={savedSearch} stateContainer={stateContainer} timefield={timeField} - viewId={viewId} - setViewId={changeViewId} + discoverViewMode={discoverViewMode} + setDiscoverViewMode={changeViewId} /> @@ -366,7 +366,7 @@ export function DiscoverLayout({ defaultMessage="Documents" />

- {viewId === 'discoverViewOptionAggregated' && ( + {discoverViewMode === DISCOVER_VIEW_MODES.FIELD_LEVEL && ( )} - {viewId === 'discoverViewOptionDocument' && + {discoverViewMode === DISCOVER_VIEW_MODES.DOCUMENT_LEVEL && isLegacy && rows && rows.length && ( @@ -400,7 +400,7 @@ export function DiscoverLayout({ useNewFieldsApi={useNewFieldsApi} /> )} - {viewId === 'discoverViewOptionDocument' && + {discoverViewMode === DISCOVER_VIEW_MODES.DOCUMENT_LEVEL && !isLegacy && rows && rows.length && ( diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx index 642388a48f6b9..6e90c702c2bfd 100644 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx +++ b/src/plugins/discover/public/application/apps/main/components/top_nav/open_options_popover.tsx @@ -20,7 +20,6 @@ import { EuiHorizontalRule, EuiButtonEmpty, EuiTextAlign, - EuiButtonGroup, } from '@elastic/eui'; import './open_options_popover.scss'; import { DOC_TABLE_LEGACY } from '../../../../../../common'; @@ -34,37 +33,6 @@ interface OptionsPopoverProps { anchorElement: HTMLElement; } -const toggleButtons = [ - { - id: `discoverViewOptionDocument`, - label: 'Document view', - }, - { - id: `discoverViewOptionAggregated`, - label: 'Aggregated view', - }, -]; - -// @todo: move this somewhere else -export const DocumentViewOption = ({ - viewId, - setViewId, -}: { - viewId: string; - setViewId: (id: string) => void; -}) => { - return ( - setViewId(id)} - /> - ); -}; export function OptionsPopover(props: OptionsPopoverProps) { const { core: { uiSettings }, diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/constants.ts b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/constants.ts new file mode 100644 index 0000000000000..e8ff88cdbbe95 --- /dev/null +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/constants.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const DISCOVER_VIEW_MODES = { + DOCUMENT_LEVEL: 'discoverViewOptionDocument', + FIELD_LEVEL: 'discoverViewOptionAggregated', +}; diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/index.ts b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/index.ts new file mode 100644 index 0000000000000..2544fe9be0b9b --- /dev/null +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { DocumentViewModeToggle } from './view_mode_toggle'; +export { DISCOVER_VIEW_MODES } from './constants'; diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx new file mode 100644 index 0000000000000..6758ec6890dd2 --- /dev/null +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { EuiButtonGroup } from '@elastic/eui'; +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { DISCOVER_VIEW_MODES } from './constants'; + +// @todo: Revisit naming/terminology +const toggleButtons = [ + { + id: DISCOVER_VIEW_MODES.DOCUMENT_LEVEL, + label: 'Document view', + }, + { + id: DISCOVER_VIEW_MODES.FIELD_LEVEL, + label: 'Aggregated view', + }, +]; + +export const DocumentViewModeToggle = ({ + discoverViewMode, + setDiscoverViewMode, +}: { + discoverViewMode: string; + setDiscoverViewMode: (id: string) => void; +}) => { + return ( + setDiscoverViewMode(id)} + /> + ); +}; From eaccec9fa02d077e871b27754b7728aac75a4ab1 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 6 Aug 2021 14:45:53 -0500 Subject: [PATCH 039/188] Resolve conflicts --- .../components/layout/discover_layout.tsx | 42 ++++++++++++------- .../components/sidebar/discover_sidebar.tsx | 1 - 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index ae963e1b0deff..bd3f62f104205 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -82,7 +82,6 @@ export function DiscoverLayout({ setDiscoverViewMode(option); }; - const scrollableDesktop = useRef(null); const collapseIcon = useRef(null); const fetchCounter = useRef(0); const { main$, charts$, totalHits$ } = savedSearchData$; @@ -279,19 +278,34 @@ export function DiscoverLayout({ /> - + {discoverViewMode === DISCOVER_VIEW_MODES.DOCUMENT_LEVEL && ( + + )} + {discoverViewMode === DISCOVER_VIEW_MODES.FIELD_LEVEL && ( + + )} )} diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx index 13e84b186221a..938b0a49b29a7 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx @@ -37,7 +37,6 @@ import { getIndexPatternFieldList } from './lib/get_index_pattern_field_list'; import { DiscoverSidebarResponsiveProps } from './discover_sidebar_responsive'; import { DiscoverIndexPatternManagement } from './discover_index_pattern_management'; import { ElasticSearchHit } from '../../../../doc_views/doc_views_types'; -import { ANOMALY_EXPLORER_CHARTS_EMBEDDABLE_TYPE } from '../../../../../../../../../x-pack/plugins/ml/public/embeddables'; /** * Default number of available fields displayed and added on scroll From 2a234fc2772c56205baef2e33e46cc319ea19eca Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 31 Aug 2021 15:19:25 -0500 Subject: [PATCH 040/188] [ML] Reduce size of chart --- .../stats_table/components/field_data_row/column_chart.scss | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss index 63603ee9bd2ec..8f0753b6bab6c 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss @@ -1,6 +1,6 @@ .dataGridChart__histogram { width: 100%; - height: $euiSizeXL + $euiSizeXXL; + height: $euiSizeM; } .dataGridChart__legend { @@ -14,6 +14,7 @@ font-style: italic; font-weight: normal; text-align: left; + line-height: 1; } .dataGridChart__legend--numeric { From f5b44e0c80a1af7eef38d59088f95b9084131a48 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 31 Aug 2021 15:34:29 -0500 Subject: [PATCH 041/188] [ML] Unbold name, switch icons of show distributions --- .../components/stats_table/data_visualizer_stats_table.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 02e4e29dcc05e..f51a8b59e9879 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -163,8 +163,8 @@ export const DataVisualizerTable = ({ const displayName = item.displayName ?? item.fieldName; return ( - - {displayName} + + {displayName} ); }, @@ -217,7 +217,7 @@ export const DataVisualizerTable = ({ iconType={showDistributions ? 'eye' : 'eyeClosed'} onClick={() => toggleShowDistribution()} aria-label={ - !showDistributions + showDistributions ? i18n.translate('xpack.dataVisualizer.dataGrid.showDistributionsAriaLabel', { defaultMessage: 'Show distributions', }) From d1e35d908945c094a2f040565432276c20898615 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 31 Aug 2021 15:58:05 -0500 Subject: [PATCH 042/188] [ML] Make size consistent --- .../apps/main/components/layout/discover_layout.tsx | 1 - .../field_data_row/number_content_preview.tsx | 10 ++++++---- .../metric_distribution_chart.tsx | 7 +++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 4a3ba62aec723..e09c7f549a9b0 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -276,7 +276,6 @@ export function DiscoverLayout({ documents$={savedSearchData$.documents$} expandedDoc={expandedDoc} indexPattern={indexPattern} - isMobile={isMobile} navigateTo={navigateTo} onAddFilter={onAddFilter as DocViewFilterFn} savedSearch={savedSearch} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/number_content_preview.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/number_content_preview.tsx index 651e41b0cbea8..eb9ffca7d9309 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/number_content_preview.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/number_content_preview.tsx @@ -6,7 +6,7 @@ */ import React, { FC, useEffect, useState } from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import classNames from 'classnames'; import { MetricDistributionChart, @@ -17,7 +17,8 @@ import { FieldVisConfig } from '../../types'; import { kibanaFieldFormat, formatSingleValue } from '../../../utils'; const METRIC_DISTRIBUTION_CHART_WIDTH = 150; -const METRIC_DISTRIBUTION_CHART_HEIGHT = 80; +// @todo: Re-enable +// const METRIC_DISTRIBUTION_CHART_HEIGHT = 80; export interface NumberContentPreviewProps { config: FieldVisConfig; @@ -48,9 +49,11 @@ export const IndexBasedNumberContentPreview: FC = ({ return (
+ {/** @todo: Use euiSizeM, make this small height only for embeddable + **/} = ({
{legendText && ( <> - {kibanaFieldFormat(legendText.min, fieldFormat)} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/metric_distribution_chart/metric_distribution_chart.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/metric_distribution_chart/metric_distribution_chart.tsx index 2c4739206d47f..c375660cdea14 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/metric_distribution_chart/metric_distribution_chart.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/metric_distribution_chart/metric_distribution_chart.tsx @@ -82,8 +82,11 @@ export const MetricDistributionChart: FC = ({ }; return ( -
- +
+ Date: Tue, 31 Aug 2021 16:34:06 -0500 Subject: [PATCH 043/188] [ML] Make page size 25 --- .../common/components/fields_stats_grid/fields_stats_grid.tsx | 2 +- .../common/components/stats_table/use_table_settings.ts | 2 +- .../index_data_visualizer_view/index_data_visualizer_view.tsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx index f1c164768d6e7..09b1e42c79a49 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx @@ -25,7 +25,7 @@ interface Props { export const getDefaultDataVisualizerListState = (): DataVisualizerTableState => ({ pageIndex: 0, - pageSize: 10, + pageSize: 25, sortField: 'fieldName', sortDirection: 'asc', visibleFieldTypes: [], diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/use_table_settings.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/use_table_settings.ts index 3fbf333bdc876..87d936edc2957 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/use_table_settings.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/use_table_settings.ts @@ -10,7 +10,7 @@ import { useCallback, useMemo } from 'react'; import { DataVisualizerTableState } from '../../../../../common'; -const PAGE_SIZE_OPTIONS = [10, 25, 50]; +const PAGE_SIZE_OPTIONS = [10, 25, 50, 100]; interface UseTableSettingsReturnValue { onTableChange: EuiBasicTableProps['onChange']; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 776877dec1b5c..6ef56f154e48b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -107,7 +107,7 @@ export const getDefaultDataVisualizerListState = ( overrides?: Partial ): Required => ({ pageIndex: 0, - pageSize: 10, + pageSize: 25, sortField: 'fieldName', sortDirection: 'asc', visibleFieldTypes: [], From e843dcd6e1a556e5ded509b462ea84be23e1391d Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 1 Sep 2021 12:09:26 -0500 Subject: [PATCH 044/188] [ML] Switch to arrow right and down --- .../components/stats_table/data_visualizer_stats_table.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index f51a8b59e9879..011d06104153b 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -105,7 +105,7 @@ export const DataVisualizerTable = ({ defaultMessage: 'Collapse details for all fields', }) } - iconType={expandAll ? 'arrowUp' : 'arrowDown'} + iconType={expandAll ? 'arrowDown' : 'arrowRight'} /> ), align: RIGHT_ALIGNMENT, @@ -114,7 +114,7 @@ export const DataVisualizerTable = ({ render: (item: DataVisualizerTableItem) => { const displayName = item.displayName ?? item.fieldName; if (item.fieldName === undefined) return null; - const direction = expandedRowItemIds.includes(item.fieldName) ? 'arrowUp' : 'arrowDown'; + const direction = expandedRowItemIds.includes(item.fieldName) ? 'arrowDown' : 'arrowRight'; return ( Date: Wed, 1 Sep 2021 12:20:56 -0500 Subject: [PATCH 045/188] [ML] Make legend font smaller --- .../stats_table/components/field_data_row/column_chart.scss | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss index 8f0753b6bab6c..46146682531ce 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss @@ -5,16 +5,15 @@ .dataGridChart__legend { @include euiTextTruncate; - @include euiFontSizeXS; color: $euiColorMediumShade; display: block; overflow-x: hidden; - margin: $euiSizeXS 0 0 0; font-style: italic; font-weight: normal; text-align: left; line-height: 1; + font-size: 10px; } .dataGridChart__legend--numeric { From cc6ea0bce0e70e33f5c4c232bf40e0028a6fa6a3 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 1 Sep 2021 14:14:47 -0500 Subject: [PATCH 046/188] [ML] Add user setting --- .../data_visualizer_grid.tsx | 36 ++++++++++++++----- src/plugins/discover/server/ui_settings.ts | 18 ++++++++++ 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index bd6e0df4a67b1..27271a73cd905 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -9,15 +9,26 @@ import React, { useEffect, useRef, useState } from 'react'; import { EuiDataGrid, EuiDataGridProps } from '@elastic/eui'; import { Filter } from '@kbn/es-query'; -import { IndexPattern, Query } from '../../../../../data/common'; +import { DataView, Query } from '../../../../../data/common'; import { DiscoverServices } from '../../../build_services'; -import { ErrorEmbeddable, IEmbeddable, isErrorEmbeddable } from '../../../../../embeddable/public'; +import { + EmbeddableInput, + EmbeddableOutput, + ErrorEmbeddable, + IEmbeddable, + isErrorEmbeddable, +} from '../../../../../embeddable/public'; import { SavedSearch } from '../../../saved_searches'; -import type { - DataVisualizerGridEmbeddableInput, - DataVisualizerGridEmbeddableOutput, - // eslint-disable-next-line @kbn/eslint/no-restricted-paths -} from '../../../../../../../x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable'; + +export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { + indexPattern: DataView; + savedSearch?: SavedSearch; + query?: Query; + visibleFieldNames?: string[]; + filters?: Filter[]; +} +export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; + export interface DiscoverDataVisualizerGridProps { /** * Determines which columns are displayed @@ -26,7 +37,7 @@ export interface DiscoverDataVisualizerGridProps { /** * The used index pattern */ - indexPattern: IndexPattern; + indexPattern: DataView; /** * The max size of the documents returned by Elasticsearch */ @@ -114,5 +125,12 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp } }, [embeddable, embeddableRoot]); - return
; + return ( +
+ ); }; diff --git a/src/plugins/discover/server/ui_settings.ts b/src/plugins/discover/server/ui_settings.ts index aa1b44da12bfc..0902abba6b3f4 100644 --- a/src/plugins/discover/server/ui_settings.ts +++ b/src/plugins/discover/server/ui_settings.ts @@ -171,6 +171,24 @@ export const getUiSettings: () => Record = () => ({ name: 'discover:useLegacyDataGrid', }, }, + // @todo + ['discover:aggregatedView']: { + name: i18n.translate('discover.advancedSettings.defaultViewName', { + defaultMessage: 'Aggregated view', + }), + value: false, + description: i18n.translate('discover.advancedSettings.defaultViewDescription', { + defaultMessage: + 'Turn on this option to use the aggregated view by default. Turn off to use the document view. ', + }), + category: ['discover'], + schema: schema.boolean(), + metric: { + type: METRIC_TYPE.CLICK, + name: 'discover:useAggregatedView', + }, + }, + [MODIFY_COLUMNS_ON_SWITCH]: { name: i18n.translate('discover.advancedSettings.discover.modifyColumnsOnSwitchTitle', { defaultMessage: 'Modify columns when changing index patterns', From 42c11052c6ab03ba6f5e45fb80a72e9a26aec748 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 1 Sep 2021 17:51:03 -0500 Subject: [PATCH 047/188] [ML] Add show preview by default setting --- .../data_visualizer_grid.tsx | 22 +++++++++++++++++-- src/plugins/discover/server/ui_settings.ts | 18 ++++++++++++++- .../data_visualizer_stats_table.tsx | 12 +++++----- .../grid_embeddable/grid_embeddable.tsx | 11 +++++++--- 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index 27271a73cd905..a9c345da3f072 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -26,6 +26,7 @@ export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { query?: Query; visibleFieldNames?: string[]; filters?: Filter[]; + showPreviewByDefault?: boolean; } export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; @@ -65,6 +66,8 @@ export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { const { services, indexPattern, savedSearch, query, columns, filters } = props; + const { uiSettings } = services; + const [embeddable, setEmbeddable] = useState< | ErrorEmbeddable | IEmbeddable @@ -86,6 +89,17 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp } }, [embeddable, indexPattern, savedSearch, query, columns, filters]); + useEffect(() => { + const showPreviewByDefault = uiSettings?.get('discover:showAggregatedPreview'); + if (showPreviewByDefault && embeddable && !isErrorEmbeddable(embeddable)) { + // Update embeddable whenever one of the important input changes + embeddable.updateInput({ + showPreviewByDefault, + }); + embeddable.reload(); + } + }, [uiSettings, embeddable]); + useEffect(() => { return () => { // Clean up embeddable upon unmounting @@ -96,7 +110,10 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp }, [embeddable]); useEffect(() => { + // @todo: handle unmounted const loadEmbeddable = async () => { + const showPreviewByDefault = uiSettings?.get('discover:showAggregatedPreview'); + if (services?.embeddable) { const factory = services.embeddable.getEmbeddableFactory< DataVisualizerGridEmbeddableInput, @@ -109,6 +126,7 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp indexPattern, savedSearch, query, + showPreviewByDefault, }); setEmbeddable(initializedEmbeddable); } @@ -123,14 +141,14 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp if (embeddableRoot.current && embeddable) { embeddable.render(embeddableRoot.current); } - }, [embeddable, embeddableRoot]); + }, [embeddable, embeddableRoot, uiSettings]); return (
); }; diff --git a/src/plugins/discover/server/ui_settings.ts b/src/plugins/discover/server/ui_settings.ts index 0902abba6b3f4..0c8426ea31a65 100644 --- a/src/plugins/discover/server/ui_settings.ts +++ b/src/plugins/discover/server/ui_settings.ts @@ -179,7 +179,7 @@ export const getUiSettings: () => Record = () => ({ value: false, description: i18n.translate('discover.advancedSettings.defaultViewDescription', { defaultMessage: - 'Turn on this option to use the aggregated view by default. Turn off to use the document view. ', + 'Turn on this option to use the aggregated view by default. Turn off to use the document view.', }), category: ['discover'], schema: schema.boolean(), @@ -188,6 +188,22 @@ export const getUiSettings: () => Record = () => ({ name: 'discover:useAggregatedView', }, }, + // @todo update terminology + ['discover:showAggregatedPreview']: { + name: i18n.translate('discover.advancedSettings.defaultViewName', { + defaultMessage: 'Show aggregated preview', + }), + value: false, + description: i18n.translate('discover.advancedSettings.defaultViewDescription', { + defaultMessage: 'Turn on this option to show the preview in the aggregated view by default.', + }), + category: ['discover'], + schema: schema.boolean(), + metric: { + type: METRIC_TYPE.CLICK, + name: 'discover:showAggregatedPreview', + }, + }, [MODIFY_COLUMNS_ON_SWITCH]: { name: i18n.translate('discover.advancedSettings.discover.modifyColumnsOnSwitchTitle', { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 011d06104153b..f75f1fcfbac08 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -49,14 +49,15 @@ interface DataVisualizerTableProps { updatePageState: (update: DataVisualizerTableState) => void; getItemIdToExpandedRowMap: (itemIds: string[], items: T[]) => ItemIdToExpandedRowMap; extendedColumns?: Array>; + showPreviewByDefault?: boolean; } - export const DataVisualizerTable = ({ items, pageState, updatePageState, getItemIdToExpandedRowMap, extendedColumns, + showPreviewByDefault, }: DataVisualizerTableProps) => { const [expandedRowItemIds, setExpandedRowItemIds] = useState([]); const [expandAll, toggleExpandAll] = useState(false); @@ -66,13 +67,10 @@ export const DataVisualizerTable = ({ pageState, updatePageState ); - const showDistributions: boolean = - ('showDistributions' in pageState && pageState.showDistributions) ?? true; + const [showDistributions, setShowDistributions] = useState(showPreviewByDefault ?? true); + const toggleShowDistribution = () => { - updatePageState({ - ...pageState, - showDistributions: !showDistributions, - }); + setShowDistributions(!showDistributions); }; function toggleDetails(item: DataVisualizerTableItem) { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 3bbb5453aedea..5ae2041a6c15e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -66,6 +66,7 @@ export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { query?: Query; visibleFieldNames?: string[]; filters?: Filter[]; + showPreviewByDefault?: boolean; } export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; @@ -635,9 +636,12 @@ export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddable Required >(restorableDefaults); - const onTableChange = (update: DataVisualizerTableState) => { - setDataVisualizerListState({ ...dataVisualizerListState, ...update }); - }; + const onTableChange = useCallback( + (update: DataVisualizerTableState) => { + setDataVisualizerListState({ ...dataVisualizerListState, ...update }); + }, + [dataVisualizerListState] + ); const { configs, searchQueryLanguage, searchString, extendedColumns } = useDataVisualizerGridData( input, dataVisualizerListState @@ -668,6 +672,7 @@ export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddable updatePageState={onTableChange} getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} extendedColumns={extendedColumns} + showPreviewByDefault={input?.showPreviewByDefault} /> ); From 680259a4166029d0a1afb3345fbd28594c7ce035 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 1 Sep 2021 18:10:34 -0500 Subject: [PATCH 048/188] [ML] Match icon --- .../main/components/sidebar/lib/get_field_type_name.ts | 9 +++++++++ .../kibana_react/public/field_icon/field_icon.tsx | 2 ++ .../stats_table/data_visualizer_stats_table.tsx | 5 +++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts b/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts index e2d4c2f7ddcf2..f68395593bd8b 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts @@ -51,6 +51,15 @@ export function getFieldTypeName(type: string) { return i18n.translate('discover.fieldNameIcons.stringFieldAriaLabel', { defaultMessage: 'String field', }); + case 'text': + return i18n.translate('discover.fieldNameIcons.textFieldAriaLabel', { + defaultMessage: 'Text field', + }); + case 'keyword': + return i18n.translate('discover.fieldNameIcons.keywordFieldAriaLabel', { + defaultMessage: 'Keyword field', + }); + case 'nested': return i18n.translate('discover.fieldNameIcons.nestedFieldAriaLabel', { defaultMessage: 'Nested field', diff --git a/src/plugins/kibana_react/public/field_icon/field_icon.tsx b/src/plugins/kibana_react/public/field_icon/field_icon.tsx index cd3c7ee9e7a99..fe7c292d3d00b 100644 --- a/src/plugins/kibana_react/public/field_icon/field_icon.tsx +++ b/src/plugins/kibana_react/public/field_icon/field_icon.tsx @@ -50,6 +50,8 @@ export const typeToEuiIconMap: Partial> = { number_range: { iconType: 'tokenNumber' }, _source: { iconType: 'editorCodeBlock', color: 'gray' }, string: { iconType: 'tokenString' }, + text: { iconType: 'tokenString' }, + keyword: { iconType: 'tokenString' }, nested: { iconType: 'tokenNested' }, }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index f75f1fcfbac08..6d923cda7f41b 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -23,7 +23,6 @@ import { import { i18n } from '@kbn/i18n'; import { EuiTableComputedColumnType } from '@elastic/eui/src/components/basic_table/table_types'; import { JOB_FIELD_TYPES, JobFieldType, DataVisualizerTableState } from '../../../../../common'; -import { FieldTypeIcon } from '../field_type_icon'; import { DocumentStat } from './components/field_data_row/document_stats'; import { DistinctValues } from './components/field_data_row/distinct_values'; import { IndexBasedNumberContentPreview } from './components/field_data_row/number_content_preview'; @@ -37,6 +36,7 @@ import { } from './types/field_vis_config'; import { FileBasedNumberContentPreview } from '../field_data_row'; import { BooleanContentPreview } from './components/field_data_row'; +import { FieldIcon } from '../../../../../../../../src/plugins/kibana_react/public'; const FIELD_NAME = 'fieldName'; @@ -143,7 +143,8 @@ export const DataVisualizerTable = ({ defaultMessage: 'Type', }), render: (fieldType: JobFieldType) => { - return ; + // @todo: fix scripted + return ; }, width: '75px', sortable: true, From 578f49c68679ca3e9b739029e36958e66f4607d2 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 3 Sep 2021 09:42:52 -0500 Subject: [PATCH 049/188] Add panels around the subcontent --- .../main/components/chart/discover_chart.tsx | 10 +++++---- .../view_mode_toggle/view_mode_toggle.tsx | 7 +++--- .../components/embedded_map/embedded_map.tsx | 2 +- .../examples_list/examples_list.tsx | 5 ++++- .../common/components/stats_table/_index.scss | 22 +++++++++++++++++++ .../field_data_expanded_row/date_content.tsx | 2 +- .../document_stats.tsx | 2 +- .../expanded_row_content.tsx | 2 +- .../keyword_content.tsx | 8 ++++++- .../number_content.tsx | 7 ++++-- .../field_data_expanded_row/text_content.tsx | 2 +- .../field_data_row/column_chart.scss | 6 ++++- .../field_data_row/column_chart.tsx | 2 +- .../field_data_row/number_content_preview.tsx | 9 +++----- .../metric_distribution_chart.tsx | 2 +- .../data_visualizer_stats_table.tsx | 5 ++++- .../components/top_values/top_values.tsx | 5 ++++- 17 files changed, 71 insertions(+), 27 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx index 2209b7db2d28f..d9e3f1f110132 100644 --- a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx +++ b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx @@ -96,6 +96,11 @@ export function DiscoverChart({ onResetQuery={resetQuery} /> + + {!state.hideChart && ( )} - + {timefield && !state.hideChart && (
void; }) => { return ( + // Table controls setDiscoverViewMode(id)} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/embedded_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/embedded_map.tsx index 29131b4a3372a..0bae10881c771 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/embedded_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/embedded_map.tsx @@ -144,7 +144,7 @@ export function EmbeddedMapComponent({ return (
); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx index 296820479437c..ff2eaf6ef7137 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx @@ -41,7 +41,10 @@ export const ExamplesList: FC = ({ examples }) => { } return ( -
+
= ({ config }) => { return ( - + {summaryTableTitle} className={'dataVisualizerSummaryTable'} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx index f4ed74193d90a..b1340e9636860 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx @@ -78,7 +78,7 @@ export const DocumentStatsTable: FC = ({ config }) => { return ( {metaTableTitle} = ({ children, dataTestSubj }) => { return ( {children} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx index 1baea4b3f2f7c..bf66d8e21eada 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx @@ -6,6 +6,7 @@ */ import React, { FC, useCallback, useEffect, useState } from 'react'; +import { EuiPanel } from '@elastic/eui'; import type { FieldDataRowProps } from '../../types/field_data_row'; import { TopValues } from '../../../top_values'; import { EMSTermJoinConfig } from '../../../../../../../../maps/public'; @@ -45,7 +46,12 @@ export const KeywordContent: FC = ({ config }) => { - {EMSSuggestion && stats && } + + {EMSSuggestion && stats && ( + + {' '} + + )} ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx index ef3ac5a267346..888ecacf0d43a 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx @@ -101,7 +101,7 @@ export const NumberContent: FC = ({ config }) => { return ( - + {summaryTableTitle} className={'dataVisualizerSummaryTable'} @@ -117,7 +117,10 @@ export const NumberContent: FC = ({ config }) => { )} {distribution && ( - + = ({ config }) => { return ( - + {numExamples > 0 && } {numExamples === 0 && ( diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss index 46146682531ce..d0c39c625a6c2 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss @@ -1,6 +1,10 @@ .dataGridChart__histogram { width: 100%; - height: $euiSizeM; +} + +.dataGridChart__column-chart { + width: 100%; + height: 12px; } .dataGridChart__legend { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.tsx index ed4b82005db29..5f5cfbfe1b282 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.tsx @@ -53,7 +53,7 @@ export const ColumnChart: FC = ({ return (
{!isUnsupportedChartData(chartData) && data.length > 0 && ( -
+
= ({ return (
- {/** @todo: Use euiSizeM, make this small height only for embeddable - **/} = ({ data-test-subj="dataVisualizerFieldDataMetricDistributionChart" className="dataGridChart__histogram" > - + ({ sortable: (item: DataVisualizerTableItem) => item?.stats?.count, align: LEFT_ALIGNMENT as HorizontalAlignment, 'data-test-subj': 'dataVisualizerTableColumnDocumentsCount', + width: '175px', }, { field: 'stats.cardinality', @@ -191,6 +192,7 @@ export const DataVisualizerTable = ({ sortable: true, align: LEFT_ALIGNMENT as HorizontalAlignment, 'data-test-subj': 'dataVisualizerTableColumnDistinctValues', + width: '175px', }, { name: ( @@ -251,7 +253,8 @@ export const DataVisualizerTable = ({ return null; }, - align: LEFT_ALIGNMENT as HorizontalAlignment, + width: '150px', + align: CENTER_ALIGNMENT as HorizontalAlignment, 'data-test-subj': 'dataVisualizerTableColumnDistribution', }, ]; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index 7a20b054462a6..9f87f099b847c 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -49,7 +49,10 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed } = stats; const progressBarMax = isTopValuesSampled === true ? topValuesSampleSize : count; return ( - + Date: Tue, 7 Sep 2021 10:42:38 -0500 Subject: [PATCH 050/188] Add preference for aggregated vs doc --- .../main/components/layout/discover_layout.tsx | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index e09c7f549a9b0..8831f513d6b7f 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -52,8 +52,6 @@ const TopNavMemoized = React.memo(DiscoverTopNav); const DiscoverChartMemoized = React.memo(DiscoverChart); const DataVisualizerGridMemoized = React.memo(DiscoverDataVisualizerGrid); -let storageViewPreference = 'discoverViewOptionDocument'; - export function DiscoverLayout({ indexPattern, indexPatternList, @@ -75,11 +73,18 @@ export function DiscoverLayout({ const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); - const [discoverViewMode, setDiscoverViewMode] = useState(storageViewPreference); + const [discoverViewMode, setDiscoverViewMode] = useState(DISCOVER_VIEW_MODES.DOCUMENT_LEVEL); + + useEffect(() => { + const userSetViewMode = uiSettings?.get('discover:aggregatedView'); + if (userSetViewMode) { + setDiscoverViewMode(DISCOVER_VIEW_MODES.FIELD_LEVEL); + } else { + setDiscoverViewMode(DISCOVER_VIEW_MODES.DOCUMENT_LEVEL); + } + }, [uiSettings]); const changeViewId = (option: string) => { - // @todo: temp hack to replace storage - storageViewPreference = option; setDiscoverViewMode(option); }; From f35332401f2585122d538587d2add7a9149f2291 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 7 Sep 2021 12:53:06 -0500 Subject: [PATCH 051/188] Fix types --- .../apps/main/components/chart/discover_chart.test.tsx | 3 +++ .../application/index_data_visualizer/embeddables/index.ts | 6 +++++- x-pack/plugins/data_visualizer/public/plugin.ts | 5 ++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.test.tsx b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.test.tsx index dc3c9ebbc75ca..a5a41fdde6ea4 100644 --- a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.test.tsx @@ -19,6 +19,7 @@ import { discoverServiceMock } from '../../../../../__mocks__/services'; import { FetchStatus } from '../../../../types'; import { Chart } from './point_series'; import { DiscoverChart } from './discover_chart'; +import { DISCOVER_VIEW_MODES } from '../view_mode_toggle'; setHeaderActionMenuMounter(jest.fn()); @@ -106,6 +107,8 @@ function getProps(timefield?: string) { state: { columns: [] }, stateContainer: {} as GetStateReturn, timefield, + discoverViewMode: DISCOVER_VIEW_MODES.DOCUMENT_LEVEL, + setDiscoverViewMode: jest.fn(), }; } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts index ef10f5bccf202..add99a8d2501d 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts @@ -8,8 +8,12 @@ import { CoreSetup } from 'kibana/public'; import { EmbeddableSetup } from '../../../../../../../src/plugins/embeddable/public'; import { DataVisualizerGridEmbeddableFactory } from './grid_embeddable/grid_embeddable_factory'; +import { DataVisualizerPluginStart, DataVisualizerStartDependencies } from '../../../plugin'; -export function registerEmbeddables(embeddable: EmbeddableSetup, core: CoreSetup) { +export function registerEmbeddables( + embeddable: EmbeddableSetup, + core: CoreSetup +) { const dataVisualizerGridEmbeddableFactory = new DataVisualizerGridEmbeddableFactory( core.getStartServices ); diff --git a/x-pack/plugins/data_visualizer/public/plugin.ts b/x-pack/plugins/data_visualizer/public/plugin.ts index e4313172e037c..0084437347965 100644 --- a/x-pack/plugins/data_visualizer/public/plugin.ts +++ b/x-pack/plugins/data_visualizer/public/plugin.ts @@ -49,7 +49,10 @@ export class DataVisualizerPlugin DataVisualizerSetupDependencies, DataVisualizerStartDependencies > { - public setup(core: CoreSetup, plugins: DataVisualizerSetupDependencies) { + public setup( + core: CoreSetup, + plugins: DataVisualizerSetupDependencies + ) { if (plugins.home) { registerHomeAddData(plugins.home); registerHomeFeatureCatalogue(plugins.home); From 222ea5a595229a2d010ddc1ddab97af431f0629e Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 8 Sep 2021 09:12:32 -0500 Subject: [PATCH 052/188] Fix types, add constants for adv settings --- src/plugins/discover/common/index.ts | 2 + .../top_nav/data_visualizer_panel.tsx | 67 ------------------- .../data_visualizer_grid.tsx | 14 ++-- src/plugins/discover/server/ui_settings.ts | 18 ++--- .../index_data_visualizer/locator/locator.ts | 12 ++-- 5 files changed, 27 insertions(+), 86 deletions(-) delete mode 100644 src/plugins/discover/public/application/apps/main/components/top_nav/data_visualizer_panel.tsx diff --git a/src/plugins/discover/common/index.ts b/src/plugins/discover/common/index.ts index b30fcf972eda5..3a01e312fd393 100644 --- a/src/plugins/discover/common/index.ts +++ b/src/plugins/discover/common/index.ts @@ -21,3 +21,5 @@ export const SEARCH_FIELDS_FROM_SOURCE = 'discover:searchFieldsFromSource'; export const MAX_DOC_FIELDS_DISPLAYED = 'discover:maxDocFieldsDisplayed'; export const SHOW_MULTIFIELDS = 'discover:showMultiFields'; export const SEARCH_EMBEDDABLE_TYPE = 'search'; +export const AGGREGATED_VIEW_SETTING = 'discover:aggregatedView'; +export const AGGREGATED_VIEW_PREVIEW = 'data_visualizer:showAggregatedPreview'; diff --git a/src/plugins/discover/public/application/apps/main/components/top_nav/data_visualizer_panel.tsx b/src/plugins/discover/public/application/apps/main/components/top_nav/data_visualizer_panel.tsx deleted file mode 100644 index 6695694259cfb..0000000000000 --- a/src/plugins/discover/public/application/apps/main/components/top_nav/data_visualizer_panel.tsx +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { EuiFlyout, EuiFlyoutBody, EuiFlyoutFooter, EuiFlyoutHeader, EuiTitle } from '@elastic/eui'; -import { FormattedMessage } from '@kbn/i18n/react'; -import React, { useMemo } from 'react'; -import { IndexPattern } from '../../../../../kibana_services'; -import { SavedSearch } from '../../../../../saved_searches'; -import { DiscoverServices } from '../../../../../build_services'; -import { GetStateReturn } from '../../services/discover_state'; -import { DiscoverDataVisualizerGrid } from '../../../../components/data_visualizer_grid'; -import { SAMPLE_SIZE_SETTING } from '../../../../../../common'; - -const DataVisualizerGridPanelMemoized = React.memo(DiscoverDataVisualizerGrid); - -interface DataVisualizerPanelProps { - onClose: () => void; - indexPattern: IndexPattern; - savedSearch: SavedSearch; - services: DiscoverServices; - state: GetStateReturn; -} - -export function DataVisualizerPanel({ - onClose, - savedSearch, - services, - indexPattern, - state, -}: DataVisualizerPanelProps) { - const { uiSettings } = services; - const sampleSize = useMemo(() => uiSettings.get(SAMPLE_SIZE_SETTING), [uiSettings]); - const appState = state.appStateContainer.getState(); - - return ( - - - -

- -

- - - - - - - - ); -} diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index a9c345da3f072..e39a34e14641f 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -19,6 +19,7 @@ import { isErrorEmbeddable, } from '../../../../../embeddable/public'; import { SavedSearch } from '../../../saved_searches'; +import { AGGREGATED_VIEW_PREVIEW } from '../../../../common'; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { indexPattern: DataView; @@ -90,7 +91,7 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp }, [embeddable, indexPattern, savedSearch, query, columns, filters]); useEffect(() => { - const showPreviewByDefault = uiSettings?.get('discover:showAggregatedPreview'); + const showPreviewByDefault = uiSettings?.get(AGGREGATED_VIEW_PREVIEW); if (showPreviewByDefault && embeddable && !isErrorEmbeddable(embeddable)) { // Update embeddable whenever one of the important input changes embeddable.updateInput({ @@ -110,9 +111,9 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp }, [embeddable]); useEffect(() => { - // @todo: handle unmounted + let unmounted = false; const loadEmbeddable = async () => { - const showPreviewByDefault = uiSettings?.get('discover:showAggregatedPreview'); + const showPreviewByDefault = uiSettings?.get(AGGREGATED_VIEW_PREVIEW); if (services?.embeddable) { const factory = services.embeddable.getEmbeddableFactory< @@ -128,11 +129,16 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp query, showPreviewByDefault, }); - setEmbeddable(initializedEmbeddable); + if (!unmounted) { + setEmbeddable(initializedEmbeddable); + } } } }; loadEmbeddable(); + return () => { + unmounted = true; + }; // eslint-disable-next-line react-hooks/exhaustive-deps }, [services?.embeddable]); diff --git a/src/plugins/discover/server/ui_settings.ts b/src/plugins/discover/server/ui_settings.ts index 0c8426ea31a65..9e73657de6665 100644 --- a/src/plugins/discover/server/ui_settings.ts +++ b/src/plugins/discover/server/ui_settings.ts @@ -26,6 +26,8 @@ import { SEARCH_FIELDS_FROM_SOURCE, MAX_DOC_FIELDS_DISPLAYED, SHOW_MULTIFIELDS, + AGGREGATED_VIEW_SETTING, + AGGREGATED_VIEW_PREVIEW, } from '../common'; export const getUiSettings: () => Record = () => ({ @@ -171,15 +173,14 @@ export const getUiSettings: () => Record = () => ({ name: 'discover:useLegacyDataGrid', }, }, - // @todo - ['discover:aggregatedView']: { - name: i18n.translate('discover.advancedSettings.defaultViewName', { + [AGGREGATED_VIEW_SETTING]: { + name: i18n.translate('discover.advancedSettings.aggregatedViewName', { defaultMessage: 'Aggregated view', }), value: false, - description: i18n.translate('discover.advancedSettings.defaultViewDescription', { + description: i18n.translate('discover.advancedSettings.aggregatedViewDescription', { defaultMessage: - 'Turn on this option to use the aggregated view by default. Turn off to use the document view.', + 'Turn on this option to show the aggregated view by default. Turn off to show the document view.', }), category: ['discover'], schema: schema.boolean(), @@ -188,13 +189,12 @@ export const getUiSettings: () => Record = () => ({ name: 'discover:useAggregatedView', }, }, - // @todo update terminology - ['discover:showAggregatedPreview']: { - name: i18n.translate('discover.advancedSettings.defaultViewName', { + [AGGREGATED_VIEW_PREVIEW]: { + name: i18n.translate('discover.advancedSettings.showAggregatedPreviewName', { defaultMessage: 'Show aggregated preview', }), value: false, - description: i18n.translate('discover.advancedSettings.defaultViewDescription', { + description: i18n.translate('discover.advancedSettings.showAggregatedPreviewDescription', { defaultMessage: 'Turn on this option to show the preview in the aggregated view by default.', }), category: ['discover'], diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts index e64fbb8e02ec1..c791d24a42f16 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts @@ -8,7 +8,7 @@ // @ts-ignore import { encode } from 'rison-node'; import { stringify } from 'query-string'; -import { SerializableState } from '../../../../../../../src/plugins/kibana_utils/common'; +import { SerializableRecord } from '@kbn/utility-types'; import { RefreshInterval, TimeRange } from '../../../../../../../src/plugins/data/common'; import { LocatorDefinition, LocatorPublic } from '../../../../../../../src/plugins/share/common'; import { QueryState } from '../../../../../../../src/plugins/data/public'; @@ -17,7 +17,7 @@ import { SearchQueryLanguage } from '../types/combined_query'; export const DATA_VISUALIZER_APP_LOCATOR = 'DATA_VISUALIZER_APP_LOCATOR'; -export interface IndexDataVisualizerLocatorParams extends SerializableState { +export interface IndexDataVisualizerLocatorParams extends SerializableRecord { /** * Optionally set saved search ID. */ @@ -36,14 +36,14 @@ export interface IndexDataVisualizerLocatorParams extends SerializableState { /** * Optionally set the refresh interval. */ - refreshInterval?: RefreshInterval & SerializableState; + refreshInterval?: RefreshInterval & SerializableRecord; /** * Optionally set a query. */ query?: { - searchQuery: SerializableState; - searchString: string | SerializableState; + searchQuery: SerializableRecord; + searchString: string | SerializableRecord; searchQueryLanguage: SearchQueryLanguage; }; @@ -84,7 +84,7 @@ export class IndexDataVisualizerLocatorDefinition const appState: { searchQuery?: { [key: string]: any }; searchQueryLanguage?: string; - searchString?: string | SerializableState; + searchString?: string | SerializableRecord; visibleFieldNames?: string[]; visibleFieldTypes?: string[]; } = {}; From 8d6d3585dda221d8709c1879fa181fb29e9d8cb4 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 8 Sep 2021 09:20:48 -0500 Subject: [PATCH 053/188] Change to data view type --- .../field_data_row/action_menu/actions.ts | 14 +++++++------- .../grid_embeddable/grid_embeddable.tsx | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/action_menu/actions.ts b/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/action_menu/actions.ts index a77ca1d589349..91509823dbf4f 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/action_menu/actions.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/action_menu/actions.ts @@ -9,7 +9,7 @@ import { i18n } from '@kbn/i18n'; import { Action } from '@elastic/eui/src/components/basic_table/action_types'; import { MutableRefObject } from 'react'; import { getCompatibleLensDataType, getLensAttributes } from './lens_utils'; -import { IndexPattern } from '../../../../../../../../../src/plugins/data/common/index_patterns/index_patterns'; +import { DataView } from '../../../../../../../../../src/plugins/data/common'; import { CombinedQuery } from '../../../../index_data_visualizer/types/combined_query'; import { FieldVisConfig } from '../../stats_table/types'; import { DataVisualizerKibanaReactContextValue } from '../../../../kibana_context'; @@ -19,12 +19,12 @@ import { } from '../../../../index_data_visualizer/services/timefilter_refresh_service'; export function getActions( - indexPattern: IndexPattern, - services: DataVisualizerKibanaReactContextValue['services'], + indexPattern: DataView, + services: Partial, combinedQuery: CombinedQuery, actionFlyoutRef: MutableRefObject<(() => void | undefined) | undefined> ): Array> { - const { lens: lensPlugin, indexPatternFieldEditor } = services; + const { lens: lensPlugin } = services; const actions: Array> = []; @@ -62,7 +62,7 @@ export function getActions( } // Allow to edit index pattern field - if (indexPatternFieldEditor?.userPermissions.editIndexPattern()) { + if (services.indexPatternFieldEditor?.userPermissions.editIndexPattern()) { actions.push({ name: i18n.translate('xpack.dataVisualizer.index.dataGrid.editIndexPatternFieldTitle', { defaultMessage: 'Edit index pattern field', @@ -76,7 +76,7 @@ export function getActions( type: 'icon', icon: 'indexEdit', onClick: (item: FieldVisConfig) => { - actionFlyoutRef.current = indexPatternFieldEditor?.openEditor({ + actionFlyoutRef.current = services.indexPatternFieldEditor?.openEditor({ ctx: { indexPattern }, fieldName: item.fieldName, onSave: refreshPage, @@ -100,7 +100,7 @@ export function getActions( return item.deletable === true; }, onClick: (item: FieldVisConfig) => { - actionFlyoutRef.current = indexPatternFieldEditor?.openDeleteModal({ + actionFlyoutRef.current = services.indexPatternFieldEditor?.openDeleteModal({ ctx: { indexPattern }, fieldName: item.fieldName!, onDelete: refreshPage, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 5ae2041a6c15e..fab7aa75ae905 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -597,7 +597,7 @@ const useDataVisualizerGridData = ( const extendedColumns = useMemo(() => { const actions = getActions( input.indexPattern, - services, + { lens: services.lens }, { searchQueryLanguage, searchString, From c81a056110fd07d63c2c39e07e3eb6c50511f07c Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 8 Sep 2021 09:22:04 -0500 Subject: [PATCH 054/188] Temp fix for Kibana/EUI table overflow issue --- .../apps/main/components/layout/discover_layout.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss index 2401325dd76f2..cf4843c32333e 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss @@ -4,6 +4,10 @@ discover-app { flex-grow: 1; } +.dscAppWrapper { + overflow: hidden; +} + .dscPage { @include euiBreakpoint('m', 'l', 'xl') { @include kibanaFullBodyHeight(); From abd8d1e107812a2ad8962d2fc2f2b66bb9cb8f91 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 8 Sep 2021 10:43:05 -0500 Subject: [PATCH 055/188] Modify line height so text is not cut off, modify widths for varying screen sizes --- .../components/layout/discover_layout.scss | 4 - .../field_data_row/column_chart.scss | 2 +- .../field_data_row/distinct_values.tsx | 15 +++- .../field_data_row/document_stats.tsx | 13 ++- .../data_visualizer_stats_table.tsx | 84 +++++++++++++++++-- .../grid_embeddable/grid_embeddable.tsx | 2 +- 6 files changed, 97 insertions(+), 23 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss index cf4843c32333e..2401325dd76f2 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss @@ -4,10 +4,6 @@ discover-app { flex-grow: 1; } -.dscAppWrapper { - overflow: hidden; -} - .dscPage { @include euiBreakpoint('m', 'l', 'xl') { @include kibanaFullBodyHeight(); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss index d0c39c625a6c2..bc2c05f407659 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss @@ -16,7 +16,7 @@ font-style: italic; font-weight: normal; text-align: left; - line-height: 1; + line-height: 1.1; font-size: 10px; } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx index 92e0d1a16229f..353a9e40eeb61 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx @@ -9,13 +9,20 @@ import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiText } from '@elastic/eui'; import React from 'react'; -export const DistinctValues = ({ cardinality }: { cardinality?: number }) => { +interface Props { + cardinality?: number; + showIcons: boolean; +} + +export const DistinctValues = ({ cardinality, showIcons }: Props) => { if (cardinality === undefined) return null; return ( - - - + {showIcons ? ( + + + + ) : null} {cardinality} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx index 7d0bda6ac47ea..07057f67a5e78 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx @@ -11,7 +11,10 @@ import React from 'react'; import type { FieldDataRowProps } from '../../types/field_data_row'; import { roundToDecimalPlace } from '../../../utils'; -export const DocumentStat = ({ config }: FieldDataRowProps) => { +interface Props extends FieldDataRowProps { + showIcons: boolean; +} +export const DocumentStat = ({ config, showIcons }: Props) => { const { stats } = config; if (stats === undefined) return null; @@ -22,9 +25,11 @@ export const DocumentStat = ({ config }: FieldDataRowProps) => { return ( - - - + {showIcons ? ( + + + + ) : null} {count} ({docsPercent}%) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index fb3dbb63eb427..3371c8a1040c8 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -16,12 +16,15 @@ import { EuiInMemoryTable, EuiText, EuiToolTip, + getBreakpoint, HorizontalAlignment, LEFT_ALIGNMENT, RIGHT_ALIGNMENT, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { EuiTableComputedColumnType } from '@elastic/eui/src/components/basic_table/table_types'; +import useWindowSize from 'react-use/lib/useWindowSize'; +import useDebounce from 'react-use/lib/useDebounce'; import { JOB_FIELD_TYPES, JobFieldType, DataVisualizerTableState } from '../../../../../common'; import { DocumentStat } from './components/field_data_row/document_stats'; import { DistinctValues } from './components/field_data_row/distinct_values'; @@ -51,6 +54,46 @@ interface DataVisualizerTableProps { extendedColumns?: Array>; showPreviewByDefault?: boolean; } + +export const calculateTableColumnsDimensions = (width: number, showDistributions: boolean) => { + const breakPoint = getBreakpoint(width - 300); + switch (breakPoint) { + case 'xs': + case 's': + return { + expander: '25px', + type: '40px', + docCount: '110px', + distinctValues: '75px', + distributions: showDistributions ? '120px' : '20px', + showIcons: false, + breakPoint, + }; + + case 'm': + case 'l': + return { + expander: '25px', + type: '40px', + docCount: '110px', + distinctValues: '75px', + distributions: showDistributions ? '120px' : '50px', + showIcons: false, + breakPoint, + }; + + default: + return { + expander: '40px', + type: '75px', + docCount: '175px', + distinctValues: '175px', + distributions: '150px', + showIcons: true, + breakPoint: 'xl', + }; + } +}; export const DataVisualizerTable = ({ items, pageState, @@ -68,6 +111,25 @@ export const DataVisualizerTable = ({ updatePageState ); const [showDistributions, setShowDistributions] = useState(showPreviewByDefault ?? true); + const [dimensions, setDimensions] = useState({ + expander: '40px', + type: '75px', + docCount: '175px', + distinctValues: '175px', + distributions: '150px', + showIcons: true, + breakPoint: 'xl', + }); + + const { width } = useWindowSize(); + + useDebounce( + () => { + setDimensions(calculateTableColumnsDimensions(width, showDistributions)); + }, + 100, + [width, showDistributions] + ); const toggleShowDistribution = () => { setShowDistributions(!showDistributions); @@ -107,7 +169,7 @@ export const DataVisualizerTable = ({ /> ), align: RIGHT_ALIGNMENT, - width: '40px', + width: dimensions.expander, isExpander: true, render: (item: DataVisualizerTableItem) => { const displayName = item.displayName ?? item.fieldName; @@ -146,7 +208,7 @@ export const DataVisualizerTable = ({ // @todo: fix scripted return ; }, - width: '75px', + width: dimensions.type, sortable: true, align: CENTER_ALIGNMENT as HorizontalAlignment, 'data-test-subj': 'dataVisualizerTableColumnType', @@ -176,28 +238,32 @@ export const DataVisualizerTable = ({ defaultMessage: 'Documents (%)', }), render: (value: number | undefined, item: DataVisualizerTableItem) => ( - + ), sortable: (item: DataVisualizerTableItem) => item?.stats?.count, align: LEFT_ALIGNMENT as HorizontalAlignment, 'data-test-subj': 'dataVisualizerTableColumnDocumentsCount', - width: '175px', + width: dimensions.docCount, }, { field: 'stats.cardinality', name: i18n.translate('xpack.dataVisualizer.dataGrid.distinctValuesColumnName', { defaultMessage: 'Distinct values', }), - render: (cardinality?: number) => , + render: (cardinality?: number) => ( + + ), sortable: true, align: LEFT_ALIGNMENT as HorizontalAlignment, 'data-test-subj': 'dataVisualizerTableColumnDistinctValues', - width: '175px', + width: dimensions.distinctValues, }, { name: (
- + {dimensions.showIcons ? ( + + ) : null} {i18n.translate('xpack.dataVisualizer.dataGrid.distributionsColumnName', { defaultMessage: 'Distributions', })} @@ -253,14 +319,14 @@ export const DataVisualizerTable = ({ return null; }, - width: '150px', + width: dimensions.distributions, align: CENTER_ALIGNMENT as HorizontalAlignment, 'data-test-subj': 'dataVisualizerTableColumnDistribution', }, ]; return extendedColumns ? [...baseColumns, ...extendedColumns] : baseColumns; // eslint-disable-next-line react-hooks/exhaustive-deps - }, [expandAll, showDistributions, updatePageState, extendedColumns]); + }, [expandAll, showDistributions, updatePageState, extendedColumns, dimensions.breakPoint]); const itemIdToExpandedRowMap = useMemo(() => { let itemIds = expandedRowItemIds; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index fab7aa75ae905..77fa939505b78 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -614,7 +614,7 @@ const useDataVisualizerGridData = ( /> ), actions, - width: '100px', + width: '75px', }; return [actionColumn]; From fda17118c44c43085fa5c5112af3d2f256a285b9 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 8 Sep 2021 10:54:46 -0500 Subject: [PATCH 056/188] Different width padders for different screens --- .../fields_stats_grid/fields_stats_grid.tsx | 4 + .../data_visualizer_stats_table.tsx | 86 +++++++++++++------ .../index_data_visualizer_view.tsx | 3 + .../grid_embeddable/grid_embeddable.tsx | 3 + 4 files changed, 68 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx index 09b1e42c79a49..74c5be99ce6e7 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx @@ -7,6 +7,7 @@ import React, { useMemo, FC, useState } from 'react'; import { EuiFlexGroup, EuiSpacer } from '@elastic/eui'; +import useWindowSize from 'react-use/lib/useWindowSize'; import type { FindFileStructureResponse } from '../../../../../../file_upload/common'; import type { DataVisualizerTableState } from '../../../../../common'; import { DataVisualizerTable, ItemIdToExpandedRowMap } from '../stats_table'; @@ -77,6 +78,8 @@ export const FieldsStatsGrid: FC = ({ results }) => { const fieldsCountStats = { visibleFieldsCount, totalFieldsCount }; const metricsStats = { visibleMetricsCount, totalMetricFieldsCount }; + const { width } = useWindowSize(); + return (
@@ -113,6 +116,7 @@ export const FieldsStatsGrid: FC = ({ results }) => { pageState={dataVisualizerListState} updatePageState={setDataVisualizerListState} getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} + width={width} />
); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 3371c8a1040c8..c4b1005ded959 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -23,7 +23,6 @@ import { } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { EuiTableComputedColumnType } from '@elastic/eui/src/components/basic_table/table_types'; -import useWindowSize from 'react-use/lib/useWindowSize'; import useDebounce from 'react-use/lib/useDebounce'; import { JOB_FIELD_TYPES, JobFieldType, DataVisualizerTableState } from '../../../../../common'; import { DocumentStat } from './components/field_data_row/document_stats'; @@ -53,10 +52,12 @@ interface DataVisualizerTableProps { getItemIdToExpandedRowMap: (itemIds: string[], items: T[]) => ItemIdToExpandedRowMap; extendedColumns?: Array>; showPreviewByDefault?: boolean; + /** Table width used to calculate the appropriate column widths **/ + width: number; } export const calculateTableColumnsDimensions = (width: number, showDistributions: boolean) => { - const breakPoint = getBreakpoint(width - 300); + const breakPoint = getBreakpoint(width); switch (breakPoint) { case 'xs': case 's': @@ -65,7 +66,7 @@ export const calculateTableColumnsDimensions = (width: number, showDistributions type: '40px', docCount: '110px', distinctValues: '75px', - distributions: showDistributions ? '120px' : '20px', + distributions: showDistributions ? '120px' : '50px', showIcons: false, breakPoint, }; @@ -101,6 +102,7 @@ export const DataVisualizerTable = ({ getItemIdToExpandedRowMap, extendedColumns, showPreviewByDefault, + width, }: DataVisualizerTableProps) => { const [expandedRowItemIds, setExpandedRowItemIds] = useState([]); const [expandAll, toggleExpandAll] = useState(false); @@ -121,8 +123,6 @@ export const DataVisualizerTable = ({ breakPoint: 'xl', }); - const { width } = useWindowSize(); - useDebounce( () => { setDimensions(calculateTableColumnsDimensions(width, showDistributions)); @@ -263,37 +263,67 @@ export const DataVisualizerTable = ({
{dimensions.showIcons ? ( - ) : null} + ) : ( + + toggleShowDistribution()} + aria-label={ + showDistributions + ? i18n.translate('xpack.dataVisualizer.dataGrid.showDistributionsAriaLabel', { + defaultMessage: 'Show distributions', + }) + : i18n.translate('xpack.dataVisualizer.dataGrid.hideDistributionsAriaLabel', { + defaultMessage: 'Hide distributions', + }) + } + /> + + )} {i18n.translate('xpack.dataVisualizer.dataGrid.distributionsColumnName', { defaultMessage: 'Distributions', })} - - toggleShowDistribution()} - aria-label={ - showDistributions - ? i18n.translate('xpack.dataVisualizer.dataGrid.showDistributionsAriaLabel', { + {dimensions.showIcons ? ( + - + > + toggleShowDistribution()} + aria-label={ + showDistributions + ? i18n.translate('xpack.dataVisualizer.dataGrid.showDistributionsAriaLabel', { + defaultMessage: 'Show distributions', + }) + : i18n.translate('xpack.dataVisualizer.dataGrid.hideDistributionsAriaLabel', { + defaultMessage: 'Hide distributions', + }) + } + /> + + ) : null}
), render: (item: DataVisualizerTableItem) => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 6ef56f154e48b..7a40a6083a494 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -23,6 +23,7 @@ import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_tab import { FormattedMessage } from '@kbn/i18n/react'; import { Required } from 'utility-types'; import { i18n } from '@kbn/i18n'; +import useWindowSize from 'react-use/lib/useWindowSize'; import { IndexPatternField, KBN_FIELD_TYPES, @@ -804,6 +805,7 @@ export const IndexDataVisualizerView: FC = (dataVi }, [currentIndexPattern, services, searchQueryLanguage, searchString]); const helpLink = docLinks.links.ml.guide; + const { width: windowWidth } = useWindowSize(); return ( @@ -896,6 +898,7 @@ export const IndexDataVisualizerView: FC = (dataVi updatePageState={setDataVisualizerListState} getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} extendedColumns={extendedColumns} + width={windowWidth - 300} /> diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 77fa939505b78..75a7ec78186f7 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -14,6 +14,7 @@ import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_tab import { FormattedMessage } from '@kbn/i18n/react'; import { Filter } from '@kbn/es-query'; import { Required } from 'utility-types'; +import useWindowSize from 'react-use/lib/useWindowSize'; import { Embeddable, EmbeddableInput, @@ -664,6 +665,7 @@ export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddable }, [input, searchQueryLanguage, searchString] ); + const { width: windowWidth } = useWindowSize(); return ( @@ -673,6 +675,7 @@ export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddable getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} extendedColumns={extendedColumns} showPreviewByDefault={input?.showPreviewByDefault} + width={windowWidth - 300} /> ); From a0e457f29a3eb2ce9b040d55f9330f5fe182ec1d Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 8 Sep 2021 11:04:32 -0500 Subject: [PATCH 057/188] Fix CI --- src/plugins/discover/common/index.ts | 2 +- src/plugins/discover/server/ui_settings.ts | 4 +- .../public/field_icon/field_icon.tsx | 2 - .../server/collectors/management/schema.ts | 8 +++ .../server/collectors/management/types.ts | 2 + src/plugins/telemetry/schema/oss_plugins.json | 12 +++++ .../data_visualizer_stats_table.tsx | 44 +--------------- .../common/components/stats_table/utils.ts | 51 +++++++++++++++++++ .../embeddables/grid_embeddable/index.ts | 8 +++ .../locator/locator.test.ts | 3 +- .../routes/new_job/index_or_search.tsx | 2 - .../functional/services/ml/custom_urls.ts | 3 +- .../services/ml/data_visualizer_table.ts | 8 +-- 13 files changed, 93 insertions(+), 56 deletions(-) diff --git a/src/plugins/discover/common/index.ts b/src/plugins/discover/common/index.ts index 3a01e312fd393..6fc073ab166f4 100644 --- a/src/plugins/discover/common/index.ts +++ b/src/plugins/discover/common/index.ts @@ -22,4 +22,4 @@ export const MAX_DOC_FIELDS_DISPLAYED = 'discover:maxDocFieldsDisplayed'; export const SHOW_MULTIFIELDS = 'discover:showMultiFields'; export const SEARCH_EMBEDDABLE_TYPE = 'search'; export const AGGREGATED_VIEW_SETTING = 'discover:aggregatedView'; -export const AGGREGATED_VIEW_PREVIEW = 'data_visualizer:showAggregatedPreview'; +export const AGGREGATED_VIEW_PREVIEW = 'dataVisualizerTable:showPreview'; diff --git a/src/plugins/discover/server/ui_settings.ts b/src/plugins/discover/server/ui_settings.ts index 9e73657de6665..977b01ab45d7f 100644 --- a/src/plugins/discover/server/ui_settings.ts +++ b/src/plugins/discover/server/ui_settings.ts @@ -186,7 +186,7 @@ export const getUiSettings: () => Record = () => ({ schema: schema.boolean(), metric: { type: METRIC_TYPE.CLICK, - name: 'discover:useAggregatedView', + name: 'discover:aggregatedView', }, }, [AGGREGATED_VIEW_PREVIEW]: { @@ -201,7 +201,7 @@ export const getUiSettings: () => Record = () => ({ schema: schema.boolean(), metric: { type: METRIC_TYPE.CLICK, - name: 'discover:showAggregatedPreview', + name: 'dataVisualizerTable:showPreview', }, }, diff --git a/src/plugins/kibana_react/public/field_icon/field_icon.tsx b/src/plugins/kibana_react/public/field_icon/field_icon.tsx index fe7c292d3d00b..cd3c7ee9e7a99 100644 --- a/src/plugins/kibana_react/public/field_icon/field_icon.tsx +++ b/src/plugins/kibana_react/public/field_icon/field_icon.tsx @@ -50,8 +50,6 @@ export const typeToEuiIconMap: Partial> = { number_range: { iconType: 'tokenNumber' }, _source: { iconType: 'editorCodeBlock', color: 'gray' }, string: { iconType: 'tokenString' }, - text: { iconType: 'tokenString' }, - keyword: { iconType: 'tokenString' }, nested: { iconType: 'tokenNested' }, }; diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index 00bb24f5293fa..e7fdb183f32e7 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -468,4 +468,12 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, }, + 'discover:aggregatedView': { + type: 'boolean', + _meta: { description: 'Non-default value of setting.' }, + }, + 'dataVisualizerTable:showPreview': { + type: 'boolean', + _meta: { description: 'Non-default value of setting.' }, + }, }; diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index 2375de4a35467..5cbc62946b903 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -68,11 +68,13 @@ export interface UsageStats { 'notifications:lifetime:banner': number; 'notifications:lifetime:info': number; 'notifications:lifetime:error': number; + 'dataVisualizerTable:showPreview': boolean; 'doc_table:highlight': boolean; 'discover:searchOnPageLoad': boolean; // eslint-disable-next-line @typescript-eslint/naming-convention 'doc_table:hideTimeColumn': boolean; 'discover:sampleSize': number; + 'discover:aggregatedView': boolean; defaultColumns: string[]; 'context:defaultSize': number; 'context:tieBreakerFields': string[]; diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index b64c2fbe41265..74ab5b03691bb 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -7717,6 +7717,18 @@ "_meta": { "description": "Non-default value of setting." } + }, + "discover:aggregatedView": { + "type": "boolean", + "_meta": { + "description": "Non-default value of setting." + } + }, + "dataVisualizerTable:showPreview": { + "type": "boolean", + "_meta": { + "description": "Non-default value of setting." + } } } }, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index c4b1005ded959..b7bd45288e9bb 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -16,7 +16,6 @@ import { EuiInMemoryTable, EuiText, EuiToolTip, - getBreakpoint, HorizontalAlignment, LEFT_ALIGNMENT, RIGHT_ALIGNMENT, @@ -39,6 +38,7 @@ import { import { FileBasedNumberContentPreview } from '../field_data_row'; import { BooleanContentPreview } from './components/field_data_row'; import { FieldIcon } from '../../../../../../../../src/plugins/kibana_react/public'; +import { calculateTableColumnsDimensions, getKibanaFieldType } from './utils'; const FIELD_NAME = 'fieldName'; @@ -56,45 +56,6 @@ interface DataVisualizerTableProps { width: number; } -export const calculateTableColumnsDimensions = (width: number, showDistributions: boolean) => { - const breakPoint = getBreakpoint(width); - switch (breakPoint) { - case 'xs': - case 's': - return { - expander: '25px', - type: '40px', - docCount: '110px', - distinctValues: '75px', - distributions: showDistributions ? '120px' : '50px', - showIcons: false, - breakPoint, - }; - - case 'm': - case 'l': - return { - expander: '25px', - type: '40px', - docCount: '110px', - distinctValues: '75px', - distributions: showDistributions ? '120px' : '50px', - showIcons: false, - breakPoint, - }; - - default: - return { - expander: '40px', - type: '75px', - docCount: '175px', - distinctValues: '175px', - distributions: '150px', - showIcons: true, - breakPoint: 'xl', - }; - } -}; export const DataVisualizerTable = ({ items, pageState, @@ -205,8 +166,7 @@ export const DataVisualizerTable = ({ defaultMessage: 'Type', }), render: (fieldType: JobFieldType) => { - // @todo: fix scripted - return ; + return ; }, width: dimensions.type, sortable: true, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts index 27da91153b3ba..539c8621b915b 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { getBreakpoint } from '@elastic/eui'; import { FileBasedFieldVisConfig } from './types'; export const getTFPercentage = (config: FileBasedFieldVisConfig) => { @@ -36,3 +37,53 @@ export const getTFPercentage = (config: FileBasedFieldVisConfig) => { falseCount, }; }; + +export const calculateTableColumnsDimensions = (width: number, showDistributions: boolean) => { + const breakPoint = getBreakpoint(width); + switch (breakPoint) { + case 'xs': + case 's': + return { + expander: '25px', + type: '40px', + docCount: '110px', + distinctValues: '75px', + distributions: showDistributions ? '120px' : '50px', + showIcons: false, + breakPoint, + }; + + case 'm': + case 'l': + return { + expander: '25px', + type: '40px', + docCount: '110px', + distinctValues: '75px', + distributions: showDistributions ? '120px' : '50px', + showIcons: false, + breakPoint, + }; + + default: + return { + expander: '40px', + type: '75px', + docCount: '175px', + distinctValues: '175px', + distributions: '150px', + showIcons: true, + breakPoint: 'xl', + }; + } +}; + +export const getKibanaFieldType = (fieldType: string) => { + switch (fieldType) { + case 'text': + case 'keyword': + return 'string'; + default: + return fieldType; + } +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts index e69de29bb2d1d..91ca8e1633eb9 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { DataVisualizerGridEmbeddable } from './grid_embeddable'; +export { DataVisualizerGridEmbeddableFactory } from './grid_embeddable_factory'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.test.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.test.ts index 5fdbcea12006c..f56c3e6b733f9 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.test.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.test.ts @@ -52,8 +52,7 @@ describe('Index data visualizer locator', () => { expect(location).toMatchObject({ app: 'ml', path: - "/jobs/new_job/datavisualizer?_a=(DATA_VISUALIZER_INDEX_VIEWER:(visibleFieldNames:!('@timestamp',responsetime),visibleFieldTypes:!(number)))&_g=()", - state: {}, + "/jobs/new_job/datavisualizer?index=3da93760-e0af-11ea-9ad3-3bcfc330e42a&_a=(DATA_VISUALIZER_INDEX_VIEWER:(visibleFieldNames:!('@timestamp',responsetime),visibleFieldTypes:!(number)))&_g=()", }); }); diff --git a/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx b/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx index 5420fbbfbec12..8500d85d5580a 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx @@ -81,11 +81,9 @@ const PageWrapper: FC = ({ nextStepPath, deps, mode }) = services: { http: { basePath }, application: { navigateToUrl }, - embeddable: embeddablePlugin, }, } = useMlKibana(); - console.log('embeddablePlugin', embeddablePlugin); const { redirectToMlAccessDeniedPage } = deps; const redirectToJobsManagementPage = useCreateAndNavigateToMlLink( ML_PAGES.ANOMALY_DETECTION_JOBS_MANAGE diff --git a/x-pack/test/functional/services/ml/custom_urls.ts b/x-pack/test/functional/services/ml/custom_urls.ts index 67640eff7129e..c9836ce21a0c4 100644 --- a/x-pack/test/functional/services/ml/custom_urls.ts +++ b/x-pack/test/functional/services/ml/custom_urls.ts @@ -169,7 +169,8 @@ export function MachineLearningCustomUrlsProvider({ async assertDiscoverCustomUrlAction(expectedHitCountFormatted: string) { await PageObjects.discover.waitForDiscoverAppOnScreen(); - await retry.tryForTime(5000, async () => { + await PageObjects.discover.toggleSidebarCollapse(); + await retry.tryForTime(10 * 1000, async () => { const hitCount = await PageObjects.discover.getHitCount(); expect(hitCount).to.eql( expectedHitCountFormatted, diff --git a/x-pack/test/functional/services/ml/data_visualizer_table.ts b/x-pack/test/functional/services/ml/data_visualizer_table.ts index 2f67a9b75e3d6..7c0409f3ec27c 100644 --- a/x-pack/test/functional/services/ml/data_visualizer_table.ts +++ b/x-pack/test/functional/services/ml/data_visualizer_table.ts @@ -110,11 +110,11 @@ export function MachineLearningDataVisualizerTableProvider( if (!(await testSubjects.exists(this.detailsSelector(fieldName)))) { const selector = this.rowSelector( fieldName, - `dataVisualizerDetailsToggle-${fieldName}-arrowDown` + `dataVisualizerDetailsToggle-${fieldName}-arrowRight` ); await testSubjects.click(selector); await testSubjects.existOrFail( - this.rowSelector(fieldName, `dataVisualizerDetailsToggle-${fieldName}-arrowUp`), + this.rowSelector(fieldName, `dataVisualizerDetailsToggle-${fieldName}-arrowDown`), { timeout: 1000, } @@ -128,10 +128,10 @@ export function MachineLearningDataVisualizerTableProvider( await retry.tryForTime(10000, async () => { if (await testSubjects.exists(this.detailsSelector(fieldName))) { await testSubjects.click( - this.rowSelector(fieldName, `dataVisualizerDetailsToggle-${fieldName}-arrowUp`) + this.rowSelector(fieldName, `dataVisualizerDetailsToggle-${fieldName}-arrowDown`) ); await testSubjects.existOrFail( - this.rowSelector(fieldName, `dataVisualizerDetailsToggle-${fieldName}-arrowDown`), + this.rowSelector(fieldName, `dataVisualizerDetailsToggle-${fieldName}-arrowRight`), { timeout: 1000, } From d23b9c4970a01d7a829d53cf461eebcf0befa398 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 8 Sep 2021 15:57:19 -0500 Subject: [PATCH 058/188] Merge latest, move button to the right --- .../apps/main/components/chart/discover_chart.tsx | 8 ++++---- .../apps/main/components/layout/discover_layout.tsx | 6 ++---- x-pack/plugins/translations/translations/ja-JP.json | 2 +- x-pack/plugins/translations/translations/zh-CN.json | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx index 46610331ca56b..868b9733b445d 100644 --- a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx +++ b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx @@ -96,10 +96,6 @@ export function DiscoverChart({ onResetQuery={resetSavedSearch} /> - {!state.hideChart && ( @@ -131,6 +127,10 @@ export function DiscoverChart({ )} + diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 9026de7c8e1ec..2ec45e2ac77a2 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -83,10 +83,6 @@ export function DiscoverLayout({ } }, [uiSettings]); - const changeViewId = (option: string) => { - setDiscoverViewMode(option); - }; - const fetchCounter = useRef(0); const dataState: DataMainMsg = useDataState(main$); @@ -270,6 +266,8 @@ export function DiscoverLayout({ services={services} stateContainer={stateContainer} timefield={timeField} + discoverViewMode={discoverViewMode} + setDiscoverViewMode={setDiscoverViewMode} /> diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index c857c98d5b398..dc9b14fd067d3 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -27093,4 +27093,4 @@ "xpack.watcher.watchEdit.thresholdWatchExpression.aggType.fieldIsRequiredValidationMessage": "フィールドを選択してください。", "xpack.watcher.watcherDescription": "アラートの作成、管理、監視によりデータへの変更を検知します。" } -} \ No newline at end of file +} diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index d5022fb43da04..9a426c9faf721 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -27540,4 +27540,4 @@ "xpack.watcher.watchEdit.thresholdWatchExpression.aggType.fieldIsRequiredValidationMessage": "此字段必填。", "xpack.watcher.watcherDescription": "通过创建、管理和监测警报来检测数据中的更改。" } -} \ No newline at end of file +} From 96d112ffd80e20b807b18c6ce2eecb7e76c745a5 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 9 Sep 2021 14:40:02 -0500 Subject: [PATCH 059/188] Fix width for bar charts previews --- .../field_data_row/column_chart.tsx | 61 +++++++++---------- .../field_data_row/number_content_preview.tsx | 4 +- .../field_data_row/use_column_chart.tsx | 6 +- .../data_visualizer_stats_table.tsx | 2 +- 4 files changed, 35 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.tsx index 5f5cfbfe1b282..453754d4d6bd4 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.tsx @@ -8,7 +8,7 @@ import React, { FC } from 'react'; import classNames from 'classnames'; -import { BarSeries, Chart, Settings } from '@elastic/charts'; +import { Axis, BarSeries, Chart, Position, ScaleType, Settings } from '@elastic/charts'; import { EuiDataGridColumn } from '@elastic/eui'; import './column_chart.scss'; @@ -25,22 +25,9 @@ interface Props { maxChartColumns?: number; } -const columnChartTheme = { - background: { color: 'transparent' }, - chartMargins: { - left: 0, - right: 0, - top: 0, - bottom: 1, - }, - chartPaddings: { - left: 0, - right: 0, - top: 0, - bottom: 0, - }, - scales: { barsPadding: 0.1 }, -}; +const zeroSize = { bottom: 0, left: 0, right: 0, top: 0 }; +const size = { width: 100, height: 10 }; + export const ColumnChart: FC = ({ chartData, columnType, @@ -48,26 +35,34 @@ export const ColumnChart: FC = ({ hideLabel, maxChartColumns, }) => { - const { data, legendText, xScaleType } = useColumnChart(chartData, columnType, maxChartColumns); + const { data, legendText } = useColumnChart(chartData, columnType, maxChartColumns); return (
{!isUnsupportedChartData(chartData) && data.length > 0 && ( -
- - - d.datum.color} - data={data} - /> - -
+ + + { + return `${data[idx]?.key_as_string ?? ''}`; + }} + hide + /> + d.datum.color} + /> + )}
({ + data = chartData.data.map((d: OrdinalDataItem, idx) => ({ ...d, + x: idx, key_as_string: d.key_as_string ?? d.key, color: getColor(d), })); } else if (isNumericChartData(chartData)) { - data = chartData.data.map((d: NumericDataItem) => ({ + data = chartData.data.map((d: NumericDataItem, idx) => ({ ...d, + x: idx, key_as_string: d.key_as_string || d.key, color: getColor(d), })); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index b7bd45288e9bb..f47911b9b7ee8 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -310,7 +310,7 @@ export const DataVisualizerTable = ({ return null; }, width: dimensions.distributions, - align: CENTER_ALIGNMENT as HorizontalAlignment, + align: LEFT_ALIGNMENT as HorizontalAlignment, 'data-test-subj': 'dataVisualizerTableColumnDistribution', }, ]; From d36b30ca41d4120d2b5389a1f96652120a4e8f0e Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 9 Sep 2021 15:17:57 -0500 Subject: [PATCH 060/188] Fix toggle buttons, fix maps --- .../components/view_mode_toggle/view_mode_toggle.tsx | 7 ++----- .../common/components/embedded_map/embedded_map.tsx | 2 +- .../geo_point_content_with_map.tsx | 7 ++----- .../field_data_expanded_row/choropleth_map.tsx | 10 ++++++---- .../field_data_expanded_row/keyword_content.tsx | 7 +------ 5 files changed, 12 insertions(+), 21 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx index 6b2b72645ff56..f386da19e95d3 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx @@ -11,7 +11,6 @@ import React from 'react'; import { i18n } from '@kbn/i18n'; import { DISCOVER_VIEW_MODES } from './constants'; -// @todo: Revisit naming/terminology const toggleButtons = [ { id: DISCOVER_VIEW_MODES.DOCUMENT_LEVEL, @@ -31,11 +30,9 @@ export const DocumentViewModeToggle = ({ setDiscoverViewMode: (id: string) => void; }) => { return ( - // Table controls ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx index b4c8c3c22f5a9..c7e776f3ac55f 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx @@ -64,11 +64,8 @@ export const GeoPointContentWithMap: FC<{ return ( - - - - - + + diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx index 6bd4de22beca4..29629b0bf86b3 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx @@ -103,10 +103,12 @@ export const ChoroplethMap: FC = ({ stats, suggestion }) => { ); return ( - -
- -
+ + {isTopValuesSampled === true && ( <> diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx index bf66d8e21eada..d77f61ca5f52c 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx @@ -6,7 +6,6 @@ */ import React, { FC, useCallback, useEffect, useState } from 'react'; -import { EuiPanel } from '@elastic/eui'; import type { FieldDataRowProps } from '../../types/field_data_row'; import { TopValues } from '../../../top_values'; import { EMSTermJoinConfig } from '../../../../../../../../maps/public'; @@ -47,11 +46,7 @@ export const KeywordContent: FC = ({ config }) => { - {EMSSuggestion && stats && ( - - {' '} - - )} + {EMSSuggestion && stats && } ); }; From 76aed6382ee52db01134a1e13e0e760b004cd9b3 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 9 Sep 2021 16:55:05 -0500 Subject: [PATCH 061/188] Delete unused file --- .../components/layout/discover_layout.tsx | 4 +- .../sidebar/lib/get_field_type_name.ts | 9 ---- .../view_mode_toggle/view_mode_toggle.tsx | 2 +- .../apps/main/services/add_top_nav_links.ts | 44 ------------------- .../functional/services/ml/custom_urls.ts | 1 - 5 files changed, 3 insertions(+), 57 deletions(-) delete mode 100644 src/plugins/discover/public/application/apps/main/services/add_top_nav_links.ts diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 2ec45e2ac77a2..8d84f47bc19ba 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -30,7 +30,7 @@ import { } from '../../../../../../../data/public'; import { DiscoverSidebarResponsive } from '../sidebar'; import { DiscoverLayoutProps } from './types'; -import { SEARCH_FIELDS_FROM_SOURCE } from '../../../../../../common'; +import { AGGREGATED_VIEW_SETTING, SEARCH_FIELDS_FROM_SOURCE } from '../../../../../../common'; import { popularizeField } from '../../../../helpers/popularize_field'; import { DiscoverTopNav } from '../top_nav/discover_topnav'; import { DocViewFilterFn, ElasticSearchHit } from '../../../../doc_views/doc_views_types'; @@ -75,7 +75,7 @@ export function DiscoverLayout({ const [discoverViewMode, setDiscoverViewMode] = useState(DISCOVER_VIEW_MODES.DOCUMENT_LEVEL); useEffect(() => { - const userSetViewMode = uiSettings?.get('discover:aggregatedView'); + const userSetViewMode = uiSettings?.get(AGGREGATED_VIEW_SETTING); if (userSetViewMode) { setDiscoverViewMode(DISCOVER_VIEW_MODES.FIELD_LEVEL); } else { diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts b/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts index f68395593bd8b..e2d4c2f7ddcf2 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts @@ -51,15 +51,6 @@ export function getFieldTypeName(type: string) { return i18n.translate('discover.fieldNameIcons.stringFieldAriaLabel', { defaultMessage: 'String field', }); - case 'text': - return i18n.translate('discover.fieldNameIcons.textFieldAriaLabel', { - defaultMessage: 'Text field', - }); - case 'keyword': - return i18n.translate('discover.fieldNameIcons.keywordFieldAriaLabel', { - defaultMessage: 'Keyword field', - }); - case 'nested': return i18n.translate('discover.fieldNameIcons.nestedFieldAriaLabel', { defaultMessage: 'Nested field', diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx index f386da19e95d3..67e402c6ec471 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx @@ -32,7 +32,7 @@ export const DocumentViewModeToggle = ({ return ( Promise; - -/** - * Service for other plugins to register additional links or actions - * within Discover (e.g. top navigation bar) - */ -export class AddDataService { - private TopNavMenuDatas: Record = {}; - - public setup() { - return { - /** - * Registers an async callback function that will return a valid top nav link action - */ - registerTopNavLinks: (id: string, callbackFn: AddNavLinkCallback) => { - if (this.TopNavMenuDatas[id]) { - throw new Error(`link ${id} already exists`); - } - this.TopNavMenuDatas[id] = callbackFn; - }, - }; - } - - public getTopNavMenuDatas() { - return Object.values(this.TopNavMenuDatas); - } -} - -export type AddDataServiceSetup = ReturnType; diff --git a/x-pack/test/functional/services/ml/custom_urls.ts b/x-pack/test/functional/services/ml/custom_urls.ts index c9836ce21a0c4..ae9a388f30bcd 100644 --- a/x-pack/test/functional/services/ml/custom_urls.ts +++ b/x-pack/test/functional/services/ml/custom_urls.ts @@ -169,7 +169,6 @@ export function MachineLearningCustomUrlsProvider({ async assertDiscoverCustomUrlAction(expectedHitCountFormatted: string) { await PageObjects.discover.waitForDiscoverAppOnScreen(); - await PageObjects.discover.toggleSidebarCollapse(); await retry.tryForTime(10 * 1000, async () => { const hitCount = await PageObjects.discover.getHitCount(); expect(hitCount).to.eql( From 6a7b1bad6fd5f97a2a361738c9d2a98268bf0dd9 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Sun, 12 Sep 2021 20:58:56 -0500 Subject: [PATCH 062/188] Fix boolean styling --- .../components/field_data_expanded_row/boolean_content.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx index 2869b5030f81b..a787b1378c5a9 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx @@ -90,7 +90,7 @@ export const BooleanContent: FC = ({ config }) => { - + {summaryTableTitle} = ({ config }) => { /> - + Date: Mon, 13 Sep 2021 10:24:31 -0500 Subject: [PATCH 063/188] Change to enum, discover mode --- .../components/chart/discover_chart.test.tsx | 4 ++-- .../main/components/chart/discover_chart.tsx | 6 +++--- .../components/layout/discover_layout.tsx | 20 +++++++------------ .../components/view_mode_toggle/constants.ts | 8 ++++---- .../main/components/view_mode_toggle/index.ts | 2 +- .../view_mode_toggle/view_mode_toggle.tsx | 6 +++--- .../data_visualizer_grid.tsx | 8 +------- 7 files changed, 21 insertions(+), 33 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.test.tsx b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.test.tsx index 489b73b93a4bb..96973708df173 100644 --- a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.test.tsx @@ -19,7 +19,7 @@ import { discoverServiceMock } from '../../../../../__mocks__/services'; import { FetchStatus } from '../../../../types'; import { Chart } from './point_series'; import { DiscoverChart } from './discover_chart'; -import { DISCOVER_VIEW_MODES } from '../view_mode_toggle'; +import { DISCOVER_VIEW_MODE } from '../view_mode_toggle'; setHeaderActionMenuMounter(jest.fn()); @@ -107,7 +107,7 @@ function getProps(timefield?: string) { state: { columns: [] }, stateContainer: {} as GetStateReturn, timefield, - discoverViewMode: DISCOVER_VIEW_MODES.DOCUMENT_LEVEL, + discoverViewMode: DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, setDiscoverViewMode: jest.fn(), }; } diff --git a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx index 868b9733b445d..aa4fa84c92025 100644 --- a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx +++ b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx @@ -17,7 +17,7 @@ import { AppState, GetStateReturn } from '../../services/discover_state'; import { DiscoverHistogram } from './histogram'; import { DataCharts$, DataTotalHits$ } from '../../services/use_saved_search'; import { DiscoverServices } from '../../../../../build_services'; -import { DocumentViewModeToggle } from '../view_mode_toggle'; +import { DISCOVER_VIEW_MODE, DocumentViewModeToggle } from '../view_mode_toggle'; const TimechartHeaderMemoized = memo(TimechartHeader); const DiscoverHistogramMemoized = memo(DiscoverHistogram); @@ -41,8 +41,8 @@ export function DiscoverChart({ state: AppState; stateContainer: GetStateReturn; timefield?: string; - discoverViewMode: string; - setDiscoverViewMode: (discoverViewMode: string) => void; + discoverViewMode: DISCOVER_VIEW_MODE; + setDiscoverViewMode: (discoverViewMode: DISCOVER_VIEW_MODE) => void; }) { const { data, uiSettings: config } = services; const chartRef = useRef<{ element: HTMLElement | null; moveFocus: boolean }>({ diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 8d84f47bc19ba..93d3c9ab04320 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -44,7 +44,7 @@ import { DiscoverDocuments } from './discover_documents'; import { FetchStatus } from '../../../../types'; import { useDataState } from '../../utils/use_data_state'; import { DiscoverDataVisualizerGrid } from '../../../../components/data_visualizer_grid'; -import { DISCOVER_VIEW_MODES } from '../view_mode_toggle'; +import { DISCOVER_VIEW_MODE } from '../view_mode_toggle'; const SidebarMemoized = React.memo(DiscoverSidebarResponsive); const TopNavMemoized = React.memo(DiscoverTopNav); @@ -72,16 +72,11 @@ export function DiscoverLayout({ const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); - const [discoverViewMode, setDiscoverViewMode] = useState(DISCOVER_VIEW_MODES.DOCUMENT_LEVEL); - - useEffect(() => { - const userSetViewMode = uiSettings?.get(AGGREGATED_VIEW_SETTING); - if (userSetViewMode) { - setDiscoverViewMode(DISCOVER_VIEW_MODES.FIELD_LEVEL); - } else { - setDiscoverViewMode(DISCOVER_VIEW_MODES.DOCUMENT_LEVEL); - } - }, [uiSettings]); + const [discoverViewMode, setDiscoverViewMode] = useState(() => + uiSettings?.get(AGGREGATED_VIEW_SETTING) + ? DISCOVER_VIEW_MODE.FIELD_LEVEL + : DISCOVER_VIEW_MODE.DOCUMENT_LEVEL + ); const fetchCounter = useRef(0); const dataState: DataMainMsg = useDataState(main$); @@ -271,7 +266,7 @@ export function DiscoverLayout({ /> - {discoverViewMode === DISCOVER_VIEW_MODES.DOCUMENT_LEVEL ? ( + {discoverViewMode === DISCOVER_VIEW_MODE.DOCUMENT_LEVEL ? ( { return () => { // Clean up embeddable upon unmounting - if (embeddable) { - embeddable.destroy(); - } + embeddable?.destroy(); }; }, [embeddable]); From b0f7d16701159bd5d337bfa6a3ac1f155adaf832 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 13 Sep 2021 11:07:30 -0500 Subject: [PATCH 064/188] Hide field stats --- .../components/layout/discover_layout.tsx | 1 + .../components/sidebar/discover_field.tsx | 42 ++++++++++++------- .../sidebar/discover_field_visualize.tsx | 30 +++++++------ .../sidebar/discover_sidebar.test.tsx | 7 ++-- .../components/sidebar/discover_sidebar.tsx | 10 +++++ .../discover_sidebar_responsive.test.tsx | 2 + .../sidebar/discover_sidebar_responsive.tsx | 5 +++ .../view_mode_toggle/view_mode_toggle.tsx | 6 +-- 8 files changed, 66 insertions(+), 37 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 93d3c9ab04320..c228fb38e0b0c 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -195,6 +195,7 @@ export function DiscoverLayout({ trackUiMetric={trackUiMetric} useNewFieldsApi={useNewFieldsApi} onEditRuntimeField={onEditRuntimeField} + discoverViewMode={discoverViewMode} /> diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx index 707514073e23e..ab62094215bb2 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx @@ -19,6 +19,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiSpacer, + EuiHorizontalRule, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { UiCounterMetricType } from '@kbn/analytics'; @@ -251,6 +252,11 @@ export interface DiscoverFieldProps { * @param fieldName name of the field to delete */ onDeleteField?: (fieldName: string) => void; + + /** + * Optionally show or hide field stats in the popover + */ + showFieldStats?: boolean; } function DiscoverFieldComponent({ @@ -266,6 +272,7 @@ function DiscoverFieldComponent({ multiFields, onEditField, onDeleteField, + showFieldStats, }: DiscoverFieldProps) { const [infoIsOpen, setOpen] = useState(false); @@ -362,15 +369,27 @@ function DiscoverFieldComponent({ const details = getDetails(field); return ( <> - + {showFieldStats && ( + <> + +
+ {i18n.translate('discover.fieldChooser.discoverField.fieldTopValuesLabel', { + defaultMessage: 'Top 5 values', + })} +
+ + + + )} + {multiFields && ( <> - + {showFieldStats && } )} + {(showFieldStats || multiFields) && } ); }; - return ( {popoverTitle} - -
- {i18n.translate('discover.fieldChooser.discoverField.fieldTopValuesLabel', { - defaultMessage: 'Top 5 values', - })} -
- {infoIsOpen && renderPopover()} ); diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field_visualize.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field_visualize.tsx index baf740531e6bf..e974a67aef60d 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field_visualize.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field_visualize.tsx @@ -7,7 +7,7 @@ */ import React, { useEffect, useState } from 'react'; -import { EuiButton, EuiPopoverFooter } from '@elastic/eui'; +import { EuiButton } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { METRIC_TYPE, UiCounterMetricType } from '@kbn/analytics'; import type { IndexPattern, IndexPatternField } from 'src/plugins/data/common'; @@ -46,21 +46,19 @@ export const DiscoverFieldVisualize: React.FC = React.memo( }; return ( - - {/* eslint-disable-next-line @elastic/eui/href-or-on-click */} - - - - + // eslint-disable-next-line @elastic/eui/href-or-on-click + + + ); } ); diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.test.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.test.tsx index a9781595d698e..fbd950baa346d 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.test.tsx @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { each, cloneDeep } from 'lodash'; +import { cloneDeep, each } from 'lodash'; import { ReactWrapper } from 'enzyme'; import { findTestSubject } from '@elastic/eui/lib/test'; // @ts-expect-error @@ -14,14 +14,14 @@ import realHits from '../../../../../__fixtures__/real_hits.js'; import { mountWithIntl } from '@kbn/test/jest'; import React from 'react'; -import { DiscoverSidebarProps } from './discover_sidebar'; +import { DiscoverSidebar, DiscoverSidebarProps } from './discover_sidebar'; import { IndexPatternAttributes } from '../../../../../../../data/common'; import { SavedObject } from '../../../../../../../../core/types'; import { getDefaultFieldFilter } from './lib/field_filter'; -import { DiscoverSidebar } from './discover_sidebar'; import { ElasticSearchHit } from '../../../../doc_views/doc_views_types'; import { discoverServiceMock as mockDiscoverServices } from '../../../../../__mocks__/services'; import { stubLogstashIndexPattern } from '../../../../../../../data/common/stubs'; +import { DISCOVER_VIEW_MODE } from '../view_mode_toggle'; jest.mock('../../../../../kibana_services', () => ({ getServices: () => mockDiscoverServices, @@ -65,6 +65,7 @@ function getCompProps(): DiscoverSidebarProps { setFieldFilter: jest.fn(), onEditRuntimeField: jest.fn(), editField: jest.fn(), + discoverViewMode: DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, }; } diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx index b42bb4fe09bf1..2ac3dbb944d9a 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx @@ -37,6 +37,7 @@ import { getIndexPatternFieldList } from './lib/get_index_pattern_field_list'; import { DiscoverSidebarResponsiveProps } from './discover_sidebar_responsive'; import { DiscoverIndexPatternManagement } from './discover_index_pattern_management'; import { ElasticSearchHit } from '../../../../doc_views/doc_views_types'; +import { DISCOVER_VIEW_MODE } from '../view_mode_toggle'; /** * Default number of available fields displayed and added on scroll @@ -74,6 +75,10 @@ export interface DiscoverSidebarProps extends Omit(null); @@ -458,6 +464,9 @@ export function DiscoverSidebar({ multiFields={multiFields?.get(field.name)} onEditField={canEditIndexPatternField ? editField : undefined} onDeleteField={canEditIndexPatternField ? deleteField : undefined} + showFieldStats={ + discoverViewMode === DISCOVER_VIEW_MODE.DOCUMENT_LEVEL + } /> ); @@ -486,6 +495,7 @@ export function DiscoverSidebar({ multiFields={multiFields?.get(field.name)} onEditField={canEditIndexPatternField ? editField : undefined} onDeleteField={canEditIndexPatternField ? deleteField : undefined} + showFieldStats={discoverViewMode === DISCOVER_VIEW_MODE.DOCUMENT_LEVEL} /> ); diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.test.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.test.tsx index fc1c09ec8c829..a784f58faba1c 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.test.tsx @@ -26,6 +26,7 @@ import { ElasticSearchHit } from '../../../../doc_views/doc_views_types'; import { FetchStatus } from '../../../../types'; import { DataDocuments$ } from '../../services/use_saved_search'; import { stubLogstashIndexPattern } from '../../../../../../../data/common/stubs'; +import { DISCOVER_VIEW_MODE } from '../view_mode_toggle'; const mockServices = ({ history: () => ({ @@ -103,6 +104,7 @@ function getCompProps(): DiscoverSidebarResponsiveProps { state: {}, trackUiMetric: jest.fn(), onEditRuntimeField: jest.fn(), + discoverViewMode: DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, }; } diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.tsx index 7533a54ade405..3c506bbe86ad9 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.tsx @@ -37,6 +37,7 @@ import { AppState } from '../../services/discover_state'; import { DiscoverIndexPatternManagement } from './discover_index_pattern_management'; import { DataDocuments$ } from '../../services/use_saved_search'; import { calcFieldCounts } from '../../utils/calc_field_counts'; +import { DISCOVER_VIEW_MODE } from '../view_mode_toggle'; export interface DiscoverSidebarResponsiveProps { /** @@ -106,6 +107,10 @@ export interface DiscoverSidebarResponsiveProps { * callback to execute on edit runtime field */ onEditRuntimeField: () => void; + /** + * Discover view mode + */ + discoverViewMode: DISCOVER_VIEW_MODE; } /** diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx index 3d9008ecc61de..3085edab0c56e 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx @@ -26,8 +26,8 @@ export const DocumentViewModeToggle = ({ discoverViewMode, setDiscoverViewMode, }: { - discoverViewMode: string; - setDiscoverViewMode: (id: string) => void; + discoverViewMode: DISCOVER_VIEW_MODE; + setDiscoverViewMode: (discoverViewMode: DISCOVER_VIEW_MODE) => void; }) => { return ( setDiscoverViewMode(id)} + onChange={(id: string) => setDiscoverViewMode(id as DISCOVER_VIEW_MODE)} /> ); }; From bed2cd6a78eabfb1626ac599dbbe6d62e6fe77d1 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 13 Sep 2021 12:30:14 -0500 Subject: [PATCH 065/188] Hide field stats --- src/plugins/discover/common/index.ts | 1 - .../main/components/layout/discover_layout.tsx | 18 ++++++++++-------- .../apps/main/services/discover_state.ts | 5 +++++ .../apps/main/utils/get_state_defaults.ts | 5 +++++ .../apps/main/utils/persist_saved_search.ts | 4 ++++ .../discover/public/saved_searches/types.ts | 2 ++ src/plugins/discover/server/ui_settings.ts | 17 ----------------- .../server/collectors/management/schema.ts | 4 ---- .../server/collectors/management/types.ts | 1 - src/plugins/telemetry/schema/oss_plugins.json | 6 ------ 10 files changed, 26 insertions(+), 37 deletions(-) diff --git a/src/plugins/discover/common/index.ts b/src/plugins/discover/common/index.ts index 6fc073ab166f4..1407efc16f68b 100644 --- a/src/plugins/discover/common/index.ts +++ b/src/plugins/discover/common/index.ts @@ -21,5 +21,4 @@ export const SEARCH_FIELDS_FROM_SOURCE = 'discover:searchFieldsFromSource'; export const MAX_DOC_FIELDS_DISPLAYED = 'discover:maxDocFieldsDisplayed'; export const SHOW_MULTIFIELDS = 'discover:showMultiFields'; export const SEARCH_EMBEDDABLE_TYPE = 'search'; -export const AGGREGATED_VIEW_SETTING = 'discover:aggregatedView'; export const AGGREGATED_VIEW_PREVIEW = 'dataVisualizerTable:showPreview'; diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index c228fb38e0b0c..efde22826afb6 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -30,7 +30,7 @@ import { } from '../../../../../../../data/public'; import { DiscoverSidebarResponsive } from '../sidebar'; import { DiscoverLayoutProps } from './types'; -import { AGGREGATED_VIEW_SETTING, SEARCH_FIELDS_FROM_SOURCE } from '../../../../../../common'; +import { SEARCH_FIELDS_FROM_SOURCE } from '../../../../../../common'; import { popularizeField } from '../../../../helpers/popularize_field'; import { DiscoverTopNav } from '../top_nav/discover_topnav'; import { DocViewFilterFn, ElasticSearchHit } from '../../../../doc_views/doc_views_types'; @@ -72,10 +72,12 @@ export function DiscoverLayout({ const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); - const [discoverViewMode, setDiscoverViewMode] = useState(() => - uiSettings?.get(AGGREGATED_VIEW_SETTING) - ? DISCOVER_VIEW_MODE.FIELD_LEVEL - : DISCOVER_VIEW_MODE.DOCUMENT_LEVEL + + const setDiscoverViewMode = useCallback( + (mode: DISCOVER_VIEW_MODE) => { + stateContainer.setAppState({ discoverViewMode: mode }); + }, + [stateContainer] ); const fetchCounter = useRef(0); @@ -195,7 +197,7 @@ export function DiscoverLayout({ trackUiMetric={trackUiMetric} useNewFieldsApi={useNewFieldsApi} onEditRuntimeField={onEditRuntimeField} - discoverViewMode={discoverViewMode} + discoverViewMode={state.discoverViewMode} /> @@ -262,12 +264,12 @@ export function DiscoverLayout({ services={services} stateContainer={stateContainer} timefield={timeField} - discoverViewMode={discoverViewMode} + discoverViewMode={state.discoverViewMode} setDiscoverViewMode={setDiscoverViewMode} /> - {discoverViewMode === DISCOVER_VIEW_MODE.DOCUMENT_LEVEL ? ( + {state.discoverViewMode === DISCOVER_VIEW_MODE.DOCUMENT_LEVEL ? ( 0) { @@ -47,6 +48,7 @@ export function getStateDefaults({ interval: 'auto', filters: cloneDeep(searchSource.getOwnField('filter')), hideChart: undefined, + discoverViewMode: DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, } as AppState; if (savedSearch.grid) { defaultState.grid = savedSearch.grid; @@ -54,6 +56,9 @@ export function getStateDefaults({ if (savedSearch.hideChart) { defaultState.hideChart = savedSearch.hideChart; } + if (savedSearch.discoverViewMode) { + defaultState.discoverViewMode = savedSearch.discoverViewMode; + } return defaultState; } diff --git a/src/plugins/discover/public/application/apps/main/utils/persist_saved_search.ts b/src/plugins/discover/public/application/apps/main/utils/persist_saved_search.ts index a5e1e2bb6c2ea..1d857d7652b93 100644 --- a/src/plugins/discover/public/application/apps/main/utils/persist_saved_search.ts +++ b/src/plugins/discover/public/application/apps/main/utils/persist_saved_search.ts @@ -51,6 +51,10 @@ export async function persistSavedSearch( savedSearch.hideChart = state.hideChart; } + if (state.discoverViewMode) { + savedSearch.discoverViewMode = state.discoverViewMode; + } + try { const id = await savedSearch.save(saveOptions); onSuccess(id); diff --git a/src/plugins/discover/public/saved_searches/types.ts b/src/plugins/discover/public/saved_searches/types.ts index b1c7b48d696b3..d70d46e8bd389 100644 --- a/src/plugins/discover/public/saved_searches/types.ts +++ b/src/plugins/discover/public/saved_searches/types.ts @@ -9,6 +9,7 @@ import { SearchSource } from '../../../data/public'; import { SavedObjectSaveOpts } from '../../../saved_objects/public'; import { DiscoverGridSettings } from '../application/components/discover_grid/types'; +import { DISCOVER_VIEW_MODE } from '../application/apps/main/components/view_mode_toggle'; export type SortOrder = [string, string]; export interface SavedSearch { @@ -24,6 +25,7 @@ export interface SavedSearch { lastSavedTitle?: string; copyOnSave?: boolean; hideChart?: boolean; + discoverViewMode: DISCOVER_VIEW_MODE; } export interface SavedSearchLoader { get: (id: string) => Promise; diff --git a/src/plugins/discover/server/ui_settings.ts b/src/plugins/discover/server/ui_settings.ts index 977b01ab45d7f..8e1e04a08c799 100644 --- a/src/plugins/discover/server/ui_settings.ts +++ b/src/plugins/discover/server/ui_settings.ts @@ -26,7 +26,6 @@ import { SEARCH_FIELDS_FROM_SOURCE, MAX_DOC_FIELDS_DISPLAYED, SHOW_MULTIFIELDS, - AGGREGATED_VIEW_SETTING, AGGREGATED_VIEW_PREVIEW, } from '../common'; @@ -173,22 +172,6 @@ export const getUiSettings: () => Record = () => ({ name: 'discover:useLegacyDataGrid', }, }, - [AGGREGATED_VIEW_SETTING]: { - name: i18n.translate('discover.advancedSettings.aggregatedViewName', { - defaultMessage: 'Aggregated view', - }), - value: false, - description: i18n.translate('discover.advancedSettings.aggregatedViewDescription', { - defaultMessage: - 'Turn on this option to show the aggregated view by default. Turn off to show the document view.', - }), - category: ['discover'], - schema: schema.boolean(), - metric: { - type: METRIC_TYPE.CLICK, - name: 'discover:aggregatedView', - }, - }, [AGGREGATED_VIEW_PREVIEW]: { name: i18n.translate('discover.advancedSettings.showAggregatedPreviewName', { defaultMessage: 'Show aggregated preview', diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index 13f5624c8e85e..a1b8d2095e6c7 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -464,10 +464,6 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, }, - 'discover:aggregatedView': { - type: 'boolean', - _meta: { description: 'Non-default value of setting.' }, - }, 'dataVisualizerTable:showPreview': { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index e17a749ae8053..2bcb0437d93cf 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -73,7 +73,6 @@ export interface UsageStats { // eslint-disable-next-line @typescript-eslint/naming-convention 'doc_table:hideTimeColumn': boolean; 'discover:sampleSize': number; - 'discover:aggregatedView': boolean; defaultColumns: string[]; 'context:defaultSize': number; 'context:tieBreakerFields': string[]; diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index ab345b1399879..25429aadbdb21 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -7796,12 +7796,6 @@ "description": "Non-default value of setting." } }, - "discover:aggregatedView": { - "type": "boolean", - "_meta": { - "description": "Non-default value of setting." - } - }, "dataVisualizerTable:showPreview": { "type": "boolean", "_meta": { From f29765bfb5902237c2f5f50b5d7a22bae970b12c Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 13 Sep 2021 14:07:57 -0500 Subject: [PATCH 066/188] Persist show mini preview/distribution settings --- src/plugins/discover/common/index.ts | 1 - .../components/layout/discover_layout.tsx | 1 + .../apps/main/services/discover_state.ts | 4 +++ .../apps/main/utils/get_state_defaults.ts | 9 +++++- .../apps/main/utils/persist_saved_search.ts | 4 +++ .../data_visualizer_grid.tsx | 31 +++++++++++++------ .../public/saved_searches/_saved_search.ts | 4 +++ .../discover/public/saved_searches/types.ts | 3 +- .../discover/server/saved_objects/search.ts | 2 ++ src/plugins/discover/server/ui_settings.ts | 16 ---------- .../server/collectors/management/schema.ts | 4 --- .../server/collectors/management/types.ts | 1 - src/plugins/telemetry/schema/oss_plugins.json | 6 ---- .../data_visualizer_stats_table.tsx | 3 ++ .../grid_embeddable/grid_embeddable.tsx | 20 +++++++++--- 15 files changed, 66 insertions(+), 43 deletions(-) diff --git a/src/plugins/discover/common/index.ts b/src/plugins/discover/common/index.ts index 1407efc16f68b..b30fcf972eda5 100644 --- a/src/plugins/discover/common/index.ts +++ b/src/plugins/discover/common/index.ts @@ -21,4 +21,3 @@ export const SEARCH_FIELDS_FROM_SOURCE = 'discover:searchFieldsFromSource'; export const MAX_DOC_FIELDS_DISPLAYED = 'discover:maxDocFieldsDisplayed'; export const SHOW_MULTIFIELDS = 'discover:showMultiFields'; export const SEARCH_EMBEDDABLE_TYPE = 'search'; -export const AGGREGATED_VIEW_PREVIEW = 'dataVisualizerTable:showPreview'; diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index efde22826afb6..bd17741fa4100 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -292,6 +292,7 @@ export function DiscoverLayout({ query={state.query} filters={state.filters} columns={columns} + stateContainer={stateContainer} /> )} diff --git a/src/plugins/discover/public/application/apps/main/services/discover_state.ts b/src/plugins/discover/public/application/apps/main/services/discover_state.ts index fe2aad1456d3c..66f1f95e8a852 100644 --- a/src/plugins/discover/public/application/apps/main/services/discover_state.ts +++ b/src/plugins/discover/public/application/apps/main/services/discover_state.ts @@ -78,6 +78,10 @@ export interface AppState { * Table view: Document view or Aggregated view */ discoverViewMode?: DISCOVER_VIEW_MODE; + /** + * Hide mini distribution/preview charts when in Aggregated view + */ + hideAggregatedPreview?: boolean; } interface GetStateParams { diff --git a/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.ts b/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.ts index 94d9aa6745894..e05ac5a48153a 100644 --- a/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.ts +++ b/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.ts @@ -48,7 +48,8 @@ export function getStateDefaults({ interval: 'auto', filters: cloneDeep(searchSource.getOwnField('filter')), hideChart: undefined, - discoverViewMode: DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, + discoverViewMode: undefined, + hideAggregatedPreview: undefined, } as AppState; if (savedSearch.grid) { defaultState.grid = savedSearch.grid; @@ -58,6 +59,12 @@ export function getStateDefaults({ } if (savedSearch.discoverViewMode) { defaultState.discoverViewMode = savedSearch.discoverViewMode; + } else { + defaultState.discoverViewMode = DISCOVER_VIEW_MODE.DOCUMENT_LEVEL; + } + + if (savedSearch.hideAggregatedPreview) { + defaultState.hideAggregatedPreview = savedSearch.hideAggregatedPreview; } return defaultState; diff --git a/src/plugins/discover/public/application/apps/main/utils/persist_saved_search.ts b/src/plugins/discover/public/application/apps/main/utils/persist_saved_search.ts index 1d857d7652b93..7f671cbe2e1e6 100644 --- a/src/plugins/discover/public/application/apps/main/utils/persist_saved_search.ts +++ b/src/plugins/discover/public/application/apps/main/utils/persist_saved_search.ts @@ -55,6 +55,10 @@ export async function persistSavedSearch( savedSearch.discoverViewMode = state.discoverViewMode; } + if (state.hideAggregatedPreview) { + savedSearch.hideAggregatedPreview = state.hideAggregatedPreview; + } + try { const id = await savedSearch.save(saveOptions); onSuccess(id); diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index e90345b309759..d00b259459579 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import React, { useEffect, useRef, useState } from 'react'; +import React, { useEffect, useMemo, useRef, useState } from 'react'; import { EuiDataGrid, EuiDataGridProps } from '@elastic/eui'; import { Filter } from '@kbn/es-query'; import { DataView, Query } from '../../../../../data/common'; @@ -19,7 +19,7 @@ import { isErrorEmbeddable, } from '../../../../../embeddable/public'; import { SavedSearch } from '../../../saved_searches'; -import { AGGREGATED_VIEW_PREVIEW } from '../../../../common'; +import { GetStateReturn } from '../../apps/main/services/discover_state'; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { indexPattern: DataView; @@ -29,7 +29,9 @@ export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { filters?: Filter[]; showPreviewByDefault?: boolean; } -export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; +export interface DataVisualizerGridEmbeddableOutput extends EmbeddableOutput { + showDistributions?: boolean; +} export interface DiscoverDataVisualizerGridProps { /** @@ -55,6 +57,7 @@ export interface DiscoverDataVisualizerGridProps { savedSearch?: SavedSearch; query?: Query; filters?: Filter[]; + stateContainer: GetStateReturn; } export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { @@ -62,7 +65,7 @@ export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { }); export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { - const { services, indexPattern, savedSearch, query, columns, filters } = props; + const { services, indexPattern, savedSearch, query, columns, filters, stateContainer } = props; const { uiSettings } = services; const [embeddable, setEmbeddable] = useState< @@ -72,6 +75,19 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp >(); const embeddableRoot: React.RefObject = useRef(null); + const showPreviewByDefault = useMemo( + () => !stateContainer.appStateContainer.getState().hideAggregatedPreview, + [stateContainer.appStateContainer] + ); + + useEffect(() => { + embeddable?.getOutput$().subscribe((output: DataVisualizerGridEmbeddableOutput) => { + if (output.showDistributions !== undefined) { + stateContainer.setAppState({ hideAggregatedPreview: !output.showDistributions }); + } + }); + }, [embeddable, stateContainer]); + useEffect(() => { if (embeddable && !isErrorEmbeddable(embeddable)) { // Update embeddable whenever one of the important input changes @@ -87,7 +103,6 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp }, [embeddable, indexPattern, savedSearch, query, columns, filters]); useEffect(() => { - const showPreviewByDefault = uiSettings?.get(AGGREGATED_VIEW_PREVIEW); if (showPreviewByDefault && embeddable && !isErrorEmbeddable(embeddable)) { // Update embeddable whenever one of the important input changes embeddable.updateInput({ @@ -95,7 +110,7 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp }); embeddable.reload(); } - }, [uiSettings, embeddable]); + }, [showPreviewByDefault, uiSettings, embeddable]); useEffect(() => { return () => { @@ -107,8 +122,6 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp useEffect(() => { let unmounted = false; const loadEmbeddable = async () => { - const showPreviewByDefault = uiSettings?.get(AGGREGATED_VIEW_PREVIEW); - if (services?.embeddable) { const factory = services.embeddable.getEmbeddableFactory< DataVisualizerGridEmbeddableInput, @@ -134,7 +147,7 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp unmounted = true; }; // eslint-disable-next-line react-hooks/exhaustive-deps - }, [services?.embeddable]); + }, [services?.embeddable, showPreviewByDefault]); // We can only render after embeddable has already initialized useEffect(() => { diff --git a/src/plugins/discover/public/saved_searches/_saved_search.ts b/src/plugins/discover/public/saved_searches/_saved_search.ts index 320332ca4ace5..4194b0bd6c08b 100644 --- a/src/plugins/discover/public/saved_searches/_saved_search.ts +++ b/src/plugins/discover/public/saved_searches/_saved_search.ts @@ -14,7 +14,9 @@ export function createSavedSearchClass(savedObjects: SavedObjectsStart) { public static mapping = { title: 'text', description: 'text', + discoverViewMode: 'keyword', hideChart: 'boolean', + hideAggregatedPreview: 'boolean', hits: 'integer', columns: 'keyword', grid: 'object', @@ -35,7 +37,9 @@ export function createSavedSearchClass(savedObjects: SavedObjectsStart) { mapping: { title: 'text', description: 'text', + discoverViewMode: 'keyword', hideChart: 'boolean', + hideAggregatedPreview: 'boolean', hits: 'integer', columns: 'keyword', grid: 'object', diff --git a/src/plugins/discover/public/saved_searches/types.ts b/src/plugins/discover/public/saved_searches/types.ts index d70d46e8bd389..843495e25b0ec 100644 --- a/src/plugins/discover/public/saved_searches/types.ts +++ b/src/plugins/discover/public/saved_searches/types.ts @@ -25,7 +25,8 @@ export interface SavedSearch { lastSavedTitle?: string; copyOnSave?: boolean; hideChart?: boolean; - discoverViewMode: DISCOVER_VIEW_MODE; + discoverViewMode?: DISCOVER_VIEW_MODE; + hideAggregatedPreview?: boolean; } export interface SavedSearchLoader { get: (id: string) => Promise; diff --git a/src/plugins/discover/server/saved_objects/search.ts b/src/plugins/discover/server/saved_objects/search.ts index 070f0253f17e0..bca45c7f8cd34 100644 --- a/src/plugins/discover/server/saved_objects/search.ts +++ b/src/plugins/discover/server/saved_objects/search.ts @@ -34,7 +34,9 @@ export const searchSavedObjectType: SavedObjectsType = { properties: { columns: { type: 'keyword', index: false, doc_values: false }, description: { type: 'text' }, + discoverViewMode: { type: 'keyword', index: false, doc_values: false }, hideChart: { type: 'boolean', index: false, doc_values: false }, + hideAggregatedPreview: { type: 'boolean', index: false, doc_values: false }, hits: { type: 'integer', index: false, doc_values: false }, kibanaSavedObjectMeta: { properties: { diff --git a/src/plugins/discover/server/ui_settings.ts b/src/plugins/discover/server/ui_settings.ts index 8e1e04a08c799..007b6b0cf52e7 100644 --- a/src/plugins/discover/server/ui_settings.ts +++ b/src/plugins/discover/server/ui_settings.ts @@ -26,7 +26,6 @@ import { SEARCH_FIELDS_FROM_SOURCE, MAX_DOC_FIELDS_DISPLAYED, SHOW_MULTIFIELDS, - AGGREGATED_VIEW_PREVIEW, } from '../common'; export const getUiSettings: () => Record = () => ({ @@ -172,21 +171,6 @@ export const getUiSettings: () => Record = () => ({ name: 'discover:useLegacyDataGrid', }, }, - [AGGREGATED_VIEW_PREVIEW]: { - name: i18n.translate('discover.advancedSettings.showAggregatedPreviewName', { - defaultMessage: 'Show aggregated preview', - }), - value: false, - description: i18n.translate('discover.advancedSettings.showAggregatedPreviewDescription', { - defaultMessage: 'Turn on this option to show the preview in the aggregated view by default.', - }), - category: ['discover'], - schema: schema.boolean(), - metric: { - type: METRIC_TYPE.CLICK, - name: 'dataVisualizerTable:showPreview', - }, - }, [MODIFY_COLUMNS_ON_SWITCH]: { name: i18n.translate('discover.advancedSettings.discover.modifyColumnsOnSwitchTitle', { diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index a1b8d2095e6c7..68d56944d9974 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -464,8 +464,4 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, }, - 'dataVisualizerTable:showPreview': { - type: 'boolean', - _meta: { description: 'Non-default value of setting.' }, - }, }; diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index 2bcb0437d93cf..ed46e6b38b283 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -67,7 +67,6 @@ export interface UsageStats { 'notifications:lifetime:banner': number; 'notifications:lifetime:info': number; 'notifications:lifetime:error': number; - 'dataVisualizerTable:showPreview': boolean; 'doc_table:highlight': boolean; 'discover:searchOnPageLoad': boolean; // eslint-disable-next-line @typescript-eslint/naming-convention diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 25429aadbdb21..07c765b493db4 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -7795,12 +7795,6 @@ "_meta": { "description": "Non-default value of setting." } - }, - "dataVisualizerTable:showPreview": { - "type": "boolean", - "_meta": { - "description": "Non-default value of setting." - } } } }, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index f47911b9b7ee8..410ed638a0734 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -54,6 +54,7 @@ interface DataVisualizerTableProps { showPreviewByDefault?: boolean; /** Table width used to calculate the appropriate column widths **/ width: number; + onChange: (update: Partial) => void; } export const DataVisualizerTable = ({ @@ -64,6 +65,7 @@ export const DataVisualizerTable = ({ extendedColumns, showPreviewByDefault, width, + onChange, }: DataVisualizerTableProps) => { const [expandedRowItemIds, setExpandedRowItemIds] = useState([]); const [expandAll, toggleExpandAll] = useState(false); @@ -94,6 +96,7 @@ export const DataVisualizerTable = ({ const toggleShowDistribution = () => { setShowDistributions(!showDistributions); + onChange({ showDistributions: !showDistributions }); }; function toggleDetails(item: DataVisualizerTableItem) { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 75a7ec78186f7..b18a3805f739c 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -632,7 +632,13 @@ const useDataVisualizerGridData = ( }; }; -export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddableInput }) => { +export const DiscoverWrapper = ({ + input, + onOutputChange, +}: { + input: DataVisualizerGridEmbeddableInput; + onOutputChange?: (ouput: any) => void; +}) => { const [dataVisualizerListState, setDataVisualizerListState] = useState< Required >(restorableDefaults); @@ -640,8 +646,11 @@ export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddable const onTableChange = useCallback( (update: DataVisualizerTableState) => { setDataVisualizerListState({ ...dataVisualizerListState, ...update }); + if (onOutputChange) { + onOutputChange(update); + } }, - [dataVisualizerListState] + [dataVisualizerListState, onOutputChange] ); const { configs, searchQueryLanguage, searchString, extendedColumns } = useDataVisualizerGridData( input, @@ -676,6 +685,7 @@ export const DiscoverWrapper = ({ input }: { input: DataVisualizerGridEmbeddable extendedColumns={extendedColumns} showPreviewByDefault={input?.showPreviewByDefault} width={windowWidth - 300} + onChange={onOutputChange} /> ); @@ -686,12 +696,13 @@ export const IndexDataVisualizerViewWrapper = (props: { id: string; embeddableContext: InstanceType; embeddableInput: Readonly>; + onOutputChange?: (output: any) => void; }) => { - const { embeddableInput } = props; + const { embeddableInput, onOutputChange } = props; const input = useObservable(embeddableInput); if (input && input.indexPattern) { - return ; + return ; } else { return
; } @@ -726,6 +737,7 @@ export class DataVisualizerGridEmbeddable extends Embeddable< id={this.input.id} embeddableContext={this} embeddableInput={this.getInput$()} + onOutputChange={(output) => this.updateOutput(output)} /> From d0d2289bcfa60e89ad6bd3e7a784e595a6c51842 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 13 Sep 2021 14:48:31 -0500 Subject: [PATCH 067/188] Remove window size, use size observer instead --- .../fields_stats_grid/fields_stats_grid.tsx | 4 - .../metric_distribution_chart.tsx | 2 +- .../data_visualizer_stats_table.tsx | 77 +++++++++++-------- .../common/components/stats_table/utils.ts | 6 +- .../index_data_visualizer_view.tsx | 3 - .../grid_embeddable/grid_embeddable.tsx | 5 -- 6 files changed, 47 insertions(+), 50 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx index 74c5be99ce6e7..09b1e42c79a49 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx @@ -7,7 +7,6 @@ import React, { useMemo, FC, useState } from 'react'; import { EuiFlexGroup, EuiSpacer } from '@elastic/eui'; -import useWindowSize from 'react-use/lib/useWindowSize'; import type { FindFileStructureResponse } from '../../../../../../file_upload/common'; import type { DataVisualizerTableState } from '../../../../../common'; import { DataVisualizerTable, ItemIdToExpandedRowMap } from '../stats_table'; @@ -78,8 +77,6 @@ export const FieldsStatsGrid: FC = ({ results }) => { const fieldsCountStats = { visibleFieldsCount, totalFieldsCount }; const metricsStats = { visibleMetricsCount, totalMetricFieldsCount }; - const { width } = useWindowSize(); - return (
@@ -116,7 +113,6 @@ export const FieldsStatsGrid: FC = ({ results }) => { pageState={dataVisualizerListState} updatePageState={setDataVisualizerListState} getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} - width={width} />
); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/metric_distribution_chart/metric_distribution_chart.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/metric_distribution_chart/metric_distribution_chart.tsx index d2a271254269e..627c206e87fb0 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/metric_distribution_chart/metric_distribution_chart.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/metric_distribution_chart/metric_distribution_chart.tsx @@ -75,7 +75,7 @@ export const MetricDistributionChart: FC = ({ return ( ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 410ed638a0734..98a5e3e0de3b6 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -5,13 +5,12 @@ * 2.0. */ -import React, { useMemo, useState } from 'react'; +import React, { useCallback, useMemo, useState } from 'react'; import { CENTER_ALIGNMENT, EuiBasicTableColumn, EuiButtonIcon, - EuiFlexItem, EuiIcon, EuiInMemoryTable, EuiText, @@ -19,10 +18,11 @@ import { HorizontalAlignment, LEFT_ALIGNMENT, RIGHT_ALIGNMENT, + EuiResizeObserver, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { EuiTableComputedColumnType } from '@elastic/eui/src/components/basic_table/table_types'; -import useDebounce from 'react-use/lib/useDebounce'; +import { throttle } from 'lodash'; import { JOB_FIELD_TYPES, JobFieldType, DataVisualizerTableState } from '../../../../../common'; import { DocumentStat } from './components/field_data_row/document_stats'; import { DistinctValues } from './components/field_data_row/distinct_values'; @@ -52,9 +52,8 @@ interface DataVisualizerTableProps { getItemIdToExpandedRowMap: (itemIds: string[], items: T[]) => ItemIdToExpandedRowMap; extendedColumns?: Array>; showPreviewByDefault?: boolean; - /** Table width used to calculate the appropriate column widths **/ - width: number; - onChange: (update: Partial) => void; + /** Callback to receive any updates when table or page state is changed **/ + onChange?: (update: Partial) => void; } export const DataVisualizerTable = ({ @@ -64,7 +63,6 @@ export const DataVisualizerTable = ({ getItemIdToExpandedRowMap, extendedColumns, showPreviewByDefault, - width, onChange, }: DataVisualizerTableProps) => { const [expandedRowItemIds, setExpandedRowItemIds] = useState([]); @@ -86,18 +84,25 @@ export const DataVisualizerTable = ({ breakPoint: 'xl', }); - useDebounce( - () => { - setDimensions(calculateTableColumnsDimensions(width, showDistributions)); - }, - 100, - [width, showDistributions] + const [tableWidth, setTableWidth] = useState(1400); + + // eslint-disable-next-line react-hooks/exhaustive-deps + const resizeHandler = useCallback( + throttle((e: { width: number; height: number }) => { + // When window or table is resized, + // update the column widths and other settings accordingly + setTableWidth(e.width); + setDimensions(calculateTableColumnsDimensions(e.width)); + }, 500), + [tableWidth] ); - const toggleShowDistribution = () => { + const toggleShowDistribution = useCallback(() => { setShowDistributions(!showDistributions); - onChange({ showDistributions: !showDistributions }); - }; + if (onChange) { + onChange({ showDistributions: !showDistributions }); + } + }, [onChange, showDistributions]); function toggleDetails(item: DataVisualizerTableItem) { if (item.fieldName === undefined) return; @@ -331,23 +336,27 @@ export const DataVisualizerTable = ({ }, [expandAll, items, expandedRowItemIds]); return ( - - - className={'dataVisualizer'} - items={items} - itemId={FIELD_NAME} - columns={columns} - pagination={pagination} - sorting={sorting} - isExpandable={true} - itemIdToExpandedRowMap={itemIdToExpandedRowMap} - isSelectable={false} - onTableChange={onTableChange} - data-test-subj={'dataVisualizerTable'} - rowProps={(item) => ({ - 'data-test-subj': `dataVisualizerRow row-${item.fieldName}`, - })} - /> - + + {(resizeRef) => ( +
+ + className={'dataVisualizer'} + items={items} + itemId={FIELD_NAME} + columns={columns} + pagination={pagination} + sorting={sorting} + isExpandable={true} + itemIdToExpandedRowMap={itemIdToExpandedRowMap} + isSelectable={false} + onTableChange={onTableChange} + data-test-subj={'dataVisualizerTable'} + rowProps={(item) => ({ + 'data-test-subj': `dataVisualizerRow row-${item.fieldName}`, + })} + /> +
+ )} + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts index 539c8621b915b..c7814255b9f16 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts @@ -38,7 +38,7 @@ export const getTFPercentage = (config: FileBasedFieldVisConfig) => { }; }; -export const calculateTableColumnsDimensions = (width: number, showDistributions: boolean) => { +export const calculateTableColumnsDimensions = (width: number) => { const breakPoint = getBreakpoint(width); switch (breakPoint) { case 'xs': @@ -48,7 +48,7 @@ export const calculateTableColumnsDimensions = (width: number, showDistributions type: '40px', docCount: '110px', distinctValues: '75px', - distributions: showDistributions ? '120px' : '50px', + distributions: '150px', showIcons: false, breakPoint, }; @@ -60,7 +60,7 @@ export const calculateTableColumnsDimensions = (width: number, showDistributions type: '40px', docCount: '110px', distinctValues: '75px', - distributions: showDistributions ? '120px' : '50px', + distributions: '150px', showIcons: false, breakPoint, }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 7a40a6083a494..6ef56f154e48b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -23,7 +23,6 @@ import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_tab import { FormattedMessage } from '@kbn/i18n/react'; import { Required } from 'utility-types'; import { i18n } from '@kbn/i18n'; -import useWindowSize from 'react-use/lib/useWindowSize'; import { IndexPatternField, KBN_FIELD_TYPES, @@ -805,7 +804,6 @@ export const IndexDataVisualizerView: FC = (dataVi }, [currentIndexPattern, services, searchQueryLanguage, searchString]); const helpLink = docLinks.links.ml.guide; - const { width: windowWidth } = useWindowSize(); return ( @@ -898,7 +896,6 @@ export const IndexDataVisualizerView: FC = (dataVi updatePageState={setDataVisualizerListState} getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} extendedColumns={extendedColumns} - width={windowWidth - 300} /> diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index b18a3805f739c..1e72d6d82334c 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -14,7 +14,6 @@ import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_tab import { FormattedMessage } from '@kbn/i18n/react'; import { Filter } from '@kbn/es-query'; import { Required } from 'utility-types'; -import useWindowSize from 'react-use/lib/useWindowSize'; import { Embeddable, EmbeddableInput, @@ -76,7 +75,6 @@ export type IDataVisualizerGridEmbeddable = typeof DataVisualizerGridEmbeddable; const restorableDefaults = getDefaultDataVisualizerListState(); const defaults = getDefaultPageState(); -// @todo: consolidate this hook with main view const useDataVisualizerGridData = ( input: DataVisualizerGridEmbeddableInput, dataVisualizerListState: Required @@ -84,7 +82,6 @@ const useDataVisualizerGridData = ( const { services } = useDataVisualizerKibana(); const { notifications, uiSettings } = services; const { toasts } = notifications; - // @todo: consolidate dataVisualizerListState and input const { samplerShardSize, visibleFieldTypes, showEmptyFields } = dataVisualizerListState; const [lastRefresh, setLastRefresh] = useState(0); @@ -674,7 +671,6 @@ export const DiscoverWrapper = ({ }, [input, searchQueryLanguage, searchString] ); - const { width: windowWidth } = useWindowSize(); return ( @@ -684,7 +680,6 @@ export const DiscoverWrapper = ({ getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} extendedColumns={extendedColumns} showPreviewByDefault={input?.showPreviewByDefault} - width={windowWidth - 300} onChange={onOutputChange} /> ); From eb9683ca5bea7e574878cf954ddc8f608aea73ff Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 13 Sep 2021 14:53:24 -0500 Subject: [PATCH 068/188] Default to document view --- .../main/components/layout/discover_layout.tsx | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index bd17741fa4100..6be4eb32d5c3b 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -6,9 +6,8 @@ * Side Public License, v 1. */ import './discover_layout.scss'; -import React, { useState, useMemo, useCallback, useEffect, useRef } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { - EuiSpacer, EuiButtonIcon, EuiFlexGroup, EuiFlexItem, @@ -17,6 +16,7 @@ import { EuiPage, EuiPageBody, EuiPageContent, + EuiSpacer, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { METRIC_TYPE } from '@kbn/analytics'; @@ -73,6 +73,11 @@ export function DiscoverLayout({ const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); + const discoverViewMode = useMemo( + () => state.discoverViewMode ?? DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, + [state.discoverViewMode] + ); + const setDiscoverViewMode = useCallback( (mode: DISCOVER_VIEW_MODE) => { stateContainer.setAppState({ discoverViewMode: mode }); @@ -197,7 +202,7 @@ export function DiscoverLayout({ trackUiMetric={trackUiMetric} useNewFieldsApi={useNewFieldsApi} onEditRuntimeField={onEditRuntimeField} - discoverViewMode={state.discoverViewMode} + discoverViewMode={discoverViewMode} /> @@ -264,12 +269,12 @@ export function DiscoverLayout({ services={services} stateContainer={stateContainer} timefield={timeField} - discoverViewMode={state.discoverViewMode} + discoverViewMode={discoverViewMode} setDiscoverViewMode={setDiscoverViewMode} /> - {state.discoverViewMode === DISCOVER_VIEW_MODE.DOCUMENT_LEVEL ? ( + {discoverViewMode === DISCOVER_VIEW_MODE.DOCUMENT_LEVEL ? ( Date: Mon, 13 Sep 2021 15:01:15 -0500 Subject: [PATCH 069/188] Remove bold, switch icon --- .../field_data_row/distinct_values.tsx | 4 +-- .../field_data_row/document_stats.tsx | 2 +- .../data_visualizer_stats_table.tsx | 32 ++----------------- 3 files changed, 4 insertions(+), 34 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx index 353a9e40eeb61..a3d527dd84be9 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx @@ -23,9 +23,7 @@ export const DistinctValues = ({ cardinality, showIcons }: Props) => { ) : null} - - {cardinality} - + {cardinality} ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx index 07057f67a5e78..6d313341af6ca 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx @@ -31,7 +31,7 @@ export const DocumentStat = ({ config, showIcons }: Props) => { ) : null} - {count} ({docsPercent}%) + {count} ({docsPercent}%) ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 98a5e3e0de3b6..60304905cf834 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -231,35 +231,7 @@ export const DataVisualizerTable = ({
{dimensions.showIcons ? ( - ) : ( - - toggleShowDistribution()} - aria-label={ - showDistributions - ? i18n.translate('xpack.dataVisualizer.dataGrid.showDistributionsAriaLabel', { - defaultMessage: 'Show distributions', - }) - : i18n.translate('xpack.dataVisualizer.dataGrid.hideDistributionsAriaLabel', { - defaultMessage: 'Hide distributions', - }) - } - /> - - )} + ) : null} {i18n.translate('xpack.dataVisualizer.dataGrid.distributionsColumnName', { defaultMessage: 'Distributions', })} @@ -278,7 +250,7 @@ export const DataVisualizerTable = ({ toggleShowDistribution()} aria-label={ showDistributions From d810f941ea5a76678efcc48ca680d16d8b7f095a Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 13 Sep 2021 16:16:33 -0500 Subject: [PATCH 070/188] Set fixed width for top values, reduce font size in table --- .../examples_list/examples_list.tsx | 3 +- .../stats_table/_field_data_row.scss | 4 -- .../common/components/stats_table/_index.scss | 18 ++++-- .../field_data_row/distinct_values.tsx | 16 +----- .../field_data_row/document_stats.tsx | 21 ++----- .../data_visualizer_stats_table.tsx | 56 +++++++++++-------- .../common/components/stats_table/utils.ts | 28 +++++----- .../components/top_values/_top_values.scss | 2 +- .../components/top_values/top_values.tsx | 2 +- .../grid_embeddable/grid_embeddable.tsx | 2 +- 10 files changed, 74 insertions(+), 78 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx index ff2eaf6ef7137..2a6bf0b341907 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx @@ -31,8 +31,7 @@ export const ExamplesList: FC = ({ examples }) => { examplesContent = examples.map((example, i) => { return ( diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_field_data_row.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_field_data_row.scss index 944c31da8cab7..595687ac48731 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_field_data_row.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_field_data_row.scss @@ -56,10 +56,6 @@ overflow: hidden; } - .fieldDataCard__codeContent { - @include euiCodeFont; - } - .fieldDataCard__geoContent { z-index: auto; flex: 1; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index 289ed848113ce..7b6e3ce66e1b0 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -15,12 +15,21 @@ } .fieldDataCard__codeContent { - @include euiCodeFont; padding-left: 4px; } } .dataVisualizer { + + .columnHeaderTitle { + display: flex; + align-items: center; + } + + .columnHeaderIcon { + padding-right: $euiSizeXS; + } + .euiTableRow > .euiTableRowCell { border-bottom: 0; border-top: $euiBorderThin; @@ -60,6 +69,10 @@ min-height: 300px; min-width: 600px; } + .dataVisualizerTopValuesWrapper { + min-width: 400px; + max-width: 600px; + } .dataVisualizerPanelWrapper { border-radius: $euiBorderRadius; @@ -70,9 +83,6 @@ .dataVisualizerTextContent { min-width: 500px; - .euiListGroupItem__text { - padding: 0; - } } } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx index a3d527dd84be9..6d6152f75ebc3 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx @@ -5,25 +5,15 @@ * 2.0. */ -import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiText } from '@elastic/eui'; +import { EuiText } from '@elastic/eui'; import React from 'react'; interface Props { cardinality?: number; - showIcons: boolean; } -export const DistinctValues = ({ cardinality, showIcons }: Props) => { +export const DistinctValues = ({ cardinality }: Props) => { if (cardinality === undefined) return null; - return ( - - {showIcons ? ( - - - - ) : null} - {cardinality} - - ); + return {cardinality}; }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx index 6d313341af6ca..9452a66eedec7 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx @@ -5,16 +5,14 @@ * 2.0. */ -import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiText } from '@elastic/eui'; +import { EuiText } from '@elastic/eui'; import React from 'react'; import type { FieldDataRowProps } from '../../types/field_data_row'; import { roundToDecimalPlace } from '../../../utils'; -interface Props extends FieldDataRowProps { - showIcons: boolean; -} -export const DocumentStat = ({ config, showIcons }: Props) => { +type Props = FieldDataRowProps; +export const DocumentStat = ({ config }: Props) => { const { stats } = config; if (stats === undefined) return null; @@ -24,15 +22,8 @@ export const DocumentStat = ({ config, showIcons }: Props) => { const docsPercent = roundToDecimalPlace((count / sampleCount) * 100); return ( - - {showIcons ? ( - - - - ) : null} - - {count} ({docsPercent}%) - - + + {count} ({docsPercent}%) + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 60304905cf834..b853a8a4a9677 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -25,7 +25,6 @@ import { EuiTableComputedColumnType } from '@elastic/eui/src/components/basic_ta import { throttle } from 'lodash'; import { JOB_FIELD_TYPES, JobFieldType, DataVisualizerTableState } from '../../../../../common'; import { DocumentStat } from './components/field_data_row/document_stats'; -import { DistinctValues } from './components/field_data_row/distinct_values'; import { IndexBasedNumberContentPreview } from './components/field_data_row/number_content_preview'; import { useTableSettings } from './use_table_settings'; @@ -39,6 +38,7 @@ import { FileBasedNumberContentPreview } from '../field_data_row'; import { BooleanContentPreview } from './components/field_data_row'; import { FieldIcon } from '../../../../../../../../src/plugins/kibana_react/public'; import { calculateTableColumnsDimensions, getKibanaFieldType } from './utils'; +import { DistinctValues } from './components/field_data_row/distinct_values'; const FIELD_NAME = 'fieldName'; @@ -74,15 +74,7 @@ export const DataVisualizerTable = ({ updatePageState ); const [showDistributions, setShowDistributions] = useState(showPreviewByDefault ?? true); - const [dimensions, setDimensions] = useState({ - expander: '40px', - type: '75px', - docCount: '175px', - distinctValues: '175px', - distributions: '150px', - showIcons: true, - breakPoint: 'xl', - }); + const [dimensions, setDimensions] = useState(calculateTableColumnsDimensions()); const [tableWidth, setTableWidth] = useState(1400); @@ -192,7 +184,7 @@ export const DataVisualizerTable = ({ const displayName = item.displayName ?? item.fieldName; return ( - + {displayName} ); @@ -202,11 +194,19 @@ export const DataVisualizerTable = ({ }, { field: 'docCount', - name: i18n.translate('xpack.dataVisualizer.dataGrid.documentsCountColumnName', { - defaultMessage: 'Documents (%)', - }), + name: ( +
+ {dimensions.showIcons ? ( + + ) : null} + {i18n.translate('xpack.dataVisualizer.dataGrid.documentsCountColumnName', { + defaultMessage: 'Documents (%)', + })} +
+ ), + render: (value: number | undefined, item: DataVisualizerTableItem) => ( - + ), sortable: (item: DataVisualizerTableItem) => item?.stats?.count, align: LEFT_ALIGNMENT as HorizontalAlignment, @@ -215,12 +215,20 @@ export const DataVisualizerTable = ({ }, { field: 'stats.cardinality', - name: i18n.translate('xpack.dataVisualizer.dataGrid.distinctValuesColumnName', { - defaultMessage: 'Distinct values', - }), - render: (cardinality?: number) => ( - + name: ( +
+ {dimensions.showIcons ? ( + + ) : null} + {i18n.translate('xpack.dataVisualizer.dataGrid.distinctValuesColumnName', { + defaultMessage: 'Distinct values', + })} +
), + render: (cardinality: number | undefined, item: DataVisualizerTableItem) => ( + + ), + sortable: true, align: LEFT_ALIGNMENT as HorizontalAlignment, 'data-test-subj': 'dataVisualizerTableColumnDistinctValues', @@ -228,14 +236,14 @@ export const DataVisualizerTable = ({ }, { name: ( -
+
{dimensions.showIcons ? ( - + ) : null} {i18n.translate('xpack.dataVisualizer.dataGrid.distributionsColumnName', { defaultMessage: 'Distributions', })} - {dimensions.showIcons ? ( + { ({ } /> - ) : null} + }
), render: (item: DataVisualizerTableItem) => { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts index c7814255b9f16..fafb65ad17227 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts @@ -38,7 +38,17 @@ export const getTFPercentage = (config: FileBasedFieldVisConfig) => { }; }; -export const calculateTableColumnsDimensions = (width: number) => { +export const calculateTableColumnsDimensions = (width?: number) => { + const defaultSettings = { + expander: '40px', + type: '75px', + docCount: '200px', + distinctValues: '200px', + distributions: '150px', + showIcons: true, + breakPoint: 'xl', + }; + if (width === undefined) return defaultSettings; const breakPoint = getBreakpoint(width); switch (breakPoint) { case 'xs': @@ -47,8 +57,8 @@ export const calculateTableColumnsDimensions = (width: number) => { expander: '25px', type: '40px', docCount: '110px', - distinctValues: '75px', - distributions: '150px', + distinctValues: '90px', + distributions: '90px', showIcons: false, breakPoint, }; @@ -59,22 +69,14 @@ export const calculateTableColumnsDimensions = (width: number) => { expander: '25px', type: '40px', docCount: '110px', - distinctValues: '75px', + distinctValues: '110px', distributions: '150px', showIcons: false, breakPoint, }; default: - return { - expander: '40px', - type: '75px', - docCount: '175px', - distinctValues: '175px', - distributions: '150px', - showIcons: true, - breakPoint: 'xl', - }; + return defaultSettings; } }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/_top_values.scss b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/_top_values.scss index 05fa1bfa94b2d..fd7f52ee67cde 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/_top_values.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/_top_values.scss @@ -9,7 +9,7 @@ } &.topValuesValueLabelContainer--large { - width: 200px; + width: 300px; } } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index 9f87f099b847c..ecaf9befb9fe2 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -51,7 +51,7 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed return ( ), actions, - width: '75px', + width: '70px', }; return [actionColumn]; From db97adf8477f646d6389115a619f89c2a83bb14e Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 13 Sep 2021 16:19:37 -0500 Subject: [PATCH 071/188] Fix custom url tests --- x-pack/test/functional/services/ml/custom_urls.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/test/functional/services/ml/custom_urls.ts b/x-pack/test/functional/services/ml/custom_urls.ts index ae9a388f30bcd..7fb9933578b8e 100644 --- a/x-pack/test/functional/services/ml/custom_urls.ts +++ b/x-pack/test/functional/services/ml/custom_urls.ts @@ -169,6 +169,9 @@ export function MachineLearningCustomUrlsProvider({ async assertDiscoverCustomUrlAction(expectedHitCountFormatted: string) { await PageObjects.discover.waitForDiscoverAppOnScreen(); + // During cloud tests, the small browser width might cause hit count to be invisible + // so temporarily collapsing the sidebar ensures the count shows + await PageObjects.discover.toggleSidebarCollapse(); await retry.tryForTime(10 * 1000, async () => { const hitCount = await PageObjects.discover.getHitCount(); expect(hitCount).to.eql( From 9d520650834195e00d9dbfd2762332aca7612efa Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 13 Sep 2021 18:40:11 -0500 Subject: [PATCH 072/188] Update width styling for panels --- .../examples_list/examples_list.tsx | 2 +- .../common/components/stats_table/_index.scss | 26 +++++++++++++++---- .../boolean_content.tsx | 2 +- .../common/components/stats_table/utils.ts | 12 ++++----- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx index 2a6bf0b341907..068d0a415df30 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx @@ -42,7 +42,7 @@ export const ExamplesList: FC = ({ examples }) => { return (
= ({ config }) => { /> - + { return { expander: '25px', type: '40px', - docCount: '110px', - distinctValues: '90px', - distributions: '90px', + docCount: 'auto', + distinctValues: 'auto', + distributions: 'auto', showIcons: false, breakPoint, }; @@ -68,9 +68,9 @@ export const calculateTableColumnsDimensions = (width?: number) => { return { expander: '25px', type: '40px', - docCount: '110px', - distinctValues: '110px', - distributions: '150px', + docCount: 'auto', + distinctValues: 'auto', + distributions: 'auto', showIcons: false, breakPoint, }; From b3bb224b6b7429da2dde29f6c1762f3a95d56bd1 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 13 Sep 2021 19:35:41 -0500 Subject: [PATCH 073/188] Fix missing flag for Discover sidebar, jest tests --- .../main/components/sidebar/discover_sidebar.tsx | 11 +++++++---- .../apps/main/utils/get_state_defaults.test.ts | 4 ++++ test/functional/apps/discover/_shared_links.ts | 13 ++++++++----- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx index 2ac3dbb944d9a..3e84053158295 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx @@ -204,6 +204,10 @@ export function DiscoverSidebar({ return result; }, [fields]); + const showFieldStats = useMemo(() => discoverViewMode === DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, [ + discoverViewMode, + ]); + const calculateMultiFields = () => { if (!useNewFieldsApi || !fields) { return undefined; @@ -405,6 +409,7 @@ export function DiscoverSidebar({ multiFields={multiFields?.get(field.name)} onEditField={canEditIndexPatternField ? editField : undefined} onDeleteField={canEditIndexPatternField ? deleteField : undefined} + showFieldStats={showFieldStats} /> ); @@ -464,9 +469,7 @@ export function DiscoverSidebar({ multiFields={multiFields?.get(field.name)} onEditField={canEditIndexPatternField ? editField : undefined} onDeleteField={canEditIndexPatternField ? deleteField : undefined} - showFieldStats={ - discoverViewMode === DISCOVER_VIEW_MODE.DOCUMENT_LEVEL - } + showFieldStats={showFieldStats} /> ); @@ -495,7 +498,7 @@ export function DiscoverSidebar({ multiFields={multiFields?.get(field.name)} onEditField={canEditIndexPatternField ? editField : undefined} onDeleteField={canEditIndexPatternField ? deleteField : undefined} - showFieldStats={discoverViewMode === DISCOVER_VIEW_MODE.DOCUMENT_LEVEL} + showFieldStats={showFieldStats} /> ); diff --git a/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.test.ts b/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.test.ts index 554aca6ddb8f1..efc93cdc74fb4 100644 --- a/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.test.ts +++ b/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.test.ts @@ -26,7 +26,9 @@ describe('getStateDefaults', () => { "columns": Array [ "default_column", ], + "discoverViewMode": "discoverViewOptionDocument", "filters": undefined, + "hideAggregatedPreview": undefined, "hideChart": undefined, "index": "index-pattern-with-timefield-id", "interval": "auto", @@ -54,7 +56,9 @@ describe('getStateDefaults', () => { "columns": Array [ "default_column", ], + "discoverViewMode": "discoverViewOptionDocument", "filters": undefined, + "hideAggregatedPreview": undefined, "hideChart": undefined, "index": "the-index-pattern-id", "interval": "auto", diff --git a/test/functional/apps/discover/_shared_links.ts b/test/functional/apps/discover/_shared_links.ts index 62364739db311..4c822d13e43a1 100644 --- a/test/functional/apps/discover/_shared_links.ts +++ b/test/functional/apps/discover/_shared_links.ts @@ -76,11 +76,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const expectedUrl = baseUrl + '/app/discover?_t=1453775307251#' + - '/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time' + - ":(from:'2015-09-19T06:31:44.000Z',to:'2015-09" + - "-23T18:31:44.000Z'))&_a=(columns:!(),filters:!(),index:'logstash-" + - "*',interval:auto,query:(language:kuery,query:'')" + - ",sort:!(!('@timestamp',desc)))"; + '/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),' + + "time:(from:'2015-09-19T06:31:44.000Z',to:'2015-09" + + "-23T18:31:44.000Z'))&_a=(columns:!()," + + 'discoverViewMode:discoverViewOptionDocument,' + + "filters:!(),index:'logstash-*'," + + "interval:auto,query:(language:kuery,query:'')," + + "sort:!(!('@timestamp',desc)))"; + const actualUrl = await PageObjects.share.getSharedUrl(); // strip the timestamp out of each URL expect(actualUrl.replace(/_t=\d{13}/, '_t=TIMESTAMP')).to.be( From 006aeadbcba0b73a0e618cc2e6c61eb81ce25b65 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 14 Sep 2021 15:00:04 -0500 Subject: [PATCH 074/188] Fix max width --- .../application/common/components/stats_table/_index.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index 7f43a110e6184..cc432c0677ff9 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -85,7 +85,8 @@ $panelWidthXL: 900px; } .dataVisualizerUniformPanel { - width: $panelWidthS; + min-width: $panelWidthS; + max-width: $panelWidthS; } .dataVisualizerPanelWrapper { @@ -93,7 +94,6 @@ $panelWidthXL: 900px; border: 1px solid $euiBorderColor; padding: 6px; margin: 3px 12px 12px 0; - height: min-content; } .dataVisualizerTextContent { From ae2c0c340f7c834e2629987129123b424f5cf4d3 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 14 Sep 2021 16:50:08 -0500 Subject: [PATCH 075/188] Workaround for sorting --- .../field_data_row/distinct_values.tsx | 12 +++++-- .../field_data_row/document_stats.tsx | 17 +++++---- .../data_visualizer_stats_table.tsx | 35 ++++++------------- .../common/components/stats_table/utils.ts | 10 +++--- 4 files changed, 35 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx index 6d6152f75ebc3..56a0390311f20 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx @@ -5,15 +5,21 @@ * 2.0. */ -import { EuiText } from '@elastic/eui'; +import { EuiIcon, EuiText } from '@elastic/eui'; import React from 'react'; interface Props { cardinality?: number; + showIcon?: boolean; } -export const DistinctValues = ({ cardinality }: Props) => { +export const DistinctValues = ({ cardinality, showIcon }: Props) => { if (cardinality === undefined) return null; - return {cardinality}; + return ( + <> + {showIcon ? : null} + {cardinality} + + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx index 9452a66eedec7..ff313b9329eed 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx @@ -5,14 +5,16 @@ * 2.0. */ -import { EuiText } from '@elastic/eui'; +import { EuiIcon, EuiText } from '@elastic/eui'; import React from 'react'; import type { FieldDataRowProps } from '../../types/field_data_row'; import { roundToDecimalPlace } from '../../../utils'; -type Props = FieldDataRowProps; -export const DocumentStat = ({ config }: Props) => { +interface Props extends FieldDataRowProps { + showIcon?: boolean; +} +export const DocumentStat = ({ config, showIcon }: Props) => { const { stats } = config; if (stats === undefined) return null; @@ -22,8 +24,11 @@ export const DocumentStat = ({ config }: Props) => { const docsPercent = roundToDecimalPlace((count / sampleCount) * 100); return ( - - {count} ({docsPercent}%) - + <> + {showIcon ? : null} + + {count} ({docsPercent}%) + + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index b853a8a4a9677..135eea68ed067 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -194,19 +194,11 @@ export const DataVisualizerTable = ({ }, { field: 'docCount', - name: ( -
- {dimensions.showIcons ? ( - - ) : null} - {i18n.translate('xpack.dataVisualizer.dataGrid.documentsCountColumnName', { - defaultMessage: 'Documents (%)', - })} -
- ), - + name: i18n.translate('xpack.dataVisualizer.dataGrid.documentsCountColumnName', { + defaultMessage: 'Documents (%)', + }), render: (value: number | undefined, item: DataVisualizerTableItem) => ( - + ), sortable: (item: DataVisualizerTableItem) => item?.stats?.count, align: LEFT_ALIGNMENT as HorizontalAlignment, @@ -215,18 +207,11 @@ export const DataVisualizerTable = ({ }, { field: 'stats.cardinality', - name: ( -
- {dimensions.showIcons ? ( - - ) : null} - {i18n.translate('xpack.dataVisualizer.dataGrid.distinctValuesColumnName', { - defaultMessage: 'Distinct values', - })} -
- ), - render: (cardinality: number | undefined, item: DataVisualizerTableItem) => ( - + name: i18n.translate('xpack.dataVisualizer.dataGrid.distinctValuesColumnName', { + defaultMessage: 'Distinct values', + }), + render: (cardinality: number | undefined) => ( + ), sortable: true, @@ -237,7 +222,7 @@ export const DataVisualizerTable = ({ { name: (
- {dimensions.showIcons ? ( + {dimensions.showIcon ? ( ) : null} {i18n.translate('xpack.dataVisualizer.dataGrid.distributionsColumnName', { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts index f7fc846567e8b..040662c1eafff 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts @@ -42,10 +42,10 @@ export const calculateTableColumnsDimensions = (width?: number) => { const defaultSettings = { expander: '40px', type: '75px', - docCount: '200px', - distinctValues: '200px', + docCount: '225px', + distinctValues: '225px', distributions: '150px', - showIcons: true, + showIcon: true, breakPoint: 'xl', }; if (width === undefined) return defaultSettings; @@ -59,7 +59,7 @@ export const calculateTableColumnsDimensions = (width?: number) => { docCount: 'auto', distinctValues: 'auto', distributions: 'auto', - showIcons: false, + showIcon: false, breakPoint, }; @@ -71,7 +71,7 @@ export const calculateTableColumnsDimensions = (width?: number) => { docCount: 'auto', distinctValues: 'auto', distributions: 'auto', - showIcons: false, + showIcon: false, breakPoint, }; From 3ac229328368dfaa0f29146a468256907050590d Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 15 Sep 2021 10:47:15 -0500 Subject: [PATCH 076/188] Fix import --- .../apps/main/components/sidebar/discover_sidebar.test.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.test.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.test.tsx index 7dad33f96b773..fbd950baa346d 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.test.tsx @@ -18,7 +18,6 @@ import { DiscoverSidebar, DiscoverSidebarProps } from './discover_sidebar'; import { IndexPatternAttributes } from '../../../../../../../data/common'; import { SavedObject } from '../../../../../../../../core/types'; import { getDefaultFieldFilter } from './lib/field_filter'; -import { DiscoverSidebarComponent as DiscoverSidebar } from './discover_sidebar'; import { ElasticSearchHit } from '../../../../doc_views/doc_views_types'; import { discoverServiceMock as mockDiscoverServices } from '../../../../../__mocks__/services'; import { stubLogstashIndexPattern } from '../../../../../../../data/common/stubs'; From ed496bf282c5b41e49b559e46d63c7cb9f95c409 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 15 Sep 2021 11:00:10 -0500 Subject: [PATCH 077/188] Fix styling --- .../application/common/components/stats_table/_index.scss | 2 ++ .../components/field_data_expanded_row/boolean_content.tsx | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index cc432c0677ff9..402dffc8eec36 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -94,6 +94,8 @@ $panelWidthXL: 900px; border: 1px solid $euiBorderColor; padding: 6px; margin: 3px 12px 12px 0; + min-height: 120px; + height: min-content; } .dataVisualizerTextContent { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx index 6999716873e24..cfb5b3dd5de20 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx @@ -35,7 +35,7 @@ function getFormattedValue(value: number, totalCount: number): string { return `${value} (${getPercentLabel(percentage)})`; } -const BOOLEAN_DISTRIBUTION_CHART_HEIGHT = 100; +const BOOLEAN_DISTRIBUTION_CHART_HEIGHT = 70; export const BooleanContent: FC = ({ config }) => { const fieldFormat = 'fieldFormat' in config ? config.fieldFormat : undefined; From 8fc42e2f2d5d53feee5ad67b7871b069b99c278e Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 16 Sep 2021 12:14:08 -0500 Subject: [PATCH 078/188] Make height uniform, center alignment, fix map and keyword map not same size Move styling --- .../geo_point_content/geo_point_content.tsx | 4 +- .../geo_point_content_with_map.tsx | 2 +- .../common/components/stats_table/_index.scss | 20 ++++---- .../expanded_row_field_header.tsx | 7 ++- .../boolean_content.tsx | 1 + .../choropleth_map.tsx | 5 +- .../field_data_expanded_row/date_content.tsx | 1 + .../document_stats.tsx | 1 + .../expanded_row_content.tsx | 1 + .../field_data_expanded_row/ip_content.tsx | 2 +- .../keyword_content.tsx | 7 ++- .../number_content.tsx | 4 +- .../components/top_values/_top_values.scss | 15 +++--- .../components/top_values/top_values.tsx | 49 ++++++++++--------- 14 files changed, 67 insertions(+), 52 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content/geo_point_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content/geo_point_content.tsx index b732e542658b5..28f47d857e944 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content/geo_point_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content/geo_point_content.tsx @@ -63,9 +63,7 @@ export const GeoPointContent: FC = ({ config }) => { {formattedResults && Array.isArray(formattedResults.examples) && ( - - - + )} {formattedResults && Array.isArray(formattedResults.layerList) && ( - + diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index 402dffc8eec36..f697e35f4cc25 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -4,7 +4,6 @@ $panelWidthS: 300px; $panelWidthL: 600px; -$panelWidthXL: 900px; .dataVisualizerFieldExpandedRow { padding-left: $euiSize * 4; @@ -70,18 +69,20 @@ $panelWidthXL: 900px; .dataVisualizerSummaryTableWrapper { min-width: $panelWidthS; max-width: $panelWidthS; - max-height: 120px; } .dataVisualizerMapWrapper { - min-height: 300px; - min-width: $panelWidthS; - max-width: $panelWidthS; + min-height: 200px; + min-width: 450px; } .dataVisualizerTopValuesWrapper { min-width: $panelWidthS; - max-width: $panelWidthL; + width: fit-content; + + &.wrapperSize--compressed { + max-width: 450px; + } } .dataVisualizerUniformPanel { @@ -92,15 +93,12 @@ $panelWidthXL: 900px; .dataVisualizerPanelWrapper { border-radius: $euiBorderRadius; border: 1px solid $euiBorderColor; - padding: 6px; + padding: 4px; margin: 3px 12px 12px 0; - min-height: 120px; - height: min-content; } .dataVisualizerTextContent { - min-width: $panelWidthS; - max-width: $panelWidthXL; + min-width: 450px; } } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/expanded_row_field_header/expanded_row_field_header.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/expanded_row_field_header/expanded_row_field_header.tsx index 7279bceb8be93..78bf7b31804e7 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/expanded_row_field_header/expanded_row_field_header.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/expanded_row_field_header/expanded_row_field_header.tsx @@ -9,7 +9,12 @@ import { EuiText } from '@elastic/eui'; import React from 'react'; export const ExpandedRowFieldHeader = ({ children }: { children: React.ReactNode }) => ( - + {children} ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx index cfb5b3dd5de20..dd23af8108852 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx @@ -71,6 +71,7 @@ export const BooleanContent: FC = ({ config }) => { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, width: '75px', + align: 'right', }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx index 2a4b8cef470d7..492cd763f2757 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx @@ -105,14 +105,13 @@ export const ChoroplethMap: FC = ({ stats, suggestion }) => { return ( {isTopValuesSampled === true && ( <> - + = ({ config }) => { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, width: '75px', + align: 'right', }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx index b1340e9636860..1798c7cd3f037 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx @@ -18,6 +18,7 @@ const metaTableColumns = [ name: '', render: (metaItem: { display: ReactNode }) => metaItem.display, width: '75px', + align: 'right', }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_content.tsx index 68930dc733cc0..fe5672b7fc557 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_content.tsx @@ -18,6 +18,7 @@ export const ExpandedRowContent: FC = ({ children, dataTestSubj }) => { data-test-subj={dataTestSubj} gutterSize={'s'} className={'dataVisualizerExpandedRow'} + responsive={true} > {children} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx index 77cf5fad5cca8..629affdb94c5b 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx @@ -21,7 +21,7 @@ export const IpContent: FC = ({ config }) => { return ( - + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx index d77f61ca5f52c..e778cf582f28f 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx @@ -44,7 +44,12 @@ export const KeywordContent: FC = ({ config }) => { return ( - + {EMSSuggestion && stats && } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx index 888ecacf0d43a..b1c66fced598c 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx @@ -84,6 +84,7 @@ export const NumberContent: FC = ({ config }) => { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, width: '75px', + align: 'right', }, { field: 'value', @@ -112,7 +113,6 @@ export const NumberContent: FC = ({ config }) => { data-test-subj={'dataVisualizerNumberSummaryTable'} /> - {stats && ( )} @@ -139,7 +139,7 @@ export const NumberContent: FC = ({ config }) => { /> - + = ({ stats, fieldFormat, barColor, compressed return ( = ({ stats, fieldFormat, barColor, compressed {Array.isArray(topValues) && topValues.map((value) => ( + {progressBarMax !== undefined && ( + + + {getPercentLabel(value.doc_count, progressBarMax)} + + + )} + + + = ({ stats, fieldFormat, barColor, compressed )} > - + {kibanaFieldFormat(value.key, fieldFormat)} - - - - {progressBarMax !== undefined && ( - - - {getPercentLabel(value.doc_count, progressBarMax)} - - - )} ))} {isTopValuesSampled === true && ( - + Date: Thu, 16 Sep 2021 12:31:12 -0500 Subject: [PATCH 079/188] Revert "Make height uniform, center alignment, fix map and keyword map not same size" This reverts commit 8fc42e2f --- .../geo_point_content/geo_point_content.tsx | 4 +- .../geo_point_content_with_map.tsx | 2 +- .../common/components/stats_table/_index.scss | 19 +++---- .../choropleth_map.tsx | 5 +- .../expanded_row_content.tsx | 1 - .../keyword_content.tsx | 7 +-- .../number_content.tsx | 4 +- .../components/top_values/_top_values.scss | 15 +++--- .../components/top_values/top_values.tsx | 49 +++++++++---------- 9 files changed, 49 insertions(+), 57 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content/geo_point_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content/geo_point_content.tsx index 28f47d857e944..b732e542658b5 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content/geo_point_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content/geo_point_content.tsx @@ -63,7 +63,9 @@ export const GeoPointContent: FC = ({ config }) => { {formattedResults && Array.isArray(formattedResults.examples) && ( - + + + )} {formattedResults && Array.isArray(formattedResults.layerList) && ( - + diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index f697e35f4cc25..1d2222e614d3a 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -4,6 +4,7 @@ $panelWidthS: 300px; $panelWidthL: 600px; +$panelWidthXL: 900px; .dataVisualizerFieldExpandedRow { padding-left: $euiSize * 4; @@ -69,20 +70,18 @@ $panelWidthL: 600px; .dataVisualizerSummaryTableWrapper { min-width: $panelWidthS; max-width: $panelWidthS; + max-height: 120px; } .dataVisualizerMapWrapper { - min-height: 200px; - min-width: 450px; + min-height: 300px; + min-width: $panelWidthS; + max-width: $panelWidthS; } .dataVisualizerTopValuesWrapper { min-width: $panelWidthS; - width: fit-content; - - &.wrapperSize--compressed { - max-width: 450px; - } + max-width: $panelWidthL; } .dataVisualizerUniformPanel { @@ -93,12 +92,14 @@ $panelWidthL: 600px; .dataVisualizerPanelWrapper { border-radius: $euiBorderRadius; border: 1px solid $euiBorderColor; - padding: 4px; + padding: 6px; margin: 3px 12px 12px 0; + min-height: 120px; } .dataVisualizerTextContent { - min-width: 450px; + min-width: $panelWidthS; + max-width: $panelWidthXL; } } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx index 492cd763f2757..2a4b8cef470d7 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx @@ -105,13 +105,14 @@ export const ChoroplethMap: FC = ({ stats, suggestion }) => { return ( {isTopValuesSampled === true && ( <> - + = ({ children, dataTestSubj }) => { data-test-subj={dataTestSubj} gutterSize={'s'} className={'dataVisualizerExpandedRow'} - responsive={true} > {children} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx index e778cf582f28f..d77f61ca5f52c 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx @@ -44,12 +44,7 @@ export const KeywordContent: FC = ({ config }) => { return ( - + {EMSSuggestion && stats && } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx index b1c66fced598c..888ecacf0d43a 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx @@ -84,7 +84,6 @@ export const NumberContent: FC = ({ config }) => { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, width: '75px', - align: 'right', }, { field: 'value', @@ -113,6 +112,7 @@ export const NumberContent: FC = ({ config }) => { data-test-subj={'dataVisualizerNumberSummaryTable'} /> + {stats && ( )} @@ -139,7 +139,7 @@ export const NumberContent: FC = ({ config }) => { /> - + = ({ stats, fieldFormat, barColor, compressed return ( = ({ stats, fieldFormat, barColor, compressed {Array.isArray(topValues) && topValues.map((value) => ( - {progressBarMax !== undefined && ( - - - {getPercentLabel(value.doc_count, progressBarMax)} - - - )} - - - = ({ stats, fieldFormat, barColor, compressed )} > - + {kibanaFieldFormat(value.key, fieldFormat)} + + + + {progressBarMax !== undefined && ( + + + {getPercentLabel(value.doc_count, progressBarMax)} + + + )} ))} {isTopValuesSampled === true && ( - + Date: Thu, 16 Sep 2021 12:34:06 -0500 Subject: [PATCH 080/188] Revert "Make height uniform, center alignment, fix map and keyword map not same size" This reverts commit 8fc42e2f --- .../expanded_row_field_header.tsx | 7 +------ .../components/field_data_expanded_row/boolean_content.tsx | 1 - .../components/field_data_expanded_row/date_content.tsx | 1 - .../components/field_data_expanded_row/document_stats.tsx | 1 - .../components/field_data_expanded_row/ip_content.tsx | 2 +- 5 files changed, 2 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/expanded_row_field_header/expanded_row_field_header.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/expanded_row_field_header/expanded_row_field_header.tsx index 78bf7b31804e7..7279bceb8be93 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/expanded_row_field_header/expanded_row_field_header.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/expanded_row_field_header/expanded_row_field_header.tsx @@ -9,12 +9,7 @@ import { EuiText } from '@elastic/eui'; import React from 'react'; export const ExpandedRowFieldHeader = ({ children }: { children: React.ReactNode }) => ( - + {children} ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx index dd23af8108852..cfb5b3dd5de20 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx @@ -71,7 +71,6 @@ export const BooleanContent: FC = ({ config }) => { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, width: '75px', - align: 'right', }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx index a38bc869a458a..4bfcd593de766 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx @@ -62,7 +62,6 @@ export const DateContent: FC = ({ config }) => { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, width: '75px', - align: 'right', }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx index 1798c7cd3f037..b1340e9636860 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx @@ -18,7 +18,6 @@ const metaTableColumns = [ name: '', render: (metaItem: { display: ReactNode }) => metaItem.display, width: '75px', - align: 'right', }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx index 629affdb94c5b..77cf5fad5cca8 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx @@ -21,7 +21,7 @@ export const IpContent: FC = ({ config }) => { return ( - + ); }; From 84c4e649f9091055154fe65371290582a435b17a Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 16 Sep 2021 12:47:52 -0500 Subject: [PATCH 081/188] Uniform height, left aligned, flex grid --- .../geo_point_content_with_map.tsx | 2 +- .../common/components/stats_table/_index.scss | 7 ++----- .../field_data_expanded_row/expanded_row_content.tsx | 10 +++------- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx index 469016ad3dd2b..34641274e3518 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx @@ -64,7 +64,7 @@ export const GeoPointContentWithMap: FC<{ - + diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index 1d2222e614d3a..fa04149c47ba1 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -70,13 +70,11 @@ $panelWidthXL: 900px; .dataVisualizerSummaryTableWrapper { min-width: $panelWidthS; max-width: $panelWidthS; - max-height: 120px; } .dataVisualizerMapWrapper { - min-height: 300px; - min-width: $panelWidthS; - max-width: $panelWidthS; + min-height: 250px; + min-width: $panelWidthL; } .dataVisualizerTopValuesWrapper { @@ -94,7 +92,6 @@ $panelWidthXL: 900px; border: 1px solid $euiBorderColor; padding: 6px; margin: 3px 12px 12px 0; - min-height: 120px; } .dataVisualizerTextContent { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_content.tsx index 68930dc733cc0..87caa0386da94 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_content.tsx @@ -6,7 +6,7 @@ */ import React, { FC, ReactNode } from 'react'; -import { EuiFlexGroup } from '@elastic/eui'; +import { EuiFlexGrid } from '@elastic/eui'; interface Props { children: ReactNode; @@ -14,12 +14,8 @@ interface Props { } export const ExpandedRowContent: FC = ({ children, dataTestSubj }) => { return ( - + {children} - + ); }; From 70c3f9dcd5a0e50554f6150b87b5b412dd3aab06 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 16 Sep 2021 13:19:36 -0500 Subject: [PATCH 082/188] Switch top values to have labels --- .../common/components/stats_table/_index.scss | 7 ++--- .../choropleth_map.tsx | 3 +- .../number_content.tsx | 6 +++- .../components/top_values/top_values.tsx | 31 ++++--------------- 4 files changed, 15 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index fa04149c47ba1..ee207fc5e3e3e 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -79,7 +79,6 @@ $panelWidthXL: 900px; .dataVisualizerTopValuesWrapper { min-width: $panelWidthS; - max-width: $panelWidthL; } .dataVisualizerUniformPanel { @@ -89,9 +88,9 @@ $panelWidthXL: 900px; .dataVisualizerPanelWrapper { border-radius: $euiBorderRadius; - border: 1px solid $euiBorderColor; - padding: 6px; - margin: 3px 12px 12px 0; + border: $euiBorderThin; + padding: $euiSizeS; + margin: $euiSizeXS $euiSizeM $euiSizeM 0; } .dataVisualizerTextContent { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx index 2a4b8cef470d7..217c7e14f072d 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx @@ -105,8 +105,7 @@ export const ChoroplethMap: FC = ({ stats, suggestion }) => { return ( {isTopValuesSampled === true && ( diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx index 888ecacf0d43a..11cbe2808597d 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx @@ -101,7 +101,10 @@ export const NumberContent: FC = ({ config }) => { return ( - + {summaryTableTitle} className={'dataVisualizerSummaryTable'} @@ -120,6 +123,7 @@ export const NumberContent: FC = ({ config }) => { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index ecaf9befb9fe2..ea5a68eeeb3cd 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -6,14 +6,7 @@ */ import React, { FC, Fragment } from 'react'; -import { - EuiFlexGroup, - EuiFlexItem, - EuiProgress, - EuiSpacer, - EuiText, - EuiToolTip, -} from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, EuiProgress, EuiSpacer, EuiText } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -51,7 +44,7 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed return ( = ({ stats, fieldFormat, barColor, compressed
{Array.isArray(topValues) && topValues.map((value) => ( - - - - {kibanaFieldFormat(value.key, fieldFormat)} - - - {progressBarMax !== undefined && ( From 3acf891c039bfd1aeb731b0463b15eee9f5bfca1 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 16 Sep 2021 13:28:33 -0500 Subject: [PATCH 083/188] Center content --- .../expanded_row_field_header.tsx | 7 ++++++- .../components/field_data_expanded_row/boolean_content.tsx | 1 + .../components/field_data_expanded_row/choropleth_map.tsx | 2 +- .../components/field_data_expanded_row/date_content.tsx | 1 + .../components/field_data_expanded_row/document_stats.tsx | 1 + .../components/field_data_expanded_row/number_content.tsx | 3 ++- .../common/components/top_values/top_values.tsx | 2 +- 7 files changed, 13 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/expanded_row_field_header/expanded_row_field_header.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/expanded_row_field_header/expanded_row_field_header.tsx index 7279bceb8be93..78bf7b31804e7 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/expanded_row_field_header/expanded_row_field_header.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/expanded_row_field_header/expanded_row_field_header.tsx @@ -9,7 +9,12 @@ import { EuiText } from '@elastic/eui'; import React from 'react'; export const ExpandedRowFieldHeader = ({ children }: { children: React.ReactNode }) => ( - + {children} ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx index cfb5b3dd5de20..dd23af8108852 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx @@ -71,6 +71,7 @@ export const BooleanContent: FC = ({ config }) => { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, width: '75px', + align: 'right', }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx index 217c7e14f072d..492cd763f2757 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx @@ -111,7 +111,7 @@ export const ChoroplethMap: FC = ({ stats, suggestion }) => { {isTopValuesSampled === true && ( <> - + = ({ config }) => { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, width: '75px', + align: 'right', }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx index b1340e9636860..1798c7cd3f037 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx @@ -18,6 +18,7 @@ const metaTableColumns = [ name: '', render: (metaItem: { display: ReactNode }) => metaItem.display, width: '75px', + align: 'right', }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx index 11cbe2808597d..9899735dd1152 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx @@ -84,6 +84,7 @@ export const NumberContent: FC = ({ config }) => { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, width: '75px', + align: 'right', }, { field: 'value', @@ -143,7 +144,7 @@ export const NumberContent: FC = ({ config }) => { /> - + = ({ stats, fieldFormat, barColor, compressed {isTopValuesSampled === true && ( - + Date: Thu, 16 Sep 2021 14:42:59 -0500 Subject: [PATCH 084/188] Replace fixed widths with percentage --- .../examples_list/examples_list.tsx | 7 ++-- .../geo_point_content_with_map.tsx | 9 ++++-- .../common/components/stats_table/_index.scss | 18 +++++------ .../boolean_content.tsx | 11 ++++--- .../choropleth_map.tsx | 9 +++--- .../field_data_expanded_row/date_content.tsx | 9 +++--- .../document_stats.tsx | 11 ++++--- .../expanded_row_panel.tsx | 32 +++++++++++++++++++ .../number_content.tsx | 15 +++++---- .../field_data_expanded_row/other_content.tsx | 6 ++-- .../field_data_expanded_row/text_content.tsx | 2 +- .../components/top_values/_top_values.scss | 7 ---- .../components/top_values/top_values.tsx | 13 +++++--- 13 files changed, 93 insertions(+), 56 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_panel.tsx diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx index 068d0a415df30..49aee5f8ce7e5 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx @@ -11,6 +11,7 @@ import { EuiListGroup, EuiListGroupItem } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { ExpandedRowFieldHeader } from '../stats_table/components/expanded_row_field_header'; +import { ExpandedRowPanel } from '../stats_table/components/field_data_expanded_row/expanded_row_panel'; interface Props { examples: Array; } @@ -40,8 +41,8 @@ export const ExamplesList: FC = ({ examples }) => { } return ( -
@@ -56,6 +57,6 @@ export const ExamplesList: FC = ({ examples }) => { {examplesContent} -
+ ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx index 34641274e3518..b8d872f80b959 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx @@ -6,7 +6,6 @@ */ import React, { FC, useEffect, useState } from 'react'; -import { EuiFlexItem } from '@elastic/eui'; import { IndexPattern } from '../../../../../../../../../src/plugins/data/common'; import { CombinedQuery } from '../../../../index_data_visualizer/types/combined_query'; import { ExpandedRowContent } from '../../stats_table/components/field_data_expanded_row/expanded_row_content'; @@ -17,6 +16,7 @@ import { useDataVisualizerKibana } from '../../../../kibana_context'; import { JOB_FIELD_TYPES } from '../../../../../../common'; import { ES_GEO_FIELD_TYPE, LayerDescriptor } from '../../../../../../../maps/common'; import { EmbeddedMapComponent } from '../../embedded_map'; +import { ExpandedRowPanel } from '../../stats_table/components/field_data_expanded_row/expanded_row_panel'; export const GeoPointContentWithMap: FC<{ config: FieldVisConfig; @@ -64,9 +64,12 @@ export const GeoPointContentWithMap: FC<{ - + - + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index ee207fc5e3e3e..78afdebe446d3 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -2,9 +2,8 @@ @import 'components/field_count_stats/index'; @import 'components/field_data_row/index'; -$panelWidthS: 300px; -$panelWidthL: 600px; -$panelWidthXL: 900px; +$panelWidthS: #{'max(20%, 225px)'}; +$panelWidthL: #{'max(40%, 450px)'}; .dataVisualizerFieldExpandedRow { padding-left: $euiSize * 4; @@ -57,8 +56,8 @@ $panelWidthXL: 900px; } .dataVisualizerSummaryTable { - max-width: 350px; - min-width: 250px; + //max-width: #{$panelWidthS - $euiSize}; + //min-width: #{$panelWidthS - $euiSize}; .euiTableRow > .euiTableRowCell { border-bottom: 0; } @@ -87,15 +86,14 @@ $panelWidthXL: 900px; } .dataVisualizerPanelWrapper { - border-radius: $euiBorderRadius; - border: $euiBorderThin; - padding: $euiSizeS; - margin: $euiSizeXS $euiSizeM $euiSizeM 0; + margin: $euiSizeXS $euiSizeS $euiSizeS 0; + &.compressed { + max-width: $panelWidthS; + } } .dataVisualizerTextContent { min-width: $panelWidthS; - max-width: $panelWidthXL; } } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx index dd23af8108852..5084d36fdac02 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx @@ -18,6 +18,7 @@ import { roundToDecimalPlace } from '../../../utils'; import { useDataVizChartTheme } from '../../hooks'; import { DocumentStatsTable } from './document_stats'; import { ExpandedRowContent } from './expanded_row_content'; +import { ExpandedRowPanel } from './expanded_row_panel'; function getPercentLabel(value: number): string { if (value === 0) { @@ -70,7 +71,7 @@ export const BooleanContent: FC = ({ config }) => { { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, - width: '75px', + width: '25px', align: 'right', }, { @@ -91,7 +92,7 @@ export const BooleanContent: FC = ({ config }) => { - + {summaryTableTitle} = ({ config }) => { columns={summaryTableColumns} tableCaption={summaryTableTitle} /> - + - + = ({ config }) => { yScaleType="linear" /> - + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx index 492cd763f2757..634bebaf8e6d6 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx @@ -6,7 +6,7 @@ */ import React, { FC, useMemo } from 'react'; -import { EuiFlexItem, EuiSpacer, EuiText, htmlIdGenerator } from '@elastic/eui'; +import { EuiSpacer, EuiText, htmlIdGenerator } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { @@ -19,6 +19,7 @@ import { import { EMSTermJoinConfig } from '../../../../../../../../maps/public'; import { EmbeddedMapComponent } from '../../../embedded_map'; import { FieldVisStats } from '../../../../../../../common/types'; +import { ExpandedRowPanel } from './expanded_row_panel'; export const getChoroplethTopValuesLayer = ( fieldName: string, @@ -103,8 +104,8 @@ export const ChoroplethMap: FC = ({ stats, suggestion }) => { ); return ( - @@ -122,6 +123,6 @@ export const ChoroplethMap: FC = ({ stats, suggestion }) => { )} - + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx index a38bc869a458a..155cfd7c63905 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx @@ -6,7 +6,7 @@ */ import React, { FC, ReactNode } from 'react'; -import { EuiBasicTable, EuiFlexItem } from '@elastic/eui'; +import { EuiBasicTable } from '@elastic/eui'; // @ts-ignore import { formatDate } from '@elastic/eui/lib/services/format'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -16,6 +16,7 @@ import type { FieldDataRowProps } from '../../types/field_data_row'; import { ExpandedRowFieldHeader } from '../expanded_row_field_header'; import { DocumentStatsTable } from './document_stats'; import { ExpandedRowContent } from './expanded_row_content'; +import { ExpandedRowPanel } from './expanded_row_panel'; const TIME_FORMAT = 'MMM D YYYY, HH:mm:ss.SSS'; interface SummaryTableItem { function: string; @@ -61,7 +62,7 @@ export const DateContent: FC = ({ config }) => { { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, - width: '75px', + width: '80px', align: 'right', }, { @@ -74,7 +75,7 @@ export const DateContent: FC = ({ config }) => { return ( - + {summaryTableTitle} className={'dataVisualizerSummaryTable'} @@ -85,7 +86,7 @@ export const DateContent: FC = ({ config }) => { tableCaption={summaryTableTitle} tableLayout="auto" /> - + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx index 1798c7cd3f037..f2714c6bc61eb 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx @@ -8,16 +8,17 @@ import { FormattedMessage } from '@kbn/i18n/react'; import React, { FC, ReactNode } from 'react'; import { i18n } from '@kbn/i18n'; -import { EuiBasicTable, EuiFlexItem } from '@elastic/eui'; +import { EuiBasicTable } from '@elastic/eui'; import { ExpandedRowFieldHeader } from '../expanded_row_field_header'; import { FieldDataRowProps } from '../../types'; import { roundToDecimalPlace } from '../../../utils'; +import { ExpandedRowPanel } from './expanded_row_panel'; const metaTableColumns = [ { name: '', render: (metaItem: { display: ReactNode }) => metaItem.display, - width: '75px', + width: '25px', align: 'right', }, { @@ -77,8 +78,8 @@ export const DocumentStatsTable: FC = ({ config }) => { ]; return ( - {metaTableTitle} @@ -89,6 +90,6 @@ export const DocumentStatsTable: FC = ({ config }) => { columns={metaTableColumns} tableCaption={metaTableTitle} /> - + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_panel.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_panel.tsx new file mode 100644 index 0000000000000..62c014ba7d417 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_panel.tsx @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, ReactNode } from 'react'; +import { EuiPanel } from '@elastic/eui'; +import { EuiFlexItemProps } from '@elastic/eui/src/components/flex/flex_item'; + +interface Props { + children: ReactNode; + dataTestSubj?: string; + // 'data-test-subj'?: string; + grow?: EuiFlexItemProps['grow']; + className?: string; +} +export const ExpandedRowPanel: FC = ({ children, dataTestSubj, grow, className }) => { + return ( + + {children} + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx index 9899735dd1152..69990aada1d06 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx @@ -21,8 +21,9 @@ import { TopValues } from '../../../top_values'; import { ExpandedRowFieldHeader } from '../expanded_row_field_header'; import { DocumentStatsTable } from './document_stats'; import { ExpandedRowContent } from './expanded_row_content'; +import { ExpandedRowPanel } from './expanded_row_panel'; -const METRIC_DISTRIBUTION_CHART_WIDTH = 325; +const METRIC_DISTRIBUTION_CHART_WIDTH = 260; const METRIC_DISTRIBUTION_CHART_HEIGHT = 200; interface SummaryTableItem { @@ -83,7 +84,7 @@ export const NumberContent: FC = ({ config }) => { { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, - width: '75px', + width: '25px', align: 'right', }, { @@ -102,7 +103,7 @@ export const NumberContent: FC = ({ config }) => { return ( - @@ -115,16 +116,16 @@ export const NumberContent: FC = ({ config }) => { tableCaption={summaryTableTitle} data-test-subj={'dataVisualizerNumberSummaryTable'} /> - + {stats && ( )} {distribution && ( - @@ -155,7 +156,7 @@ export const NumberContent: FC = ({ config }) => { /> - + )} ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/other_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/other_content.tsx index 98d5cb2ec0fc9..38b22925ea5e2 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/other_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/other_content.tsx @@ -6,11 +6,11 @@ */ import React, { FC } from 'react'; -import { EuiFlexItem } from '@elastic/eui'; import type { FieldDataRowProps } from '../../types/field_data_row'; import { ExamplesList } from '../../../examples_list'; import { DocumentStatsTable } from './document_stats'; import { ExpandedRowContent } from './expanded_row_content'; +import { ExpandedRowPanel } from './expanded_row_panel'; export const OtherContent: FC = ({ config }) => { const { stats } = config; @@ -19,9 +19,9 @@ export const OtherContent: FC = ({ config }) => { {Array.isArray(stats.examples) && ( - + - + )} ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx index 9dd17411e36d5..fd939844218fe 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx @@ -6,7 +6,7 @@ */ import React, { FC, Fragment } from 'react'; -import { EuiCallOut, EuiFlexItem, EuiSpacer } from '@elastic/eui'; +import { EuiCallOut, EuiSpacer, EuiFlexItem } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/_top_values.scss b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/_top_values.scss index fd7f52ee67cde..1db0bc8270e97 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/_top_values.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/_top_values.scss @@ -4,13 +4,6 @@ .topValuesValueLabelContainer { margin-right: $euiSizeM; - &.topValuesValueLabelContainer--small { - width:70px; - } - - &.topValuesValueLabelContainer--large { - width: 300px; - } } .topValuesPercentLabelContainer { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index aacb840615601..08322bf0d2c6e 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -14,6 +14,7 @@ import classNames from 'classnames'; import { roundToDecimalPlace, kibanaFieldFormat } from '../utils'; import { ExpandedRowFieldHeader } from '../stats_table/components/expanded_row_field_header'; import { FieldVisStats } from '../../../../../common/types'; +import { ExpandedRowPanel } from '../stats_table/components/field_data_expanded_row/expanded_row_panel'; interface Props { stats: FieldVisStats | undefined; @@ -42,9 +43,9 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed } = stats; const progressBarMax = isTopValuesSampled === true ? topValuesSampleSize : count; return ( - = ({ stats, fieldFormat, barColor, compressed
{Array.isArray(topValues) && topValues.map((value) => ( @@ -97,6 +102,6 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed )}
- + ); }; From 6eceb57ec69958596656dc13af4ddc60f0a69e12 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 16 Sep 2021 15:29:43 -0500 Subject: [PATCH 085/188] Fix table missing field types --- .../common/components/stats_table/_index.scss | 8 +------- .../field_data_expanded_row/boolean_content.tsx | 7 ++++--- .../field_data_expanded_row/date_content.tsx | 8 +++++--- .../field_data_expanded_row/document_stats.tsx | 7 ++++--- .../field_data_expanded_row/number_content.tsx | 10 ++++++++-- .../index_data_visualizer/utils/saved_search_utils.ts | 2 +- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index 78afdebe446d3..a5cea0f56bfa6 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -16,10 +16,6 @@ $panelWidthL: #{'max(40%, 450px)'}; font-weight: bold; padding-bottom: $euiSizeS; } - - .fieldDataCard__codeContent { - padding-left: 4px; - } } .dataVisualizer { @@ -56,8 +52,6 @@ $panelWidthL: #{'max(40%, 450px)'}; } .dataVisualizerSummaryTable { - //max-width: #{$panelWidthS - $euiSize}; - //min-width: #{$panelWidthS - $euiSize}; .euiTableRow > .euiTableRowCell { border-bottom: 0; } @@ -86,7 +80,7 @@ $panelWidthL: #{'max(40%, 450px)'}; } .dataVisualizerPanelWrapper { - margin: $euiSizeXS $euiSizeS $euiSizeS 0; + margin: $euiSizeXS $euiSizeM $euiSizeM 0; &.compressed { max-width: $panelWidthS; } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx index 5084d36fdac02..96ecd33663500 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx @@ -6,7 +6,7 @@ */ import React, { FC, ReactNode, useMemo } from 'react'; -import { EuiBasicTable, EuiFlexItem, EuiSpacer } from '@elastic/eui'; +import { EuiBasicTable, EuiSpacer, RIGHT_ALIGNMENT, HorizontalAlignment } from '@elastic/eui'; import { Axis, BarSeries, Chart, Settings } from '@elastic/charts'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -69,10 +69,11 @@ export const BooleanContent: FC = ({ config }) => { ]; const summaryTableColumns = [ { + field: 'function', name: '', - render: (summaryItem: { display: ReactNode }) => summaryItem.display, + render: (_: string, summaryItem: { display: ReactNode }) => summaryItem.display, width: '25px', - align: 'right', + align: RIGHT_ALIGNMENT as HorizontalAlignment, }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx index 155cfd7c63905..19869ce7626e2 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx @@ -6,12 +6,13 @@ */ import React, { FC, ReactNode } from 'react'; -import { EuiBasicTable } from '@elastic/eui'; +import { EuiBasicTable, HorizontalAlignment } from '@elastic/eui'; // @ts-ignore import { formatDate } from '@elastic/eui/lib/services/format'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; +import { RIGHT_ALIGNMENT } from '@elastic/eui'; import type { FieldDataRowProps } from '../../types/field_data_row'; import { ExpandedRowFieldHeader } from '../expanded_row_field_header'; import { DocumentStatsTable } from './document_stats'; @@ -61,9 +62,10 @@ export const DateContent: FC = ({ config }) => { const summaryTableColumns = [ { name: '', - render: (summaryItem: { display: ReactNode }) => summaryItem.display, + field: 'function', + render: (func: string, summaryItem: { display: ReactNode }) => summaryItem.display, width: '80px', - align: 'right', + align: RIGHT_ALIGNMENT as HorizontalAlignment, }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx index f2714c6bc61eb..fd43cfae06f05 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx @@ -8,7 +8,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import React, { FC, ReactNode } from 'react'; import { i18n } from '@kbn/i18n'; -import { EuiBasicTable } from '@elastic/eui'; +import { EuiBasicTable, HorizontalAlignment, RIGHT_ALIGNMENT } from '@elastic/eui'; import { ExpandedRowFieldHeader } from '../expanded_row_field_header'; import { FieldDataRowProps } from '../../types'; import { roundToDecimalPlace } from '../../../utils'; @@ -16,10 +16,11 @@ import { ExpandedRowPanel } from './expanded_row_panel'; const metaTableColumns = [ { + field: 'function', name: '', - render: (metaItem: { display: ReactNode }) => metaItem.display, + render: (_: string, metaItem: { display: ReactNode }) => metaItem.display, width: '25px', - align: 'right', + align: RIGHT_ALIGNMENT as HorizontalAlignment, }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx index 69990aada1d06..62f7ced98743e 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx @@ -6,7 +6,13 @@ */ import React, { FC, ReactNode, useEffect, useState } from 'react'; -import { EuiBasicTable, EuiFlexItem, EuiText } from '@elastic/eui'; +import { + EuiBasicTable, + EuiFlexItem, + EuiText, + HorizontalAlignment, + RIGHT_ALIGNMENT, +} from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; @@ -85,7 +91,7 @@ export const NumberContent: FC = ({ config }) => { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, width: '25px', - align: 'right', + align: RIGHT_ALIGNMENT as HorizontalAlignment, }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index f3437df12ed62..2f182b1b0a615 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -16,7 +16,7 @@ import { Filter, } from '@kbn/es-query'; import { isSavedSearchSavedObject, SavedSearchSavedObject } from '../../../../common/types'; -import { IndexPattern } from '../../../../../../../src/plugins/data/common/index_patterns/index_patterns'; +import { IndexPattern } from '../../../../../../../src/plugins/data/common'; import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; import { SavedSearch } from '../../../../../../../src/plugins/discover/public'; import { getEsQueryConfig } from '../../../../../../../src/plugins/data/common'; From b8a31eab04cf7a510de582147aef84bf4d30282d Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 16 Sep 2021 16:03:56 -0500 Subject: [PATCH 086/188] Add dashboard embeddable and filter support --- .../components/layout/discover_layout.tsx | 3 +- .../components/view_mode_toggle/constants.ts | 4 +- .../view_mode_toggle/view_mode_toggle.tsx | 4 +- .../apps/main/services/discover_state.ts | 4 +- .../data_visualizer_grid.tsx | 36 +++++++-- .../field_stats_table_embeddable.tsx | 31 ++++++++ .../embeddable/saved_search_embeddable.tsx | 29 ++++++- .../expanded_row/index_based_expanded_row.tsx | 12 ++- .../common/components/stats_table/_index.scss | 4 - .../field_data_expanded_row/ip_content.tsx | 9 ++- .../keyword_content.tsx | 13 +++- .../number_content.tsx | 10 ++- .../stats_table/types/field_data_row.ts | 2 + .../components/top_values/_top_values.scss | 2 +- .../components/top_values/top_values.tsx | 78 +++++++++++++++++-- .../grid_embeddable/grid_embeddable.tsx | 10 ++- 16 files changed, 207 insertions(+), 44 deletions(-) create mode 100644 src/plugins/discover/public/application/components/data_visualizer_grid/field_stats_table_embeddable.tsx diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 6be4eb32d5c3b..4d6ce05560712 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -292,12 +292,11 @@ export function DiscoverLayout({ savedSearch={savedSearch} services={services} indexPattern={indexPattern} - searchDescription={savedSearch.description} - searchTitle={savedSearch.lastSavedTitle} query={state.query} filters={state.filters} columns={columns} stateContainer={stateContainer} + onAddFilter={onAddFilter} /> )} diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/constants.ts b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/constants.ts index 2bf6b30d03ada..be16ff7c6367f 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/constants.ts +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/constants.ts @@ -7,6 +7,6 @@ */ export enum DISCOVER_VIEW_MODE { - DOCUMENT_LEVEL = 'discoverViewOptionDocument', - FIELD_LEVEL = 'discoverViewOptionAggregated', + DOCUMENT_LEVEL = 'documents', + FIELD_LEVEL = 'fields', } diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx index 3085edab0c56e..81d978171d92b 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx @@ -14,11 +14,11 @@ import { DISCOVER_VIEW_MODE } from './constants'; const toggleButtons = [ { id: DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, - label: 'Document view', + label: 'Documents', }, { id: DISCOVER_VIEW_MODE.FIELD_LEVEL, - label: 'Aggregated view', + label: 'Field Statistics', }, ]; diff --git a/src/plugins/discover/public/application/apps/main/services/discover_state.ts b/src/plugins/discover/public/application/apps/main/services/discover_state.ts index 66f1f95e8a852..138b5ce566563 100644 --- a/src/plugins/discover/public/application/apps/main/services/discover_state.ts +++ b/src/plugins/discover/public/application/apps/main/services/discover_state.ts @@ -75,11 +75,11 @@ export interface AppState { */ savedQuery?: string; /** - * Table view: Document view or Aggregated view + * Table view: Documents vs Field Statistics */ discoverViewMode?: DISCOVER_VIEW_MODE; /** - * Hide mini distribution/preview charts when in Aggregated view + * Hide mini distribution/preview charts when in Field Statistics mode */ hideAggregatedPreview?: boolean; } diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index d00b259459579..a565f4bf7fee8 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -9,7 +9,7 @@ import React, { useEffect, useMemo, useRef, useState } from 'react'; import { EuiDataGrid, EuiDataGridProps } from '@elastic/eui'; import { Filter } from '@kbn/es-query'; -import { DataView, Query } from '../../../../../data/common'; +import { DataView, DataViewField, Query } from '../../../../../data/common'; import { DiscoverServices } from '../../../build_services'; import { EmbeddableInput, @@ -28,6 +28,10 @@ export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { visibleFieldNames?: string[]; filters?: Filter[]; showPreviewByDefault?: boolean; + /** + * Callback to add a filter to filter bar + */ + onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void; } export interface DataVisualizerGridEmbeddableOutput extends EmbeddableOutput { showDistributions?: boolean; @@ -39,7 +43,7 @@ export interface DiscoverDataVisualizerGridProps { */ columns: string[]; /** - * The used index pattern + * The used data view */ indexPattern: DataView; /** @@ -57,7 +61,11 @@ export interface DiscoverDataVisualizerGridProps { savedSearch?: SavedSearch; query?: Query; filters?: Filter[]; - stateContainer: GetStateReturn; + stateContainer?: GetStateReturn; + /** + * Callback to add a filter to filter bar + */ + onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void; } export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { @@ -65,7 +73,16 @@ export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { }); export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { - const { services, indexPattern, savedSearch, query, columns, filters, stateContainer } = props; + const { + services, + indexPattern, + savedSearch, + query, + columns, + filters, + stateContainer, + onAddFilter, + } = props; const { uiSettings } = services; const [embeddable, setEmbeddable] = useState< @@ -76,13 +93,14 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp const embeddableRoot: React.RefObject = useRef(null); const showPreviewByDefault = useMemo( - () => !stateContainer.appStateContainer.getState().hideAggregatedPreview, - [stateContainer.appStateContainer] + () => + stateContainer ? !stateContainer.appStateContainer.getState().hideAggregatedPreview : true, + [stateContainer] ); useEffect(() => { embeddable?.getOutput$().subscribe((output: DataVisualizerGridEmbeddableOutput) => { - if (output.showDistributions !== undefined) { + if (output.showDistributions !== undefined && stateContainer) { stateContainer.setAppState({ hideAggregatedPreview: !output.showDistributions }); } }); @@ -97,10 +115,11 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp query, filters, visibleFieldNames: columns, + onAddFilter, }); embeddable.reload(); } - }, [embeddable, indexPattern, savedSearch, query, columns, filters]); + }, [embeddable, indexPattern, savedSearch, query, columns, filters, onAddFilter]); useEffect(() => { if (showPreviewByDefault && embeddable && !isErrorEmbeddable(embeddable)) { @@ -135,6 +154,7 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp savedSearch, query, showPreviewByDefault, + onAddFilter, }); if (!unmounted) { setEmbeddable(initializedEmbeddable); diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/field_stats_table_embeddable.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/field_stats_table_embeddable.tsx new file mode 100644 index 0000000000000..099f45bf988cc --- /dev/null +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/field_stats_table_embeddable.tsx @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React from 'react'; +import { I18nProvider } from '@kbn/i18n/react'; +import { + DiscoverDataVisualizerGrid, + DiscoverDataVisualizerGridProps, +} from './data_visualizer_grid'; + +export function FieldStatsTableEmbeddable(renderProps: DiscoverDataVisualizerGridProps) { + return ( + + + + ); +} diff --git a/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx b/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx index e060a0fbae871..618de5ebda7fe 100644 --- a/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx +++ b/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx @@ -19,12 +19,12 @@ import { SEARCH_EMBEDDABLE_TYPE } from './constants'; import { APPLY_FILTER_TRIGGER, esFilters, FilterManager } from '../../../../data/public'; import { DiscoverServices } from '../../build_services'; import { - Query, - TimeRange, Filter, IndexPattern, - ISearchSource, IndexPatternField, + ISearchSource, + Query, + TimeRange, } from '../../../../data/common'; import { ElasticSearchHit } from '../doc_views/doc_views_types'; import { SavedSearchEmbeddableComponent } from './saved_search_embeddable_component'; @@ -45,6 +45,8 @@ import { DocTableProps } from '../apps/main/components/doc_table/doc_table_wrapp import { getDefaultSort } from '../apps/main/components/doc_table'; import { SortOrder } from '../apps/main/components/doc_table/components/table_header/helpers'; import { updateSearchSource } from './helpers/update_search_source'; +import { DISCOVER_VIEW_MODE } from '../apps/main/components/view_mode_toggle'; +import { FieldStatsTableEmbeddable } from '../components/data_visualizer_grid/field_stats_table_embeddable'; export type SearchProps = Partial & Partial & { @@ -378,6 +380,27 @@ export class SavedSearchEmbeddable if (!this.searchProps) { return; } + + if ( + this.savedSearch.discoverViewMode === DISCOVER_VIEW_MODE.FIELD_LEVEL && + searchProps.services && + searchProps.indexPattern && + Array.isArray(searchProps.columns) + ) { + ReactDOM.render( + , + domNode + ); + return; + } const useLegacyTable = this.services.uiSettings.get(DOC_TABLE_LEGACY); const props = { searchProps, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx index ca9c8301bcfba..d115c0431859a 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx @@ -22,15 +22,21 @@ import { FieldVisConfig } from '../stats_table/types'; import { IndexPattern } from '../../../../../../../../src/plugins/data/common'; import { CombinedQuery } from '../../../index_data_visualizer/types/combined_query'; import { LoadingIndicator } from '../loading_indicator'; +import { DataViewField } from '../../../../../../../../src/plugins/data/common'; export const IndexBasedDataVisualizerExpandedRow = ({ item, indexPattern, combinedQuery, + onAddFilter, }: { item: FieldVisConfig; indexPattern: IndexPattern | undefined; combinedQuery: CombinedQuery; + /** + * Callback to add a filter to filter bar + */ + onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void; }) => { const config = item; const { loading, type, existsInDocs, fieldName } = config; @@ -42,7 +48,7 @@ export const IndexBasedDataVisualizerExpandedRow = ({ switch (type) { case JOB_FIELD_TYPES.NUMBER: - return ; + return ; case JOB_FIELD_TYPES.BOOLEAN: return ; @@ -61,10 +67,10 @@ export const IndexBasedDataVisualizerExpandedRow = ({ ); case JOB_FIELD_TYPES.IP: - return ; + return ; case JOB_FIELD_TYPES.KEYWORD: - return ; + return ; case JOB_FIELD_TYPES.TEXT: return ; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index a5cea0f56bfa6..950c3733068f4 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -70,10 +70,6 @@ $panelWidthL: #{'max(40%, 450px)'}; min-width: $panelWidthL; } - .dataVisualizerTopValuesWrapper { - min-width: $panelWidthS; - } - .dataVisualizerUniformPanel { min-width: $panelWidthS; max-width: $panelWidthS; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx index 77cf5fad5cca8..a5db86e0c30a0 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx @@ -11,7 +11,7 @@ import { TopValues } from '../../../top_values'; import { DocumentStatsTable } from './document_stats'; import { ExpandedRowContent } from './expanded_row_content'; -export const IpContent: FC = ({ config }) => { +export const IpContent: FC = ({ config, onAddFilter }) => { const { stats } = config; if (stats === undefined) return null; const { count, sampleCount, cardinality } = stats; @@ -21,7 +21,12 @@ export const IpContent: FC = ({ config }) => { return ( - + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx index d77f61ca5f52c..2753373be1aa5 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx @@ -13,8 +13,12 @@ import { useDataVisualizerKibana } from '../../../../../kibana_context'; import { DocumentStatsTable } from './document_stats'; import { ExpandedRowContent } from './expanded_row_content'; import { ChoroplethMap } from './choropleth_map'; +import { DataViewField } from '../../../../../../../../../../src/plugins/data/common'; -export const KeywordContent: FC = ({ config }) => { +interface Props extends FieldDataRowProps { + onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void; +} +export const KeywordContent: FC = ({ config, onAddFilter }) => { const [EMSSuggestion, setEMSSuggestion] = useState(); const { stats, fieldName } = config; const fieldFormat = 'fieldFormat' in config ? config.fieldFormat : undefined; @@ -44,7 +48,12 @@ export const KeywordContent: FC = ({ config }) => { return ( - + {EMSSuggestion && stats && } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx index 62f7ced98743e..9ae120ec087e6 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx @@ -38,7 +38,7 @@ interface SummaryTableItem { value: number | string | undefined | null; } -export const NumberContent: FC = ({ config }) => { +export const NumberContent: FC = ({ config, onAddFilter }) => { const { stats } = config; useEffect(() => { @@ -125,7 +125,13 @@ export const NumberContent: FC = ({ config }) => { {stats && ( - + )} {distribution && ( void; } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/_top_values.scss b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/_top_values.scss index 1db0bc8270e97..b86974946bb38 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/_top_values.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/_top_values.scss @@ -8,5 +8,5 @@ .topValuesPercentLabelContainer { margin-left: $euiSizeM; - width:70px; + width: 40px; } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index 08322bf0d2c6e..408fd47e25520 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -6,7 +6,14 @@ */ import React, { FC, Fragment } from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiProgress, EuiSpacer, EuiText } from '@elastic/eui'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiProgress, + EuiSpacer, + EuiText, + EuiButtonIcon, +} from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -15,12 +22,14 @@ import { roundToDecimalPlace, kibanaFieldFormat } from '../utils'; import { ExpandedRowFieldHeader } from '../stats_table/components/expanded_row_field_header'; import { FieldVisStats } from '../../../../../common/types'; import { ExpandedRowPanel } from '../stats_table/components/field_data_expanded_row/expanded_row_panel'; +import { i18n } from '../../../../../../../../../../../../../private/var/tmp/_bazel_quynhnguyen/bd5cc7ce3740c1abb2c63a2609d8bb9f/execroot/kibana/bazel-out/darwin-fastbuild/bin/packages/kbn-i18n'; interface Props { stats: FieldVisStats | undefined; fieldFormat?: any; barColor?: 'primary' | 'secondary' | 'danger' | 'subdued' | 'accent'; compressed?: boolean; + onAddFilter?: (field: string, value: string, type: '+' | '-') => void; } function getPercentLabel(docCount: number, topValuesSampleSize: number): string { @@ -32,7 +41,7 @@ function getPercentLabel(docCount: number, topValuesSampleSize: number): string } } -export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed }) => { +export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed, onAddFilter }) => { if (stats === undefined) return null; const { topValues, @@ -40,6 +49,7 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed topValuesSamplerShardSize, count, isTopValuesSampled, + fieldName, } = stats; const progressBarMax = isTopValuesSampled === true ? topValuesSampleSize : count; return ( @@ -56,11 +66,7 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed
{Array.isArray(topValues) && topValues.map((value) => ( @@ -85,6 +91,64 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed )} + {fieldName !== undefined && value.key !== undefined && onAddFilter !== undefined ? ( + <> + + onAddFilter( + fieldName, + typeof value.key === 'number' ? value.key.toString() : value.key, + '+' + ) + } + aria-label={i18n.translate( + 'xpack.dataVisualizer.dataGrid.field.addFilterAriaLabel', + { + defaultMessage: 'Filter for {fieldName}: "{value}"', + values: { fieldName, value: value.key }, + } + )} + data-test-subj={`dvFieldDataTopValuesAddFilterButton-${value.key}-${value.key}`} + style={{ + minHeight: 'auto', + minWidth: 'auto', + paddingRight: 2, + paddingLeft: 2, + paddingTop: 0, + paddingBottom: 0, + }} + /> + + onAddFilter( + fieldName, + typeof value.key === 'number' ? value.key.toString() : value.key, + '-' + ) + } + aria-label={i18n.translate( + 'xpack.dataVisualizer.dataGrid.field.removeFilterAriaLabel', + { + defaultMessage: 'Filter out {fieldName}: "{value}"', + values: { fieldName, value: value.key }, + } + )} + data-test-subj={`dvFieldDataTopValuesExcludeFilterButton-${value.key}-${value.key}`} + style={{ + minHeight: 'auto', + minWidth: 'auto', + paddingTop: 0, + paddingBottom: 0, + paddingRight: 2, + paddingLeft: 2, + }} + /> + + ) : null} ))} {isTopValuesSampled === true && ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 406547566f3ce..8674d742efb92 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -25,8 +25,8 @@ import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; import { EmbeddableLoading } from './embeddable_loading_fallback'; import { DataVisualizerStartDependencies } from '../../../../plugin'; import { - IndexPattern, - IndexPatternField, + DataView, + DataViewField, KBN_FIELD_TYPES, Query, UI_SETTINGS, @@ -61,12 +61,13 @@ import { getActions } from '../../../common/components/field_data_row/action_men import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerStartDependencies]; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { - indexPattern: IndexPattern; + indexPattern: DataView; savedSearch?: SavedSearch; query?: Query; visibleFieldNames?: string[]; filters?: Filter[]; showPreviewByDefault?: boolean; + onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void; } export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; @@ -175,7 +176,7 @@ const useDataVisualizerGridData = ( }); }, [uiSettings]); - const indexPatternFields: IndexPatternField[] = useMemo(() => currentIndexPattern.fields, [ + const indexPatternFields: DataViewField[] = useMemo(() => currentIndexPattern.fields, [ currentIndexPattern, ]); @@ -663,6 +664,7 @@ export const DiscoverWrapper = ({ item={item} indexPattern={input.indexPattern} combinedQuery={{ searchQueryLanguage, searchString }} + onAddFilter={input.onAddFilter} /> ); } From 90c8259ab7190a9df0f5f6c22c7ad44c8e30f904 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 16 Sep 2021 15:29:43 -0500 Subject: [PATCH 087/188] Fix file data viz styling and tests, lean up imports, remove hard coded pixels --- .../components/view_mode_toggle/_index.scss | 1 + .../view_mode_toggle/_view_mode_toggle.scss | 4 +++ .../view_mode_toggle/view_mode_toggle.tsx | 3 ++- .../data_visualizer_grid.tsx | 1 + .../field_data_row/number_content_preview.tsx | 26 ++++++++++++------- .../common/components/stats_table/_index.scss | 11 +++----- .../boolean_content.tsx | 7 ++--- .../field_data_expanded_row/date_content.tsx | 10 ++++--- .../document_stats.tsx | 7 ++--- .../expanded_row_panel.tsx | 1 - .../keyword_content.tsx | 1 - .../number_content.tsx | 12 ++++++--- .../field_data_row/column_chart.scss | 3 +-- .../common/components/stats_table/utils.ts | 2 +- .../components/top_values/top_values.tsx | 8 ++---- .../utils/saved_search_utils.ts | 2 +- 16 files changed, 55 insertions(+), 44 deletions(-) create mode 100644 src/plugins/discover/public/application/apps/main/components/view_mode_toggle/_index.scss create mode 100644 src/plugins/discover/public/application/apps/main/components/view_mode_toggle/_view_mode_toggle.scss diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/_index.scss b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/_index.scss new file mode 100644 index 0000000000000..a76c3453de32a --- /dev/null +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/_index.scss @@ -0,0 +1 @@ +@import 'view_mode_toggle'; diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/_view_mode_toggle.scss b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/_view_mode_toggle.scss new file mode 100644 index 0000000000000..8831e7a57dbfb --- /dev/null +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/_view_mode_toggle.scss @@ -0,0 +1,4 @@ + +.dscViewModeToggle { + padding-right: $euiSize; +} diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx index 3085edab0c56e..ac98cddf8fc71 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx @@ -10,6 +10,7 @@ import { EuiButtonGroup } from '@elastic/eui'; import React from 'react'; import { i18n } from '@kbn/i18n'; import { DISCOVER_VIEW_MODE } from './constants'; +import './_index.scss'; const toggleButtons = [ { @@ -31,8 +32,8 @@ export const DocumentViewModeToggle = ({ }) => { return ( ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/number_content_preview.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/number_content_preview.tsx index 08d2d42c6c027..45ba0d2c90130 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/number_content_preview.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/number_content_preview.tsx @@ -7,7 +7,7 @@ import React from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui'; import { FileBasedFieldVisConfig } from '../stats_table/types'; export const FileBasedNumberContentPreview = ({ config }: { config: FileBasedFieldVisConfig }) => { @@ -23,28 +23,34 @@ export const FileBasedNumberContentPreview = ({ config }: { config: FileBasedFie - + - + - + - + - + - + - {stats.min} - {stats.median} - {stats.max} + + {stats.min} + + + {stats.median} + + + {stats.max} + ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index 78afdebe446d3..b3a2a7d84b5a9 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -16,10 +16,6 @@ $panelWidthL: #{'max(40%, 450px)'}; font-weight: bold; padding-bottom: $euiSizeS; } - - .fieldDataCard__codeContent { - padding-left: 4px; - } } .dataVisualizer { @@ -56,8 +52,6 @@ $panelWidthL: #{'max(40%, 450px)'}; } .dataVisualizerSummaryTable { - //max-width: #{$panelWidthS - $euiSize}; - //min-width: #{$panelWidthS - $euiSize}; .euiTableRow > .euiTableRowCell { border-bottom: 0; } @@ -86,9 +80,10 @@ $panelWidthL: #{'max(40%, 450px)'}; } .dataVisualizerPanelWrapper { - margin: $euiSizeXS $euiSizeS $euiSizeS 0; + margin: $euiSizeXS $euiSizeM $euiSizeM 0; + height: fit-content; &.compressed { - max-width: $panelWidthS; + width: $panelWidthS; } } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx index 5084d36fdac02..96ecd33663500 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx @@ -6,7 +6,7 @@ */ import React, { FC, ReactNode, useMemo } from 'react'; -import { EuiBasicTable, EuiFlexItem, EuiSpacer } from '@elastic/eui'; +import { EuiBasicTable, EuiSpacer, RIGHT_ALIGNMENT, HorizontalAlignment } from '@elastic/eui'; import { Axis, BarSeries, Chart, Settings } from '@elastic/charts'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -69,10 +69,11 @@ export const BooleanContent: FC = ({ config }) => { ]; const summaryTableColumns = [ { + field: 'function', name: '', - render: (summaryItem: { display: ReactNode }) => summaryItem.display, + render: (_: string, summaryItem: { display: ReactNode }) => summaryItem.display, width: '25px', - align: 'right', + align: RIGHT_ALIGNMENT as HorizontalAlignment, }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx index 155cfd7c63905..df90d897d2d3d 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx @@ -6,12 +6,13 @@ */ import React, { FC, ReactNode } from 'react'; -import { EuiBasicTable } from '@elastic/eui'; +import { EuiBasicTable, HorizontalAlignment } from '@elastic/eui'; // @ts-ignore import { formatDate } from '@elastic/eui/lib/services/format'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; +import { RIGHT_ALIGNMENT } from '@elastic/eui'; import type { FieldDataRowProps } from '../../types/field_data_row'; import { ExpandedRowFieldHeader } from '../expanded_row_field_header'; import { DocumentStatsTable } from './document_stats'; @@ -61,9 +62,10 @@ export const DateContent: FC = ({ config }) => { const summaryTableColumns = [ { name: '', - render: (summaryItem: { display: ReactNode }) => summaryItem.display, - width: '80px', - align: 'right', + field: 'function', + render: (func: string, summaryItem: { display: ReactNode }) => summaryItem.display, + width: '70px', + align: RIGHT_ALIGNMENT as HorizontalAlignment, }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx index f2714c6bc61eb..fd43cfae06f05 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx @@ -8,7 +8,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import React, { FC, ReactNode } from 'react'; import { i18n } from '@kbn/i18n'; -import { EuiBasicTable } from '@elastic/eui'; +import { EuiBasicTable, HorizontalAlignment, RIGHT_ALIGNMENT } from '@elastic/eui'; import { ExpandedRowFieldHeader } from '../expanded_row_field_header'; import { FieldDataRowProps } from '../../types'; import { roundToDecimalPlace } from '../../../utils'; @@ -16,10 +16,11 @@ import { ExpandedRowPanel } from './expanded_row_panel'; const metaTableColumns = [ { + field: 'function', name: '', - render: (metaItem: { display: ReactNode }) => metaItem.display, + render: (_: string, metaItem: { display: ReactNode }) => metaItem.display, width: '25px', - align: 'right', + align: RIGHT_ALIGNMENT as HorizontalAlignment, }, { field: 'value', diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_panel.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_panel.tsx index 62c014ba7d417..b738dbdf67178 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/expanded_row_panel.tsx @@ -12,7 +12,6 @@ import { EuiFlexItemProps } from '@elastic/eui/src/components/flex/flex_item'; interface Props { children: ReactNode; dataTestSubj?: string; - // 'data-test-subj'?: string; grow?: EuiFlexItemProps['grow']; className?: string; } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx index d77f61ca5f52c..1baea4b3f2f7c 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx @@ -45,7 +45,6 @@ export const KeywordContent: FC = ({ config }) => { - {EMSSuggestion && stats && } ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx index 69990aada1d06..4e6e32a7ae285 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx @@ -6,7 +6,13 @@ */ import React, { FC, ReactNode, useEffect, useState } from 'react'; -import { EuiBasicTable, EuiFlexItem, EuiText } from '@elastic/eui'; +import { + EuiBasicTable, + EuiFlexItem, + EuiText, + HorizontalAlignment, + RIGHT_ALIGNMENT, +} from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; @@ -85,7 +91,7 @@ export const NumberContent: FC = ({ config }) => { name: '', render: (summaryItem: { display: ReactNode }) => summaryItem.display, width: '25px', - align: 'right', + align: RIGHT_ALIGNMENT as HorizontalAlignment, }, { field: 'value', @@ -123,7 +129,7 @@ export const NumberContent: FC = ({ config }) => { )} {distribution && ( diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss index bc2c05f407659..ea11841a85887 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss @@ -4,7 +4,6 @@ .dataGridChart__column-chart { width: 100%; - height: 12px; } .dataGridChart__legend { @@ -17,7 +16,7 @@ font-weight: normal; text-align: left; line-height: 1.1; - font-size: 10px; + font-size: #{$euiFontSizeL / 2}; // 10px } .dataGridChart__legend--numeric { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts index 040662c1eafff..0eeb3a3ae31d1 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts @@ -44,7 +44,7 @@ export const calculateTableColumnsDimensions = (width?: number) => { type: '75px', docCount: '225px', distinctValues: '225px', - distributions: '150px', + distributions: '225px', showIcon: true, breakPoint: 'xl', }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index 08322bf0d2c6e..72766e64b2e88 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -44,7 +44,7 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed const progressBarMax = isTopValuesSampled === true ? topValuesSampleSize : count; return ( @@ -56,11 +56,7 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed
{Array.isArray(topValues) && topValues.map((value) => ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index f3437df12ed62..2f182b1b0a615 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -16,7 +16,7 @@ import { Filter, } from '@kbn/es-query'; import { isSavedSearchSavedObject, SavedSearchSavedObject } from '../../../../common/types'; -import { IndexPattern } from '../../../../../../../src/plugins/data/common/index_patterns/index_patterns'; +import { IndexPattern } from '../../../../../../../src/plugins/data/common'; import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; import { SavedSearch } from '../../../../../../../src/plugins/discover/public'; import { getEsQueryConfig } from '../../../../../../../src/plugins/data/common'; From a9db7505e1c143e7139c63206fff396154d641d7 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 17 Sep 2021 10:55:59 -0500 Subject: [PATCH 088/188] Add search panel/kql filter bar --- .../multi_select_picker.tsx | 2 +- .../index_data_visualizer_view.tsx | 2 - .../components/search_panel/_index.scss | 1 + .../components/search_panel/search_panel.scss | 13 +++ .../components/search_panel/search_panel.tsx | 107 +++++++++--------- 5 files changed, 69 insertions(+), 56 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/_index.scss create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.scss diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/multi_select_picker/multi_select_picker.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/multi_select_picker/multi_select_picker.tsx index caa58009fda5d..ff4701e22953f 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/multi_select_picker/multi_select_picker.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/multi_select_picker/multi_select_picker.tsx @@ -98,7 +98,7 @@ export const MultiSelectPicker: FC<{ ); return ( - + = (dataVi /> )} - - = ({ setSearchParams, showEmptyFields, }) => { + const { + services: { + notifications: { toasts }, + data: { + ui: { SearchBar }, + }, + }, + } = useDataVisualizerKibana(); + // The internal state of the input query bar updated on every key stroke. const [searchInput, setSearchInput] = useState({ query: searchString || '', @@ -75,7 +84,8 @@ export const SearchPanel: FC = ({ }); }, [searchQueryLanguage, searchString]); - const searchHandler = (query: Query) => { + const searchHandler = (query?: Query) => { + if (!query) return; let filterQuery; try { if (query.language === SEARCH_QUERY_LANGUAGE.KUERY) { @@ -93,66 +103,57 @@ export const SearchPanel: FC = ({ } catch (e) { console.log('Invalid syntax', JSON.stringify(e, null, 2)); // eslint-disable-line no-console setErrorMessage({ query: query.query as string, message: e.message }); + toasts.addError(e, { + title: i18n.translate('xpack.dataVisualizer.searchPanel.invalidSyntax', { + defaultMessage: 'Invalid syntax', + }), + }); } }; - const searchChangeHandler = (query: Query) => setSearchInput(query); return ( - - - setErrorMessage(undefined)} - input={ - - } - isOpen={errorMessage?.query === searchInput.query && errorMessage?.message !== ''} - > - - {i18n.translate( - 'xpack.dataVisualizer.searchPanel.invalidKuerySyntaxErrorMessageQueryBar', - { - defaultMessage: 'Invalid query', - } - )} - {': '} - {errorMessage?.message.split('\n')[0]} - - + + + searchHandler(params.query)} + indexPatterns={[indexPattern]} + placeholder={i18n.translate('xpack.dataVisualizer.searchPanel.queryBarPlaceholderText', { + defaultMessage: 'Search… (e.g. status:200 AND extension:"PHP")', + })} + displayStyle={'inPage'} + isClearable={true} + /> - + + + + - - ); }; From b743a960ccf9ed71ed53ae2c1bd1400be237a0c0 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 20 Sep 2021 11:29:04 -0500 Subject: [PATCH 089/188] Temporarily fix scrolling --- .../apps/main/components/layout/discover_layout.scss | 4 ++++ src/plugins/discover/public/plugin.tsx | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss index 2401325dd76f2..d9d3bce046705 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss @@ -4,6 +4,10 @@ discover-app { flex-grow: 1; } +.dscAppWrapper { + overflow-y: hidden; +} + .dscPage { @include euiBreakpoint('m', 'l', 'xl') { @include kibanaFullBodyHeight(); diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index 4624fa79ca14b..7387f192105cb 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -345,6 +345,9 @@ export class DiscoverPlugin await depsStart.data.indexPatterns.clearCache(); const { renderApp } = await import('./application'); + + params.element.classList.add('dscAppWrapper'); + const unmount = renderApp(params.element); return () => { unlistenParentHistory(); From 3459f0212c9795e93a2f5bac7dde9747e251a1ed Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 20 Sep 2021 11:34:21 -0500 Subject: [PATCH 090/188] New kql filters for data visualizer --- .../index_data_visualizer_view.tsx | 28 ++++++++++++++-- .../components/search_panel/search_panel.tsx | 33 ++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 90185ea0e085f..628b5afd034de 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -29,6 +29,7 @@ import { UI_SETTINGS, Query, IndexPattern, + generateFilters, } from '../../../../../../../../src/plugins/data/public'; import { FullTimeRangeSelector } from '../full_time_range_selector'; import { usePageUrlState, useUrlState } from '../../../common/util/url_state'; @@ -69,6 +70,7 @@ import { extractSearchData } from '../../utils/saved_search_utils'; import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; import { ResultLink } from '../../../common/components/results_links'; import { extractErrorProperties } from '../../utils/error_utils'; +import { DataViewField } from '../../../../../../../../src/plugins/data/common'; interface DataVisualizerPageState { overallStats: OverallStats; @@ -131,7 +133,7 @@ const restorableDefaults = getDefaultDataVisualizerListState(); export const IndexDataVisualizerView: FC = (dataVisualizerProps) => { const { services } = useDataVisualizerKibana(); - const { docLinks, notifications, uiSettings } = services; + const { docLinks, notifications, uiSettings, data } = services; const { toasts } = notifications; const [dataVisualizerListState, setDataVisualizerListState] = usePageUrlState( @@ -307,6 +309,26 @@ export const IndexDataVisualizerView: FC = (dataVi const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); + const onAddFilter = useCallback( + (field: DataViewField | string, values: string, operation: '+' | '-') => { + // const fieldName = typeof field === 'string' ? field : field.name; + // popularizeField(indexPattern, fieldName, indexPatterns, capabilities); + const newFilters = generateFilters( + data.query.filterManager, + field, + values, + operation, + String(currentIndexPattern.id) + ); + // if (trackUiMetric) { + // trackUiMetric(METRIC_TYPE.CLICK, 'filter_added'); + // } + + return data.query.filterManager.addFilters(newFilters); + }, + [data.query] + ); + useEffect(() => { const timeUpdateSubscription = merge( timefilter.getTimeUpdate$(), @@ -753,13 +775,14 @@ export const IndexDataVisualizerView: FC = (dataVi item={item} indexPattern={currentIndexPattern} combinedQuery={{ searchQueryLanguage, searchString }} + onAddFilter={onAddFilter} /> ); } return m; }, {} as ItemIdToExpandedRowMap); }, - [currentIndexPattern, searchQueryLanguage, searchString] + [currentIndexPattern, searchQueryLanguage, searchString, onAddFilter] ); // Some actions open up fly-out or popup @@ -879,6 +902,7 @@ export const IndexDataVisualizerView: FC = (dataVi visibleFieldNames={visibleFieldNames} setVisibleFieldNames={setVisibleFieldNames} showEmptyFields={showEmptyFields} + onAddFilter={onAddFilter} /> void; } export const SearchPanel: FC = ({ @@ -63,6 +65,7 @@ export const SearchPanel: FC = ({ }) => { const { services: { + uiSettings, notifications: { toasts }, data: { ui: { SearchBar }, @@ -75,6 +78,9 @@ export const SearchPanel: FC = ({ query: searchString || '', language: searchQueryLanguage, }); + const [filters, setFilters] = useState([]); + // const [query, setQuery] = useState(); + const [errorMessage, setErrorMessage] = useState(undefined); useEffect(() => { @@ -85,6 +91,8 @@ export const SearchPanel: FC = ({ }, [searchQueryLanguage, searchString]); const searchHandler = (query?: Query) => { + console.log('searchHandler query', query); + if (!query) return; let filterQuery; try { @@ -111,6 +119,24 @@ export const SearchPanel: FC = ({ } }; + // useEffect(() => { + // console.log('useEffect filters', filters); + // const userQuery = query ?? searchInput; + // const combinedQuery = createCombinedQuery( + // userQuery, + // Array.isArray(filters) ? filters : [], + // indexPattern, + // uiSettings + // ); + // console.log('combinedQuery', combinedQuery); + // + // setSearchParams({ + // searchQuery: combinedQuery, + // searchString: userQuery.query, + // queryLanguage: userQuery.language as SearchQueryLanguage, + // }); + // }, [filters, query]); + return ( = ({ showQueryInput={true} query={searchInput} onQuerySubmit={(params) => searchHandler(params.query)} + // onQueryChange={(params) => console.log('params', params)} + onFiltersUpdated={(filters) => { + console.log('onFiltersUpdated', filters); + setFilters(filters); + }} indexPatterns={[indexPattern]} placeholder={i18n.translate('xpack.dataVisualizer.searchPanel.queryBarPlaceholderText', { defaultMessage: 'Search… (e.g. status:200 AND extension:"PHP")', From ed1d4e5ed8df7b69a2d529218300a3cb5806983f Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 20 Sep 2021 12:52:44 -0500 Subject: [PATCH 091/188] Set map height so it will fit the sampler shard size text --- .../geo_point_content/geo_point_content.tsx | 5 +- .../common/components/stats_table/_index.scss | 10 ++-- .../choropleth_map.tsx | 47 ++++++++++++------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content/geo_point_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content/geo_point_content.tsx index b732e542658b5..f3d053acd328f 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content/geo_point_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content/geo_point_content.tsx @@ -68,10 +68,7 @@ export const GeoPointContent: FC = ({ config }) => { )} {formattedResults && Array.isArray(formattedResults.layerList) && ( - + )} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index b3a2a7d84b5a9..c0fb26d602df6 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -65,11 +65,6 @@ $panelWidthL: #{'max(40%, 450px)'}; max-width: $panelWidthS; } - .dataVisualizerMapWrapper { - min-height: 250px; - min-width: $panelWidthL; - } - .dataVisualizerTopValuesWrapper { min-width: $panelWidthS; } @@ -87,6 +82,11 @@ $panelWidthL: #{'max(40%, 450px)'}; } } + .dataVisualizerMapWrapper { + height: $euiSize * 15; //240px + min-width: $panelWidthL; + } + .dataVisualizerTextContent { min-width: $panelWidthS; } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx index 634bebaf8e6d6..5eb37c2983c7d 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx @@ -106,23 +106,38 @@ export const ChoroplethMap: FC = ({ stats, suggestion }) => { return ( - - {isTopValuesSampled === true && ( - <> - - - - - - )} +
+ +
+
+ + + +
+ + {/* {isTopValuesSampled === true && (*/} + {/* <>*/} + {/* */} + {/* */} + {/* */} + {/* */} + {/* */} + {/* )}*/} ); }; From 22a35c0ad2ba75cc9efbb8f3aa819e78479c7f8c Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 20 Sep 2021 12:57:24 -0500 Subject: [PATCH 092/188] Use eui progress labels --- .../common/components/top_values/top_values.tsx | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index 72766e64b2e88..2c928b18a382e 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -69,18 +69,13 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed size="xs" label={kibanaFieldFormat(value.key, fieldFormat)} className={classNames('eui-textTruncate', 'topValuesValueLabelContainer')} + valueText={ + progressBarMax !== undefined + ? getPercentLabel(value.doc_count, progressBarMax) + : undefined + } /> - {progressBarMax !== undefined && ( - - - {getPercentLabel(value.doc_count, progressBarMax)} - - - )} ))} {isTopValuesSampled === true && ( From 71df07082720e15d4978efbe7b2bc667c85131da Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 20 Sep 2021 14:41:27 -0500 Subject: [PATCH 093/188] Fix spacer --- .../choropleth_map.tsx | 39 +++++++------------ 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx index 5eb37c2983c7d..bc5795a53f6e4 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx @@ -112,32 +112,21 @@ export const ChoroplethMap: FC = ({ stats, suggestion }) => {
-
- - - -
- {/* {isTopValuesSampled === true && (*/} - {/* <>*/} - {/* */} - {/* */} - {/* */} - {/* */} - {/* */} - {/* )}*/} + {isTopValuesSampled === true && ( +
+ + + + +
+ )} ); }; From c16cb8f477c95a9f3765e5ac9cbbba1d8812eec9 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 20 Sep 2021 15:16:44 -0500 Subject: [PATCH 094/188] Add beta badge --- .../view_mode_toggle/_view_mode_toggle.scss | 10 +++- .../view_mode_toggle/view_mode_toggle.tsx | 47 ++++++++++++++----- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/_view_mode_toggle.scss b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/_view_mode_toggle.scss index 8831e7a57dbfb..1009ab0511957 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/_view_mode_toggle.scss +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/_view_mode_toggle.scss @@ -1,4 +1,12 @@ - .dscViewModeToggle { padding-right: $euiSize; } + +.fieldStatsButton { + display: flex; + align-items: center; +} + +.fieldStatsBetaBadge { + margin-left: $euiSizeXS; +} diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx index ac98cddf8fc71..62dc8c7ed463b 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx @@ -6,23 +6,13 @@ * Side Public License, v 1. */ -import { EuiButtonGroup } from '@elastic/eui'; -import React from 'react'; +import { EuiButtonGroup, EuiBetaBadge } from '@elastic/eui'; +import React, { useMemo } from 'react'; import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; import { DISCOVER_VIEW_MODE } from './constants'; import './_index.scss'; -const toggleButtons = [ - { - id: DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, - label: 'Document view', - }, - { - id: DISCOVER_VIEW_MODE.FIELD_LEVEL, - label: 'Aggregated view', - }, -]; - export const DocumentViewModeToggle = ({ discoverViewMode, setDiscoverViewMode, @@ -30,6 +20,37 @@ export const DocumentViewModeToggle = ({ discoverViewMode: DISCOVER_VIEW_MODE; setDiscoverViewMode: (discoverViewMode: DISCOVER_VIEW_MODE) => void; }) => { + const toggleButtons = useMemo( + () => [ + { + id: DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, + label: i18n.translate('discover.viewModes.document.label', { + defaultMessage: 'Documents', + }), + }, + { + id: DISCOVER_VIEW_MODE.FIELD_LEVEL, + label: ( +
+ + +
+ ), + }, + ], + [] + ); + return ( Date: Mon, 20 Sep 2021 11:29:04 -0500 Subject: [PATCH 095/188] Temporarily fix scrolling --- .../apps/main/components/layout/discover_layout.scss | 4 ++++ src/plugins/discover/public/plugin.tsx | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss index 2401325dd76f2..d9d3bce046705 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss @@ -4,6 +4,10 @@ discover-app { flex-grow: 1; } +.dscAppWrapper { + overflow-y: hidden; +} + .dscPage { @include euiBreakpoint('m', 'l', 'xl') { @include kibanaFullBodyHeight(); diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index 0327b25fd864e..ff8a8cadfb069 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -346,6 +346,12 @@ export class DiscoverPlugin await depsStart.data.indexPatterns.clearCache(); const { renderApp } = await import('./application'); + + // FIXME: Temporarily hide overflow-y in Discover app when Field Stats table is shown + // due to EUI bug https://github.com/elastic/eui/pull/5152 + // until EUI is bumped to 38.0.0 + params.element.classList.add('dscAppWrapper'); + const unmount = renderApp(params.element); return () => { unlistenParentHistory(); From 00c3499b8f363b3b5b5d81acbea985cc059e8308 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 21 Sep 2021 09:33:55 -0500 Subject: [PATCH 096/188] Fix grow for Top Values for --- .../application/common/components/top_values/top_values.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index 91c5e0829bdd4..e51cc2367612a 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -41,6 +41,7 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed Date: Wed, 22 Sep 2021 09:52:37 -0500 Subject: [PATCH 097/188] [ML] Update functional tests to reflect new arrow icons --- .../test/functional/services/ml/data_visualizer_table.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/test/functional/services/ml/data_visualizer_table.ts b/x-pack/test/functional/services/ml/data_visualizer_table.ts index 2f67a9b75e3d6..7c0409f3ec27c 100644 --- a/x-pack/test/functional/services/ml/data_visualizer_table.ts +++ b/x-pack/test/functional/services/ml/data_visualizer_table.ts @@ -110,11 +110,11 @@ export function MachineLearningDataVisualizerTableProvider( if (!(await testSubjects.exists(this.detailsSelector(fieldName)))) { const selector = this.rowSelector( fieldName, - `dataVisualizerDetailsToggle-${fieldName}-arrowDown` + `dataVisualizerDetailsToggle-${fieldName}-arrowRight` ); await testSubjects.click(selector); await testSubjects.existOrFail( - this.rowSelector(fieldName, `dataVisualizerDetailsToggle-${fieldName}-arrowUp`), + this.rowSelector(fieldName, `dataVisualizerDetailsToggle-${fieldName}-arrowDown`), { timeout: 1000, } @@ -128,10 +128,10 @@ export function MachineLearningDataVisualizerTableProvider( await retry.tryForTime(10000, async () => { if (await testSubjects.exists(this.detailsSelector(fieldName))) { await testSubjects.click( - this.rowSelector(fieldName, `dataVisualizerDetailsToggle-${fieldName}-arrowUp`) + this.rowSelector(fieldName, `dataVisualizerDetailsToggle-${fieldName}-arrowDown`) ); await testSubjects.existOrFail( - this.rowSelector(fieldName, `dataVisualizerDetailsToggle-${fieldName}-arrowDown`), + this.rowSelector(fieldName, `dataVisualizerDetailsToggle-${fieldName}-arrowRight`), { timeout: 1000, } From cd9f68191529c68012ab557831a3aa1ab46b3487 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 22 Sep 2021 11:18:41 -0500 Subject: [PATCH 098/188] [ML] Add filter buttons and KQL bars --- .../keyword_content.tsx | 9 +- .../number_content.tsx | 10 +- .../components/top_values/top_values.tsx | 83 +++++++++++++- .../index_data_visualizer_view.tsx | 21 +++- .../components/search_panel/_index.scss | 1 + .../components/search_panel/search_panel.scss | 13 +++ .../components/search_panel/search_panel.tsx | 107 +++++++++--------- .../types/index_data_visualizer_state.ts | 2 +- 8 files changed, 182 insertions(+), 64 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/_index.scss create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.scss diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx index 1baea4b3f2f7c..2bae49323a6bb 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/keyword_content.tsx @@ -14,7 +14,7 @@ import { DocumentStatsTable } from './document_stats'; import { ExpandedRowContent } from './expanded_row_content'; import { ChoroplethMap } from './choropleth_map'; -export const KeywordContent: FC = ({ config }) => { +export const KeywordContent: FC = ({ config, onAddFilter }) => { const [EMSSuggestion, setEMSSuggestion] = useState(); const { stats, fieldName } = config; const fieldFormat = 'fieldFormat' in config ? config.fieldFormat : undefined; @@ -44,7 +44,12 @@ export const KeywordContent: FC = ({ config }) => { return ( - + {EMSSuggestion && stats && } ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx index 4e6e32a7ae285..4592a7ee6a60c 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/number_content.tsx @@ -38,7 +38,7 @@ interface SummaryTableItem { value: number | string | undefined | null; } -export const NumberContent: FC = ({ config }) => { +export const NumberContent: FC = ({ config, onAddFilter }) => { const { stats } = config; useEffect(() => { @@ -125,7 +125,13 @@ export const NumberContent: FC = ({ config }) => { {stats && ( - + )} {distribution && ( void; } function getPercentLabel(docCount: number, topValuesSampleSize: number): string { @@ -32,10 +42,17 @@ function getPercentLabel(docCount: number, topValuesSampleSize: number): string } } -export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed }) => { +export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed, onAddFilter }) => { if (stats === undefined) return null; - const { topValues, topValuesSampleSize, topValuesSamplerShardSize, count, isTopValuesSampled } = - stats; + const { + topValues, + topValuesSampleSize, + topValuesSamplerShardSize, + count, + isTopValuesSampled, + fieldName, + } = stats; + const progressBarMax = isTopValuesSampled === true ? topValuesSampleSize : count; return ( = ({ stats, fieldFormat, barColor, compressed } /> + {fieldName !== undefined && value.key !== undefined && onAddFilter !== undefined ? ( + <> + + onAddFilter( + fieldName, + typeof value.key === 'number' ? value.key.toString() : value.key, + '+' + ) + } + aria-label={i18n.translate( + 'xpack.dataVisualizer.dataGrid.field.addFilterAriaLabel', + { + defaultMessage: 'Filter for {fieldName}: "{value}"', + values: { fieldName, value: value.key }, + } + )} + data-test-subj={`dvFieldDataTopValuesAddFilterButton-${value.key}-${value.key}`} + style={{ + minHeight: 'auto', + minWidth: 'auto', + paddingRight: 2, + paddingLeft: 2, + paddingTop: 0, + paddingBottom: 0, + }} + /> + + onAddFilter( + fieldName, + typeof value.key === 'number' ? value.key.toString() : value.key, + '-' + ) + } + aria-label={i18n.translate( + 'xpack.dataVisualizer.dataGrid.field.removeFilterAriaLabel', + { + defaultMessage: 'Filter out {fieldName}: "{value}"', + values: { fieldName, value: value.key }, + } + )} + data-test-subj={`dvFieldDataTopValuesExcludeFilterButton-${value.key}-${value.key}`} + style={{ + minHeight: 'auto', + minWidth: 'auto', + paddingTop: 0, + paddingBottom: 0, + paddingRight: 2, + paddingLeft: 2, + }} + /> + + ) : null} ))} {isTopValuesSampled === true && ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 5abca1170e733..f235726f0ed85 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -27,6 +27,7 @@ import { KBN_FIELD_TYPES, UI_SETTINGS, Query, + generateFilters, } from '../../../../../../../../src/plugins/data/public'; import { FullTimeRangeSelector } from '../full_time_range_selector'; import { usePageUrlState, useUrlState } from '../../../common/util/url_state'; @@ -130,7 +131,7 @@ const restorableDefaults = getDefaultDataVisualizerListState(); export const IndexDataVisualizerView: FC = (dataVisualizerProps) => { const { services } = useDataVisualizerKibana(); - const { docLinks, notifications, uiSettings } = services; + const { docLinks, notifications, uiSettings, data } = services; const { toasts } = notifications; const [dataVisualizerListState, setDataVisualizerListState] = usePageUrlState( @@ -306,6 +307,20 @@ export const IndexDataVisualizerView: FC = (dataVi const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); + const onAddFilter = useCallback( + (field: DataViewField | string, values: string, operation: '+' | '-') => { + const newFilters = generateFilters( + data.query.filterManager, + field, + values, + operation, + String(currentIndexPattern.id) + ); + return data.query.filterManager.addFilters(newFilters); + }, + [currentIndexPattern.id, data.query.filterManager] + ); + useEffect(() => { const timeUpdateSubscription = merge( timefilter.getTimeUpdate$(), @@ -752,13 +767,14 @@ export const IndexDataVisualizerView: FC = (dataVi item={item} indexPattern={currentIndexPattern} combinedQuery={{ searchQueryLanguage, searchString }} + onAddFilter={onAddFilter} /> ); } return m; }, {} as ItemIdToExpandedRowMap); }, - [currentIndexPattern, searchQueryLanguage, searchString] + [currentIndexPattern, searchQueryLanguage, searchString, onAddFilter] ); // Some actions open up fly-out or popup @@ -878,6 +894,7 @@ export const IndexDataVisualizerView: FC = (dataVi visibleFieldNames={visibleFieldNames} setVisibleFieldNames={setVisibleFieldNames} showEmptyFields={showEmptyFields} + onAddFilter={onAddFilter} /> = ({ setSearchParams, showEmptyFields, }) => { + const { + services: { + notifications: { toasts }, + data: { + ui: { SearchBar }, + }, + }, + } = useDataVisualizerKibana(); + // The internal state of the input query bar updated on every key stroke. const [searchInput, setSearchInput] = useState({ query: searchString || '', @@ -75,7 +84,8 @@ export const SearchPanel: FC = ({ }); }, [searchQueryLanguage, searchString]); - const searchHandler = (query: Query) => { + const searchHandler = (query?: Query) => { + if (!query) return; let filterQuery; try { if (query.language === SEARCH_QUERY_LANGUAGE.KUERY) { @@ -93,66 +103,57 @@ export const SearchPanel: FC = ({ } catch (e) { console.log('Invalid syntax', JSON.stringify(e, null, 2)); // eslint-disable-line no-console setErrorMessage({ query: query.query as string, message: e.message }); + toasts.addError(e, { + title: i18n.translate('xpack.dataVisualizer.searchPanel.invalidSyntax', { + defaultMessage: 'Invalid syntax', + }), + }); } }; - const searchChangeHandler = (query: Query) => setSearchInput(query); return ( - - - setErrorMessage(undefined)} - input={ - - } - isOpen={errorMessage?.query === searchInput.query && errorMessage?.message !== ''} - > - - {i18n.translate( - 'xpack.dataVisualizer.searchPanel.invalidKuerySyntaxErrorMessageQueryBar', - { - defaultMessage: 'Invalid query', - } - )} - {': '} - {errorMessage?.message.split('\n')[0]} - - + + + searchHandler(params.query)} + indexPatterns={[indexPattern]} + placeholder={i18n.translate('xpack.dataVisualizer.searchPanel.queryBarPlaceholderText', { + defaultMessage: 'Search… (e.g. status:200 AND extension:"PHP")', + })} + displayStyle={'inPage'} + isClearable={true} + /> - + + + + - - ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts index 7cd1c2bb3ce09..b0f3fa4f409db 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { Query } from '../../../../../../../src/plugins/data/common/query'; +export { Query } from '@kbn/es-query'; import { SearchQueryLanguage } from './combined_query'; export interface ListingPageUrlState { From 5bfb9e24a73e72181f17568e00312fdbc4657c10 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 22 Sep 2021 13:09:56 -0500 Subject: [PATCH 099/188] [ML] Update filter bar onChange behavior --- .../components/search_panel/search_panel.tsx | 50 ++++++++++--------- .../utils/saved_search_utils.ts | 14 +++--- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx index 4aeb83e810103..c1a42728ef1b7 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx @@ -8,21 +8,18 @@ import React, { FC, useEffect, useState } from 'react'; import { EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { Query, fromKueryExpression, luceneStringToDsl, toElasticsearchQuery } from '@kbn/es-query'; +import { Query, Filter } from '@kbn/es-query'; import { ShardSizeFilter } from './shard_size_select'; import { DataVisualizerFieldNamesFilter } from './field_name_filter'; import { DatavisualizerFieldTypeFilter } from './field_type_filter'; -import { IndexPattern } from '../../../../../../../../src/plugins/data/common'; +import { DataView, TimeRange } from '../../../../../../../../src/plugins/data/common'; import { JobFieldType } from '../../../../../common/types'; -import { - ErrorMessage, - SEARCH_QUERY_LANGUAGE, - SearchQueryLanguage, -} from '../../types/combined_query'; +import { SearchQueryLanguage } from '../../types/combined_query'; import { useDataVisualizerKibana } from '../../../kibana_context'; import './_index.scss'; +import { createCombinedQuery } from '../../utils/saved_search_utils'; interface Props { - indexPattern: IndexPattern; + indexPattern: DataView; searchString: Query['query']; searchQuery: Query['query']; searchQueryLanguage: SearchQueryLanguage; @@ -63,8 +60,10 @@ export const SearchPanel: FC = ({ }) => { const { services: { + uiSettings, notifications: { toasts }, data: { + query: queryManager, ui: { SearchBar }, }, }, @@ -75,7 +74,6 @@ export const SearchPanel: FC = ({ query: searchString || '', language: searchQueryLanguage, }); - const [errorMessage, setErrorMessage] = useState(undefined); useEffect(() => { setSearchInput({ @@ -84,25 +82,27 @@ export const SearchPanel: FC = ({ }); }, [searchQueryLanguage, searchString]); - const searchHandler = (query?: Query) => { - if (!query) return; - let filterQuery; + const searchHandler = ({ query, filters }: { query?: Query; filters?: Filter[] }) => { + const mergedQuery = query ?? searchInput; try { - if (query.language === SEARCH_QUERY_LANGUAGE.KUERY) { - filterQuery = toElasticsearchQuery(fromKueryExpression(query.query), indexPattern); - } else if (query.language === SEARCH_QUERY_LANGUAGE.LUCENE) { - filterQuery = luceneStringToDsl(query.query); - } else { - filterQuery = {}; + if (filters) { + queryManager.filterManager.setFilters(filters); } + + const combinedQuery = createCombinedQuery( + mergedQuery, + queryManager.filterManager.getFilters() ?? [], + indexPattern, + uiSettings + ); + setSearchParams({ - searchQuery: filterQuery, - searchString: query.query, - queryLanguage: query.language as SearchQueryLanguage, + searchQuery: combinedQuery, + searchString: mergedQuery.query, + queryLanguage: mergedQuery.language as SearchQueryLanguage, }); } catch (e) { console.log('Invalid syntax', JSON.stringify(e, null, 2)); // eslint-disable-line no-console - setErrorMessage({ query: query.query as string, message: e.message }); toasts.addError(e, { title: i18n.translate('xpack.dataVisualizer.searchPanel.invalidSyntax', { defaultMessage: 'Invalid syntax', @@ -126,7 +126,11 @@ export const SearchPanel: FC = ({ showDatePicker={false} showQueryInput={true} query={searchInput} - onQuerySubmit={(params) => searchHandler(params.query)} + onQuerySubmit={(params: { dateRange: TimeRange; query?: Query | undefined }) => + searchHandler({ query: params.query }) + } + // @ts-expect-error onFiltersUpdated is a valid prop on SearchBar + onFiltersUpdated={(filters: Filter[]) => searchHandler({ filters })} indexPatterns={[indexPattern]} placeholder={i18n.translate('xpack.dataVisualizer.searchPanel.queryBarPlaceholderText', { defaultMessage: 'Search… (e.g. status:200 AND extension:"PHP")', diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index 3f6992ef82a3f..bda9550427e69 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -36,14 +36,14 @@ export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject | Sa } export function createCombinedQuery( - query: Query, - filters: Filter[], + query?: Query, + filters?: Filter[], indexPattern?: DataView, uiSettings?: IUiSettingsClient ) { - let combinedQuery: any = getDefaultDatafeedQuery(); + let combinedQuery: any = getDefaultQuery(); - if (query.language === SEARCH_QUERY_LANGUAGE.KUERY) { + if (query && query.language === SEARCH_QUERY_LANGUAGE.KUERY) { const ast = fromKueryExpression(query.query); if (query.query !== '') { combinedQuery = toElasticsearchQuery(ast, indexPattern); @@ -65,8 +65,8 @@ export function createCombinedQuery( } else { combinedQuery = buildEsQuery( indexPattern, - [query], - filters, + query ? [query] : [], + filters ? filters : [], uiSettings ? getEsQueryConfig(uiSettings) : undefined ); } @@ -142,6 +142,6 @@ const DEFAULT_QUERY = { }, }; -export function getDefaultDatafeedQuery() { +export function getDefaultQuery() { return cloneDeep(DEFAULT_QUERY); } From 16298ab5f6df983229d1a6dafbbc9330ca7618a1 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 22 Sep 2021 13:33:01 -0500 Subject: [PATCH 100/188] [ML] Update top values filter onChange behavior --- .../index_data_visualizer_view.tsx | 74 ++++++++++++++----- .../components/search_panel/search_panel.tsx | 16 +++- 2 files changed, 67 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index f235726f0ed85..1aebe8acfc87a 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -64,7 +64,7 @@ import { DatePickerWrapper } from '../../../common/components/date_picker_wrappe import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; import { HelpMenu } from '../../../common/components/help_menu'; import { TimeBuckets } from '../../services/time_buckets'; -import { extractSearchData } from '../../utils/saved_search_utils'; +import { createCombinedQuery, extractSearchData } from '../../utils/saved_search_utils'; import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; import { ResultLink } from '../../../common/components/results_links'; import { extractErrorProperties } from '../../utils/error_utils'; @@ -251,24 +251,27 @@ export const IndexDataVisualizerView: FC = (dataVi // eslint-disable-next-line react-hooks/exhaustive-deps }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState]); - const setSearchParams = (searchParams: { - searchQuery: Query['query']; - searchString: Query['query']; - queryLanguage: SearchQueryLanguage; - }) => { - // When the user loads saved search and then clear or modify the query - // we should remove the saved search and replace it with the index pattern id - if (currentSavedSearch !== null) { - setCurrentSavedSearch(null); - } + const setSearchParams = useCallback( + (searchParams: { + searchQuery: Query['query']; + searchString: Query['query']; + queryLanguage: SearchQueryLanguage; + }) => { + // When the user loads saved search and then clear or modify the query + // we should remove the saved search and replace it with the index pattern id + if (currentSavedSearch !== null) { + setCurrentSavedSearch(null); + } - setDataVisualizerListState({ - ...dataVisualizerListState, - searchQuery: searchParams.searchQuery, - searchString: searchParams.searchString, - searchQueryLanguage: searchParams.queryLanguage, - }); - }; + setDataVisualizerListState({ + ...dataVisualizerListState, + searchQuery: searchParams.searchQuery, + searchString: searchParams.searchString, + searchQueryLanguage: searchParams.queryLanguage, + }); + }, + [currentSavedSearch, dataVisualizerListState, setDataVisualizerListState] + ); const samplerShardSize = dataVisualizerListState.samplerShardSize ?? restorableDefaults.samplerShardSize; @@ -316,9 +319,40 @@ export const IndexDataVisualizerView: FC = (dataVi operation, String(currentIndexPattern.id) ); - return data.query.filterManager.addFilters(newFilters); + if (newFilters) { + data.query.filterManager.addFilters(newFilters); + } + + // Merge current query with new filters + const mergedQuery = { + query: searchString || '', + language: searchQueryLanguage, + }; + + const combinedQuery = createCombinedQuery( + { + query: searchString || '', + language: searchQueryLanguage, + }, + data.query.filterManager.getFilters() ?? [], + currentIndexPattern, + uiSettings + ); + + setSearchParams({ + searchQuery: combinedQuery, + searchString: mergedQuery.query, + queryLanguage: mergedQuery.language as SearchQueryLanguage, + }); }, - [currentIndexPattern.id, data.query.filterManager] + [ + currentIndexPattern, + data.query.filterManager, + searchQueryLanguage, + searchString, + setSearchParams, + uiSettings, + ] ); useEffect(() => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx index c1a42728ef1b7..d3eed5b41d563 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx @@ -12,7 +12,11 @@ import { Query, Filter } from '@kbn/es-query'; import { ShardSizeFilter } from './shard_size_select'; import { DataVisualizerFieldNamesFilter } from './field_name_filter'; import { DatavisualizerFieldTypeFilter } from './field_type_filter'; -import { DataView, TimeRange } from '../../../../../../../../src/plugins/data/common'; +import { + DataView, + DataViewField, + TimeRange, +} from '../../../../../../../../src/plugins/data/common'; import { JobFieldType } from '../../../../../common/types'; import { SearchQueryLanguage } from '../../types/combined_query'; import { useDataVisualizerKibana } from '../../../kibana_context'; @@ -41,6 +45,7 @@ interface Props { queryLanguage: SearchQueryLanguage; }): void; showEmptyFields: boolean; + onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void; } export const SearchPanel: FC = ({ @@ -68,7 +73,6 @@ export const SearchPanel: FC = ({ }, }, } = useDataVisualizerKibana(); - // The internal state of the input query bar updated on every key stroke. const [searchInput, setSearchInput] = useState({ query: searchString || '', @@ -80,7 +84,12 @@ export const SearchPanel: FC = ({ query: searchString || '', language: searchQueryLanguage, }); - }, [searchQueryLanguage, searchString]); + + return () => { + // Reset previously added filters when moving away from the page + queryManager.filterManager.removeAll(); + }; + }, [searchQueryLanguage, searchString, queryManager.filterManager]); const searchHandler = ({ query, filters }: { query?: Query; filters?: Filter[] }) => { const mergedQuery = query ?? searchInput; @@ -129,6 +138,7 @@ export const SearchPanel: FC = ({ onQuerySubmit={(params: { dateRange: TimeRange; query?: Query | undefined }) => searchHandler({ query: params.query }) } + // filters={queryManager.filterManager.getFilters()} // @ts-expect-error onFiltersUpdated is a valid prop on SearchBar onFiltersUpdated={(filters: Filter[]) => searchHandler({ filters })} indexPatterns={[indexPattern]} From 0a412da520953cfb7af47ffccdc1d604c38203dd Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 22 Sep 2021 13:41:41 -0500 Subject: [PATCH 101/188] [ML] Update search filters when opening saved search --- .../index_data_visualizer_view/index_data_visualizer_view.tsx | 3 ++- .../index_data_visualizer/utils/saved_search_utils.ts | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 1aebe8acfc87a..7d2f4c85a2bc0 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -233,6 +233,7 @@ export const IndexDataVisualizerView: FC = (dataVi indexPattern: currentIndexPattern, uiSettings, savedSearch: currentSavedSearch, + filterManager: data.query.filterManager, }); if (searchData === undefined || dataVisualizerListState.searchString !== '') { @@ -249,7 +250,7 @@ export const IndexDataVisualizerView: FC = (dataVi }; } // eslint-disable-next-line react-hooks/exhaustive-deps - }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState]); + }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState, data.query]); const setSearchParams = useCallback( (searchParams: { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index bda9550427e69..3aa0d29f4ecdf 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -20,6 +20,7 @@ import { DataView } from '../../../../../../../src/plugins/data/common'; import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; import { SavedSearch } from '../../../../../../../src/plugins/discover/public'; import { getEsQueryConfig } from '../../../../../../../src/plugins/data/common'; +import { FilterManager } from '../../../../../../../src/plugins/data/public'; export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject | SavedSearch) { const search = isSavedSearchSavedObject(savedSearch) @@ -83,12 +84,14 @@ export function extractSearchData({ savedSearch, query, filters, + filterManager, }: { indexPattern: DataView; uiSettings: IUiSettingsClient; savedSearch: SavedSearchSavedObject | SavedSearch | null | undefined; query?: Query; filters?: Filter[]; + filterManager: FilterManager; }) { if (!indexPattern || !savedSearch) return; @@ -116,6 +119,7 @@ export function extractSearchData({ if (savedSearchData) { const currentQuery = userQuery ?? savedSearchData?.query; const currentFilters = userFilters ?? savedSearchData?.filter; + filterManager.setFilters(currentFilters); const combinedQuery = createCombinedQuery( currentQuery, From a956d89cf4f55cb5e294b18003ba85128952696a Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 22 Sep 2021 13:52:27 -0500 Subject: [PATCH 102/188] [ML] Clean up --- .../components/search_panel/search_panel.tsx | 2 +- .../types/index_data_visualizer_state.ts | 2 +- .../index_data_visualizer/utils/saved_search_utils.ts | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx index d3eed5b41d563..1ea09817a9db1 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx @@ -138,7 +138,6 @@ export const SearchPanel: FC = ({ onQuerySubmit={(params: { dateRange: TimeRange; query?: Query | undefined }) => searchHandler({ query: params.query }) } - // filters={queryManager.filterManager.getFilters()} // @ts-expect-error onFiltersUpdated is a valid prop on SearchBar onFiltersUpdated={(filters: Filter[]) => searchHandler({ filters })} indexPatterns={[indexPattern]} @@ -147,6 +146,7 @@ export const SearchPanel: FC = ({ })} displayStyle={'inPage'} isClearable={true} + customSubmitButton={
} /> diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts index b0f3fa4f409db..7cd1c2bb3ce09 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts @@ -5,7 +5,7 @@ * 2.0. */ -export { Query } from '@kbn/es-query'; +import { Query } from '../../../../../../../src/plugins/data/common/query'; import { SearchQueryLanguage } from './combined_query'; export interface ListingPageUrlState { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index 3aa0d29f4ecdf..01cd8a766b409 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -91,7 +91,7 @@ export function extractSearchData({ savedSearch: SavedSearchSavedObject | SavedSearch | null | undefined; query?: Query; filters?: Filter[]; - filterManager: FilterManager; + filterManager?: FilterManager; }) { if (!indexPattern || !savedSearch) return; @@ -119,7 +119,8 @@ export function extractSearchData({ if (savedSearchData) { const currentQuery = userQuery ?? savedSearchData?.query; const currentFilters = userFilters ?? savedSearchData?.filter; - filterManager.setFilters(currentFilters); + + if (filterManager) filterManager.setFilters(currentFilters); const combinedQuery = createCombinedQuery( currentQuery, From cf25f85d77dc8a2f60d3a96278384b6faffa8bb2 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 22 Sep 2021 13:59:27 -0500 Subject: [PATCH 103/188] [ML] Remove fit content for height --- .../public/application/common/components/stats_table/_index.scss | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index 5a468155cc4b8..83357eecc9a89 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -76,7 +76,6 @@ $panelWidthL: #{'max(40%, 450px)'}; .dataVisualizerPanelWrapper { margin: $euiSizeXS $euiSizeM $euiSizeM 0; - height: fit-content; &.compressed { width: $panelWidthS; min-width: fit-content; From f78c0e0cdf6322a0806572a22e2297346665e61e Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 22 Sep 2021 14:51:12 -0500 Subject: [PATCH 104/188] [ML] Fix boolean legend --- .../components/field_data_row/column_chart.scss | 4 +--- .../components/field_data_row/use_column_chart.tsx | 14 +++++++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss index ea11841a85887..8a0b9cc992c3e 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/column_chart.scss @@ -24,9 +24,7 @@ } .dataGridChart__legendBoolean { - width: 100%; - min-width: $euiButtonMinWidth; - td { text-align: center } + width: #{$euiSizeXS * 2.5} // 10px } /* Override to align column header to bottom of cell when no chart is available */ diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.tsx index 0055ad9a7f0e2..2c0817228655e 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.tsx @@ -94,11 +94,19 @@ export const getLegendText = ( if (chartData.type === 'boolean') { return ( - +
- {chartData.data[0] !== undefined && } - {chartData.data[1] !== undefined && } + {chartData.data[0] !== undefined && ( + + )} + {chartData.data[1] !== undefined && ( + + )}
{chartData.data[0].key_as_string}{chartData.data[1].key_as_string} + {chartData.data[0].key_as_string?.slice(0, 1) ?? ''} + + {chartData.data[1].key_as_string?.slice(0, 1) ?? ''} +
From 436d849341a791a6adce7d712ddd27b3fcc43022 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 22 Sep 2021 15:44:46 -0500 Subject: [PATCH 105/188] [ML] Fix header section when browser width is small to large and when index pattern title is too large --- .../field_count_panel/field_count_panel.tsx | 5 +- .../common/components/stats_table/_index.scss | 107 +++++++++--------- .../components/field_count_stats/_index.scss | 9 ++ .../field_count_stats/metric_fields_count.tsx | 1 + .../field_count_stats/total_fields_count.tsx | 1 + .../field_data_row/number_content_preview.tsx | 6 +- .../data_visualizer_stats_table.tsx | 6 +- .../components/top_values/top_values.tsx | 2 +- .../index_data_visualizer_view/_index.scss | 1 + .../_index_data_visualizer_view.scss | 13 +++ .../index_data_visualizer_view.tsx | 49 ++++---- .../components/search_panel/search_panel.scss | 7 ++ .../components/search_panel/search_panel.tsx | 1 + 13 files changed, 122 insertions(+), 86 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/_index.scss create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/_index_data_visualizer_view.scss diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_count_panel/field_count_panel.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_count_panel/field_count_panel.tsx index c79ed4ade7092..978e59d0cf957 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_count_panel/field_count_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_count_panel/field_count_panel.tsx @@ -28,12 +28,13 @@ export const FieldCountPanel: FC = ({ - + .euiTableRowCell { - border-bottom: 0; - border-top: $euiBorderThin; + .euiTableRow > .euiTableRowCell { + border-bottom: 0; + border-top: $euiBorderThin; - } + } - .euiTableCellContent { - padding: 4px; - } + .euiTableCellContent { + padding: 4px; + } - .euiTableRow-isExpandedRow { + .euiTableRow-isExpandedRow { - .euiTableRowCell { - background-color: $euiColorEmptyShade !important; - border-top: 0; - border-bottom: $euiBorderThin; - &:hover { + .euiTableRowCell { background-color: $euiColorEmptyShade !important; + border-top: 0; + border-bottom: $euiBorderThin; + &:hover { + background-color: $euiColorEmptyShade !important; + } } } - } - .dataVisualizerSummaryTable { - .euiTableRow > .euiTableRowCell { - border-bottom: 0; - } - .euiTableHeaderCell { - display: none; + .dataVisualizerSummaryTable { + .euiTableRow > .euiTableRowCell { + border-bottom: 0; + } + .euiTableHeaderCell { + display: none; + } } - } - .dataVisualizerSummaryTableWrapper { - min-width: $panelWidthS; - max-width: $panelWidthS; - } + .dataVisualizerSummaryTableWrapper { + min-width: $panelWidthS; + max-width: $panelWidthS; + } - .dataVisualizerTopValuesWrapper { - min-width: $panelWidthS; - } + .dataVisualizerTopValuesWrapper { + min-width: fit-content; + } - .dataVisualizerUniformPanel { - min-width: $panelWidthS; - max-width: $panelWidthS; - } + .dataVisualizerUniformPanel { + min-width: $panelWidthS; + max-width: $panelWidthS; + } - .dataVisualizerPanelWrapper { - margin: $euiSizeXS $euiSizeM $euiSizeM 0; - &.compressed { - width: $panelWidthS; - min-width: fit-content; + .dataVisualizerPanelWrapper { + margin: $euiSizeXS $euiSizeM $euiSizeM 0; + &.compressed { + width: $panelWidthS; + min-width: fit-content; + } } - } - .dataVisualizerMapWrapper { - height: $euiSize * 15; //240px - min-width: $panelWidthL; - } + .dataVisualizerMapWrapper { + height: $euiSize * 15; //240px + min-width: $panelWidthL; + } - .dataVisualizerTextContent { - min-width: $panelWidthS; + .dataVisualizerTextContent { + min-width: $panelWidthS; + } } - } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/_index.scss index e44082c90ba32..9e26d79e1f685 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/_index.scss @@ -1,3 +1,12 @@ +.dataVisualizerFieldCountPanel { + margin-left: $euiSizeXS; + @include euiBreakpoint('xs', 's') { + flex-direction: column; + align-items: flex-start; + } +} + .dataVisualizerFieldCountContainer { max-width: 300px; + min-width: 300px; } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/metric_fields_count.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/metric_fields_count.tsx index 7996e6366c497..351b862a48ec0 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/metric_fields_count.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/metric_fields_count.tsx @@ -32,6 +32,7 @@ export const MetricFieldsCount: FC = ({ metricsStats }) alignItems="center" className="dataVisualizerFieldCountContainer" data-test-subj="dataVisualizerMetricFieldsSummary" + responsive={false} > diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/total_fields_count.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/total_fields_count.tsx index 8e9e3e59f1281..62f2aaf0b4528 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/total_fields_count.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/total_fields_count.tsx @@ -32,6 +32,7 @@ export const TotalFieldsCount: FC = ({ fieldsCountStats } alignItems="center" className="dataVisualizerFieldCountContainer" data-test-subj="dataVisualizerFieldsSummary" + responsive={false} > diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/number_content_preview.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/number_content_preview.tsx index 58d51439e6035..dd8685fdb9380 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/number_content_preview.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/number_content_preview.tsx @@ -59,7 +59,11 @@ export const IndexBasedNumberContentPreview: FC = ({
{legendText && ( <> - + {kibanaFieldFormat(legendText.min, fieldFormat)} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 135eea68ed067..59dd90cfceb9f 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -111,7 +111,7 @@ export const DataVisualizerTable = ({ const columns = useMemo(() => { const expanderColumn: EuiTableComputedColumnType = { - name: ( + name: dimensions.showIcon ? ( ({ } iconType={expandAll ? 'arrowDown' : 'arrowRight'} /> - ), + ) : null, align: RIGHT_ALIGNMENT, width: dimensions.expander, isExpander: true, @@ -305,7 +305,7 @@ export const DataVisualizerTable = ({ {(resizeRef) => (
- className={'dataVisualizer'} + className={'dataVisualizerTable'} items={items} itemId={FIELD_NAME} columns={columns} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index a8c3ec8fdec0e..031016acc36ca 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -18,11 +18,11 @@ import { import { FormattedMessage } from '@kbn/i18n/react'; import classNames from 'classnames'; +import { i18n } from '@kbn/i18n'; import { roundToDecimalPlace, kibanaFieldFormat } from '../utils'; import { ExpandedRowFieldHeader } from '../stats_table/components/expanded_row_field_header'; import { FieldVisStats } from '../../../../../common/types'; import { ExpandedRowPanel } from '../stats_table/components/field_data_expanded_row/expanded_row_panel'; -import { i18n } from '../../../../../../../../../../../../../private/var/tmp/_bazel_quynhnguyen/bd5cc7ce3740c1abb2c63a2609d8bb9f/execroot/kibana/bazel-out/darwin-fastbuild/bin/packages/kbn-i18n'; import { DataViewField } from '../../../../../../../../src/plugins/data/common/data_views/fields'; interface Props { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/_index.scss b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/_index.scss new file mode 100644 index 0000000000000..c9b1d78320aee --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/_index.scss @@ -0,0 +1 @@ +@import 'index_data_visualizer_view'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/_index_data_visualizer_view.scss b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/_index_data_visualizer_view.scss new file mode 100644 index 0000000000000..f49cb73454178 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/_index_data_visualizer_view.scss @@ -0,0 +1,13 @@ +.dataViewTitleHeader { + min-width: 300px; + display: flex; + flex-direction: row; + align-items: center; +} + +@include euiBreakpoint('xs', 's', 'm', 'l') { + .dataVisualizerPageHeader { + flex-direction: column; + align-items: flex-start; + } +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 7d2f4c85a2bc0..a0d6c16923bfd 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -69,6 +69,7 @@ import { DataVisualizerIndexPatternManagement } from '../index_pattern_managemen import { ResultLink } from '../../../common/components/results_links'; import { extractErrorProperties } from '../../utils/error_utils'; import { DataViewField, DataView } from '../../../../../../../../src/plugins/data/common'; +import './_index.scss'; interface DataVisualizerPageState { overallStats: OverallStats; @@ -861,17 +862,10 @@ export const IndexDataVisualizerView: FC = (dataVi - + -
- +
+

{currentIndexPattern.title}

 = (dataVi
- - - {currentIndexPattern.timeFieldName !== undefined && ( - - - - )} + + {currentIndexPattern.timeFieldName !== undefined && ( - + - - + )} + + + + @@ -931,7 +928,7 @@ export const IndexDataVisualizerView: FC = (dataVi showEmptyFields={showEmptyFields} onAddFilter={onAddFilter} /> - + = ({ alignItems="flexStart" data-test-subj="dataVisualizerSearchPanel" className={'dvSearchPanel'} + responsive={false} > Date: Wed, 22 Sep 2021 15:50:15 -0500 Subject: [PATCH 106/188] [ML] Hide expander icon when dimension is xs or s & css fixes --- .../common/components/stats_table/_index.scss | 6 +-- .../data_visualizer_stats_table.tsx | 37 ++++++++++--------- .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - 4 files changed, 22 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index 86f189bbcbc48..8e120140e0e0d 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -7,7 +7,9 @@ $panelWidthL: #{'max(40%, 450px)'}; .dataVisualizerFieldExpandedRow { padding-left: $euiSize * 4; - width: 100%; + // Width is subtracting the left padding + // to ensure the content wraps correctly + width: calc(100% - #{$euiSize * 2}); .fieldDataCard__valuesTitle { text-transform: uppercase; @@ -79,13 +81,11 @@ $panelWidthL: #{'max(40%, 450px)'}; margin: $euiSizeXS $euiSizeM $euiSizeM 0; &.compressed { width: $panelWidthS; - min-width: fit-content; } } .dataVisualizerMapWrapper { height: $euiSize * 15; //240px - min-width: $panelWidthL; } .dataVisualizerTextContent { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 59dd90cfceb9f..499ee74adab2b 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -111,24 +111,25 @@ export const DataVisualizerTable = ({ const columns = useMemo(() => { const expanderColumn: EuiTableComputedColumnType = { - name: dimensions.showIcon ? ( - toggleExpandAll(!expandAll)} - aria-label={ - !expandAll - ? i18n.translate('xpack.dataVisualizer.dataGrid.expandDetailsForAllAriaLabel', { - defaultMessage: 'Expand details for all fields', - }) - : i18n.translate('xpack.dataVisualizer.dataGrid.collapseDetailsForAllAriaLabel', { - defaultMessage: 'Collapse details for all fields', - }) - } - iconType={expandAll ? 'arrowDown' : 'arrowRight'} - /> - ) : null, + name: + dimensions.breakPoint !== 'xs' && dimensions.breakPoint !== 's' ? ( + toggleExpandAll(!expandAll)} + aria-label={ + !expandAll + ? i18n.translate('xpack.dataVisualizer.dataGrid.expandDetailsForAllAriaLabel', { + defaultMessage: 'Expand details for all fields', + }) + : i18n.translate('xpack.dataVisualizer.dataGrid.collapseDetailsForAllAriaLabel', { + defaultMessage: 'Collapse details for all fields', + }) + } + iconType={expandAll ? 'arrowDown' : 'arrowRight'} + /> + ) : null, align: RIGHT_ALIGNMENT, width: dimensions.expander, isExpander: true, diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 38c957292ec7d..d51eab5d1be67 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -9064,7 +9064,6 @@ "xpack.dataVisualizer.removeCombinedFieldsLabel": "結合されたフィールドを削除", "xpack.dataVisualizer.searchPanel.allFieldsLabel": "すべてのフィールド", "xpack.dataVisualizer.searchPanel.allOptionLabel": "すべて検索", - "xpack.dataVisualizer.searchPanel.invalidKuerySyntaxErrorMessageQueryBar": "無効なクエリ", "xpack.dataVisualizer.searchPanel.numberFieldsLabel": "数値フィールド", "xpack.dataVisualizer.searchPanel.ofFieldsTotal": "合計 {totalCount}", "xpack.dataVisualizer.searchPanel.queryBarPlaceholder": "小さいサンプルサイズを選択することで、クエリの実行時間を短縮しクラスターへの負荷を軽減できます。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 9f046ffc53cc0..7c8a85fc30eb0 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -9151,7 +9151,6 @@ "xpack.dataVisualizer.removeCombinedFieldsLabel": "移除组合字段", "xpack.dataVisualizer.searchPanel.allFieldsLabel": "所有字段", "xpack.dataVisualizer.searchPanel.allOptionLabel": "搜索全部", - "xpack.dataVisualizer.searchPanel.invalidKuerySyntaxErrorMessageQueryBar": "无效查询", "xpack.dataVisualizer.searchPanel.numberFieldsLabel": "字段数目", "xpack.dataVisualizer.searchPanel.ofFieldsTotal": "，共 {totalCount} 个", "xpack.dataVisualizer.searchPanel.queryBarPlaceholder": "选择较小的样例大小将减少查询运行时间和集群上的负载。", From 06c2d40aa8e53433ad8a95244aaaf2f6341b7858 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 22 Sep 2021 16:31:32 -0500 Subject: [PATCH 107/188] [ML] Delete embeddables because they are not use --- .../embeddables/grid_embeddable/constants.ts | 8 - .../embeddable_loading_fallback.tsx | 20 - .../grid_embeddable/grid_embeddable.tsx | 758 ------------------ .../grid_embeddable_factory.tsx | 67 -- .../embeddables/grid_embeddable/index.ts | 8 - .../embeddables/index.ts | 24 - .../plugins/data_visualizer/public/plugin.ts | 7 +- 7 files changed, 1 insertion(+), 891 deletions(-) delete mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/constants.ts delete mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_loading_fallback.tsx delete mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx delete mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx delete mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts delete mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/constants.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/constants.ts deleted file mode 100644 index 26004db8fd529..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/constants.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE = 'data_visualizer_grid'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_loading_fallback.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_loading_fallback.tsx deleted file mode 100644 index 01644efd6652c..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_loading_fallback.tsx +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; - -import { EuiLoadingSpinner, EuiSpacer, EuiText } from '@elastic/eui'; - -export const EmbeddableLoading = () => { - return ( - - - - - - ); -}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx deleted file mode 100644 index e7e1956acc0ae..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ /dev/null @@ -1,758 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { merge, Observable, Subject } from 'rxjs'; -import { CoreStart } from 'kibana/public'; -import ReactDOM from 'react-dom'; -import React, { Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import useObservable from 'react-use/lib/useObservable'; -import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { Filter } from '@kbn/es-query'; -import { Required } from 'utility-types'; -import { - Embeddable, - EmbeddableInput, - EmbeddableOutput, - IContainer, -} from '../../../../../../../../src/plugins/embeddable/public'; -import { KibanaContextProvider } from '../../../../../../../../src/plugins/kibana_react/public'; -import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; -import { EmbeddableLoading } from './embeddable_loading_fallback'; -import { DataVisualizerStartDependencies } from '../../../../plugin'; -import { - DataView, - IndexPatternField, - KBN_FIELD_TYPES, - Query, - UI_SETTINGS, -} from '../../../../../../../../src/plugins/data/common'; -import { SavedSearch } from '../../../../../../../../src/plugins/discover/public'; -import { - DataVisualizerTable, - ItemIdToExpandedRowMap, -} from '../../../common/components/stats_table'; -import { FieldVisConfig } from '../../../common/components/stats_table/types'; -import { MetricFieldsStats } from '../../../common/components/stats_table/components/field_count_stats'; -import { - getDefaultDataVisualizerListState, - getDefaultPageState, -} from '../../components/index_data_visualizer_view/index_data_visualizer_view'; - -import { extractSearchData } from '../../utils/saved_search_utils'; -import { useDataVisualizerKibana } from '../../../kibana_context'; -import { extractErrorProperties } from '../../utils/error_utils'; -import { DataLoader } from '../../data_loader/data_loader'; -import { useTimefilter } from '../../hooks/use_time_filter'; -import { - DataVisualizerTableState, - FieldRequestConfig, - JOB_FIELD_TYPES, -} from '../../../../../common'; -import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; -import { TimeBuckets } from '../../services/time_buckets'; -import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; -import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; -import { getActions } from '../../../common/components/field_data_row/action_menu'; -import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; -export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerStartDependencies]; -export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { - indexPattern: DataView; - savedSearch?: SavedSearch; - query?: Query; - visibleFieldNames?: string[]; - filters?: Filter[]; - showPreviewByDefault?: boolean; -} -export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; - -export type IDataVisualizerGridEmbeddable = typeof DataVisualizerGridEmbeddable; - -const restorableDefaults = getDefaultDataVisualizerListState(); -const defaults = getDefaultPageState(); - -const useDataVisualizerGridData = ( - input: DataVisualizerGridEmbeddableInput, - dataVisualizerListState: Required -) => { - const { services } = useDataVisualizerKibana(); - const { notifications, uiSettings } = services; - const { toasts } = notifications; - const { samplerShardSize, visibleFieldTypes, showEmptyFields } = dataVisualizerListState; - - const [lastRefresh, setLastRefresh] = useState(0); - - const { - currentSavedSearch, - currentIndexPattern, - currentQuery, - currentFilters, - visibleFieldNames, - } = useMemo( - () => ({ - currentSavedSearch: input?.savedSearch, - currentIndexPattern: input.indexPattern, - currentQuery: input?.query, - visibleFieldNames: input?.visibleFieldNames ?? [], - currentFilters: input?.filters, - }), - [input] - ); - - const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - const searchData = extractSearchData({ - indexPattern: currentIndexPattern, - uiSettings, - savedSearch: currentSavedSearch, - query: currentQuery, - filters: currentFilters, - }); - - if (searchData === undefined || dataVisualizerListState.searchString !== '') { - return { - searchQuery: dataVisualizerListState.searchQuery, - searchString: dataVisualizerListState.searchString, - searchQueryLanguage: dataVisualizerListState.searchQueryLanguage, - }; - } else { - return { - searchQuery: searchData.searchQuery, - searchString: searchData.searchString, - searchQueryLanguage: searchData.queryLanguage, - }; - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [ - currentSavedSearch, - currentIndexPattern, - dataVisualizerListState, - currentQuery, - currentFilters, - ]); - - const [overallStats, setOverallStats] = useState(defaults.overallStats); - - const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); - const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); - const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); - const [metricsStats, setMetricsStats] = useState(); - - const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); - const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); - - const dataLoader = useMemo( - () => new DataLoader(currentIndexPattern, toasts), - [currentIndexPattern, toasts] - ); - - const timefilter = useTimefilter({ - timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, - autoRefreshSelector: true, - }); - - useEffect(() => { - const timeUpdateSubscription = merge( - timefilter.getTimeUpdate$(), - dataVisualizerRefresh$ - ).subscribe(() => { - setLastRefresh(Date.now()); - }); - return () => { - timeUpdateSubscription.unsubscribe(); - }; - }); - - const getTimeBuckets = useCallback(() => { - return new TimeBuckets({ - [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), - [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), - dateFormat: uiSettings.get('dateFormat'), - 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), - }); - }, [uiSettings]); - - const indexPatternFields: IndexPatternField[] = useMemo( - () => currentIndexPattern.fields, - [currentIndexPattern] - ); - - async function loadOverallStats() { - const tf = timefilter as any; - let earliest; - let latest; - - const activeBounds = tf.getActiveBounds(); - - if (currentIndexPattern.timeFieldName !== undefined && activeBounds === undefined) { - return; - } - - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = activeBounds.min.valueOf(); - latest = activeBounds.max.valueOf(); - } - - try { - const allStats = await dataLoader.loadOverallData( - searchQuery, - samplerShardSize, - earliest, - latest - ); - // Because load overall stats perform queries in batches - // there could be multiple errors - if (Array.isArray(allStats.errors) && allStats.errors.length > 0) { - allStats.errors.forEach((err: any) => { - dataLoader.displayError(extractErrorProperties(err)); - }); - } - setOverallStats(allStats); - } catch (err) { - dataLoader.displayError(err.body ?? err); - } - } - - const createMetricCards = useCallback(() => { - const configs: FieldVisConfig[] = []; - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; - - const allMetricFields = indexPatternFields.filter((f) => { - return ( - f.type === KBN_FIELD_TYPES.NUMBER && - f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true - ); - }); - const metricExistsFields = allMetricFields.filter((f) => { - return aggregatableExistsFields.find((existsF) => { - return existsF.fieldName === f.spec.name; - }); - }); - - // Add a config for 'document count', identified by no field name if indexpattern is time based. - if (currentIndexPattern.timeFieldName !== undefined) { - configs.push({ - type: JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - loading: true, - aggregatable: true, - }); - } - - if (metricsLoaded === false) { - setMetricsLoaded(true); - return; - } - - let aggregatableFields: any[] = overallStats.aggregatableExistsFields; - if (allMetricFields.length !== metricExistsFields.length && metricsLoaded === true) { - aggregatableFields = aggregatableFields.concat(overallStats.aggregatableNotExistsFields); - } - - const metricFieldsToShow = - metricsLoaded === true && showEmptyFields === true ? allMetricFields : metricExistsFields; - - metricFieldsToShow.forEach((field) => { - const fieldData = aggregatableFields.find((f) => { - return f.fieldName === field.spec.name; - }); - - const metricConfig: FieldVisConfig = { - ...(fieldData ? fieldData : {}), - fieldFormat: currentIndexPattern.getFormatterForField(field), - type: JOB_FIELD_TYPES.NUMBER, - loading: true, - aggregatable: true, - deletable: field.runtimeField !== undefined, - }; - if (field.displayName !== metricConfig.fieldName) { - metricConfig.displayName = field.displayName; - } - - configs.push(metricConfig); - }); - - setMetricsStats({ - totalMetricFieldsCount: allMetricFields.length, - visibleMetricsCount: metricFieldsToShow.length, - }); - setMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - metricsLoaded, - overallStats, - showEmptyFields, - ]); - - const createNonMetricCards = useCallback(() => { - const allNonMetricFields = indexPatternFields.filter((f) => { - return ( - f.type !== KBN_FIELD_TYPES.NUMBER && - f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true - ); - }); - // Obtain the list of all non-metric fields which appear in documents - // (aggregatable or not aggregatable). - const populatedNonMetricFields: any[] = []; // Kibana index pattern non metric fields. - let nonMetricFieldData: any[] = []; // Basic non metric field data loaded from requesting overall stats. - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; - const nonAggregatableExistsFields: any[] = overallStats.nonAggregatableExistsFields || []; - - allNonMetricFields.forEach((f) => { - const checkAggregatableField = aggregatableExistsFields.find( - (existsField) => existsField.fieldName === f.spec.name - ); - - if (checkAggregatableField !== undefined) { - populatedNonMetricFields.push(f); - nonMetricFieldData.push(checkAggregatableField); - } else { - const checkNonAggregatableField = nonAggregatableExistsFields.find( - (existsField) => existsField.fieldName === f.spec.name - ); - - if (checkNonAggregatableField !== undefined) { - populatedNonMetricFields.push(f); - nonMetricFieldData.push(checkNonAggregatableField); - } - } - }); - - if (nonMetricsLoaded === false) { - setNonMetricsLoaded(true); - return; - } - - if (allNonMetricFields.length !== nonMetricFieldData.length && showEmptyFields === true) { - // Combine the field data obtained from Elasticsearch into a single array. - nonMetricFieldData = nonMetricFieldData.concat( - overallStats.aggregatableNotExistsFields, - overallStats.nonAggregatableNotExistsFields - ); - } - - const nonMetricFieldsToShow = showEmptyFields ? allNonMetricFields : populatedNonMetricFields; - - const configs: FieldVisConfig[] = []; - - nonMetricFieldsToShow.forEach((field) => { - const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); - - const nonMetricConfig = { - ...fieldData, - fieldFormat: currentIndexPattern.getFormatterForField(field), - aggregatable: field.aggregatable, - scripted: field.scripted, - loading: fieldData.existsInDocs, - deletable: field.runtimeField !== undefined, - }; - - // Map the field type from the Kibana index pattern to the field type - // used in the data visualizer. - const dataVisualizerType = kbnTypeToJobType(field); - if (dataVisualizerType !== undefined) { - nonMetricConfig.type = dataVisualizerType; - } else { - // Add a flag to indicate that this is one of the 'other' Kibana - // field types that do not yet have a specific card type. - nonMetricConfig.type = field.type; - nonMetricConfig.isUnsupportedType = true; - } - - if (field.displayName !== nonMetricConfig.fieldName) { - nonMetricConfig.displayName = field.displayName; - } - - configs.push(nonMetricConfig); - }); - - setNonMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - nonMetricsLoaded, - overallStats, - showEmptyFields, - ]); - - async function loadMetricFieldStats() { - // Only request data for fields that exist in documents. - if (metricConfigs.length === 0) { - return; - } - - const configsToLoad = metricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - // Obtain the interval to use for date histogram aggregations - // (such as the document count chart). Aim for 75 bars. - const buckets = getTimeBuckets(); - - const tf = timefilter as any; - let earliest: number | undefined; - let latest: number | undefined; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - const bounds = tf.getActiveBounds(); - const BAR_TARGET = 75; - buckets.setInterval('auto'); - buckets.setBounds(bounds); - buckets.setBarTarget(BAR_TARGET); - const aggInterval = buckets.getInterval(); - - try { - const metricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existMetricFields, - aggInterval.asMilliseconds() - ); - - // Add the metric stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - metricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - configWithStats.loading = false; - configs.push(configWithStats); - } else { - // Document count card. - configWithStats.stats = metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === undefined - ); - - if (configWithStats.stats !== undefined) { - // Add earliest / latest of timefilter for setting x axis domain. - configWithStats.stats.timeRangeEarliest = earliest; - configWithStats.stats.timeRangeLatest = latest; - } - setDocumentCountStats(configWithStats); - } - }); - - setMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - async function loadNonMetricFieldStats() { - // Only request data for fields that exist in documents. - if (nonMetricConfigs.length === 0) { - return; - } - - const configsToLoad = nonMetricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existNonMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - const tf = timefilter as any; - let earliest; - let latest; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - try { - const nonMetricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existNonMetricFields - ); - - // Add the field stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - nonMetricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...nonMetricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - } - configWithStats.loading = false; - configs.push(configWithStats); - }); - - setNonMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - useEffect(() => { - loadOverallStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchQuery, samplerShardSize, lastRefresh]); - - useEffect(() => { - createMetricCards(); - createNonMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [overallStats, showEmptyFields]); - - useEffect(() => { - loadMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricConfigs]); - - useEffect(() => { - loadNonMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricConfigs]); - - useEffect(() => { - createMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricsLoaded]); - - useEffect(() => { - createNonMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricsLoaded]); - - const configs = useMemo(() => { - let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; - if (visibleFieldTypes && visibleFieldTypes.length > 0) { - combinedConfigs = combinedConfigs.filter( - (config) => visibleFieldTypes.findIndex((field) => field === config.type) > -1 - ); - } - if (visibleFieldNames && visibleFieldNames.length > 0) { - combinedConfigs = combinedConfigs.filter( - (config) => visibleFieldNames.findIndex((field) => field === config.fieldName) > -1 - ); - } - - return combinedConfigs; - }, [nonMetricConfigs, metricConfigs, visibleFieldTypes, visibleFieldNames]); - - // Some actions open up fly-out or popup - // This variable is used to keep track of them and clean up when unmounting - const actionFlyoutRef = useRef<() => void | undefined>(); - useEffect(() => { - const ref = actionFlyoutRef; - return () => { - // Clean up any of the flyout/editor opened from the actions - if (ref.current) { - ref.current(); - } - }; - }, []); - - // Inject custom action column for the index based visualizer - // Hide the column completely if no access to any of the plugins - const extendedColumns = useMemo(() => { - const actions = getActions( - input.indexPattern, - { lens: services.lens }, - { - searchQueryLanguage, - searchString, - }, - actionFlyoutRef - ); - if (!Array.isArray(actions) || actions.length < 1) return; - - const actionColumn: EuiTableActionsColumnType = { - name: ( - - ), - actions, - width: '70px', - }; - - return [actionColumn]; - }, [input.indexPattern, services, searchQueryLanguage, searchString]); - - return { - configs, - searchQueryLanguage, - searchString, - searchQuery, - extendedColumns, - documentCountStats, - metricsStats, - }; -}; - -export const DiscoverWrapper = ({ - input, - onOutputChange, -}: { - input: DataVisualizerGridEmbeddableInput; - onOutputChange?: (ouput: any) => void; -}) => { - const [dataVisualizerListState, setDataVisualizerListState] = - useState>(restorableDefaults); - - const onTableChange = useCallback( - (update: DataVisualizerTableState) => { - setDataVisualizerListState({ ...dataVisualizerListState, ...update }); - if (onOutputChange) { - onOutputChange(update); - } - }, - [dataVisualizerListState, onOutputChange] - ); - const { configs, searchQueryLanguage, searchString, extendedColumns } = useDataVisualizerGridData( - input, - dataVisualizerListState - ); - const getItemIdToExpandedRowMap = useCallback( - function (itemIds: string[], items: FieldVisConfig[]): ItemIdToExpandedRowMap { - return itemIds.reduce((m: ItemIdToExpandedRowMap, fieldName: string) => { - const item = items.find((fieldVisConfig) => fieldVisConfig.fieldName === fieldName); - if (item !== undefined) { - m[fieldName] = ( - - ); - } - return m; - }, {} as ItemIdToExpandedRowMap); - }, - [input, searchQueryLanguage, searchString] - ); - - return ( - - items={configs} - pageState={dataVisualizerListState} - updatePageState={onTableChange} - getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} - extendedColumns={extendedColumns} - showPreviewByDefault={input?.showPreviewByDefault} - onChange={onOutputChange} - /> - ); - - return
; -}; - -export const IndexDataVisualizerViewWrapper = (props: { - id: string; - embeddableContext: InstanceType; - embeddableInput: Readonly>; - onOutputChange?: (output: any) => void; -}) => { - const { embeddableInput, onOutputChange } = props; - - const input = useObservable(embeddableInput); - if (input && input.indexPattern) { - return ; - } else { - return
; - } -}; -export class DataVisualizerGridEmbeddable extends Embeddable< - DataVisualizerGridEmbeddableInput, - DataVisualizerGridEmbeddableOutput -> { - private node?: HTMLElement; - private reload$ = new Subject(); - public readonly type: string = DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE; - - constructor( - initialInput: DataVisualizerGridEmbeddableInput, - public services: DataVisualizerGridEmbeddableServices, - parent?: IContainer - ) { - super(initialInput, {}, parent); - } - - public render(node: HTMLElement) { - super.render(node); - this.node = node; - - const I18nContext = this.services[0].i18n.Context; - - ReactDOM.render( - - - }> - this.updateOutput(output)} - /> - - - , - node - ); - } - - public destroy() { - super.destroy(); - if (this.node) { - ReactDOM.unmountComponentAtNode(this.node); - } - } - - public reload() { - this.reload$.next(); - } - - public supportedTriggers() { - return []; - } -} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx deleted file mode 100644 index e4df251bdcec0..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// import { StartServicesAccessor } from 'kibana/public'; -import { i18n } from '@kbn/i18n'; -import { StartServicesAccessor } from 'kibana/public'; -import { - EmbeddableFactoryDefinition, - IContainer, -} from '../../../../../../../../src/plugins/embeddable/public'; -import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; -import { - DataVisualizerGridEmbeddableInput, - DataVisualizerGridEmbeddableServices, -} from './grid_embeddable'; -import { DataVisualizerPluginStart, DataVisualizerStartDependencies } from '../../../../plugin'; - -export class DataVisualizerGridEmbeddableFactory - implements EmbeddableFactoryDefinition -{ - public readonly type = DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE; - - public readonly grouping = [ - { - id: 'data_visualizer_grid', - getDisplayName: () => 'Data Visualizer Grid', - }, - ]; - - constructor( - private getStartServices: StartServicesAccessor< - DataVisualizerStartDependencies, - DataVisualizerPluginStart - > - ) {} - - public async isEditable() { - return false; - } - - public getDisplayName() { - return i18n.translate('xpack.dataVisualizer.index.components.grid.displayName', { - defaultMessage: 'Data Visualizer Grid', - }); - } - - public getDescription() { - return i18n.translate('xpack.dataVisualizer.index.components.grid.description', { - defaultMessage: 'Visualize data', - }); - } - - private async getServices(): Promise { - const [coreStart, pluginsStart] = await this.getStartServices(); - return [coreStart, pluginsStart]; - } - - public async create(initialInput: DataVisualizerGridEmbeddableInput, parent?: IContainer) { - const [coreStart, pluginsStart] = await this.getServices(); - const { DataVisualizerGridEmbeddable } = await import('./grid_embeddable'); - return new DataVisualizerGridEmbeddable(initialInput, [coreStart, pluginsStart], parent); - } -} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts deleted file mode 100644 index 91ca8e1633eb9..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -export { DataVisualizerGridEmbeddable } from './grid_embeddable'; -export { DataVisualizerGridEmbeddableFactory } from './grid_embeddable_factory'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts deleted file mode 100644 index add99a8d2501d..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { CoreSetup } from 'kibana/public'; -import { EmbeddableSetup } from '../../../../../../../src/plugins/embeddable/public'; -import { DataVisualizerGridEmbeddableFactory } from './grid_embeddable/grid_embeddable_factory'; -import { DataVisualizerPluginStart, DataVisualizerStartDependencies } from '../../../plugin'; - -export function registerEmbeddables( - embeddable: EmbeddableSetup, - core: CoreSetup -) { - const dataVisualizerGridEmbeddableFactory = new DataVisualizerGridEmbeddableFactory( - core.getStartServices - ); - embeddable.registerEmbeddableFactory( - dataVisualizerGridEmbeddableFactory.type, - dataVisualizerGridEmbeddableFactory - ); -} diff --git a/x-pack/plugins/data_visualizer/public/plugin.ts b/x-pack/plugins/data_visualizer/public/plugin.ts index df1a5ea406d76..112294f4b246f 100644 --- a/x-pack/plugins/data_visualizer/public/plugin.ts +++ b/x-pack/plugins/data_visualizer/public/plugin.ts @@ -6,7 +6,7 @@ */ import { CoreSetup, CoreStart } from 'kibana/public'; -import type { EmbeddableSetup, EmbeddableStart } from '../../../../src/plugins/embeddable/public'; +import type { EmbeddableStart } from '../../../../src/plugins/embeddable/public'; import type { SharePluginStart } from '../../../../src/plugins/share/public'; import { Plugin } from '../../../../src/core/public'; @@ -21,11 +21,9 @@ import type { IndexPatternFieldEditorStart } from '../../../../src/plugins/index import { getFileDataVisualizerComponent, getIndexDataVisualizerComponent } from './api'; import { getMaxBytesFormatted } from './application/common/util/get_max_bytes'; import { registerHomeAddData, registerHomeFeatureCatalogue } from './register_home'; -import { registerEmbeddables } from './application/index_data_visualizer/embeddables'; export interface DataVisualizerSetupDependencies { home?: HomePublicPluginSetup; - embeddable: EmbeddableSetup; } export interface DataVisualizerStartDependencies { data: DataPublicPluginStart; @@ -58,9 +56,6 @@ export class DataVisualizerPlugin registerHomeAddData(plugins.home); registerHomeFeatureCatalogue(plugins.home); } - if (plugins.embeddable) { - registerEmbeddables(plugins.embeddable, core); - } } public start(core: CoreStart, plugins: DataVisualizerStartDependencies) { From ba1cad7ff6ee7b3391fb42000306c6b3b96096d5 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 22 Sep 2021 17:20:49 -0500 Subject: [PATCH 108/188] [ML] Rename view mode, refactor to separate hook, add error prompt if can't show, rename wrapper, clean up & fix tests --- .../components/chart/discover_chart.test.tsx | 4 +- .../main/components/chart/discover_chart.tsx | 13 +- .../components/layout/discover_layout.tsx | 19 +- .../sidebar/discover_sidebar.test.tsx | 4 +- .../components/sidebar/discover_sidebar.tsx | 11 +- .../discover_sidebar_responsive.test.tsx | 4 +- .../sidebar/discover_sidebar_responsive.tsx | 4 +- .../components/view_mode_toggle/constants.ts | 4 +- .../main/components/view_mode_toggle/index.ts | 2 +- .../view_mode_toggle/view_mode_toggle.tsx | 16 +- .../apps/main/services/discover_state.ts | 4 +- .../main/utils/get_state_defaults.test.ts | 4 +- .../apps/main/utils/get_state_defaults.ts | 9 +- .../apps/main/utils/persist_saved_search.ts | 4 +- .../data_visualizer_grid.tsx | 27 +- .../public/saved_searches/_saved_search.ts | 4 +- .../discover/public/saved_searches/types.ts | 4 +- .../discover/server/saved_objects/search.ts | 2 +- .../functional/apps/discover/_shared_links.ts | 13 +- .../grid_embeddable/grid_embeddable.tsx | 625 +----------------- .../grid_embeddable_factory.tsx | 7 +- .../use_data_visualizer_grid_data.ts | 586 ++++++++++++++++ .../index_data_visualizer.tsx | 1 - .../index_data_visualizer/locator/locator.ts | 1 - 24 files changed, 691 insertions(+), 681 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts diff --git a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.test.tsx b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.test.tsx index b6f9fedf51e33..d22473a8df6ed 100644 --- a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.test.tsx @@ -19,7 +19,7 @@ import { discoverServiceMock } from '../../../../../__mocks__/services'; import { FetchStatus } from '../../../../types'; import { Chart } from './point_series'; import { DiscoverChart } from './discover_chart'; -import { DISCOVER_VIEW_MODE } from '../view_mode_toggle'; +import { VIEW_MODE } from '../view_mode_toggle'; setHeaderActionMenuMounter(jest.fn()); @@ -95,7 +95,7 @@ function getProps(timefield?: string) { state: { columns: [] }, stateContainer: {} as GetStateReturn, timefield, - discoverViewMode: DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, + viewMode: VIEW_MODE.DOCUMENT_LEVEL, setDiscoverViewMode: jest.fn(), }; } diff --git a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx index aa4fa84c92025..419ea3eedd2b3 100644 --- a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx +++ b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx @@ -17,7 +17,7 @@ import { AppState, GetStateReturn } from '../../services/discover_state'; import { DiscoverHistogram } from './histogram'; import { DataCharts$, DataTotalHits$ } from '../../services/use_saved_search'; import { DiscoverServices } from '../../../../../build_services'; -import { DISCOVER_VIEW_MODE, DocumentViewModeToggle } from '../view_mode_toggle'; +import { VIEW_MODE, DocumentViewModeToggle } from '../view_mode_toggle'; const TimechartHeaderMemoized = memo(TimechartHeader); const DiscoverHistogramMemoized = memo(DiscoverHistogram); @@ -30,7 +30,7 @@ export function DiscoverChart({ state, stateContainer, timefield, - discoverViewMode, + viewMode, setDiscoverViewMode, }: { resetSavedSearch: () => void; @@ -41,8 +41,8 @@ export function DiscoverChart({ state: AppState; stateContainer: GetStateReturn; timefield?: string; - discoverViewMode: DISCOVER_VIEW_MODE; - setDiscoverViewMode: (discoverViewMode: DISCOVER_VIEW_MODE) => void; + viewMode: VIEW_MODE; + setDiscoverViewMode: (viewMode: VIEW_MODE) => void; }) { const { data, uiSettings: config } = services; const chartRef = useRef<{ element: HTMLElement | null; moveFocus: boolean }>({ @@ -127,10 +127,7 @@ export function DiscoverChart({ )} - + diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 6be4eb32d5c3b..edefb63038fa4 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -44,7 +44,7 @@ import { DiscoverDocuments } from './discover_documents'; import { FetchStatus } from '../../../../types'; import { useDataState } from '../../utils/use_data_state'; import { DiscoverDataVisualizerGrid } from '../../../../components/data_visualizer_grid'; -import { DISCOVER_VIEW_MODE } from '../view_mode_toggle'; +import { VIEW_MODE } from '../view_mode_toggle'; const SidebarMemoized = React.memo(DiscoverSidebarResponsive); const TopNavMemoized = React.memo(DiscoverTopNav); @@ -73,14 +73,11 @@ export function DiscoverLayout({ const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); - const discoverViewMode = useMemo( - () => state.discoverViewMode ?? DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, - [state.discoverViewMode] - ); + const viewMode = useMemo(() => state.viewMode ?? VIEW_MODE.DOCUMENT_LEVEL, [state.viewMode]); const setDiscoverViewMode = useCallback( - (mode: DISCOVER_VIEW_MODE) => { - stateContainer.setAppState({ discoverViewMode: mode }); + (mode: VIEW_MODE) => { + stateContainer.setAppState({ viewMode: mode }); }, [stateContainer] ); @@ -202,7 +199,7 @@ export function DiscoverLayout({ trackUiMetric={trackUiMetric} useNewFieldsApi={useNewFieldsApi} onEditRuntimeField={onEditRuntimeField} - discoverViewMode={discoverViewMode} + viewMode={viewMode} /> @@ -269,12 +266,12 @@ export function DiscoverLayout({ services={services} stateContainer={stateContainer} timefield={timeField} - discoverViewMode={discoverViewMode} + viewMode={viewMode} setDiscoverViewMode={setDiscoverViewMode} /> - {discoverViewMode === DISCOVER_VIEW_MODE.DOCUMENT_LEVEL ? ( + {viewMode === VIEW_MODE.DOCUMENT_LEVEL ? ( ({ getServices: () => mockDiscoverServices, @@ -65,7 +65,7 @@ function getCompProps(): DiscoverSidebarProps { setFieldFilter: jest.fn(), onEditRuntimeField: jest.fn(), editField: jest.fn(), - discoverViewMode: DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, + viewMode: VIEW_MODE.DOCUMENT_LEVEL, }; } diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx index 50da3dd1f67c2..24f630183cdf9 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.tsx @@ -37,7 +37,7 @@ import { getIndexPatternFieldList } from './lib/get_index_pattern_field_list'; import { DiscoverSidebarResponsiveProps } from './discover_sidebar_responsive'; import { DiscoverIndexPatternManagement } from './discover_index_pattern_management'; import { ElasticSearchHit } from '../../../../doc_views/doc_views_types'; -import { DISCOVER_VIEW_MODE } from '../view_mode_toggle'; +import { VIEW_MODE } from '../view_mode_toggle'; /** * Default number of available fields displayed and added on scroll @@ -78,7 +78,7 @@ export interface DiscoverSidebarProps extends Omit(null); @@ -208,10 +208,7 @@ export function DiscoverSidebarComponent({ return result; }, [fields]); - const showFieldStats = useMemo( - () => discoverViewMode === DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, - [discoverViewMode] - ); + const showFieldStats = useMemo(() => viewMode === VIEW_MODE.DOCUMENT_LEVEL, [viewMode]); const calculateMultiFields = () => { if (!useNewFieldsApi || !fields) { diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.test.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.test.tsx index 2e40c03d6c840..c8d387e80250a 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.test.tsx @@ -26,7 +26,7 @@ import { ElasticSearchHit } from '../../../../doc_views/doc_views_types'; import { FetchStatus } from '../../../../types'; import { DataDocuments$ } from '../../services/use_saved_search'; import { stubLogstashIndexPattern } from '../../../../../../../data/common/stubs'; -import { DISCOVER_VIEW_MODE } from '../view_mode_toggle'; +import { VIEW_MODE } from '../view_mode_toggle'; const mockServices = { history: () => ({ @@ -104,7 +104,7 @@ function getCompProps(): DiscoverSidebarResponsiveProps { state: {}, trackUiMetric: jest.fn(), onEditRuntimeField: jest.fn(), - discoverViewMode: DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, + viewMode: VIEW_MODE.DOCUMENT_LEVEL, }; } diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.tsx index 43383591672fb..ec9a899fab991 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.tsx @@ -37,7 +37,7 @@ import { AppState } from '../../services/discover_state'; import { DiscoverIndexPatternManagement } from './discover_index_pattern_management'; import { DataDocuments$ } from '../../services/use_saved_search'; import { calcFieldCounts } from '../../utils/calc_field_counts'; -import { DISCOVER_VIEW_MODE } from '../view_mode_toggle'; +import { VIEW_MODE } from '../view_mode_toggle'; export interface DiscoverSidebarResponsiveProps { /** @@ -110,7 +110,7 @@ export interface DiscoverSidebarResponsiveProps { /** * Discover view mode */ - discoverViewMode: DISCOVER_VIEW_MODE; + viewMode: VIEW_MODE; } /** diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/constants.ts b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/constants.ts index be16ff7c6367f..d03c0710d12b3 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/constants.ts +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/constants.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -export enum DISCOVER_VIEW_MODE { +export enum VIEW_MODE { DOCUMENT_LEVEL = 'documents', - FIELD_LEVEL = 'fields', + AGGREGATED_LEVEL = 'aggregated', } diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/index.ts b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/index.ts index 52e4ce3d8d3ec..95b76f5879d19 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/index.ts +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/index.ts @@ -7,4 +7,4 @@ */ export { DocumentViewModeToggle } from './view_mode_toggle'; -export { DISCOVER_VIEW_MODE } from './constants'; +export { VIEW_MODE } from './constants'; diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx index 62dc8c7ed463b..e73a9783f503b 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx @@ -10,26 +10,26 @@ import { EuiButtonGroup, EuiBetaBadge } from '@elastic/eui'; import React, { useMemo } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { DISCOVER_VIEW_MODE } from './constants'; +import { VIEW_MODE } from './constants'; import './_index.scss'; export const DocumentViewModeToggle = ({ - discoverViewMode, + viewMode, setDiscoverViewMode, }: { - discoverViewMode: DISCOVER_VIEW_MODE; - setDiscoverViewMode: (discoverViewMode: DISCOVER_VIEW_MODE) => void; + viewMode: VIEW_MODE; + setDiscoverViewMode: (viewMode: VIEW_MODE) => void; }) => { const toggleButtons = useMemo( () => [ { - id: DISCOVER_VIEW_MODE.DOCUMENT_LEVEL, + id: VIEW_MODE.DOCUMENT_LEVEL, label: i18n.translate('discover.viewModes.document.label', { defaultMessage: 'Documents', }), }, { - id: DISCOVER_VIEW_MODE.FIELD_LEVEL, + id: VIEW_MODE.AGGREGATED_LEVEL, label: (
setDiscoverViewMode(id as DISCOVER_VIEW_MODE)} + idSelected={viewMode} + onChange={(id: string) => setDiscoverViewMode(id as VIEW_MODE)} /> ); }; diff --git a/src/plugins/discover/public/application/apps/main/services/discover_state.ts b/src/plugins/discover/public/application/apps/main/services/discover_state.ts index 138b5ce566563..0014765ad28c4 100644 --- a/src/plugins/discover/public/application/apps/main/services/discover_state.ts +++ b/src/plugins/discover/public/application/apps/main/services/discover_state.ts @@ -35,7 +35,7 @@ import { DiscoverGridSettings } from '../../../components/discover_grid/types'; import { DISCOVER_APP_URL_GENERATOR, DiscoverUrlGeneratorState } from '../../../../url_generator'; import { SavedSearch } from '../../../../saved_searches'; import { handleSourceColumnState } from '../../../helpers/state_helpers'; -import { DISCOVER_VIEW_MODE } from '../components/view_mode_toggle'; +import { VIEW_MODE } from '../components/view_mode_toggle'; export interface AppState { /** @@ -77,7 +77,7 @@ export interface AppState { /** * Table view: Documents vs Field Statistics */ - discoverViewMode?: DISCOVER_VIEW_MODE; + viewMode?: VIEW_MODE; /** * Hide mini distribution/preview charts when in Field Statistics mode */ diff --git a/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.test.ts b/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.test.ts index efc93cdc74fb4..01526c2125142 100644 --- a/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.test.ts +++ b/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.test.ts @@ -26,7 +26,6 @@ describe('getStateDefaults', () => { "columns": Array [ "default_column", ], - "discoverViewMode": "discoverViewOptionDocument", "filters": undefined, "hideAggregatedPreview": undefined, "hideChart": undefined, @@ -39,6 +38,7 @@ describe('getStateDefaults', () => { "desc", ], ], + "viewMode": undefined, } `); }); @@ -56,7 +56,6 @@ describe('getStateDefaults', () => { "columns": Array [ "default_column", ], - "discoverViewMode": "discoverViewOptionDocument", "filters": undefined, "hideAggregatedPreview": undefined, "hideChart": undefined, @@ -64,6 +63,7 @@ describe('getStateDefaults', () => { "interval": "auto", "query": undefined, "sort": Array [], + "viewMode": undefined, } `); }); diff --git a/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.ts b/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.ts index e05ac5a48153a..79e1198f154fd 100644 --- a/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.ts +++ b/src/plugins/discover/public/application/apps/main/utils/get_state_defaults.ts @@ -14,7 +14,6 @@ import { DataPublicPluginStart } from '../../../../../../data/public'; import { AppState } from '../services/discover_state'; import { getDefaultSort, getSortArray } from '../components/doc_table'; -import { DISCOVER_VIEW_MODE } from '../components/view_mode_toggle'; function getDefaultColumns(savedSearch: SavedSearch, config: IUiSettingsClient) { if (savedSearch.columns && savedSearch.columns.length > 0) { @@ -48,7 +47,7 @@ export function getStateDefaults({ interval: 'auto', filters: cloneDeep(searchSource.getOwnField('filter')), hideChart: undefined, - discoverViewMode: undefined, + viewMode: undefined, hideAggregatedPreview: undefined, } as AppState; if (savedSearch.grid) { @@ -57,10 +56,8 @@ export function getStateDefaults({ if (savedSearch.hideChart) { defaultState.hideChart = savedSearch.hideChart; } - if (savedSearch.discoverViewMode) { - defaultState.discoverViewMode = savedSearch.discoverViewMode; - } else { - defaultState.discoverViewMode = DISCOVER_VIEW_MODE.DOCUMENT_LEVEL; + if (savedSearch.viewMode) { + defaultState.viewMode = savedSearch.viewMode; } if (savedSearch.hideAggregatedPreview) { diff --git a/src/plugins/discover/public/application/apps/main/utils/persist_saved_search.ts b/src/plugins/discover/public/application/apps/main/utils/persist_saved_search.ts index 7f671cbe2e1e6..e8beb681260ad 100644 --- a/src/plugins/discover/public/application/apps/main/utils/persist_saved_search.ts +++ b/src/plugins/discover/public/application/apps/main/utils/persist_saved_search.ts @@ -51,8 +51,8 @@ export async function persistSavedSearch( savedSearch.hideChart = state.hideChart; } - if (state.discoverViewMode) { - savedSearch.discoverViewMode = state.discoverViewMode; + if (state.viewMode) { + savedSearch.viewMode = state.viewMode; } if (state.hideAggregatedPreview) { diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index 00d9b9e63c6b6..101b4c2b9cdc6 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -7,7 +7,6 @@ */ import React, { useEffect, useMemo, useRef, useState } from 'react'; -import { EuiDataGrid, EuiDataGridProps } from '@elastic/eui'; import { Filter } from '@kbn/es-query'; import { DataView, Query } from '../../../../../data/common'; import { DiscoverServices } from '../../../build_services'; @@ -43,27 +42,27 @@ export interface DiscoverDataVisualizerGridProps { */ indexPattern: DataView; /** - * Saved search description + * Discover plugin services */ - searchDescription?: string; + services: DiscoverServices; /** - * Saved search title + * Optional saved search */ - searchTitle?: string; + savedSearch?: SavedSearch; /** - * Discover plugin services + * Optional query to update the table content */ - services: DiscoverServices; - savedSearch?: SavedSearch; query?: Query; + /** + * Filters query to update the table content + */ filters?: Filter[]; + /** + * stateContainer to access and update app state preferences like to show preview or not + */ stateContainer?: GetStateReturn; } -export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => { - return ; -}); - export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { const { services, indexPattern, savedSearch, query, columns, filters, stateContainer } = props; const { uiSettings } = services; @@ -123,7 +122,7 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp useEffect(() => { let unmounted = false; const loadEmbeddable = async () => { - if (services?.embeddable) { + if (services.embeddable) { const factory = services.embeddable.getEmbeddableFactory< DataVisualizerGridEmbeddableInput, DataVisualizerGridEmbeddableOutput @@ -148,7 +147,7 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp unmounted = true; }; // eslint-disable-next-line react-hooks/exhaustive-deps - }, [services?.embeddable, showPreviewByDefault]); + }, [services.embeddable, showPreviewByDefault]); // We can only render after embeddable has already initialized useEffect(() => { diff --git a/src/plugins/discover/public/saved_searches/_saved_search.ts b/src/plugins/discover/public/saved_searches/_saved_search.ts index c104b02213793..75b58e7b6169a 100644 --- a/src/plugins/discover/public/saved_searches/_saved_search.ts +++ b/src/plugins/discover/public/saved_searches/_saved_search.ts @@ -14,7 +14,7 @@ export function createSavedSearchClass(savedObjects: SavedObjectsStart) { public static mapping = { title: 'text', description: 'text', - discoverViewMode: 'keyword', + viewMode: 'keyword', hideChart: 'boolean', hideAggregatedPreview: 'boolean', hits: 'integer', @@ -37,7 +37,7 @@ export function createSavedSearchClass(savedObjects: SavedObjectsStart) { mapping: { title: 'text', description: 'text', - discoverViewMode: 'keyword', + viewMode: 'keyword', hideChart: 'boolean', hideAggregatedPreview: 'boolean', hits: 'integer', diff --git a/src/plugins/discover/public/saved_searches/types.ts b/src/plugins/discover/public/saved_searches/types.ts index 843495e25b0ec..e0734e9d6621b 100644 --- a/src/plugins/discover/public/saved_searches/types.ts +++ b/src/plugins/discover/public/saved_searches/types.ts @@ -9,7 +9,7 @@ import { SearchSource } from '../../../data/public'; import { SavedObjectSaveOpts } from '../../../saved_objects/public'; import { DiscoverGridSettings } from '../application/components/discover_grid/types'; -import { DISCOVER_VIEW_MODE } from '../application/apps/main/components/view_mode_toggle'; +import { VIEW_MODE } from '../application/apps/main/components/view_mode_toggle'; export type SortOrder = [string, string]; export interface SavedSearch { @@ -25,7 +25,7 @@ export interface SavedSearch { lastSavedTitle?: string; copyOnSave?: boolean; hideChart?: boolean; - discoverViewMode?: DISCOVER_VIEW_MODE; + viewMode?: VIEW_MODE; hideAggregatedPreview?: boolean; } export interface SavedSearchLoader { diff --git a/src/plugins/discover/server/saved_objects/search.ts b/src/plugins/discover/server/saved_objects/search.ts index bca45c7f8cd34..950d23ec38a31 100644 --- a/src/plugins/discover/server/saved_objects/search.ts +++ b/src/plugins/discover/server/saved_objects/search.ts @@ -34,7 +34,7 @@ export const searchSavedObjectType: SavedObjectsType = { properties: { columns: { type: 'keyword', index: false, doc_values: false }, description: { type: 'text' }, - discoverViewMode: { type: 'keyword', index: false, doc_values: false }, + viewMode: { type: 'keyword', index: false, doc_values: false }, hideChart: { type: 'boolean', index: false, doc_values: false }, hideAggregatedPreview: { type: 'boolean', index: false, doc_values: false }, hits: { type: 'integer', index: false, doc_values: false }, diff --git a/test/functional/apps/discover/_shared_links.ts b/test/functional/apps/discover/_shared_links.ts index 4c822d13e43a1..62364739db311 100644 --- a/test/functional/apps/discover/_shared_links.ts +++ b/test/functional/apps/discover/_shared_links.ts @@ -76,14 +76,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const expectedUrl = baseUrl + '/app/discover?_t=1453775307251#' + - '/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),' + - "time:(from:'2015-09-19T06:31:44.000Z',to:'2015-09" + - "-23T18:31:44.000Z'))&_a=(columns:!()," + - 'discoverViewMode:discoverViewOptionDocument,' + - "filters:!(),index:'logstash-*'," + - "interval:auto,query:(language:kuery,query:'')," + - "sort:!(!('@timestamp',desc)))"; - + '/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time' + + ":(from:'2015-09-19T06:31:44.000Z',to:'2015-09" + + "-23T18:31:44.000Z'))&_a=(columns:!(),filters:!(),index:'logstash-" + + "*',interval:auto,query:(language:kuery,query:'')" + + ",sort:!(!('@timestamp',desc)))"; const actualUrl = await PageObjects.share.getSharedUrl(); // strip the timestamp out of each URL expect(actualUrl.replace(/_t=\d{13}/, '_t=TIMESTAMP')).to.be( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index a5c0f6e1412c0..cd9df10d67247 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -5,15 +5,15 @@ * 2.0. */ -import { merge, Observable, Subject } from 'rxjs'; +import { Observable, Subject } from 'rxjs'; import { CoreStart } from 'kibana/public'; import ReactDOM from 'react-dom'; -import React, { Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import React, { Suspense, useCallback, useState } from 'react'; import useObservable from 'react-use/lib/useObservable'; -import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; -import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiEmptyPrompt } from '@elastic/eui'; import { Filter } from '@kbn/es-query'; import { Required } from 'utility-types'; +import { FormattedMessage } from '@kbn/i18n/react'; import { Embeddable, EmbeddableInput, @@ -24,41 +24,19 @@ import { KibanaContextProvider } from '../../../../../../../../src/plugins/kiban import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; import { EmbeddableLoading } from './embeddable_loading_fallback'; import { DataVisualizerStartDependencies } from '../../../../plugin'; -import { - IndexPattern, - IndexPatternField, - KBN_FIELD_TYPES, - Query, - UI_SETTINGS, -} from '../../../../../../../../src/plugins/data/common'; +import { IndexPattern, Query } from '../../../../../../../../src/plugins/data/common'; import { SavedSearch } from '../../../../../../../../src/plugins/discover/public'; import { DataVisualizerTable, ItemIdToExpandedRowMap, } from '../../../common/components/stats_table'; import { FieldVisConfig } from '../../../common/components/stats_table/types'; -import { MetricFieldsStats } from '../../../common/components/stats_table/components/field_count_stats'; -import { - getDefaultDataVisualizerListState, - getDefaultPageState, -} from '../../components/index_data_visualizer_view/index_data_visualizer_view'; - -import { extractSearchData } from '../../utils/saved_search_utils'; -import { useDataVisualizerKibana } from '../../../kibana_context'; -import { extractErrorProperties } from '../../utils/error_utils'; -import { DataLoader } from '../../data_loader/data_loader'; -import { useTimefilter } from '../../hooks/use_time_filter'; -import { - DataVisualizerTableState, - FieldRequestConfig, - JOB_FIELD_TYPES, -} from '../../../../../common'; -import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; -import { TimeBuckets } from '../../services/time_buckets'; +import { getDefaultDataVisualizerListState } from '../../components/index_data_visualizer_view/index_data_visualizer_view'; +import { DataVisualizerTableState } from '../../../../../common'; import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; -import { getActions } from '../../../common/components/field_data_row/action_menu'; -import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; +import { useDataVisualizerGridData } from './use_data_visualizer_grid_data'; + export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerStartDependencies]; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { indexPattern: IndexPattern; @@ -73,564 +51,8 @@ export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; export type IDataVisualizerGridEmbeddable = typeof DataVisualizerGridEmbeddable; const restorableDefaults = getDefaultDataVisualizerListState(); -const defaults = getDefaultPageState(); - -const useDataVisualizerGridData = ( - input: DataVisualizerGridEmbeddableInput, - dataVisualizerListState: Required -) => { - const { services } = useDataVisualizerKibana(); - const { notifications, uiSettings } = services; - const { toasts } = notifications; - const { samplerShardSize, visibleFieldTypes, showEmptyFields } = dataVisualizerListState; - - const [lastRefresh, setLastRefresh] = useState(0); - - const { - currentSavedSearch, - currentIndexPattern, - currentQuery, - currentFilters, - visibleFieldNames, - } = useMemo( - () => ({ - currentSavedSearch: input?.savedSearch, - currentIndexPattern: input.indexPattern, - currentQuery: input?.query, - visibleFieldNames: input?.visibleFieldNames ?? [], - currentFilters: input?.filters, - }), - [input] - ); - - const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - const searchData = extractSearchData({ - indexPattern: currentIndexPattern, - uiSettings, - savedSearch: currentSavedSearch, - query: currentQuery, - filters: currentFilters, - }); - - if (searchData === undefined || dataVisualizerListState.searchString !== '') { - return { - searchQuery: dataVisualizerListState.searchQuery, - searchString: dataVisualizerListState.searchString, - searchQueryLanguage: dataVisualizerListState.searchQueryLanguage, - }; - } else { - return { - searchQuery: searchData.searchQuery, - searchString: searchData.searchString, - searchQueryLanguage: searchData.queryLanguage, - }; - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [ - currentSavedSearch, - currentIndexPattern, - dataVisualizerListState, - currentQuery, - currentFilters, - ]); - - const [overallStats, setOverallStats] = useState(defaults.overallStats); - - const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); - const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); - const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); - const [metricsStats, setMetricsStats] = useState(); - - const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); - const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); - - const dataLoader = useMemo( - () => new DataLoader(currentIndexPattern, toasts), - [currentIndexPattern, toasts] - ); - - const timefilter = useTimefilter({ - timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, - autoRefreshSelector: true, - }); - - useEffect(() => { - const timeUpdateSubscription = merge( - timefilter.getTimeUpdate$(), - dataVisualizerRefresh$ - ).subscribe(() => { - setLastRefresh(Date.now()); - }); - return () => { - timeUpdateSubscription.unsubscribe(); - }; - }); - - const getTimeBuckets = useCallback(() => { - return new TimeBuckets({ - [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), - [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), - dateFormat: uiSettings.get('dateFormat'), - 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), - }); - }, [uiSettings]); - - const indexPatternFields: IndexPatternField[] = useMemo( - () => currentIndexPattern.fields, - [currentIndexPattern] - ); - - async function loadOverallStats() { - const tf = timefilter as any; - let earliest; - let latest; - - const activeBounds = tf.getActiveBounds(); - - if (currentIndexPattern.timeFieldName !== undefined && activeBounds === undefined) { - return; - } - - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = activeBounds.min.valueOf(); - latest = activeBounds.max.valueOf(); - } - - try { - const allStats = await dataLoader.loadOverallData( - searchQuery, - samplerShardSize, - earliest, - latest - ); - // Because load overall stats perform queries in batches - // there could be multiple errors - if (Array.isArray(allStats.errors) && allStats.errors.length > 0) { - allStats.errors.forEach((err: any) => { - dataLoader.displayError(extractErrorProperties(err)); - }); - } - setOverallStats(allStats); - } catch (err) { - dataLoader.displayError(err.body ?? err); - } - } - - const createMetricCards = useCallback(() => { - const configs: FieldVisConfig[] = []; - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; - - const allMetricFields = indexPatternFields.filter((f) => { - return ( - f.type === KBN_FIELD_TYPES.NUMBER && - f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true - ); - }); - const metricExistsFields = allMetricFields.filter((f) => { - return aggregatableExistsFields.find((existsF) => { - return existsF.fieldName === f.spec.name; - }); - }); - - // Add a config for 'document count', identified by no field name if indexpattern is time based. - if (currentIndexPattern.timeFieldName !== undefined) { - configs.push({ - type: JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - loading: true, - aggregatable: true, - }); - } - - if (metricsLoaded === false) { - setMetricsLoaded(true); - return; - } - - let aggregatableFields: any[] = overallStats.aggregatableExistsFields; - if (allMetricFields.length !== metricExistsFields.length && metricsLoaded === true) { - aggregatableFields = aggregatableFields.concat(overallStats.aggregatableNotExistsFields); - } - - const metricFieldsToShow = - metricsLoaded === true && showEmptyFields === true ? allMetricFields : metricExistsFields; - - metricFieldsToShow.forEach((field) => { - const fieldData = aggregatableFields.find((f) => { - return f.fieldName === field.spec.name; - }); - - const metricConfig: FieldVisConfig = { - ...(fieldData ? fieldData : {}), - fieldFormat: currentIndexPattern.getFormatterForField(field), - type: JOB_FIELD_TYPES.NUMBER, - loading: true, - aggregatable: true, - deletable: field.runtimeField !== undefined, - }; - if (field.displayName !== metricConfig.fieldName) { - metricConfig.displayName = field.displayName; - } - - configs.push(metricConfig); - }); - - setMetricsStats({ - totalMetricFieldsCount: allMetricFields.length, - visibleMetricsCount: metricFieldsToShow.length, - }); - setMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - metricsLoaded, - overallStats, - showEmptyFields, - ]); - - const createNonMetricCards = useCallback(() => { - const allNonMetricFields = indexPatternFields.filter((f) => { - return ( - f.type !== KBN_FIELD_TYPES.NUMBER && - f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true - ); - }); - // Obtain the list of all non-metric fields which appear in documents - // (aggregatable or not aggregatable). - const populatedNonMetricFields: any[] = []; // Kibana index pattern non metric fields. - let nonMetricFieldData: any[] = []; // Basic non metric field data loaded from requesting overall stats. - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; - const nonAggregatableExistsFields: any[] = overallStats.nonAggregatableExistsFields || []; - - allNonMetricFields.forEach((f) => { - const checkAggregatableField = aggregatableExistsFields.find( - (existsField) => existsField.fieldName === f.spec.name - ); - - if (checkAggregatableField !== undefined) { - populatedNonMetricFields.push(f); - nonMetricFieldData.push(checkAggregatableField); - } else { - const checkNonAggregatableField = nonAggregatableExistsFields.find( - (existsField) => existsField.fieldName === f.spec.name - ); - - if (checkNonAggregatableField !== undefined) { - populatedNonMetricFields.push(f); - nonMetricFieldData.push(checkNonAggregatableField); - } - } - }); - - if (nonMetricsLoaded === false) { - setNonMetricsLoaded(true); - return; - } - - if (allNonMetricFields.length !== nonMetricFieldData.length && showEmptyFields === true) { - // Combine the field data obtained from Elasticsearch into a single array. - nonMetricFieldData = nonMetricFieldData.concat( - overallStats.aggregatableNotExistsFields, - overallStats.nonAggregatableNotExistsFields - ); - } - - const nonMetricFieldsToShow = showEmptyFields ? allNonMetricFields : populatedNonMetricFields; - - const configs: FieldVisConfig[] = []; - - nonMetricFieldsToShow.forEach((field) => { - const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); - - const nonMetricConfig = { - ...fieldData, - fieldFormat: currentIndexPattern.getFormatterForField(field), - aggregatable: field.aggregatable, - scripted: field.scripted, - loading: fieldData.existsInDocs, - deletable: field.runtimeField !== undefined, - }; - - // Map the field type from the Kibana index pattern to the field type - // used in the data visualizer. - const dataVisualizerType = kbnTypeToJobType(field); - if (dataVisualizerType !== undefined) { - nonMetricConfig.type = dataVisualizerType; - } else { - // Add a flag to indicate that this is one of the 'other' Kibana - // field types that do not yet have a specific card type. - nonMetricConfig.type = field.type; - nonMetricConfig.isUnsupportedType = true; - } - - if (field.displayName !== nonMetricConfig.fieldName) { - nonMetricConfig.displayName = field.displayName; - } - - configs.push(nonMetricConfig); - }); - - setNonMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - nonMetricsLoaded, - overallStats, - showEmptyFields, - ]); - - async function loadMetricFieldStats() { - // Only request data for fields that exist in documents. - if (metricConfigs.length === 0) { - return; - } - - const configsToLoad = metricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - // Obtain the interval to use for date histogram aggregations - // (such as the document count chart). Aim for 75 bars. - const buckets = getTimeBuckets(); - - const tf = timefilter as any; - let earliest: number | undefined; - let latest: number | undefined; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - const bounds = tf.getActiveBounds(); - const BAR_TARGET = 75; - buckets.setInterval('auto'); - buckets.setBounds(bounds); - buckets.setBarTarget(BAR_TARGET); - const aggInterval = buckets.getInterval(); - try { - const metricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existMetricFields, - aggInterval.asMilliseconds() - ); - - // Add the metric stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - metricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - configWithStats.loading = false; - configs.push(configWithStats); - } else { - // Document count card. - configWithStats.stats = metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === undefined - ); - - if (configWithStats.stats !== undefined) { - // Add earliest / latest of timefilter for setting x axis domain. - configWithStats.stats.timeRangeEarliest = earliest; - configWithStats.stats.timeRangeLatest = latest; - } - setDocumentCountStats(configWithStats); - } - }); - - setMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - async function loadNonMetricFieldStats() { - // Only request data for fields that exist in documents. - if (nonMetricConfigs.length === 0) { - return; - } - - const configsToLoad = nonMetricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existNonMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - const tf = timefilter as any; - let earliest; - let latest; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - try { - const nonMetricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existNonMetricFields - ); - - // Add the field stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - nonMetricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...nonMetricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - } - configWithStats.loading = false; - configs.push(configWithStats); - }); - - setNonMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - useEffect(() => { - loadOverallStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchQuery, samplerShardSize, lastRefresh]); - - useEffect(() => { - createMetricCards(); - createNonMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [overallStats, showEmptyFields]); - - useEffect(() => { - loadMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricConfigs]); - - useEffect(() => { - loadNonMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricConfigs]); - - useEffect(() => { - createMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricsLoaded]); - - useEffect(() => { - createNonMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricsLoaded]); - - const configs = useMemo(() => { - let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; - if (visibleFieldTypes && visibleFieldTypes.length > 0) { - combinedConfigs = combinedConfigs.filter( - (config) => visibleFieldTypes.findIndex((field) => field === config.type) > -1 - ); - } - if (visibleFieldNames && visibleFieldNames.length > 0) { - combinedConfigs = combinedConfigs.filter( - (config) => visibleFieldNames.findIndex((field) => field === config.fieldName) > -1 - ); - } - - return combinedConfigs; - }, [nonMetricConfigs, metricConfigs, visibleFieldTypes, visibleFieldNames]); - - // Some actions open up fly-out or popup - // This variable is used to keep track of them and clean up when unmounting - const actionFlyoutRef = useRef<() => void | undefined>(); - useEffect(() => { - const ref = actionFlyoutRef; - return () => { - // Clean up any of the flyout/editor opened from the actions - if (ref.current) { - ref.current(); - } - }; - }, []); - - // Inject custom action column for the index based visualizer - // Hide the column completely if no access to any of the plugins - const extendedColumns = useMemo(() => { - const actions = getActions( - input.indexPattern, - { lens: services.lens }, - { - searchQueryLanguage, - searchString, - }, - actionFlyoutRef - ); - if (!Array.isArray(actions) || actions.length < 1) return; - - const actionColumn: EuiTableActionsColumnType = { - name: ( - - ), - actions, - width: '70px', - }; - - return [actionColumn]; - }, [input.indexPattern, services, searchQueryLanguage, searchString]); - - return { - configs, - searchQueryLanguage, - searchString, - searchQuery, - extendedColumns, - documentCountStats, - metricsStats, - }; -}; - -export const DiscoverWrapper = ({ +export const EmbeddableWrapper = ({ input, onOutputChange, }: { @@ -683,8 +105,6 @@ export const DiscoverWrapper = ({ onChange={onOutputChange} /> ); - - return
; }; export const IndexDataVisualizerViewWrapper = (props: { @@ -697,9 +117,30 @@ export const IndexDataVisualizerViewWrapper = (props: { const input = useObservable(embeddableInput); if (input && input.indexPattern) { - return ; + return ; } else { - return
; + return ( + + + + } + body={ +

+ +

+ } + /> + ); } }; export class DataVisualizerGridEmbeddable extends Embeddable< diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx index e4df251bdcec0..08ddc2d5fe3c2 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable_factory.tsx @@ -5,7 +5,6 @@ * 2.0. */ -// import { StartServicesAccessor } from 'kibana/public'; import { i18n } from '@kbn/i18n'; import { StartServicesAccessor } from 'kibana/public'; import { @@ -42,9 +41,13 @@ export class DataVisualizerGridEmbeddableFactory return false; } + public canCreateNew() { + return false; + } + public getDisplayName() { return i18n.translate('xpack.dataVisualizer.index.components.grid.displayName', { - defaultMessage: 'Data Visualizer Grid', + defaultMessage: 'Data visualizer grid', }); } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts new file mode 100644 index 0000000000000..94873964fe140 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts @@ -0,0 +1,586 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Required } from 'utility-types'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { merge } from 'rxjs'; +import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; +import { i18n } from '@kbn/i18n'; +import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; +import { useDataVisualizerKibana } from '../../../kibana_context'; +import { extractSearchData } from '../../utils/saved_search_utils'; +import { MetricFieldsStats } from '../../../common/components/stats_table/components/field_count_stats'; +import { DataLoader } from '../../data_loader/data_loader'; +import { useTimefilter } from '../../hooks/use_time_filter'; +import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; +import { TimeBuckets } from '../../services/time_buckets'; +import { + DataViewField, + KBN_FIELD_TYPES, + UI_SETTINGS, +} from '../../../../../../../../src/plugins/data/common'; +import { extractErrorProperties } from '../../utils/error_utils'; +import { FieldVisConfig } from '../../../common/components/stats_table/types'; +import { FieldRequestConfig, JOB_FIELD_TYPES } from '../../../../../common'; +import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; +import { getActions } from '../../../common/components/field_data_row/action_menu'; +import { DataVisualizerGridEmbeddableInput } from './grid_embeddable'; +import { getDefaultPageState } from '../../components/index_data_visualizer_view/index_data_visualizer_view'; + +const defaults = getDefaultPageState(); + +export const useDataVisualizerGridData = ( + input: DataVisualizerGridEmbeddableInput, + dataVisualizerListState: Required +) => { + const { services } = useDataVisualizerKibana(); + const { notifications, uiSettings } = services; + const { toasts } = notifications; + const { samplerShardSize, visibleFieldTypes, showEmptyFields } = dataVisualizerListState; + + const [lastRefresh, setLastRefresh] = useState(0); + + const { + currentSavedSearch, + currentIndexPattern, + currentQuery, + currentFilters, + visibleFieldNames, + } = useMemo( + () => ({ + currentSavedSearch: input?.savedSearch, + currentIndexPattern: input.indexPattern, + currentQuery: input?.query, + visibleFieldNames: input?.visibleFieldNames ?? [], + currentFilters: input?.filters, + }), + [input] + ); + + const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { + const searchData = extractSearchData({ + indexPattern: currentIndexPattern, + uiSettings, + savedSearch: currentSavedSearch, + query: currentQuery, + filters: currentFilters, + }); + + if (searchData === undefined || dataVisualizerListState.searchString !== '') { + return { + searchQuery: dataVisualizerListState.searchQuery, + searchString: dataVisualizerListState.searchString, + searchQueryLanguage: dataVisualizerListState.searchQueryLanguage, + }; + } else { + return { + searchQuery: searchData.searchQuery, + searchString: searchData.searchString, + searchQueryLanguage: searchData.queryLanguage, + }; + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [ + currentSavedSearch, + currentIndexPattern, + dataVisualizerListState, + currentQuery, + currentFilters, + ]); + + const [overallStats, setOverallStats] = useState(defaults.overallStats); + + const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); + const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); + const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); + const [metricsStats, setMetricsStats] = useState(); + + const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); + const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); + + const dataLoader = useMemo( + () => new DataLoader(currentIndexPattern, toasts), + [currentIndexPattern, toasts] + ); + + const timefilter = useTimefilter({ + timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, + autoRefreshSelector: true, + }); + + useEffect(() => { + const timeUpdateSubscription = merge( + timefilter.getTimeUpdate$(), + dataVisualizerRefresh$ + ).subscribe(() => { + setLastRefresh(Date.now()); + }); + return () => { + timeUpdateSubscription.unsubscribe(); + }; + }); + + const getTimeBuckets = useCallback(() => { + return new TimeBuckets({ + [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), + [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), + dateFormat: uiSettings.get('dateFormat'), + 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), + }); + }, [uiSettings]); + + const indexPatternFields: DataViewField[] = useMemo( + () => currentIndexPattern.fields, + [currentIndexPattern] + ); + + async function loadOverallStats() { + const tf = timefilter as any; + let earliest; + let latest; + + const activeBounds = tf.getActiveBounds(); + + if (currentIndexPattern.timeFieldName !== undefined && activeBounds === undefined) { + return; + } + + if (currentIndexPattern.timeFieldName !== undefined) { + earliest = activeBounds.min.valueOf(); + latest = activeBounds.max.valueOf(); + } + + try { + const allStats = await dataLoader.loadOverallData( + searchQuery, + samplerShardSize, + earliest, + latest + ); + // Because load overall stats perform queries in batches + // there could be multiple errors + if (Array.isArray(allStats.errors) && allStats.errors.length > 0) { + allStats.errors.forEach((err: any) => { + dataLoader.displayError(extractErrorProperties(err)); + }); + } + setOverallStats(allStats); + } catch (err) { + dataLoader.displayError(err.body ?? err); + } + } + + const createMetricCards = useCallback(() => { + const configs: FieldVisConfig[] = []; + const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; + + const allMetricFields = indexPatternFields.filter((f) => { + return ( + f.type === KBN_FIELD_TYPES.NUMBER && + f.displayName !== undefined && + dataLoader.isDisplayField(f.displayName) === true + ); + }); + const metricExistsFields = allMetricFields.filter((f) => { + return aggregatableExistsFields.find((existsF) => { + return existsF.fieldName === f.spec.name; + }); + }); + + // Add a config for 'document count', identified by no field name if indexpattern is time based. + if (currentIndexPattern.timeFieldName !== undefined) { + configs.push({ + type: JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + loading: true, + aggregatable: true, + }); + } + + if (metricsLoaded === false) { + setMetricsLoaded(true); + return; + } + + let aggregatableFields: any[] = overallStats.aggregatableExistsFields; + if (allMetricFields.length !== metricExistsFields.length && metricsLoaded === true) { + aggregatableFields = aggregatableFields.concat(overallStats.aggregatableNotExistsFields); + } + + const metricFieldsToShow = + metricsLoaded === true && showEmptyFields === true ? allMetricFields : metricExistsFields; + + metricFieldsToShow.forEach((field) => { + const fieldData = aggregatableFields.find((f) => { + return f.fieldName === field.spec.name; + }); + + const metricConfig: FieldVisConfig = { + ...(fieldData ? fieldData : {}), + fieldFormat: currentIndexPattern.getFormatterForField(field), + type: JOB_FIELD_TYPES.NUMBER, + loading: true, + aggregatable: true, + deletable: field.runtimeField !== undefined, + }; + if (field.displayName !== metricConfig.fieldName) { + metricConfig.displayName = field.displayName; + } + + configs.push(metricConfig); + }); + + setMetricsStats({ + totalMetricFieldsCount: allMetricFields.length, + visibleMetricsCount: metricFieldsToShow.length, + }); + setMetricConfigs(configs); + }, [ + currentIndexPattern, + dataLoader, + indexPatternFields, + metricsLoaded, + overallStats, + showEmptyFields, + ]); + + const createNonMetricCards = useCallback(() => { + const allNonMetricFields = indexPatternFields.filter((f) => { + return ( + f.type !== KBN_FIELD_TYPES.NUMBER && + f.displayName !== undefined && + dataLoader.isDisplayField(f.displayName) === true + ); + }); + // Obtain the list of all non-metric fields which appear in documents + // (aggregatable or not aggregatable). + const populatedNonMetricFields: any[] = []; // Kibana index pattern non metric fields. + let nonMetricFieldData: any[] = []; // Basic non metric field data loaded from requesting overall stats. + const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; + const nonAggregatableExistsFields: any[] = overallStats.nonAggregatableExistsFields || []; + + allNonMetricFields.forEach((f) => { + const checkAggregatableField = aggregatableExistsFields.find( + (existsField) => existsField.fieldName === f.spec.name + ); + + if (checkAggregatableField !== undefined) { + populatedNonMetricFields.push(f); + nonMetricFieldData.push(checkAggregatableField); + } else { + const checkNonAggregatableField = nonAggregatableExistsFields.find( + (existsField) => existsField.fieldName === f.spec.name + ); + + if (checkNonAggregatableField !== undefined) { + populatedNonMetricFields.push(f); + nonMetricFieldData.push(checkNonAggregatableField); + } + } + }); + + if (nonMetricsLoaded === false) { + setNonMetricsLoaded(true); + return; + } + + if (allNonMetricFields.length !== nonMetricFieldData.length && showEmptyFields === true) { + // Combine the field data obtained from Elasticsearch into a single array. + nonMetricFieldData = nonMetricFieldData.concat( + overallStats.aggregatableNotExistsFields, + overallStats.nonAggregatableNotExistsFields + ); + } + + const nonMetricFieldsToShow = showEmptyFields ? allNonMetricFields : populatedNonMetricFields; + + const configs: FieldVisConfig[] = []; + + nonMetricFieldsToShow.forEach((field) => { + const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); + + const nonMetricConfig = { + ...fieldData, + fieldFormat: currentIndexPattern.getFormatterForField(field), + aggregatable: field.aggregatable, + scripted: field.scripted, + loading: fieldData.existsInDocs, + deletable: field.runtimeField !== undefined, + }; + + // Map the field type from the Kibana index pattern to the field type + // used in the data visualizer. + const dataVisualizerType = kbnTypeToJobType(field); + if (dataVisualizerType !== undefined) { + nonMetricConfig.type = dataVisualizerType; + } else { + // Add a flag to indicate that this is one of the 'other' Kibana + // field types that do not yet have a specific card type. + nonMetricConfig.type = field.type; + nonMetricConfig.isUnsupportedType = true; + } + + if (field.displayName !== nonMetricConfig.fieldName) { + nonMetricConfig.displayName = field.displayName; + } + + configs.push(nonMetricConfig); + }); + + setNonMetricConfigs(configs); + }, [ + currentIndexPattern, + dataLoader, + indexPatternFields, + nonMetricsLoaded, + overallStats, + showEmptyFields, + ]); + + async function loadMetricFieldStats() { + // Only request data for fields that exist in documents. + if (metricConfigs.length === 0) { + return; + } + + const configsToLoad = metricConfigs.filter( + (config) => config.existsInDocs === true && config.loading === true + ); + if (configsToLoad.length === 0) { + return; + } + + // Pass the field name, type and cardinality in the request. + // Top values will be obtained on a sample if cardinality > 100000. + const existMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { + const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; + if (config.stats !== undefined && config.stats.cardinality !== undefined) { + props.cardinality = config.stats.cardinality; + } + return props; + }); + + // Obtain the interval to use for date histogram aggregations + // (such as the document count chart). Aim for 75 bars. + const buckets = getTimeBuckets(); + + const tf = timefilter as any; + let earliest: number | undefined; + let latest: number | undefined; + if (currentIndexPattern.timeFieldName !== undefined) { + earliest = tf.getActiveBounds().min.valueOf(); + latest = tf.getActiveBounds().max.valueOf(); + } + + const bounds = tf.getActiveBounds(); + const BAR_TARGET = 75; + buckets.setInterval('auto'); + buckets.setBounds(bounds); + buckets.setBarTarget(BAR_TARGET); + const aggInterval = buckets.getInterval(); + + try { + const metricFieldStats = await dataLoader.loadFieldStats( + searchQuery, + samplerShardSize, + earliest, + latest, + existMetricFields, + aggInterval.asMilliseconds() + ); + + // Add the metric stats to the existing stats in the corresponding config. + const configs: FieldVisConfig[] = []; + metricConfigs.forEach((config) => { + const configWithStats = { ...config }; + if (config.fieldName !== undefined) { + configWithStats.stats = { + ...configWithStats.stats, + ...metricFieldStats.find( + (fieldStats: any) => fieldStats.fieldName === config.fieldName + ), + }; + configWithStats.loading = false; + configs.push(configWithStats); + } else { + // Document count card. + configWithStats.stats = metricFieldStats.find( + (fieldStats: any) => fieldStats.fieldName === undefined + ); + + if (configWithStats.stats !== undefined) { + // Add earliest / latest of timefilter for setting x axis domain. + configWithStats.stats.timeRangeEarliest = earliest; + configWithStats.stats.timeRangeLatest = latest; + } + setDocumentCountStats(configWithStats); + } + }); + + setMetricConfigs(configs); + } catch (err) { + dataLoader.displayError(err); + } + } + + async function loadNonMetricFieldStats() { + // Only request data for fields that exist in documents. + if (nonMetricConfigs.length === 0) { + return; + } + + const configsToLoad = nonMetricConfigs.filter( + (config) => config.existsInDocs === true && config.loading === true + ); + if (configsToLoad.length === 0) { + return; + } + + // Pass the field name, type and cardinality in the request. + // Top values will be obtained on a sample if cardinality > 100000. + const existNonMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { + const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; + if (config.stats !== undefined && config.stats.cardinality !== undefined) { + props.cardinality = config.stats.cardinality; + } + return props; + }); + + const tf = timefilter as any; + let earliest; + let latest; + if (currentIndexPattern.timeFieldName !== undefined) { + earliest = tf.getActiveBounds().min.valueOf(); + latest = tf.getActiveBounds().max.valueOf(); + } + + try { + const nonMetricFieldStats = await dataLoader.loadFieldStats( + searchQuery, + samplerShardSize, + earliest, + latest, + existNonMetricFields + ); + + // Add the field stats to the existing stats in the corresponding config. + const configs: FieldVisConfig[] = []; + nonMetricConfigs.forEach((config) => { + const configWithStats = { ...config }; + if (config.fieldName !== undefined) { + configWithStats.stats = { + ...configWithStats.stats, + ...nonMetricFieldStats.find( + (fieldStats: any) => fieldStats.fieldName === config.fieldName + ), + }; + } + configWithStats.loading = false; + configs.push(configWithStats); + }); + + setNonMetricConfigs(configs); + } catch (err) { + dataLoader.displayError(err); + } + } + + useEffect(() => { + loadOverallStats(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [searchQuery, samplerShardSize, lastRefresh]); + + useEffect(() => { + createMetricCards(); + createNonMetricCards(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [overallStats, showEmptyFields]); + + useEffect(() => { + loadMetricFieldStats(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [metricConfigs]); + + useEffect(() => { + loadNonMetricFieldStats(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [nonMetricConfigs]); + + useEffect(() => { + createMetricCards(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [metricsLoaded]); + + useEffect(() => { + createNonMetricCards(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [nonMetricsLoaded]); + + const configs = useMemo(() => { + let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; + if (visibleFieldTypes && visibleFieldTypes.length > 0) { + combinedConfigs = combinedConfigs.filter( + (config) => visibleFieldTypes.findIndex((field) => field === config.type) > -1 + ); + } + if (visibleFieldNames && visibleFieldNames.length > 0) { + combinedConfigs = combinedConfigs.filter( + (config) => visibleFieldNames.findIndex((field) => field === config.fieldName) > -1 + ); + } + + return combinedConfigs; + }, [nonMetricConfigs, metricConfigs, visibleFieldTypes, visibleFieldNames]); + + // Some actions open up fly-out or popup + // This variable is used to keep track of them and clean up when unmounting + const actionFlyoutRef = useRef<() => void | undefined>(); + useEffect(() => { + const ref = actionFlyoutRef; + return () => { + // Clean up any of the flyout/editor opened from the actions + if (ref.current) { + ref.current(); + } + }; + }, []); + + // Inject custom action column for the index based visualizer + // Hide the column completely if no access to any of the plugins + const extendedColumns = useMemo(() => { + const actions = getActions( + input.indexPattern, + { lens: services.lens }, + { + searchQueryLanguage, + searchString, + }, + actionFlyoutRef + ); + if (!Array.isArray(actions) || actions.length < 1) return; + + const actionColumn: EuiTableActionsColumnType = { + name: i18n.translate('xpack.dataVisualizer.index.dataGrid.actionsColumnLabel', { + defaultMessage: 'Actions', + }), + actions, + width: '70px', + }; + + return [actionColumn]; + }, [input.indexPattern, services, searchQueryLanguage, searchString]); + + return { + configs, + searchQueryLanguage, + searchString, + searchQuery, + extendedColumns, + documentCountStats, + metricsStats, + }; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx index c3f3d744a3978..1374cffb71db5 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx @@ -9,7 +9,6 @@ import React, { FC, useCallback, useEffect, useState } from 'react'; import { useHistory, useLocation } from 'react-router-dom'; import { parse, stringify } from 'query-string'; import { isEqual } from 'lodash'; -// @ts-ignore import { encode } from 'rison-node'; import { SimpleSavedObject } from 'kibana/public'; import { i18n } from '@kbn/i18n'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts index c26a668bd04ab..acc4d3056d8aa 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts @@ -5,7 +5,6 @@ * 2.0. */ -// @ts-ignore import { encode } from 'rison-node'; import { stringify } from 'query-string'; import { SerializableRecord } from '@kbn/utility-types'; From fc32cd985e267256c3890b739b332ae182a7b1fd Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 23 Sep 2021 10:18:22 -0500 Subject: [PATCH 109/188] [ML] Make doc count 0 for empty fields, update t/f test --- .../components/field_data_row/document_stats.tsx | 9 ++++++--- .../components/field_data_row/use_column_chart.test.tsx | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx index ff313b9329eed..203c7da834f65 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx @@ -19,15 +19,18 @@ export const DocumentStat = ({ config, showIcon }: Props) => { if (stats === undefined) return null; const { count, sampleCount } = stats; - if (count === undefined || sampleCount === undefined) return null; - const docsPercent = roundToDecimalPlace((count / sampleCount) * 100); + const docsCount = count ?? 0; + const docsPercent = + count !== undefined && sampleCount !== undefined + ? roundToDecimalPlace((count / sampleCount) * 100) + : 0; return ( <> {showIcon ? : null} - {count} ({docsPercent}%) + {docsCount} ({docsPercent}%) ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.test.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.test.tsx index aff4d6d62c6c8..228719552da9e 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.test.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.test.tsx @@ -122,8 +122,8 @@ describe('getLegendText()', () => { })} ); - expect(getByText('true')).toBeInTheDocument(); - expect(getByText('false')).toBeInTheDocument(); + expect(getByText('t')).toBeInTheDocument(); + expect(getByText('f')).toBeInTheDocument(); }); it('should return the chart legend text for ordinal chart data with less than max categories', () => { expect(getLegendText({ ...validOrdinalChartData, data: [{ key: 'cat', doc_count: 10 }] })).toBe( From 6c91692b1adc5b76d6a471f10ef48ad8d9ea8f1d Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 23 Sep 2021 11:48:21 -0500 Subject: [PATCH 110/188] [ML] Add unit testing for search utils --- .../index_data_visualizer_view.tsx | 8 +- .../components/search_panel/search_panel.tsx | 4 +- .../utils/saved_search_utils.test.ts | 313 ++++++++++++++++++ .../utils/saved_search_utils.ts | 18 +- 4 files changed, 332 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index a0d6c16923bfd..5de44ed308888 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -64,7 +64,7 @@ import { DatePickerWrapper } from '../../../common/components/date_picker_wrappe import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; import { HelpMenu } from '../../../common/components/help_menu'; import { TimeBuckets } from '../../services/time_buckets'; -import { createCombinedQuery, extractSearchData } from '../../utils/saved_search_utils'; +import { createMergedEsQuery, getEsQueryFromSavedSearch } from '../../utils/saved_search_utils'; import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; import { ResultLink } from '../../../common/components/results_links'; import { extractErrorProperties } from '../../utils/error_utils'; @@ -230,7 +230,7 @@ export const IndexDataVisualizerView: FC = (dataVi const defaults = getDefaultPageState(); const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - const searchData = extractSearchData({ + const searchData = getEsQueryFromSavedSearch({ indexPattern: currentIndexPattern, uiSettings, savedSearch: currentSavedSearch, @@ -331,7 +331,7 @@ export const IndexDataVisualizerView: FC = (dataVi language: searchQueryLanguage, }; - const combinedQuery = createCombinedQuery( + const combinedQuery = createMergedEsQuery( { query: searchString || '', language: searchQueryLanguage, @@ -722,7 +722,7 @@ export const IndexDataVisualizerView: FC = (dataVi fieldFormat: currentIndexPattern.getFormatterForField(field), aggregatable: field.aggregatable, scripted: field.scripted, - loading: fieldData.existsInDocs, + loading: fieldData?.existsInDocs ?? false, deletable: field.runtimeField !== undefined, }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx index ea8299a117325..2c1475f9fe834 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx @@ -21,7 +21,7 @@ import { JobFieldType } from '../../../../../common/types'; import { SearchQueryLanguage } from '../../types/combined_query'; import { useDataVisualizerKibana } from '../../../kibana_context'; import './_index.scss'; -import { createCombinedQuery } from '../../utils/saved_search_utils'; +import { createMergedEsQuery } from '../../utils/saved_search_utils'; interface Props { indexPattern: DataView; searchString: Query['query']; @@ -98,7 +98,7 @@ export const SearchPanel: FC = ({ queryManager.filterManager.setFilters(filters); } - const combinedQuery = createCombinedQuery( + const combinedQuery = createMergedEsQuery( mergedQuery, queryManager.filterManager.getFilters() ?? [], indexPattern, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts new file mode 100644 index 0000000000000..432e771d4a42e --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts @@ -0,0 +1,313 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + getQueryFromSavedSearch, + createMergedEsQuery, + getEsQueryFromSavedSearch, +} from './saved_search_utils'; +import type { SavedSearchSavedObject } from '../../../../common'; +import type { SavedSearch } from '../../../../../../../src/plugins/discover/public'; +import type { Filter, FilterStateStore } from '@kbn/es-query'; +import { stubbedSavedObjectIndexPattern } from '../../../../../../../src/plugins/data/common/data_views/data_view.stub'; +import { DataView } from '../../../../../../../src/plugins/data/common'; +import { fieldFormatsMock } from '../../../../../../../src/plugins/field_formats/common/mocks'; +import { uiSettingsServiceMock } from 'src/core/public/mocks'; + +// helper function to create index patterns +function createMockDataView(id: string) { + const { + type, + version, + attributes: { timeFieldName, fields, title }, + } = stubbedSavedObjectIndexPattern(id); + + return new DataView({ + spec: { + id, + type, + version, + timeFieldName, + fields: JSON.parse(fields), + title, + runtimeFieldMap: {}, + }, + fieldFormats: fieldFormatsMock, + shortDotsEnable: false, + metaFields: [], + }); +} + +const mockDataView = createMockDataView('test-mock-data-view'); +const mockUiSettings = uiSettingsServiceMock.createStartContract(); + +// @ts-expect-error We don't need the full object here +const luceneSavedSearchObj: SavedSearchSavedObject = { + attributes: { + title: 'farequote_filter_and_lucene', + columns: ['_source'], + sort: ['@timestamp', 'desc'], + kibanaSavedObjectMeta: { + searchSourceJSON: + '{"highlightAll":true,"version":true,"query":{"query":"responsetime:>50","language":"lucene"},"filter":[{"meta":{"index":"90a978e0-1c80-11ec-b1d7-f7e5cf21b9e0","negate":false,"disabled":false,"alias":null,"type":"phrase","key":"airline","value":"ASA","params":{"query":"ASA","type":"phrase"}},"query":{"match":{"airline":{"query":"ASA","type":"phrase"}}},"$state":{"store":"appState"}}],"indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index"}', + }, + }, + id: '93fc4d60-1c80-11ec-b1d7-f7e5cf21b9e0', + type: 'search', +}; + +// @ts-expect-error We don't need the full object here +const luceneInvalidSavedSearchObj: SavedSearchSavedObject = { + attributes: { + kibanaSavedObjectMeta: { + searchSourceJSON: null, + }, + }, + id: '93fc4d60-1c80-11ec-b1d7-f7e5cf21b9e0', + type: 'search', +}; + +const kqlSavedSearch: SavedSearch = { + title: 'farequote_filter_and_kuery', + description: '', + columns: ['_source'], + // @ts-expect-error We don't need the full object here + kibanaSavedObjectMeta: { + searchSourceJSON: + '{"highlightAll":true,"version":true,"query":{"query":"responsetime > 49","language":"kuery"},"filter":[{"meta":{"index":"90a978e0-1c80-11ec-b1d7-f7e5cf21b9e0","negate":false,"disabled":false,"alias":null,"type":"phrase","key":"airline","value":"ASA","params":{"query":"ASA","type":"phrase"}},"query":{"match":{"airline":{"query":"ASA","type":"phrase"}}},"$state":{"store":"appState"}}],"indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index"}', + }, +}; + +describe('getQueryFromSavedSearch()', () => { + it('should return parsed searchSourceJSON with query and filter', () => { + expect(getQueryFromSavedSearch(luceneSavedSearchObj)).toEqual({ + filter: [ + { + $state: { store: 'appState' }, + meta: { + alias: null, + disabled: false, + index: '90a978e0-1c80-11ec-b1d7-f7e5cf21b9e0', + key: 'airline', + negate: false, + params: { query: 'ASA', type: 'phrase' }, + type: 'phrase', + value: 'ASA', + }, + query: { match: { airline: { query: 'ASA', type: 'phrase' } } }, + }, + ], + highlightAll: true, + indexRefName: 'kibanaSavedObjectMeta.searchSourceJSON.index', + query: { language: 'lucene', query: 'responsetime:>50' }, + version: true, + }); + expect(getQueryFromSavedSearch(kqlSavedSearch)).toEqual({ + filter: [ + { + $state: { store: 'appState' }, + meta: { + alias: null, + disabled: false, + index: '90a978e0-1c80-11ec-b1d7-f7e5cf21b9e0', + key: 'airline', + negate: false, + params: { query: 'ASA', type: 'phrase' }, + type: 'phrase', + value: 'ASA', + }, + query: { match: { airline: { query: 'ASA', type: 'phrase' } } }, + }, + ], + highlightAll: true, + indexRefName: 'kibanaSavedObjectMeta.searchSourceJSON.index', + query: { language: 'kuery', query: 'responsetime > 49' }, + version: true, + }); + }); + it('should return undefined if invalid searchSourceJSON', () => { + expect(getQueryFromSavedSearch(luceneInvalidSavedSearchObj)).toEqual(undefined); + }); +}); + +describe('createMergedEsQuery()', () => { + const luceneQuery = { + query: 'responsetime:>50', + language: 'lucene', + }; + const kqlQuery = { + query: 'responsetime > 49', + language: 'kuery', + }; + const mockFilters: Filter[] = [ + { + meta: { + index: '90a978e0-1c80-11ec-b1d7-f7e5cf21b9e0', + negate: false, + disabled: false, + alias: null, + type: 'phrase', + key: 'airline', + params: { + query: 'ASA', + }, + }, + query: { + match: { + airline: { + query: 'ASA', + type: 'phrase', + }, + }, + }, + $state: { + store: 'appState' as FilterStateStore, + }, + }, + ]; + + it('return formatted ES bool query with both the original query and filters combined', () => { + expect(createMergedEsQuery(luceneQuery, mockFilters)).toEqual({ + bool: { + filter: [{ match_phrase: { airline: { query: 'ASA' } } }], + must: [{ query_string: { query: 'responsetime:>50' } }], + must_not: [], + should: [], + }, + }); + expect(createMergedEsQuery(kqlQuery, mockFilters)).toEqual({ + bool: { + filter: [{ match_phrase: { airline: { query: 'ASA' } } }], + minimum_should_match: 1, + must_not: [], + should: [{ range: { responsetime: { gt: '49' } } }], + }, + }); + }); + it('return formatted ES bool query without filters ', () => { + expect(createMergedEsQuery(luceneQuery)).toEqual({ + bool: { + filter: [], + must: [{ query_string: { query: 'responsetime:>50' } }], + must_not: [], + should: [], + }, + }); + expect(createMergedEsQuery(kqlQuery)).toEqual({ + bool: { + filter: [], + minimum_should_match: 1, + must_not: [], + should: [{ range: { responsetime: { gt: '49' } } }], + }, + }); + }); +}); + +describe('getEsQueryFromSavedSearch()', () => { + it('return undefined if saved search is not provided', () => { + expect( + getEsQueryFromSavedSearch({ + indexPattern: mockDataView, + savedSearch: undefined, + uiSettings: mockUiSettings, + }) + ).toEqual(undefined); + }); + it('return search data from saved search if neither query nor filter is provided ', () => { + expect( + getEsQueryFromSavedSearch({ + indexPattern: mockDataView, + savedSearch: luceneSavedSearchObj, + uiSettings: mockUiSettings, + }) + ).toEqual({ + queryLanguage: 'lucene', + searchQuery: { + bool: { + filter: [{ match_phrase: { airline: { query: 'ASA' } } }], + must: [{ query_string: { query: 'responsetime:>50' } }], + must_not: [], + should: [], + }, + }, + searchString: 'responsetime:>50', + }); + }); + it('should override original saved search with the provided query ', () => { + expect( + getEsQueryFromSavedSearch({ + indexPattern: mockDataView, + savedSearch: luceneSavedSearchObj, + uiSettings: mockUiSettings, + query: { + query: 'responsetime:>100', + language: 'lucene', + }, + }) + ).toEqual({ + queryLanguage: 'lucene', + searchQuery: { + bool: { + filter: [{ match_phrase: { airline: { query: 'ASA' } } }], + must: [{ query_string: { query: 'responsetime:>100' } }], + must_not: [], + should: [], + }, + }, + searchString: 'responsetime:>100', + }); + }); + + it('should override original saved search with the provided filters ', () => { + expect( + getEsQueryFromSavedSearch({ + indexPattern: mockDataView, + savedSearch: luceneSavedSearchObj, + uiSettings: mockUiSettings, + query: { + query: 'responsetime:>100', + language: 'lucene', + }, + filters: [ + { + meta: { + index: '90a978e0-1c80-11ec-b1d7-f7e5cf21b9e0', + alias: null, + negate: true, + disabled: false, + type: 'phrase', + key: 'airline', + params: { + query: 'JZA', + }, + }, + query: { + match_phrase: { + airline: 'JZA', + }, + }, + $state: { + store: 'appState' as FilterStateStore, + }, + }, + ], + }) + ).toEqual({ + queryLanguage: 'lucene', + searchQuery: { + bool: { + filter: [], + must: [{ query_string: { query: 'responsetime:>100' } }], + must_not: [{ match_phrase: { airline: 'JZA' } }], + should: [], + }, + }, + searchString: 'responsetime:>100', + }); + }); +}); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index 01cd8a766b409..bba7f87cf9696 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -22,6 +22,10 @@ import { SavedSearch } from '../../../../../../../src/plugins/discover/public'; import { getEsQueryConfig } from '../../../../../../../src/plugins/data/common'; import { FilterManager } from '../../../../../../../src/plugins/data/public'; +/** + * Parse the stringified searchSourceJSON + * from a saved search or saved search object + */ export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject | SavedSearch) { const search = isSavedSearchSavedObject(savedSearch) ? savedSearch?.attributes?.kibanaSavedObjectMeta @@ -36,7 +40,11 @@ export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject | Sa : undefined; } -export function createCombinedQuery( +/** + * Create an Elasticsearch query that combines both lucene/kql query string and filters + * Should also form a valid query if only the query or filters is provided + */ +export function createMergedEsQuery( query?: Query, filters?: Filter[], indexPattern?: DataView, @@ -76,9 +84,9 @@ export function createCombinedQuery( /** * Extract query data from the saved search object - * and merge with query data and filters + * with overrides from the provided query data and/or filters */ -export function extractSearchData({ +export function getEsQueryFromSavedSearch({ indexPattern, uiSettings, savedSearch, @@ -101,7 +109,7 @@ export function extractSearchData({ // If no saved search available, use user's query and filters if (!savedSearchData && userQuery) { - const combinedQuery = createCombinedQuery( + const combinedQuery = createMergedEsQuery( userQuery, Array.isArray(userFilters) ? userFilters : [], indexPattern, @@ -122,7 +130,7 @@ export function extractSearchData({ if (filterManager) filterManager.setFilters(currentFilters); - const combinedQuery = createCombinedQuery( + const combinedQuery = createMergedEsQuery( currentQuery, Array.isArray(currentFilters) ? currentFilters : [], indexPattern, From 5f2a22803133b5cb7def597f237c41a0324f27d3 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 23 Sep 2021 12:22:22 -0500 Subject: [PATCH 111/188] Fix missing unsubscribe for embeddable output --- .../data_visualizer_grid/data_visualizer_grid.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index 101b4c2b9cdc6..3cae73fd466f1 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -81,11 +81,15 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp ); useEffect(() => { - embeddable?.getOutput$().subscribe((output: DataVisualizerGridEmbeddableOutput) => { + const sub = embeddable?.getOutput$().subscribe((output: DataVisualizerGridEmbeddableOutput) => { if (output.showDistributions !== undefined && stateContainer) { stateContainer.setAppState({ hideAggregatedPreview: !output.showDistributions }); } }); + + return () => { + sub?.unsubscribe(); + }; }, [embeddable, stateContainer]); useEffect(() => { From 747aa407bf2ed70868a44a80b31f8ce4ea74a446 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 23 Sep 2021 12:27:16 -0500 Subject: [PATCH 112/188] Remove redundant onAddFilter for this PR, fix width --- .../common/components/stats_table/_index.scss | 4 +- .../index_data_visualizer_view.tsx | 52 +------------------ 2 files changed, 3 insertions(+), 53 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss index 8e120140e0e0d..0cca157b76672 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/_index.scss @@ -7,9 +7,7 @@ $panelWidthL: #{'max(40%, 450px)'}; .dataVisualizerFieldExpandedRow { padding-left: $euiSize * 4; - // Width is subtracting the left padding - // to ensure the content wraps correctly - width: calc(100% - #{$euiSize * 2}); + width: 100%; .fieldDataCard__valuesTitle { text-transform: uppercase; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 86c616ea07bbf..4fc7598ad568b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -27,7 +27,6 @@ import { KBN_FIELD_TYPES, UI_SETTINGS, Query, - generateFilters, } from '../../../../../../../../src/plugins/data/public'; import { FullTimeRangeSelector } from '../full_time_range_selector'; import { usePageUrlState, useUrlState } from '../../../common/util/url_state'; @@ -64,7 +63,7 @@ import { DatePickerWrapper } from '../../../common/components/date_picker_wrappe import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; import { HelpMenu } from '../../../common/components/help_menu'; import { TimeBuckets } from '../../services/time_buckets'; -import { createMergedEsQuery, getEsQueryFromSavedSearch } from '../../utils/saved_search_utils'; +import { getEsQueryFromSavedSearch } from '../../utils/saved_search_utils'; import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; import { ResultLink } from '../../../common/components/results_links'; import { extractErrorProperties } from '../../utils/error_utils'; @@ -312,51 +311,6 @@ export const IndexDataVisualizerView: FC = (dataVi const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); - const onAddFilter = useCallback( - (field: DataViewField | string, values: string, operation: '+' | '-') => { - const newFilters = generateFilters( - data.query.filterManager, - field, - values, - operation, - String(currentIndexPattern.id) - ); - if (newFilters) { - data.query.filterManager.addFilters(newFilters); - } - - // Merge current query with new filters - const mergedQuery = { - query: searchString || '', - language: searchQueryLanguage, - }; - - const combinedQuery = createMergedEsQuery( - { - query: searchString || '', - language: searchQueryLanguage, - }, - data.query.filterManager.getFilters() ?? [], - currentIndexPattern, - uiSettings - ); - - setSearchParams({ - searchQuery: combinedQuery, - searchString: mergedQuery.query, - queryLanguage: mergedQuery.language as SearchQueryLanguage, - }); - }, - [ - currentIndexPattern, - data.query.filterManager, - searchQueryLanguage, - searchString, - setSearchParams, - uiSettings, - ] - ); - useEffect(() => { const timeUpdateSubscription = merge( timefilter.getTimeUpdate$(), @@ -803,14 +757,13 @@ export const IndexDataVisualizerView: FC = (dataVi item={item} indexPattern={currentIndexPattern} combinedQuery={{ searchQueryLanguage, searchString }} - onAddFilter={onAddFilter} /> ); } return m; }, {} as ItemIdToExpandedRowMap); }, - [currentIndexPattern, searchQueryLanguage, searchString, onAddFilter] + [currentIndexPattern, searchQueryLanguage, searchString] ); // Some actions open up fly-out or popup @@ -926,7 +879,6 @@ export const IndexDataVisualizerView: FC = (dataVi visibleFieldNames={visibleFieldNames} setVisibleFieldNames={setVisibleFieldNames} showEmptyFields={showEmptyFields} - onAddFilter={onAddFilter} /> Date: Thu, 23 Sep 2021 12:37:27 -0500 Subject: [PATCH 113/188] Rename Field Stats to Field stats to match convention --- .../apps/main/components/view_mode_toggle/view_mode_toggle.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx index e73a9783f503b..0003083ea650b 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx @@ -34,7 +34,7 @@ export const DocumentViewModeToggle = ({
Date: Thu, 23 Sep 2021 13:12:24 -0500 Subject: [PATCH 114/188] [ML] Fix expand all/collapse all behavior to override individual setting --- .../data_visualizer_stats_table.tsx | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 499ee74adab2b..92618149d4ad1 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -66,7 +66,7 @@ export const DataVisualizerTable = ({ onChange, }: DataVisualizerTableProps) => { const [expandedRowItemIds, setExpandedRowItemIds] = useState([]); - const [expandAll, toggleExpandAll] = useState(false); + const [expandAll, setExpandAll] = useState(false); const { onTableChange, pagination, sorting } = useTableSettings( items, @@ -75,9 +75,22 @@ export const DataVisualizerTable = ({ ); const [showDistributions, setShowDistributions] = useState(showPreviewByDefault ?? true); const [dimensions, setDimensions] = useState(calculateTableColumnsDimensions()); - const [tableWidth, setTableWidth] = useState(1400); + const toggleExpandAll = useCallback( + (shouldExpandAll: boolean) => { + setExpandedRowItemIds( + shouldExpandAll + ? // Update list of ids in expandedRowIds to include all + (items.map((item) => item.fieldName).filter((id) => id !== undefined) as string[]) + : // Otherwise, reset list of ids in expandedRowIds + [] + ); + setExpandAll(shouldExpandAll); + }, + [items] + ); + // eslint-disable-next-line react-hooks/exhaustive-deps const resizeHandler = useCallback( throttle((e: { width: number; height: number }) => { @@ -290,16 +303,19 @@ export const DataVisualizerTable = ({ ]; return extendedColumns ? [...baseColumns, ...extendedColumns] : baseColumns; // eslint-disable-next-line react-hooks/exhaustive-deps - }, [expandAll, showDistributions, updatePageState, extendedColumns, dimensions.breakPoint]); + }, [ + expandAll, + showDistributions, + updatePageState, + extendedColumns, + dimensions.breakPoint, + toggleExpandAll, + ]); const itemIdToExpandedRowMap = useMemo(() => { - let itemIds = expandedRowItemIds; - if (expandAll) { - itemIds = items.map((i) => i[FIELD_NAME]).filter((f) => f !== undefined) as string[]; - } + const itemIds = expandedRowItemIds; return getItemIdToExpandedRowMap(itemIds, items); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [expandAll, items, expandedRowItemIds]); + }, [items, expandedRowItemIds, getItemIdToExpandedRowMap]); return ( From fb9ec3263a0ce234e4d44a660901a08fc19c3fc0 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 23 Sep 2021 16:14:58 -0500 Subject: [PATCH 115/188] [ML] Fix functional tests should be 0/0% --- .../apps/ml/data_visualizer/index_data_visualizer.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts index 031074876f39c..56a9547f4c664 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts @@ -92,7 +92,7 @@ export default function ({ getService }: FtrProviderContext) { aggregatable: false, loading: false, exampleCount: 1, - docCountFormatted: '', + docCountFormatted: '0 (0%)', viewableInLens: false, }, { @@ -122,7 +122,7 @@ export default function ({ getService }: FtrProviderContext) { aggregatable: false, loading: false, exampleCount: 1, - docCountFormatted: '', + docCountFormatted: '0 (0%)', viewableInLens: false, }, { @@ -188,7 +188,7 @@ export default function ({ getService }: FtrProviderContext) { aggregatable: false, loading: false, exampleCount: 1, - docCountFormatted: '', + docCountFormatted: '0 (0%)', viewableInLens: false, }, { @@ -218,7 +218,7 @@ export default function ({ getService }: FtrProviderContext) { aggregatable: false, loading: false, exampleCount: 1, - docCountFormatted: '', + docCountFormatted: '0 (0%)', viewableInLens: false, }, { @@ -284,7 +284,7 @@ export default function ({ getService }: FtrProviderContext) { aggregatable: false, loading: false, exampleCount: 1, - docCountFormatted: '', + docCountFormatted: '0 (0%)', viewableInLens: false, }, { @@ -314,7 +314,7 @@ export default function ({ getService }: FtrProviderContext) { aggregatable: false, loading: false, exampleCount: 1, - docCountFormatted: '', + docCountFormatted: '0 (0%)', viewableInLens: false, }, { From c591dc207aff721715078d290af7a2a010b788b4 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 28 Sep 2021 10:49:48 -0500 Subject: [PATCH 116/188] [ML] Fix docs content spacing, rename classnames, add filters to Discover, lens, and maps --- .../common/components/embedded_map/embedded_map.tsx | 4 ++-- .../geo_point_content_with_map.tsx | 5 +++-- .../components/field_data_row/action_menu/actions.ts | 5 +++-- .../field_data_row/action_menu/lens_utils.ts | 5 ++++- .../not_in_docs_content/not_in_docs_context.tsx | 3 --- .../components/stats_table/_field_data_row.scss | 8 ++++---- .../common/components/stats_table/_index.scss | 2 +- .../expanded_row_field_header.tsx | 2 +- .../field_data_expanded_row/choropleth_map.tsx | 2 +- .../field_data_expanded_row/other_content.tsx | 11 ++++------- .../field_data_expanded_row/text_content.tsx | 10 +++++----- .../components/field_data_row/document_stats.tsx | 5 ++++- .../stats_table/data_visualizer_stats_table.tsx | 6 +++--- .../common/components/stats_table/utils.ts | 10 ---------- .../components/actions_panel/actions_panel.tsx | 5 +++++ .../utils/saved_search_utils.test.ts | 2 +- 16 files changed, 41 insertions(+), 44 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/embedded_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/embedded_map.tsx index 13aab06640bd5..0d17918f76f4f 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/embedded_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/embedded_map.tsx @@ -39,7 +39,7 @@ export function EmbeddedMapComponent({ const baseLayers = useRef(); const { - services: { embeddable: embeddablePlugin, maps: mapsPlugin }, + services: { embeddable: embeddablePlugin, maps: mapsPlugin, data }, } = useDataVisualizerKibana(); const factory: @@ -73,7 +73,7 @@ export function EmbeddedMapComponent({ const input: MapEmbeddableInput = { id: htmlIdGenerator()(), attributes: { title: '' }, - filters: [], + filters: data.query.filterManager.getFilters() ?? [], hidePanelTitles: true, viewMode: ViewMode.VIEW, isLayerTOCOpen: false, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx index 89b55a66c71fe..e0047c16fd2f5 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx @@ -26,7 +26,7 @@ export const GeoPointContentWithMap: FC<{ const { stats } = config; const [layerList, setLayerList] = useState([]); const { - services: { maps: mapsPlugin }, + services: { maps: mapsPlugin, data }, } = useDataVisualizerKibana(); // Update the layer list with updated geo points upon refresh @@ -42,6 +42,7 @@ export const GeoPointContentWithMap: FC<{ indexPatternId: indexPattern.id, geoFieldName: config.fieldName, geoFieldType: config.type as ES_GEO_FIELD_TYPE, + filters: data.query.filterManager.getFilters() ?? [], query: { query: combinedQuery.searchString, language: combinedQuery.searchQueryLanguage, @@ -57,7 +58,7 @@ export const GeoPointContentWithMap: FC<{ } updateIndexPatternSearchLayer(); // eslint-disable-next-line react-hooks/exhaustive-deps - }, [indexPattern, combinedQuery, config, mapsPlugin]); + }, [indexPattern, combinedQuery, config, mapsPlugin, data.query]); if (stats?.examples === undefined) return null; return ( diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/action_menu/actions.ts b/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/action_menu/actions.ts index 91509823dbf4f..058516839062a 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/action_menu/actions.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/action_menu/actions.ts @@ -24,9 +24,10 @@ export function getActions( combinedQuery: CombinedQuery, actionFlyoutRef: MutableRefObject<(() => void | undefined) | undefined> ): Array> { - const { lens: lensPlugin } = services; + const { lens: lensPlugin, data } = services; const actions: Array> = []; + const filters = data?.query.filterManager.getFilters() ?? []; const refreshPage = () => { const refresh: Refresh = { @@ -49,7 +50,7 @@ export function getActions( available: (item: FieldVisConfig) => getCompatibleLensDataType(item.type) !== undefined && canUseLensEditor, onClick: (item: FieldVisConfig) => { - const lensAttributes = getLensAttributes(indexPattern, combinedQuery, item); + const lensAttributes = getLensAttributes(indexPattern, combinedQuery, filters, item); if (lensAttributes) { lensPlugin.navigateToPrefilledEditor({ id: `dataVisualizer-${item.fieldName}`, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/action_menu/lens_utils.ts b/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/action_menu/lens_utils.ts index 3f80bbefcc259..615ba84afb5b7 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/action_menu/lens_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_data_row/action_menu/lens_utils.ts @@ -6,6 +6,7 @@ */ import { i18n } from '@kbn/i18n'; +import type { Filter } from '@kbn/es-query'; import type { IndexPattern } from '../../../../../../../../../src/plugins/data/common'; import type { CombinedQuery } from '../../../../index_data_visualizer/types/combined_query'; import type { @@ -15,6 +16,7 @@ import type { } from '../../../../../../../lens/public'; import { FieldVisConfig } from '../../stats_table/types'; import { JOB_FIELD_TYPES } from '../../../../../../common'; + interface ColumnsAndLayer { columns: Record; layer: XYLayerConfig; @@ -241,6 +243,7 @@ function getColumnsAndLayer( export function getLensAttributes( defaultIndexPattern: IndexPattern | undefined, combinedQuery: CombinedQuery, + filters: Filter[], item: FieldVisConfig ): TypedLensByValueInput['attributes'] | undefined { if (defaultIndexPattern === undefined || item.type === undefined || item.fieldName === undefined) @@ -279,7 +282,7 @@ export function getLensAttributes( }, }, }, - filters: [], + filters, query: { language: combinedQuery.searchQueryLanguage, query: combinedQuery.searchString }, visualization: { axisTitlesVisibilitySettings: { x: true, yLeft: true, yRight: true }, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/not_in_docs_content/not_in_docs_context.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/not_in_docs_content/not_in_docs_context.tsx index e4fd3b96405df..2baa52b0baf0a 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/not_in_docs_content/not_in_docs_context.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/not_in_docs_content/not_in_docs_context.tsx @@ -12,12 +12,9 @@ import { FormattedMessage } from '@kbn/i18n/react'; export const NotInDocsContent: FC = () => ( - - - {children} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx index d3188569044a0..9ea197d72ef56 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx @@ -107,7 +107,7 @@ export const ChoroplethMap: FC = ({ stats, suggestion }) => { return (
diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/other_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/other_content.tsx index 38b22925ea5e2..4307da33523ed 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/other_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/other_content.tsx @@ -10,19 +10,16 @@ import type { FieldDataRowProps } from '../../types/field_data_row'; import { ExamplesList } from '../../../examples_list'; import { DocumentStatsTable } from './document_stats'; import { ExpandedRowContent } from './expanded_row_content'; -import { ExpandedRowPanel } from './expanded_row_panel'; export const OtherContent: FC = ({ config }) => { const { stats } = config; if (stats === undefined) return null; - return ( + return stats.count === undefined ? ( + <>{Array.isArray(stats.examples) && } + ) : ( - {Array.isArray(stats.examples) && ( - - - - )} + {Array.isArray(stats.examples) && } ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx index fd939844218fe..a45a93d032cde 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx @@ -44,7 +44,7 @@ export const TextContent: FC = ({ config }) => { id="xpack.dataVisualizer.dataGrid.fieldText.fieldNotPresentDescription" defaultMessage="This field was not present in the {sourceParam} field of documents queried." values={{ - sourceParam: _source, + sourceParam: _source, }} /> @@ -54,10 +54,10 @@ export const TextContent: FC = ({ config }) => { id="xpack.dataVisualizer.dataGrid.fieldText.fieldMayBePopulatedDescription" defaultMessage="It may be populated, for example, using a {copyToParam} parameter in the document mapping, or be pruned from the {sourceParam} field after indexing through the use of {includesParam} and {excludesParam} parameters." values={{ - copyToParam: copy_to, - sourceParam: _source, - includesParam: includes, - excludesParam: excludes, + copyToParam: copy_to, + sourceParam: _source, + includesParam: includes, + excludesParam: excludes, }} /> diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx index 203c7da834f65..e5249debe1cf6 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx @@ -10,13 +10,16 @@ import { EuiIcon, EuiText } from '@elastic/eui'; import React from 'react'; import type { FieldDataRowProps } from '../../types/field_data_row'; import { roundToDecimalPlace } from '../../../utils'; +import { isIndexBasedFieldVisConfig } from '../../types'; interface Props extends FieldDataRowProps { showIcon?: boolean; } export const DocumentStat = ({ config, showIcon }: Props) => { const { stats } = config; - if (stats === undefined) return null; + + if (stats === undefined || (isIndexBasedFieldVisConfig(config) && config.aggregatable === false)) + return null; const { count, sampleCount } = stats; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 92618149d4ad1..99c253e8aeaaf 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -36,9 +36,9 @@ import { } from './types/field_vis_config'; import { FileBasedNumberContentPreview } from '../field_data_row'; import { BooleanContentPreview } from './components/field_data_row'; -import { FieldIcon } from '../../../../../../../../src/plugins/kibana_react/public'; -import { calculateTableColumnsDimensions, getKibanaFieldType } from './utils'; +import { calculateTableColumnsDimensions } from './utils'; import { DistinctValues } from './components/field_data_row/distinct_values'; +import { FieldTypeIcon } from '../field_type_icon'; const FIELD_NAME = 'fieldName'; @@ -180,7 +180,7 @@ export const DataVisualizerTable = ({ defaultMessage: 'Type', }), render: (fieldType: JobFieldType) => { - return ; + return ; }, width: dimensions.type, sortable: true, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts index 0eeb3a3ae31d1..d30a33a96c590 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/utils.ts @@ -79,13 +79,3 @@ export const calculateTableColumnsDimensions = (width?: number) => { return defaultSettings; } }; - -export const getKibanaFieldType = (fieldType: string) => { - switch (fieldType) { - case 'text': - case 'keyword': - return 'string'; - default: - return fieldType; - } -}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/actions_panel/actions_panel.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/actions_panel/actions_panel.tsx index bc68bdf4b6ce0..186d3ef840c21 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/actions_panel/actions_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/actions_panel/actions_panel.tsx @@ -40,6 +40,7 @@ export const ActionsPanel: FC = ({ const { services: { + data, application: { capabilities }, share: { urlGenerators: { getUrlGenerator }, @@ -60,6 +61,9 @@ export const ActionsPanel: FC = ({ const state: DiscoverUrlGeneratorState = { indexPatternId, }; + + state.filters = data.query.filterManager.getFilters() ?? []; + if (searchString && searchQueryLanguage !== undefined) { state.query = { query: searchString, language: searchQueryLanguage }; } @@ -113,6 +117,7 @@ export const ActionsPanel: FC = ({ capabilities, getUrlGenerator, additionalLinks, + data.query, ]); // Note we use display:none for the DataRecognizer section as it needs to be diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts index 432e771d4a42e..696be7a74479d 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts @@ -18,7 +18,7 @@ import { DataView } from '../../../../../../../src/plugins/data/common'; import { fieldFormatsMock } from '../../../../../../../src/plugins/field_formats/common/mocks'; import { uiSettingsServiceMock } from 'src/core/public/mocks'; -// helper function to create index patterns +// helper function to create data views function createMockDataView(id: string) { const { type, From 58e68b948988ac3672108b7545cc9e9ccdf33cbf Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 28 Sep 2021 14:30:27 -0500 Subject: [PATCH 117/188] [ML] Fix doc count for fields that exists but have no stats --- .../field_data_row/document_stats.tsx | 18 +++++++++--------- .../index_data_visualizer_view.tsx | 4 ++-- .../use_data_visualizer_grid_data.ts | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx index e5249debe1cf6..aab75611267b7 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx @@ -17,24 +17,24 @@ interface Props extends FieldDataRowProps { } export const DocumentStat = ({ config, showIcon }: Props) => { const { stats } = config; - - if (stats === undefined || (isIndexBasedFieldVisConfig(config) && config.aggregatable === false)) - return null; - + if (stats === undefined) return null; const { count, sampleCount } = stats; - const docsCount = count ?? 0; + // If field exists is docs but we don't have count stats then don't show + // Otherwise if field doesn't appear in docs at all, show 0% + const docsCount = + count ?? (isIndexBasedFieldVisConfig(config) && config.existsInDocs === true ? undefined : 0); const docsPercent = - count !== undefined && sampleCount !== undefined - ? roundToDecimalPlace((count / sampleCount) * 100) + docsCount !== undefined && sampleCount !== undefined + ? roundToDecimalPlace((docsCount / sampleCount) * 100) : 0; - return ( + return docsCount !== undefined ? ( <> {showIcon ? : null} {docsCount} ({docsPercent}%) - ); + ) : null; }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 4fc7598ad568b..f63cd0a238f35 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -672,11 +672,11 @@ export const IndexDataVisualizerView: FC = (dataVi const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); const nonMetricConfig = { - ...fieldData, + ...(fieldData ? fieldData : {}), fieldFormat: currentIndexPattern.getFormatterForField(field), aggregatable: field.aggregatable, scripted: field.scripted, - loading: fieldData.existsInDocs, + loading: fieldData?.existsInDocs, deletable: field.runtimeField !== undefined, }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts index 3eb32ba695882..0c57a1bf25b6a 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts @@ -304,11 +304,11 @@ export const useDataVisualizerGridData = ( const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); const nonMetricConfig = { - ...fieldData, + ...(fieldData ? fieldData : {}), fieldFormat: currentIndexPattern.getFormatterForField(field), aggregatable: field.aggregatable, scripted: field.scripted, - loading: fieldData.existsInDocs, + loading: fieldData?.existsInDocs, deletable: field.runtimeField !== undefined, }; From e92dc2a5c541902be33f8b2b790e3837b3b6a9f2 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 28 Sep 2021 11:26:11 -0500 Subject: [PATCH 118/188] [ML] Fix icon styling to match Discover but have text/keyword/histogram --- .../field_type_icon/field_type_icon.tsx | 88 +++++++------------ .../field_types_filter/field_types_filter.tsx | 7 +- .../not_in_docs_context.tsx | 2 +- .../search_panel/field_type_filter.tsx | 9 +- .../components/search_panel/search_panel.tsx | 4 +- 5 files changed, 36 insertions(+), 74 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx index ee4b4f8171d7d..719b5a263afb9 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx @@ -6,86 +6,59 @@ */ import React, { FC } from 'react'; - import { EuiToken, EuiToolTip } from '@elastic/eui'; - import { i18n } from '@kbn/i18n'; - import { getJobTypeAriaLabel } from '../../util/field_types_utils'; -import { JOB_FIELD_TYPES } from '../../../../../common'; import type { JobFieldType } from '../../../../../common'; interface FieldTypeIconProps { tooltipEnabled: boolean; type: JobFieldType; - fieldName?: string; needsAria: boolean; } interface FieldTypeIconContainerProps { ariaLabel: string | null; iconType: string; - color: string; + color?: string; needsAria: boolean; [key: string]: any; } +// defaultIcon => a unknown datatype +const defaultIcon = { iconType: 'questionInCircle', color: 'gray' }; + +// Extended & modified version of src/plugins/kibana_react/public/field_icon/field_icon.tsx +export const typeToEuiIconMap: Record = { + boolean: { iconType: 'tokenBoolean' }, + // icon for an index pattern mapping conflict in discover + conflict: { iconType: 'alert', color: 'euiColorVis9' }, + date: { iconType: 'tokenDate' }, + date_range: { iconType: 'tokenDate' }, + geo_point: { iconType: 'tokenGeo' }, + geo_shape: { iconType: 'tokenGeo' }, + ip: { iconType: 'tokenIP' }, + ip_range: { iconType: 'tokenIP' }, + // is a plugin's data type https://www.elastic.co/guide/en/elasticsearch/plugins/current/mapper-murmur3-usage.html + murmur3: { iconType: 'tokenFile' }, + number: { iconType: 'tokenNumber', color: 'euiColorVis0' }, + number_range: { iconType: 'tokenNumber', color: 'euiColorVis0' }, + histogram: { iconType: 'tokenHistogram', color: 'euiColorVis0' }, + _source: { iconType: 'editorCodeBlock', color: 'gray' }, + string: { iconType: 'tokenString' }, + text: { iconType: 'tokenString', color: 'euiColorVis1' }, + keyword: { iconType: 'tokenText', color: 'euiColorVis1' }, + nested: { iconType: 'tokenNested' }, +}; + export const FieldTypeIcon: FC = ({ tooltipEnabled = false, type, - fieldName, needsAria = true, }) => { const ariaLabel = getJobTypeAriaLabel(type); - - let iconType = 'questionInCircle'; - let color = 'euiColorVis6'; - - switch (type) { - // Set icon types and colors - case JOB_FIELD_TYPES.BOOLEAN: - iconType = 'tokenBoolean'; - color = 'euiColorVis5'; - break; - case JOB_FIELD_TYPES.DATE: - iconType = 'tokenDate'; - color = 'euiColorVis7'; - break; - case JOB_FIELD_TYPES.GEO_POINT: - case JOB_FIELD_TYPES.GEO_SHAPE: - iconType = 'tokenGeo'; - color = 'euiColorVis8'; - break; - case JOB_FIELD_TYPES.TEXT: - iconType = 'document'; - color = 'euiColorVis9'; - break; - case JOB_FIELD_TYPES.IP: - iconType = 'tokenIP'; - color = 'euiColorVis3'; - break; - case JOB_FIELD_TYPES.KEYWORD: - iconType = 'tokenText'; - color = 'euiColorVis0'; - break; - case JOB_FIELD_TYPES.NUMBER: - iconType = 'tokenNumber'; - color = fieldName !== undefined ? 'euiColorVis1' : 'euiColorVis2'; - break; - case JOB_FIELD_TYPES.HISTOGRAM: - iconType = 'tokenHistogram'; - color = 'euiColorVis7'; - case JOB_FIELD_TYPES.UNKNOWN: - // Use defaults - break; - } - - const containerProps = { - ariaLabel, - iconType, - color, - needsAria, - }; + const token = typeToEuiIconMap[type] || defaultIcon; + const containerProps = { ...token, ariaLabel, needsAria }; if (tooltipEnabled === true) { // wrap the inner component inside because EuiToolTip doesn't seem @@ -122,11 +95,10 @@ const FieldTypeIconContainer: FC = ({ if (needsAria && ariaLabel) { wrapperProps['aria-label'] = ariaLabel; } - return ( - + ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx index 511a068f305f9..fa1114e05100f 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx @@ -54,12 +54,7 @@ export const DataVisualizerFieldTypesFilter: FC = ({ {item.name} {type && ( - + )} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/not_in_docs_content/not_in_docs_context.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/not_in_docs_content/not_in_docs_context.tsx index 2baa52b0baf0a..9d9801890349f 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/not_in_docs_content/not_in_docs_context.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/not_in_docs_content/not_in_docs_context.tsx @@ -6,7 +6,7 @@ */ import React, { FC, Fragment } from 'react'; -import { EuiIcon, EuiSpacer, EuiText } from '@elastic/eui'; +import { EuiIcon, EuiText } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/field_type_filter.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/field_type_filter.tsx index a4286bc4e09d1..e14185d968fa7 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/field_type_filter.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/field_type_filter.tsx @@ -12,7 +12,7 @@ import { JOB_FIELD_TYPES_OPTIONS, JobFieldType } from '../../../../../common'; import { FieldTypeIcon } from '../../../common/components/field_type_icon'; import { MultiSelectPicker, Option } from '../../../common/components/multi_select_picker'; -export const DatavisualizerFieldTypeFilter: FC<{ +export const DataVisualizerFieldTypeFilter: FC<{ indexedFieldTypes: JobFieldType[]; setVisibleFieldTypes(q: string[]): void; visibleFieldTypes: string[]; @@ -28,12 +28,7 @@ export const DatavisualizerFieldTypeFilter: FC<{ {item.name} {indexedFieldName && ( - + )} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx index 91ec1e449bb38..19102ef8eac00 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx @@ -12,7 +12,7 @@ import { Query, fromKueryExpression, luceneStringToDsl, toElasticsearchQuery } f import { QueryStringInput } from '../../../../../../../../src/plugins/data/public'; import { ShardSizeFilter } from './shard_size_select'; import { DataVisualizerFieldNamesFilter } from './field_name_filter'; -import { DatavisualizerFieldTypeFilter } from './field_type_filter'; +import { DataVisualizerFieldTypeFilter } from './field_type_filter'; import { IndexPattern } from '../../../../../../../../src/plugins/data/common'; import { JobFieldType } from '../../../../../common/types'; import { @@ -148,7 +148,7 @@ export const SearchPanel: FC = ({ visibleFieldNames={visibleFieldNames} showEmptyFields={showEmptyFields} /> - Date: Tue, 28 Sep 2021 14:30:27 -0500 Subject: [PATCH 119/188] [ML] Fix doc count for fields that exists but have no stats --- .../apps/ml/data_visualizer/index_data_visualizer.ts | 12 ++++++------ .../functional/services/ml/data_visualizer_table.ts | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts index 56a9547f4c664..031074876f39c 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts @@ -92,7 +92,7 @@ export default function ({ getService }: FtrProviderContext) { aggregatable: false, loading: false, exampleCount: 1, - docCountFormatted: '0 (0%)', + docCountFormatted: '', viewableInLens: false, }, { @@ -122,7 +122,7 @@ export default function ({ getService }: FtrProviderContext) { aggregatable: false, loading: false, exampleCount: 1, - docCountFormatted: '0 (0%)', + docCountFormatted: '', viewableInLens: false, }, { @@ -188,7 +188,7 @@ export default function ({ getService }: FtrProviderContext) { aggregatable: false, loading: false, exampleCount: 1, - docCountFormatted: '0 (0%)', + docCountFormatted: '', viewableInLens: false, }, { @@ -218,7 +218,7 @@ export default function ({ getService }: FtrProviderContext) { aggregatable: false, loading: false, exampleCount: 1, - docCountFormatted: '0 (0%)', + docCountFormatted: '', viewableInLens: false, }, { @@ -284,7 +284,7 @@ export default function ({ getService }: FtrProviderContext) { aggregatable: false, loading: false, exampleCount: 1, - docCountFormatted: '0 (0%)', + docCountFormatted: '', viewableInLens: false, }, { @@ -314,7 +314,7 @@ export default function ({ getService }: FtrProviderContext) { aggregatable: false, loading: false, exampleCount: 1, - docCountFormatted: '0 (0%)', + docCountFormatted: '', viewableInLens: false, }, { diff --git a/x-pack/test/functional/services/ml/data_visualizer_table.ts b/x-pack/test/functional/services/ml/data_visualizer_table.ts index 7c0409f3ec27c..8094f0ad1f8d2 100644 --- a/x-pack/test/functional/services/ml/data_visualizer_table.ts +++ b/x-pack/test/functional/services/ml/data_visualizer_table.ts @@ -150,7 +150,7 @@ export function MachineLearningDataVisualizerTableProvider( const docCount = await testSubjects.getVisibleText(docCountFormattedSelector); expect(docCount).to.eql( docCountFormatted, - `Expected field document count to be '${docCountFormatted}' (got '${docCount}')` + `Expected field ${fieldName}'s document count to be '${docCountFormatted}' (got '${docCount}')` ); } From 4645ff33ff0ca0e5291e823ad6cde8d94ed67cb1 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 28 Sep 2021 13:31:25 -0500 Subject: [PATCH 120/188] [ML] Rename classnames to BEM style --- .../embedded_map/_embedded_map.scss | 2 +- .../components/embedded_map/embedded_map.tsx | 2 +- .../examples_list/examples_list.tsx | 2 +- .../expanded_row/file_based_expanded_row.tsx | 5 +-- .../geo_point_content_with_map.tsx | 5 +-- .../expanded_row/index_based_expanded_row.tsx | 5 +-- .../field_count_panel/field_count_panel.tsx | 4 +-- .../common/components/stats_table/_index.scss | 32 +++++++++---------- .../components/field_count_stats/_index.scss | 4 +-- .../field_count_stats/metric_fields_count.tsx | 2 +- .../field_count_stats/total_fields_count.tsx | 2 +- .../boolean_content.tsx | 6 ++-- .../choropleth_map.tsx | 4 +-- .../field_data_expanded_row/date_content.tsx | 4 +-- .../document_stats.tsx | 4 +-- .../number_content.tsx | 9 ++---- .../field_data_expanded_row/text_content.tsx | 2 +- .../field_data_row/distinct_values.tsx | 2 +- .../field_data_row/document_stats.tsx | 2 +- .../data_visualizer_stats_table.tsx | 6 ++-- .../components/top_values/top_values.tsx | 4 +-- 21 files changed, 47 insertions(+), 61 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/_embedded_map.scss b/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/_embedded_map.scss index 99ee60f62bb21..a3682bfd7d96c 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/_embedded_map.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/_embedded_map.scss @@ -1,4 +1,4 @@ -.embeddedMapContent { +.embeddedMap__content { width: 100%; height: 100%; display: flex; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/embedded_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/embedded_map.tsx index 0d17918f76f4f..cf357a462d9b3 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/embedded_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/embedded_map/embedded_map.tsx @@ -143,7 +143,7 @@ export function EmbeddedMapComponent({ return (
); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx index 49aee5f8ce7e5..1aa14a88a5248 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/examples_list/examples_list.tsx @@ -43,7 +43,7 @@ export const ExamplesList: FC = ({ examples }) => { return ( +
{getCardContent()}
); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx index e0047c16fd2f5..2fb1136ad5454 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/geo_point_content_with_map/geo_point_content_with_map.tsx @@ -65,10 +65,7 @@ export const GeoPointContentWithMap: FC<{ - + diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx index fe0a5e6c08404..170263ebbb11d 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx @@ -81,10 +81,7 @@ export const IndexBasedDataVisualizerExpandedRow = ({ } return ( -
+
{loading === true ? : getCardContent()}
); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_count_panel/field_count_panel.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_count_panel/field_count_panel.tsx index 978e59d0cf957..ce98ecd2fa5c7 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_count_panel/field_count_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_count_panel/field_count_panel.tsx @@ -30,11 +30,11 @@ export const FieldCountPanel: FC = ({ gutterSize="xs" data-test-subj="dataVisualizerFieldCountPanel" responsive={false} - className="dataVisualizerFieldCountPanel" + className="dvFieldCount__panel" > - + .euiTableRowCell { border-bottom: 0; } @@ -61,32 +60,31 @@ $panelWidthL: #{'max(40%, 450px)'}; } } - .dataVisualizerSummaryTableWrapper { + .dvSummaryTable__wrapper { min-width: $panelWidthS; max-width: $panelWidthS; } - .dataVisualizerTopValuesWrapper { + .dvTopValues__wrapper { min-width: fit-content; } - .dataVisualizerUniformPanel { - min-width: $panelWidthS; - max-width: $panelWidthS; - } - - .dataVisualizerPanelWrapper { + .dvPanel__wrapper { margin: $euiSizeXS $euiSizeM $euiSizeM 0; - &.compressed { + &.dvPanel--compressed { width: $panelWidthS; } + &.dvPanel--uniform { + min-width: $panelWidthS; + max-width: $panelWidthS; + } } - .dataVisualizerMapWrapper { + .dvMap__wrapper { height: $euiSize * 15; //240px } - .dataVisualizerTextContent { + .dvText__wrapper { min-width: $panelWidthS; } } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/_index.scss b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/_index.scss index 9e26d79e1f685..0774cb198ea90 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/_index.scss +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/_index.scss @@ -1,4 +1,4 @@ -.dataVisualizerFieldCountPanel { +.dvFieldCount__panel { margin-left: $euiSizeXS; @include euiBreakpoint('xs', 's') { flex-direction: column; @@ -6,7 +6,7 @@ } } -.dataVisualizerFieldCountContainer { +.dvFieldCount__item { max-width: 300px; min-width: 300px; } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/metric_fields_count.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/metric_fields_count.tsx index 351b862a48ec0..3b1dbf0c6376d 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/metric_fields_count.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/metric_fields_count.tsx @@ -30,7 +30,7 @@ export const MetricFieldsCount: FC = ({ metricsStats }) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/total_fields_count.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/total_fields_count.tsx index 62f2aaf0b4528..53aa84c09d3a7 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/total_fields_count.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_count_stats/total_fields_count.tsx @@ -30,7 +30,7 @@ export const TotalFieldsCount: FC = ({ fieldsCountStats } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx index 96ecd33663500..754d0e470fe40 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx @@ -93,10 +93,10 @@ export const BooleanContent: FC = ({ config }) => { - + {summaryTableTitle} = ({ config }) => { /> - + = ({ stats, suggestion }) => { return ( -
+
diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx index df90d897d2d3d..8d5704fc16fd5 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/date_content.tsx @@ -77,10 +77,10 @@ export const DateContent: FC = ({ config }) => { return ( - + {summaryTableTitle} - className={'dataVisualizerSummaryTable'} + className={'dvSummaryTable'} data-test-subj={'dataVisualizerDateSummaryTable'} compressed items={summaryTableItems} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx index fd43cfae06f05..5995b81555f9b 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/document_stats.tsx @@ -81,11 +81,11 @@ export const DocumentStatsTable: FC = ({ config }) => { return ( {metaTableTitle} = ({ config, onAddFilter }) => return ( - + {summaryTableTitle} - className={'dataVisualizerSummaryTable'} + className={'dvSummaryTable'} compressed items={summaryTableItems} columns={summaryTableColumns} @@ -136,7 +133,7 @@ export const NumberContent: FC = ({ config, onAddFilter }) => {distribution && ( diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx index a45a93d032cde..6f946fc1025ed 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx @@ -26,7 +26,7 @@ export const TextContent: FC = ({ config }) => { return ( - + {numExamples > 0 && } {numExamples === 0 && ( diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx index 56a0390311f20..cb6eca62cd655 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/distinct_values.tsx @@ -18,7 +18,7 @@ export const DistinctValues = ({ cardinality, showIcon }: Props) => { if (cardinality === undefined) return null; return ( <> - {showIcon ? : null} + {showIcon ? : null} {cardinality} ); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx index aab75611267b7..01b8f0af9538d 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/document_stats.tsx @@ -31,7 +31,7 @@ export const DocumentStat = ({ config, showIcon }: Props) => { return docsCount !== undefined ? ( <> - {showIcon ? : null} + {showIcon ? : null} {docsCount} ({docsPercent}%) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 99c253e8aeaaf..4e1c03aa987bd 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -235,9 +235,9 @@ export const DataVisualizerTable = ({ }, { name: ( -
+
{dimensions.showIcon ? ( - + ) : null} {i18n.translate('xpack.dataVisualizer.dataGrid.distributionsColumnName', { defaultMessage: 'Distributions', @@ -322,7 +322,7 @@ export const DataVisualizerTable = ({ {(resizeRef) => (
- className={'dataVisualizerTable'} + className={'dvTable'} items={items} itemId={FIELD_NAME} columns={columns} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index 031016acc36ca..13dc2f7d190c2 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -57,7 +57,7 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed, return ( = ({ stats, fieldFormat, barColor, compressed,
{Array.isArray(topValues) && topValues.map((value) => ( From 6835aeed34867fc567440d6a739d167984a48887 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 30 Sep 2021 17:26:27 -0500 Subject: [PATCH 121/188] Resolve latest changes --- .../components/layout/discover_layout.tsx | 1 + .../data_visualizer_grid.tsx | 40 +++++++++++++++---- .../field_stats_table_embeddable.tsx | 31 ++++++++++++++ .../embeddable/saved_search_embeddable.tsx | 29 ++++++++++++-- .../field_type_icon.test.tsx.snap | 2 +- .../field_type_icon/field_type_icon.tsx | 2 +- .../grid_embeddable/grid_embeddable.tsx | 11 ++++- 7 files changed, 102 insertions(+), 14 deletions(-) create mode 100644 src/plugins/discover/public/application/components/data_visualizer_grid/field_stats_table_embeddable.tsx diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index f8b11b08a46e7..0b6a877377b92 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -289,6 +289,7 @@ export function DiscoverLayout({ filters={state.filters} columns={columns} stateContainer={stateContainer} + onAddFilter={onAddFilter} /> )} diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx index 3cae73fd466f1..5492fac014b74 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx @@ -8,7 +8,7 @@ import React, { useEffect, useMemo, useRef, useState } from 'react'; import { Filter } from '@kbn/es-query'; -import { DataView, Query } from '../../../../../data/common'; +import { IndexPatternField, IndexPattern, DataView, Query } from '../../../../../data/common'; import { DiscoverServices } from '../../../build_services'; import { EmbeddableInput, @@ -21,12 +21,16 @@ import { SavedSearch } from '../../../saved_searches'; import { GetStateReturn } from '../../apps/main/services/discover_state'; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { - indexPattern: DataView; + indexPattern: IndexPattern; savedSearch?: SavedSearch; query?: Query; visibleFieldNames?: string[]; filters?: Filter[]; showPreviewByDefault?: boolean; + /** + * Callback to add a filter to filter bar + */ + onAddFilter?: (field: IndexPatternField | string, value: string, type: '+' | '-') => void; } export interface DataVisualizerGridEmbeddableOutput extends EmbeddableOutput { showDistributions?: boolean; @@ -38,9 +42,17 @@ export interface DiscoverDataVisualizerGridProps { */ columns: string[]; /** - * The used data view + * The used index pattern */ indexPattern: DataView; + /** + * Saved search description + */ + searchDescription?: string; + /** + * Saved search title + */ + searchTitle?: string; /** * Discover plugin services */ @@ -57,14 +69,24 @@ export interface DiscoverDataVisualizerGridProps { * Filters query to update the table content */ filters?: Filter[]; + stateContainer?: GetStateReturn; /** - * stateContainer to access and update app state preferences like to show preview or not + * Callback to add a filter to filter bar */ - stateContainer?: GetStateReturn; + onAddFilter?: (field: IndexPatternField | string, value: string, type: '+' | '-') => void; } export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { - const { services, indexPattern, savedSearch, query, columns, filters, stateContainer } = props; + const { + services, + indexPattern, + savedSearch, + query, + columns, + filters, + stateContainer, + onAddFilter, + } = props; const { uiSettings } = services; const [embeddable, setEmbeddable] = useState< @@ -101,10 +123,11 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp query, filters, visibleFieldNames: columns, + onAddFilter, }); embeddable.reload(); } - }, [embeddable, indexPattern, savedSearch, query, columns, filters]); + }, [embeddable, indexPattern, savedSearch, query, columns, filters, onAddFilter]); useEffect(() => { if (showPreviewByDefault && embeddable && !isErrorEmbeddable(embeddable)) { @@ -139,6 +162,7 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp savedSearch, query, showPreviewByDefault, + onAddFilter, }); if (!unmounted) { setEmbeddable(initializedEmbeddable); @@ -162,7 +186,7 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp return (
+ + + ); +} diff --git a/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx b/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx index ef8670f976672..c7c50b8872718 100644 --- a/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx +++ b/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx @@ -19,12 +19,12 @@ import { SEARCH_EMBEDDABLE_TYPE } from './constants'; import { APPLY_FILTER_TRIGGER, esFilters, FilterManager } from '../../../../data/public'; import { DiscoverServices } from '../../build_services'; import { - Query, - TimeRange, Filter, IndexPattern, - ISearchSource, IndexPatternField, + ISearchSource, + Query, + TimeRange, } from '../../../../data/common'; import { ElasticSearchHit } from '../doc_views/doc_views_types'; import { SavedSearchEmbeddableComponent } from './saved_search_embeddable_component'; @@ -45,6 +45,8 @@ import { DocTableProps } from '../apps/main/components/doc_table/doc_table_wrapp import { getDefaultSort } from '../apps/main/components/doc_table'; import { SortOrder } from '../apps/main/components/doc_table/components/table_header/helpers'; import { updateSearchSource } from './helpers/update_search_source'; +import { VIEW_MODE } from '../apps/main/components/view_mode_toggle'; +import { FieldStatsTableEmbeddable } from '../components/data_visualizer_grid/field_stats_table_embeddable'; export type SearchProps = Partial & Partial & { @@ -379,6 +381,27 @@ export class SavedSearchEmbeddable if (!this.searchProps) { return; } + + if ( + this.savedSearch.viewMode === VIEW_MODE.AGGREGATED_LEVEL && + searchProps.services && + searchProps.indexPattern && + Array.isArray(searchProps.columns) + ) { + ReactDOM.render( + , + domNode + ); + return; + } const useLegacyTable = this.services.uiSettings.get(DOC_TABLE_LEGACY); const props = { searchProps, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/__snapshots__/field_type_icon.test.tsx.snap b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/__snapshots__/field_type_icon.test.tsx.snap index 927d8ddb7a851..398dc5dad2dc7 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/__snapshots__/field_type_icon.test.tsx.snap +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/__snapshots__/field_type_icon.test.tsx.snap @@ -10,7 +10,7 @@ exports[`FieldTypeIcon render component when type matches a field type 1`] = ` > diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx index 3f84950c1345b..d5ec7b45edb6d 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx @@ -48,7 +48,7 @@ export const typeToEuiIconMap: Record void; } export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; @@ -85,6 +93,7 @@ export const EmbeddableWrapper = ({ item={item} indexPattern={input.indexPattern} combinedQuery={{ searchQueryLanguage, searchString }} + onAddFilter={input.onAddFilter} /> ); } From 0aef3a32b18f6c894a818c2ff8c32c0568550145 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 27 Sep 2021 13:01:11 -0500 Subject: [PATCH 122/188] Add in place ss --- .../common/search_strategy/constants.ts | 8 + .../common/search_strategy/types.ts | 23 +++ .../data_loader/data_loader.ts | 15 ++ .../grid_embeddable/grid_embeddable.tsx | 2 +- .../use_data_visualizer_grid_data.ts | 66 +++++--- .../hooks/use_search_strategy.ts | 151 ++++++++++++++++++ .../services/visualizer_stats.ts | 8 +- .../plugins/data_visualizer/server/plugin.ts | 29 ++++ .../field_stats_search_strategy.ts | 110 +++++++++++++ .../field_stats_state_provider.ts | 91 +++++++++++ .../data_visualizer/server/types/deps.ts | 6 + 11 files changed, 486 insertions(+), 23 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/common/search_strategy/constants.ts create mode 100644 x-pack/plugins/data_visualizer/common/search_strategy/types.ts rename x-pack/plugins/data_visualizer/public/application/index_data_visualizer/{embeddables/grid_embeddable => hooks}/use_data_visualizer_grid_data.ts (87%) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts create mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts create mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts diff --git a/x-pack/plugins/data_visualizer/common/search_strategy/constants.ts b/x-pack/plugins/data_visualizer/common/search_strategy/constants.ts new file mode 100644 index 0000000000000..f77efc81f897c --- /dev/null +++ b/x-pack/plugins/data_visualizer/common/search_strategy/constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const FIELD_STATS_SEARCH_STRATEGY = 'dataVisualizerFieldStatsSearchStrategy'; diff --git a/x-pack/plugins/data_visualizer/common/search_strategy/types.ts b/x-pack/plugins/data_visualizer/common/search_strategy/types.ts new file mode 100644 index 0000000000000..ae87a673c0b7f --- /dev/null +++ b/x-pack/plugins/data_visualizer/common/search_strategy/types.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + IKibanaSearchRequest, + IKibanaSearchResponse, +} from '../../../../../src/plugins/data/common'; + +export interface FieldStatRawResponse { + loading?: boolean; + ccsWarning: false; + took: 0; +} +export type FieldStatsRequest = IKibanaSearchRequest<{ + index: string; + sessionId?: string; +}>; + +export type FieldStatsResponse = IKibanaSearchResponse; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts index c4db51dcd81bc..ca5afeebb00be 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts @@ -84,6 +84,21 @@ export class DataLoader { fields: FieldRequestConfig[], interval?: number ): Promise { + console.log('loadFieldStats'); + console.log( + JSON.stringify({ + indexPatternTitle: this._indexPatternTitle, + query, + timeFieldName: this._indexPattern.timeFieldName, + earliest, + latest, + samplerShardSize, + interval, + fields, + maxExamples: this._maxExamples, + runtimeMappings: this._runtimeMappings, + }) + ); const stats = await getVisualizerFieldStats({ indexPatternTitle: this._indexPatternTitle, query, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index e083a6781fad7..39eecb860bc10 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -39,7 +39,7 @@ import { getDefaultDataVisualizerListState } from '../../components/index_data_v import { DataVisualizerTableState } from '../../../../../common'; import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; -import { useDataVisualizerGridData } from './use_data_visualizer_grid_data'; +import { useDataVisualizerGridData } from '../../hooks/use_data_visualizer_grid_data'; export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerStartDependencies]; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts similarity index 87% rename from x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts rename to x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 0c57a1bf25b6a..1a7f44fdffef2 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -9,27 +9,28 @@ import { Required } from 'utility-types'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { merge } from 'rxjs'; import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; -import { i18n } from '@kbn/i18n'; -import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; -import { useDataVisualizerKibana } from '../../../kibana_context'; -import { getEsQueryFromSavedSearch } from '../../utils/saved_search_utils'; -import { MetricFieldsStats } from '../../../common/components/stats_table/components/field_count_stats'; -import { DataLoader } from '../../data_loader/data_loader'; -import { useTimefilter } from '../../hooks/use_time_filter'; -import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; -import { TimeBuckets } from '../../services/time_buckets'; +import { i18n } from '../../../../../../../../../../../../private/var/tmp/_bazel_quynhnguyen/bd5cc7ce3740c1abb2c63a2609d8bb9f/execroot/kibana/bazel-out/darwin-fastbuild/bin/packages/kbn-i18n'; +import { DataVisualizerIndexBasedAppState } from '../types/index_data_visualizer_state'; +import { useDataVisualizerKibana } from '../../kibana_context'; +import { getEsQueryFromSavedSearch } from '../utils/saved_search_utils'; +import { MetricFieldsStats } from '../../common/components/stats_table/components/field_count_stats'; +import { DataLoader } from '../data_loader/data_loader'; +import { useTimefilter } from './use_time_filter'; +import { dataVisualizerRefresh$ } from '../services/timefilter_refresh_service'; +import { TimeBuckets } from '../services/time_buckets'; import { DataViewField, KBN_FIELD_TYPES, UI_SETTINGS, -} from '../../../../../../../../src/plugins/data/common'; -import { extractErrorProperties } from '../../utils/error_utils'; -import { FieldVisConfig } from '../../../common/components/stats_table/types'; -import { FieldRequestConfig, JOB_FIELD_TYPES } from '../../../../../common'; -import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; -import { getActions } from '../../../common/components/field_data_row/action_menu'; -import { DataVisualizerGridEmbeddableInput } from './grid_embeddable'; -import { getDefaultPageState } from '../../components/index_data_visualizer_view/index_data_visualizer_view'; +} from '../../../../../../../src/plugins/data/common'; +import { extractErrorProperties } from '../utils/error_utils'; +import { FieldVisConfig } from '../../common/components/stats_table/types'; +import { FieldRequestConfig, JOB_FIELD_TYPES } from '../../../../common'; +import { kbnTypeToJobType } from '../../common/util/field_types_utils'; +import { getActions } from '../../common/components/field_data_row/action_menu'; +import { DataVisualizerGridEmbeddableInput } from '../embeddables/grid_embeddable/grid_embeddable'; +import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; +import { useFieldStatsSearchStrategy } from './use_search_strategy'; const defaults = getDefaultPageState(); @@ -38,11 +39,12 @@ export const useDataVisualizerGridData = ( dataVisualizerListState: Required ) => { const { services } = useDataVisualizerKibana(); - const { notifications, uiSettings } = services; + const { notifications, uiSettings, data } = services; const { toasts } = notifications; const { samplerShardSize, visibleFieldTypes, showEmptyFields } = dataVisualizerListState; const [lastRefresh, setLastRefresh] = useState(0); + const [searchSessionId, setSearchSessionId] = useState(); const { currentSavedSearch, @@ -92,6 +94,34 @@ export const useDataVisualizerGridData = ( currentFilters, ]); + useEffect(() => { + const currentSearchSessionId = data.search?.session?.getSessionId(); + if (currentSearchSessionId !== undefined) { + setSearchSessionId(currentSearchSessionId); + } + console.log('currentSearchSessionId', currentSearchSessionId); + }, [data]); + + const _timeBuckets = useMemo(() => { + return new TimeBuckets({ + [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), + [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), + dateFormat: uiSettings.get('dateFormat'), + 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), + }); + }, [uiSettings]); + + /** Search strategy**/ + const strategyResponse = useFieldStatsSearchStrategy({ + query: searchQuery, + sessionId: searchSessionId, + timeBuckets: _timeBuckets, + indexPattern: currentIndexPattern, + savedSearch: currentSavedSearch, + timeFilter: timefilter, + }); + + console.log('strategyResponse', strategyResponse); const [overallStats, setOverallStats] = useState(defaults.overallStats); const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts new file mode 100644 index 0000000000000..630a2eafb8709 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useEffect, useReducer, useRef } from 'react'; +import { Subscription } from 'rxjs'; +import { isCompleteResponse, isErrorResponse } from '../../../../../../../src/plugins/data/common'; +import { FIELD_STATS_SEARCH_STRATEGY } from '../../../../common/search_strategy/constants'; +import { + FieldStatRawResponse, + FieldStatsRequest, + FieldStatsResponse, +} from '../../../../common/search_strategy/types'; +import { useDataVisualizerKibana } from '../../kibana_context'; + +interface SearchStrategyReturnBase { + progress: SearchStrategyProgress; + response: TRawResponse; + startFetch: () => void; + cancelFetch: () => void; +} + +interface SearchStrategyProgress { + error?: Error; + isRunning: boolean; + loaded: number; + total: number; +} + +const getInitialRawResponse = (): TRawResponse => + ({ + ccsWarning: false, + took: 0, + } as TRawResponse); + +const getInitialProgress = (): SearchStrategyProgress => ({ + isRunning: false, + loaded: 0, + total: 100, +}); + +const getReducer = + () => + (prev: T, update: Partial): T => ({ + ...prev, + ...update, + }); + +interface SearchStrategyParams { + sessionId?: string; + // @todo update type + query?: any; +} +export function useFieldStatsSearchStrategy< + TRawResponse extends FieldStatRawResponse, + TParams extends SearchStrategyParams +>(searchStrategyParams?: TParams): SearchStrategyReturnBase { + console.log('searchStrategyParams', searchStrategyParams); + const { + services: { data }, + } = useDataVisualizerKibana(); + + const [rawResponse, setRawResponse] = useReducer( + getReducer(), + getInitialRawResponse() + ); + + const [fetchState, setFetchState] = useReducer( + getReducer(), + getInitialProgress() + ); + + const abortCtrl = useRef(new AbortController()); + const searchSubscription$ = useRef(); + const searchStrategyParamsRef = useRef(searchStrategyParams); + + const startFetch = useCallback(() => { + searchSubscription$.current?.unsubscribe(); + abortCtrl.current.abort(); + abortCtrl.current = new AbortController(); + setFetchState({ + ...getInitialProgress(), + error: undefined, + }); + + const request = { + params: {}, + }; + + // Submit the search request using the `data.search` service. + searchSubscription$.current = data.search + .search(request, { + strategy: FIELD_STATS_SEARCH_STRATEGY, + abortSignal: abortCtrl.current.signal, + }) + .subscribe({ + next: (response: FieldStatsResponse) => { + setRawResponse(response.rawResponse); + setFetchState({ + isRunning: response.isRunning || false, + ...(response.loaded ? { loaded: response.loaded } : {}), + ...(response.total ? { total: response.total } : {}), + }); + + if (isCompleteResponse(response)) { + searchSubscription$.current?.unsubscribe(); + setFetchState({ + isRunning: false, + }); + } else if (isErrorResponse(response)) { + searchSubscription$.current?.unsubscribe(); + setFetchState({ + error: response as unknown as Error, + isRunning: false, + }); + } + }, + error: (error: Error) => { + setFetchState({ + error, + isRunning: false, + }); + }, + }); + }, [data.search]); + + const cancelFetch = useCallback(() => { + searchSubscription$.current?.unsubscribe(); + searchSubscription$.current = undefined; + abortCtrl.current.abort(); + setFetchState({ + isRunning: false, + }); + }, []); + + // auto-update + useEffect(() => { + startFetch(); + return cancelFetch; + }, [startFetch, cancelFetch]); + + return { + progress: fetchState, + response: rawResponse, + startFetch, + cancelFetch, + }; +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/visualizer_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/visualizer_stats.ts index 8db267a1dc837..1b3057231bfef 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/visualizer_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/visualizer_stats.ts @@ -46,8 +46,8 @@ export async function getVisualizerOverallStats({ runtimeMappings, }); - const fileUploadModules = await lazyLoadModules(); - return await fileUploadModules.getHttp().fetch({ + const dataVisualizerModules = await lazyLoadModules(); + return await dataVisualizerModules.getHttp().fetch({ path: `${basePath()}/get_overall_stats/${indexPatternTitle}`, method: 'POST', body, @@ -89,8 +89,8 @@ export async function getVisualizerFieldStats({ runtimeMappings, }); - const fileUploadModules = await lazyLoadModules(); - return await fileUploadModules.getHttp().fetch<[DocumentCounts, FieldVisStats]>({ + const dataVisualizerModules = await lazyLoadModules(); + return await dataVisualizerModules.getHttp().fetch<[DocumentCounts, FieldVisStats]>({ path: `${basePath()}/get_field_stats/${indexPatternTitle}`, method: 'POST', body, diff --git a/x-pack/plugins/data_visualizer/server/plugin.ts b/x-pack/plugins/data_visualizer/server/plugin.ts index 9db580959b116..9adfc9718ee70 100644 --- a/x-pack/plugins/data_visualizer/server/plugin.ts +++ b/x-pack/plugins/data_visualizer/server/plugin.ts @@ -8,12 +8,41 @@ import { CoreSetup, CoreStart, Plugin } from 'src/core/server'; import { StartDeps, SetupDeps } from './types'; import { dataVisualizerRoutes } from './routes'; +import { ENHANCED_ES_SEARCH_STRATEGY } from '../../../../src/plugins/data/common'; +import { FIELD_STATS_SEARCH_STRATEGY } from '../common/search_strategy/constants'; +import type { FieldStatsRequest, FieldStatsResponse } from '../common/search_strategy/types'; +import type { ISearchStrategy } from '../../../../src/plugins/data/server'; + +// @todo: rename +export const myEnhancedSearchStrategyProvider = ( + data: StartDeps['data'] +): ISearchStrategy => { + // Get the default search strategy + const ese = data.search.getSearchStrategy(ENHANCED_ES_SEARCH_STRATEGY); + return { + search: (request, options, deps) => { + // search will be called multiple times, + // be sure your response formatting is capable of handling partial results, as well as the final result. + return ese.search(request, options, deps); + }, + cancel: async (id, options, deps) => { + // call the cancel method of the async strategy you are using or implement your own cancellation function. + if (ese.cancel) { + await ese.cancel(id, options, deps); + } + }, + }; +}; export class DataVisualizerPlugin implements Plugin { constructor() {} setup(coreSetup: CoreSetup, plugins: SetupDeps) { dataVisualizerRoutes(coreSetup); + coreSetup.getStartServices().then(([_, depsStart]) => { + const myStrategy = myEnhancedSearchStrategyProvider(depsStart.data); + plugins.data.search.registerSearchStrategy(FIELD_STATS_SEARCH_STRATEGY, myStrategy); + }); } start(core: CoreStart) {} diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts new file mode 100644 index 0000000000000..a3e4216d8d739 --- /dev/null +++ b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { fieldStatsSearchServiceStateProvider } from './field_stats_state_provider'; + +export interface RawResponseBase { + ccsWarning: boolean; + took: number; +} + +export interface FieldStatsRequestParams { + sessionId?: string; +} + +export interface SearchStrategyServerParams { + includeFrozen?: boolean; +} + +export interface FieldStatsRawResponse extends RawResponseBase { + log: string[]; +} + +interface SearchServiceState { + cancel: () => void; + error: Error; + meta: { + loaded: number; + total: number; + isRunning: boolean; + isPartial: boolean; + }; + rawResponse: TRawResponse; +} + +type GetSearchServiceState = + () => SearchServiceState; + +export type SearchServiceProvider< + TSearchStrategyClientParams, + TRawResponse extends RawResponseBase +> = ( + esClient: ElasticsearchClient, + searchServiceParams: TSearchStrategyClientParams, + includeFrozen: boolean +) => GetSearchServiceState; + +export type FieldStatsSearchServiceProvider = SearchServiceProvider< + FieldStatsRequestParams, + FieldStatsRawResponse +>; + +export const fieldStatsSearchServiceProvider: FieldStatsSearchServiceProvider = ( + esClient: ElasticsearchClient, + searchServiceParams: FieldStatsRequestParams, + includeFrozen: boolean +) => { + const state = fieldStatsSearchServiceStateProvider(); + + async function fetchCorrelations() { + let params: (FieldStatsRequestParams & SearchStrategyServerParams) | undefined; + + try { + params = { + ...searchServiceParams, + includeFrozen, + }; + } catch (e) { + state.setError(e); + } + + if (state.getState().error !== undefined) { + state.setCcsWarning(true); + } + + state.setIsRunning(false); + } + + function cancel() { + // addLogMessage(`Service cancelled.`); + state.setIsCancelled(true); + } + + fetchCorrelations(); + + return () => { + const { ccsWarning, error, isRunning } = state.getState(); + + return { + cancel, + error, + meta: { + loaded: Math.round(state.getOverallProgress() * 100), + total: 100, + isRunning, + isPartial: isRunning, + }, + // @todo + rawResponse: { + ccsWarning, + log: [], + took: 0, + }, + }; + }; +}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts new file mode 100644 index 0000000000000..9d099cdf22e1e --- /dev/null +++ b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginStart as DataPluginStart } from '../../../../../src/plugins/data/server'; + +export interface LatencyCorrelationSearchServiceProgress { + started: number; + loadedHistogramStepsize: number; + loadedOverallHistogram: number; + loadedFieldCandidates: number; + loadedFieldValuePairs: number; + loadedHistograms: number; +} + +export const fieldStatsSearchServiceStateProvider = (dataPlugin: DataPluginStart) => { + let ccsWarning = false; + function setCcsWarning(d: boolean) { + ccsWarning = d; + } + + let error: Error; + function setError(d: Error) { + error = d; + } + + let isCancelled = false; + function getIsCancelled() { + return isCancelled; + } + function setIsCancelled(d: boolean) { + isCancelled = d; + } + + let isRunning = true; + function setIsRunning(d: boolean) { + isRunning = d; + } + + let progress: LatencyCorrelationSearchServiceProgress = { + started: Date.now(), + loadedHistogramStepsize: 0, + loadedOverallHistogram: 0, + loadedFieldCandidates: 0, + loadedFieldValuePairs: 0, + loadedHistograms: 0, + }; + function getOverallProgress() { + return ( + progress.loadedHistogramStepsize * 0.025 + + progress.loadedOverallHistogram * 0.025 + + progress.loadedFieldCandidates * 0.025 + + progress.loadedFieldValuePairs * 0.025 + + progress.loadedHistograms * 0.9 + ); + } + function setProgress(d: Partial>) { + progress = { + ...progress, + ...d, + }; + } + + function getState() { + return { + ccsWarning, + error, + isCancelled, + isRunning, + progress, + }; + } + + return { + getIsCancelled, + getOverallProgress, + getState, + setCcsWarning, + setError, + setIsCancelled, + setIsRunning, + setProgress, + }; +}; + +export type LatencyCorrelationsSearchServiceState = ReturnType< + typeof fieldStatsSearchServiceStateProvider +>; diff --git a/x-pack/plugins/data_visualizer/server/types/deps.ts b/x-pack/plugins/data_visualizer/server/types/deps.ts index fe982b1fa5e1a..4c8e630ca8630 100644 --- a/x-pack/plugins/data_visualizer/server/types/deps.ts +++ b/x-pack/plugins/data_visualizer/server/types/deps.ts @@ -7,10 +7,16 @@ import type { SecurityPluginStart } from '../../../security/server'; import type { UsageCollectionSetup } from '../../../../../src/plugins/usage_collection/server'; +import { + PluginSetup as DataPluginSetup, + PluginStart as DataPluginStart, +} from '../../../../../src/plugins/data/server'; export interface StartDeps { security?: SecurityPluginStart; + data: DataPluginStart; } export interface SetupDeps { usageCollection: UsageCollectionSetup; + data: DataPluginSetup; } From 4019b5d80e22ed89fc99d643ee9f4684ccaf9eb7 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 27 Sep 2021 17:36:52 -0500 Subject: [PATCH 123/188] Refactor helper functions --- .../common/search_strategy/types.ts | 61 ++++++- .../services/time_buckets.d.ts | 0 .../services/time_buckets.js | 6 +- .../types/field_vis_config.ts | 4 +- .../util => common/utils}/parse_interval.ts | 0 .../expanded_row/file_based_expanded_row.tsx | 2 +- .../field_names_filter/field_names_filter.tsx | 2 +- .../field_types_filter/field_types_filter.tsx | 2 +- .../fields_stats_grid/fields_stats_grid.tsx | 2 +- .../fields_stats_grid/filter_fields.ts | 2 +- .../data_visualizer_stats_table.tsx | 2 +- .../components/stats_table/types/index.ts | 2 +- .../common/util/parse_interval.test.ts | 2 +- .../index_data_visualizer_view.tsx | 2 +- .../data_loader/data_loader.ts | 15 -- .../hooks/use_data_visualizer_grid_data.ts | 90 ++++++++-- .../hooks/use_search_strategy.ts | 46 ++--- .../models/data_visualizer/constants.ts | 3 + .../plugins/data_visualizer/server/plugin.ts | 25 +-- .../field_stats_search_strategy.ts | 136 ++++++++++++--- .../field_stats_state_provider.ts | 34 ++-- .../requests/get_boolean_field_stats.ts | 88 ++++++++++ .../requests/get_date_field_stats.ts | 80 +++++++++ .../requests/get_document_stats.ts | 77 ++++++++ .../requests/get_field_examples.ts | 81 +++++++++ .../requests/get_field_stats.ts | 72 ++++++++ .../requests/get_numeric_field_stats.ts | 165 ++++++++++++++++++ .../requests/get_string_field_stats.ts | 114 ++++++++++++ .../server/types/chart_data.ts | 7 + 29 files changed, 978 insertions(+), 144 deletions(-) rename x-pack/plugins/data_visualizer/{public/application/index_data_visualizer => common}/services/time_buckets.d.ts (100%) rename x-pack/plugins/data_visualizer/{public/application/index_data_visualizer => common}/services/time_buckets.js (98%) rename x-pack/plugins/data_visualizer/{public/application/common/components/stats_table => common}/types/field_vis_config.ts (94%) rename x-pack/plugins/data_visualizer/{public/application/common/util => common/utils}/parse_interval.ts (100%) create mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_boolean_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_date_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_document_stats.ts create mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts create mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_numeric_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_string_field_stats.ts diff --git a/x-pack/plugins/data_visualizer/common/search_strategy/types.ts b/x-pack/plugins/data_visualizer/common/search_strategy/types.ts index ae87a673c0b7f..a65869202dab4 100644 --- a/x-pack/plugins/data_visualizer/common/search_strategy/types.ts +++ b/x-pack/plugins/data_visualizer/common/search_strategy/types.ts @@ -5,19 +5,68 @@ * 2.0. */ -import { +import type { IKibanaSearchRequest, IKibanaSearchResponse, } from '../../../../../src/plugins/data/common'; +import type { TimeBucketsInterval } from '../services/time_buckets'; +import { RuntimeField } from '../../../../../src/plugins/data/common'; +import { FieldRequestConfig } from '../types'; +import { ISearchStrategy } from '../../../../../src/plugins/data/server'; +import { isPopulatedObject } from '../utils/object_utils'; + +export interface FieldStatsCommonRequestParams { + index: string; + query: any; + samplerShardSize: number; + timeFieldName?: string; + earliestMs?: number | undefined; + latestMs?: number | undefined; + runtimeFieldMap?: Record; + intervalMs?: number; +} + +export interface FieldStatsSearchStrategyParams { + sessionId?: string; + earliest?: number; + latest?: number; + aggInterval: TimeBucketsInterval; + intervalMs?: number; + searchQuery?: any; + samplerShardSize: number; + index: string; + metricConfigs: FieldRequestConfig[]; + nonMetricConfigs: FieldRequestConfig[]; + timeFieldName?: string; + runtimeFieldMap: Record; +} + +export function isFieldStatsSearchStrategyParams( + arg: unknown +): arg is FieldStatsSearchStrategyParams { + return isPopulatedObject(arg, ['index', 'samplerShardSize', 'metricConfigs', 'nonMetricConfigs']); +} export interface FieldStatRawResponse { loading?: boolean; ccsWarning: false; took: 0; } -export type FieldStatsRequest = IKibanaSearchRequest<{ - index: string; - sessionId?: string; -}>; - +export type FieldStatsRequest = IKibanaSearchRequest; export type FieldStatsResponse = IKibanaSearchResponse; + +export interface FieldStatsSearchStrategyReturnBase { + progress: FieldStatsSearchStrategyProgress; + response: TRawResponse; + startFetch: () => void; + cancelFetch: () => void; +} + +export interface FieldStatsSearchStrategyProgress { + error?: Error; + isRunning: boolean; + loaded: number; + total: number; +} + +export type FieldStatsSearchStrategy = ISearchStrategy; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/time_buckets.d.ts b/x-pack/plugins/data_visualizer/common/services/time_buckets.d.ts similarity index 100% rename from x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/time_buckets.d.ts rename to x-pack/plugins/data_visualizer/common/services/time_buckets.d.ts diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/time_buckets.js b/x-pack/plugins/data_visualizer/common/services/time_buckets.js similarity index 98% rename from x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/time_buckets.js rename to x-pack/plugins/data_visualizer/common/services/time_buckets.js index 5d54b6c936fb2..49de535ee6c26 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/time_buckets.js +++ b/x-pack/plugins/data_visualizer/common/services/time_buckets.js @@ -5,12 +5,12 @@ * 2.0. */ -import { FIELD_FORMAT_IDS } from '../../../../../../../src/plugins/field_formats/common'; -import { UI_SETTINGS } from '../../../../../../../src/plugins/data/common'; +import { FIELD_FORMAT_IDS } from '../../../../../src/plugins/field_formats/common'; +import { UI_SETTINGS } from '../../../../../src/plugins/data/common'; import { ary, assign, isPlainObject, isString, sortBy } from 'lodash'; import moment from 'moment'; import dateMath from '@elastic/datemath'; -import { parseInterval } from '../../common/util/parse_interval'; +import { parseInterval } from '../utils/parse_interval'; const { duration: d } = moment; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_vis_config.ts b/x-pack/plugins/data_visualizer/common/types/field_vis_config.ts similarity index 94% rename from x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_vis_config.ts rename to x-pack/plugins/data_visualizer/common/types/field_vis_config.ts index eeb9fe12692fd..b88ea8c8fbbc4 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_vis_config.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_vis_config.ts @@ -5,8 +5,8 @@ * 2.0. */ -import type { Percentile, JobFieldType, FieldVisStats } from '../../../../../../common/types'; - +import type { Percentile, JobFieldType, FieldVisStats } from './index'; +// @todo: move this back? export interface MetricFieldVisStats { avg?: number; distribution?: { diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/parse_interval.ts b/x-pack/plugins/data_visualizer/common/utils/parse_interval.ts similarity index 100% rename from x-pack/plugins/data_visualizer/public/application/common/util/parse_interval.ts rename to x-pack/plugins/data_visualizer/common/utils/parse_interval.ts diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/file_based_expanded_row.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/file_based_expanded_row.tsx index 8a9f9a25c16fa..7ba1615e22b43 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/file_based_expanded_row.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/file_based_expanded_row.tsx @@ -17,7 +17,7 @@ import { } from '../stats_table/components/field_data_expanded_row'; import { GeoPointContent } from './geo_point_content/geo_point_content'; import { JOB_FIELD_TYPES } from '../../../../../common'; -import type { FileBasedFieldVisConfig } from '../stats_table/types/field_vis_config'; +import type { FileBasedFieldVisConfig } from '../../../../../common/types/field_vis_config'; export const FileBasedDataVisualizerExpandedRow = ({ item }: { item: FileBasedFieldVisConfig }) => { const config = item; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_names_filter/field_names_filter.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_names_filter/field_names_filter.tsx index 88b4cd406b33c..58e9b9b5740dc 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_names_filter/field_names_filter.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_names_filter/field_names_filter.tsx @@ -11,7 +11,7 @@ import { MultiSelectPicker } from '../multi_select_picker'; import type { FileBasedFieldVisConfig, FileBasedUnknownFieldVisConfig, -} from '../stats_table/types/field_vis_config'; +} from '../../../../../common/types/field_vis_config'; interface Props { fields: Array; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx index 97dc2077d5931..23360bd7a0c87 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx @@ -12,7 +12,7 @@ import { MultiSelectPicker, Option } from '../multi_select_picker'; import type { FileBasedFieldVisConfig, FileBasedUnknownFieldVisConfig, -} from '../stats_table/types/field_vis_config'; +} from '../../../../../common/types/field_vis_config'; import { FieldTypeIcon } from '../field_type_icon'; import { jobTypeLabels } from '../../util/field_types_utils'; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx index b57072eed2944..1173ede84e631 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx @@ -10,7 +10,7 @@ import { EuiFlexGroup, EuiSpacer } from '@elastic/eui'; import type { FindFileStructureResponse } from '../../../../../../file_upload/common'; import type { DataVisualizerTableState } from '../../../../../common'; import { DataVisualizerTable, ItemIdToExpandedRowMap } from '../stats_table'; -import type { FileBasedFieldVisConfig } from '../stats_table/types/field_vis_config'; +import type { FileBasedFieldVisConfig } from '../../../../../common/types/field_vis_config'; import { FileBasedDataVisualizerExpandedRow } from '../expanded_row'; import { DataVisualizerFieldNamesFilter } from '../field_names_filter'; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/filter_fields.ts b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/filter_fields.ts index 6c164233bdbc1..9f1ea4af22537 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/filter_fields.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/filter_fields.ts @@ -9,7 +9,7 @@ import { JOB_FIELD_TYPES } from '../../../../../common'; import type { FileBasedFieldVisConfig, FileBasedUnknownFieldVisConfig, -} from '../stats_table/types/field_vis_config'; +} from '../../../../../common/types/field_vis_config'; export function filterFields( fields: Array, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 4e1c03aa987bd..61de8ef363e72 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -33,7 +33,7 @@ import { FieldVisConfig, FileBasedFieldVisConfig, isIndexBasedFieldVisConfig, -} from './types/field_vis_config'; +} from '../../../../../common/types/field_vis_config'; import { FileBasedNumberContentPreview } from '../field_data_row'; import { BooleanContentPreview } from './components/field_data_row'; import { calculateTableColumnsDimensions } from './utils'; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts index 171d029482e27..7770e6da5d700 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts @@ -12,4 +12,4 @@ export { MetricFieldVisStats, isFileBasedFieldVisConfig, isIndexBasedFieldVisConfig, -} from './field_vis_config'; +} from '../../../../../../common/types/field_vis_config'; diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/parse_interval.test.ts b/x-pack/plugins/data_visualizer/public/application/common/util/parse_interval.test.ts index a1608960a91bc..c259f82d12bfb 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/util/parse_interval.test.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/util/parse_interval.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { parseInterval } from './parse_interval'; +import { parseInterval } from '../../../../common/utils/parse_interval'; describe('ML parse interval util', () => { test('should correctly parse an interval containing a valid unit and value', () => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index fdd723dea3487..56d338babd346 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -64,13 +64,13 @@ import { ActionsPanel } from '../actions_panel'; import { DatePickerWrapper } from '../../../common/components/date_picker_wrapper'; import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; import { HelpMenu } from '../../../common/components/help_menu'; -import { TimeBuckets } from '../../services/time_buckets'; import { createMergedEsQuery, getEsQueryFromSavedSearch } from '../../utils/saved_search_utils'; import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; import { ResultLink } from '../../../common/components/results_links'; import { extractErrorProperties } from '../../utils/error_utils'; import { IndexPatternField, IndexPattern } from '../../../../../../../../src/plugins/data/common'; import './_index.scss'; +import { TimeBuckets } from '../../../../../common/services/time_buckets'; interface DataVisualizerPageState { overallStats: OverallStats; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts index ca5afeebb00be..c4db51dcd81bc 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts @@ -84,21 +84,6 @@ export class DataLoader { fields: FieldRequestConfig[], interval?: number ): Promise { - console.log('loadFieldStats'); - console.log( - JSON.stringify({ - indexPatternTitle: this._indexPatternTitle, - query, - timeFieldName: this._indexPattern.timeFieldName, - earliest, - latest, - samplerShardSize, - interval, - fields, - maxExamples: this._maxExamples, - runtimeMappings: this._runtimeMappings, - }) - ); const stats = await getVisualizerFieldStats({ indexPatternTitle: this._indexPatternTitle, query, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 1a7f44fdffef2..ab52c4a05e74d 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -17,7 +17,7 @@ import { MetricFieldsStats } from '../../common/components/stats_table/component import { DataLoader } from '../data_loader/data_loader'; import { useTimefilter } from './use_time_filter'; import { dataVisualizerRefresh$ } from '../services/timefilter_refresh_service'; -import { TimeBuckets } from '../services/time_buckets'; +import { TimeBuckets } from '../../../../common/services/time_buckets'; import { DataViewField, KBN_FIELD_TYPES, @@ -63,6 +63,7 @@ export const useDataVisualizerGridData = ( [input] ); + /** Prepare required params to pass to search strategy **/ const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { const searchData = getEsQueryFromSavedSearch({ indexPattern: currentIndexPattern, @@ -99,7 +100,6 @@ export const useDataVisualizerGridData = ( if (currentSearchSessionId !== undefined) { setSearchSessionId(currentSearchSessionId); } - console.log('currentSearchSessionId', currentSearchSessionId); }, [data]); const _timeBuckets = useMemo(() => { @@ -111,17 +111,11 @@ export const useDataVisualizerGridData = ( }); }, [uiSettings]); - /** Search strategy**/ - const strategyResponse = useFieldStatsSearchStrategy({ - query: searchQuery, - sessionId: searchSessionId, - timeBuckets: _timeBuckets, - indexPattern: currentIndexPattern, - savedSearch: currentSavedSearch, - timeFilter: timefilter, + const timefilter = useTimefilter({ + timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, + autoRefreshSelector: true, }); - console.log('strategyResponse', strategyResponse); const [overallStats, setOverallStats] = useState(defaults.overallStats); const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); @@ -132,16 +126,80 @@ export const useDataVisualizerGridData = ( const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); + /** Search strategy **/ + const fieldStatsRequest = useMemo(() => { + // Obtain the interval to use for date histogram aggregations + // (such as the document count chart). Aim for 75 bars. + const buckets = _timeBuckets; + + const tf = timefilter as any; + + if (!buckets || !tf || !currentIndexPattern) return; + + const activeBounds = tf.getActiveBounds(); + let earliest: number | undefined; + let latest: number | undefined; + if (activeBounds !== undefined && currentIndexPattern.timeFieldName !== undefined) { + earliest = tf.getActiveBounds().min.valueOf(); + latest = tf.getActiveBounds().max.valueOf(); + } + + const bounds = tf.getActiveBounds(); + const BAR_TARGET = 75; + buckets.setInterval('auto'); + buckets.setBounds(bounds); + buckets.setBarTarget(BAR_TARGET); + const aggInterval = buckets.getInterval(); + + const existMetricFields: FieldRequestConfig[] = metricConfigs.map((config) => { + const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; + if (config.stats !== undefined && config.stats.cardinality !== undefined) { + props.cardinality = config.stats.cardinality; + } + return props; + }); + + // Pass the field name, type and cardinality in the request. + // Top values will be obtained on a sample if cardinality > 100000. + const existNonMetricFields: FieldRequestConfig[] = nonMetricConfigs.map((config) => { + const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; + if (config.stats !== undefined && config.stats.cardinality !== undefined) { + props.cardinality = config.stats.cardinality; + } + return props; + }); + + return { + earliest, + latest, + aggInterval, + intervalMs: aggInterval?.asMilliseconds(), + searchQuery, + samplerShardSize, + sessionId: searchSessionId, + index: currentIndexPattern.title, + timeFieldName: currentIndexPattern.timeFieldName, + runtimeFieldMap: currentIndexPattern.getComputedFields().runtimeFields, + metricConfigs: existMetricFields, + nonMetricConfigs: existNonMetricFields, + }; + }, [ + _timeBuckets, + timefilter, + currentIndexPattern, + searchQuery, + samplerShardSize, + searchSessionId, + metricConfigs, + nonMetricConfigs, + ]); + + const strategyResponse = useFieldStatsSearchStrategy(fieldStatsRequest); const dataLoader = useMemo( () => new DataLoader(currentIndexPattern, toasts), [currentIndexPattern, toasts] ); - const timefilter = useTimefilter({ - timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, - autoRefreshSelector: true, - }); - useEffect(() => { const timeUpdateSubscription = merge( timefilter.getTimeUpdate$(), diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts index 630a2eafb8709..c3870fae27f74 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts @@ -9,34 +9,23 @@ import { useCallback, useEffect, useReducer, useRef } from 'react'; import { Subscription } from 'rxjs'; import { isCompleteResponse, isErrorResponse } from '../../../../../../../src/plugins/data/common'; import { FIELD_STATS_SEARCH_STRATEGY } from '../../../../common/search_strategy/constants'; -import { +import type { FieldStatRawResponse, FieldStatsRequest, FieldStatsResponse, + FieldStatsSearchStrategyParams, + FieldStatsSearchStrategyProgress, + FieldStatsSearchStrategyReturnBase, } from '../../../../common/search_strategy/types'; import { useDataVisualizerKibana } from '../../kibana_context'; -interface SearchStrategyReturnBase { - progress: SearchStrategyProgress; - response: TRawResponse; - startFetch: () => void; - cancelFetch: () => void; -} - -interface SearchStrategyProgress { - error?: Error; - isRunning: boolean; - loaded: number; - total: number; -} - const getInitialRawResponse = (): TRawResponse => ({ ccsWarning: false, took: 0, } as TRawResponse); -const getInitialProgress = (): SearchStrategyProgress => ({ +const getInitialProgress = (): FieldStatsSearchStrategyProgress => ({ isRunning: false, loaded: 0, total: 100, @@ -49,16 +38,10 @@ const getReducer = ...update, }); -interface SearchStrategyParams { - sessionId?: string; - // @todo update type - query?: any; -} export function useFieldStatsSearchStrategy< TRawResponse extends FieldStatRawResponse, - TParams extends SearchStrategyParams ->(searchStrategyParams?: TParams): SearchStrategyReturnBase { - console.log('searchStrategyParams', searchStrategyParams); + TParams extends FieldStatsSearchStrategyParams +>(searchStrategyParams: TParams | undefined): FieldStatsSearchStrategyReturnBase { const { services: { data }, } = useDataVisualizerKibana(); @@ -69,13 +52,14 @@ export function useFieldStatsSearchStrategy< ); const [fetchState, setFetchState] = useReducer( - getReducer(), + getReducer(), getInitialProgress() ); const abortCtrl = useRef(new AbortController()); const searchSubscription$ = useRef(); - const searchStrategyParamsRef = useRef(searchStrategyParams); + + console.log('rawResponse', rawResponse); const startFetch = useCallback(() => { searchSubscription$.current?.unsubscribe(); @@ -87,7 +71,7 @@ export function useFieldStatsSearchStrategy< }); const request = { - params: {}, + params: searchStrategyParams, }; // Submit the search request using the `data.search` service. @@ -97,7 +81,8 @@ export function useFieldStatsSearchStrategy< abortSignal: abortCtrl.current.signal, }) .subscribe({ - next: (response: FieldStatsResponse) => { + next: (response) => { + // Setting results to latest even if the response is still partial setRawResponse(response.rawResponse); setFetchState({ isRunning: response.isRunning || false, @@ -106,6 +91,7 @@ export function useFieldStatsSearchStrategy< }); if (isCompleteResponse(response)) { + // If the whole request is completed searchSubscription$.current?.unsubscribe(); setFetchState({ isRunning: false, @@ -116,6 +102,8 @@ export function useFieldStatsSearchStrategy< error: response as unknown as Error, isRunning: false, }); + } else { + // If is partial response } }, error: (error: Error) => { @@ -125,7 +113,7 @@ export function useFieldStatsSearchStrategy< }); }, }); - }, [data.search]); + }, [data.search, searchStrategyParams]); const cancelFetch = useCallback(() => { searchSubscription$.current?.unsubscribe(); diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/constants.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/constants.ts index 91bd394aee797..4dbe51e5c0838 100644 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/constants.ts +++ b/x-pack/plugins/data_visualizer/server/models/data_visualizer/constants.ts @@ -11,3 +11,6 @@ export const AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE = 200; export const FIELDS_REQUEST_BATCH_SIZE = 10; export const MAX_CHART_COLUMNS = 20; + +export const MAX_PERCENT = 100; +export const PERCENTILE_SPACING = 5; diff --git a/x-pack/plugins/data_visualizer/server/plugin.ts b/x-pack/plugins/data_visualizer/server/plugin.ts index 9adfc9718ee70..8a98793274aa2 100644 --- a/x-pack/plugins/data_visualizer/server/plugin.ts +++ b/x-pack/plugins/data_visualizer/server/plugin.ts @@ -8,31 +8,8 @@ import { CoreSetup, CoreStart, Plugin } from 'src/core/server'; import { StartDeps, SetupDeps } from './types'; import { dataVisualizerRoutes } from './routes'; -import { ENHANCED_ES_SEARCH_STRATEGY } from '../../../../src/plugins/data/common'; import { FIELD_STATS_SEARCH_STRATEGY } from '../common/search_strategy/constants'; -import type { FieldStatsRequest, FieldStatsResponse } from '../common/search_strategy/types'; -import type { ISearchStrategy } from '../../../../src/plugins/data/server'; - -// @todo: rename -export const myEnhancedSearchStrategyProvider = ( - data: StartDeps['data'] -): ISearchStrategy => { - // Get the default search strategy - const ese = data.search.getSearchStrategy(ENHANCED_ES_SEARCH_STRATEGY); - return { - search: (request, options, deps) => { - // search will be called multiple times, - // be sure your response formatting is capable of handling partial results, as well as the final result. - return ese.search(request, options, deps); - }, - cancel: async (id, options, deps) => { - // call the cancel method of the async strategy you are using or implement your own cancellation function. - if (ese.cancel) { - await ese.cancel(id, options, deps); - } - }, - }; -}; +import { myEnhancedSearchStrategyProvider } from './search_strategy/field_stats_search_strategy'; export class DataVisualizerPlugin implements Plugin { constructor() {} diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts index a3e4216d8d739..969a6d6c7e2d3 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts @@ -6,16 +6,27 @@ */ import { ElasticsearchClient } from 'kibana/server'; +import uuid from 'uuid'; +import { of } from 'rxjs'; import { fieldStatsSearchServiceStateProvider } from './field_stats_state_provider'; +import { Field, StartDeps } from '../types'; +import { ISearchStrategy } from '../../../../../src/plugins/data/server'; +import { + FieldStatRawResponse, + FieldStatsRequest, + FieldStatsResponse, + FieldStatsSearchStrategyParams, + isFieldStatsSearchStrategyParams, +} from '../../common/search_strategy/types'; +import { ENHANCED_ES_SEARCH_STRATEGY } from '../../../../../src/plugins/data/common'; +import { getFieldStats } from './requests/get_field_stats'; export interface RawResponseBase { ccsWarning: boolean; took: number; } -export interface FieldStatsRequestParams { - sessionId?: string; -} +export type FieldStatsRequestParams = FieldStatsSearchStrategyParams; export interface SearchStrategyServerParams { includeFrozen?: boolean; @@ -37,7 +48,7 @@ interface SearchServiceState { rawResponse: TRawResponse; } -type GetSearchServiceState = +type GetSearchServiceState = () => SearchServiceState; export type SearchServiceProvider< @@ -45,30 +56,112 @@ export type SearchServiceProvider< TRawResponse extends RawResponseBase > = ( esClient: ElasticsearchClient, - searchServiceParams: TSearchStrategyClientParams, - includeFrozen: boolean -) => GetSearchServiceState; - -export type FieldStatsSearchServiceProvider = SearchServiceProvider< - FieldStatsRequestParams, - FieldStatsRawResponse ->; + searchServiceParams: TSearchStrategyClientParams +) => GetSearchServiceState; + +// @todo: rename +export const myEnhancedSearchStrategyProvider = ( + data: StartDeps['data'] +): ISearchStrategy => { + // Get the default search strategy + const ese = data.search.getSearchStrategy(ENHANCED_ES_SEARCH_STRATEGY); + const searchServiceMap = new Map>(); + + return { + search: (request, options, deps) => { + if (request.params === undefined) { + throw new Error('Invalid request parameters.'); + } + + // return formatResponse(ese.search(preprocessRequest(request.params), options, deps)); + + // The function to fetch the current state of the search service. + // This will be either an existing service for a follow up fetch or a new one for new requests. + let getSearchServiceState: GetSearchServiceState; + + // If the request includes an ID, we require that the search service already exists + // otherwise we throw an error. The client should never poll a service that's been cancelled or finished. + // This also avoids instantiating search services when the service gets called with random IDs. + if (typeof request.id === 'string') { + const existingGetSearchServiceState = searchServiceMap.get(request.id); + + if (typeof existingGetSearchServiceState === 'undefined') { + throw new Error(`SearchService with ID '${request.id}' does not exist.`); + } + + getSearchServiceState = existingGetSearchServiceState; + } else { + getSearchServiceState = fieldStatsSearchServiceProvider( + deps.esClient.asCurrentUser, + request + ); + } + // Reuse the request's id or create a new one. + const id = request.id ?? uuid(); + + const { error, meta, rawResponse } = getSearchServiceState(); + + if (error instanceof Error) { + searchServiceMap.delete(id); + throw error; + } else if (meta.isRunning) { + searchServiceMap.set(id, getSearchServiceState); + } else { + searchServiceMap.delete(id); + } + + return of({ + id, + ...meta, + rawResponse, + }); + // search will be called multiple times, + // be sure your response formatting is capable of handling partial results, as well as the final result. + // return ese.search(request, options, deps); + }, + cancel: async (id, options, deps) => { + // call the cancel method of the async strategy you are using or implement your own cancellation function. + if (ese.cancel) { + await ese.cancel(id, options, deps); + } + }, + }; +}; -export const fieldStatsSearchServiceProvider: FieldStatsSearchServiceProvider = ( +export const fieldStatsSearchServiceProvider = ( esClient: ElasticsearchClient, - searchServiceParams: FieldStatsRequestParams, - includeFrozen: boolean + request: FieldStatsRequest ) => { const state = fieldStatsSearchServiceStateProvider(); async function fetchCorrelations() { - let params: (FieldStatsRequestParams & SearchStrategyServerParams) | undefined; - try { - params = { - ...searchServiceParams, - includeFrozen, - }; + if (isFieldStatsSearchStrategyParams(request.params)) { + const params = request.params; + const fields = [...params.metricConfigs, ...params.nonMetricConfigs]; + const fieldToLoadCnt = fields.length; + let fieldsLoadedCnt = 0; + + for (let idx = 0; idx < fieldToLoadCnt; idx++) { + const field = { + fieldName: fields[idx].fieldName, + type: fields[idx].type, + cardinality: fields[idx].cardinality, + identifier: idx, + }; + + try { + const testMetricFieldResult = await getFieldStats(esClient, params, field, idx); + state.addFieldStats(testMetricFieldResult); + } catch (e) { + console.error(e); + // @todo: Log error + } + + fieldsLoadedCnt += 1; + state.setProgress({ loadedFieldStats: fieldsLoadedCnt / fieldToLoadCnt }); + } + } } catch (e) { state.setError(e); } @@ -104,6 +197,7 @@ export const fieldStatsSearchServiceProvider: FieldStatsSearchServiceProvider = ccsWarning, log: [], took: 0, + fieldStats: state.getState().fieldsStats, }, }; }; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts index 9d099cdf22e1e..fff7190cc108b 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts @@ -5,18 +5,16 @@ * 2.0. */ -import { PluginStart as DataPluginStart } from '../../../../../src/plugins/data/server'; +import type { AggregationsAggregate } from '@elastic/elasticsearch/api/types'; export interface LatencyCorrelationSearchServiceProgress { started: number; - loadedHistogramStepsize: number; - loadedOverallHistogram: number; - loadedFieldCandidates: number; - loadedFieldValuePairs: number; - loadedHistograms: number; + loadedOverallStats: number; + loadedFieldStats: number; } -export const fieldStatsSearchServiceStateProvider = (dataPlugin: DataPluginStart) => { +type FieldStat = Record | undefined; +export const fieldStatsSearchServiceStateProvider = () => { let ccsWarning = false; function setCcsWarning(d: boolean) { ccsWarning = d; @@ -40,22 +38,18 @@ export const fieldStatsSearchServiceStateProvider = (dataPlugin: DataPluginStart isRunning = d; } + const fieldsStats: FieldStat[] = []; + function addFieldStats(d: FieldStat) { + fieldsStats.push(d); + } + let progress: LatencyCorrelationSearchServiceProgress = { started: Date.now(), - loadedHistogramStepsize: 0, - loadedOverallHistogram: 0, - loadedFieldCandidates: 0, - loadedFieldValuePairs: 0, - loadedHistograms: 0, + loadedOverallStats: 0, + loadedFieldStats: 0, }; function getOverallProgress() { - return ( - progress.loadedHistogramStepsize * 0.025 + - progress.loadedOverallHistogram * 0.025 + - progress.loadedFieldCandidates * 0.025 + - progress.loadedFieldValuePairs * 0.025 + - progress.loadedHistograms * 0.9 - ); + return progress.loadedOverallStats * 0.1 + progress.loadedFieldStats * 0.9; } function setProgress(d: Partial>) { progress = { @@ -71,6 +65,7 @@ export const fieldStatsSearchServiceStateProvider = (dataPlugin: DataPluginStart isCancelled, isRunning, progress, + fieldsStats, }; } @@ -83,6 +78,7 @@ export const fieldStatsSearchServiceStateProvider = (dataPlugin: DataPluginStart setIsCancelled, setIsRunning, setProgress, + addFieldStats, }; }; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_boolean_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_boolean_field_stats.ts new file mode 100644 index 0000000000000..619ef27419c56 --- /dev/null +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_boolean_field_stats.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { get } from 'lodash'; +import { FieldStatsCommonRequestParams } from '../../../common/search_strategy/types'; +import { Field, BooleanFieldStats, Aggs } from '../../types'; +import { + buildBaseFilterCriteria, + buildSamplerAggregation, + getSafeAggregationName, + getSamplerAggregationsResponsePath, +} from '../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../common/utils/object_utils'; + +export const getBooleanFieldStatsRequest = ( + params: FieldStatsCommonRequestParams, + field: Field +) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = + params; + + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + const aggs: Aggs = {}; + + const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + aggs[`${safeFieldName}_value_count`] = { + filter: { exists: { field: field.fieldName } }, + }; + aggs[`${safeFieldName}_values`] = { + terms: { + field: field.fieldName, + size: 2, + }, + }; + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchBooleanFieldStats = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams, + field: Field +): Promise => { + const { samplerShardSize } = params; + const request: SearchRequest = getBooleanFieldStatsRequest(params, field); + const { body } = await esClient.search(request); + const aggregations = body.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + + const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + const stats: BooleanFieldStats = { + fieldName: field.fieldName, + count: get(aggregations, [...aggsPath, `${safeFieldName}_value_count`, 'doc_count'], 0), + trueCount: 0, + falseCount: 0, + }; + + const valueBuckets: Array<{ [key: string]: number }> = get( + aggregations, + [...aggsPath, `${safeFieldName}_values`, 'buckets'], + [] + ); + valueBuckets.forEach((bucket) => { + stats[`${bucket.key_as_string}Count`] = bucket.doc_count; + }); + return stats; +}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_date_field_stats.ts new file mode 100644 index 0000000000000..9e61febe127cd --- /dev/null +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_date_field_stats.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { get } from 'lodash'; +import { FieldStatsCommonRequestParams } from '../../../common/search_strategy/types'; +import { Aggs, DateFieldStats, Field } from '../../types'; +import { + buildBaseFilterCriteria, + buildSamplerAggregation, + getSafeAggregationName, + getSamplerAggregationsResponsePath, +} from '../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../common/utils/object_utils'; + +export const getDateFieldStatsRequest = (params: FieldStatsCommonRequestParams, field: Field) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = + params; + + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + const aggs: Aggs = {}; + const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + aggs[`${safeFieldName}_field_stats`] = { + filter: { exists: { field: field.fieldName } }, + aggs: { + actual_stats: { + stats: { field: field.fieldName }, + }, + }, + }; + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchDateFieldStats = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams, + field: Field +): Promise => { + const { samplerShardSize } = params; + + const request: SearchRequest = getDateFieldStatsRequest(params, field); + const { body } = await esClient.search(request); + + const aggregations = body.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + const docCount = get(aggregations, [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], 0); + const fieldStatsResp = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], + {} + ); + return { + fieldName: field.fieldName, + count: docCount, + earliest: get(fieldStatsResp, 'min', 0), + latest: get(fieldStatsResp, 'max', 0), + }; +}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_document_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_document_stats.ts new file mode 100644 index 0000000000000..9b11d4a7ac2d5 --- /dev/null +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_document_stats.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { each, get } from 'lodash'; +import { FieldStatsCommonRequestParams } from '../../../common/search_strategy/types'; +import { buildBaseFilterCriteria } from '../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../common/utils/object_utils'; +import { DocumentCountStats } from '../../types'; + +export const getDocumentCountStatsRequest = (params: FieldStatsCommonRequestParams) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, intervalMs } = params; + + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + // Don't use the sampler aggregation as this can lead to some potentially + // confusing date histogram results depending on the date range of data amongst shards. + + const aggs = { + eventRate: { + date_histogram: { + field: timeFieldName, + fixed_interval: `${intervalMs}ms`, + min_doc_count: 1, + }, + }, + }; + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs, + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchDocumentCountStats = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams +): Promise => { + const { intervalMs } = params; + const request: SearchRequest = getDocumentCountStatsRequest(params); + + const { body } = await esClient.search(request); + + const buckets: { [key: string]: number } = {}; + const dataByTimeBucket: Array<{ key: string; doc_count: number }> = get( + body, + ['aggregations', 'eventRate', 'buckets'], + [] + ); + each(dataByTimeBucket, (dataForTime) => { + const time = dataForTime.key; + buckets[time] = dataForTime.doc_count; + }); + + return { + documentCounts: { + interval: intervalMs, + buckets, + }, + }; +}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts new file mode 100644 index 0000000000000..8bd3b9c80f88a --- /dev/null +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { get } from 'lodash'; +import { FieldStatsCommonRequestParams } from '../../../common/search_strategy/types'; +import { Field, FieldExamples } from '../../types'; +import { buildBaseFilterCriteria } from '../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../common/utils/object_utils'; + +// @todo +const maxExamples = 10; +export const getFieldExamplesRequest = (params: FieldStatsCommonRequestParams, field: Field) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap } = params; + + // Request at least 100 docs so that we have a chance of obtaining + // 'maxExamples' of the field. + const size = Math.max(100, maxExamples); + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + // Use an exists filter to return examples of the field. + filterCriteria.push({ + exists: { field }, + }); + + const searchBody = { + fields: [field], + _source: false, + query: { + bool: { + filter: filterCriteria, + }, + }, + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchFieldExamples = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams, + field: Field +): Promise => { + const request: SearchRequest = getFieldExamplesRequest(params, field); + const { body } = await esClient.search(request); + + const stats = { + fieldName: field, + examples: [] as any[], + }; + // @ts-expect-error incorrect search response type + if (body.hits.total.value > 0) { + const hits = body.hits.hits; + for (let i = 0; i < hits.length; i++) { + // Use lodash get() to support field names containing dots. + const doc: object[] | undefined = get(hits[i].fields, field); + // the results from fields query is always an array + if (Array.isArray(doc) && doc.length > 0) { + const example = doc[0]; + if (example !== undefined && stats.examples.indexOf(example) === -1) { + stats.examples.push(example); + if (stats.examples.length === maxExamples) { + break; + } + } + } + } + } + + return stats; +}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts new file mode 100644 index 0000000000000..47c256c6e2d2e --- /dev/null +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { JOB_FIELD_TYPES } from '../../../common'; +import { + FieldStatsCommonRequestParams, + FieldStatsSearchStrategyParams, +} from '../../../common/search_strategy/types'; +import { isValidField } from '../../types'; +import { fetchDocumentCountStats } from './get_document_stats'; +import { fetchNumericFieldStats } from './get_numeric_field_stats'; +import { fetchStringFieldStats } from './get_string_field_stats'; +import { fetchDateFieldStats } from './get_date_field_stats'; +import { fetchBooleanFieldStats } from './get_boolean_field_stats'; +import { fetchFieldExamples } from './get_field_examples'; + +export const getFieldStats = async ( + esClient: ElasticsearchClient, + searchStrategyParams: FieldStatsSearchStrategyParams, + field: { + fieldName?: string; + type: string; + cardinality: number; + } +) => { + const params: FieldStatsCommonRequestParams = { + index: searchStrategyParams.index, + query: searchStrategyParams.searchQuery, + samplerShardSize: searchStrategyParams.samplerShardSize, + timeFieldName: searchStrategyParams.timeFieldName, + earliestMs: searchStrategyParams.earliest, + latestMs: searchStrategyParams.latest, + runtimeFieldMap: searchStrategyParams.runtimeFieldMap, + intervalMs: searchStrategyParams.intervalMs, + }; + + // An invalid field with undefined fieldName is used for a document count request. + if (!isValidField(field)) { + // Will only ever be one document count card, + // so no value in batching up the single request. + if (field.type === JOB_FIELD_TYPES.NUMBER && params.intervalMs !== undefined) { + return fetchDocumentCountStats(esClient, params); + } + } else { + switch (field.type) { + case JOB_FIELD_TYPES.NUMBER: + return fetchNumericFieldStats(esClient, params, field); + break; + case JOB_FIELD_TYPES.KEYWORD: + case JOB_FIELD_TYPES.IP: + return fetchStringFieldStats(esClient, params, field); + break; + case JOB_FIELD_TYPES.DATE: + return fetchDateFieldStats(esClient, params, field); + break; + case JOB_FIELD_TYPES.BOOLEAN: + return fetchBooleanFieldStats(esClient, params, field); + break; + case JOB_FIELD_TYPES.TEXT: + default: + // Use an exists filter on the the field name to get + // examples of the field, so cannot batch up. + return fetchFieldExamples(esClient, params, field); + break; + } + } +}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_numeric_field_stats.ts new file mode 100644 index 0000000000000..d26f937b51c74 --- /dev/null +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_numeric_field_stats.ts @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { find, get } from 'lodash'; +import { Bucket, Field, NumericFieldStats } from '../../types'; +import { + buildBaseFilterCriteria, + buildSamplerAggregation, + getSafeAggregationName, + getSamplerAggregationsResponsePath, +} from '../../../common/utils/query_utils'; +import { + MAX_PERCENT, + PERCENTILE_SPACING, + SAMPLER_TOP_TERMS_SHARD_SIZE, + SAMPLER_TOP_TERMS_THRESHOLD, +} from '../../models/data_visualizer/constants'; +import { isPopulatedObject } from '../../../common/utils/object_utils'; +import { FieldStatsCommonRequestParams } from '../../../common/search_strategy/types'; +import { processDistributionData } from '../../models/data_visualizer/process_distribution_data'; + +export const getNumericFieldStatsRequest = ( + params: FieldStatsCommonRequestParams, + field: Field +) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = + params; + + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + // Build the percents parameter which defines the percentiles to query + // for the metric distribution data. + // Use a fixed percentile spacing of 5%. + let count = 0; + const percents = Array.from( + Array(MAX_PERCENT / PERCENTILE_SPACING), + () => (count += PERCENTILE_SPACING) + ); + + const aggs: { [key: string]: any } = {}; + // @todo: check if field.cardinality is a proper replacement for index + const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + aggs[`${safeFieldName}_field_stats`] = { + filter: { exists: { field: field.fieldName } }, + aggs: { + actual_stats: { + stats: { field: field.fieldName }, + }, + }, + }; + aggs[`${safeFieldName}_percentiles`] = { + percentiles: { + field: field.fieldName, + percents, + keyed: false, + }, + }; + + const top = { + terms: { + field: field.fieldName, + size: 10, + order: { + _count: 'desc', + }, + }, + }; + + // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation + // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + aggs[`${safeFieldName}_top`] = { + sampler: { + shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, + }, + aggs: { + top, + }, + }; + } else { + aggs[`${safeFieldName}_top`] = top; + } + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchNumericFieldStats = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams, + field: Field +): Promise => { + const { samplerShardSize } = params; + const request: SearchRequest = getNumericFieldStatsRequest(params, field); + const { body } = await esClient.search(request); + const aggregations = body.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + + const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + const docCount = get(aggregations, [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], 0); + const fieldStatsResp = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], + {} + ); + + const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + topAggsPath.push('top'); + } + + const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); + + const stats: NumericFieldStats = { + fieldName: field.fieldName, + count: docCount, + min: get(fieldStatsResp, 'min', 0), + max: get(fieldStatsResp, 'max', 0), + avg: get(fieldStatsResp, 'avg', 0), + isTopValuesSampled: field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, + topValues, + topValuesSampleSize: topValues.reduce( + (acc, curr) => acc + curr.doc_count, + get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) + ), + topValuesSamplerShardSize: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? SAMPLER_TOP_TERMS_SHARD_SIZE + : samplerShardSize, + }; + + if (stats.count > 0) { + const percentiles = get( + aggregations, + [...aggsPath, `${safeFieldName}_percentiles`, 'values'], + [] + ); + const medianPercentile: { value: number; key: number } | undefined = find(percentiles, { + key: 50, + }); + stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; + stats.distribution = processDistributionData(percentiles, PERCENTILE_SPACING, stats.min); + } + return stats; +}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_string_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_string_field_stats.ts new file mode 100644 index 0000000000000..59613e821cf1b --- /dev/null +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_string_field_stats.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { get } from 'lodash'; +import { FieldStatsCommonRequestParams } from '../../../common/search_strategy/types'; +import { Aggs, Bucket, Field, StringFieldStats } from '../../types'; +import { + buildBaseFilterCriteria, + buildSamplerAggregation, + getSafeAggregationName, + getSamplerAggregationsResponsePath, +} from '../../../common/utils/query_utils'; +import { + SAMPLER_TOP_TERMS_SHARD_SIZE, + SAMPLER_TOP_TERMS_THRESHOLD, +} from '../../models/data_visualizer/constants'; +import { isPopulatedObject } from '../../../common/utils/object_utils'; + +export const getStringFieldStatsRequest = (params: FieldStatsCommonRequestParams, field: Field) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = + params; + + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + const aggs: Aggs = {}; + + const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + const top = { + terms: { + field: field.fieldName, + size: 10, + order: { + _count: 'desc', + }, + }, + }; + + // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation + // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + aggs[`${safeFieldName}_top`] = { + sampler: { + shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, + }, + aggs: { + top, + }, + }; + } else { + aggs[`${safeFieldName}_top`] = top; + } + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchStringFieldStats = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams, + field: Field +): Promise => { + const { samplerShardSize } = params; + const request: SearchRequest = getStringFieldStatsRequest(params, field); + + const { body } = await esClient.search(request); + + const aggregations = body.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + + const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + + const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + topAggsPath.push('top'); + } + + const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); + + const stats = { + fieldName: field.fieldName, + isTopValuesSampled: field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, + topValues, + topValuesSampleSize: topValues.reduce( + (acc, curr) => acc + curr.doc_count, + get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) + ), + topValuesSamplerShardSize: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? SAMPLER_TOP_TERMS_SHARD_SIZE + : samplerShardSize, + }; + + return stats; +}; diff --git a/x-pack/plugins/data_visualizer/server/types/chart_data.ts b/x-pack/plugins/data_visualizer/server/types/chart_data.ts index 99c23cf88b5ba..f43114f68952c 100644 --- a/x-pack/plugins/data_visualizer/server/types/chart_data.ts +++ b/x-pack/plugins/data_visualizer/server/types/chart_data.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { isPopulatedObject } from '../../common/utils/object_utils'; + export interface FieldData { fieldName: string; existsInDocs: boolean; @@ -19,6 +21,11 @@ export interface Field { fieldName: string; type: string; cardinality: number; + identifier: number; +} + +export function isValidField(arg: unknown): arg is Field { + return isPopulatedObject(arg, ['fieldName', 'type']) && typeof arg.fieldName === 'string'; } export interface HistogramField { From 2759e5784edc2a646297102806a0f788d8641978 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 4 Oct 2021 18:30:24 -0500 Subject: [PATCH 124/188] Refactor helper functions --- .../apm/public/hooks/use_search_strategy.ts | 1 + .../common/search_strategy/types.ts | 13 +- .../common/utils/query_utils.ts | 2 +- .../hooks/use_data_visualizer_grid_data.ts | 206 +++--------------- .../hooks/use_search_strategy.ts | 11 +- .../field_stats_search_strategy.ts | 136 ++++++++---- .../field_stats_state_provider.ts | 11 +- .../requests/get_boolean_field_stats.ts | 5 +- .../requests/get_date_field_stats.ts | 5 +- .../requests/get_field_examples.ts | 6 +- .../requests/get_field_stats.ts | 29 +-- .../requests/get_numeric_field_stats.ts | 6 +- .../requests/get_string_field_stats.ts | 5 +- .../server/types/chart_data.ts | 14 +- 14 files changed, 177 insertions(+), 273 deletions(-) diff --git a/x-pack/plugins/apm/public/hooks/use_search_strategy.ts b/x-pack/plugins/apm/public/hooks/use_search_strategy.ts index ca8d28b106f84..ab0ea214c62bd 100644 --- a/x-pack/plugins/apm/public/hooks/use_search_strategy.ts +++ b/x-pack/plugins/apm/public/hooks/use_search_strategy.ts @@ -171,6 +171,7 @@ export function useSearchStrategy< error: response as unknown as Error, isRunning: false, }); + } else { } }, error: (error: Error) => { diff --git a/x-pack/plugins/data_visualizer/common/search_strategy/types.ts b/x-pack/plugins/data_visualizer/common/search_strategy/types.ts index a65869202dab4..c3e67ecc9213b 100644 --- a/x-pack/plugins/data_visualizer/common/search_strategy/types.ts +++ b/x-pack/plugins/data_visualizer/common/search_strategy/types.ts @@ -5,25 +5,27 @@ * 2.0. */ +import { estypes } from '@elastic/elasticsearch'; import type { IKibanaSearchRequest, IKibanaSearchResponse, } from '../../../../../src/plugins/data/common'; import type { TimeBucketsInterval } from '../services/time_buckets'; -import { RuntimeField } from '../../../../../src/plugins/data/common'; -import { FieldRequestConfig } from '../types'; -import { ISearchStrategy } from '../../../../../src/plugins/data/server'; +import type { RuntimeField } from '../../../../../src/plugins/data/common'; +import type { FieldRequestConfig } from '../types'; +import type { ISearchStrategy } from '../../../../../src/plugins/data/server'; import { isPopulatedObject } from '../utils/object_utils'; +import { FieldStats } from '../../server/types'; export interface FieldStatsCommonRequestParams { index: string; - query: any; samplerShardSize: number; timeFieldName?: string; earliestMs?: number | undefined; latestMs?: number | undefined; runtimeFieldMap?: Record; intervalMs?: number; + query: estypes.QueryDslQueryContainer; } export interface FieldStatsSearchStrategyParams { @@ -49,8 +51,9 @@ export function isFieldStatsSearchStrategyParams( export interface FieldStatRawResponse { loading?: boolean; - ccsWarning: false; + ccsWarning: boolean; took: 0; + fieldStats: Record; } export type FieldStatsRequest = IKibanaSearchRequest; export type FieldStatsResponse = IKibanaSearchResponse; diff --git a/x-pack/plugins/data_visualizer/common/utils/query_utils.ts b/x-pack/plugins/data_visualizer/common/utils/query_utils.ts index d2785072f419d..647bf1781d34a 100644 --- a/x-pack/plugins/data_visualizer/common/utils/query_utils.ts +++ b/x-pack/plugins/data_visualizer/common/utils/query_utils.ts @@ -17,7 +17,7 @@ export function buildBaseFilterCriteria( earliestMs?: number, latestMs?: number, query?: object -) { +): estypes.QueryDslBoolQuery['filter'] { const filterCriteria = []; if (timeFieldName && earliestMs && latestMs) { filterCriteria.push({ diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index ab52c4a05e74d..fc7d92255180b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -90,7 +90,9 @@ export const useDataVisualizerGridData = ( }, [ currentSavedSearch, currentIndexPattern, - dataVisualizerListState, + dataVisualizerListState.searchQuery, + dataVisualizerListState.searchString, + dataVisualizerListState.searchQueryLanguage, currentQuery, currentFilters, ]); @@ -279,15 +281,16 @@ export const useDataVisualizerGridData = ( }); }); + // @todo: remove // Add a config for 'document count', identified by no field name if indexpattern is time based. - if (currentIndexPattern.timeFieldName !== undefined) { - configs.push({ - type: JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - loading: true, - aggregatable: true, - }); - } + // if (currentIndexPattern.timeFieldName !== undefined) { + // configs.push({ + // type: JOB_FIELD_TYPES.NUMBER, + // existsInDocs: true, + // loading: true, + // aggregatable: true, + // }); + // } if (metricsLoaded === false) { setMetricsLoaded(true); @@ -429,154 +432,6 @@ export const useDataVisualizerGridData = ( showEmptyFields, ]); - async function loadMetricFieldStats() { - // Only request data for fields that exist in documents. - if (metricConfigs.length === 0) { - return; - } - - const configsToLoad = metricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - // Obtain the interval to use for date histogram aggregations - // (such as the document count chart). Aim for 75 bars. - const buckets = getTimeBuckets(); - - const tf = timefilter as any; - let earliest: number | undefined; - let latest: number | undefined; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - const bounds = tf.getActiveBounds(); - const BAR_TARGET = 75; - buckets.setInterval('auto'); - buckets.setBounds(bounds); - buckets.setBarTarget(BAR_TARGET); - const aggInterval = buckets.getInterval(); - - try { - const metricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existMetricFields, - aggInterval.asMilliseconds() - ); - - // Add the metric stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - metricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - configWithStats.loading = false; - configs.push(configWithStats); - } else { - // Document count card. - configWithStats.stats = metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === undefined - ); - - if (configWithStats.stats !== undefined) { - // Add earliest / latest of timefilter for setting x axis domain. - configWithStats.stats.timeRangeEarliest = earliest; - configWithStats.stats.timeRangeLatest = latest; - } - setDocumentCountStats(configWithStats); - } - }); - - setMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - async function loadNonMetricFieldStats() { - // Only request data for fields that exist in documents. - if (nonMetricConfigs.length === 0) { - return; - } - - const configsToLoad = nonMetricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existNonMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - const tf = timefilter as any; - let earliest; - let latest; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - try { - const nonMetricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existNonMetricFields - ); - - // Add the field stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - nonMetricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...nonMetricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - } - configWithStats.loading = false; - configs.push(configWithStats); - }); - - setNonMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - useEffect(() => { loadOverallStats(); // eslint-disable-next-line react-hooks/exhaustive-deps @@ -588,27 +443,8 @@ export const useDataVisualizerGridData = ( // eslint-disable-next-line react-hooks/exhaustive-deps }, [overallStats, showEmptyFields]); - useEffect(() => { - loadMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricConfigs]); - - useEffect(() => { - loadNonMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricConfigs]); - - useEffect(() => { - createMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricsLoaded]); - - useEffect(() => { - createNonMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricsLoaded]); - const configs = useMemo(() => { + const fieldStats = strategyResponse.response.fieldStats; let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; if (visibleFieldTypes && visibleFieldTypes.length > 0) { combinedConfigs = combinedConfigs.filter( @@ -621,8 +457,22 @@ export const useDataVisualizerGridData = ( ); } + if (Array.isArray(fieldStats)) { + combinedConfigs = combinedConfigs.map((c) => ({ + ...c, + loading: false, + stats: { ...c.stats, ...fieldStats.find((r) => r.fieldName === c.fieldName) }, + })); + } + return combinedConfigs; - }, [nonMetricConfigs, metricConfigs, visibleFieldTypes, visibleFieldNames]); + }, [ + nonMetricConfigs, + metricConfigs, + visibleFieldTypes, + visibleFieldNames, + strategyResponse.response.fieldStats, + ]); // Some actions open up fly-out or popup // This variable is used to keep track of them and clean up when unmounting diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts index c3870fae27f74..44455b9a59239 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts @@ -46,6 +46,10 @@ export function useFieldStatsSearchStrategy< services: { data }, } = useDataVisualizerKibana(); + useEffect(() => { + console.log('searchStrategyParams updated'); + }, [searchStrategyParams]); + const [rawResponse, setRawResponse] = useReducer( getReducer(), getInitialRawResponse() @@ -59,8 +63,6 @@ export function useFieldStatsSearchStrategy< const abortCtrl = useRef(new AbortController()); const searchSubscription$ = useRef(); - console.log('rawResponse', rawResponse); - const startFetch = useCallback(() => { searchSubscription$.current?.unsubscribe(); abortCtrl.current.abort(); @@ -79,11 +81,14 @@ export function useFieldStatsSearchStrategy< .search(request, { strategy: FIELD_STATS_SEARCH_STRATEGY, abortSignal: abortCtrl.current.signal, + sessionId: searchStrategyParams?.sessionId, }) .subscribe({ next: (response) => { // Setting results to latest even if the response is still partial + setRawResponse(response.rawResponse); + setFetchState({ isRunning: response.isRunning || false, ...(response.loaded ? { loaded: response.loaded } : {}), @@ -102,8 +107,6 @@ export function useFieldStatsSearchStrategy< error: response as unknown as Error, isRunning: false, }); - } else { - // If is partial response } }, error: (error: Error) => { diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts index 969a6d6c7e2d3..77abf145dbe58 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts @@ -6,20 +6,21 @@ */ import { ElasticsearchClient } from 'kibana/server'; -import uuid from 'uuid'; import { of } from 'rxjs'; +import { chunk } from 'lodash'; import { fieldStatsSearchServiceStateProvider } from './field_stats_state_provider'; -import { Field, StartDeps } from '../types'; +import { FieldStats, StartDeps } from '../types'; import { ISearchStrategy } from '../../../../../src/plugins/data/server'; import { FieldStatRawResponse, + FieldStatsCommonRequestParams, FieldStatsRequest, FieldStatsResponse, FieldStatsSearchStrategyParams, isFieldStatsSearchStrategyParams, } from '../../common/search_strategy/types'; -import { ENHANCED_ES_SEARCH_STRATEGY } from '../../../../../src/plugins/data/common'; import { getFieldStats } from './requests/get_field_stats'; +import { buildBaseFilterCriteria, getSafeAggregationName } from '../../common/utils/query_utils'; export interface RawResponseBase { ccsWarning: boolean; @@ -63,18 +64,14 @@ export type SearchServiceProvider< export const myEnhancedSearchStrategyProvider = ( data: StartDeps['data'] ): ISearchStrategy => { - // Get the default search strategy - const ese = data.search.getSearchStrategy(ENHANCED_ES_SEARCH_STRATEGY); const searchServiceMap = new Map>(); return { search: (request, options, deps) => { - if (request.params === undefined) { + if (options.sessionId === undefined || request.params === undefined) { throw new Error('Invalid request parameters.'); } - // return formatResponse(ese.search(preprocessRequest(request.params), options, deps)); - // The function to fetch the current state of the search service. // This will be either an existing service for a follow up fetch or a new one for new requests. let getSearchServiceState: GetSearchServiceState; @@ -82,22 +79,19 @@ export const myEnhancedSearchStrategyProvider = ( // If the request includes an ID, we require that the search service already exists // otherwise we throw an error. The client should never poll a service that's been cancelled or finished. // This also avoids instantiating search services when the service gets called with random IDs. - if (typeof request.id === 'string') { - const existingGetSearchServiceState = searchServiceMap.get(request.id); - - if (typeof existingGetSearchServiceState === 'undefined') { - throw new Error(`SearchService with ID '${request.id}' does not exist.`); - } + const existingGetSearchServiceState = searchServiceMap.get(options.sessionId); - getSearchServiceState = existingGetSearchServiceState; - } else { + if (typeof existingGetSearchServiceState === 'undefined') { getSearchServiceState = fieldStatsSearchServiceProvider( deps.esClient.asCurrentUser, request ); + } else { + getSearchServiceState = existingGetSearchServiceState; } + // Reuse the request's id or create a new one. - const id = request.id ?? uuid(); + const id = options.sessionId; const { error, meta, rawResponse } = getSearchServiceState(); @@ -120,52 +114,102 @@ export const myEnhancedSearchStrategyProvider = ( // return ese.search(request, options, deps); }, cancel: async (id, options, deps) => { - // call the cancel method of the async strategy you are using or implement your own cancellation function. - if (ese.cancel) { - await ese.cancel(id, options, deps); + const getSearchServiceState = searchServiceMap.get(id); + if (getSearchServiceState !== undefined) { + getSearchServiceState().cancel(); + searchServiceMap.delete(id); } }, }; }; +// @todo: remove +function timeout(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + export const fieldStatsSearchServiceProvider = ( esClient: ElasticsearchClient, request: FieldStatsRequest ) => { const state = fieldStatsSearchServiceStateProvider(); - async function fetchCorrelations() { + async function fetchFieldStats() { try { if (isFieldStatsSearchStrategyParams(request.params)) { - const params = request.params; - const fields = [...params.metricConfigs, ...params.nonMetricConfigs]; + const searchStrategyParams = request.params; + const fields = [ + ...searchStrategyParams.metricConfigs, + ...searchStrategyParams.nonMetricConfigs, + ]; + + const filterCriteria = buildBaseFilterCriteria( + searchStrategyParams.timeFieldName, + searchStrategyParams.earliest, + searchStrategyParams.latest, + searchStrategyParams.searchQuery + ); + + const params: FieldStatsCommonRequestParams = { + index: searchStrategyParams.index, + samplerShardSize: searchStrategyParams.samplerShardSize, + timeFieldName: searchStrategyParams.timeFieldName, + earliestMs: searchStrategyParams.earliest, + latestMs: searchStrategyParams.latest, + runtimeFieldMap: searchStrategyParams.runtimeFieldMap, + intervalMs: searchStrategyParams.intervalMs, + query: { + bool: { + filter: filterCriteria, + }, + }, + }; + const fieldToLoadCnt = fields.length; + const fieldsWithError: any[] = []; let fieldsLoadedCnt = 0; - for (let idx = 0; idx < fieldToLoadCnt; idx++) { - const field = { - fieldName: fields[idx].fieldName, - type: fields[idx].type, - cardinality: fields[idx].cardinality, - identifier: idx, - }; - - try { - const testMetricFieldResult = await getFieldStats(esClient, params, field, idx); - state.addFieldStats(testMetricFieldResult); - } catch (e) { - console.error(e); - // @todo: Log error + if (params !== undefined && fields.length > 0) { + const batches = chunk(fields, 5); + for (let i = 0; i < batches.length; i++) { + const batchedResults: FieldStats[] = []; + + try { + const results = await Promise.allSettled([ + ...batches[i].map((field, idx) => { + return getFieldStats(esClient, params, { + fieldName: field.fieldName, + type: field.type, + cardinality: field.cardinality, + safeFieldName: getSafeAggregationName(field.fieldName ?? '', idx), + }); + }), + timeout(2000), + ]); + + results.forEach((r) => { + if (r.status === 'fulfilled' && r.value !== undefined) { + batchedResults.push(r.value); + } else { + fieldsWithError.push(r); + } + }); + + state.addFieldsStats(batchedResults); + } catch (e) { + state.setError(e); + } + + fieldsLoadedCnt += batches[i].length; + state.setProgress({ + loadedFieldStats: fieldsLoadedCnt / fieldToLoadCnt, + }); } - - fieldsLoadedCnt += 1; - state.setProgress({ loadedFieldStats: fieldsLoadedCnt / fieldToLoadCnt }); } } } catch (e) { state.setError(e); } - if (state.getState().error !== undefined) { state.setCcsWarning(true); } @@ -178,17 +222,17 @@ export const fieldStatsSearchServiceProvider = ( state.setIsCancelled(true); } - fetchCorrelations(); + fetchFieldStats(); return () => { - const { ccsWarning, error, isRunning } = state.getState(); + const { ccsWarning, error, isRunning, fieldsStats } = state.getState(); return { cancel, error, meta: { - loaded: Math.round(state.getOverallProgress() * 100), - total: 100, + loaded: state.getOverallProgress(), + total: 1, isRunning, isPartial: isRunning, }, @@ -197,7 +241,7 @@ export const fieldStatsSearchServiceProvider = ( ccsWarning, log: [], took: 0, - fieldStats: state.getState().fieldsStats, + fieldStats: fieldsStats, }, }; }; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts index fff7190cc108b..417fe4122ce0c 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts @@ -5,15 +5,13 @@ * 2.0. */ -import type { AggregationsAggregate } from '@elastic/elasticsearch/api/types'; - export interface LatencyCorrelationSearchServiceProgress { started: number; loadedOverallStats: number; loadedFieldStats: number; } -type FieldStat = Record | undefined; +export type FieldStat = Record | undefined; export const fieldStatsSearchServiceStateProvider = () => { let ccsWarning = false; function setCcsWarning(d: boolean) { @@ -43,9 +41,13 @@ export const fieldStatsSearchServiceStateProvider = () => { fieldsStats.push(d); } + function addFieldsStats(d: FieldStat[]) { + fieldsStats.push(...d); + } + let progress: LatencyCorrelationSearchServiceProgress = { started: Date.now(), - loadedOverallStats: 0, + loadedOverallStats: 1, loadedFieldStats: 0, }; function getOverallProgress() { @@ -78,6 +80,7 @@ export const fieldStatsSearchServiceStateProvider = () => { setIsCancelled, setIsRunning, setProgress, + addFieldsStats, addFieldStats, }; }; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_boolean_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_boolean_field_stats.ts index 619ef27419c56..c8a564c5428ea 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_boolean_field_stats.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_boolean_field_stats.ts @@ -13,7 +13,6 @@ import { Field, BooleanFieldStats, Aggs } from '../../types'; import { buildBaseFilterCriteria, buildSamplerAggregation, - getSafeAggregationName, getSamplerAggregationsResponsePath, } from '../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../common/utils/object_utils'; @@ -29,7 +28,7 @@ export const getBooleanFieldStatsRequest = ( const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); const aggs: Aggs = {}; - const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + const safeFieldName = field.safeFieldName; aggs[`${safeFieldName}_value_count`] = { filter: { exists: { field: field.fieldName } }, }; @@ -68,7 +67,7 @@ export const fetchBooleanFieldStats = async ( const aggregations = body.aggregations; const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + const safeFieldName = field.safeFieldName; const stats: BooleanFieldStats = { fieldName: field.fieldName, count: get(aggregations, [...aggsPath, `${safeFieldName}_value_count`, 'doc_count'], 0), diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_date_field_stats.ts index 9e61febe127cd..4033995660f25 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_date_field_stats.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_date_field_stats.ts @@ -13,7 +13,6 @@ import { Aggs, DateFieldStats, Field } from '../../types'; import { buildBaseFilterCriteria, buildSamplerAggregation, - getSafeAggregationName, getSamplerAggregationsResponsePath, } from '../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../common/utils/object_utils'; @@ -26,7 +25,7 @@ export const getDateFieldStatsRequest = (params: FieldStatsCommonRequestParams, const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); const aggs: Aggs = {}; - const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + const safeFieldName = field.safeFieldName; aggs[`${safeFieldName}_field_stats`] = { filter: { exists: { field: field.fieldName } }, aggs: { @@ -64,7 +63,7 @@ export const fetchDateFieldStats = async ( const aggregations = body.aggregations; const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + const safeFieldName = field.safeFieldName; const docCount = get(aggregations, [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], 0); const fieldStatsResp = get( aggregations, diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts index 8bd3b9c80f88a..48cc37d5fe424 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts @@ -29,7 +29,7 @@ export const getFieldExamplesRequest = (params: FieldStatsCommonRequestParams, f }); const searchBody = { - fields: [field], + fields: [field.fieldName], _source: false, query: { bool: { @@ -55,7 +55,7 @@ export const fetchFieldExamples = async ( const { body } = await esClient.search(request); const stats = { - fieldName: field, + fieldName: field.fieldName, examples: [] as any[], }; // @ts-expect-error incorrect search response type @@ -63,7 +63,7 @@ export const fetchFieldExamples = async ( const hits = body.hits.hits; for (let i = 0; i < hits.length; i++) { // Use lodash get() to support field names containing dots. - const doc: object[] | undefined = get(hits[i].fields, field); + const doc: object[] | undefined = get(hits[i].fields, field.fieldName); // the results from fields query is always an array if (Array.isArray(doc) && doc.length > 0) { const example = doc[0]; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts index 47c256c6e2d2e..24abc6e0aea30 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts @@ -11,7 +11,7 @@ import { FieldStatsCommonRequestParams, FieldStatsSearchStrategyParams, } from '../../../common/search_strategy/types'; -import { isValidField } from '../../types'; +import { FieldStats, isValidField } from '../../types'; import { fetchDocumentCountStats } from './get_document_stats'; import { fetchNumericFieldStats } from './get_numeric_field_stats'; import { fetchStringFieldStats } from './get_string_field_stats'; @@ -21,30 +21,21 @@ import { fetchFieldExamples } from './get_field_examples'; export const getFieldStats = async ( esClient: ElasticsearchClient, - searchStrategyParams: FieldStatsSearchStrategyParams, + params: FieldStatsCommonRequestParams, field: { fieldName?: string; type: string; cardinality: number; + safeFieldName: string; } -) => { - const params: FieldStatsCommonRequestParams = { - index: searchStrategyParams.index, - query: searchStrategyParams.searchQuery, - samplerShardSize: searchStrategyParams.samplerShardSize, - timeFieldName: searchStrategyParams.timeFieldName, - earliestMs: searchStrategyParams.earliest, - latestMs: searchStrategyParams.latest, - runtimeFieldMap: searchStrategyParams.runtimeFieldMap, - intervalMs: searchStrategyParams.intervalMs, - }; - +): Promise => { // An invalid field with undefined fieldName is used for a document count request. if (!isValidField(field)) { + // @todo // Will only ever be one document count card, // so no value in batching up the single request. if (field.type === JOB_FIELD_TYPES.NUMBER && params.intervalMs !== undefined) { - return fetchDocumentCountStats(esClient, params); + // return fetchDocumentCountStats(esClient, params); } } else { switch (field.type) { @@ -62,11 +53,13 @@ export const getFieldStats = async ( return fetchBooleanFieldStats(esClient, params, field); break; case JOB_FIELD_TYPES.TEXT: - default: - // Use an exists filter on the the field name to get - // examples of the field, so cannot batch up. return fetchFieldExamples(esClient, params, field); break; + // default: + // // Use an exists filter on the the field name to get + // // examples of the field, so cannot batch up. + // return fetchFieldExamples(esClient, params, field); + // break; } } }; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_numeric_field_stats.ts index d26f937b51c74..4ac0001e6d60a 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_numeric_field_stats.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_numeric_field_stats.ts @@ -12,7 +12,6 @@ import { Bucket, Field, NumericFieldStats } from '../../types'; import { buildBaseFilterCriteria, buildSamplerAggregation, - getSafeAggregationName, getSamplerAggregationsResponsePath, } from '../../../common/utils/query_utils'; import { @@ -45,8 +44,7 @@ export const getNumericFieldStatsRequest = ( ); const aggs: { [key: string]: any } = {}; - // @todo: check if field.cardinality is a proper replacement for index - const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + const safeFieldName = field.safeFieldName; aggs[`${safeFieldName}_field_stats`] = { filter: { exists: { field: field.fieldName } }, aggs: { @@ -116,7 +114,7 @@ export const fetchNumericFieldStats = async ( const aggregations = body.aggregations; const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + const safeFieldName = field.safeFieldName; const docCount = get(aggregations, [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], 0); const fieldStatsResp = get( aggregations, diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_string_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_string_field_stats.ts index 59613e821cf1b..66c6efcdb1864 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_string_field_stats.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_string_field_stats.ts @@ -13,7 +13,6 @@ import { Aggs, Bucket, Field, StringFieldStats } from '../../types'; import { buildBaseFilterCriteria, buildSamplerAggregation, - getSafeAggregationName, getSamplerAggregationsResponsePath, } from '../../../common/utils/query_utils'; import { @@ -31,7 +30,7 @@ export const getStringFieldStatsRequest = (params: FieldStatsCommonRequestParams const aggs: Aggs = {}; - const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + const safeFieldName = field.safeFieldName; const top = { terms: { field: field.fieldName, @@ -87,7 +86,7 @@ export const fetchStringFieldStats = async ( const aggregations = body.aggregations; const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const safeFieldName = getSafeAggregationName(field.fieldName, field.identifier); + const safeFieldName = field.safeFieldName; const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { diff --git a/x-pack/plugins/data_visualizer/server/types/chart_data.ts b/x-pack/plugins/data_visualizer/server/types/chart_data.ts index f43114f68952c..182e4511d2353 100644 --- a/x-pack/plugins/data_visualizer/server/types/chart_data.ts +++ b/x-pack/plugins/data_visualizer/server/types/chart_data.ts @@ -21,7 +21,7 @@ export interface Field { fieldName: string; type: string; cardinality: number; - identifier: number; + safeFieldName: string; } export function isValidField(arg: unknown): arg is Field { @@ -173,3 +173,15 @@ export type BatchStats = | DateFieldStats | DocumentCountStats | FieldExamples; + +export type FieldStats = + | NumericFieldStats + | StringFieldStats + | BooleanFieldStats + | DateFieldStats + // | DocumentCountStats + | FieldExamples; + +export function isValidFieldStats(arg: unknown): arg is FieldStats { + return isPopulatedObject(arg, ['fieldName', 'type', 'count']); +} From 6e5095a27a0701a0a6a2a3ea380018e2282083d3 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 5 Oct 2021 09:26:29 -0500 Subject: [PATCH 125/188] Add error log --- .../hooks/use_data_visualizer_grid_data.ts | 11 +--------- .../hooks/use_search_strategy.ts | 20 ++++++++----------- .../field_stats_search_strategy.ts | 20 +++++++++++++------ .../field_stats_state_provider.ts | 12 ++++++----- .../requests/get_field_examples.ts | 2 +- .../requests/get_field_stats.ts | 6 +++++- 6 files changed, 36 insertions(+), 35 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index fc7d92255180b..5cadd443b8d1f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -9,7 +9,7 @@ import { Required } from 'utility-types'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { merge } from 'rxjs'; import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; -import { i18n } from '../../../../../../../../../../../../private/var/tmp/_bazel_quynhnguyen/bd5cc7ce3740c1abb2c63a2609d8bb9f/execroot/kibana/bazel-out/darwin-fastbuild/bin/packages/kbn-i18n'; +import { i18n } from '@kbn/i18n'; import { DataVisualizerIndexBasedAppState } from '../types/index_data_visualizer_state'; import { useDataVisualizerKibana } from '../../kibana_context'; import { getEsQueryFromSavedSearch } from '../utils/saved_search_utils'; @@ -214,15 +214,6 @@ export const useDataVisualizerGridData = ( }; }); - const getTimeBuckets = useCallback(() => { - return new TimeBuckets({ - [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), - [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), - dateFormat: uiSettings.get('dateFormat'), - 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), - }); - }, [uiSettings]); - const indexPatternFields: DataViewField[] = useMemo( () => currentIndexPattern.fields, [currentIndexPattern] diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts index 44455b9a59239..f03307d1f053c 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts @@ -19,11 +19,11 @@ import type { } from '../../../../common/search_strategy/types'; import { useDataVisualizerKibana } from '../../kibana_context'; -const getInitialRawResponse = (): TRawResponse => +const getInitialRawResponse = (): FieldStatRawResponse => ({ ccsWarning: false, took: 0, - } as TRawResponse); + } as FieldStatRawResponse); const getInitialProgress = (): FieldStatsSearchStrategyProgress => ({ isRunning: false, @@ -38,21 +38,16 @@ const getReducer = ...update, }); -export function useFieldStatsSearchStrategy< - TRawResponse extends FieldStatRawResponse, - TParams extends FieldStatsSearchStrategyParams ->(searchStrategyParams: TParams | undefined): FieldStatsSearchStrategyReturnBase { +export function useFieldStatsSearchStrategy( + searchStrategyParams: TParams | undefined +): FieldStatsSearchStrategyReturnBase { const { services: { data }, } = useDataVisualizerKibana(); - useEffect(() => { - console.log('searchStrategyParams updated'); - }, [searchStrategyParams]); - const [rawResponse, setRawResponse] = useReducer( - getReducer(), - getInitialRawResponse() + getReducer(), + getInitialRawResponse() ); const [fetchState, setFetchState] = useReducer( @@ -87,6 +82,7 @@ export function useFieldStatsSearchStrategy< next: (response) => { // Setting results to latest even if the response is still partial + console.log('response.rawResponse', response); setRawResponse(response.rawResponse); setFetchState({ diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts index 77abf145dbe58..9a2ae8126f4ac 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts @@ -170,7 +170,7 @@ export const fieldStatsSearchServiceProvider = ( let fieldsLoadedCnt = 0; if (params !== undefined && fields.length > 0) { - const batches = chunk(fields, 5); + const batches = chunk(fields, 10); for (let i = 0; i < batches.length; i++) { const batchedResults: FieldStats[] = []; @@ -184,14 +184,21 @@ export const fieldStatsSearchServiceProvider = ( safeFieldName: getSafeAggregationName(field.fieldName ?? '', idx), }); }), - timeout(2000), + // @todo: throttle + timeout(1000), ]); - results.forEach((r) => { + results.forEach((r, idx) => { if (r.status === 'fulfilled' && r.value !== undefined) { batchedResults.push(r.value); } else { - fieldsWithError.push(r); + if (r.status === 'rejected' && r.reason) { + const fieldWithError = batches[i][idx]; + const reason = r.reason.meta.body.error?.reason ?? r.reason; + state.addErrorMessage( + `Error fetching field stats for ${fieldWithError.type} field ${fieldWithError.fieldName} because '${reason}'` + ); + } } }); @@ -210,6 +217,7 @@ export const fieldStatsSearchServiceProvider = ( } catch (e) { state.setError(e); } + if (state.getState().error !== undefined) { state.setCcsWarning(true); } @@ -225,8 +233,7 @@ export const fieldStatsSearchServiceProvider = ( fetchFieldStats(); return () => { - const { ccsWarning, error, isRunning, fieldsStats } = state.getState(); - + const { ccsWarning, error, isRunning, fieldsStats, errorLog } = state.getState(); return { cancel, error, @@ -242,6 +249,7 @@ export const fieldStatsSearchServiceProvider = ( log: [], took: 0, fieldStats: fieldsStats, + errorLog, }, }; }; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts index 417fe4122ce0c..1816dee26e163 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts @@ -18,6 +18,11 @@ export const fieldStatsSearchServiceStateProvider = () => { ccsWarning = d; } + const errorLog: string[] = []; + function addErrorMessage(message: string) { + errorLog.push(message); + } + let error: Error; function setError(d: Error) { error = d; @@ -37,10 +42,6 @@ export const fieldStatsSearchServiceStateProvider = () => { } const fieldsStats: FieldStat[] = []; - function addFieldStats(d: FieldStat) { - fieldsStats.push(d); - } - function addFieldsStats(d: FieldStat[]) { fieldsStats.push(...d); } @@ -68,6 +69,7 @@ export const fieldStatsSearchServiceStateProvider = () => { isRunning, progress, fieldsStats, + errorLog, }; } @@ -81,7 +83,7 @@ export const fieldStatsSearchServiceStateProvider = () => { setIsRunning, setProgress, addFieldsStats, - addFieldStats, + addErrorMessage, }; }; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts index 48cc37d5fe424..2a6198078bff4 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts @@ -25,7 +25,7 @@ export const getFieldExamplesRequest = (params: FieldStatsCommonRequestParams, f // Use an exists filter to return examples of the field. filterCriteria.push({ - exists: { field }, + exists: { field: field.fieldName }, }); const searchBody = { diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts index 24abc6e0aea30..40e602336f256 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts @@ -43,7 +43,7 @@ export const getFieldStats = async ( return fetchNumericFieldStats(esClient, params, field); break; case JOB_FIELD_TYPES.KEYWORD: - case JOB_FIELD_TYPES.IP: + // case JOB_FIELD_TYPES.IP: return fetchStringFieldStats(esClient, params, field); break; case JOB_FIELD_TYPES.DATE: @@ -55,6 +55,10 @@ export const getFieldStats = async ( case JOB_FIELD_TYPES.TEXT: return fetchFieldExamples(esClient, params, field); break; + // @todo: fix field.fieldName & move to keyword + case JOB_FIELD_TYPES.IP: + return fetchStringFieldStats(esClient, params, field.fieldName); + break; // default: // // Use an exists filter on the the field name to get // // examples of the field, so cannot batch up. From b159e2956ba10ee2b32e48997d4c4c99fad5771f Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 6 Oct 2021 17:13:49 -0500 Subject: [PATCH 126/188] Migrate overall stats to data's search --- .../common/search_strategy/types.ts | 53 ++++- .../data_visualizer_stats_table.tsx | 1 + .../hooks/use_data_visualizer_grid_data.ts | 122 +++++----- ..._search_strategy.ts => use_field_stats.ts} | 28 ++- .../hooks/use_overall_stats.ts | 153 +++++++++++++ .../search_strategy/requests/overall_stats.ts | 211 ++++++++++++++++++ .../types/overall_stats.ts | 4 +- 7 files changed, 486 insertions(+), 86 deletions(-) rename x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/{use_search_strategy.ts => use_field_stats.ts} (86%) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts diff --git a/x-pack/plugins/data_visualizer/common/search_strategy/types.ts b/x-pack/plugins/data_visualizer/common/search_strategy/types.ts index c3e67ecc9213b..d07a0c0620770 100644 --- a/x-pack/plugins/data_visualizer/common/search_strategy/types.ts +++ b/x-pack/plugins/data_visualizer/common/search_strategy/types.ts @@ -28,6 +28,21 @@ export interface FieldStatsCommonRequestParams { query: estypes.QueryDslQueryContainer; } +export interface OverallStatsSearchStrategyParams { + sessionId?: string; + earliest?: number; + latest?: number; + aggInterval: TimeBucketsInterval; + intervalMs?: number; + searchQuery?: any; + samplerShardSize: number; + index: string; + timeFieldName?: string; + runtimeFieldMap: Record; + aggregatableFields: string[]; + nonAggregatableFields: string[]; +} + export interface FieldStatsSearchStrategyParams { sessionId?: string; earliest?: number; @@ -37,10 +52,10 @@ export interface FieldStatsSearchStrategyParams { searchQuery?: any; samplerShardSize: number; index: string; - metricConfigs: FieldRequestConfig[]; - nonMetricConfigs: FieldRequestConfig[]; timeFieldName?: string; runtimeFieldMap: Record; + metricConfigs: FieldRequestConfig[]; + nonMetricConfigs: FieldRequestConfig[]; } export function isFieldStatsSearchStrategyParams( @@ -73,3 +88,37 @@ export interface FieldStatsSearchStrategyProgress { } export type FieldStatsSearchStrategy = ISearchStrategy; + +export interface FieldData { + fieldName: string; + existsInDocs: boolean; + stats?: { + sampleCount?: number; + count?: number; + cardinality?: number; + }; +} + +export interface Field { + fieldName: string; + type: string; + cardinality: number; + safeFieldName: string; +} + +export interface Aggs { + [key: string]: any; +} + +export interface FieldAggCardinality { + field: string; + percent?: any; +} + +export interface ScriptAggCardinality { + script: any; +} + +export interface AggCardinality { + cardinality: FieldAggCardinality | ScriptAggCardinality; +} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 61de8ef363e72..b4608006dfd83 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -39,6 +39,7 @@ import { BooleanContentPreview } from './components/field_data_row'; import { calculateTableColumnsDimensions } from './utils'; import { DistinctValues } from './components/field_data_row/distinct_values'; import { FieldTypeIcon } from '../field_type_icon'; +import './_index.scss'; const FIELD_NAME = 'fieldName'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 5cadd443b8d1f..9b98308b70771 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -23,14 +23,20 @@ import { KBN_FIELD_TYPES, UI_SETTINGS, } from '../../../../../../../src/plugins/data/common'; -import { extractErrorProperties } from '../utils/error_utils'; import { FieldVisConfig } from '../../common/components/stats_table/types'; -import { FieldRequestConfig, JOB_FIELD_TYPES } from '../../../../common'; +import { + FieldRequestConfig, + JOB_FIELD_TYPES, + NON_AGGREGATABLE_FIELD_TYPES, + OMIT_FIELDS, +} from '../../../../common'; import { kbnTypeToJobType } from '../../common/util/field_types_utils'; import { getActions } from '../../common/components/field_data_row/action_menu'; import { DataVisualizerGridEmbeddableInput } from '../embeddables/grid_embeddable/grid_embeddable'; import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; -import { useFieldStatsSearchStrategy } from './use_search_strategy'; +import { useFieldStatsSearchStrategy } from './use_field_stats'; +import { useOverallStats } from './use_overall_stats'; +import { OverallStatsSearchStrategyParams } from '../../../../common/search_strategy/types'; const defaults = getDefaultPageState(); @@ -62,6 +68,10 @@ export const useDataVisualizerGridData = ( }), [input] ); + const dataLoader = useMemo( + () => new DataLoader(currentIndexPattern, toasts), + [currentIndexPattern, toasts] + ); /** Prepare required params to pass to search strategy **/ const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { @@ -118,8 +128,6 @@ export const useDataVisualizerGridData = ( autoRefreshSelector: true, }); - const [overallStats, setOverallStats] = useState(defaults.overallStats); - const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); @@ -129,7 +137,7 @@ export const useDataVisualizerGridData = ( const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); /** Search strategy **/ - const fieldStatsRequest = useMemo(() => { + const fieldStatsRequest: OverallStatsSearchStrategyParams = useMemo(() => { // Obtain the interval to use for date histogram aggregations // (such as the document count chart). Aim for 75 bars. const buckets = _timeBuckets; @@ -153,24 +161,18 @@ export const useDataVisualizerGridData = ( buckets.setBarTarget(BAR_TARGET); const aggInterval = buckets.getInterval(); - const existMetricFields: FieldRequestConfig[] = metricConfigs.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existNonMetricFields: FieldRequestConfig[] = nonMetricConfigs.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; + const aggregatableFields: string[] = []; + const nonAggregatableFields: string[] = []; + currentIndexPattern.fields.forEach((field) => { + const fieldName = field.displayName !== undefined ? field.displayName : field.name; + if (!OMIT_FIELDS.includes(fieldName)) { + if (field.aggregatable === true && !NON_AGGREGATABLE_FIELD_TYPES.has(field.type)) { + aggregatableFields.push(field.name); + } else { + nonAggregatableFields.push(field.name); + } } - return props; }); - return { earliest, latest, @@ -182,8 +184,9 @@ export const useDataVisualizerGridData = ( index: currentIndexPattern.title, timeFieldName: currentIndexPattern.timeFieldName, runtimeFieldMap: currentIndexPattern.getComputedFields().runtimeFields, - metricConfigs: existMetricFields, - nonMetricConfigs: existNonMetricFields, + aggregatableFields, + nonAggregatableFields, + lastRefresh, }; }, [ _timeBuckets, @@ -192,15 +195,31 @@ export const useDataVisualizerGridData = ( searchQuery, samplerShardSize, searchSessionId, - metricConfigs, - nonMetricConfigs, + lastRefresh, ]); - const strategyResponse = useFieldStatsSearchStrategy(fieldStatsRequest); - const dataLoader = useMemo( - () => new DataLoader(currentIndexPattern, toasts), - [currentIndexPattern, toasts] - ); + const configsWithoutStats = useMemo(() => { + const existMetricFields: FieldRequestConfig[] = metricConfigs.map((config) => { + const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; + if (config.stats !== undefined && config.stats.cardinality !== undefined) { + props.cardinality = config.stats.cardinality; + } + return props; + }); + + // Pass the field name, type and cardinality in the request. + // Top values will be obtained on a sample if cardinality > 100000. + const existNonMetricFields: FieldRequestConfig[] = nonMetricConfigs.map((config) => { + const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; + if (config.stats !== undefined && config.stats.cardinality !== undefined) { + props.cardinality = config.stats.cardinality; + } + return props; + }); + return { metricConfigs: existMetricFields, nonMetricConfigs: existNonMetricFields }; + }, [metricConfigs, nonMetricConfigs]); + const overallStats = useOverallStats(fieldStatsRequest); + const strategyResponse = useFieldStatsSearchStrategy(fieldStatsRequest, configsWithoutStats); useEffect(() => { const timeUpdateSubscription = merge( @@ -219,42 +238,6 @@ export const useDataVisualizerGridData = ( [currentIndexPattern] ); - async function loadOverallStats() { - const tf = timefilter as any; - let earliest; - let latest; - - const activeBounds = tf.getActiveBounds(); - - if (currentIndexPattern.timeFieldName !== undefined && activeBounds === undefined) { - return; - } - - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = activeBounds.min.valueOf(); - latest = activeBounds.max.valueOf(); - } - - try { - const allStats = await dataLoader.loadOverallData( - searchQuery, - samplerShardSize, - earliest, - latest - ); - // Because load overall stats perform queries in batches - // there could be multiple errors - if (Array.isArray(allStats.errors) && allStats.errors.length > 0) { - allStats.errors.forEach((err: any) => { - dataLoader.displayError(extractErrorProperties(err)); - }); - } - setOverallStats(allStats); - } catch (err) { - dataLoader.displayError(err.body ?? err); - } - } - const createMetricCards = useCallback(() => { const configs: FieldVisConfig[] = []; const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; @@ -423,11 +406,6 @@ export const useDataVisualizerGridData = ( showEmptyFields, ]); - useEffect(() => { - loadOverallStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchQuery, samplerShardSize, lastRefresh]); - useEffect(() => { createMetricCards(); createNonMetricCards(); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts similarity index 86% rename from x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts rename to x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index f03307d1f053c..1578ac4705497 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_search_strategy.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -8,16 +8,15 @@ import { useCallback, useEffect, useReducer, useRef } from 'react'; import { Subscription } from 'rxjs'; import { isCompleteResponse, isErrorResponse } from '../../../../../../../src/plugins/data/common'; -import { FIELD_STATS_SEARCH_STRATEGY } from '../../../../common/search_strategy/constants'; import type { FieldStatRawResponse, - FieldStatsRequest, - FieldStatsResponse, FieldStatsSearchStrategyParams, FieldStatsSearchStrategyProgress, FieldStatsSearchStrategyReturnBase, } from '../../../../common/search_strategy/types'; import { useDataVisualizerKibana } from '../../kibana_context'; +import { FieldRequestConfig } from '../../../../common'; +import { FIELD_STATS_SEARCH_STRATEGY } from '../../../../common/search_strategy/constants'; const getInitialRawResponse = (): FieldStatRawResponse => ({ @@ -37,9 +36,14 @@ const getReducer = ...prev, ...update, }); +interface FieldStatsParams { + metricConfigs: FieldRequestConfig[]; + nonMetricConfigs: FieldRequestConfig[]; +} export function useFieldStatsSearchStrategy( - searchStrategyParams: TParams | undefined + searchStrategyParams: TParams | undefined, + fieldStatsParams: FieldStatsParams | undefined ): FieldStatsSearchStrategyReturnBase { const { services: { data }, @@ -67,13 +71,19 @@ export function useFieldStatsSearchStrategy(request, { + .search(request, { strategy: FIELD_STATS_SEARCH_STRATEGY, abortSignal: abortCtrl.current.signal, sessionId: searchStrategyParams?.sessionId, @@ -81,8 +91,6 @@ export function useFieldStatsSearchStrategy { // Setting results to latest even if the response is still partial - - console.log('response.rawResponse', response); setRawResponse(response.rawResponse); setFetchState({ @@ -112,7 +120,7 @@ export function useFieldStatsSearchStrategy { searchSubscription$.current?.unsubscribe(); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts new file mode 100644 index 0000000000000..bf91780f42df3 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useEffect, useState, useRef, useMemo } from 'react'; +import { combineLatest, forkJoin, of, Subscription } from 'rxjs'; +import { mergeMap, switchMap } from 'rxjs/operators'; +import { + FieldStatsSearchStrategyParams, + OverallStatsSearchStrategyParams, +} from '../../../../common/search_strategy/types'; +import { useDataVisualizerKibana } from '../../kibana_context'; +import { + checkAggregatableFieldsExistRequest, + checkNonAggregatableFieldExistsRequest, + processAggregatableFieldsExistResponse, + processNonAggregatableFieldsExistResponse, +} from '../search_strategy/requests/overall_stats'; +import { + IKibanaSearchRequest, + IKibanaSearchResponse, +} from '../../../../../../../src/plugins/data/common'; +import { OverallStats } from '../types/overall_stats'; +import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; + +export function useOverallStats( + searchStrategyParams: TParams | undefined +): OverallStats { + const { + services: { data }, + } = useDataVisualizerKibana(); + + const [stats, setOverallStats] = useState(getDefaultPageState().overallStats); + + const abortCtrl = useRef(new AbortController()); + const searchSubscription$ = useRef(); + + const startFetch = useCallback(() => { + searchSubscription$.current?.unsubscribe(); + abortCtrl.current.abort(); + abortCtrl.current = new AbortController(); + if (!searchStrategyParams) return; + console.log('searchStrategyParams', searchStrategyParams); + + const { + aggregatableFields, + nonAggregatableFields, + index, + searchQuery, + timeFieldName, + earliest, + latest, + runtimeFieldMap, + samplerShardSize, + } = searchStrategyParams; + + const nonAggregatableOverallStats$ = combineLatest( + nonAggregatableFields.map((fieldName: string) => + data.search + .search( + { + params: checkNonAggregatableFieldExistsRequest( + index, + searchQuery, + fieldName, + timeFieldName, + earliest, + latest, + runtimeFieldMap + ), + }, + { + abortSignal: abortCtrl.current.signal, + sessionId: searchStrategyParams?.sessionId, + } + ) + .pipe( + switchMap((resp) => { + return of({ + ...resp, + rawResponse: { ...resp.rawResponse, fieldName }, + } as IKibanaSearchResponse); + }) + ) + ) + ); + const sub = forkJoin({ + nonAggregatableOverallStatsResp: nonAggregatableOverallStats$, + aggregatableOverallStatsResp: data.search.search( + { + params: checkAggregatableFieldsExistRequest( + index, + searchQuery, + aggregatableFields, + samplerShardSize, + timeFieldName, + earliest, + latest, + undefined, + runtimeFieldMap + ), + }, + { + abortSignal: abortCtrl.current.signal, + sessionId: searchStrategyParams?.sessionId, + } + ), + }).pipe( + mergeMap(({ nonAggregatableOverallStatsResp, aggregatableOverallStatsResp }) => { + const aggregatableOverallStats = processAggregatableFieldsExistResponse( + aggregatableOverallStatsResp.rawResponse, + aggregatableFields, + samplerShardSize + ); + const nonAggregatableOverallStats = processNonAggregatableFieldsExistResponse( + nonAggregatableOverallStatsResp, + nonAggregatableFields + ); + return of({ + ...nonAggregatableOverallStats, + ...aggregatableOverallStats, + }); + }) + ); + + searchSubscription$.current = sub.subscribe({ + next: (overallStats) => { + setOverallStats(overallStats); + }, + error: (error) => { + // @todo: handle error + // import { extractErrorProperties } from '../utils/error_utils'; + }, + }); + }, [data.search, searchStrategyParams]); + + const cancelFetch = useCallback(() => { + searchSubscription$.current?.unsubscribe(); + searchSubscription$.current = undefined; + abortCtrl.current.abort(); + }, []); + + // auto-update + useEffect(() => { + startFetch(); + return cancelFetch; + }, [startFetch, cancelFetch]); + + return useMemo(() => stats, [stats]); +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts new file mode 100644 index 0000000000000..22d1541f53b3b --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts @@ -0,0 +1,211 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { get } from 'lodash'; +import { + buildBaseFilterCriteria, + buildSamplerAggregation, + getSafeAggregationName, + getSamplerAggregationsResponsePath, +} from '../../../../../common/utils/query_utils'; +import { getDatafeedAggregations } from '../../../../../common/utils/datafeed_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import { IKibanaSearchResponse } from '../../../../../../../../src/plugins/data/common'; +import { AggregatableField, NonAggregatableField } from '../../types/overall_stats'; +import { AggCardinality, FieldData, Aggs } from '../../../../../common/search_strategy/types'; + +export const checkAggregatableFieldsExistRequest = ( + indexPatternTitle: string, + query: any, + aggregatableFields: string[], + samplerShardSize: number, + timeFieldName: string | undefined, + earliestMs?: number, + latestMs?: number, + datafeedConfig?: estypes.MlDatafeed, + runtimeMappings?: estypes.MappingRuntimeFields +): estypes.SearchRequest => { + const index = indexPatternTitle; + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + const datafeedAggregations = getDatafeedAggregations(datafeedConfig); + + // Value count aggregation faster way of checking if field exists than using + // filter aggregation with exists query. + const aggs: Aggs = datafeedAggregations !== undefined ? { ...datafeedAggregations } : {}; + + // Combine runtime fields from the index pattern as well as the datafeed + const combinedRuntimeMappings: estypes.MappingRuntimeFields = { + ...(isPopulatedObject(runtimeMappings) ? runtimeMappings : {}), + ...(isPopulatedObject(datafeedConfig) && isPopulatedObject(datafeedConfig.runtime_mappings) + ? datafeedConfig.runtime_mappings + : {}), + }; + + aggregatableFields.forEach((field, i) => { + const safeFieldName = getSafeAggregationName(field, i); + aggs[`${safeFieldName}_count`] = { + filter: { exists: { field } }, + }; + + let cardinalityField: AggCardinality; + if (datafeedConfig?.script_fields?.hasOwnProperty(field)) { + cardinalityField = aggs[`${safeFieldName}_cardinality`] = { + cardinality: { script: datafeedConfig?.script_fields[field].script }, + }; + } else { + cardinalityField = { + cardinality: { field }, + }; + } + aggs[`${safeFieldName}_cardinality`] = cardinalityField; + }); + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + ...(isPopulatedObject(aggs) ? { aggs: buildSamplerAggregation(aggs, samplerShardSize) } : {}), + ...(isPopulatedObject(combinedRuntimeMappings) + ? { runtime_mappings: combinedRuntimeMappings } + : {}), + }; + + return { + index, + track_total_hits: true, + size, + body: searchBody, + }; +}; + +export const processAggregatableFieldsExistResponse = ( + body: estypes.SearchResponse, + aggregatableFields: string[], + samplerShardSize: number, + datafeedConfig?: estypes.MlDatafeed +) => { + const aggregations = body.aggregations; + // @ts-expect-error incorrect search response type + const totalCount = body.hits.total; + const stats = { + totalCount, + aggregatableExistsFields: [] as FieldData[], + aggregatableNotExistsFields: [] as FieldData[], + }; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const sampleCount = + samplerShardSize > 0 ? get(aggregations, ['sample', 'doc_count'], 0) : totalCount; + aggregatableFields.forEach((field, i) => { + const safeFieldName = getSafeAggregationName(field, i); + const count = get(aggregations, [...aggsPath, `${safeFieldName}_count`, 'doc_count'], 0); + if (count > 0) { + const cardinality = get( + aggregations, + [...aggsPath, `${safeFieldName}_cardinality`, 'value'], + 0 + ); + stats.aggregatableExistsFields.push({ + fieldName: field, + existsInDocs: true, + stats: { + sampleCount, + count, + cardinality, + }, + }); + } else { + if ( + datafeedConfig?.script_fields?.hasOwnProperty(field) || + datafeedConfig?.runtime_mappings?.hasOwnProperty(field) + ) { + const cardinality = get( + aggregations, + [...aggsPath, `${safeFieldName}_cardinality`, 'value'], + 0 + ); + stats.aggregatableExistsFields.push({ + fieldName: field, + existsInDocs: true, + stats: { + sampleCount, + count, + cardinality, + }, + }); + } else { + stats.aggregatableNotExistsFields.push({ + fieldName: field, + existsInDocs: false, + }); + } + } + }); + + return stats as { + totalCount: number; + aggregatableExistsFields: AggregatableField[]; + aggregatableNotExistsFields: AggregatableField[]; + }; +}; + +export const checkNonAggregatableFieldExistsRequest = ( + indexPatternTitle: string, + query: any, + field: string, + timeFieldName: string | undefined, + earliestMs: number | undefined, + latestMs: number | undefined, + runtimeMappings?: estypes.MappingRuntimeFields +): estypes.SearchRequest => { + const index = indexPatternTitle; + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), + }; + filterCriteria.push({ exists: { field } }); + + return { + index, + size, + body: searchBody, + }; +}; + +export const processNonAggregatableFieldsExistResponse = ( + results: IKibanaSearchResponse[], + nonAggregatableFields: string[] +) => { + const stats = { + nonAggregatableExistsFields: [] as NonAggregatableField[], + nonAggregatableNotExistsFields: [] as NonAggregatableField[], + }; + + nonAggregatableFields.forEach((fieldName) => { + const existsInDocs = results.find((r) => r.rawResponse.fieldName === fieldName) !== undefined; + const fieldData: NonAggregatableField = { + fieldName, + existsInDocs, + }; + if (existsInDocs === true) { + stats.nonAggregatableExistsFields.push(fieldData); + } else { + stats.nonAggregatableNotExistsFields.push(fieldData); + } + }); + return stats; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts index 2672dc69ac29a..286703afec5f9 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts @@ -20,7 +20,7 @@ export type NonAggregatableField = Omit; export interface OverallStats { totalCount: number; aggregatableExistsFields: AggregatableField[]; - aggregatableNotExistsFields: NonAggregatableField[]; - nonAggregatableExistsFields: AggregatableField[]; + aggregatableNotExistsFields: AggregatableField[]; + nonAggregatableExistsFields: NonAggregatableField[]; nonAggregatableNotExistsFields: NonAggregatableField[]; } From 4a4df4cdb499c6c29ae512c7e058f11c51c07923 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 7 Oct 2021 09:47:31 -0500 Subject: [PATCH 127/188] Better handle errors --- .../hooks/use_overall_stats.ts | 130 +++++++++++------- .../search_strategy/requests/overall_stats.ts | 28 ++-- 2 files changed, 102 insertions(+), 56 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index bf91780f42df3..833d8e5683462 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -8,10 +8,9 @@ import { useCallback, useEffect, useState, useRef, useMemo } from 'react'; import { combineLatest, forkJoin, of, Subscription } from 'rxjs'; import { mergeMap, switchMap } from 'rxjs/operators'; -import { - FieldStatsSearchStrategyParams, - OverallStatsSearchStrategyParams, -} from '../../../../common/search_strategy/types'; +import { i18n } from '@kbn/i18n'; +import { ToastsStart } from 'kibana/public'; +import { OverallStatsSearchStrategyParams } from '../../../../common/search_strategy/types'; import { useDataVisualizerKibana } from '../../kibana_context'; import { checkAggregatableFieldsExistRequest, @@ -25,12 +24,42 @@ import { } from '../../../../../../../src/plugins/data/common'; import { OverallStats } from '../types/overall_stats'; import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; +import { extractErrorProperties } from '../utils/error_utils'; + +function displayError(toastNotifications: ToastsStart, indexPattern: string, err: any) { + if (err.statusCode === 500) { + toastNotifications.addError(err, { + title: i18n.translate('xpack.dataVisualizer.index.dataLoader.internalServerErrorMessage', { + defaultMessage: + 'Error loading data in index {index}. {message}. ' + + 'The request may have timed out. Try using a smaller sample size or narrowing the time range.', + values: { + index: indexPattern, + message: err.error ?? err.message, + }, + }), + }); + } else { + toastNotifications.addError(err, { + title: i18n.translate('xpack.dataVisualizer.index.errorLoadingDataMessage', { + defaultMessage: 'Error loading data in index {index}. {message}.', + values: { + index: indexPattern, + message: err.error ?? err.message, + }, + }), + }); + } +} export function useOverallStats( searchStrategyParams: TParams | undefined ): OverallStats { const { - services: { data }, + services: { + data, + notifications: { toasts }, + }, } = useDataVisualizerKibana(); const [stats, setOverallStats] = useState(getDefaultPageState().overallStats); @@ -43,7 +72,6 @@ export function useOverallStats - data.search - .search( + const nonAggregatableOverallStats$ = + nonAggregatableFields.length > 0 + ? combineLatest( + nonAggregatableFields.map((fieldName: string) => + data.search + .search( + { + params: checkNonAggregatableFieldExistsRequest( + index, + searchQuery, + fieldName, + timeFieldName, + earliest, + latest, + runtimeFieldMap + ), + }, + { + abortSignal: abortCtrl.current.signal, + sessionId: searchStrategyParams?.sessionId, + } + ) + .pipe( + switchMap((resp) => { + return of({ + ...resp, + rawResponse: { ...resp.rawResponse, fieldName }, + } as IKibanaSearchResponse); + }) + ) + ) + ) + : of(undefined); + + const aggregatableOverallStats$ = + aggregatableFields.length > 0 + ? data.search.search( { - params: checkNonAggregatableFieldExistsRequest( + params: checkAggregatableFieldsExistRequest( index, searchQuery, - fieldName, + aggregatableFields, + samplerShardSize, timeFieldName, earliest, latest, + undefined, runtimeFieldMap ), }, @@ -77,41 +140,15 @@ export function useOverallStats { - return of({ - ...resp, - rawResponse: { ...resp.rawResponse, fieldName }, - } as IKibanaSearchResponse); - }) - ) - ) - ); + : of(undefined); + const sub = forkJoin({ nonAggregatableOverallStatsResp: nonAggregatableOverallStats$, - aggregatableOverallStatsResp: data.search.search( - { - params: checkAggregatableFieldsExistRequest( - index, - searchQuery, - aggregatableFields, - samplerShardSize, - timeFieldName, - earliest, - latest, - undefined, - runtimeFieldMap - ), - }, - { - abortSignal: abortCtrl.current.signal, - sessionId: searchStrategyParams?.sessionId, - } - ), + aggregatableOverallStatsResp: aggregatableOverallStats$, }).pipe( mergeMap(({ nonAggregatableOverallStatsResp, aggregatableOverallStatsResp }) => { const aggregatableOverallStats = processAggregatableFieldsExistResponse( - aggregatableOverallStatsResp.rawResponse, + aggregatableOverallStatsResp?.rawResponse, aggregatableFields, samplerShardSize ); @@ -128,14 +165,15 @@ export function useOverallStats { - setOverallStats(overallStats); + if (overallStats) { + setOverallStats(overallStats); + } }, error: (error) => { - // @todo: handle error - // import { extractErrorProperties } from '../utils/error_utils'; + displayError(toasts, searchStrategyParams.index, extractErrorProperties(error)); }, }); - }, [data.search, searchStrategyParams]); + }, [data.search, searchStrategyParams, toasts]); const cancelFetch = useCallback(() => { searchSubscription$.current?.unsubscribe(); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts index 22d1541f53b3b..1296554d984e1 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts @@ -17,7 +17,7 @@ import { getDatafeedAggregations } from '../../../../../common/utils/datafeed_ut import { isPopulatedObject } from '../../../../../common/utils/object_utils'; import { IKibanaSearchResponse } from '../../../../../../../../src/plugins/data/common'; import { AggregatableField, NonAggregatableField } from '../../types/overall_stats'; -import { AggCardinality, FieldData, Aggs } from '../../../../../common/search_strategy/types'; +import { AggCardinality, Aggs } from '../../../../../common/search_strategy/types'; export const checkAggregatableFieldsExistRequest = ( indexPatternTitle: string, @@ -87,19 +87,23 @@ export const checkAggregatableFieldsExistRequest = ( }; export const processAggregatableFieldsExistResponse = ( - body: estypes.SearchResponse, + body: estypes.SearchResponse | undefined, aggregatableFields: string[], samplerShardSize: number, datafeedConfig?: estypes.MlDatafeed ) => { - const aggregations = body.aggregations; - // @ts-expect-error incorrect search response type - const totalCount = body.hits.total; const stats = { - totalCount, - aggregatableExistsFields: [] as FieldData[], - aggregatableNotExistsFields: [] as FieldData[], + totalCount: 0, + aggregatableExistsFields: [] as AggregatableField[], + aggregatableNotExistsFields: [] as AggregatableField[], }; + + if (!body || aggregatableFields.length === 0) return stats; + + const aggregations = body.aggregations; + const totalCount = (body.hits.total as estypes.SearchTotalHits).value ?? body.hits.total; + stats.totalCount = totalCount as number; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); const sampleCount = samplerShardSize > 0 ? get(aggregations, ['sample', 'doc_count'], 0) : totalCount; @@ -144,6 +148,7 @@ export const processAggregatableFieldsExistResponse = ( stats.aggregatableNotExistsFields.push({ fieldName: field, existsInDocs: false, + stats: {}, }); } } @@ -177,7 +182,9 @@ export const checkNonAggregatableFieldExistsRequest = ( }, ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), }; - filterCriteria.push({ exists: { field } }); + if (Array.isArray(filterCriteria)) { + filterCriteria.push({ exists: { field } }); + } return { index, @@ -187,7 +194,7 @@ export const checkNonAggregatableFieldExistsRequest = ( }; export const processNonAggregatableFieldsExistResponse = ( - results: IKibanaSearchResponse[], + results: IKibanaSearchResponse[] | undefined, nonAggregatableFields: string[] ) => { const stats = { @@ -195,6 +202,7 @@ export const processNonAggregatableFieldsExistResponse = ( nonAggregatableNotExistsFields: [] as NonAggregatableField[], }; + if (!results || nonAggregatableFields.length === 0) return stats; nonAggregatableFields.forEach((fieldName) => { const existsInDocs = results.find((r) => r.rawResponse.fieldName === fieldName) !== undefined; const fieldData: NonAggregatableField = { From 3d40532f5a17ec61d204df1688a221c7ce4434d8 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 7 Oct 2021 10:20:19 -0500 Subject: [PATCH 128/188] Fix url so restore session brings back correct view --- .../apps/main/services/discover_state.ts | 2 ++ src/plugins/discover/public/url_generator.ts | 9 ++++++ .../hooks/use_data_visualizer_grid_data.ts | 2 +- .../plugins/data_visualizer/public/plugin.ts | 29 ++++++++++++++++++- 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/services/discover_state.ts b/src/plugins/discover/public/application/apps/main/services/discover_state.ts index 0014765ad28c4..bc08032d98ecb 100644 --- a/src/plugins/discover/public/application/apps/main/services/discover_state.ts +++ b/src/plugins/discover/public/application/apps/main/services/discover_state.ts @@ -408,5 +408,7 @@ function createUrlGeneratorState({ } : undefined, useHash: false, + viewMode: appState.viewMode, + hideAggregatedPreview: appState.hideAggregatedPreview, }; } diff --git a/src/plugins/discover/public/url_generator.ts b/src/plugins/discover/public/url_generator.ts index 7cc729fd7f7e5..53f6064231ab3 100644 --- a/src/plugins/discover/public/url_generator.ts +++ b/src/plugins/discover/public/url_generator.ts @@ -10,6 +10,7 @@ import type { UrlGeneratorsDefinition } from '../../share/public'; import type { TimeRange, Filter, Query, QueryState, RefreshInterval } from '../../data/public'; import { esFilters } from '../../data/public'; import { setStateToKbnUrl } from '../../kibana_utils/public'; +import { VIEW_MODE } from './application/apps/main/components/view_mode_toggle'; export const DISCOVER_APP_URL_GENERATOR = 'DISCOVER_APP_URL_GENERATOR'; @@ -75,6 +76,8 @@ export interface DiscoverUrlGeneratorState { * id of the used saved query */ savedQuery?: string; + viewMode?: VIEW_MODE; + hideAggregatedPreview?: boolean; } interface Params { @@ -104,6 +107,8 @@ export class DiscoverUrlGenerator savedQuery, sort, interval, + viewMode, + hideAggregatedPreview, }: DiscoverUrlGeneratorState): Promise => { const savedSearchPath = savedSearchId ? `view/${encodeURIComponent(savedSearchId)}` : ''; const appState: { @@ -114,6 +119,8 @@ export class DiscoverUrlGenerator interval?: string; sort?: string[][]; savedQuery?: string; + viewMode?: VIEW_MODE; + hideAggregatedPreview?: boolean; } = {}; const queryState: QueryState = {}; @@ -130,6 +137,8 @@ export class DiscoverUrlGenerator if (filters && filters.length) queryState.filters = filters?.filter((f) => esFilters.isFilterPinned(f)); if (refreshInterval) queryState.refreshInterval = refreshInterval; + if (viewMode) appState.viewMode = viewMode; + if (hideAggregatedPreview) appState.hideAggregatedPreview = hideAggregatedPreview; let url = `${this.params.appBasePath}#/${savedSearchPath}`; url = setStateToKbnUrl('_g', queryState, { useHash }, url); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 9b98308b70771..108080b8bdeeb 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -137,7 +137,7 @@ export const useDataVisualizerGridData = ( const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); /** Search strategy **/ - const fieldStatsRequest: OverallStatsSearchStrategyParams = useMemo(() => { + const fieldStatsRequest: OverallStatsSearchStrategyParams | undefined = useMemo(() => { // Obtain the interval to use for date histogram aggregations // (such as the document count chart). Aim for 75 bars. const buckets = _timeBuckets; diff --git a/x-pack/plugins/data_visualizer/public/plugin.ts b/x-pack/plugins/data_visualizer/public/plugin.ts index df1a5ea406d76..8b2a878e6c1f7 100644 --- a/x-pack/plugins/data_visualizer/public/plugin.ts +++ b/x-pack/plugins/data_visualizer/public/plugin.ts @@ -11,7 +11,10 @@ import type { SharePluginStart } from '../../../../src/plugins/share/public'; import { Plugin } from '../../../../src/core/public'; import { setStartServices } from './kibana_services'; -import type { DataPublicPluginStart } from '../../../../src/plugins/data/public'; +import type { + DataPublicPluginStart, + SearchSessionInfoProvider, +} from '../../../../src/plugins/data/public'; import type { HomePublicPluginSetup } from '../../../../src/plugins/home/public'; import type { FileUploadPluginStart } from '../../file_upload/public'; import type { MapsStartApi } from '../../maps/public'; @@ -22,6 +25,7 @@ import { getFileDataVisualizerComponent, getIndexDataVisualizerComponent } from import { getMaxBytesFormatted } from './application/common/util/get_max_bytes'; import { registerHomeAddData, registerHomeFeatureCatalogue } from './register_home'; import { registerEmbeddables } from './application/index_data_visualizer/embeddables'; +import { DATA_VISUALIZER_APP_LOCATOR } from './application/index_data_visualizer/locator'; export interface DataVisualizerSetupDependencies { home?: HomePublicPluginSetup; @@ -65,6 +69,29 @@ export class DataVisualizerPlugin public start(core: CoreStart, plugins: DataVisualizerStartDependencies) { setStartServices(core, plugins); + if (plugins.data) { + // const sessionRestorationDataProvider: SearchSessionInfoProvider = { + // data: plugins.data, + // }; + + plugins.data.search.session.enableStorage({ + getName: async () => { + // return the name you want to give the saved Search Session + return `dataVisualizer_${Math.random()}`; + }, + getUrlGeneratorData: async () => { + return { + urlGeneratorId: DATA_VISUALIZER_APP_LOCATOR, + initialState: { shouldRestoreSearchSession: false }, + restoreState: { shouldRestoreSearchSession: true }, + + // initialState: getUrlGeneratorState({ ...deps, shouldRestoreSearchSession: false }), + // restoreState: getUrlGeneratorState({ ...deps, shouldRestoreSearchSession: true }), + }; + }, + }); + } + return { getFileDataVisualizerComponent, getIndexDataVisualizerComponent, From 50b864250d98aa5cbcef8203e52733fabfbbc661 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 7 Oct 2021 10:29:43 -0500 Subject: [PATCH 129/188] Add progress bar --- .../grid_embeddable/grid_embeddable.tsx | 29 ++++++++++--------- .../hooks/use_data_visualizer_grid_data.ts | 1 + 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 39eecb860bc10..b19b5dd391559 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -10,7 +10,7 @@ import { CoreStart } from 'kibana/public'; import ReactDOM from 'react-dom'; import React, { Suspense, useCallback, useState } from 'react'; import useObservable from 'react-use/lib/useObservable'; -import { EuiEmptyPrompt } from '@elastic/eui'; +import { EuiEmptyPrompt, EuiProgress } from '@elastic/eui'; import { Filter } from '@kbn/es-query'; import { Required } from 'utility-types'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -79,10 +79,8 @@ export const EmbeddableWrapper = ({ }, [dataVisualizerListState, onOutputChange] ); - const { configs, searchQueryLanguage, searchString, extendedColumns } = useDataVisualizerGridData( - input, - dataVisualizerListState - ); + const { configs, searchQueryLanguage, searchString, extendedColumns, progress } = + useDataVisualizerGridData(input, dataVisualizerListState); const getItemIdToExpandedRowMap = useCallback( function (itemIds: string[], items: FieldVisConfig[]): ItemIdToExpandedRowMap { return itemIds.reduce((m: ItemIdToExpandedRowMap, fieldName: string) => { @@ -104,15 +102,18 @@ export const EmbeddableWrapper = ({ ); return ( - - items={configs} - pageState={dataVisualizerListState} - updatePageState={onTableChange} - getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} - extendedColumns={extendedColumns} - showPreviewByDefault={input?.showPreviewByDefault} - onChange={onOutputChange} - /> +
+ + + items={configs} + pageState={dataVisualizerListState} + updatePageState={onTableChange} + getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} + extendedColumns={extendedColumns} + showPreviewByDefault={input?.showPreviewByDefault} + onChange={onOutputChange} + /> +
); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 108080b8bdeeb..6e849cfb86c1e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -482,6 +482,7 @@ export const useDataVisualizerGridData = ( }, [input.indexPattern, services, searchQueryLanguage, searchString]); return { + progress: strategyResponse.progress, configs, searchQueryLanguage, searchString, From 570972dea5299669eb41d6f20de08dc1cbb746a2 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 8 Oct 2021 15:04:28 -0500 Subject: [PATCH 130/188] [ML] Add tests for data viz in Discover --- .../view_mode_toggle/view_mode_toggle.tsx | 4 +- test/functional/page_objects/discover_page.ts | 7 + .../apps/ml/data_visualizer/index.ts | 1 + .../data_visualizer/index_data_visualizer.ts | 39 +- .../index_data_visualizer_grid_in_discover.ts | 651 ++++++++++++++++++ .../apps/ml/data_visualizer/types.ts | 47 ++ .../services/ml/data_visualizer_table.ts | 43 +- 7 files changed, 747 insertions(+), 45 deletions(-) create mode 100644 x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts create mode 100644 x-pack/test/functional/apps/ml/data_visualizer/types.ts diff --git a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx index 0003083ea650b..3aa24c05e98d4 100644 --- a/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx +++ b/src/plugins/discover/public/application/apps/main/components/view_mode_toggle/view_mode_toggle.tsx @@ -27,11 +27,12 @@ export const DocumentViewModeToggle = ({ label: i18n.translate('discover.viewModes.document.label', { defaultMessage: 'Documents', }), + 'data-test-subj': 'dscViewModeDocumentButton', }, { id: VIEW_MODE.AGGREGATED_LEVEL, label: ( -
+
setDiscoverViewMode(id as VIEW_MODE)} + data-test-subj={'dscViewModeToggle'} /> ); }; diff --git a/test/functional/page_objects/discover_page.ts b/test/functional/page_objects/discover_page.ts index 497c5c959ee0d..c9e2c54ddf7cb 100644 --- a/test/functional/page_objects/discover_page.ts +++ b/test/functional/page_objects/discover_page.ts @@ -296,6 +296,13 @@ export class DiscoverPageObject extends FtrService { return await this.testSubjects.click('collapseSideBarButton'); } + public async closeSidebar() { + await this.retry.tryForTime(2 * 1000, async () => { + await this.toggleSidebarCollapse(); + await this.testSubjects.missingOrFail('discover-sidebar'); + }); + } + public async getAllFieldNames() { const sidebar = await this.testSubjects.find('discover-sidebar'); const $ = await sidebar.parseDomContent(); diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index.ts b/x-pack/test/functional/apps/ml/data_visualizer/index.ts index 3e6b644a0b494..363bdb50e072b 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index.ts @@ -14,6 +14,7 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./index_data_visualizer')); loadTestFile(require.resolve('./index_data_visualizer_actions_panel')); loadTestFile(require.resolve('./index_data_visualizer_index_pattern_management')); + loadTestFile(require.resolve('./index_data_visualizer_grid_in_discover')); loadTestFile(require.resolve('./file_data_visualizer')); }); } diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts index 031074876f39c..da6bf3e28a9e4 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts @@ -7,44 +7,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; import { ML_JOB_FIELD_TYPES } from '../../../../../plugins/ml/common/constants/field_types'; -import { FieldVisConfig } from '../../../../../plugins/data_visualizer/public/application/common/components/stats_table/types'; - -interface MetricFieldVisConfig extends FieldVisConfig { - statsMaxDecimalPlaces: number; - docCountFormatted: string; - topValuesCount: number; - viewableInLens: boolean; -} - -interface NonMetricFieldVisConfig extends FieldVisConfig { - docCountFormatted: string; - exampleCount: number; - viewableInLens: boolean; -} - -interface TestData { - suiteTitle: string; - sourceIndexOrSavedSearch: string; - fieldNameFilters: string[]; - fieldTypeFilters: string[]; - rowsPerPage?: 10 | 25 | 50; - sampleSizeValidations: Array<{ - size: number; - expected: { field: string; docCountFormatted: string }; - }>; - expected: { - totalDocCountFormatted: string; - metricFields?: MetricFieldVisConfig[]; - nonMetricFields?: NonMetricFieldVisConfig[]; - emptyFields: string[]; - visibleMetricFieldsCount: number; - totalMetricFieldsCount: number; - populatedFieldsCount: number; - totalFieldsCount: number; - fieldNameFiltersResultCount: number; - fieldTypeFiltersResultCount: number; - }; -} +import { TestData, MetricFieldVisConfig } from './types'; export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts new file mode 100644 index 0000000000000..ddfcd67cd711e --- /dev/null +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts @@ -0,0 +1,651 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { ML_JOB_FIELD_TYPES } from '../../../../../plugins/ml/common/constants/field_types'; +import { TestData, MetricFieldVisConfig } from './types'; + +const farequoteIndexPatternTestData: TestData = { + suiteTitle: 'index pattern', + isSavedSearch: false, + sourceIndexOrSavedSearch: 'ft_farequote', + fieldNameFilters: ['airline', '@timestamp'], + fieldTypeFilters: [ML_JOB_FIELD_TYPES.KEYWORD], + sampleSizeValidations: [ + { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, + { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, + ], + expected: { + totalDocCountFormatted: '86,274', + metricFields: [ + { + fieldName: 'responsetime', + type: ML_JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + statsMaxDecimalPlaces: 3, + topValuesCount: 10, + viewableInLens: true, + }, + ], + nonMetricFields: [ + { + fieldName: '@timestamp', + type: ML_JOB_FIELD_TYPES.DATE, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + exampleCount: 2, + viewableInLens: true, + }, + { + fieldName: '@version', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: '@version.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'airline', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 10, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'type', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: 'type.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + ], + emptyFields: ['sourcetype'], + visibleMetricFieldsCount: 1, + totalMetricFieldsCount: 1, + populatedFieldsCount: 7, + totalFieldsCount: 8, + fieldNameFiltersResultCount: 2, + fieldTypeFiltersResultCount: 3, + }, +}; + +const farequoteKQLSearchTestData: TestData = { + suiteTitle: 'KQL saved search', + isSavedSearch: true, + sourceIndexOrSavedSearch: 'ft_farequote_kuery', + fieldNameFilters: ['@version'], + fieldTypeFilters: [ML_JOB_FIELD_TYPES.DATE, ML_JOB_FIELD_TYPES.TEXT], + sampleSizeValidations: [ + { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, + { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, + ], + expected: { + totalDocCountFormatted: '34,415', + metricFields: [ + { + fieldName: 'responsetime', + type: ML_JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + statsMaxDecimalPlaces: 3, + topValuesCount: 10, + viewableInLens: true, + }, + ], + nonMetricFields: [ + { + fieldName: '@timestamp', + type: ML_JOB_FIELD_TYPES.DATE, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + exampleCount: 2, + viewableInLens: true, + }, + { + fieldName: '@version', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: '@version.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'airline', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 5, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'type', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: 'type.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + ], + emptyFields: ['sourcetype'], + visibleMetricFieldsCount: 1, + totalMetricFieldsCount: 1, + populatedFieldsCount: 7, + totalFieldsCount: 8, + fieldNameFiltersResultCount: 1, + fieldTypeFiltersResultCount: 3, + }, +}; + +const farequoteKQLFiltersSearchTestData: TestData = { + suiteTitle: 'KQL saved search and filters', + isSavedSearch: true, + sourceIndexOrSavedSearch: 'ft_farequote_filter_and_kuery', + fieldNameFilters: ['@version'], + fieldTypeFilters: [ML_JOB_FIELD_TYPES.DATE, ML_JOB_FIELD_TYPES.TEXT], + sampleSizeValidations: [ + { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, + { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, + ], + expected: { + totalDocCountFormatted: '5,674', + metricFields: [ + { + fieldName: 'responsetime', + type: ML_JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + statsMaxDecimalPlaces: 3, + topValuesCount: 10, + viewableInLens: true, + }, + ], + nonMetricFields: [ + { + fieldName: '@timestamp', + type: ML_JOB_FIELD_TYPES.DATE, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + exampleCount: 2, + viewableInLens: true, + }, + { + fieldName: '@version', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: '@version.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'airline', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + exampleContent: ['ASA'], + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'type', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: 'type.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + ], + emptyFields: ['sourcetype'], + visibleMetricFieldsCount: 1, + totalMetricFieldsCount: 1, + populatedFieldsCount: 7, + totalFieldsCount: 8, + fieldNameFiltersResultCount: 1, + fieldTypeFiltersResultCount: 3, + }, +}; + +const farequoteLuceneSearchTestData: TestData = { + suiteTitle: 'lucene saved search', + isSavedSearch: true, + sourceIndexOrSavedSearch: 'ft_farequote_lucene', + fieldNameFilters: ['@version.keyword', 'type'], + fieldTypeFilters: [ML_JOB_FIELD_TYPES.NUMBER], + sampleSizeValidations: [ + { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, + { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, + ], + expected: { + totalDocCountFormatted: '34,416', + metricFields: [ + { + fieldName: 'responsetime', + type: ML_JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + statsMaxDecimalPlaces: 3, + topValuesCount: 10, + viewableInLens: true, + }, + ], + nonMetricFields: [ + { + fieldName: '@timestamp', + type: ML_JOB_FIELD_TYPES.DATE, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + exampleCount: 2, + viewableInLens: true, + }, + { + fieldName: '@version', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: '@version.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'airline', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 5, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'type', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: 'type.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + ], + emptyFields: ['sourcetype'], + visibleMetricFieldsCount: 1, + totalMetricFieldsCount: 1, + populatedFieldsCount: 7, + totalFieldsCount: 8, + fieldNameFiltersResultCount: 2, + fieldTypeFiltersResultCount: 1, + }, +}; + +const farequoteLuceneFiltersSearchTestData: TestData = { + suiteTitle: 'lucene saved search and filter', + isSavedSearch: true, + sourceIndexOrSavedSearch: 'ft_farequote_filter_and_lucene', + fieldNameFilters: ['@version.keyword', 'type'], + fieldTypeFilters: [ML_JOB_FIELD_TYPES.NUMBER], + sampleSizeValidations: [ + { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, + { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, + ], + expected: { + totalDocCountFormatted: '5,673', + metricFields: [ + { + fieldName: 'responsetime', + type: ML_JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + statsMaxDecimalPlaces: 3, + topValuesCount: 10, + viewableInLens: true, + }, + ], + nonMetricFields: [ + { + fieldName: '@timestamp', + type: ML_JOB_FIELD_TYPES.DATE, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + exampleCount: 2, + viewableInLens: true, + }, + { + fieldName: '@version', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: '@version.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'airline', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + exampleContent: ['ASA'], + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'type', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: 'type.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + ], + emptyFields: ['sourcetype'], + visibleMetricFieldsCount: 1, + totalMetricFieldsCount: 1, + populatedFieldsCount: 7, + totalFieldsCount: 8, + fieldNameFiltersResultCount: 2, + fieldTypeFiltersResultCount: 1, + }, +}; + +const sampleLogTestData: TestData = { + suiteTitle: 'geo point field', + isSavedSearch: false, + sourceIndexOrSavedSearch: 'ft_module_sample_logs', + fieldNameFilters: ['geo.coordinates'], + fieldTypeFilters: [ML_JOB_FIELD_TYPES.GEO_POINT], + rowsPerPage: 50, + expected: { + totalDocCountFormatted: '408', + metricFields: [], + // only testing the geo_point fields + nonMetricFields: [ + { + fieldName: 'geo.coordinates', + type: ML_JOB_FIELD_TYPES.GEO_POINT, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '408 (100%)', + exampleCount: 10, + viewableInLens: false, + }, + ], + emptyFields: [], + visibleMetricFieldsCount: 4, + totalMetricFieldsCount: 5, + populatedFieldsCount: 35, + totalFieldsCount: 36, + fieldNameFiltersResultCount: 1, + fieldTypeFiltersResultCount: 1, + }, + sampleSizeValidations: [ + { size: 1000, expected: { field: 'geo.coordinates', docCountFormatted: '408 (100%)' } }, + { size: 5000, expected: { field: '@timestamp', docCountFormatted: '408 (100%)' } }, + ], +}; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const PageObjects = getPageObjects(['common', 'discover', 'timePicker']); + const ml = getService('ml'); + const testSubjects = getService('testSubjects'); + const retry = getService('retry'); + + const assertHitCount = async (expectedHitCount: string) => { + await retry.tryForTime(2 * 1000, async () => { + // Close side bar to ensure Discover hit count shows + // edge case for when browser width is small + await PageObjects.discover.closeSidebar(); + const hitCount = await PageObjects.discover.getHitCount(); + expect(hitCount).to.eql( + expectedHitCount, + `Expected Discover hit count to be ${expectedHitCount} but got ${hitCount}.` + ); + }); + }; + + const assertViewModeToggleExists = async () => { + await retry.tryForTime(2 * 1000, async () => { + await testSubjects.existOrFail('dscViewModeToggle'); + }); + }; + + const clickViewModeFieldStatsButton = async () => { + await retry.tryForTime(2 * 1000, async () => { + await testSubjects.existOrFail('dscViewModeFieldStatsButton'); + await testSubjects.clickWhenNotDisabled('dscViewModeFieldStatsButton'); + await testSubjects.existOrFail('dscFieldStatsEmbeddedContent'); + }); + }; + + function runTests(testData: TestData) { + describe(`with ${testData.suiteTitle}`, function () { + it('displays the Field statistics table', async function () { + await PageObjects.common.navigateToApp('discover'); + if (testData.isSavedSearch) { + await retry.tryForTime(2 * 1000, async () => { + await PageObjects.discover.loadSavedSearch(testData.sourceIndexOrSavedSearch); + }); + } else { + await retry.tryForTime(2 * 1000, async () => { + await PageObjects.discover.selectIndexPattern(testData.sourceIndexOrSavedSearch); + const indexPatternTitle = await testSubjects.getVisibleText('indexPattern-switch-link'); + expect(indexPatternTitle).to.be(testData.sourceIndexOrSavedSearch); + }); + } + await PageObjects.timePicker.setAbsoluteRange( + 'Jan 1, 2016 @ 00:00:00.000', + 'Nov 1, 2020 @ 00:00:00.000' + ); + + await assertHitCount(testData.expected.totalDocCountFormatted); + await assertViewModeToggleExists(); + await clickViewModeFieldStatsButton(); + await ml.testExecution.logTestStep( + 'displays details for metric fields and non-metric fields correctly' + ); + for (const fieldRow of testData.expected.metricFields as Array< + Required + >) { + await ml.dataVisualizerTable.assertNumberFieldContents( + fieldRow.fieldName, + fieldRow.docCountFormatted, + fieldRow.topValuesCount, + fieldRow.viewableInLens + ); + } + + for (const fieldRow of testData.expected.nonMetricFields!) { + await ml.dataVisualizerTable.assertNonMetricFieldContents( + fieldRow.type, + fieldRow.fieldName!, + fieldRow.docCountFormatted, + fieldRow.exampleCount, + fieldRow.viewableInLens, + false, + fieldRow.exampleContent + ); + } + }); + }); + } + + describe('field statistics', function () { + before(async function () { + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/module_sample_logs'); + await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp'); + await ml.testResources.createIndexPatternIfNeeded('ft_module_sample_logs', '@timestamp'); + await ml.testResources.createSavedSearchFarequoteKueryIfNeeded(); + await ml.testResources.createSavedSearchFarequoteLuceneIfNeeded(); + await ml.testResources.createSavedSearchFarequoteFilterAndLuceneIfNeeded(); + await ml.testResources.createSavedSearchFarequoteFilterAndKueryIfNeeded(); + }); + + after(async function () { + await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote'); + await esArchiver.unload('x-pack/test/functional/es_archives/ml/module_sample_logs'); + + await ml.testResources.deleteIndexPatternByTitle('ft_farequote'); + await ml.testResources.deleteIndexPatternByTitle('ft_module_sample_logs'); + await ml.testResources.deleteSavedSearches(); + }); + + runTests(farequoteIndexPatternTestData); + runTests(farequoteKQLSearchTestData); + runTests(farequoteLuceneSearchTestData); + runTests(farequoteKQLFiltersSearchTestData); + runTests(farequoteLuceneFiltersSearchTestData); + runTests(sampleLogTestData); + }); +} diff --git a/x-pack/test/functional/apps/ml/data_visualizer/types.ts b/x-pack/test/functional/apps/ml/data_visualizer/types.ts new file mode 100644 index 0000000000000..5c3f890dba561 --- /dev/null +++ b/x-pack/test/functional/apps/ml/data_visualizer/types.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FieldVisConfig } from '../../../../../plugins/data_visualizer/public/application/common/components/stats_table/types'; + +export interface MetricFieldVisConfig extends FieldVisConfig { + statsMaxDecimalPlaces: number; + docCountFormatted: string; + topValuesCount: number; + viewableInLens: boolean; +} + +export interface NonMetricFieldVisConfig extends FieldVisConfig { + docCountFormatted: string; + exampleCount: number; + exampleContent?: string[]; + viewableInLens: boolean; +} + +export interface TestData { + suiteTitle: string; + isSavedSearch?: boolean; + sourceIndexOrSavedSearch: string; + fieldNameFilters: string[]; + fieldTypeFilters: string[]; + rowsPerPage?: 10 | 25 | 50; + sampleSizeValidations: Array<{ + size: number; + expected: { field: string; docCountFormatted: string }; + }>; + expected: { + totalDocCountFormatted: string; + metricFields?: MetricFieldVisConfig[]; + nonMetricFields?: NonMetricFieldVisConfig[]; + emptyFields: string[]; + visibleMetricFieldsCount: number; + totalMetricFieldsCount: number; + populatedFieldsCount: number; + totalFieldsCount: number; + fieldNameFiltersResultCount: number; + fieldTypeFiltersResultCount: number; + }; +} diff --git a/x-pack/test/functional/services/ml/data_visualizer_table.ts b/x-pack/test/functional/services/ml/data_visualizer_table.ts index 8094f0ad1f8d2..860f2bd86bec7 100644 --- a/x-pack/test/functional/services/ml/data_visualizer_table.ts +++ b/x-pack/test/functional/services/ml/data_visualizer_table.ts @@ -361,7 +361,27 @@ export function MachineLearningDataVisualizerTableProvider( }); } - public async assertTopValuesContents(fieldName: string, expectedTopValuesCount: number) { + public async assertTopValuesContent(fieldName: string, expectedTopValues: string[]) { + const selector = this.detailsSelector(fieldName, 'dataVisualizerFieldDataTopValuesContent'); + const topValuesElement = await testSubjects.find(selector); + const topValuesBars = await topValuesElement.findAllByTestSubject( + 'dataVisualizerFieldDataTopValueBar' + ); + + const topValuesBarsValues = await Promise.all( + topValuesBars.map(async (bar) => { + const visibleText = await bar.getVisibleText(); + return visibleText ? visibleText.split('\n')[0] : undefined; + }) + ); + + expect(topValuesBarsValues).to.eql( + expectedTopValues, + `Expected top values for field '${fieldName}' to equal '${expectedTopValues}' (got '${topValuesBarsValues}')` + ); + } + + public async assertTopValuesCount(fieldName: string, expectedTopValuesCount: number) { const selector = this.detailsSelector(fieldName, 'dataVisualizerFieldDataTopValuesContent'); const topValuesElement = await testSubjects.find(selector); const topValuesBars = await topValuesElement.findAllByTestSubject( @@ -401,7 +421,7 @@ export function MachineLearningDataVisualizerTableProvider( await testSubjects.existOrFail( this.detailsSelector(fieldName, 'dataVisualizerFieldDataTopValues') ); - await this.assertTopValuesContents(fieldName, topValuesCount); + await this.assertTopValuesCount(fieldName, topValuesCount); if (checkDistributionPreviewExist) { await this.assertDistributionPreviewExist(fieldName); @@ -433,7 +453,8 @@ export function MachineLearningDataVisualizerTableProvider( public async assertKeywordFieldContents( fieldName: string, docCountFormatted: string, - topValuesCount: number + topValuesCount: number, + exampleContent?: string[] ) { await this.assertRowExists(fieldName); await this.assertFieldDocCount(fieldName, docCountFormatted); @@ -442,7 +463,11 @@ export function MachineLearningDataVisualizerTableProvider( await testSubjects.existOrFail( this.detailsSelector(fieldName, 'dataVisualizerFieldDataTopValuesContent') ); - await this.assertTopValuesContents(fieldName, topValuesCount); + await this.assertTopValuesCount(fieldName, topValuesCount); + + if (exampleContent) { + await this.assertTopValuesContent(fieldName, exampleContent); + } await this.ensureDetailsClosed(fieldName); } @@ -508,13 +533,19 @@ export function MachineLearningDataVisualizerTableProvider( docCountFormatted: string, exampleCount: number, viewableInLens: boolean, - hasActionMenu?: boolean + hasActionMenu?: boolean, + exampleContent?: string[] ) { // Currently the data used in the data visualizer tests only contains these field types. if (fieldType === ML_JOB_FIELD_TYPES.DATE) { await this.assertDateFieldContents(fieldName, docCountFormatted); } else if (fieldType === ML_JOB_FIELD_TYPES.KEYWORD) { - await this.assertKeywordFieldContents(fieldName, docCountFormatted, exampleCount); + await this.assertKeywordFieldContents( + fieldName, + docCountFormatted, + exampleCount, + exampleContent + ); } else if (fieldType === ML_JOB_FIELD_TYPES.TEXT) { await this.assertTextFieldContents(fieldName, docCountFormatted, exampleCount); } else if (fieldType === ML_JOB_FIELD_TYPES.GEO_POINT) { From 6ae6ba6d5f78ad5daeae7323e7de14656a2713f0 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 11 Oct 2021 15:30:35 -0500 Subject: [PATCH 131/188] [ML] Change to combinelatest --- .../index_data_visualizer/hooks/use_overall_stats.ts | 7 ++----- .../server/search_strategy/field_stats_search_strategy.ts | 2 -- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 833d8e5683462..62471cc7eb07f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -142,11 +142,8 @@ export function useOverallStats { + const sub = combineLatest([nonAggregatableOverallStats$, aggregatableOverallStats$]).pipe( + mergeMap(([nonAggregatableOverallStatsResp, aggregatableOverallStatsResp]) => { const aggregatableOverallStats = processAggregatableFieldsExistResponse( aggregatableOverallStatsResp?.rawResponse, aggregatableFields, diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts index 9a2ae8126f4ac..bf39199c8b018 100644 --- a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts +++ b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts @@ -184,8 +184,6 @@ export const fieldStatsSearchServiceProvider = ( safeFieldName: getSafeAggregationName(field.fieldName ?? '', idx), }); }), - // @todo: throttle - timeout(1000), ]); results.forEach((r, idx) => { From fe9cd1660a6aa0d416750ba9572d75df2bb23dc1 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 11 Oct 2021 16:51:16 -0500 Subject: [PATCH 132/188] Update tests & dashboard behavior to reflect new advanced settings --- src/plugins/discover/common/index.ts | 1 + .../main/components/chart/discover_chart.tsx | 13 +- .../components/layout/discover_layout.tsx | 7 +- .../embeddable/saved_search_embeddable.tsx | 2 + src/plugins/discover/server/ui_settings.ts | 19 +++ .../server/collectors/management/schema.ts | 4 + .../server/collectors/management/types.ts | 1 + .../index_data_visualizer_grid_in_discover.ts | 111 +++++++++++++++--- .../apps/ml/data_visualizer/index.ts | 5 + 9 files changed, 143 insertions(+), 20 deletions(-) diff --git a/src/plugins/discover/common/index.ts b/src/plugins/discover/common/index.ts index b30fcf972eda5..32704d95423f7 100644 --- a/src/plugins/discover/common/index.ts +++ b/src/plugins/discover/common/index.ts @@ -19,5 +19,6 @@ export const DOC_TABLE_LEGACY = 'doc_table:legacy'; export const MODIFY_COLUMNS_ON_SWITCH = 'discover:modifyColumnsOnSwitch'; export const SEARCH_FIELDS_FROM_SOURCE = 'discover:searchFieldsFromSource'; export const MAX_DOC_FIELDS_DISPLAYED = 'discover:maxDocFieldsDisplayed'; +export const SHOW_FIELD_STATISTICS = 'discover:showFieldStatistics'; export const SHOW_MULTIFIELDS = 'discover:showMultiFields'; export const SEARCH_EMBEDDABLE_TYPE = 'search'; diff --git a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx index 7f330264c934b..85fbd4268b21e 100644 --- a/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx +++ b/src/plugins/discover/public/application/apps/main/components/chart/discover_chart.tsx @@ -24,6 +24,7 @@ import { DataCharts$, DataTotalHits$ } from '../../services/use_saved_search'; import { DiscoverServices } from '../../../../../build_services'; import { useChartPanels } from './use_chart_panels'; import { VIEW_MODE, DocumentViewModeToggle } from '../view_mode_toggle'; +import { SHOW_FIELD_STATISTICS } from '../../../../../../common'; const DiscoverHistogramMemoized = memo(DiscoverHistogram); @@ -51,6 +52,7 @@ export function DiscoverChart({ setDiscoverViewMode: (viewMode: VIEW_MODE) => void; }) { const [showChartOptionsPopover, setShowChartOptionsPopover] = useState(false); + const showViewModeToggle = services.uiSettings.get(SHOW_FIELD_STATISTICS) ?? false; const { data } = services; const chartRef = useRef<{ element: HTMLElement | null; moveFocus: boolean }>({ @@ -111,9 +113,14 @@ export function DiscoverChart({ /> - - - + {showViewModeToggle && ( + + + + )} {timefield && ( diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 06b007099591b..4e4ac7ceadaf6 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -26,7 +26,7 @@ import { LoadingSpinner } from '../loading_spinner/loading_spinner'; import { esFilters, IndexPatternField } from '../../../../../../../data/public'; import { DiscoverSidebarResponsive } from '../sidebar'; import { DiscoverLayoutProps } from './types'; -import { SEARCH_FIELDS_FROM_SOURCE } from '../../../../../../common'; +import { SEARCH_FIELDS_FROM_SOURCE, SHOW_FIELD_STATISTICS } from '../../../../../../common'; import { popularizeField } from '../../../../helpers/popularize_field'; import { DiscoverTopNav } from '../top_nav/discover_topnav'; import { DocViewFilterFn, ElasticSearchHit } from '../../../../doc_views/doc_views_types'; @@ -75,7 +75,10 @@ export function DiscoverLayout({ const [expandedDoc, setExpandedDoc] = useState(undefined); const [inspectorSession, setInspectorSession] = useState(undefined); - const viewMode = useMemo(() => state.viewMode ?? VIEW_MODE.DOCUMENT_LEVEL, [state.viewMode]); + const viewMode = useMemo(() => { + if (uiSettings.get(SHOW_FIELD_STATISTICS) !== true) return VIEW_MODE.DOCUMENT_LEVEL; + return state.viewMode ?? VIEW_MODE.DOCUMENT_LEVEL; + }, [uiSettings, state.viewMode]); const setDiscoverViewMode = useCallback( (mode: VIEW_MODE) => { diff --git a/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx b/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx index c7c50b8872718..46766c5e2b243 100644 --- a/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx +++ b/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx @@ -35,6 +35,7 @@ import { DOC_TABLE_LEGACY, SAMPLE_SIZE_SETTING, SEARCH_FIELDS_FROM_SOURCE, + SHOW_FIELD_STATISTICS, SORT_DEFAULT_ORDER_SETTING, } from '../../../common'; import * as columnActions from '../apps/main/components/doc_table/actions/columns'; @@ -383,6 +384,7 @@ export class SavedSearchEmbeddable } if ( + this.services.uiSettings.get(SHOW_FIELD_STATISTICS) === true && this.savedSearch.viewMode === VIEW_MODE.AGGREGATED_LEVEL && searchProps.services && searchProps.indexPattern && diff --git a/src/plugins/discover/server/ui_settings.ts b/src/plugins/discover/server/ui_settings.ts index 007b6b0cf52e7..ce11a37eac148 100644 --- a/src/plugins/discover/server/ui_settings.ts +++ b/src/plugins/discover/server/ui_settings.ts @@ -26,6 +26,7 @@ import { SEARCH_FIELDS_FROM_SOURCE, MAX_DOC_FIELDS_DISPLAYED, SHOW_MULTIFIELDS, + SHOW_FIELD_STATISTICS, } from '../common'; export const getUiSettings: () => Record = () => ({ @@ -201,6 +202,24 @@ export const getUiSettings: () => Record = () => ({ category: ['discover'], schema: schema.boolean(), }, + [SHOW_FIELD_STATISTICS]: { + name: i18n.translate('discover.advancedSettings.discover.showFieldStatistics', { + defaultMessage: 'Show field statistics', + }), + description: i18n.translate( + 'discover.advancedSettings.discover.showFieldStatisticsDescription', + { + defaultMessage: `Enable Field statistics table in Discover.`, + } + ), + value: false, + category: ['discover'], + schema: schema.boolean(), + metric: { + type: METRIC_TYPE.CLICK, + name: 'discover:showFieldStatistics', + }, + }, [SHOW_MULTIFIELDS]: { name: i18n.translate('discover.advancedSettings.discover.showMultifields', { defaultMessage: 'Show multi-fields', diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index a8a391995b005..bf936b2ae8dbe 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -448,6 +448,10 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, }, + 'discover:showFieldStatistics': { + type: 'boolean', + _meta: { description: 'Non-default value of setting.' }, + }, 'discover:showMultiFields': { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index 7ea80ffb77dda..7575fa5d2b3f3 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -31,6 +31,7 @@ export interface UsageStats { 'doc_table:legacy': boolean; 'discover:modifyColumnsOnSwitch': boolean; 'discover:searchFieldsFromSource': boolean; + 'discover:showFieldStatistics': boolean; 'discover:showMultiFields': boolean; 'discover:maxDocFieldsDisplayed': number; 'securitySolution:rulesTableRefresh': string; diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts index ddfcd67cd711e..0d50da07f91d0 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts @@ -10,8 +10,9 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; import { ML_JOB_FIELD_TYPES } from '../../../../../plugins/ml/common/constants/field_types'; import { TestData, MetricFieldVisConfig } from './types'; +const SHOW_FIELD_STATISTICS = 'discover:showFieldStatistics'; const farequoteIndexPatternTestData: TestData = { - suiteTitle: 'index pattern', + suiteTitle: 'farequote index pattern', isSavedSearch: false, sourceIndexOrSavedSearch: 'ft_farequote', fieldNameFilters: ['airline', '@timestamp'], @@ -536,11 +537,13 @@ const sampleLogTestData: TestData = { export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); - const PageObjects = getPageObjects(['common', 'discover', 'timePicker']); + const PageObjects = getPageObjects(['common', 'discover', 'timePicker', 'settings']); const ml = getService('ml'); const testSubjects = getService('testSubjects'); const retry = getService('retry'); + const toasts = getService('toasts'); + /** Discover page helpers **/ const assertHitCount = async (expectedHitCount: string) => { await retry.tryForTime(2 * 1000, async () => { // Close side bar to ensure Discover hit count shows @@ -554,12 +557,22 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }; + const assertViewModeToggleNotExists = async () => { + await retry.tryForTime(2 * 1000, async () => { + await testSubjects.missingOrFail('dscViewModeToggle'); + }); + }; + const assertViewModeToggleExists = async () => { await retry.tryForTime(2 * 1000, async () => { await testSubjects.existOrFail('dscViewModeToggle'); }); }; + const assertFieldStatsTableNotExists = async () => { + await testSubjects.missingOrFail('dscFieldStatsEmbeddedContent'); + }; + const clickViewModeFieldStatsButton = async () => { await retry.tryForTime(2 * 1000, async () => { await testSubjects.existOrFail('dscViewModeFieldStatsButton'); @@ -568,20 +581,67 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }; + const selectIndexPattern = async (indexPattern: string) => { + await retry.tryForTime(2 * 1000, async () => { + await PageObjects.discover.selectIndexPattern(indexPattern); + const indexPatternTitle = await testSubjects.getVisibleText('indexPattern-switch-link'); + expect(indexPatternTitle).to.be(indexPattern); + }); + }; + + const clearAdvancedSetting = async (propertyName: string) => { + await retry.tryForTime(2 * 1000, async () => { + await PageObjects.common.navigateToUrl('management', 'kibana/settings', { + shouldUseHashForSubUrl: false, + }); + if ((await PageObjects.settings.getAdvancedSettingCheckbox(propertyName)) === 'true') { + await PageObjects.settings.clearAdvancedSettings(propertyName); + } + }); + }; + + const setAdvancedSettingCheckbox = async (propertyName: string, checkedState: boolean) => { + await retry.tryForTime(2 * 1000, async () => { + await PageObjects.common.navigateToUrl('management', 'kibana/settings', { + shouldUseHashForSubUrl: false, + }); + await testSubjects.click('settings'); + await toasts.dismissAllToasts(); + await PageObjects.settings.toggleAdvancedSettingCheckbox(propertyName, checkedState); + }); + }; + + function runTestsWhenDisabled(testData: TestData) { + it('should not show view mode toggle or Field stats table', async function () { + await PageObjects.common.navigateToApp('discover'); + if (testData.isSavedSearch) { + await retry.tryForTime(2 * 1000, async () => { + await PageObjects.discover.loadSavedSearch(testData.sourceIndexOrSavedSearch); + }); + } else { + await selectIndexPattern(testData.sourceIndexOrSavedSearch); + } + + await PageObjects.timePicker.setAbsoluteRange( + 'Jan 1, 2016 @ 00:00:00.000', + 'Nov 1, 2020 @ 00:00:00.000' + ); + + await assertViewModeToggleNotExists(); + await assertFieldStatsTableNotExists(); + }); + } + function runTests(testData: TestData) { describe(`with ${testData.suiteTitle}`, function () { - it('displays the Field statistics table', async function () { + it(`displays the 'Field statistics' table content correctly`, async function () { await PageObjects.common.navigateToApp('discover'); if (testData.isSavedSearch) { await retry.tryForTime(2 * 1000, async () => { await PageObjects.discover.loadSavedSearch(testData.sourceIndexOrSavedSearch); }); } else { - await retry.tryForTime(2 * 1000, async () => { - await PageObjects.discover.selectIndexPattern(testData.sourceIndexOrSavedSearch); - const indexPatternTitle = await testSubjects.getVisibleText('indexPattern-switch-link'); - expect(indexPatternTitle).to.be(testData.sourceIndexOrSavedSearch); - }); + await selectIndexPattern(testData.sourceIndexOrSavedSearch); } await PageObjects.timePicker.setAbsoluteRange( 'Jan 1, 2016 @ 00:00:00.000', @@ -620,7 +680,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); } - describe('field statistics', function () { + describe('field statistics in Discover', function () { before(async function () { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/module_sample_logs'); @@ -639,13 +699,34 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await ml.testResources.deleteIndexPatternByTitle('ft_farequote'); await ml.testResources.deleteIndexPatternByTitle('ft_module_sample_logs'); await ml.testResources.deleteSavedSearches(); + await clearAdvancedSetting(SHOW_FIELD_STATISTICS); }); - runTests(farequoteIndexPatternTestData); - runTests(farequoteKQLSearchTestData); - runTests(farequoteLuceneSearchTestData); - runTests(farequoteKQLFiltersSearchTestData); - runTests(farequoteLuceneFiltersSearchTestData); - runTests(sampleLogTestData); + describe('when enabled', function () { + before(async function () { + await setAdvancedSettingCheckbox(SHOW_FIELD_STATISTICS, true); + }); + + after(async function () { + await clearAdvancedSetting(SHOW_FIELD_STATISTICS); + }); + + runTests(farequoteIndexPatternTestData); + runTests(farequoteKQLSearchTestData); + runTests(farequoteLuceneSearchTestData); + runTests(farequoteKQLFiltersSearchTestData); + runTests(farequoteLuceneFiltersSearchTestData); + runTests(sampleLogTestData); + }); + + describe('when disabled', function () { + before(async function () { + // Ensure that the setting is set to default state which is false + await setAdvancedSettingCheckbox(SHOW_FIELD_STATISTICS, false); + }); + + runTestsWhenDisabled(farequoteIndexPatternTestData); + runTestsWhenDisabled(farequoteKQLSearchTestData); + }); }); } diff --git a/x-pack/test/functional_basic/apps/ml/data_visualizer/index.ts b/x-pack/test/functional_basic/apps/ml/data_visualizer/index.ts index 57a44a0b7952d..4d38e6a144a78 100644 --- a/x-pack/test/functional_basic/apps/ml/data_visualizer/index.ts +++ b/x-pack/test/functional_basic/apps/ml/data_visualizer/index.ts @@ -19,6 +19,11 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile( require.resolve('../../../../functional/apps/ml/data_visualizer/index_data_visualizer') ); + loadTestFile( + require.resolve( + '../../../../functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover' + ) + ); loadTestFile(require.resolve('./index_data_visualizer_actions_panel')); }); } From cad14c22bdc10e671bef9f2fb7076cfeadc74c63 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 11 Oct 2021 17:24:48 -0500 Subject: [PATCH 133/188] Update telemetry --- src/plugins/telemetry/schema/oss_plugins.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index c6724056f77a5..f9ca99a26ec19 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -7689,6 +7689,12 @@ "description": "Non-default value of setting." } }, + "discover:showFieldStatistics": { + "type": "boolean", + "_meta": { + "description": "Non-default value of setting." + } + }, "discover:showMultiFields": { "type": "boolean", "_meta": { From 122c6ca5ff2ca76ff246efaf0a92e13108e647bf Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 11 Oct 2021 17:35:23 -0500 Subject: [PATCH 134/188] Remove workaround after eui bump fix --- .../apps/main/components/layout/discover_layout.scss | 4 ---- src/plugins/discover/public/plugin.tsx | 5 ----- 2 files changed, 9 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss index 37ff50e333124..743f0fa5ec9fd 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss @@ -4,10 +4,6 @@ discover-app { flex-grow: 1; } -.dscAppWrapper { - overflow-y: hidden; -} - .dscPage { @include euiBreakpoint('m', 'l', 'xl') { @include kibanaFullBodyHeight(); diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index 46d0d43dd5df8..e4d6d5f255b39 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -348,11 +348,6 @@ export class DiscoverPlugin const { renderApp } = await import('./application'); - // FIXME: Temporarily hide overflow-y in Discover app when Field Stats table is shown - // due to EUI bug https://github.com/elastic/eui/pull/5152 - // until EUI is bumped to 38.0.0 - params.element.classList.add('dscAppWrapper'); - const unmount = renderApp(params.element); return () => { unlistenParentHistory(); From b6d9df897e30527907dfa9a0eb51b2956de593a1 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 12 Oct 2021 11:19:32 -0500 Subject: [PATCH 135/188] Remove dataloader --- .../data_loader/data_loader.ts | 1 + .../hooks/use_data_visualizer_grid_data.ts | 35 ++++++------------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts index c4db51dcd81bc..816f058964de3 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts @@ -19,6 +19,7 @@ type SavedSearchQuery = Record | null | undefined; const MAX_EXAMPLES_DEFAULT: number = 10; +// @TODO: remove export class DataLoader { private _indexPattern: IndexPattern; private _runtimeMappings: estypes.MappingRuntimeFields; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 6e849cfb86c1e..95ca7b1b5b2e9 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -14,7 +14,6 @@ import { DataVisualizerIndexBasedAppState } from '../types/index_data_visualizer import { useDataVisualizerKibana } from '../../kibana_context'; import { getEsQueryFromSavedSearch } from '../utils/saved_search_utils'; import { MetricFieldsStats } from '../../common/components/stats_table/components/field_count_stats'; -import { DataLoader } from '../data_loader/data_loader'; import { useTimefilter } from './use_time_filter'; import { dataVisualizerRefresh$ } from '../services/timefilter_refresh_service'; import { TimeBuckets } from '../../../../common/services/time_buckets'; @@ -40,13 +39,16 @@ import { OverallStatsSearchStrategyParams } from '../../../../common/search_stra const defaults = getDefaultPageState(); +function isDisplayField(fieldName: string): boolean { + return !OMIT_FIELDS.includes(fieldName); +} + export const useDataVisualizerGridData = ( input: DataVisualizerGridEmbeddableInput, dataVisualizerListState: Required ) => { const { services } = useDataVisualizerKibana(); - const { notifications, uiSettings, data } = services; - const { toasts } = notifications; + const { uiSettings, data } = services; const { samplerShardSize, visibleFieldTypes, showEmptyFields } = dataVisualizerListState; const [lastRefresh, setLastRefresh] = useState(0); @@ -68,10 +70,6 @@ export const useDataVisualizerGridData = ( }), [input] ); - const dataLoader = useMemo( - () => new DataLoader(currentIndexPattern, toasts), - [currentIndexPattern, toasts] - ); /** Prepare required params to pass to search strategy **/ const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { @@ -218,6 +216,7 @@ export const useDataVisualizerGridData = ( }); return { metricConfigs: existMetricFields, nonMetricConfigs: existNonMetricFields }; }, [metricConfigs, nonMetricConfigs]); + const overallStats = useOverallStats(fieldStatsRequest); const strategyResponse = useFieldStatsSearchStrategy(fieldStatsRequest, configsWithoutStats); @@ -246,7 +245,7 @@ export const useDataVisualizerGridData = ( return ( f.type === KBN_FIELD_TYPES.NUMBER && f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true + isDisplayField(f.displayName) === true ); }); const metricExistsFields = allMetricFields.filter((f) => { @@ -304,21 +303,14 @@ export const useDataVisualizerGridData = ( visibleMetricsCount: metricFieldsToShow.length, }); setMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - metricsLoaded, - overallStats, - showEmptyFields, - ]); + }, [currentIndexPattern, indexPatternFields, metricsLoaded, overallStats, showEmptyFields]); const createNonMetricCards = useCallback(() => { const allNonMetricFields = indexPatternFields.filter((f) => { return ( f.type !== KBN_FIELD_TYPES.NUMBER && f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true + isDisplayField(f.displayName) === true ); }); // Obtain the list of all non-metric fields which appear in documents @@ -397,14 +389,7 @@ export const useDataVisualizerGridData = ( }); setNonMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - nonMetricsLoaded, - overallStats, - showEmptyFields, - ]); + }, [currentIndexPattern, indexPatternFields, nonMetricsLoaded, overallStats, showEmptyFields]); useEffect(() => { createMetricCards(); From d7b2de1f9e1c7f3fdaf91c25bf1c5f475d62f4dc Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 12 Oct 2021 11:52:45 -0500 Subject: [PATCH 136/188] Snapshot --- .../hooks/use_data_visualizer_grid_data.ts | 7 +- .../hooks/use_field_stats.ts | 27 ++- .../hooks/use_overall_stats.ts | 2 +- .../search_strategy/requests/constants.ts | 16 ++ .../requests/get_boolean_field_stats.ts | 86 ++++++++ .../requests/get_date_field_stats.ts | 79 ++++++++ .../requests/get_document_stats.ts | 77 ++++++++ .../requests/get_field_examples.ts | 81 ++++++++ .../requests/get_field_stats.ts | 67 +++++++ .../requests/get_numeric_field_stats.ts | 163 +++++++++++++++ .../requests/get_string_field_stats.ts | 110 +++++++++++ .../types/field_stats.ts | 187 ++++++++++++++++++ .../utils/process_distribution_data.ts | 108 ++++++++++ .../process_distribution_data.ts | 1 + .../server/types/chart_data.ts | 1 + 15 files changed, 1008 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/constants.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/field_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 95ca7b1b5b2e9..0819087715b15 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -50,6 +50,7 @@ export const useDataVisualizerGridData = ( const { services } = useDataVisualizerKibana(); const { uiSettings, data } = services; const { samplerShardSize, visibleFieldTypes, showEmptyFields } = dataVisualizerListState; + const dataVisualizerListStateRef = useRef(dataVisualizerListState); const [lastRefresh, setLastRefresh] = useState(0); const [searchSessionId, setSearchSessionId] = useState(); @@ -218,7 +219,11 @@ export const useDataVisualizerGridData = ( }, [metricConfigs, nonMetricConfigs]); const overallStats = useOverallStats(fieldStatsRequest); - const strategyResponse = useFieldStatsSearchStrategy(fieldStatsRequest, configsWithoutStats); + const strategyResponse = useFieldStatsSearchStrategy( + fieldStatsRequest, + configsWithoutStats, + dataVisualizerListStateRef.current + ); useEffect(() => { const timeUpdateSubscription = merge( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index 1578ac4705497..552a5d5150759 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -7,6 +7,7 @@ import { useCallback, useEffect, useReducer, useRef } from 'react'; import { Subscription } from 'rxjs'; +import { chunk } from 'lodash'; import { isCompleteResponse, isErrorResponse } from '../../../../../../../src/plugins/data/common'; import type { FieldStatRawResponse, @@ -17,6 +18,7 @@ import type { import { useDataVisualizerKibana } from '../../kibana_context'; import { FieldRequestConfig } from '../../../../common'; import { FIELD_STATS_SEARCH_STRATEGY } from '../../../../common/search_strategy/constants'; +import { DataVisualizerIndexBasedAppState } from '../types/index_data_visualizer_state'; const getInitialRawResponse = (): FieldStatRawResponse => ({ @@ -43,12 +45,18 @@ interface FieldStatsParams { export function useFieldStatsSearchStrategy( searchStrategyParams: TParams | undefined, - fieldStatsParams: FieldStatsParams | undefined + fieldStatsParams: FieldStatsParams | undefined, + initialDataVisualizerListState: DataVisualizerIndexBasedAppState ): FieldStatsSearchStrategyReturnBase { const { services: { data }, } = useDataVisualizerKibana(); + useEffect( + () => console.log('initial dataVisualizerListState', initialDataVisualizerListState), + [initialDataVisualizerListState] + ); + const [rawResponse, setRawResponse] = useReducer( getReducer(), getInitialRawResponse() @@ -78,6 +86,21 @@ export function useFieldStatsSearchStrategy a[sortField].localeCompare(b[sortField])); + if (sortDirection === 'desc') { + sortedCnfigs = sortedCnfigs.reverse(); + } + const chunks = chunk(sortedCnfigs, pageSize); + console.log('chunks', chunks); const request = { params: { ...searchStrategyParams, ...fieldStatsParams }, }; @@ -120,7 +143,7 @@ export function useFieldStatsSearchStrategy { searchSubscription$.current?.unsubscribe(); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 62471cc7eb07f..fe15a378bf4a1 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -6,7 +6,7 @@ */ import { useCallback, useEffect, useState, useRef, useMemo } from 'react'; -import { combineLatest, forkJoin, of, Subscription } from 'rxjs'; +import { combineLatest, of, Subscription } from 'rxjs'; import { mergeMap, switchMap } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; import { ToastsStart } from 'kibana/public'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/constants.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/constants.ts new file mode 100644 index 0000000000000..4dbe51e5c0838 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/constants.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const SAMPLER_TOP_TERMS_THRESHOLD = 100000; +export const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000; +export const AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE = 200; +export const FIELDS_REQUEST_BATCH_SIZE = 10; + +export const MAX_CHART_COLUMNS = 20; + +export const MAX_PERCENT = 100; +export const PERCENTILE_SPACING = 5; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts new file mode 100644 index 0000000000000..d620ec8e907e2 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { get } from 'lodash'; +import type { ElasticsearchClient } from 'kibana/server'; +import type { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { + buildBaseFilterCriteria, + buildSamplerAggregation, + getSamplerAggregationsResponsePath, +} from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; +import type { Field, BooleanFieldStats, Aggs } from '../../types/field_stats'; + +export const getBooleanFieldStatsRequest = ( + params: FieldStatsCommonRequestParams, + field: Field +) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = + params; + + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + const aggs: Aggs = {}; + + const safeFieldName = field.safeFieldName; + aggs[`${safeFieldName}_value_count`] = { + filter: { exists: { field: field.fieldName } }, + }; + aggs[`${safeFieldName}_values`] = { + terms: { + field: field.fieldName, + size: 2, + }, + }; + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchBooleanFieldStats = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams, + field: Field +): Promise => { + const { samplerShardSize } = params; + const request: SearchRequest = getBooleanFieldStatsRequest(params, field); + const { body } = await esClient.search(request); + const aggregations = body.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + + const safeFieldName = field.safeFieldName; + const stats: BooleanFieldStats = { + fieldName: field.fieldName, + count: get(aggregations, [...aggsPath, `${safeFieldName}_value_count`, 'doc_count'], 0), + trueCount: 0, + falseCount: 0, + }; + + const valueBuckets: Array<{ [key: string]: number }> = get( + aggregations, + [...aggsPath, `${safeFieldName}_values`, 'buckets'], + [] + ); + valueBuckets.forEach((bucket) => { + stats[`${bucket.key_as_string}Count`] = bucket.doc_count; + }); + return stats; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts new file mode 100644 index 0000000000000..dc648a62c5870 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { get } from 'lodash'; +import { + buildBaseFilterCriteria, + buildSamplerAggregation, + getSamplerAggregationsResponsePath, +} from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; +import type { Field, DateFieldStats, Aggs } from '../../types/field_stats'; + +export const getDateFieldStatsRequest = (params: FieldStatsCommonRequestParams, field: Field) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = + params; + + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + const aggs: Aggs = {}; + const safeFieldName = field.safeFieldName; + aggs[`${safeFieldName}_field_stats`] = { + filter: { exists: { field: field.fieldName } }, + aggs: { + actual_stats: { + stats: { field: field.fieldName }, + }, + }, + }; + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchDateFieldStats = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams, + field: Field +): Promise => { + const { samplerShardSize } = params; + + const request: SearchRequest = getDateFieldStatsRequest(params, field); + const { body } = await esClient.search(request); + + const aggregations = body.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const safeFieldName = field.safeFieldName; + const docCount = get(aggregations, [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], 0); + const fieldStatsResp = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], + {} + ); + return { + fieldName: field.fieldName, + count: docCount, + earliest: get(fieldStatsResp, 'min', 0), + latest: get(fieldStatsResp, 'max', 0), + }; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts new file mode 100644 index 0000000000000..91b08dfb829a4 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { each, get } from 'lodash'; +import { buildBaseFilterCriteria } from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; +import type { DocumentCountStats } from '../../types/field_stats'; + +export const getDocumentCountStatsRequest = (params: FieldStatsCommonRequestParams) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, intervalMs } = params; + + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + // Don't use the sampler aggregation as this can lead to some potentially + // confusing date histogram results depending on the date range of data amongst shards. + + const aggs = { + eventRate: { + date_histogram: { + field: timeFieldName, + fixed_interval: `${intervalMs}ms`, + min_doc_count: 1, + }, + }, + }; + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs, + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchDocumentCountStats = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams +): Promise => { + const { intervalMs } = params; + const request: SearchRequest = getDocumentCountStatsRequest(params); + + const { body } = await esClient.search(request); + + const buckets: { [key: string]: number } = {}; + const dataByTimeBucket: Array<{ key: string; doc_count: number }> = get( + body, + ['aggregations', 'eventRate', 'buckets'], + [] + ); + each(dataByTimeBucket, (dataForTime) => { + const time = dataForTime.key; + buckets[time] = dataForTime.doc_count; + }); + + return { + documentCounts: { + interval: intervalMs, + buckets, + }, + }; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts new file mode 100644 index 0000000000000..10176c1630c73 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { get } from 'lodash'; +import { buildBaseFilterCriteria } from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; +import type { Field, FieldExamples } from '../../types/field_stats'; + +// @todo +const maxExamples = 10; +export const getFieldExamplesRequest = (params: FieldStatsCommonRequestParams, field: Field) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap } = params; + + // Request at least 100 docs so that we have a chance of obtaining + // 'maxExamples' of the field. + const size = Math.max(100, maxExamples); + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + // Use an exists filter to return examples of the field. + filterCriteria.push({ + exists: { field: field.fieldName }, + }); + + const searchBody = { + fields: [field.fieldName], + _source: false, + query: { + bool: { + filter: filterCriteria, + }, + }, + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchFieldExamples = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams, + field: Field +): Promise => { + const request: SearchRequest = getFieldExamplesRequest(params, field); + const { body } = await esClient.search(request); + + const stats = { + fieldName: field.fieldName, + examples: [] as any[], + }; + // @ts-expect-error incorrect search response type + if (body.hits.total.value > 0) { + const hits = body.hits.hits; + for (let i = 0; i < hits.length; i++) { + // Use lodash get() to support field names containing dots. + const doc: object[] | undefined = get(hits[i].fields, field.fieldName); + // the results from fields query is always an array + if (Array.isArray(doc) && doc.length > 0) { + const example = doc[0]; + if (example !== undefined && stats.examples.indexOf(example) === -1) { + stats.examples.push(example); + if (stats.examples.length === maxExamples) { + break; + } + } + } + } + } + + return stats; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts new file mode 100644 index 0000000000000..3b171b58805e0 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from 'kibana/server'; +import { estypes } from '@elastic/elasticsearch'; +import { FieldStats, isValidField } from '../../types/field_stats'; +import { fetchDocumentCountStats } from './get_document_stats'; +import { getNumericFieldStatsRequest } from './get_numeric_field_stats'; +import { getStringFieldStatsRequest } from './get_string_field_stats'; +import { getDateFieldStatsRequest } from './get_date_field_stats'; +import { getBooleanFieldStatsRequest } from './get_boolean_field_stats'; +import { getFieldExamplesRequest } from './get_field_examples'; +import { JOB_FIELD_TYPES } from '../../../../../common'; +import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; + +export const getFieldStats = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams, + field: { + fieldName?: string; + type: string; + cardinality: number; + safeFieldName: string; + } +): Promise => { + // An invalid field with undefined fieldName is used for a document count request. + if (!isValidField(field)) { + // @todo + // Will only ever be one document count card, + // so no value in batching up the single request. + if (field.type === JOB_FIELD_TYPES.NUMBER && params.intervalMs !== undefined) { + // return fetchDocumentCountStats(esClient, params); + } + } else { + switch (field.type) { + case JOB_FIELD_TYPES.NUMBER: + return getNumericFieldStatsRequest(esClient, params, field); + break; + case JOB_FIELD_TYPES.KEYWORD: + // case JOB_FIELD_TYPES.IP: + return getStringFieldStatsRequest(esClient, params, field); + break; + case JOB_FIELD_TYPES.DATE: + return getDateFieldStatsRequest(esClient, params, field); + break; + case JOB_FIELD_TYPES.BOOLEAN: + return getBooleanFieldStatsRequest(esClient, params, field); + break; + case JOB_FIELD_TYPES.TEXT: + return getFieldExamplesRequest(esClient, params, field); + break; + // @todo: fix field.fieldName & move to keyword + case JOB_FIELD_TYPES.IP: + return getStringFieldStatsRequest(esClient, params, field.fieldName); + break; + // default: + // // Use an exists filter on the the field name to get + // // examples of the field, so cannot batch up. + // return getFieldExamplesRequest(esClient, params, field); + // break; + } + } +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts new file mode 100644 index 0000000000000..26565ba7c7730 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { find, get } from 'lodash'; +import { + MAX_PERCENT, + PERCENTILE_SPACING, + SAMPLER_TOP_TERMS_SHARD_SIZE, + SAMPLER_TOP_TERMS_THRESHOLD, +} from './constants'; +import { + buildBaseFilterCriteria, + buildSamplerAggregation, + getSamplerAggregationsResponsePath, +} from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; +import type { Field, NumericFieldStats, Bucket } from '../../types/field_stats'; +import { processDistributionData } from '../../utils/process_distribution_data'; + +export const getNumericFieldStatsRequest = ( + params: FieldStatsCommonRequestParams, + field: Field +) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = + params; + + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + // Build the percents parameter which defines the percentiles to query + // for the metric distribution data. + // Use a fixed percentile spacing of 5%. + let count = 0; + const percents = Array.from( + Array(MAX_PERCENT / PERCENTILE_SPACING), + () => (count += PERCENTILE_SPACING) + ); + + const aggs: { [key: string]: any } = {}; + const safeFieldName = field.safeFieldName; + aggs[`${safeFieldName}_field_stats`] = { + filter: { exists: { field: field.fieldName } }, + aggs: { + actual_stats: { + stats: { field: field.fieldName }, + }, + }, + }; + aggs[`${safeFieldName}_percentiles`] = { + percentiles: { + field: field.fieldName, + percents, + keyed: false, + }, + }; + + const top = { + terms: { + field: field.fieldName, + size: 10, + order: { + _count: 'desc', + }, + }, + }; + + // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation + // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + aggs[`${safeFieldName}_top`] = { + sampler: { + shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, + }, + aggs: { + top, + }, + }; + } else { + aggs[`${safeFieldName}_top`] = top; + } + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchNumericFieldStats = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams, + field: Field +): Promise => { + const { samplerShardSize } = params; + const request: SearchRequest = getNumericFieldStatsRequest(params, field); + const { body } = await esClient.search(request); + const aggregations = body.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + + const safeFieldName = field.safeFieldName; + const docCount = get(aggregations, [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], 0); + const fieldStatsResp = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], + {} + ); + + const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + topAggsPath.push('top'); + } + + const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); + + const stats: NumericFieldStats = { + fieldName: field.fieldName, + count: docCount, + min: get(fieldStatsResp, 'min', 0), + max: get(fieldStatsResp, 'max', 0), + avg: get(fieldStatsResp, 'avg', 0), + isTopValuesSampled: field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, + topValues, + topValuesSampleSize: topValues.reduce( + (acc, curr) => acc + curr.doc_count, + get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) + ), + topValuesSamplerShardSize: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? SAMPLER_TOP_TERMS_SHARD_SIZE + : samplerShardSize, + }; + + if (stats.count > 0) { + const percentiles = get( + aggregations, + [...aggsPath, `${safeFieldName}_percentiles`, 'values'], + [] + ); + const medianPercentile: { value: number; key: number } | undefined = find(percentiles, { + key: 50, + }); + stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; + stats.distribution = processDistributionData(percentiles, PERCENTILE_SPACING, stats.min); + } + return stats; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts new file mode 100644 index 0000000000000..557843affc74a --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { get } from 'lodash'; +import { SAMPLER_TOP_TERMS_SHARD_SIZE, SAMPLER_TOP_TERMS_THRESHOLD } from './constants'; +import { + buildBaseFilterCriteria, + buildSamplerAggregation, + getSamplerAggregationsResponsePath, +} from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; +import type { Aggs, Bucket, Field, StringFieldStats } from '../../types/field_stats'; + +export const getStringFieldStatsRequest = (params: FieldStatsCommonRequestParams, field: Field) => { + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = + params; + + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + + const aggs: Aggs = {}; + + const safeFieldName = field.safeFieldName; + const top = { + terms: { + field: field.fieldName, + size: 10, + order: { + _count: 'desc', + }, + }, + }; + + // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation + // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + aggs[`${safeFieldName}_top`] = { + sampler: { + shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, + }, + aggs: { + top, + }, + }; + } else { + aggs[`${safeFieldName}_top`] = top; + } + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchStringFieldStats = async ( + esClient: ElasticsearchClient, + params: FieldStatsCommonRequestParams, + field: Field +): Promise => { + const { samplerShardSize } = params; + const request: SearchRequest = getStringFieldStatsRequest(params, field); + + const { body } = await esClient.search(request); + + const aggregations = body.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + + const safeFieldName = field.safeFieldName; + + const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + topAggsPath.push('top'); + } + + const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); + + const stats = { + fieldName: field.fieldName, + isTopValuesSampled: field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, + topValues, + topValuesSampleSize: topValues.reduce( + (acc, curr) => acc + curr.doc_count, + get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) + ), + topValuesSamplerShardSize: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? SAMPLER_TOP_TERMS_SHARD_SIZE + : samplerShardSize, + }; + + return stats; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/field_stats.ts new file mode 100644 index 0000000000000..56ca8920f0efb --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/field_stats.ts @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isPopulatedObject } from '../../../../common/utils/object_utils'; + +export interface FieldData { + fieldName: string; + existsInDocs: boolean; + stats?: { + sampleCount?: number; + count?: number; + cardinality?: number; + }; +} + +export interface Field { + fieldName: string; + type: string; + cardinality: number; + safeFieldName: string; +} + +export function isValidField(arg: unknown): arg is Field { + return isPopulatedObject(arg, ['fieldName', 'type']) && typeof arg.fieldName === 'string'; +} + +export interface HistogramField { + fieldName: string; + type: string; +} + +export interface Distribution { + percentiles: any[]; + minPercentile: number; + maxPercentile: number; +} + +export interface Aggs { + [key: string]: any; +} + +export interface Bucket { + doc_count: number; +} + +export interface NumericFieldStats { + fieldName: string; + count: number; + min: number; + max: number; + avg: number; + isTopValuesSampled: boolean; + topValues: Bucket[]; + topValuesSampleSize: number; + topValuesSamplerShardSize: number; + median?: number; + distribution?: Distribution; +} + +export interface StringFieldStats { + fieldName: string; + isTopValuesSampled: boolean; + topValues: Bucket[]; + topValuesSampleSize: number; + topValuesSamplerShardSize: number; +} + +export interface DateFieldStats { + fieldName: string; + count: number; + earliest: number; + latest: number; +} + +export interface BooleanFieldStats { + fieldName: string; + count: number; + trueCount: number; + falseCount: number; + [key: string]: number | string; +} + +export interface DocumentCountStats { + documentCounts: { + interval: number; + buckets: { [key: string]: number }; + }; +} + +export interface FieldExamples { + fieldName: string; + examples: any[]; +} + +export interface NumericColumnStats { + interval: number; + min: number; + max: number; +} +export type NumericColumnStatsMap = Record; + +export interface AggHistogram { + histogram: { + field: string; + interval: number; + }; +} + +export interface AggTerms { + terms: { + field: string; + size: number; + }; +} + +export interface NumericDataItem { + key: number; + key_as_string?: string; + doc_count: number; +} + +export interface NumericChartData { + data: NumericDataItem[]; + id: string; + interval: number; + stats: [number, number]; + type: 'numeric'; +} + +export interface OrdinalDataItem { + key: string; + key_as_string?: string; + doc_count: number; +} + +export interface OrdinalChartData { + type: 'ordinal' | 'boolean'; + cardinality: number; + data: OrdinalDataItem[]; + id: string; +} + +export interface UnsupportedChartData { + id: string; + type: 'unsupported'; +} + +export interface FieldAggCardinality { + field: string; + percent?: any; +} + +export interface ScriptAggCardinality { + script: any; +} + +export interface AggCardinality { + cardinality: FieldAggCardinality | ScriptAggCardinality; +} + +export type ChartRequestAgg = AggHistogram | AggCardinality | AggTerms; + +export type ChartData = NumericChartData | OrdinalChartData | UnsupportedChartData; + +export type BatchStats = + | NumericFieldStats + | StringFieldStats + | BooleanFieldStats + | DateFieldStats + | DocumentCountStats + | FieldExamples; + +export type FieldStats = + | NumericFieldStats + | StringFieldStats + | BooleanFieldStats + | DateFieldStats + // | DocumentCountStats + | FieldExamples; + +export function isValidFieldStats(arg: unknown): arg is FieldStats { + return isPopulatedObject(arg, ['fieldName', 'type', 'count']); +} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts new file mode 100644 index 0000000000000..5fdbc44373e40 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { last } from 'lodash'; +import type { Distribution } from '../types/field_stats'; + +export const processDistributionData = ( + percentiles: Array<{ value: number }>, + percentileSpacing: number, + minValue: number +): Distribution => { + const distribution: Distribution = { percentiles: [], minPercentile: 0, maxPercentile: 100 }; + if (percentiles.length === 0) { + return distribution; + } + + let percentileBuckets: Array<{ value: number }> = []; + let lowerBound = minValue; + if (lowerBound >= 0) { + // By default return results for 0 - 90% percentiles. + distribution.minPercentile = 0; + distribution.maxPercentile = 90; + percentileBuckets = percentiles.slice(0, percentiles.length - 2); + + // Look ahead to the last percentiles and process these too if + // they don't add more than 50% to the value range. + const lastValue = (last(percentileBuckets) as any).value; + const upperBound = lowerBound + 1.5 * (lastValue - lowerBound); + const filteredLength = percentileBuckets.length; + for (let i = filteredLength; i < percentiles.length; i++) { + if (percentiles[i].value < upperBound) { + percentileBuckets.push(percentiles[i]); + distribution.maxPercentile += percentileSpacing; + } else { + break; + } + } + } else { + // By default return results for 5 - 95% percentiles. + const dataMin = lowerBound; + lowerBound = percentiles[0].value; + distribution.minPercentile = 5; + distribution.maxPercentile = 95; + percentileBuckets = percentiles.slice(1, percentiles.length - 1); + + // Add in 0-5 and 95-100% if they don't add more + // than 25% to the value range at either end. + const lastValue: number = (last(percentileBuckets) as any).value; + const maxDiff = 0.25 * (lastValue - lowerBound); + if (lowerBound - dataMin < maxDiff) { + percentileBuckets.splice(0, 0, percentiles[0]); + distribution.minPercentile = 0; + lowerBound = dataMin; + } + + if (percentiles[percentiles.length - 1].value - lastValue < maxDiff) { + percentileBuckets.push(percentiles[percentiles.length - 1]); + distribution.maxPercentile = 100; + } + } + + // Combine buckets with the same value. + const totalBuckets = percentileBuckets.length; + let lastBucketValue = lowerBound; + let numEqualValueBuckets = 0; + for (let i = 0; i < totalBuckets; i++) { + const bucket = percentileBuckets[i]; + + // Results from the percentiles aggregation can have precision rounding + // artifacts e.g returning 200 and 200.000000000123, so check for equality + // around double floating point precision i.e. 15 sig figs. + if (bucket.value.toPrecision(15) !== lastBucketValue.toPrecision(15)) { + // Create a bucket for any 'equal value' buckets which had a value <= last bucket + if (numEqualValueBuckets > 0) { + distribution.percentiles.push({ + percent: numEqualValueBuckets * percentileSpacing, + minValue: lastBucketValue, + maxValue: lastBucketValue, + }); + } + + distribution.percentiles.push({ + percent: percentileSpacing, + minValue: lastBucketValue, + maxValue: bucket.value, + }); + + lastBucketValue = bucket.value; + numEqualValueBuckets = 0; + } else { + numEqualValueBuckets++; + if (i === totalBuckets - 1) { + // If at the last bucket, create a final bucket for the equal value buckets. + distribution.percentiles.push({ + percent: numEqualValueBuckets * percentileSpacing, + minValue: lastBucketValue, + maxValue: lastBucketValue, + }); + } + } + } + + return distribution; +}; diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts index 4e40c2baaf701..432fe30e64cb3 100644 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts +++ b/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts @@ -8,6 +8,7 @@ import { last } from 'lodash'; import { Distribution } from '../../types'; +// @todo: REMOVE export const processDistributionData = ( percentiles: Array<{ value: number }>, percentileSpacing: number, diff --git a/x-pack/plugins/data_visualizer/server/types/chart_data.ts b/x-pack/plugins/data_visualizer/server/types/chart_data.ts index 182e4511d2353..4f1774e5eea34 100644 --- a/x-pack/plugins/data_visualizer/server/types/chart_data.ts +++ b/x-pack/plugins/data_visualizer/server/types/chart_data.ts @@ -7,6 +7,7 @@ import { isPopulatedObject } from '../../common/utils/object_utils'; +// @todo: Remove export interface FieldData { fieldName: string; existsInDocs: boolean; From cf65db4f080199701d3e6a6d6e8529968e164e5d Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 12 Oct 2021 14:27:27 -0500 Subject: [PATCH 137/188] Migrate search to client side --- .../common/search_strategy/constants.ts | 8 - .../common/search_strategy/types.ts | 45 +--- .../common/types/field_request_config.ts | 10 +- .../common/types/field_vis_config.ts | 2 +- .../expanded_row/index_based_expanded_row.tsx | 5 + .../field_data_expanded_row/error_message.tsx | 29 ++ .../field_data_expanded_row/ip_content.tsx | 14 +- .../components/top_values/top_values.tsx | 2 +- .../hooks/use_data_visualizer_grid_data.ts | 21 +- .../hooks/use_field_stats.ts | 171 ++++++------ .../requests/get_boolean_field_stats.ts | 82 +++--- .../requests/get_date_field_stats.ts | 80 +++--- .../requests/get_field_examples.ts | 86 +++--- .../requests/get_field_stats.ts | 84 +++--- .../requests/get_numeric_field_stats.ts | 142 +++++----- .../requests/get_string_field_stats.ts | 101 ++++--- .../types/field_stats.ts | 16 +- .../utils/error_utils.ts | 2 +- .../plugins/data_visualizer/server/plugin.ts | 6 - .../field_stats_search_strategy.ts | 254 ------------------ .../field_stats_state_provider.ts | 92 ------- .../requests/get_boolean_field_stats.ts | 87 ------ .../requests/get_date_field_stats.ts | 79 ------ .../requests/get_document_stats.ts | 77 ------ .../requests/get_field_examples.ts | 81 ------ .../requests/get_field_stats.ts | 69 ----- .../requests/get_numeric_field_stats.ts | 163 ----------- .../requests/get_string_field_stats.ts | 113 -------- .../server/types/chart_data.ts | 12 - 29 files changed, 505 insertions(+), 1428 deletions(-) delete mode 100644 x-pack/plugins/data_visualizer/common/search_strategy/constants.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/error_message.tsx delete mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts delete mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts delete mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_boolean_field_stats.ts delete mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_date_field_stats.ts delete mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_document_stats.ts delete mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts delete mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts delete mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_numeric_field_stats.ts delete mode 100644 x-pack/plugins/data_visualizer/server/search_strategy/requests/get_string_field_stats.ts diff --git a/x-pack/plugins/data_visualizer/common/search_strategy/constants.ts b/x-pack/plugins/data_visualizer/common/search_strategy/constants.ts deleted file mode 100644 index f77efc81f897c..0000000000000 --- a/x-pack/plugins/data_visualizer/common/search_strategy/constants.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const FIELD_STATS_SEARCH_STRATEGY = 'dataVisualizerFieldStatsSearchStrategy'; diff --git a/x-pack/plugins/data_visualizer/common/search_strategy/types.ts b/x-pack/plugins/data_visualizer/common/search_strategy/types.ts index d07a0c0620770..19cd94178dac3 100644 --- a/x-pack/plugins/data_visualizer/common/search_strategy/types.ts +++ b/x-pack/plugins/data_visualizer/common/search_strategy/types.ts @@ -6,16 +6,9 @@ */ import { estypes } from '@elastic/elasticsearch'; -import type { - IKibanaSearchRequest, - IKibanaSearchResponse, -} from '../../../../../src/plugins/data/common'; import type { TimeBucketsInterval } from '../services/time_buckets'; import type { RuntimeField } from '../../../../../src/plugins/data/common'; -import type { FieldRequestConfig } from '../types'; -import type { ISearchStrategy } from '../../../../../src/plugins/data/server'; -import { isPopulatedObject } from '../utils/object_utils'; -import { FieldStats } from '../../server/types'; +import { FieldStats } from '../../public/application/index_data_visualizer/types/field_stats'; export interface FieldStatsCommonRequestParams { index: string; @@ -43,39 +36,9 @@ export interface OverallStatsSearchStrategyParams { nonAggregatableFields: string[]; } -export interface FieldStatsSearchStrategyParams { - sessionId?: string; - earliest?: number; - latest?: number; - aggInterval: TimeBucketsInterval; - intervalMs?: number; - searchQuery?: any; - samplerShardSize: number; - index: string; - timeFieldName?: string; - runtimeFieldMap: Record; - metricConfigs: FieldRequestConfig[]; - nonMetricConfigs: FieldRequestConfig[]; -} - -export function isFieldStatsSearchStrategyParams( - arg: unknown -): arg is FieldStatsSearchStrategyParams { - return isPopulatedObject(arg, ['index', 'samplerShardSize', 'metricConfigs', 'nonMetricConfigs']); -} - -export interface FieldStatRawResponse { - loading?: boolean; - ccsWarning: boolean; - took: 0; - fieldStats: Record; -} -export type FieldStatsRequest = IKibanaSearchRequest; -export type FieldStatsResponse = IKibanaSearchResponse; - -export interface FieldStatsSearchStrategyReturnBase { +export interface FieldStatsSearchStrategyReturnBase { progress: FieldStatsSearchStrategyProgress; - response: TRawResponse; + fieldStats: Map | undefined; startFetch: () => void; cancelFetch: () => void; } @@ -87,8 +50,6 @@ export interface FieldStatsSearchStrategyProgress { total: number; } -export type FieldStatsSearchStrategy = ISearchStrategy; - export interface FieldData { fieldName: string; existsInDocs: boolean; diff --git a/x-pack/plugins/data_visualizer/common/types/field_request_config.ts b/x-pack/plugins/data_visualizer/common/types/field_request_config.ts index 36e8fe14b7002..f0ea7079bf750 100644 --- a/x-pack/plugins/data_visualizer/common/types/field_request_config.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_request_config.ts @@ -14,7 +14,7 @@ export interface Percentile { } export interface FieldRequestConfig { - fieldName?: string; + fieldName: string; type: JobFieldType; cardinality: number; } @@ -29,6 +29,7 @@ export interface DocumentCounts { } export interface FieldVisStats { + error?: Error; cardinality?: number; count?: number; sampleCount?: number; @@ -58,3 +59,10 @@ export interface FieldVisStats { timeRangeEarliest?: number; timeRangeLatest?: number; } + +export interface DVErrorObject { + causedBy?: string; + message: string; + statusCode?: number; + fullError?: Error; +} diff --git a/x-pack/plugins/data_visualizer/common/types/field_vis_config.ts b/x-pack/plugins/data_visualizer/common/types/field_vis_config.ts index b88ea8c8fbbc4..5340fe4f77d15 100644 --- a/x-pack/plugins/data_visualizer/common/types/field_vis_config.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_vis_config.ts @@ -23,7 +23,7 @@ export interface MetricFieldVisStats { // which display the field information. export interface FieldVisConfig { type: JobFieldType; - fieldName?: string; + fieldName: string; displayName?: string; existsInDocs: boolean; aggregatable: boolean; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx index 79af35f1c8005..b87da2b3da789 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx @@ -23,6 +23,7 @@ import { IndexPattern } from '../../../../../../../../src/plugins/data/common'; import { CombinedQuery } from '../../../index_data_visualizer/types/combined_query'; import { LoadingIndicator } from '../loading_indicator'; import { IndexPatternField } from '../../../../../../../../src/plugins/data/common'; +import { ErrorMessageContent } from '../stats_table/components/field_data_expanded_row/error_message'; export const IndexBasedDataVisualizerExpandedRow = ({ item, @@ -46,6 +47,10 @@ export const IndexBasedDataVisualizerExpandedRow = ({ return ; } + if (config.stats?.error) { + return ; + } + switch (type) { case JOB_FIELD_TYPES.NUMBER: return ; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/error_message.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/error_message.tsx new file mode 100644 index 0000000000000..1d4a685457e25 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/error_message.tsx @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiText } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import React from 'react'; +import { DVErrorObject } from '../../../../../index_data_visualizer/utils/error_utils'; + +export const ErrorMessageContent = ({ + fieldName, + error, +}: { + fieldName: string; + error: DVErrorObject; +}) => { + return ( + + + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx index a5db86e0c30a0..d32a8a6dfb907 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/ip_content.tsx @@ -21,12 +21,14 @@ export const IpContent: FC = ({ config, onAddFilter }) => { return ( - + {stats && ( + + )} ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index e2793512e23df..e128e1db24dd5 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -43,7 +43,7 @@ function getPercentLabel(docCount: number, topValuesSampleSize: number): string } export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed, onAddFilter }) => { - if (stats === undefined) return null; + if (stats === undefined || !stats.topValues) return null; const { topValues, topValuesSampleSize, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 0819087715b15..dd50b1b3263b9 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -403,7 +403,7 @@ export const useDataVisualizerGridData = ( }, [overallStats, showEmptyFields]); const configs = useMemo(() => { - const fieldStats = strategyResponse.response.fieldStats; + const fieldStats = strategyResponse.fieldStats; let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; if (visibleFieldTypes && visibleFieldTypes.length > 0) { combinedConfigs = combinedConfigs.filter( @@ -416,12 +416,17 @@ export const useDataVisualizerGridData = ( ); } - if (Array.isArray(fieldStats)) { - combinedConfigs = combinedConfigs.map((c) => ({ - ...c, - loading: false, - stats: { ...c.stats, ...fieldStats.find((r) => r.fieldName === c.fieldName) }, - })); + if (fieldStats) { + combinedConfigs = combinedConfigs.map((c) => { + const loadedFullStats = fieldStats.get(c.fieldName); + return loadedFullStats + ? { + ...c, + loading: false, + stats: { ...c.stats, ...loadedFullStats }, + } + : c; + }); } return combinedConfigs; @@ -430,7 +435,7 @@ export const useDataVisualizerGridData = ( metricConfigs, visibleFieldTypes, visibleFieldNames, - strategyResponse.response.fieldStats, + strategyResponse.fieldStats, ]); // Some actions open up fly-out or popup diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index 552a5d5150759..48c8ddb71508b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -5,26 +5,24 @@ * 2.0. */ -import { useCallback, useEffect, useReducer, useRef } from 'react'; -import { Subscription } from 'rxjs'; -import { chunk } from 'lodash'; -import { isCompleteResponse, isErrorResponse } from '../../../../../../../src/plugins/data/common'; +import { useCallback, useEffect, useReducer, useRef, useState } from 'react'; +import { combineLatest, Observable, Subscription } from 'rxjs'; +import { i18n } from '@kbn/i18n'; import type { - FieldStatRawResponse, - FieldStatsSearchStrategyParams, FieldStatsSearchStrategyProgress, FieldStatsSearchStrategyReturnBase, + OverallStatsSearchStrategyParams, } from '../../../../common/search_strategy/types'; import { useDataVisualizerKibana } from '../../kibana_context'; -import { FieldRequestConfig } from '../../../../common'; -import { FIELD_STATS_SEARCH_STRATEGY } from '../../../../common/search_strategy/constants'; -import { DataVisualizerIndexBasedAppState } from '../types/index_data_visualizer_state'; - -const getInitialRawResponse = (): FieldStatRawResponse => - ({ - ccsWarning: false, - took: 0, - } as FieldStatRawResponse); +import type { FieldRequestConfig } from '../../../../common'; +import type { DataVisualizerIndexBasedAppState } from '../types/index_data_visualizer_state'; +import type { FieldStatsCommonRequestParams } from '../../../../common/search_strategy/types'; +import { + buildBaseFilterCriteria, + getSafeAggregationName, +} from '../../../../common/utils/query_utils'; +import { getFieldStats } from '../search_strategy/requests/get_field_stats'; +import type { FieldStats, FieldStatsError } from '../types/field_stats'; const getInitialProgress = (): FieldStatsSearchStrategyProgress => ({ isRunning: false, @@ -43,24 +41,19 @@ interface FieldStatsParams { nonMetricConfigs: FieldRequestConfig[]; } -export function useFieldStatsSearchStrategy( - searchStrategyParams: TParams | undefined, +export function useFieldStatsSearchStrategy( + searchStrategyParams: OverallStatsSearchStrategyParams | undefined, fieldStatsParams: FieldStatsParams | undefined, initialDataVisualizerListState: DataVisualizerIndexBasedAppState -): FieldStatsSearchStrategyReturnBase { +): FieldStatsSearchStrategyReturnBase { const { - services: { data }, + services: { + data, + notifications: { toasts }, + }, } = useDataVisualizerKibana(); - useEffect( - () => console.log('initial dataVisualizerListState', initialDataVisualizerListState), - [initialDataVisualizerListState] - ); - - const [rawResponse, setRawResponse] = useReducer( - getReducer(), - getInitialRawResponse() - ); + const [fieldStats, setFieldStats] = useState>(); const [fetchState, setFetchState] = useReducer( getReducer(), @@ -86,64 +79,84 @@ export function useFieldStatsSearchStrategy a[sortField].localeCompare(b[sortField])); + let sortedConfigs = [...fieldStatsParams.metricConfigs, ...fieldStatsParams.nonMetricConfigs]; + + if (sortField === 'fieldName' || sortField === 'type') { + sortedConfigs = sortedConfigs.sort((a, b) => a[sortField].localeCompare(b[sortField])); + } if (sortDirection === 'desc') { - sortedCnfigs = sortedCnfigs.reverse(); + sortedConfigs = sortedConfigs.reverse(); } - const chunks = chunk(sortedCnfigs, pageSize); - console.log('chunks', chunks); - const request = { - params: { ...searchStrategyParams, ...fieldStatsParams }, - }; - // Submit the search request using the `data.search` service. - searchSubscription$.current = data.search - .search(request, { - strategy: FIELD_STATS_SEARCH_STRATEGY, - abortSignal: abortCtrl.current.signal, - sessionId: searchStrategyParams?.sessionId, - }) - .subscribe({ - next: (response) => { - // Setting results to latest even if the response is still partial - setRawResponse(response.rawResponse); - - setFetchState({ - isRunning: response.isRunning || false, - ...(response.loaded ? { loaded: response.loaded } : {}), - ...(response.total ? { total: response.total } : {}), - }); - - if (isCompleteResponse(response)) { - // If the whole request is completed - searchSubscription$.current?.unsubscribe(); - setFetchState({ - isRunning: false, - }); - } else if (isErrorResponse(response)) { - searchSubscription$.current?.unsubscribe(); - setFetchState({ - error: response as unknown as Error, - isRunning: false, - }); - } - }, - error: (error: Error) => { - setFetchState({ - error, - isRunning: false, - }); + + const filterCriteria = buildBaseFilterCriteria( + searchStrategyParams.timeFieldName, + searchStrategyParams.earliest, + searchStrategyParams.latest, + searchStrategyParams.searchQuery + ); + + const params: FieldStatsCommonRequestParams = { + index: searchStrategyParams.index, + samplerShardSize: searchStrategyParams.samplerShardSize, + timeFieldName: searchStrategyParams.timeFieldName, + earliestMs: searchStrategyParams.earliest, + latestMs: searchStrategyParams.latest, + runtimeFieldMap: searchStrategyParams.runtimeFieldMap, + intervalMs: searchStrategyParams.intervalMs, + query: { + bool: { + filter: filterCriteria, }, - }); - }, [data.search, searchStrategyParams, fieldStatsParams, initialDataVisualizerListState]); + }, + }; + const searchOptions = { + abortSignal: abortCtrl.current.signal, + sessionId: searchStrategyParams?.sessionId, + }; + const sub = combineLatest( + sortedConfigs + .map((config, idx) => + getFieldStats( + data, + params, + { + fieldName: config.fieldName, + type: config.type, + cardinality: config.cardinality, + safeFieldName: getSafeAggregationName(config.fieldName, idx), + }, + searchOptions + ) + ) + .filter((obs) => obs !== undefined) as Array> + ); + + searchSubscription$.current = sub.subscribe({ + next: (resp) => { + if (resp) { + const statsMap = resp.reduce((map, field) => { + map.set(field.fieldName, field); + return map; + }, new Map()); + + setFieldStats(statsMap); + } + }, + error: (error) => { + toasts.addError(error, { + title: i18n.translate('xpack.dataVisualizer.index.errorFetchingFieldStatisticsMessage', { + defaultMessage: 'Error fetching field statistics', + }), + }); + }, + }); + }, [data, toasts, searchStrategyParams, fieldStatsParams, initialDataVisualizerListState]); const cancelFetch = useCallback(() => { searchSubscription$.current?.unsubscribe(); @@ -162,7 +175,7 @@ export function useFieldStatsSearchStrategy { - const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = - params; + const { index, query, runtimeFieldMap, samplerShardSize } = params; const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); const aggs: Aggs = {}; const safeFieldName = field.safeFieldName; @@ -39,11 +45,7 @@ export const getBooleanFieldStatsRequest = ( }; const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, + query, aggs: buildSamplerAggregation(aggs, samplerShardSize), ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), }; @@ -55,32 +57,46 @@ export const getBooleanFieldStatsRequest = ( }; }; -export const fetchBooleanFieldStats = async ( - esClient: ElasticsearchClient, +export const fetchBooleanFieldStats = ( + data: DataPublicPluginStart, params: FieldStatsCommonRequestParams, - field: Field -): Promise => { + field: Field, + options: ISearchOptions +): Observable => { const { samplerShardSize } = params; const request: SearchRequest = getBooleanFieldStatsRequest(params, field); - const { body } = await esClient.search(request); - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + return data.search + .search({ params: request }, options) + .pipe( + catchError((e) => + of({ + fieldName: field.fieldName, + error: extractErrorProperties(e), + } as FieldStatsError) + ), + switchMap((resp) => { + if (!isIKibanaSearchResponse(resp)) return of(resp); - const safeFieldName = field.safeFieldName; - const stats: BooleanFieldStats = { - fieldName: field.fieldName, - count: get(aggregations, [...aggsPath, `${safeFieldName}_value_count`, 'doc_count'], 0), - trueCount: 0, - falseCount: 0, - }; + const aggregations = resp.rawResponse.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + + const safeFieldName = field.safeFieldName; + const stats: BooleanFieldStats = { + fieldName: field.fieldName, + count: get(aggregations, [...aggsPath, `${safeFieldName}_value_count`, 'doc_count'], 0), + trueCount: 0, + falseCount: 0, + }; - const valueBuckets: Array<{ [key: string]: number }> = get( - aggregations, - [...aggsPath, `${safeFieldName}_values`, 'buckets'], - [] - ); - valueBuckets.forEach((bucket) => { - stats[`${bucket.key_as_string}Count`] = bucket.doc_count; - }); - return stats; + const valueBuckets: Array<{ [key: string]: number }> = get( + aggregations, + [...aggsPath, `${safeFieldName}_values`, 'buckets'], + [] + ); + valueBuckets.forEach((bucket) => { + stats[`${bucket.key_as_string}Count`] = bucket.doc_count; + }); + return of(stats); + }) + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts index dc648a62c5870..9aa0f3011288f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts @@ -5,24 +5,30 @@ * 2.0. */ -import { ElasticsearchClient } from 'kibana/server'; import { SearchRequest } from '@elastic/elasticsearch/api/types'; import { get } from 'lodash'; +import { Observable, of } from 'rxjs'; +import { catchError, switchMap } from 'rxjs/operators'; import { - buildBaseFilterCriteria, buildSamplerAggregation, getSamplerAggregationsResponsePath, } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; import type { Field, DateFieldStats, Aggs } from '../../types/field_stats'; +import { + DataPublicPluginStart, + IKibanaSearchRequest, + IKibanaSearchResponse, + ISearchOptions, +} from '../../../../../../../../src/plugins/data/public'; +import { FieldStatsError, isIKibanaSearchResponse } from '../../types/field_stats'; +import { extractErrorProperties } from '../../utils/error_utils'; export const getDateFieldStatsRequest = (params: FieldStatsCommonRequestParams, field: Field) => { - const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = - params; + const { index, query, runtimeFieldMap, samplerShardSize } = params; const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); const aggs: Aggs = {}; const safeFieldName = field.safeFieldName; @@ -36,11 +42,7 @@ export const getDateFieldStatsRequest = (params: FieldStatsCommonRequestParams, }; const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, + query, aggs: buildSamplerAggregation(aggs, samplerShardSize), ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), }; @@ -51,29 +53,45 @@ export const getDateFieldStatsRequest = (params: FieldStatsCommonRequestParams, }; }; -export const fetchDateFieldStats = async ( - esClient: ElasticsearchClient, +export const fetchDateFieldStats = ( + data: DataPublicPluginStart, params: FieldStatsCommonRequestParams, - field: Field -): Promise => { + field: Field, + options: ISearchOptions +): Observable => { const { samplerShardSize } = params; const request: SearchRequest = getDateFieldStatsRequest(params, field); - const { body } = await esClient.search(request); - - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const safeFieldName = field.safeFieldName; - const docCount = get(aggregations, [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], 0); - const fieldStatsResp = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], - {} - ); - return { - fieldName: field.fieldName, - count: docCount, - earliest: get(fieldStatsResp, 'min', 0), - latest: get(fieldStatsResp, 'max', 0), - }; + return data.search + .search({ params: request }, options) + .pipe( + catchError((e) => + of({ + fieldName: field.fieldName, + error: extractErrorProperties(e), + } as FieldStatsError) + ), + switchMap((resp) => { + if (!isIKibanaSearchResponse(resp)) return of(resp); + const aggregations = resp.rawResponse.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const safeFieldName = field.safeFieldName; + const docCount = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], + 0 + ); + const fieldStatsResp = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], + {} + ); + return of({ + fieldName: field.fieldName, + count: docCount, + earliest: get(fieldStatsResp, 'min', 0), + latest: get(fieldStatsResp, 'max', 0), + }); + }) + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts index 10176c1630c73..fd47a6176a1a7 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts @@ -5,13 +5,22 @@ * 2.0. */ -import { ElasticsearchClient } from 'kibana/server'; import { SearchRequest } from '@elastic/elasticsearch/api/types'; import { get } from 'lodash'; +import { Observable, of } from 'rxjs'; +import { catchError, switchMap } from 'rxjs/operators'; import { buildBaseFilterCriteria } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; import type { Field, FieldExamples } from '../../types/field_stats'; +import { + DataPublicPluginStart, + IKibanaSearchRequest, + IKibanaSearchResponse, + ISearchOptions, +} from '../../../../../../../../src/plugins/data/public'; +import { FieldStatsError, isIKibanaSearchResponse } from '../../types/field_stats'; +import { extractErrorProperties } from '../../utils/error_utils'; // @todo const maxExamples = 10; @@ -24,9 +33,11 @@ export const getFieldExamplesRequest = (params: FieldStatsCommonRequestParams, f const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); // Use an exists filter to return examples of the field. - filterCriteria.push({ - exists: { field: field.fieldName }, - }); + if (Array.isArray(filterCriteria)) { + filterCriteria.push({ + exists: { field: field.fieldName }, + }); + } const searchBody = { fields: [field.fieldName], @@ -46,36 +57,49 @@ export const getFieldExamplesRequest = (params: FieldStatsCommonRequestParams, f }; }; -export const fetchFieldExamples = async ( - esClient: ElasticsearchClient, +export const fetchFieldExamples = ( + data: DataPublicPluginStart, params: FieldStatsCommonRequestParams, - field: Field -): Promise => { + field: Field, + options: ISearchOptions +): Observable => { const request: SearchRequest = getFieldExamplesRequest(params, field); - const { body } = await esClient.search(request); - - const stats = { - fieldName: field.fieldName, - examples: [] as any[], - }; - // @ts-expect-error incorrect search response type - if (body.hits.total.value > 0) { - const hits = body.hits.hits; - for (let i = 0; i < hits.length; i++) { - // Use lodash get() to support field names containing dots. - const doc: object[] | undefined = get(hits[i].fields, field.fieldName); - // the results from fields query is always an array - if (Array.isArray(doc) && doc.length > 0) { - const example = doc[0]; - if (example !== undefined && stats.examples.indexOf(example) === -1) { - stats.examples.push(example); - if (stats.examples.length === maxExamples) { - break; + return data.search + .search({ params: request }, options) + .pipe( + catchError((e) => + of({ + fieldName: field.fieldName, + error: extractErrorProperties(e), + } as FieldStatsError) + ), + switchMap((resp) => { + if (!isIKibanaSearchResponse(resp)) return of(resp); + const body = resp.rawResponse; + const stats = { + fieldName: field.fieldName, + examples: [] as any[], + } as FieldExamples; + // @ts-expect-error incorrect search response type + if (body.hits.total.value > 0) { + const hits = body.hits.hits; + for (let i = 0; i < hits.length; i++) { + // Use lodash get() to support field names containing dots. + const doc: object[] | undefined = get(hits[i].fields, field.fieldName); + // the results from fields query is always an array + if (Array.isArray(doc) && doc.length > 0) { + const example = doc[0]; + if (example !== undefined && stats.examples.indexOf(example) === -1) { + stats.examples.push(example); + if (stats.examples.length === maxExamples) { + break; + } + } + } } } - } - } - } - return stats; + return of(stats); + }) + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts index 3b171b58805e0..52f737ca1d531 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts @@ -5,63 +5,49 @@ * 2.0. */ -import type { ElasticsearchClient } from 'kibana/server'; -import { estypes } from '@elastic/elasticsearch'; -import { FieldStats, isValidField } from '../../types/field_stats'; -import { fetchDocumentCountStats } from './get_document_stats'; -import { getNumericFieldStatsRequest } from './get_numeric_field_stats'; -import { getStringFieldStatsRequest } from './get_string_field_stats'; -import { getDateFieldStatsRequest } from './get_date_field_stats'; -import { getBooleanFieldStatsRequest } from './get_boolean_field_stats'; -import { getFieldExamplesRequest } from './get_field_examples'; +import { Observable } from 'rxjs'; +import { FieldStats, FieldStatsError, isValidField } from '../../types/field_stats'; +import { fetchNumericFieldStats } from './get_numeric_field_stats'; +import { fetchStringFieldStats } from './get_string_field_stats'; +import { fetchDateFieldStats } from './get_date_field_stats'; +import { fetchBooleanFieldStats } from './get_boolean_field_stats'; +import { fetchFieldExamples } from './get_field_examples'; import { JOB_FIELD_TYPES } from '../../../../../common'; import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; +import { ISearchOptions } from '../../../../../../../../src/plugins/data/common'; +import { DataPublicPluginStart } from '../../../../../../../../src/plugins/data/public'; -export const getFieldStats = async ( - esClient: ElasticsearchClient, +export const getFieldStats = ( + dataPlugin: DataPublicPluginStart, params: FieldStatsCommonRequestParams, field: { - fieldName?: string; + fieldName: string; type: string; cardinality: number; safeFieldName: string; - } -): Promise => { + }, + options: ISearchOptions +): Observable | undefined => { // An invalid field with undefined fieldName is used for a document count request. - if (!isValidField(field)) { - // @todo - // Will only ever be one document count card, - // so no value in batching up the single request. - if (field.type === JOB_FIELD_TYPES.NUMBER && params.intervalMs !== undefined) { - // return fetchDocumentCountStats(esClient, params); - } - } else { - switch (field.type) { - case JOB_FIELD_TYPES.NUMBER: - return getNumericFieldStatsRequest(esClient, params, field); - break; - case JOB_FIELD_TYPES.KEYWORD: - // case JOB_FIELD_TYPES.IP: - return getStringFieldStatsRequest(esClient, params, field); - break; - case JOB_FIELD_TYPES.DATE: - return getDateFieldStatsRequest(esClient, params, field); - break; - case JOB_FIELD_TYPES.BOOLEAN: - return getBooleanFieldStatsRequest(esClient, params, field); - break; - case JOB_FIELD_TYPES.TEXT: - return getFieldExamplesRequest(esClient, params, field); - break; - // @todo: fix field.fieldName & move to keyword - case JOB_FIELD_TYPES.IP: - return getStringFieldStatsRequest(esClient, params, field.fieldName); - break; - // default: - // // Use an exists filter on the the field name to get - // // examples of the field, so cannot batch up. - // return getFieldExamplesRequest(esClient, params, field); - // break; - } + if (!isValidField(field)) return; + + switch (field.type) { + case JOB_FIELD_TYPES.NUMBER: + return fetchNumericFieldStats(dataPlugin, params, field, options); + case JOB_FIELD_TYPES.KEYWORD: + // case JOB_FIELD_TYPES.IP: + return fetchStringFieldStats(dataPlugin, params, field, options); + case JOB_FIELD_TYPES.DATE: + return fetchDateFieldStats(dataPlugin, params, field, options); + case JOB_FIELD_TYPES.BOOLEAN: + return fetchBooleanFieldStats(dataPlugin, params, field, options); + case JOB_FIELD_TYPES.TEXT: + return fetchFieldExamples(dataPlugin, params, field, options); + case JOB_FIELD_TYPES.IP: + return fetchStringFieldStats(dataPlugin, params, field, options); + default: + // Use an exists filter on the the field name to get + // examples of the field, so cannot batch up. + return fetchFieldExamples(dataPlugin, params, field, options); } }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts index 26565ba7c7730..6e68d21ed8527 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts @@ -5,9 +5,10 @@ * 2.0. */ -import { ElasticsearchClient } from 'kibana/server'; import { SearchRequest } from '@elastic/elasticsearch/api/types'; import { find, get } from 'lodash'; +import { catchError, switchMap } from 'rxjs/operators'; +import { Observable, of } from 'rxjs'; import { MAX_PERCENT, PERCENTILE_SPACING, @@ -15,24 +16,29 @@ import { SAMPLER_TOP_TERMS_THRESHOLD, } from './constants'; import { - buildBaseFilterCriteria, buildSamplerAggregation, getSamplerAggregationsResponsePath, } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; -import type { Field, NumericFieldStats, Bucket } from '../../types/field_stats'; +import type { Field, NumericFieldStats, Bucket, FieldStatsError } from '../../types/field_stats'; import { processDistributionData } from '../../utils/process_distribution_data'; +import { + IKibanaSearchRequest, + IKibanaSearchResponse, + ISearchOptions, +} from '../../../../../../../../src/plugins/data/common'; +import { DataPublicPluginStart } from '../../../../../../../../src/plugins/data/public'; +import { extractErrorProperties } from '../../utils/error_utils'; +import { isIKibanaSearchResponse } from '../../types/field_stats'; export const getNumericFieldStatsRequest = ( params: FieldStatsCommonRequestParams, field: Field ) => { - const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = - params; + const { index, query, runtimeFieldMap, samplerShardSize } = params; const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); // Build the percents parameter which defines the percentiles to query // for the metric distribution data. @@ -87,11 +93,7 @@ export const getNumericFieldStatsRequest = ( } const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, + query, aggs: buildSamplerAggregation(aggs, samplerShardSize), ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), }; @@ -103,61 +105,81 @@ export const getNumericFieldStatsRequest = ( }; }; -export const fetchNumericFieldStats = async ( - esClient: ElasticsearchClient, +export const fetchNumericFieldStats = ( + data: DataPublicPluginStart, params: FieldStatsCommonRequestParams, - field: Field -): Promise => { + field: Field, + options: ISearchOptions +): Observable => { const { samplerShardSize } = params; const request: SearchRequest = getNumericFieldStatsRequest(params, field); - const { body } = await esClient.search(request); - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const safeFieldName = field.safeFieldName; - const docCount = get(aggregations, [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], 0); - const fieldStatsResp = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], - {} - ); + return data.search + .search({ params: request }, options) + .pipe( + catchError((e) => + of({ + fieldName: field.fieldName, + error: extractErrorProperties(e), + } as FieldStatsError) + ), + switchMap((resp) => { + if (!isIKibanaSearchResponse(resp)) return of(resp); - const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } + const aggregations = resp.rawResponse.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); - - const stats: NumericFieldStats = { - fieldName: field.fieldName, - count: docCount, - min: get(fieldStatsResp, 'min', 0), - max: get(fieldStatsResp, 'max', 0), - avg: get(fieldStatsResp, 'avg', 0), - isTopValuesSampled: field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, - topValues, - topValuesSampleSize: topValues.reduce( - (acc, curr) => acc + curr.doc_count, - get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) - ), - topValuesSamplerShardSize: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? SAMPLER_TOP_TERMS_SHARD_SIZE - : samplerShardSize, - }; + const safeFieldName = field.safeFieldName; + const docCount = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], + 0 + ); + const fieldStatsResp = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], + {} + ); + + const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + topAggsPath.push('top'); + } - if (stats.count > 0) { - const percentiles = get( - aggregations, - [...aggsPath, `${safeFieldName}_percentiles`, 'values'], - [] + const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); + + const stats: NumericFieldStats = { + fieldName: field.fieldName, + count: docCount, + min: get(fieldStatsResp, 'min', 0), + max: get(fieldStatsResp, 'max', 0), + avg: get(fieldStatsResp, 'avg', 0), + isTopValuesSampled: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, + topValues, + topValuesSampleSize: topValues.reduce( + (acc, curr) => acc + curr.doc_count, + get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) + ), + topValuesSamplerShardSize: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? SAMPLER_TOP_TERMS_SHARD_SIZE + : samplerShardSize, + }; + + if (stats.count > 0) { + const percentiles = get( + aggregations, + [...aggsPath, `${safeFieldName}_percentiles`, 'values'], + [] + ); + const medianPercentile: { value: number; key: number } | undefined = find(percentiles, { + key: 50, + }); + stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; + stats.distribution = processDistributionData(percentiles, PERCENTILE_SPACING, stats.min); + } + return of(stats); + }) ); - const medianPercentile: { value: number; key: number } | undefined = find(percentiles, { - key: 50, - }); - stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; - stats.distribution = processDistributionData(percentiles, PERCENTILE_SPACING, stats.min); - } - return stats; }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts index 557843affc74a..592b06ecee7e6 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts @@ -5,25 +5,31 @@ * 2.0. */ -import { ElasticsearchClient } from 'kibana/server'; import { SearchRequest } from '@elastic/elasticsearch/api/types'; import { get } from 'lodash'; +import { Observable, of, throwError } from 'rxjs'; +import { catchError, switchMap } from 'rxjs/operators'; import { SAMPLER_TOP_TERMS_SHARD_SIZE, SAMPLER_TOP_TERMS_THRESHOLD } from './constants'; import { - buildBaseFilterCriteria, buildSamplerAggregation, getSamplerAggregationsResponsePath, } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; import type { Aggs, Bucket, Field, StringFieldStats } from '../../types/field_stats'; +import { + DataPublicPluginStart, + IKibanaSearchRequest, + IKibanaSearchResponse, + ISearchOptions, +} from '../../../../../../../../src/plugins/data/public'; +import { FieldStatsError, isIKibanaSearchResponse } from '../../types/field_stats'; +import { extractErrorProperties } from '../../utils/error_utils'; export const getStringFieldStatsRequest = (params: FieldStatsCommonRequestParams, field: Field) => { - const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = - params; + const { index, query, runtimeFieldMap, samplerShardSize } = params; const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); const aggs: Aggs = {}; @@ -54,11 +60,7 @@ export const getStringFieldStatsRequest = (params: FieldStatsCommonRequestParams } const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, + query, aggs: buildSamplerAggregation(aggs, samplerShardSize), ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), }; @@ -70,41 +72,54 @@ export const getStringFieldStatsRequest = (params: FieldStatsCommonRequestParams }; }; -export const fetchStringFieldStats = async ( - esClient: ElasticsearchClient, +export const fetchStringFieldStats = ( + data: DataPublicPluginStart, params: FieldStatsCommonRequestParams, - field: Field -): Promise => { + field: Field, + options: ISearchOptions +): Observable => { const { samplerShardSize } = params; const request: SearchRequest = getStringFieldStatsRequest(params, field); - const { body } = await esClient.search(request); - - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - - const safeFieldName = field.safeFieldName; - - const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } - - const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); - - const stats = { - fieldName: field.fieldName, - isTopValuesSampled: field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, - topValues, - topValuesSampleSize: topValues.reduce( - (acc, curr) => acc + curr.doc_count, - get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) - ), - topValuesSamplerShardSize: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? SAMPLER_TOP_TERMS_SHARD_SIZE - : samplerShardSize, - }; - - return stats; + return data.search + .search({ params: request }, options) + .pipe( + catchError((e) => + of({ + fieldName: field.fieldName, + error: extractErrorProperties(e), + } as FieldStatsError) + ), + switchMap((resp) => { + if (!isIKibanaSearchResponse(resp)) return of(resp); + const aggregations = resp.rawResponse.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + + const safeFieldName = field.safeFieldName; + + const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + topAggsPath.push('top'); + } + + const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); + + const stats = { + fieldName: field.fieldName, + isTopValuesSampled: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, + topValues, + topValuesSampleSize: topValues.reduce( + (acc, curr) => acc + curr.doc_count, + get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) + ), + topValuesSamplerShardSize: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? SAMPLER_TOP_TERMS_SHARD_SIZE + : samplerShardSize, + }; + + return of(stats); + }) + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/field_stats.ts index 56ca8920f0efb..f29cec0a9ddce 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/field_stats.ts @@ -5,7 +5,11 @@ * 2.0. */ +import { estypes } from '@elastic/elasticsearch'; import { isPopulatedObject } from '../../../../common/utils/object_utils'; +import { DVErrorObject } from '../utils/error_utils'; +import { IKibanaSearchResponse, RuntimeField } from '../../../../../../../src/plugins/data/common'; +import { TimeBucketsInterval } from '../../../../common/services/time_buckets'; export interface FieldData { fieldName: string; @@ -47,6 +51,15 @@ export interface Bucket { doc_count: number; } +export interface FieldStatsError { + fieldName: string; + error: DVErrorObject; +} + +export const isIKibanaSearchResponse = (arg: unknown): arg is IKibanaSearchResponse => { + return isPopulatedObject(arg, ['rawResponse']); +}; + export interface NumericFieldStats { fieldName: string; count: number; @@ -180,7 +193,8 @@ export type FieldStats = | BooleanFieldStats | DateFieldStats // | DocumentCountStats - | FieldExamples; + | FieldExamples + | FieldStatsError; export function isValidFieldStats(arg: unknown): arg is FieldStats { return isPopulatedObject(arg, ['fieldName', 'type', 'count']); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/error_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/error_utils.ts index 9bb36496a149e..58c4c820c28d7 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/error_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/error_utils.ts @@ -85,7 +85,7 @@ export function isDVResponseError(error: any): error is DVResponseError { } export function isBoomError(error: any): error is Boom.Boom { - return error.isBoom === true; + return error?.isBoom === true; } export function isWrappedError(error: any): error is WrappedError { diff --git a/x-pack/plugins/data_visualizer/server/plugin.ts b/x-pack/plugins/data_visualizer/server/plugin.ts index 8a98793274aa2..9db580959b116 100644 --- a/x-pack/plugins/data_visualizer/server/plugin.ts +++ b/x-pack/plugins/data_visualizer/server/plugin.ts @@ -8,18 +8,12 @@ import { CoreSetup, CoreStart, Plugin } from 'src/core/server'; import { StartDeps, SetupDeps } from './types'; import { dataVisualizerRoutes } from './routes'; -import { FIELD_STATS_SEARCH_STRATEGY } from '../common/search_strategy/constants'; -import { myEnhancedSearchStrategyProvider } from './search_strategy/field_stats_search_strategy'; export class DataVisualizerPlugin implements Plugin { constructor() {} setup(coreSetup: CoreSetup, plugins: SetupDeps) { dataVisualizerRoutes(coreSetup); - coreSetup.getStartServices().then(([_, depsStart]) => { - const myStrategy = myEnhancedSearchStrategyProvider(depsStart.data); - plugins.data.search.registerSearchStrategy(FIELD_STATS_SEARCH_STRATEGY, myStrategy); - }); } start(core: CoreStart) {} diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts deleted file mode 100644 index bf39199c8b018..0000000000000 --- a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_search_strategy.ts +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from 'kibana/server'; -import { of } from 'rxjs'; -import { chunk } from 'lodash'; -import { fieldStatsSearchServiceStateProvider } from './field_stats_state_provider'; -import { FieldStats, StartDeps } from '../types'; -import { ISearchStrategy } from '../../../../../src/plugins/data/server'; -import { - FieldStatRawResponse, - FieldStatsCommonRequestParams, - FieldStatsRequest, - FieldStatsResponse, - FieldStatsSearchStrategyParams, - isFieldStatsSearchStrategyParams, -} from '../../common/search_strategy/types'; -import { getFieldStats } from './requests/get_field_stats'; -import { buildBaseFilterCriteria, getSafeAggregationName } from '../../common/utils/query_utils'; - -export interface RawResponseBase { - ccsWarning: boolean; - took: number; -} - -export type FieldStatsRequestParams = FieldStatsSearchStrategyParams; - -export interface SearchStrategyServerParams { - includeFrozen?: boolean; -} - -export interface FieldStatsRawResponse extends RawResponseBase { - log: string[]; -} - -interface SearchServiceState { - cancel: () => void; - error: Error; - meta: { - loaded: number; - total: number; - isRunning: boolean; - isPartial: boolean; - }; - rawResponse: TRawResponse; -} - -type GetSearchServiceState = - () => SearchServiceState; - -export type SearchServiceProvider< - TSearchStrategyClientParams, - TRawResponse extends RawResponseBase -> = ( - esClient: ElasticsearchClient, - searchServiceParams: TSearchStrategyClientParams -) => GetSearchServiceState; - -// @todo: rename -export const myEnhancedSearchStrategyProvider = ( - data: StartDeps['data'] -): ISearchStrategy => { - const searchServiceMap = new Map>(); - - return { - search: (request, options, deps) => { - if (options.sessionId === undefined || request.params === undefined) { - throw new Error('Invalid request parameters.'); - } - - // The function to fetch the current state of the search service. - // This will be either an existing service for a follow up fetch or a new one for new requests. - let getSearchServiceState: GetSearchServiceState; - - // If the request includes an ID, we require that the search service already exists - // otherwise we throw an error. The client should never poll a service that's been cancelled or finished. - // This also avoids instantiating search services when the service gets called with random IDs. - const existingGetSearchServiceState = searchServiceMap.get(options.sessionId); - - if (typeof existingGetSearchServiceState === 'undefined') { - getSearchServiceState = fieldStatsSearchServiceProvider( - deps.esClient.asCurrentUser, - request - ); - } else { - getSearchServiceState = existingGetSearchServiceState; - } - - // Reuse the request's id or create a new one. - const id = options.sessionId; - - const { error, meta, rawResponse } = getSearchServiceState(); - - if (error instanceof Error) { - searchServiceMap.delete(id); - throw error; - } else if (meta.isRunning) { - searchServiceMap.set(id, getSearchServiceState); - } else { - searchServiceMap.delete(id); - } - - return of({ - id, - ...meta, - rawResponse, - }); - // search will be called multiple times, - // be sure your response formatting is capable of handling partial results, as well as the final result. - // return ese.search(request, options, deps); - }, - cancel: async (id, options, deps) => { - const getSearchServiceState = searchServiceMap.get(id); - if (getSearchServiceState !== undefined) { - getSearchServiceState().cancel(); - searchServiceMap.delete(id); - } - }, - }; -}; - -// @todo: remove -function timeout(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -export const fieldStatsSearchServiceProvider = ( - esClient: ElasticsearchClient, - request: FieldStatsRequest -) => { - const state = fieldStatsSearchServiceStateProvider(); - - async function fetchFieldStats() { - try { - if (isFieldStatsSearchStrategyParams(request.params)) { - const searchStrategyParams = request.params; - const fields = [ - ...searchStrategyParams.metricConfigs, - ...searchStrategyParams.nonMetricConfigs, - ]; - - const filterCriteria = buildBaseFilterCriteria( - searchStrategyParams.timeFieldName, - searchStrategyParams.earliest, - searchStrategyParams.latest, - searchStrategyParams.searchQuery - ); - - const params: FieldStatsCommonRequestParams = { - index: searchStrategyParams.index, - samplerShardSize: searchStrategyParams.samplerShardSize, - timeFieldName: searchStrategyParams.timeFieldName, - earliestMs: searchStrategyParams.earliest, - latestMs: searchStrategyParams.latest, - runtimeFieldMap: searchStrategyParams.runtimeFieldMap, - intervalMs: searchStrategyParams.intervalMs, - query: { - bool: { - filter: filterCriteria, - }, - }, - }; - - const fieldToLoadCnt = fields.length; - const fieldsWithError: any[] = []; - let fieldsLoadedCnt = 0; - - if (params !== undefined && fields.length > 0) { - const batches = chunk(fields, 10); - for (let i = 0; i < batches.length; i++) { - const batchedResults: FieldStats[] = []; - - try { - const results = await Promise.allSettled([ - ...batches[i].map((field, idx) => { - return getFieldStats(esClient, params, { - fieldName: field.fieldName, - type: field.type, - cardinality: field.cardinality, - safeFieldName: getSafeAggregationName(field.fieldName ?? '', idx), - }); - }), - ]); - - results.forEach((r, idx) => { - if (r.status === 'fulfilled' && r.value !== undefined) { - batchedResults.push(r.value); - } else { - if (r.status === 'rejected' && r.reason) { - const fieldWithError = batches[i][idx]; - const reason = r.reason.meta.body.error?.reason ?? r.reason; - state.addErrorMessage( - `Error fetching field stats for ${fieldWithError.type} field ${fieldWithError.fieldName} because '${reason}'` - ); - } - } - }); - - state.addFieldsStats(batchedResults); - } catch (e) { - state.setError(e); - } - - fieldsLoadedCnt += batches[i].length; - state.setProgress({ - loadedFieldStats: fieldsLoadedCnt / fieldToLoadCnt, - }); - } - } - } - } catch (e) { - state.setError(e); - } - - if (state.getState().error !== undefined) { - state.setCcsWarning(true); - } - - state.setIsRunning(false); - } - - function cancel() { - // addLogMessage(`Service cancelled.`); - state.setIsCancelled(true); - } - - fetchFieldStats(); - - return () => { - const { ccsWarning, error, isRunning, fieldsStats, errorLog } = state.getState(); - return { - cancel, - error, - meta: { - loaded: state.getOverallProgress(), - total: 1, - isRunning, - isPartial: isRunning, - }, - // @todo - rawResponse: { - ccsWarning, - log: [], - took: 0, - fieldStats: fieldsStats, - errorLog, - }, - }; - }; -}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts b/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts deleted file mode 100644 index 1816dee26e163..0000000000000 --- a/x-pack/plugins/data_visualizer/server/search_strategy/field_stats_state_provider.ts +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export interface LatencyCorrelationSearchServiceProgress { - started: number; - loadedOverallStats: number; - loadedFieldStats: number; -} - -export type FieldStat = Record | undefined; -export const fieldStatsSearchServiceStateProvider = () => { - let ccsWarning = false; - function setCcsWarning(d: boolean) { - ccsWarning = d; - } - - const errorLog: string[] = []; - function addErrorMessage(message: string) { - errorLog.push(message); - } - - let error: Error; - function setError(d: Error) { - error = d; - } - - let isCancelled = false; - function getIsCancelled() { - return isCancelled; - } - function setIsCancelled(d: boolean) { - isCancelled = d; - } - - let isRunning = true; - function setIsRunning(d: boolean) { - isRunning = d; - } - - const fieldsStats: FieldStat[] = []; - function addFieldsStats(d: FieldStat[]) { - fieldsStats.push(...d); - } - - let progress: LatencyCorrelationSearchServiceProgress = { - started: Date.now(), - loadedOverallStats: 1, - loadedFieldStats: 0, - }; - function getOverallProgress() { - return progress.loadedOverallStats * 0.1 + progress.loadedFieldStats * 0.9; - } - function setProgress(d: Partial>) { - progress = { - ...progress, - ...d, - }; - } - - function getState() { - return { - ccsWarning, - error, - isCancelled, - isRunning, - progress, - fieldsStats, - errorLog, - }; - } - - return { - getIsCancelled, - getOverallProgress, - getState, - setCcsWarning, - setError, - setIsCancelled, - setIsRunning, - setProgress, - addFieldsStats, - addErrorMessage, - }; -}; - -export type LatencyCorrelationsSearchServiceState = ReturnType< - typeof fieldStatsSearchServiceStateProvider ->; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_boolean_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_boolean_field_stats.ts deleted file mode 100644 index c8a564c5428ea..0000000000000 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_boolean_field_stats.ts +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from 'kibana/server'; -import { SearchRequest } from '@elastic/elasticsearch/api/types'; -import { get } from 'lodash'; -import { FieldStatsCommonRequestParams } from '../../../common/search_strategy/types'; -import { Field, BooleanFieldStats, Aggs } from '../../types'; -import { - buildBaseFilterCriteria, - buildSamplerAggregation, - getSamplerAggregationsResponsePath, -} from '../../../common/utils/query_utils'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; - -export const getBooleanFieldStatsRequest = ( - params: FieldStatsCommonRequestParams, - field: Field -) => { - const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = - params; - - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - const aggs: Aggs = {}; - - const safeFieldName = field.safeFieldName; - aggs[`${safeFieldName}_value_count`] = { - filter: { exists: { field: field.fieldName } }, - }; - aggs[`${safeFieldName}_values`] = { - terms: { - field: field.fieldName, - size: 2, - }, - }; - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), - }; - - return { - index, - size, - body: searchBody, - }; -}; - -export const fetchBooleanFieldStats = async ( - esClient: ElasticsearchClient, - params: FieldStatsCommonRequestParams, - field: Field -): Promise => { - const { samplerShardSize } = params; - const request: SearchRequest = getBooleanFieldStatsRequest(params, field); - const { body } = await esClient.search(request); - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - - const safeFieldName = field.safeFieldName; - const stats: BooleanFieldStats = { - fieldName: field.fieldName, - count: get(aggregations, [...aggsPath, `${safeFieldName}_value_count`, 'doc_count'], 0), - trueCount: 0, - falseCount: 0, - }; - - const valueBuckets: Array<{ [key: string]: number }> = get( - aggregations, - [...aggsPath, `${safeFieldName}_values`, 'buckets'], - [] - ); - valueBuckets.forEach((bucket) => { - stats[`${bucket.key_as_string}Count`] = bucket.doc_count; - }); - return stats; -}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_date_field_stats.ts deleted file mode 100644 index 4033995660f25..0000000000000 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_date_field_stats.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from 'kibana/server'; -import { SearchRequest } from '@elastic/elasticsearch/api/types'; -import { get } from 'lodash'; -import { FieldStatsCommonRequestParams } from '../../../common/search_strategy/types'; -import { Aggs, DateFieldStats, Field } from '../../types'; -import { - buildBaseFilterCriteria, - buildSamplerAggregation, - getSamplerAggregationsResponsePath, -} from '../../../common/utils/query_utils'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; - -export const getDateFieldStatsRequest = (params: FieldStatsCommonRequestParams, field: Field) => { - const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = - params; - - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - const aggs: Aggs = {}; - const safeFieldName = field.safeFieldName; - aggs[`${safeFieldName}_field_stats`] = { - filter: { exists: { field: field.fieldName } }, - aggs: { - actual_stats: { - stats: { field: field.fieldName }, - }, - }, - }; - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), - }; - return { - index, - size, - body: searchBody, - }; -}; - -export const fetchDateFieldStats = async ( - esClient: ElasticsearchClient, - params: FieldStatsCommonRequestParams, - field: Field -): Promise => { - const { samplerShardSize } = params; - - const request: SearchRequest = getDateFieldStatsRequest(params, field); - const { body } = await esClient.search(request); - - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const safeFieldName = field.safeFieldName; - const docCount = get(aggregations, [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], 0); - const fieldStatsResp = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], - {} - ); - return { - fieldName: field.fieldName, - count: docCount, - earliest: get(fieldStatsResp, 'min', 0), - latest: get(fieldStatsResp, 'max', 0), - }; -}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_document_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_document_stats.ts deleted file mode 100644 index 9b11d4a7ac2d5..0000000000000 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_document_stats.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from 'kibana/server'; -import { SearchRequest } from '@elastic/elasticsearch/api/types'; -import { each, get } from 'lodash'; -import { FieldStatsCommonRequestParams } from '../../../common/search_strategy/types'; -import { buildBaseFilterCriteria } from '../../../common/utils/query_utils'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; -import { DocumentCountStats } from '../../types'; - -export const getDocumentCountStatsRequest = (params: FieldStatsCommonRequestParams) => { - const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, intervalMs } = params; - - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - // Don't use the sampler aggregation as this can lead to some potentially - // confusing date histogram results depending on the date range of data amongst shards. - - const aggs = { - eventRate: { - date_histogram: { - field: timeFieldName, - fixed_interval: `${intervalMs}ms`, - min_doc_count: 1, - }, - }, - }; - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs, - ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), - }; - return { - index, - size, - body: searchBody, - }; -}; - -export const fetchDocumentCountStats = async ( - esClient: ElasticsearchClient, - params: FieldStatsCommonRequestParams -): Promise => { - const { intervalMs } = params; - const request: SearchRequest = getDocumentCountStatsRequest(params); - - const { body } = await esClient.search(request); - - const buckets: { [key: string]: number } = {}; - const dataByTimeBucket: Array<{ key: string; doc_count: number }> = get( - body, - ['aggregations', 'eventRate', 'buckets'], - [] - ); - each(dataByTimeBucket, (dataForTime) => { - const time = dataForTime.key; - buckets[time] = dataForTime.doc_count; - }); - - return { - documentCounts: { - interval: intervalMs, - buckets, - }, - }; -}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts deleted file mode 100644 index 2a6198078bff4..0000000000000 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_examples.ts +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from 'kibana/server'; -import { SearchRequest } from '@elastic/elasticsearch/api/types'; -import { get } from 'lodash'; -import { FieldStatsCommonRequestParams } from '../../../common/search_strategy/types'; -import { Field, FieldExamples } from '../../types'; -import { buildBaseFilterCriteria } from '../../../common/utils/query_utils'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; - -// @todo -const maxExamples = 10; -export const getFieldExamplesRequest = (params: FieldStatsCommonRequestParams, field: Field) => { - const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap } = params; - - // Request at least 100 docs so that we have a chance of obtaining - // 'maxExamples' of the field. - const size = Math.max(100, maxExamples); - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - // Use an exists filter to return examples of the field. - filterCriteria.push({ - exists: { field: field.fieldName }, - }); - - const searchBody = { - fields: [field.fieldName], - _source: false, - query: { - bool: { - filter: filterCriteria, - }, - }, - ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), - }; - - return { - index, - size, - body: searchBody, - }; -}; - -export const fetchFieldExamples = async ( - esClient: ElasticsearchClient, - params: FieldStatsCommonRequestParams, - field: Field -): Promise => { - const request: SearchRequest = getFieldExamplesRequest(params, field); - const { body } = await esClient.search(request); - - const stats = { - fieldName: field.fieldName, - examples: [] as any[], - }; - // @ts-expect-error incorrect search response type - if (body.hits.total.value > 0) { - const hits = body.hits.hits; - for (let i = 0; i < hits.length; i++) { - // Use lodash get() to support field names containing dots. - const doc: object[] | undefined = get(hits[i].fields, field.fieldName); - // the results from fields query is always an array - if (Array.isArray(doc) && doc.length > 0) { - const example = doc[0]; - if (example !== undefined && stats.examples.indexOf(example) === -1) { - stats.examples.push(example); - if (stats.examples.length === maxExamples) { - break; - } - } - } - } - } - - return stats; -}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts deleted file mode 100644 index 40e602336f256..0000000000000 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_field_stats.ts +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from 'kibana/server'; -import { JOB_FIELD_TYPES } from '../../../common'; -import { - FieldStatsCommonRequestParams, - FieldStatsSearchStrategyParams, -} from '../../../common/search_strategy/types'; -import { FieldStats, isValidField } from '../../types'; -import { fetchDocumentCountStats } from './get_document_stats'; -import { fetchNumericFieldStats } from './get_numeric_field_stats'; -import { fetchStringFieldStats } from './get_string_field_stats'; -import { fetchDateFieldStats } from './get_date_field_stats'; -import { fetchBooleanFieldStats } from './get_boolean_field_stats'; -import { fetchFieldExamples } from './get_field_examples'; - -export const getFieldStats = async ( - esClient: ElasticsearchClient, - params: FieldStatsCommonRequestParams, - field: { - fieldName?: string; - type: string; - cardinality: number; - safeFieldName: string; - } -): Promise => { - // An invalid field with undefined fieldName is used for a document count request. - if (!isValidField(field)) { - // @todo - // Will only ever be one document count card, - // so no value in batching up the single request. - if (field.type === JOB_FIELD_TYPES.NUMBER && params.intervalMs !== undefined) { - // return fetchDocumentCountStats(esClient, params); - } - } else { - switch (field.type) { - case JOB_FIELD_TYPES.NUMBER: - return fetchNumericFieldStats(esClient, params, field); - break; - case JOB_FIELD_TYPES.KEYWORD: - // case JOB_FIELD_TYPES.IP: - return fetchStringFieldStats(esClient, params, field); - break; - case JOB_FIELD_TYPES.DATE: - return fetchDateFieldStats(esClient, params, field); - break; - case JOB_FIELD_TYPES.BOOLEAN: - return fetchBooleanFieldStats(esClient, params, field); - break; - case JOB_FIELD_TYPES.TEXT: - return fetchFieldExamples(esClient, params, field); - break; - // @todo: fix field.fieldName & move to keyword - case JOB_FIELD_TYPES.IP: - return fetchStringFieldStats(esClient, params, field.fieldName); - break; - // default: - // // Use an exists filter on the the field name to get - // // examples of the field, so cannot batch up. - // return fetchFieldExamples(esClient, params, field); - // break; - } - } -}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_numeric_field_stats.ts deleted file mode 100644 index 4ac0001e6d60a..0000000000000 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_numeric_field_stats.ts +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from 'kibana/server'; -import { SearchRequest } from '@elastic/elasticsearch/api/types'; -import { find, get } from 'lodash'; -import { Bucket, Field, NumericFieldStats } from '../../types'; -import { - buildBaseFilterCriteria, - buildSamplerAggregation, - getSamplerAggregationsResponsePath, -} from '../../../common/utils/query_utils'; -import { - MAX_PERCENT, - PERCENTILE_SPACING, - SAMPLER_TOP_TERMS_SHARD_SIZE, - SAMPLER_TOP_TERMS_THRESHOLD, -} from '../../models/data_visualizer/constants'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; -import { FieldStatsCommonRequestParams } from '../../../common/search_strategy/types'; -import { processDistributionData } from '../../models/data_visualizer/process_distribution_data'; - -export const getNumericFieldStatsRequest = ( - params: FieldStatsCommonRequestParams, - field: Field -) => { - const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = - params; - - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - // Build the percents parameter which defines the percentiles to query - // for the metric distribution data. - // Use a fixed percentile spacing of 5%. - let count = 0; - const percents = Array.from( - Array(MAX_PERCENT / PERCENTILE_SPACING), - () => (count += PERCENTILE_SPACING) - ); - - const aggs: { [key: string]: any } = {}; - const safeFieldName = field.safeFieldName; - aggs[`${safeFieldName}_field_stats`] = { - filter: { exists: { field: field.fieldName } }, - aggs: { - actual_stats: { - stats: { field: field.fieldName }, - }, - }, - }; - aggs[`${safeFieldName}_percentiles`] = { - percentiles: { - field: field.fieldName, - percents, - keyed: false, - }, - }; - - const top = { - terms: { - field: field.fieldName, - size: 10, - order: { - _count: 'desc', - }, - }, - }; - - // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation - // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - aggs[`${safeFieldName}_top`] = { - sampler: { - shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, - }, - aggs: { - top, - }, - }; - } else { - aggs[`${safeFieldName}_top`] = top; - } - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), - }; - - return { - index, - size, - body: searchBody, - }; -}; - -export const fetchNumericFieldStats = async ( - esClient: ElasticsearchClient, - params: FieldStatsCommonRequestParams, - field: Field -): Promise => { - const { samplerShardSize } = params; - const request: SearchRequest = getNumericFieldStatsRequest(params, field); - const { body } = await esClient.search(request); - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - - const safeFieldName = field.safeFieldName; - const docCount = get(aggregations, [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], 0); - const fieldStatsResp = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], - {} - ); - - const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } - - const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); - - const stats: NumericFieldStats = { - fieldName: field.fieldName, - count: docCount, - min: get(fieldStatsResp, 'min', 0), - max: get(fieldStatsResp, 'max', 0), - avg: get(fieldStatsResp, 'avg', 0), - isTopValuesSampled: field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, - topValues, - topValuesSampleSize: topValues.reduce( - (acc, curr) => acc + curr.doc_count, - get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) - ), - topValuesSamplerShardSize: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? SAMPLER_TOP_TERMS_SHARD_SIZE - : samplerShardSize, - }; - - if (stats.count > 0) { - const percentiles = get( - aggregations, - [...aggsPath, `${safeFieldName}_percentiles`, 'values'], - [] - ); - const medianPercentile: { value: number; key: number } | undefined = find(percentiles, { - key: 50, - }); - stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; - stats.distribution = processDistributionData(percentiles, PERCENTILE_SPACING, stats.min); - } - return stats; -}; diff --git a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_string_field_stats.ts b/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_string_field_stats.ts deleted file mode 100644 index 66c6efcdb1864..0000000000000 --- a/x-pack/plugins/data_visualizer/server/search_strategy/requests/get_string_field_stats.ts +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from 'kibana/server'; -import { SearchRequest } from '@elastic/elasticsearch/api/types'; -import { get } from 'lodash'; -import { FieldStatsCommonRequestParams } from '../../../common/search_strategy/types'; -import { Aggs, Bucket, Field, StringFieldStats } from '../../types'; -import { - buildBaseFilterCriteria, - buildSamplerAggregation, - getSamplerAggregationsResponsePath, -} from '../../../common/utils/query_utils'; -import { - SAMPLER_TOP_TERMS_SHARD_SIZE, - SAMPLER_TOP_TERMS_THRESHOLD, -} from '../../models/data_visualizer/constants'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; - -export const getStringFieldStatsRequest = (params: FieldStatsCommonRequestParams, field: Field) => { - const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, samplerShardSize } = - params; - - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - const aggs: Aggs = {}; - - const safeFieldName = field.safeFieldName; - const top = { - terms: { - field: field.fieldName, - size: 10, - order: { - _count: 'desc', - }, - }, - }; - - // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation - // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - aggs[`${safeFieldName}_top`] = { - sampler: { - shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, - }, - aggs: { - top, - }, - }; - } else { - aggs[`${safeFieldName}_top`] = top; - } - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), - }; - - return { - index, - size, - body: searchBody, - }; -}; - -export const fetchStringFieldStats = async ( - esClient: ElasticsearchClient, - params: FieldStatsCommonRequestParams, - field: Field -): Promise => { - const { samplerShardSize } = params; - const request: SearchRequest = getStringFieldStatsRequest(params, field); - - const { body } = await esClient.search(request); - - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - - const safeFieldName = field.safeFieldName; - - const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } - - const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); - - const stats = { - fieldName: field.fieldName, - isTopValuesSampled: field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, - topValues, - topValuesSampleSize: topValues.reduce( - (acc, curr) => acc + curr.doc_count, - get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) - ), - topValuesSamplerShardSize: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? SAMPLER_TOP_TERMS_SHARD_SIZE - : samplerShardSize, - }; - - return stats; -}; diff --git a/x-pack/plugins/data_visualizer/server/types/chart_data.ts b/x-pack/plugins/data_visualizer/server/types/chart_data.ts index 4f1774e5eea34..c9d5fa051a257 100644 --- a/x-pack/plugins/data_visualizer/server/types/chart_data.ts +++ b/x-pack/plugins/data_visualizer/server/types/chart_data.ts @@ -174,15 +174,3 @@ export type BatchStats = | DateFieldStats | DocumentCountStats | FieldExamples; - -export type FieldStats = - | NumericFieldStats - | StringFieldStats - | BooleanFieldStats - | DateFieldStats - // | DocumentCountStats - | FieldExamples; - -export function isValidFieldStats(arg: unknown): arg is FieldStats { - return isPopulatedObject(arg, ['fieldName', 'type', 'count']); -} From 02465b07bfddf6d638d21c73532373dd93b93e37 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 12 Oct 2021 14:53:21 -0500 Subject: [PATCH 138/188] Consolidate types --- .../common/search_strategy/types.ts | 85 --------- .../types/field_stats.ts | 95 ++++++++-- .../common/utils/query_utils.ts | 3 +- .../stats_table/types/field_data_row.ts | 5 +- .../index_data_visualizer_view.tsx | 19 +- .../hooks/use_data_visualizer_grid_data.ts | 4 +- .../hooks/use_field_stats.ts | 6 +- .../hooks/use_overall_stats.ts | 2 +- .../search_strategy/requests/constants.ts | 1 + .../requests/get_boolean_field_stats.ts | 10 +- .../requests/get_date_field_stats.ts | 6 +- .../requests/get_document_stats.ts | 9 +- .../requests/get_field_examples.ts | 19 +- .../requests/get_field_stats.ts | 8 +- .../requests/get_numeric_field_stats.ts | 11 +- .../requests/get_string_field_stats.ts | 17 +- .../search_strategy/requests/overall_stats.ts | 2 +- .../utils/process_distribution_data.ts | 2 +- .../plugins/data_visualizer/public/plugin.ts | 12 +- .../data_visualizer/check_fields_exist.ts | 6 +- .../models/data_visualizer/data_visualizer.ts | 2 +- .../data_visualizer/get_field_examples.ts | 12 +- .../data_visualizer/get_fields_stats.ts | 5 +- .../get_histogram_for_fields.ts | 7 +- .../process_distribution_data.ts | 2 +- .../data_visualizer/server/routes/routes.ts | 3 +- .../server/types/chart_data.ts | 176 ------------------ .../data_visualizer/server/types/index.ts | 1 - 28 files changed, 184 insertions(+), 346 deletions(-) delete mode 100644 x-pack/plugins/data_visualizer/common/search_strategy/types.ts rename x-pack/plugins/data_visualizer/{public/application/index_data_visualizer => common}/types/field_stats.ts (66%) delete mode 100644 x-pack/plugins/data_visualizer/server/types/chart_data.ts diff --git a/x-pack/plugins/data_visualizer/common/search_strategy/types.ts b/x-pack/plugins/data_visualizer/common/search_strategy/types.ts deleted file mode 100644 index 19cd94178dac3..0000000000000 --- a/x-pack/plugins/data_visualizer/common/search_strategy/types.ts +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { estypes } from '@elastic/elasticsearch'; -import type { TimeBucketsInterval } from '../services/time_buckets'; -import type { RuntimeField } from '../../../../../src/plugins/data/common'; -import { FieldStats } from '../../public/application/index_data_visualizer/types/field_stats'; - -export interface FieldStatsCommonRequestParams { - index: string; - samplerShardSize: number; - timeFieldName?: string; - earliestMs?: number | undefined; - latestMs?: number | undefined; - runtimeFieldMap?: Record; - intervalMs?: number; - query: estypes.QueryDslQueryContainer; -} - -export interface OverallStatsSearchStrategyParams { - sessionId?: string; - earliest?: number; - latest?: number; - aggInterval: TimeBucketsInterval; - intervalMs?: number; - searchQuery?: any; - samplerShardSize: number; - index: string; - timeFieldName?: string; - runtimeFieldMap: Record; - aggregatableFields: string[]; - nonAggregatableFields: string[]; -} - -export interface FieldStatsSearchStrategyReturnBase { - progress: FieldStatsSearchStrategyProgress; - fieldStats: Map | undefined; - startFetch: () => void; - cancelFetch: () => void; -} - -export interface FieldStatsSearchStrategyProgress { - error?: Error; - isRunning: boolean; - loaded: number; - total: number; -} - -export interface FieldData { - fieldName: string; - existsInDocs: boolean; - stats?: { - sampleCount?: number; - count?: number; - cardinality?: number; - }; -} - -export interface Field { - fieldName: string; - type: string; - cardinality: number; - safeFieldName: string; -} - -export interface Aggs { - [key: string]: any; -} - -export interface FieldAggCardinality { - field: string; - percent?: any; -} - -export interface ScriptAggCardinality { - script: any; -} - -export interface AggCardinality { - cardinality: FieldAggCardinality | ScriptAggCardinality; -} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/field_stats.ts b/x-pack/plugins/data_visualizer/common/types/field_stats.ts similarity index 66% rename from x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/field_stats.ts rename to x-pack/plugins/data_visualizer/common/types/field_stats.ts index f29cec0a9ddce..a6ac7dcf91ef8 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/field_stats.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_stats.ts @@ -6,10 +6,10 @@ */ import { estypes } from '@elastic/elasticsearch'; -import { isPopulatedObject } from '../../../../common/utils/object_utils'; -import { DVErrorObject } from '../utils/error_utils'; -import { IKibanaSearchResponse, RuntimeField } from '../../../../../../../src/plugins/data/common'; -import { TimeBucketsInterval } from '../../../../common/services/time_buckets'; +import { AggregationsHistogramAggregation } from '@elastic/elasticsearch/api/types'; +import { isPopulatedObject } from '../utils/object_utils'; +import { IKibanaSearchResponse } from '../../../../../src/plugins/data/common'; +import { TimeBucketsInterval } from '../services/time_buckets'; export interface FieldData { fieldName: string; @@ -43,17 +43,13 @@ export interface Distribution { maxPercentile: number; } -export interface Aggs { - [key: string]: any; -} - export interface Bucket { doc_count: number; } export interface FieldStatsError { fieldName: string; - error: DVErrorObject; + error: Error; } export const isIKibanaSearchResponse = (arg: unknown): arg is IKibanaSearchResponse => { @@ -117,10 +113,7 @@ export interface NumericColumnStats { export type NumericColumnStatsMap = Record; export interface AggHistogram { - histogram: { - field: string; - interval: number; - }; + histogram: AggregationsHistogramAggregation; } export interface AggTerms { @@ -192,10 +185,84 @@ export type FieldStats = | StringFieldStats | BooleanFieldStats | DateFieldStats - // | DocumentCountStats | FieldExamples | FieldStatsError; export function isValidFieldStats(arg: unknown): arg is FieldStats { return isPopulatedObject(arg, ['fieldName', 'type', 'count']); } + +export interface FieldStatsCommonRequestParams { + index: string; + samplerShardSize: number; + timeFieldName?: string; + earliestMs?: number | undefined; + latestMs?: number | undefined; + runtimeFieldMap?: estypes.MappingRuntimeFields; + intervalMs?: number; + query: estypes.QueryDslQueryContainer; + maxExamples?: number; +} + +export interface OverallStatsSearchStrategyParams { + sessionId?: string; + earliest?: number; + latest?: number; + aggInterval: TimeBucketsInterval; + intervalMs?: number; + searchQuery?: any; + samplerShardSize: number; + index: string; + timeFieldName?: string; + runtimeFieldMap?: estypes.MappingRuntimeFields; + aggregatableFields: string[]; + nonAggregatableFields: string[]; +} + +export interface FieldStatsSearchStrategyReturnBase { + progress: FieldStatsSearchStrategyProgress; + fieldStats: Map | undefined; + startFetch: () => void; + cancelFetch: () => void; +} + +export interface FieldStatsSearchStrategyProgress { + error?: Error; + isRunning: boolean; + loaded: number; + total: number; +} + +export interface FieldData { + fieldName: string; + existsInDocs: boolean; + stats?: { + sampleCount?: number; + count?: number; + cardinality?: number; + }; +} + +export interface Field { + fieldName: string; + type: string; + cardinality: number; + safeFieldName: string; +} + +export interface Aggs { + [key: string]: estypes.AggregationsAggregationContainer; +} + +export interface FieldAggCardinality { + field: string; + percent?: any; +} + +export interface ScriptAggCardinality { + script: any; +} + +export interface AggCardinality { + cardinality: FieldAggCardinality | ScriptAggCardinality; +} diff --git a/x-pack/plugins/data_visualizer/common/utils/query_utils.ts b/x-pack/plugins/data_visualizer/common/utils/query_utils.ts index 647bf1781d34a..a6cedfdee1512 100644 --- a/x-pack/plugins/data_visualizer/common/utils/query_utils.ts +++ b/x-pack/plugins/data_visualizer/common/utils/query_utils.ts @@ -6,6 +6,7 @@ */ import { estypes } from '@elastic/elasticsearch'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/api/types'; /* * Contains utility functions for building and processing queries. */ @@ -17,7 +18,7 @@ export function buildBaseFilterCriteria( earliestMs?: number, latestMs?: number, query?: object -): estypes.QueryDslBoolQuery['filter'] { +): QueryDslQueryContainer[] { const filterCriteria = []; if (timeFieldName && earliestMs && latestMs) { filterCriteria.push({ diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_data_row.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_data_row.ts index 94b704764c93b..3d7678c7b60a5 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_data_row.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/field_data_row.ts @@ -5,8 +5,11 @@ * 2.0. */ -import type { FieldVisConfig, FileBasedFieldVisConfig } from './field_vis_config'; import { IndexPatternField } from '../../../../../../../../../src/plugins/data/common'; +import { + FieldVisConfig, + FileBasedFieldVisConfig, +} from '../../../../../../common/types/field_vis_config'; export interface FieldDataRowProps { config: FieldVisConfig | FileBasedFieldVisConfig; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 56d338babd346..3643fede715d4 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -622,15 +622,16 @@ export const IndexDataVisualizerView: FC = (dataVi }); }); - // Add a config for 'document count', identified by no field name if indexpattern is time based. - if (currentIndexPattern.timeFieldName !== undefined) { - configs.push({ - type: JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - loading: true, - aggregatable: true, - }); - } + // @todo: REMOVE + // // Add a config for 'document count', identified by no field name if indexpattern is time based. + // if (currentIndexPattern.timeFieldName !== undefined) { + // configs.push({ + // type: JOB_FIELD_TYPES.NUMBER, + // existsInDocs: true, + // loading: true, + // aggregatable: true, + // }); + // } if (metricsLoaded === false) { setMetricsLoaded(true); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index dd50b1b3263b9..6280226f49221 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -35,7 +35,7 @@ import { DataVisualizerGridEmbeddableInput } from '../embeddables/grid_embeddabl import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; import { useFieldStatsSearchStrategy } from './use_field_stats'; import { useOverallStats } from './use_overall_stats'; -import { OverallStatsSearchStrategyParams } from '../../../../common/search_strategy/types'; +import { OverallStatsSearchStrategyParams } from '../../../../common/types/field_stats'; const defaults = getDefaultPageState(); @@ -417,6 +417,8 @@ export const useDataVisualizerGridData = ( } if (fieldStats) { + // @todo + // @ts-ignore combinedConfigs = combinedConfigs.map((c) => { const loadedFullStats = fieldStats.get(c.fieldName); return loadedFullStats diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index 48c8ddb71508b..8cc20a4033075 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -12,17 +12,17 @@ import type { FieldStatsSearchStrategyProgress, FieldStatsSearchStrategyReturnBase, OverallStatsSearchStrategyParams, -} from '../../../../common/search_strategy/types'; + FieldStatsCommonRequestParams, +} from '../../../../common/types/field_stats'; import { useDataVisualizerKibana } from '../../kibana_context'; import type { FieldRequestConfig } from '../../../../common'; import type { DataVisualizerIndexBasedAppState } from '../types/index_data_visualizer_state'; -import type { FieldStatsCommonRequestParams } from '../../../../common/search_strategy/types'; import { buildBaseFilterCriteria, getSafeAggregationName, } from '../../../../common/utils/query_utils'; import { getFieldStats } from '../search_strategy/requests/get_field_stats'; -import type { FieldStats, FieldStatsError } from '../types/field_stats'; +import type { FieldStats, FieldStatsError } from '../../../../common/types/field_stats'; const getInitialProgress = (): FieldStatsSearchStrategyProgress => ({ isRunning: false, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index fe15a378bf4a1..6670c946e08ce 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -10,7 +10,7 @@ import { combineLatest, of, Subscription } from 'rxjs'; import { mergeMap, switchMap } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; import { ToastsStart } from 'kibana/public'; -import { OverallStatsSearchStrategyParams } from '../../../../common/search_strategy/types'; +import { OverallStatsSearchStrategyParams } from '../../../../common/types/field_stats'; import { useDataVisualizerKibana } from '../../kibana_context'; import { checkAggregatableFieldsExistRequest, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/constants.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/constants.ts index 4dbe51e5c0838..6da11fd850acc 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/constants.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/constants.ts @@ -12,5 +12,6 @@ export const FIELDS_REQUEST_BATCH_SIZE = 10; export const MAX_CHART_COLUMNS = 20; +export const MAX_EXAMPLES_DEFAULT = 10; export const MAX_PERCENT = 100; export const PERCENTILE_SPACING = 5; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts index 279ef09c13a24..ccf9f87df2bfb 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts @@ -13,9 +13,13 @@ import { getSamplerAggregationsResponsePath, } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; -import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; -import type { Field, BooleanFieldStats, Aggs } from '../../types/field_stats'; -import { FieldStatsError, isIKibanaSearchResponse } from '../../types/field_stats'; +import type { + Field, + BooleanFieldStats, + Aggs, + FieldStatsCommonRequestParams, +} from '../../../../../common/types/field_stats'; +import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; import { DataPublicPluginStart, IKibanaSearchRequest, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts index 9aa0f3011288f..d796083d3c004 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts @@ -14,15 +14,15 @@ import { getSamplerAggregationsResponsePath, } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; -import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; -import type { Field, DateFieldStats, Aggs } from '../../types/field_stats'; +import type { FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats'; +import type { Field, DateFieldStats, Aggs } from '../../../../../common/types/field_stats'; import { DataPublicPluginStart, IKibanaSearchRequest, IKibanaSearchResponse, ISearchOptions, } from '../../../../../../../../src/plugins/data/public'; -import { FieldStatsError, isIKibanaSearchResponse } from '../../types/field_stats'; +import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; import { extractErrorProperties } from '../../utils/error_utils'; export const getDateFieldStatsRequest = (params: FieldStatsCommonRequestParams, field: Field) => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts index 91b08dfb829a4..d7a54823f8c6e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts @@ -10,8 +10,10 @@ import { SearchRequest } from '@elastic/elasticsearch/api/types'; import { each, get } from 'lodash'; import { buildBaseFilterCriteria } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; -import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; -import type { DocumentCountStats } from '../../types/field_stats'; +import type { + DocumentCountStats, + FieldStatsCommonRequestParams, +} from '../../../../../common/types/field_stats'; export const getDocumentCountStatsRequest = (params: FieldStatsCommonRequestParams) => { const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, intervalMs } = params; @@ -70,7 +72,8 @@ export const fetchDocumentCountStats = async ( return { documentCounts: { - interval: intervalMs, + // @todo: confirm + interval: intervalMs ?? 0, buckets, }, }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts index fd47a6176a1a7..902cafb67a9c1 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts @@ -11,25 +11,28 @@ import { Observable, of } from 'rxjs'; import { catchError, switchMap } from 'rxjs/operators'; import { buildBaseFilterCriteria } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; -import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; -import type { Field, FieldExamples } from '../../types/field_stats'; +import type { + Field, + FieldExamples, + FieldStatsCommonRequestParams, +} from '../../../../../common/types/field_stats'; import { DataPublicPluginStart, IKibanaSearchRequest, IKibanaSearchResponse, ISearchOptions, } from '../../../../../../../../src/plugins/data/public'; -import { FieldStatsError, isIKibanaSearchResponse } from '../../types/field_stats'; +import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; import { extractErrorProperties } from '../../utils/error_utils'; +import { MAX_EXAMPLES_DEFAULT } from './constants'; -// @todo -const maxExamples = 10; export const getFieldExamplesRequest = (params: FieldStatsCommonRequestParams, field: Field) => { - const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap } = params; + const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, maxExamples } = + params; // Request at least 100 docs so that we have a chance of obtaining // 'maxExamples' of the field. - const size = Math.max(100, maxExamples); + const size = Math.max(100, maxExamples ?? MAX_EXAMPLES_DEFAULT); const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); // Use an exists filter to return examples of the field. @@ -64,6 +67,7 @@ export const fetchFieldExamples = ( options: ISearchOptions ): Observable => { const request: SearchRequest = getFieldExamplesRequest(params, field); + const { maxExamples } = params; return data.search .search({ params: request }, options) .pipe( @@ -80,7 +84,6 @@ export const fetchFieldExamples = ( fieldName: field.fieldName, examples: [] as any[], } as FieldExamples; - // @ts-expect-error incorrect search response type if (body.hits.total.value > 0) { const hits = body.hits.hits; for (let i = 0; i < hits.length; i++) { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts index 52f737ca1d531..2483ef5b4624a 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts @@ -6,14 +6,18 @@ */ import { Observable } from 'rxjs'; -import { FieldStats, FieldStatsError, isValidField } from '../../types/field_stats'; +import { + FieldStats, + FieldStatsError, + isValidField, + FieldStatsCommonRequestParams, +} from '../../../../../common/types/field_stats'; import { fetchNumericFieldStats } from './get_numeric_field_stats'; import { fetchStringFieldStats } from './get_string_field_stats'; import { fetchDateFieldStats } from './get_date_field_stats'; import { fetchBooleanFieldStats } from './get_boolean_field_stats'; import { fetchFieldExamples } from './get_field_examples'; import { JOB_FIELD_TYPES } from '../../../../../common'; -import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; import { ISearchOptions } from '../../../../../../../../src/plugins/data/common'; import { DataPublicPluginStart } from '../../../../../../../../src/plugins/data/public'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts index 6e68d21ed8527..9780f6adafc37 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts @@ -20,8 +20,13 @@ import { getSamplerAggregationsResponsePath, } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; -import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; -import type { Field, NumericFieldStats, Bucket, FieldStatsError } from '../../types/field_stats'; +import type { FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats'; +import type { + Field, + NumericFieldStats, + Bucket, + FieldStatsError, +} from '../../../../../common/types/field_stats'; import { processDistributionData } from '../../utils/process_distribution_data'; import { IKibanaSearchRequest, @@ -30,7 +35,7 @@ import { } from '../../../../../../../../src/plugins/data/common'; import { DataPublicPluginStart } from '../../../../../../../../src/plugins/data/public'; import { extractErrorProperties } from '../../utils/error_utils'; -import { isIKibanaSearchResponse } from '../../types/field_stats'; +import { isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; export const getNumericFieldStatsRequest = ( params: FieldStatsCommonRequestParams, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts index 592b06ecee7e6..668bc9abd081e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { AggregationsAggregationContainer, SearchRequest } from '@elastic/elasticsearch/api/types'; import { get } from 'lodash'; -import { Observable, of, throwError } from 'rxjs'; +import { Observable, of } from 'rxjs'; import { catchError, switchMap } from 'rxjs/operators'; import { SAMPLER_TOP_TERMS_SHARD_SIZE, SAMPLER_TOP_TERMS_THRESHOLD } from './constants'; import { @@ -15,15 +15,20 @@ import { getSamplerAggregationsResponsePath, } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; -import type { FieldStatsCommonRequestParams } from '../../../../../common/search_strategy/types'; -import type { Aggs, Bucket, Field, StringFieldStats } from '../../types/field_stats'; +import type { + Aggs, + Bucket, + Field, + FieldStatsCommonRequestParams, + StringFieldStats, +} from '../../../../../common/types/field_stats'; import { DataPublicPluginStart, IKibanaSearchRequest, IKibanaSearchResponse, ISearchOptions, } from '../../../../../../../../src/plugins/data/public'; -import { FieldStatsError, isIKibanaSearchResponse } from '../../types/field_stats'; +import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; import { extractErrorProperties } from '../../utils/error_utils'; export const getStringFieldStatsRequest = (params: FieldStatsCommonRequestParams, field: Field) => { @@ -34,7 +39,7 @@ export const getStringFieldStatsRequest = (params: FieldStatsCommonRequestParams const aggs: Aggs = {}; const safeFieldName = field.safeFieldName; - const top = { + const top: AggregationsAggregationContainer = { terms: { field: field.fieldName, size: 10, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts index 1296554d984e1..3a1e9a553190c 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts @@ -17,7 +17,7 @@ import { getDatafeedAggregations } from '../../../../../common/utils/datafeed_ut import { isPopulatedObject } from '../../../../../common/utils/object_utils'; import { IKibanaSearchResponse } from '../../../../../../../../src/plugins/data/common'; import { AggregatableField, NonAggregatableField } from '../../types/overall_stats'; -import { AggCardinality, Aggs } from '../../../../../common/search_strategy/types'; +import { AggCardinality, Aggs } from '../../../../../common/types/field_stats'; export const checkAggregatableFieldsExistRequest = ( indexPatternTitle: string, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts index 5fdbc44373e40..ef05d19b78758 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts @@ -6,7 +6,7 @@ */ import { last } from 'lodash'; -import type { Distribution } from '../types/field_stats'; +import type { Distribution } from '../../../../common/types/field_stats'; export const processDistributionData = ( percentiles: Array<{ value: number }>, diff --git a/x-pack/plugins/data_visualizer/public/plugin.ts b/x-pack/plugins/data_visualizer/public/plugin.ts index 8b2a878e6c1f7..fa652fa4a7026 100644 --- a/x-pack/plugins/data_visualizer/public/plugin.ts +++ b/x-pack/plugins/data_visualizer/public/plugin.ts @@ -11,10 +11,7 @@ import type { SharePluginStart } from '../../../../src/plugins/share/public'; import { Plugin } from '../../../../src/core/public'; import { setStartServices } from './kibana_services'; -import type { - DataPublicPluginStart, - SearchSessionInfoProvider, -} from '../../../../src/plugins/data/public'; +import type { DataPublicPluginStart } from '../../../../src/plugins/data/public'; import type { HomePublicPluginSetup } from '../../../../src/plugins/home/public'; import type { FileUploadPluginStart } from '../../file_upload/public'; import type { MapsStartApi } from '../../maps/public'; @@ -70,10 +67,6 @@ export class DataVisualizerPlugin public start(core: CoreStart, plugins: DataVisualizerStartDependencies) { setStartServices(core, plugins); if (plugins.data) { - // const sessionRestorationDataProvider: SearchSessionInfoProvider = { - // data: plugins.data, - // }; - plugins.data.search.session.enableStorage({ getName: async () => { // return the name you want to give the saved Search Session @@ -84,9 +77,6 @@ export class DataVisualizerPlugin urlGeneratorId: DATA_VISUALIZER_APP_LOCATOR, initialState: { shouldRestoreSearchSession: false }, restoreState: { shouldRestoreSearchSession: true }, - - // initialState: getUrlGeneratorState({ ...deps, shouldRestoreSearchSession: false }), - // restoreState: getUrlGeneratorState({ ...deps, shouldRestoreSearchSession: true }), }; }, }); diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/check_fields_exist.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/check_fields_exist.ts index ca7287394b8e3..02f62c3b3f296 100644 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/check_fields_exist.ts +++ b/x-pack/plugins/data_visualizer/server/models/data_visualizer/check_fields_exist.ts @@ -8,7 +8,7 @@ import { estypes } from '@elastic/elasticsearch'; import { get } from 'lodash'; import { IScopedClusterClient } from 'kibana/server'; -import { AggCardinality, Aggs, FieldData } from '../../types'; +import { AggCardinality, Aggs, FieldData } from '../../../common/types/field_stats'; import { buildBaseFilterCriteria, buildSamplerAggregation, @@ -171,7 +171,9 @@ export const checkNonAggregatableFieldExists = async ( }, ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), }; - filterCriteria.push({ exists: { field } }); + if (Array.isArray(filterCriteria)) { + filterCriteria.push({ exists: { field } }); + } const { body } = await asCurrentUser.search({ index, diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/data_visualizer.ts index 155cf09ebb8db..7c2ef2cd49551 100644 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/data_visualizer.ts +++ b/x-pack/plugins/data_visualizer/server/models/data_visualizer/data_visualizer.ts @@ -16,7 +16,7 @@ import type { Field, DocumentCountStats, FieldExamples, -} from '../../types'; +} from '../../../common/types/field_stats'; import { getHistogramsForFields } from './get_histogram_for_fields'; import { checkAggregatableFieldsExist, diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_field_examples.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_field_examples.ts index 69476e254068f..61722570c4a06 100644 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_field_examples.ts +++ b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_field_examples.ts @@ -10,7 +10,7 @@ import { get } from 'lodash'; import { IScopedClusterClient } from 'kibana/server'; import { buildBaseFilterCriteria } from '../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../common/utils/object_utils'; -import { FieldExamples } from '../../types/chart_data'; +import { FieldExamples } from '../../../common/types/field_stats'; export const getFieldExamples = async ( client: IScopedClusterClient, @@ -32,10 +32,12 @@ export const getFieldExamples = async ( const size = Math.max(100, maxExamples); const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - // Use an exists filter to return examples of the field. - filterCriteria.push({ - exists: { field }, - }); + if (Array.isArray(filterCriteria)) { + // Use an exists filter to return examples of the field. + filterCriteria.push({ + exists: { field }, + }); + } const searchBody = { fields: [field], diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts index 6968aa97ab938..ec5ff2f62df44 100644 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts +++ b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts @@ -8,6 +8,7 @@ import { estypes } from '@elastic/elasticsearch'; import { each, find, get } from 'lodash'; import { IScopedClusterClient } from 'kibana/server'; +import { AggregationsAggregationContainer } from '@elastic/elasticsearch/api/types'; import { Aggs, BooleanFieldStats, @@ -17,7 +18,7 @@ import { Field, NumericFieldStats, StringFieldStats, -} from '../../types'; +} from '../../../common/types/field_stats'; import { buildBaseFilterCriteria, buildSamplerAggregation, @@ -259,7 +260,7 @@ export const getStringFieldsStats = async ( const aggs: Aggs = {}; fields.forEach((field, i) => { const safeFieldName = getSafeAggregationName(field.fieldName, i); - const top = { + const top: AggregationsAggregationContainer = { terms: { field: field.fieldName, size: 10, diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_histogram_for_fields.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_histogram_for_fields.ts index 6621c793c0017..8db886a9e86c2 100644 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_histogram_for_fields.ts +++ b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_histogram_for_fields.ts @@ -8,7 +8,12 @@ import { IScopedClusterClient } from 'kibana/server'; import { estypes } from '@elastic/elasticsearch'; import { get } from 'lodash'; -import { ChartData, ChartRequestAgg, HistogramField, NumericColumnStatsMap } from '../../types'; +import { + ChartData, + ChartRequestAgg, + HistogramField, + NumericColumnStatsMap, +} from '../../../common/types/field_stats'; import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/common'; import { stringHash } from '../../../common/utils/string_utils'; import { diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts index 432fe30e64cb3..dcbe6e06738cc 100644 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts +++ b/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts @@ -6,7 +6,7 @@ */ import { last } from 'lodash'; -import { Distribution } from '../../types'; +import { Distribution } from '../../../common/types/field_stats'; // @todo: REMOVE export const processDistributionData = ( diff --git a/x-pack/plugins/data_visualizer/server/routes/routes.ts b/x-pack/plugins/data_visualizer/server/routes/routes.ts index 13a0031a25dfd..582dab1ad7d97 100644 --- a/x-pack/plugins/data_visualizer/server/routes/routes.ts +++ b/x-pack/plugins/data_visualizer/server/routes/routes.ts @@ -13,7 +13,8 @@ import { dataVisualizerOverallStatsSchema, indexPatternTitleSchema, } from './schemas'; -import type { Field, StartDeps, HistogramField } from '../types'; +import type { StartDeps } from '../types'; +import type { Field, HistogramField } from '../../common/types/field_stats'; import { DataVisualizer } from '../models/data_visualizer'; import { wrapError } from '../utils/error_wrapper'; diff --git a/x-pack/plugins/data_visualizer/server/types/chart_data.ts b/x-pack/plugins/data_visualizer/server/types/chart_data.ts deleted file mode 100644 index c9d5fa051a257..0000000000000 --- a/x-pack/plugins/data_visualizer/server/types/chart_data.ts +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isPopulatedObject } from '../../common/utils/object_utils'; - -// @todo: Remove -export interface FieldData { - fieldName: string; - existsInDocs: boolean; - stats?: { - sampleCount?: number; - count?: number; - cardinality?: number; - }; -} - -export interface Field { - fieldName: string; - type: string; - cardinality: number; - safeFieldName: string; -} - -export function isValidField(arg: unknown): arg is Field { - return isPopulatedObject(arg, ['fieldName', 'type']) && typeof arg.fieldName === 'string'; -} - -export interface HistogramField { - fieldName: string; - type: string; -} - -export interface Distribution { - percentiles: any[]; - minPercentile: number; - maxPercentile: number; -} - -export interface Aggs { - [key: string]: any; -} - -export interface Bucket { - doc_count: number; -} - -export interface NumericFieldStats { - fieldName: string; - count: number; - min: number; - max: number; - avg: number; - isTopValuesSampled: boolean; - topValues: Bucket[]; - topValuesSampleSize: number; - topValuesSamplerShardSize: number; - median?: number; - distribution?: Distribution; -} - -export interface StringFieldStats { - fieldName: string; - isTopValuesSampled: boolean; - topValues: Bucket[]; - topValuesSampleSize: number; - topValuesSamplerShardSize: number; -} - -export interface DateFieldStats { - fieldName: string; - count: number; - earliest: number; - latest: number; -} - -export interface BooleanFieldStats { - fieldName: string; - count: number; - trueCount: number; - falseCount: number; - [key: string]: number | string; -} - -export interface DocumentCountStats { - documentCounts: { - interval: number; - buckets: { [key: string]: number }; - }; -} - -export interface FieldExamples { - fieldName: string; - examples: any[]; -} - -export interface NumericColumnStats { - interval: number; - min: number; - max: number; -} -export type NumericColumnStatsMap = Record; - -export interface AggHistogram { - histogram: { - field: string; - interval: number; - }; -} - -export interface AggTerms { - terms: { - field: string; - size: number; - }; -} - -export interface NumericDataItem { - key: number; - key_as_string?: string; - doc_count: number; -} - -export interface NumericChartData { - data: NumericDataItem[]; - id: string; - interval: number; - stats: [number, number]; - type: 'numeric'; -} - -export interface OrdinalDataItem { - key: string; - key_as_string?: string; - doc_count: number; -} - -export interface OrdinalChartData { - type: 'ordinal' | 'boolean'; - cardinality: number; - data: OrdinalDataItem[]; - id: string; -} - -export interface UnsupportedChartData { - id: string; - type: 'unsupported'; -} - -export interface FieldAggCardinality { - field: string; - percent?: any; -} - -export interface ScriptAggCardinality { - script: any; -} - -export interface AggCardinality { - cardinality: FieldAggCardinality | ScriptAggCardinality; -} - -export type ChartRequestAgg = AggHistogram | AggCardinality | AggTerms; - -export type ChartData = NumericChartData | OrdinalChartData | UnsupportedChartData; - -export type BatchStats = - | NumericFieldStats - | StringFieldStats - | BooleanFieldStats - | DateFieldStats - | DocumentCountStats - | FieldExamples; diff --git a/x-pack/plugins/data_visualizer/server/types/index.ts b/x-pack/plugins/data_visualizer/server/types/index.ts index e0379b514de32..2fc0fb2a6173b 100644 --- a/x-pack/plugins/data_visualizer/server/types/index.ts +++ b/x-pack/plugins/data_visualizer/server/types/index.ts @@ -5,4 +5,3 @@ * 2.0. */ export * from './deps'; -export * from './chart_data'; From 86c911556a3c551325976ce4d5f12702e4f7ab56 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 12 Oct 2021 15:11:38 -0500 Subject: [PATCH 139/188] Change back to forkjoin instead of combinelatest for overallstats --- .../index_data_visualizer/hooks/use_overall_stats.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 6670c946e08ce..833d8e5683462 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -6,11 +6,11 @@ */ import { useCallback, useEffect, useState, useRef, useMemo } from 'react'; -import { combineLatest, of, Subscription } from 'rxjs'; +import { combineLatest, forkJoin, of, Subscription } from 'rxjs'; import { mergeMap, switchMap } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; import { ToastsStart } from 'kibana/public'; -import { OverallStatsSearchStrategyParams } from '../../../../common/types/field_stats'; +import { OverallStatsSearchStrategyParams } from '../../../../common/search_strategy/types'; import { useDataVisualizerKibana } from '../../kibana_context'; import { checkAggregatableFieldsExistRequest, @@ -142,8 +142,11 @@ export function useOverallStats { + const sub = forkJoin({ + nonAggregatableOverallStatsResp: nonAggregatableOverallStats$, + aggregatableOverallStatsResp: aggregatableOverallStats$, + }).pipe( + mergeMap(({ nonAggregatableOverallStatsResp, aggregatableOverallStatsResp }) => { const aggregatableOverallStats = processAggregatableFieldsExistResponse( aggregatableOverallStatsResp?.rawResponse, aggregatableFields, From 99fc665dc6885d5cc51c40dab9b07fd606271336 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 13 Oct 2021 15:21:28 -0500 Subject: [PATCH 140/188] Fix missing bool clause --- .../utils/saved_search_utils.ts | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index 80a2069aab1a8..e6288cde25fbd 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -69,20 +69,22 @@ export function createMergedEsQuery( if (query.query !== '') { combinedQuery = toElasticsearchQuery(ast, indexPattern); } - const filterQuery = buildQueryFromFilters(filters, indexPattern); + if (combinedQuery.bool !== undefined) { + const filterQuery = buildQueryFromFilters(filters, indexPattern); - if (Array.isArray(combinedQuery.bool.filter) === false) { - combinedQuery.bool.filter = - combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; - } + if (Array.isArray(combinedQuery.bool.filter) === false) { + combinedQuery.bool.filter = + combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; + } - if (Array.isArray(combinedQuery.bool.must_not) === false) { - combinedQuery.bool.must_not = - combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; - } + if (Array.isArray(combinedQuery.bool.must_not) === false) { + combinedQuery.bool.must_not = + combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; + } - combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; - combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; + combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; + combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; + } } else { combinedQuery = buildEsQuery( indexPattern, From f9f89ad65be37d8654241d68fa1170c6bfabf8c9 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 13 Oct 2021 17:04:18 -0500 Subject: [PATCH 141/188] Add login --- .../data_visualizer/index_data_visualizer_grid_in_discover.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts index 0d50da07f91d0..8019b69281c4a 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts @@ -690,6 +690,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await ml.testResources.createSavedSearchFarequoteLuceneIfNeeded(); await ml.testResources.createSavedSearchFarequoteFilterAndLuceneIfNeeded(); await ml.testResources.createSavedSearchFarequoteFilterAndKueryIfNeeded(); + + await ml.securityUI.loginAsMlPowerUser(); }); after(async function () { From b3ab9fa6f8f92f4404248a7d261555f52c3e4a17 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 13 Oct 2021 17:27:50 -0500 Subject: [PATCH 142/188] Fix saved search attributes broken with latest changes --- .../apps/main/components/layout/discover_layout.scss | 4 ++++ src/plugins/discover/public/plugin.tsx | 5 +++++ .../discover/public/saved_searches/saved_searches_utils.ts | 4 ++++ src/plugins/discover/public/saved_searches/types.ts | 2 ++ 4 files changed, 15 insertions(+) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss index 743f0fa5ec9fd..37ff50e333124 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.scss @@ -4,6 +4,10 @@ discover-app { flex-grow: 1; } +.dscAppWrapper { + overflow-y: hidden; +} + .dscPage { @include euiBreakpoint('m', 'l', 'xl') { @include kibanaFullBodyHeight(); diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index cd2c3a1bf46cc..72244b08d03d8 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -352,6 +352,11 @@ export class DiscoverPlugin const { renderApp } = await import('./application'); + // FIXME: Temporarily hide overflow-y in Discover app when Field Stats table is shown + // due to EUI bug https://github.com/elastic/eui/pull/5152 + // until EUI is bumped to 38.0.0 + params.element.classList.add('dscAppWrapper'); + const unmount = renderApp(params.element); return () => { unlistenParentHistory(); diff --git a/src/plugins/discover/public/saved_searches/saved_searches_utils.ts b/src/plugins/discover/public/saved_searches/saved_searches_utils.ts index 98ab2267a875e..064ee6afe0e99 100644 --- a/src/plugins/discover/public/saved_searches/saved_searches_utils.ts +++ b/src/plugins/discover/public/saved_searches/saved_searches_utils.ts @@ -41,6 +41,8 @@ export const fromSavedSearchAttributes = ( description: attributes.description, grid: attributes.grid, hideChart: attributes.hideChart, + viewMode: attributes.viewMode, + hideAggregatedPreview: attributes.hideAggregatedPreview, }); export const toSavedSearchAttributes = ( @@ -54,4 +56,6 @@ export const toSavedSearchAttributes = ( description: savedSearch.description ?? '', grid: savedSearch.grid ?? {}, hideChart: savedSearch.hideChart ?? false, + viewMode: savedSearch.viewMode, + hideAggregatedPreview: savedSearch.hideAggregatedPreview, }); diff --git a/src/plugins/discover/public/saved_searches/types.ts b/src/plugins/discover/public/saved_searches/types.ts index fcd9a5e5beed2..bb5e89f6ac3d4 100644 --- a/src/plugins/discover/public/saved_searches/types.ts +++ b/src/plugins/discover/public/saved_searches/types.ts @@ -44,4 +44,6 @@ export interface SavedSearch { aliasTargetId?: string; errorJSON?: string; }; + viewMode?: VIEW_MODE; + hideAggregatedPreview?: boolean; } From d70cfd388cabbad169754f3cf7537f17bf9b794a Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 14 Oct 2021 13:13:13 -0500 Subject: [PATCH 143/188] Update tests --- .../saved_searches/get_saved_searches.test.ts | 2 + .../saved_searches_utils.test.ts | 4 + test/functional/page_objects/discover_page.ts | 34 + .../apps/ml/data_visualizer/index.ts | 4 +- .../data_visualizer/index_data_visualizer.ts | 333 +--------- .../index_data_visualizer_grid_in_discover.ts | 596 +----------------- .../ml/data_visualizer/index_test_data.ts | 533 ++++++++++++++++ .../functional/services/ml/custom_urls.ts | 2 +- 8 files changed, 600 insertions(+), 908 deletions(-) create mode 100644 x-pack/test/functional/apps/ml/data_visualizer/index_test_data.ts diff --git a/src/plugins/discover/public/saved_searches/get_saved_searches.test.ts b/src/plugins/discover/public/saved_searches/get_saved_searches.test.ts index 755831e7009ed..560e16b12e5ed 100644 --- a/src/plugins/discover/public/saved_searches/get_saved_searches.test.ts +++ b/src/plugins/discover/public/saved_searches/get_saved_searches.test.ts @@ -101,6 +101,7 @@ describe('getSavedSearch', () => { ], "description": "description", "grid": Object {}, + "hideAggregatedPreview": undefined, "hideChart": false, "id": "ccf1af80-2297-11ec-86e0-1155ffb9c7a7", "searchSource": Object { @@ -138,6 +139,7 @@ describe('getSavedSearch', () => { ], ], "title": "test1", + "viewMode": undefined, } `); }); diff --git a/src/plugins/discover/public/saved_searches/saved_searches_utils.test.ts b/src/plugins/discover/public/saved_searches/saved_searches_utils.test.ts index 12c73e86b3dc4..82510340f30f1 100644 --- a/src/plugins/discover/public/saved_searches/saved_searches_utils.test.ts +++ b/src/plugins/discover/public/saved_searches/saved_searches_utils.test.ts @@ -54,6 +54,7 @@ describe('saved_searches_utils', () => { ], "description": "foo", "grid": Object {}, + "hideAggregatedPreview": undefined, "hideChart": true, "id": "id", "searchSource": SearchSource { @@ -74,6 +75,7 @@ describe('saved_searches_utils', () => { "sharingSavedObjectProps": Object {}, "sort": Array [], "title": "saved search", + "viewMode": undefined, } `); }); @@ -122,6 +124,7 @@ describe('saved_searches_utils', () => { ], "description": "description", "grid": Object {}, + "hideAggregatedPreview": undefined, "hideChart": true, "kibanaSavedObjectMeta": Object { "searchSourceJSON": "{}", @@ -133,6 +136,7 @@ describe('saved_searches_utils', () => { ], ], "title": "title", + "viewMode": undefined, } `); }); diff --git a/test/functional/page_objects/discover_page.ts b/test/functional/page_objects/discover_page.ts index c9e2c54ddf7cb..a9ee816f374fa 100644 --- a/test/functional/page_objects/discover_page.ts +++ b/test/functional/page_objects/discover_page.ts @@ -7,6 +7,7 @@ */ import { FtrService } from '../ftr_provider_context'; +import expect from '../../../../../../../../private/var/tmp/_bazel_quynhnguyen/bd5cc7ce3740c1abb2c63a2609d8bb9f/execroot/kibana/bazel-out/darwin-fastbuild/bin/packages/kbn-expect'; export class DiscoverPageObject extends FtrService { private readonly retry = this.ctx.getService('retry'); @@ -541,4 +542,37 @@ export class DiscoverPageObject extends FtrService { public async clearSavedQuery() { await this.testSubjects.click('saved-query-management-clear-button'); } + + public async assertHitCount(expectedHitCount: string) { + await this.retry.tryForTime(2 * 1000, async () => { + // Close side bar to ensure Discover hit count shows + // edge case for when browser width is small + await this.closeSidebar(); + const hitCount = await this.getHitCount(); + expect(hitCount).to.eql( + expectedHitCount, + `Expected Discover hit count to be ${expectedHitCount} but got ${hitCount}.` + ); + }); + } + + public async assertViewModeToggleNotExists() { + await this.testSubjects.missingOrFail('dscViewModeToggle', { timeout: 2 * 1000 }); + } + + public async assertViewModeToggleExists() { + await this.testSubjects.existOrFail('dscViewModeToggle', { timeout: 2 * 1000 }); + } + + public async assertFieldStatsTableNotExists() { + await this.testSubjects.missingOrFail('dscFieldStatsEmbeddedContent', { timeout: 2 * 1000 }); + } + + public async clickViewModeFieldStatsButton() { + await this.retry.tryForTime(2 * 1000, async () => { + await this.testSubjects.existOrFail('dscViewModeFieldStatsButton'); + await this.testSubjects.clickWhenNotDisabled('dscViewModeFieldStatsButton'); + await this.testSubjects.existOrFail('dscFieldStatsEmbeddedContent'); + }); + } } diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index.ts b/x-pack/test/functional/apps/ml/data_visualizer/index.ts index 363bdb50e072b..c1e5d0b4b6aae 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index.ts @@ -9,12 +9,12 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('data visualizer', function () { - this.tags(['skipFirefox']); + this.tags(['skipFirefox', 'mlqa']); loadTestFile(require.resolve('./index_data_visualizer')); + loadTestFile(require.resolve('./index_data_visualizer_grid_in_discover')); loadTestFile(require.resolve('./index_data_visualizer_actions_panel')); loadTestFile(require.resolve('./index_data_visualizer_index_pattern_management')); - loadTestFile(require.resolve('./index_data_visualizer_grid_in_discover')); loadTestFile(require.resolve('./file_data_visualizer')); }); } diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts index da6bf3e28a9e4..f622318879cae 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer.ts @@ -6,337 +6,18 @@ */ import { FtrProviderContext } from '../../../ftr_provider_context'; -import { ML_JOB_FIELD_TYPES } from '../../../../../plugins/ml/common/constants/field_types'; import { TestData, MetricFieldVisConfig } from './types'; +import { + farequoteIndexPatternTestData, + farequoteKQLSearchTestData, + farequoteLuceneSearchTestData, + sampleLogTestData, +} from './index_test_data'; export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const ml = getService('ml'); - const farequoteIndexPatternTestData: TestData = { - suiteTitle: 'index pattern', - sourceIndexOrSavedSearch: 'ft_farequote', - fieldNameFilters: ['airline', '@timestamp'], - fieldTypeFilters: [ML_JOB_FIELD_TYPES.KEYWORD], - sampleSizeValidations: [ - { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, - { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, - ], - expected: { - totalDocCountFormatted: '86,274', - metricFields: [ - { - fieldName: 'responsetime', - type: ML_JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - statsMaxDecimalPlaces: 3, - topValuesCount: 10, - viewableInLens: true, - }, - ], - nonMetricFields: [ - { - fieldName: '@timestamp', - type: ML_JOB_FIELD_TYPES.DATE, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - exampleCount: 2, - viewableInLens: true, - }, - { - fieldName: '@version', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: '@version.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'airline', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 10, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'type', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: 'type.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - ], - emptyFields: ['sourcetype'], - visibleMetricFieldsCount: 1, - totalMetricFieldsCount: 1, - populatedFieldsCount: 7, - totalFieldsCount: 8, - fieldNameFiltersResultCount: 2, - fieldTypeFiltersResultCount: 3, - }, - }; - - const farequoteKQLSearchTestData: TestData = { - suiteTitle: 'KQL saved search', - sourceIndexOrSavedSearch: 'ft_farequote_kuery', - fieldNameFilters: ['@version'], - fieldTypeFilters: [ML_JOB_FIELD_TYPES.DATE, ML_JOB_FIELD_TYPES.TEXT], - sampleSizeValidations: [ - { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, - { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, - ], - expected: { - totalDocCountFormatted: '34,415', - metricFields: [ - { - fieldName: 'responsetime', - type: ML_JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - statsMaxDecimalPlaces: 3, - topValuesCount: 10, - viewableInLens: true, - }, - ], - nonMetricFields: [ - { - fieldName: '@timestamp', - type: ML_JOB_FIELD_TYPES.DATE, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - exampleCount: 2, - viewableInLens: true, - }, - { - fieldName: '@version', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: '@version.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'airline', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 5, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'type', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: 'type.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - ], - emptyFields: ['sourcetype'], - visibleMetricFieldsCount: 1, - totalMetricFieldsCount: 1, - populatedFieldsCount: 7, - totalFieldsCount: 8, - fieldNameFiltersResultCount: 1, - fieldTypeFiltersResultCount: 3, - }, - }; - - const farequoteLuceneSearchTestData: TestData = { - suiteTitle: 'lucene saved search', - sourceIndexOrSavedSearch: 'ft_farequote_lucene', - fieldNameFilters: ['@version.keyword', 'type'], - fieldTypeFilters: [ML_JOB_FIELD_TYPES.NUMBER], - sampleSizeValidations: [ - { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, - { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, - ], - expected: { - totalDocCountFormatted: '34,416', - metricFields: [ - { - fieldName: 'responsetime', - type: ML_JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - statsMaxDecimalPlaces: 3, - topValuesCount: 10, - viewableInLens: true, - }, - ], - nonMetricFields: [ - { - fieldName: '@timestamp', - type: ML_JOB_FIELD_TYPES.DATE, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - exampleCount: 2, - viewableInLens: true, - }, - { - fieldName: '@version', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: '@version.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'airline', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 5, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'type', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: 'type.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - ], - emptyFields: ['sourcetype'], - visibleMetricFieldsCount: 1, - totalMetricFieldsCount: 1, - populatedFieldsCount: 7, - totalFieldsCount: 8, - fieldNameFiltersResultCount: 2, - fieldTypeFiltersResultCount: 1, - }, - }; - - const sampleLogTestData: TestData = { - suiteTitle: 'geo point field', - sourceIndexOrSavedSearch: 'ft_module_sample_logs', - fieldNameFilters: ['geo.coordinates'], - fieldTypeFilters: [ML_JOB_FIELD_TYPES.GEO_POINT], - rowsPerPage: 50, - expected: { - totalDocCountFormatted: '408', - metricFields: [], - // only testing the geo_point fields - nonMetricFields: [ - { - fieldName: 'geo.coordinates', - type: ML_JOB_FIELD_TYPES.GEO_POINT, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '408 (100%)', - exampleCount: 10, - viewableInLens: false, - }, - ], - emptyFields: [], - visibleMetricFieldsCount: 4, - totalMetricFieldsCount: 5, - populatedFieldsCount: 35, - totalFieldsCount: 36, - fieldNameFiltersResultCount: 1, - fieldTypeFiltersResultCount: 1, - }, - sampleSizeValidations: [ - { size: 1000, expected: { field: 'geo.coordinates', docCountFormatted: '408 (100%)' } }, - { size: 5000, expected: { field: '@timestamp', docCountFormatted: '408 (100%)' } }, - ], - }; - function runTests(testData: TestData) { it(`${testData.suiteTitle} loads the source data in the data visualizer`, async () => { await ml.testExecution.logTestStep( @@ -504,7 +185,7 @@ export default function ({ getService }: FtrProviderContext) { }); describe('with module_sample_logs ', function () { - // Run tests on full farequote index. + // Run tests on full ft_module_sample_logs index. it(`${sampleLogTestData.suiteTitle} loads the data visualizer selector page`, async () => { // Start navigation from the base of the ML app. await ml.navigation.navigateToMl(); diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts index 8019b69281c4a..57cef8fd2ad8b 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts @@ -7,534 +7,17 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../ftr_provider_context'; -import { ML_JOB_FIELD_TYPES } from '../../../../../plugins/ml/common/constants/field_types'; import { TestData, MetricFieldVisConfig } from './types'; const SHOW_FIELD_STATISTICS = 'discover:showFieldStatistics'; -const farequoteIndexPatternTestData: TestData = { - suiteTitle: 'farequote index pattern', - isSavedSearch: false, - sourceIndexOrSavedSearch: 'ft_farequote', - fieldNameFilters: ['airline', '@timestamp'], - fieldTypeFilters: [ML_JOB_FIELD_TYPES.KEYWORD], - sampleSizeValidations: [ - { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, - { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, - ], - expected: { - totalDocCountFormatted: '86,274', - metricFields: [ - { - fieldName: 'responsetime', - type: ML_JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - statsMaxDecimalPlaces: 3, - topValuesCount: 10, - viewableInLens: true, - }, - ], - nonMetricFields: [ - { - fieldName: '@timestamp', - type: ML_JOB_FIELD_TYPES.DATE, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - exampleCount: 2, - viewableInLens: true, - }, - { - fieldName: '@version', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: '@version.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'airline', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 10, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'type', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: 'type.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - ], - emptyFields: ['sourcetype'], - visibleMetricFieldsCount: 1, - totalMetricFieldsCount: 1, - populatedFieldsCount: 7, - totalFieldsCount: 8, - fieldNameFiltersResultCount: 2, - fieldTypeFiltersResultCount: 3, - }, -}; - -const farequoteKQLSearchTestData: TestData = { - suiteTitle: 'KQL saved search', - isSavedSearch: true, - sourceIndexOrSavedSearch: 'ft_farequote_kuery', - fieldNameFilters: ['@version'], - fieldTypeFilters: [ML_JOB_FIELD_TYPES.DATE, ML_JOB_FIELD_TYPES.TEXT], - sampleSizeValidations: [ - { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, - { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, - ], - expected: { - totalDocCountFormatted: '34,415', - metricFields: [ - { - fieldName: 'responsetime', - type: ML_JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - statsMaxDecimalPlaces: 3, - topValuesCount: 10, - viewableInLens: true, - }, - ], - nonMetricFields: [ - { - fieldName: '@timestamp', - type: ML_JOB_FIELD_TYPES.DATE, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - exampleCount: 2, - viewableInLens: true, - }, - { - fieldName: '@version', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: '@version.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'airline', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 5, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'type', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: 'type.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - ], - emptyFields: ['sourcetype'], - visibleMetricFieldsCount: 1, - totalMetricFieldsCount: 1, - populatedFieldsCount: 7, - totalFieldsCount: 8, - fieldNameFiltersResultCount: 1, - fieldTypeFiltersResultCount: 3, - }, -}; - -const farequoteKQLFiltersSearchTestData: TestData = { - suiteTitle: 'KQL saved search and filters', - isSavedSearch: true, - sourceIndexOrSavedSearch: 'ft_farequote_filter_and_kuery', - fieldNameFilters: ['@version'], - fieldTypeFilters: [ML_JOB_FIELD_TYPES.DATE, ML_JOB_FIELD_TYPES.TEXT], - sampleSizeValidations: [ - { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, - { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, - ], - expected: { - totalDocCountFormatted: '5,674', - metricFields: [ - { - fieldName: 'responsetime', - type: ML_JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - statsMaxDecimalPlaces: 3, - topValuesCount: 10, - viewableInLens: true, - }, - ], - nonMetricFields: [ - { - fieldName: '@timestamp', - type: ML_JOB_FIELD_TYPES.DATE, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - exampleCount: 2, - viewableInLens: true, - }, - { - fieldName: '@version', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: '@version.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'airline', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - exampleContent: ['ASA'], - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'type', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: 'type.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - ], - emptyFields: ['sourcetype'], - visibleMetricFieldsCount: 1, - totalMetricFieldsCount: 1, - populatedFieldsCount: 7, - totalFieldsCount: 8, - fieldNameFiltersResultCount: 1, - fieldTypeFiltersResultCount: 3, - }, -}; - -const farequoteLuceneSearchTestData: TestData = { - suiteTitle: 'lucene saved search', - isSavedSearch: true, - sourceIndexOrSavedSearch: 'ft_farequote_lucene', - fieldNameFilters: ['@version.keyword', 'type'], - fieldTypeFilters: [ML_JOB_FIELD_TYPES.NUMBER], - sampleSizeValidations: [ - { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, - { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, - ], - expected: { - totalDocCountFormatted: '34,416', - metricFields: [ - { - fieldName: 'responsetime', - type: ML_JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - statsMaxDecimalPlaces: 3, - topValuesCount: 10, - viewableInLens: true, - }, - ], - nonMetricFields: [ - { - fieldName: '@timestamp', - type: ML_JOB_FIELD_TYPES.DATE, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - exampleCount: 2, - viewableInLens: true, - }, - { - fieldName: '@version', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: '@version.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'airline', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 5, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'type', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: 'type.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - ], - emptyFields: ['sourcetype'], - visibleMetricFieldsCount: 1, - totalMetricFieldsCount: 1, - populatedFieldsCount: 7, - totalFieldsCount: 8, - fieldNameFiltersResultCount: 2, - fieldTypeFiltersResultCount: 1, - }, -}; - -const farequoteLuceneFiltersSearchTestData: TestData = { - suiteTitle: 'lucene saved search and filter', - isSavedSearch: true, - sourceIndexOrSavedSearch: 'ft_farequote_filter_and_lucene', - fieldNameFilters: ['@version.keyword', 'type'], - fieldTypeFilters: [ML_JOB_FIELD_TYPES.NUMBER], - sampleSizeValidations: [ - { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, - { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, - ], - expected: { - totalDocCountFormatted: '5,673', - metricFields: [ - { - fieldName: 'responsetime', - type: ML_JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - statsMaxDecimalPlaces: 3, - topValuesCount: 10, - viewableInLens: true, - }, - ], - nonMetricFields: [ - { - fieldName: '@timestamp', - type: ML_JOB_FIELD_TYPES.DATE, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '5000 (100%)', - exampleCount: 2, - viewableInLens: true, - }, - { - fieldName: '@version', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: '@version.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'airline', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - exampleContent: ['ASA'], - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - { - fieldName: 'type', - type: ML_JOB_FIELD_TYPES.TEXT, - existsInDocs: true, - aggregatable: false, - loading: false, - exampleCount: 1, - docCountFormatted: '', - viewableInLens: false, - }, - { - fieldName: 'type.keyword', - type: ML_JOB_FIELD_TYPES.KEYWORD, - existsInDocs: true, - aggregatable: true, - loading: false, - exampleCount: 1, - docCountFormatted: '5000 (100%)', - viewableInLens: true, - }, - ], - emptyFields: ['sourcetype'], - visibleMetricFieldsCount: 1, - totalMetricFieldsCount: 1, - populatedFieldsCount: 7, - totalFieldsCount: 8, - fieldNameFiltersResultCount: 2, - fieldTypeFiltersResultCount: 1, - }, -}; - -const sampleLogTestData: TestData = { - suiteTitle: 'geo point field', - isSavedSearch: false, - sourceIndexOrSavedSearch: 'ft_module_sample_logs', - fieldNameFilters: ['geo.coordinates'], - fieldTypeFilters: [ML_JOB_FIELD_TYPES.GEO_POINT], - rowsPerPage: 50, - expected: { - totalDocCountFormatted: '408', - metricFields: [], - // only testing the geo_point fields - nonMetricFields: [ - { - fieldName: 'geo.coordinates', - type: ML_JOB_FIELD_TYPES.GEO_POINT, - existsInDocs: true, - aggregatable: true, - loading: false, - docCountFormatted: '408 (100%)', - exampleCount: 10, - viewableInLens: false, - }, - ], - emptyFields: [], - visibleMetricFieldsCount: 4, - totalMetricFieldsCount: 5, - populatedFieldsCount: 35, - totalFieldsCount: 36, - fieldNameFiltersResultCount: 1, - fieldTypeFiltersResultCount: 1, - }, - sampleSizeValidations: [ - { size: 1000, expected: { field: 'geo.coordinates', docCountFormatted: '408 (100%)' } }, - { size: 5000, expected: { field: '@timestamp', docCountFormatted: '408 (100%)' } }, - ], -}; - +import { + farequoteIndexPatternTestData, + farequoteKQLSearchTestData, + farequoteLuceneFiltersSearchTestData, + farequoteKQLFiltersSearchTestData, + farequoteLuceneSearchTestData, + sampleLogTestData, +} from './index_test_data'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const PageObjects = getPageObjects(['common', 'discover', 'timePicker', 'settings']); @@ -543,44 +26,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const retry = getService('retry'); const toasts = getService('toasts'); - /** Discover page helpers **/ - const assertHitCount = async (expectedHitCount: string) => { - await retry.tryForTime(2 * 1000, async () => { - // Close side bar to ensure Discover hit count shows - // edge case for when browser width is small - await PageObjects.discover.closeSidebar(); - const hitCount = await PageObjects.discover.getHitCount(); - expect(hitCount).to.eql( - expectedHitCount, - `Expected Discover hit count to be ${expectedHitCount} but got ${hitCount}.` - ); - }); - }; - - const assertViewModeToggleNotExists = async () => { - await retry.tryForTime(2 * 1000, async () => { - await testSubjects.missingOrFail('dscViewModeToggle'); - }); - }; - - const assertViewModeToggleExists = async () => { - await retry.tryForTime(2 * 1000, async () => { - await testSubjects.existOrFail('dscViewModeToggle'); - }); - }; - - const assertFieldStatsTableNotExists = async () => { - await testSubjects.missingOrFail('dscFieldStatsEmbeddedContent'); - }; - - const clickViewModeFieldStatsButton = async () => { - await retry.tryForTime(2 * 1000, async () => { - await testSubjects.existOrFail('dscViewModeFieldStatsButton'); - await testSubjects.clickWhenNotDisabled('dscViewModeFieldStatsButton'); - await testSubjects.existOrFail('dscFieldStatsEmbeddedContent'); - }); - }; - const selectIndexPattern = async (indexPattern: string) => { await retry.tryForTime(2 * 1000, async () => { await PageObjects.discover.selectIndexPattern(indexPattern); @@ -627,8 +72,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'Nov 1, 2020 @ 00:00:00.000' ); - await assertViewModeToggleNotExists(); - await assertFieldStatsTableNotExists(); + await PageObjects.discover.assertViewModeToggleNotExists(); + await PageObjects.discover.assertFieldStatsTableNotExists(); }); } @@ -648,9 +93,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'Nov 1, 2020 @ 00:00:00.000' ); - await assertHitCount(testData.expected.totalDocCountFormatted); - await assertViewModeToggleExists(); - await clickViewModeFieldStatsButton(); + await PageObjects.discover.assertHitCount(testData.expected.totalDocCountFormatted); + await PageObjects.discover.assertViewModeToggleExists(); + await PageObjects.discover.clickViewModeFieldStatsButton(); await ml.testExecution.logTestStep( 'displays details for metric fields and non-metric fields correctly' ); @@ -695,12 +140,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); after(async function () { - await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote'); - await esArchiver.unload('x-pack/test/functional/es_archives/ml/module_sample_logs'); - - await ml.testResources.deleteIndexPatternByTitle('ft_farequote'); - await ml.testResources.deleteIndexPatternByTitle('ft_module_sample_logs'); - await ml.testResources.deleteSavedSearches(); await clearAdvancedSetting(SHOW_FIELD_STATISTICS); }); @@ -715,10 +154,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { runTests(farequoteIndexPatternTestData); runTests(farequoteKQLSearchTestData); - runTests(farequoteLuceneSearchTestData); - runTests(farequoteKQLFiltersSearchTestData); - runTests(farequoteLuceneFiltersSearchTestData); - runTests(sampleLogTestData); + // runTests(farequoteLuceneSearchTestData); + // runTests(farequoteKQLFiltersSearchTestData); + // runTests(farequoteLuceneFiltersSearchTestData); + // runTests(sampleLogTestData); }); describe('when disabled', function () { @@ -728,7 +167,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); runTestsWhenDisabled(farequoteIndexPatternTestData); - runTestsWhenDisabled(farequoteKQLSearchTestData); }); }); } diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_test_data.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_test_data.ts new file mode 100644 index 0000000000000..66ffb50e1a58b --- /dev/null +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_test_data.ts @@ -0,0 +1,533 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TestData } from './types'; +import { ML_JOB_FIELD_TYPES } from '../../../../../plugins/ml/common/constants/field_types'; + +export const farequoteIndexPatternTestData: TestData = { + suiteTitle: 'farequote index pattern', + isSavedSearch: false, + sourceIndexOrSavedSearch: 'ft_farequote', + fieldNameFilters: ['airline', '@timestamp'], + fieldTypeFilters: [ML_JOB_FIELD_TYPES.KEYWORD], + sampleSizeValidations: [ + { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, + { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, + ], + expected: { + totalDocCountFormatted: '86,274', + metricFields: [ + { + fieldName: 'responsetime', + type: ML_JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + statsMaxDecimalPlaces: 3, + topValuesCount: 10, + viewableInLens: true, + }, + ], + nonMetricFields: [ + { + fieldName: '@timestamp', + type: ML_JOB_FIELD_TYPES.DATE, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + exampleCount: 2, + viewableInLens: true, + }, + { + fieldName: '@version', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: '@version.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'airline', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 10, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'type', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: 'type.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + ], + emptyFields: ['sourcetype'], + visibleMetricFieldsCount: 1, + totalMetricFieldsCount: 1, + populatedFieldsCount: 7, + totalFieldsCount: 8, + fieldNameFiltersResultCount: 2, + fieldTypeFiltersResultCount: 3, + }, +}; + +export const farequoteKQLSearchTestData: TestData = { + suiteTitle: 'KQL saved search', + isSavedSearch: true, + sourceIndexOrSavedSearch: 'ft_farequote_kuery', + fieldNameFilters: ['@version'], + fieldTypeFilters: [ML_JOB_FIELD_TYPES.DATE, ML_JOB_FIELD_TYPES.TEXT], + sampleSizeValidations: [ + { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, + { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, + ], + expected: { + totalDocCountFormatted: '34,415', + metricFields: [ + { + fieldName: 'responsetime', + type: ML_JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + statsMaxDecimalPlaces: 3, + topValuesCount: 10, + viewableInLens: true, + }, + ], + nonMetricFields: [ + { + fieldName: '@timestamp', + type: ML_JOB_FIELD_TYPES.DATE, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + exampleCount: 2, + viewableInLens: true, + }, + { + fieldName: '@version', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: '@version.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'airline', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 5, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'type', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: 'type.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + ], + emptyFields: ['sourcetype'], + visibleMetricFieldsCount: 1, + totalMetricFieldsCount: 1, + populatedFieldsCount: 7, + totalFieldsCount: 8, + fieldNameFiltersResultCount: 1, + fieldTypeFiltersResultCount: 3, + }, +}; + +export const farequoteKQLFiltersSearchTestData: TestData = { + suiteTitle: 'KQL saved search and filters', + isSavedSearch: true, + sourceIndexOrSavedSearch: 'ft_farequote_filter_and_kuery', + fieldNameFilters: ['@version'], + fieldTypeFilters: [ML_JOB_FIELD_TYPES.DATE, ML_JOB_FIELD_TYPES.TEXT], + sampleSizeValidations: [ + { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, + { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, + ], + expected: { + totalDocCountFormatted: '5,674', + metricFields: [ + { + fieldName: 'responsetime', + type: ML_JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + statsMaxDecimalPlaces: 3, + topValuesCount: 10, + viewableInLens: true, + }, + ], + nonMetricFields: [ + { + fieldName: '@timestamp', + type: ML_JOB_FIELD_TYPES.DATE, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + exampleCount: 2, + viewableInLens: true, + }, + { + fieldName: '@version', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: '@version.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'airline', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + exampleContent: ['ASA'], + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'type', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: 'type.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + ], + emptyFields: ['sourcetype'], + visibleMetricFieldsCount: 1, + totalMetricFieldsCount: 1, + populatedFieldsCount: 7, + totalFieldsCount: 8, + fieldNameFiltersResultCount: 1, + fieldTypeFiltersResultCount: 3, + }, +}; + +export const farequoteLuceneSearchTestData: TestData = { + suiteTitle: 'lucene saved search', + isSavedSearch: true, + sourceIndexOrSavedSearch: 'ft_farequote_lucene', + fieldNameFilters: ['@version.keyword', 'type'], + fieldTypeFilters: [ML_JOB_FIELD_TYPES.NUMBER], + sampleSizeValidations: [ + { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, + { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, + ], + expected: { + totalDocCountFormatted: '34,416', + metricFields: [ + { + fieldName: 'responsetime', + type: ML_JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + statsMaxDecimalPlaces: 3, + topValuesCount: 10, + viewableInLens: true, + }, + ], + nonMetricFields: [ + { + fieldName: '@timestamp', + type: ML_JOB_FIELD_TYPES.DATE, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + exampleCount: 2, + viewableInLens: true, + }, + { + fieldName: '@version', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: '@version.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'airline', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 5, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'type', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: 'type.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + ], + emptyFields: ['sourcetype'], + visibleMetricFieldsCount: 1, + totalMetricFieldsCount: 1, + populatedFieldsCount: 7, + totalFieldsCount: 8, + fieldNameFiltersResultCount: 2, + fieldTypeFiltersResultCount: 1, + }, +}; + +export const farequoteLuceneFiltersSearchTestData: TestData = { + suiteTitle: 'lucene saved search and filter', + isSavedSearch: true, + sourceIndexOrSavedSearch: 'ft_farequote_filter_and_lucene', + fieldNameFilters: ['@version.keyword', 'type'], + fieldTypeFilters: [ML_JOB_FIELD_TYPES.NUMBER], + sampleSizeValidations: [ + { size: 1000, expected: { field: 'airline', docCountFormatted: '1000 (100%)' } }, + { size: 5000, expected: { field: '@timestamp', docCountFormatted: '5000 (100%)' } }, + ], + expected: { + totalDocCountFormatted: '5,673', + metricFields: [ + { + fieldName: 'responsetime', + type: ML_JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + statsMaxDecimalPlaces: 3, + topValuesCount: 10, + viewableInLens: true, + }, + ], + nonMetricFields: [ + { + fieldName: '@timestamp', + type: ML_JOB_FIELD_TYPES.DATE, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '5000 (100%)', + exampleCount: 2, + viewableInLens: true, + }, + { + fieldName: '@version', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: '@version.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'airline', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + exampleContent: ['ASA'], + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + { + fieldName: 'type', + type: ML_JOB_FIELD_TYPES.TEXT, + existsInDocs: true, + aggregatable: false, + loading: false, + exampleCount: 1, + docCountFormatted: '', + viewableInLens: false, + }, + { + fieldName: 'type.keyword', + type: ML_JOB_FIELD_TYPES.KEYWORD, + existsInDocs: true, + aggregatable: true, + loading: false, + exampleCount: 1, + docCountFormatted: '5000 (100%)', + viewableInLens: true, + }, + ], + emptyFields: ['sourcetype'], + visibleMetricFieldsCount: 1, + totalMetricFieldsCount: 1, + populatedFieldsCount: 7, + totalFieldsCount: 8, + fieldNameFiltersResultCount: 2, + fieldTypeFiltersResultCount: 1, + }, +}; + +export const sampleLogTestData: TestData = { + suiteTitle: 'geo point field', + isSavedSearch: false, + sourceIndexOrSavedSearch: 'ft_module_sample_logs', + fieldNameFilters: ['geo.coordinates'], + fieldTypeFilters: [ML_JOB_FIELD_TYPES.GEO_POINT], + rowsPerPage: 50, + expected: { + totalDocCountFormatted: '408', + metricFields: [], + // only testing the geo_point fields + nonMetricFields: [ + { + fieldName: 'geo.coordinates', + type: ML_JOB_FIELD_TYPES.GEO_POINT, + existsInDocs: true, + aggregatable: true, + loading: false, + docCountFormatted: '408 (100%)', + exampleCount: 10, + viewableInLens: false, + }, + ], + emptyFields: [], + visibleMetricFieldsCount: 4, + totalMetricFieldsCount: 5, + populatedFieldsCount: 35, + totalFieldsCount: 36, + fieldNameFiltersResultCount: 1, + fieldTypeFiltersResultCount: 1, + }, + sampleSizeValidations: [ + { size: 1000, expected: { field: 'geo.coordinates', docCountFormatted: '408 (100%)' } }, + { size: 5000, expected: { field: '@timestamp', docCountFormatted: '408 (100%)' } }, + ], +}; diff --git a/x-pack/test/functional/services/ml/custom_urls.ts b/x-pack/test/functional/services/ml/custom_urls.ts index 7a562693d883f..3d26236741a8a 100644 --- a/x-pack/test/functional/services/ml/custom_urls.ts +++ b/x-pack/test/functional/services/ml/custom_urls.ts @@ -171,7 +171,7 @@ export function MachineLearningCustomUrlsProvider({ await PageObjects.discover.waitForDiscoverAppOnScreen(); // During cloud tests, the small browser width might cause hit count to be invisible // so temporarily collapsing the sidebar ensures the count shows - await PageObjects.discover.toggleSidebarCollapse(); + await PageObjects.discover.closeSidebar(); await retry.tryForTime(10 * 1000, async () => { const hitCount = await PageObjects.discover.getHitCount(); expect(hitCount).to.eql( From 75486b71b809e7c9dc3f8940372df7516070a85a Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 14 Oct 2021 13:33:33 -0500 Subject: [PATCH 144/188] Fix import --- test/functional/page_objects/discover_page.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/page_objects/discover_page.ts b/test/functional/page_objects/discover_page.ts index 274a95fa93b5e..a45c1a23ed3a5 100644 --- a/test/functional/page_objects/discover_page.ts +++ b/test/functional/page_objects/discover_page.ts @@ -6,8 +6,8 @@ * Side Public License, v 1. */ +import expect from '@kbn/expect'; import { FtrService } from '../ftr_provider_context'; -import expect from '../../../../../../../../private/var/tmp/_bazel_quynhnguyen/bd5cc7ce3740c1abb2c63a2609d8bb9f/execroot/kibana/bazel-out/darwin-fastbuild/bin/packages/kbn-expect'; export class DiscoverPageObject extends FtrService { private readonly retry = this.ctx.getService('retry'); From d704232b138e97df013fb44c9d320054bb2860e6 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 14 Oct 2021 18:34:23 -0500 Subject: [PATCH 145/188] Match the no results found --- .../grid_embeddable/grid_embeddable.tsx | 36 ++++++++++++++++--- .../use_data_visualizer_grid_data.ts | 1 + .../index_data_visualizer_grid_in_discover.ts | 8 ++--- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index e083a6781fad7..c5d4e4c30c2ac 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -10,7 +10,7 @@ import { CoreStart } from 'kibana/public'; import ReactDOM from 'react-dom'; import React, { Suspense, useCallback, useState } from 'react'; import useObservable from 'react-use/lib/useObservable'; -import { EuiEmptyPrompt } from '@elastic/eui'; +import { EuiEmptyPrompt, EuiIcon, EuiSpacer, EuiText } from '@elastic/eui'; import { Filter } from '@kbn/es-query'; import { Required } from 'utility-types'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -79,10 +79,8 @@ export const EmbeddableWrapper = ({ }, [dataVisualizerListState, onOutputChange] ); - const { configs, searchQueryLanguage, searchString, extendedColumns } = useDataVisualizerGridData( - input, - dataVisualizerListState - ); + const { configs, searchQueryLanguage, searchString, extendedColumns, loaded } = + useDataVisualizerGridData(input, dataVisualizerListState); const getItemIdToExpandedRowMap = useCallback( function (itemIds: string[], items: FieldVisConfig[]): ItemIdToExpandedRowMap { return itemIds.reduce((m: ItemIdToExpandedRowMap, fieldName: string) => { @@ -103,6 +101,34 @@ export const EmbeddableWrapper = ({ [input, searchQueryLanguage, searchString] ); + if ( + loaded && + (configs.length === 0 || + // FIXME: Configs might have a placeholder document count stats field + // This will be removed in the future + (configs.length === 1 && configs[0].fieldName === undefined)) + ) { + return ( +
+ + + + + +
+ ); + } return ( items={configs} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts index 0c57a1bf25b6a..fc0fc7a2134b4 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/use_data_visualizer_grid_data.ts @@ -582,5 +582,6 @@ export const useDataVisualizerGridData = ( extendedColumns, documentCountStats, metricsStats, + loaded: metricsLoaded && nonMetricsLoaded, }; }; diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts index 57cef8fd2ad8b..77f92136724f5 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_grid_in_discover.ts @@ -154,10 +154,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { runTests(farequoteIndexPatternTestData); runTests(farequoteKQLSearchTestData); - // runTests(farequoteLuceneSearchTestData); - // runTests(farequoteKQLFiltersSearchTestData); - // runTests(farequoteLuceneFiltersSearchTestData); - // runTests(sampleLogTestData); + runTests(farequoteLuceneSearchTestData); + runTests(farequoteKQLFiltersSearchTestData); + runTests(farequoteLuceneFiltersSearchTestData); + runTests(sampleLogTestData); }); describe('when disabled', function () { From 3b99fba7d51c83a0f0ec8c9aa360cd3f895d9b40 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 15 Oct 2021 12:54:38 -0500 Subject: [PATCH 146/188] Reset field stats so it reloads when query is refreshed --- .../application/index_data_visualizer/hooks/use_field_stats.ts | 1 + .../index_data_visualizer/hooks/use_overall_stats.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index 8cc20a4033075..4f814625684b3 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -71,6 +71,7 @@ export function useFieldStatsSearchStrategy( ...getInitialProgress(), error: undefined, }); + setFieldStats(undefined); if (!searchStrategyParams || !fieldStatsParams) return; if ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 833d8e5683462..46eac36928b45 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -10,7 +10,6 @@ import { combineLatest, forkJoin, of, Subscription } from 'rxjs'; import { mergeMap, switchMap } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; import { ToastsStart } from 'kibana/public'; -import { OverallStatsSearchStrategyParams } from '../../../../common/search_strategy/types'; import { useDataVisualizerKibana } from '../../kibana_context'; import { checkAggregatableFieldsExistRequest, @@ -25,6 +24,7 @@ import { import { OverallStats } from '../types/overall_stats'; import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; import { extractErrorProperties } from '../utils/error_utils'; +import { OverallStatsSearchStrategyParams } from '../../../../common/types/field_stats'; function displayError(toastNotifications: ToastsStart, indexPattern: string, err: any) { if (err.statusCode === 500) { From 56d6f3ead451e0e612b885e8c37f5dd22c77f59b Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 15 Oct 2021 13:10:40 -0500 Subject: [PATCH 147/188] Reset field stats so it reloads when query is refreshed --- .../hooks/use_data_visualizer_grid_data.ts | 32 +++++++++++++++---- .../hooks/use_field_stats.ts | 1 + .../hooks/use_overall_stats.ts | 1 + 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 6280226f49221..1db53f4f4d03e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -97,13 +97,18 @@ export const useDataVisualizerGridData = ( } // eslint-disable-next-line react-hooks/exhaustive-deps }, [ - currentSavedSearch, - currentIndexPattern, - dataVisualizerListState.searchQuery, + currentSavedSearch?.id, + currentIndexPattern.id, + // dataVisualizerListState.searchQuery, dataVisualizerListState.searchString, dataVisualizerListState.searchQueryLanguage, - currentQuery, - currentFilters, + JSON.stringify({ + searchQuery: dataVisualizerListState.searchQuery, + currentQuery, + currentFilters, + }), + // currentQuery, + // currentFilters, ]); useEffect(() => { @@ -190,13 +195,26 @@ export const useDataVisualizerGridData = ( }, [ _timeBuckets, timefilter, - currentIndexPattern, - searchQuery, + currentIndexPattern.id, + JSON.stringify(searchQuery), samplerShardSize, searchSessionId, lastRefresh, ]); + useEffect(() => console.log('_timeBuckets', _timeBuckets), [_timeBuckets]); + useEffect(() => console.log('timefilter', timefilter), [timefilter]); + useEffect( + () => console.log('currentIndexPattern', currentIndexPattern.id), + [currentIndexPattern.id] + ); + useEffect( + () => console.log('searchQuery', JSON.stringify(searchQuery)), + [JSON.stringify(searchQuery)] + ); + useEffect(() => console.log('samplerShardSize', samplerShardSize), [samplerShardSize]); + useEffect(() => console.log('searchSessionId', searchSessionId), [searchSessionId]); + const configsWithoutStats = useMemo(() => { const existMetricFields: FieldRequestConfig[] = metricConfigs.map((config) => { const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index 4f814625684b3..f276a6a9f224f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -156,6 +156,7 @@ export function useFieldStatsSearchStrategy( }), }); }, + complete: () => console.log('useOverallStats'), }); }, [data, toasts, searchStrategyParams, fieldStatsParams, initialDataVisualizerListState]); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 46eac36928b45..707a9d16fcfe8 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -172,6 +172,7 @@ export function useOverallStats { displayError(toasts, searchStrategyParams.index, extractErrorProperties(error)); }, + complete: () => console.log('useOverallStats'), }); }, [data.search, searchStrategyParams, toasts]); From 8ee47775f4baee1f5586efe09fa0e1059a19285a Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 15 Oct 2021 14:17:31 -0500 Subject: [PATCH 148/188] Add doc stats --- .../index_data_visualizer_view.tsx | 4 +- .../hooks/use_data_visualizer_grid_data.ts | 3 +- .../hooks/use_overall_stats.ts | 56 ++++++++++++++----- .../requests/get_document_stats.ts | 43 ++++++++------ .../types/overall_stats.ts | 3 + 5 files changed, 76 insertions(+), 33 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 3643fede715d4..5949668a82dfc 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -321,6 +321,8 @@ export const IndexDataVisualizerView: FC = (dataVi const [overallStats, setOverallStats] = useState(defaults.overallStats); const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); + + console.log('documentCountStats', documentCountStats); const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); const [metricsStats, setMetricsStats] = useState(); @@ -623,7 +625,7 @@ export const IndexDataVisualizerView: FC = (dataVi }); // @todo: REMOVE - // // Add a config for 'document count', identified by no field name if indexpattern is time based. + // Add a config for 'document count', identified by no field name if indexpattern is time based. // if (currentIndexPattern.timeFieldName !== undefined) { // configs.push({ // type: JOB_FIELD_TYPES.NUMBER, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 1db53f4f4d03e..d395b2b56d25f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -496,6 +496,7 @@ export const useDataVisualizerGridData = ( return [actionColumn]; }, [input.indexPattern, services, searchQueryLanguage, searchString]); + console.log('overallStats.documentCountStats', overallStats.documentCountStats); return { progress: strategyResponse.progress, configs, @@ -503,7 +504,7 @@ export const useDataVisualizerGridData = ( searchString, searchQuery, extendedColumns, - documentCountStats, + documentCountStats: overallStats.documentCountStats, metricsStats, }; }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 707a9d16fcfe8..9a6d0abe558a1 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -25,6 +25,10 @@ import { OverallStats } from '../types/overall_stats'; import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; import { extractErrorProperties } from '../utils/error_utils'; import { OverallStatsSearchStrategyParams } from '../../../../common/types/field_stats'; +import { + getDocumentCountStatsRequest, + processDocumentCountStats, +} from '../search_strategy/requests/get_document_stats'; function displayError(toastNotifications: ToastsStart, indexPattern: string, err: any) { if (err.statusCode === 500) { @@ -81,6 +85,7 @@ export function useOverallStats 0 + ? data.search.search( + { + params: getDocumentCountStatsRequest(searchStrategyParams), + }, + { + abortSignal: abortCtrl.current.signal, + sessionId: searchStrategyParams?.sessionId, + } + ) + : of(undefined); const sub = forkJoin({ + documentCountStatsResp: documentCountStats$, nonAggregatableOverallStatsResp: nonAggregatableOverallStats$, aggregatableOverallStatsResp: aggregatableOverallStats$, }).pipe( - mergeMap(({ nonAggregatableOverallStatsResp, aggregatableOverallStatsResp }) => { - const aggregatableOverallStats = processAggregatableFieldsExistResponse( - aggregatableOverallStatsResp?.rawResponse, - aggregatableFields, - samplerShardSize - ); - const nonAggregatableOverallStats = processNonAggregatableFieldsExistResponse( + mergeMap( + ({ + documentCountStatsResp, nonAggregatableOverallStatsResp, - nonAggregatableFields - ); - return of({ - ...nonAggregatableOverallStats, - ...aggregatableOverallStats, - }); - }) + aggregatableOverallStatsResp, + }) => { + const aggregatableOverallStats = processAggregatableFieldsExistResponse( + aggregatableOverallStatsResp?.rawResponse, + aggregatableFields, + samplerShardSize + ); + const nonAggregatableOverallStats = processNonAggregatableFieldsExistResponse( + nonAggregatableOverallStatsResp, + nonAggregatableFields + ); + return of({ + documentCountStats: processDocumentCountStats( + documentCountStatsResp?.rawResponse, + intervalMs + ).documentCounts, + ...nonAggregatableOverallStats, + ...aggregatableOverallStats, + }); + } + ) ); searchSubscription$.current = sub.subscribe({ diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts index d7a54823f8c6e..06ef80140ed8f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts @@ -5,21 +5,28 @@ * 2.0. */ -import { ElasticsearchClient } from 'kibana/server'; -import { SearchRequest } from '@elastic/elasticsearch/api/types'; import { each, get } from 'lodash'; +import { estypes } from '@elastic/elasticsearch'; import { buildBaseFilterCriteria } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; import type { DocumentCountStats, - FieldStatsCommonRequestParams, + OverallStatsSearchStrategyParams, } from '../../../../../common/types/field_stats'; -export const getDocumentCountStatsRequest = (params: FieldStatsCommonRequestParams) => { - const { index, timeFieldName, earliestMs, latestMs, query, runtimeFieldMap, intervalMs } = params; +export const getDocumentCountStatsRequest = (params: OverallStatsSearchStrategyParams) => { + const { + index, + timeFieldName, + earliest: earliestMs, + latest: latestMs, + runtimeFieldMap, + searchQuery, + intervalMs, + } = params; const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, searchQuery); // Don't use the sampler aggregation as this can lead to some potentially // confusing date histogram results depending on the date range of data amongst shards. @@ -50,15 +57,18 @@ export const getDocumentCountStatsRequest = (params: FieldStatsCommonRequestPara }; }; -export const fetchDocumentCountStats = async ( - esClient: ElasticsearchClient, - params: FieldStatsCommonRequestParams -): Promise => { - const { intervalMs } = params; - const request: SearchRequest = getDocumentCountStatsRequest(params); - - const { body } = await esClient.search(request); - +export const processDocumentCountStats = ( + body: estypes.SearchResponse | undefined, + intervalMs: number | undefined +): DocumentCountStats => { + if (!body || intervalMs === undefined) { + return { + documentCounts: { + interval: 0, + buckets: {}, + }, + }; + } const buckets: { [key: string]: number } = {}; const dataByTimeBucket: Array<{ key: string; doc_count: number }> = get( body, @@ -72,8 +82,7 @@ export const fetchDocumentCountStats = async ( return { documentCounts: { - // @todo: confirm - interval: intervalMs ?? 0, + interval: intervalMs, buckets, }, }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts index 286703afec5f9..76574fcbb0a23 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { DocumentCountStats } from '../../../../common/types/field_stats'; + export interface AggregatableField { fieldName: string; stats: { @@ -19,6 +21,7 @@ export type NonAggregatableField = Omit; export interface OverallStats { totalCount: number; + documentCountStats?: DocumentCountStats['documentCounts']; aggregatableExistsFields: AggregatableField[]; aggregatableNotExistsFields: AggregatableField[]; nonAggregatableExistsFields: NonAggregatableField[]; From 083211b23b6900652cee2b39682cc7522dae9b67 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 15 Oct 2021 15:04:02 -0500 Subject: [PATCH 149/188] Merge to use hook completely --- .../common/types/field_stats.ts | 8 +- .../document_count_content.tsx | 24 +- .../index_data_visualizer_view.tsx | 555 +--------- .../index_data_visualizer_view_temp.tsx | 984 ++++++++++++++++++ .../grid_embeddable/grid_embeddable.tsx | 7 +- .../hooks/use_data_visualizer_grid_data.ts | 19 +- .../hooks/use_overall_stats.ts | 4 +- .../requests/get_document_stats.ts | 26 +- .../types/overall_stats.ts | 2 +- .../utils/saved_search_utils.ts | 24 +- 10 files changed, 1090 insertions(+), 563 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx diff --git a/x-pack/plugins/data_visualizer/common/types/field_stats.ts b/x-pack/plugins/data_visualizer/common/types/field_stats.ts index a6ac7dcf91ef8..22642074f703d 100644 --- a/x-pack/plugins/data_visualizer/common/types/field_stats.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_stats.ts @@ -94,10 +94,10 @@ export interface BooleanFieldStats { } export interface DocumentCountStats { - documentCounts: { - interval: number; - buckets: { [key: string]: number }; - }; + interval: number; + buckets: { [key: string]: number }; + timeRangeEarliest: number; + timeRangeLatest: number; } export interface FieldExamples { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx index d49dbdc7cb446..1f010dcba1b1b 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx @@ -7,30 +7,26 @@ import React, { FC } from 'react'; import { DocumentCountChart, DocumentCountChartPoint } from './document_count_chart'; -import { FieldVisConfig, FileBasedFieldVisConfig } from '../stats_table/types'; import { TotalCountHeader } from './total_count_header'; +import { DocumentCountStats } from '../../../../../common/types/field_stats'; export interface Props { - config?: FieldVisConfig | FileBasedFieldVisConfig; + documentCountStats?: DocumentCountStats; totalCount: number; } -export const DocumentCountContent: FC = ({ config, totalCount }) => { - if (config?.stats === undefined) { +// @todo: update document count stats for file based +export const DocumentCountContent: FC = ({ documentCountStats, totalCount }) => { + if (documentCountStats === undefined) { return totalCount !== undefined ? : null; } - const { documentCounts, timeRangeEarliest, timeRangeLatest } = config.stats; - if ( - documentCounts === undefined || - timeRangeEarliest === undefined || - timeRangeLatest === undefined - ) - return null; + const { timeRangeEarliest, timeRangeLatest } = documentCountStats; + if (timeRangeEarliest === undefined || timeRangeLatest === undefined) return null; let chartPoints: DocumentCountChartPoint[] = []; - if (documentCounts.buckets !== undefined) { - const buckets: Record = documentCounts?.buckets; + if (documentCountStats.buckets !== undefined) { + const buckets: Record = documentCountStats?.buckets; chartPoints = Object.entries(buckets).map(([time, value]) => ({ time: +time, value })); } @@ -41,7 +37,7 @@ export const DocumentCountContent: FC = ({ config, totalCount }) => { chartPoints={chartPoints} timeRangeEarliest={timeRangeEarliest} timeRangeLatest={timeRangeLatest} - interval={documentCounts.interval} + interval={documentCountStats.interval} /> ); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 5949668a82dfc..241ce196bff20 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -6,7 +6,6 @@ */ import React, { FC, Fragment, useEffect, useMemo, useState, useCallback, useRef } from 'react'; -import { merge } from 'rxjs'; import { EuiFlexGroup, EuiFlexItem, @@ -24,12 +23,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { Required } from 'utility-types'; import { i18n } from '@kbn/i18n'; import { Filter } from '@kbn/es-query'; -import { - KBN_FIELD_TYPES, - UI_SETTINGS, - Query, - generateFilters, -} from '../../../../../../../../src/plugins/data/public'; +import { Query, generateFilters } from '../../../../../../../../src/plugins/data/public'; import { FullTimeRangeSelector } from '../full_time_range_selector'; import { usePageUrlState, useUrlState } from '../../../common/util/url_state'; import { @@ -37,40 +31,31 @@ import { ItemIdToExpandedRowMap, } from '../../../common/components/stats_table'; import { FieldVisConfig } from '../../../common/components/stats_table/types'; -import type { - MetricFieldsStats, - TotalFieldsStats, -} from '../../../common/components/stats_table/components/field_count_stats'; +import type { TotalFieldsStats } from '../../../common/components/stats_table/components/field_count_stats'; import { OverallStats } from '../../types/overall_stats'; import { getActions } from '../../../common/components/field_data_row/action_menu'; import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; import { DATA_VISUALIZER_INDEX_VIEWER } from '../../constants/index_data_visualizer_viewer'; import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../../types/combined_query'; -import { - FieldRequestConfig, - JobFieldType, - SavedSearchSavedObject, -} from '../../../../../common/types'; +import { JobFieldType, SavedSearchSavedObject } from '../../../../../common/types'; import { useDataVisualizerKibana } from '../../../kibana_context'; import { FieldCountPanel } from '../../../common/components/field_count_panel'; import { DocumentCountContent } from '../../../common/components/document_count_content'; -import { DataLoader } from '../../data_loader/data_loader'; -import { JOB_FIELD_TYPES, OMIT_FIELDS } from '../../../../../common'; +import { OMIT_FIELDS } from '../../../../../common'; import { useTimefilter } from '../../hooks/use_time_filter'; import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; import { SearchPanel } from '../search_panel'; import { ActionsPanel } from '../actions_panel'; import { DatePickerWrapper } from '../../../common/components/date_picker_wrapper'; -import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; import { HelpMenu } from '../../../common/components/help_menu'; -import { createMergedEsQuery, getEsQueryFromSavedSearch } from '../../utils/saved_search_utils'; +import { createMergedEsQuery } from '../../utils/saved_search_utils'; import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; import { ResultLink } from '../../../common/components/results_links'; -import { extractErrorProperties } from '../../utils/error_utils'; import { IndexPatternField, IndexPattern } from '../../../../../../../../src/plugins/data/common'; +import { useDataVisualizerGridData } from '../../hooks/use_data_visualizer_grid_data'; +import { DataVisualizerGridInput } from '../../embeddables/grid_embeddable/grid_embeddable'; import './_index.scss'; -import { TimeBuckets } from '../../../../../common/services/time_buckets'; interface DataVisualizerPageState { overallStats: OverallStats; @@ -164,43 +149,10 @@ export const IndexDataVisualizerView: FC = (dataVi }; }, [currentIndexPattern.id, data.query.filterManager]); - const getTimeBuckets = useCallback(() => { - return new TimeBuckets({ - [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), - [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), - dateFormat: uiSettings.get('dateFormat'), - 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), - }); - }, [uiSettings]); - - const timefilter = useTimefilter({ - timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, - autoRefreshSelector: true, - }); - - const dataLoader = useMemo( - () => new DataLoader(currentIndexPattern, toasts), - [currentIndexPattern, toasts] - ); - - useEffect(() => { - if (globalState?.time !== undefined) { - timefilter.setTime({ - from: globalState.time.from, - to: globalState.time.to, - }); - setLastRefresh(Date.now()); - } - }, [globalState, timefilter]); - - useEffect(() => { - if (globalState?.refreshInterval !== undefined) { - timefilter.setRefreshInterval(globalState.refreshInterval); - setLastRefresh(Date.now()); - } - }, [globalState, timefilter]); - - const [lastRefresh, setLastRefresh] = useState(0); + // const timefilter = useTimefilter({ + // timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, + // autoRefreshSelector: true, + // }); useEffect(() => { if (!currentIndexPattern.isTimeBased()) { @@ -238,35 +190,6 @@ export const IndexDataVisualizerView: FC = (dataVi return indexedFieldTypes.sort(); }, [indexPatternFields]); - const defaults = getDefaultPageState(); - - const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - const searchData = getEsQueryFromSavedSearch({ - indexPattern: currentIndexPattern, - uiSettings, - savedSearch: currentSavedSearch, - filterManager: data.query.filterManager, - }); - - if (searchData === undefined || dataVisualizerListState.searchString !== '') { - if (dataVisualizerListState.filters) { - data.query.filterManager.setFilters(dataVisualizerListState.filters); - } - return { - searchQuery: dataVisualizerListState.searchQuery, - searchString: dataVisualizerListState.searchString, - searchQueryLanguage: dataVisualizerListState.searchQueryLanguage, - }; - } else { - return { - searchQuery: searchData.searchQuery, - searchString: searchData.searchString, - searchQueryLanguage: searchData.queryLanguage, - }; - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState, data.query]); - const setSearchParams = useCallback( (searchParams: { searchQuery: Query['query']; @@ -318,17 +241,40 @@ export const IndexDataVisualizerView: FC = (dataVi }); }; - const [overallStats, setOverallStats] = useState(defaults.overallStats); + const input: DataVisualizerGridInput = useMemo(() => { + return { + indexPattern: currentIndexPattern, + savedSearch: currentSavedSearch, + visibleFieldNames, + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentIndexPattern.id, currentSavedSearch?.id, visibleFieldNames]); - const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); + const { + configs, + searchQueryLanguage, + searchString, + overallStats, + searchQuery, + documentCountStats, + metricsStats, + timefilter, + } = useDataVisualizerGridData(input, dataVisualizerListState, globalState, setGlobalState); - console.log('documentCountStats', documentCountStats); - const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); - const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); - const [metricsStats, setMetricsStats] = useState(); + useEffect(() => { + if (globalState?.time !== undefined) { + timefilter.setTime({ + from: globalState.time.from, + to: globalState.time.to, + }); + } + }, [globalState, timefilter]); - const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); - const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); + useEffect(() => { + if (globalState?.refreshInterval !== undefined) { + timefilter.setRefreshInterval(globalState.refreshInterval); + } + }, [globalState, timefilter]); const onAddFilter = useCallback( (field: IndexPatternField | string, values: string, operation: '+' | '-') => { @@ -376,423 +322,8 @@ export const IndexDataVisualizerView: FC = (dataVi ] ); - useEffect(() => { - const timeUpdateSubscription = merge( - timefilter.getTimeUpdate$(), - dataVisualizerRefresh$ - ).subscribe(() => { - setGlobalState({ - time: timefilter.getTime(), - refreshInterval: timefilter.getRefreshInterval(), - }); - setLastRefresh(Date.now()); - }); - return () => { - timeUpdateSubscription.unsubscribe(); - }; - }); - - useEffect(() => { - loadOverallStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchQuery, samplerShardSize, lastRefresh]); - - useEffect(() => { - createMetricCards(); - createNonMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [overallStats, showEmptyFields]); - - useEffect(() => { - loadMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricConfigs]); - - useEffect(() => { - loadNonMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricConfigs]); - - useEffect(() => { - createMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricsLoaded]); - - useEffect(() => { - createNonMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricsLoaded]); - - async function loadOverallStats() { - const tf = timefilter as any; - let earliest; - let latest; - - const activeBounds = tf.getActiveBounds(); - - if (currentIndexPattern.timeFieldName !== undefined && activeBounds === undefined) { - return; - } - - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = activeBounds.min.valueOf(); - latest = activeBounds.max.valueOf(); - } - - try { - const allStats = await dataLoader.loadOverallData( - searchQuery, - samplerShardSize, - earliest, - latest - ); - // Because load overall stats perform queries in batches - // there could be multiple errors - if (Array.isArray(allStats.errors) && allStats.errors.length > 0) { - allStats.errors.forEach((err: any) => { - dataLoader.displayError(extractErrorProperties(err)); - }); - } - setOverallStats(allStats); - } catch (err) { - dataLoader.displayError(err.body ?? err); - } - } - - async function loadMetricFieldStats() { - // Only request data for fields that exist in documents. - if (metricConfigs.length === 0) { - return; - } - - const configsToLoad = metricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - // Obtain the interval to use for date histogram aggregations - // (such as the document count chart). Aim for 75 bars. - const buckets = getTimeBuckets(); - - const tf = timefilter as any; - let earliest: number | undefined; - let latest: number | undefined; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - const bounds = tf.getActiveBounds(); - const BAR_TARGET = 75; - buckets.setInterval('auto'); - buckets.setBounds(bounds); - buckets.setBarTarget(BAR_TARGET); - const aggInterval = buckets.getInterval(); - - try { - const metricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existMetricFields, - aggInterval.asMilliseconds() - ); - - // Add the metric stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - metricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - configWithStats.loading = false; - configs.push(configWithStats); - } else { - // Document count card. - configWithStats.stats = metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === undefined - ); - - if (configWithStats.stats !== undefined) { - // Add earliest / latest of timefilter for setting x axis domain. - configWithStats.stats.timeRangeEarliest = earliest; - configWithStats.stats.timeRangeLatest = latest; - } - setDocumentCountStats(configWithStats); - } - }); - - setMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - async function loadNonMetricFieldStats() { - // Only request data for fields that exist in documents. - if (nonMetricConfigs.length === 0) { - return; - } - - const configsToLoad = nonMetricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existNonMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - const tf = timefilter as any; - let earliest; - let latest; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - try { - const nonMetricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existNonMetricFields - ); - - // Add the field stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - nonMetricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...nonMetricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - } - configWithStats.loading = false; - configs.push(configWithStats); - }); - - setNonMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - const createMetricCards = useCallback(() => { - const configs: FieldVisConfig[] = []; - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; - - const allMetricFields = indexPatternFields.filter((f) => { - return ( - f.type === KBN_FIELD_TYPES.NUMBER && - f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true - ); - }); - const metricExistsFields = allMetricFields.filter((f) => { - return aggregatableExistsFields.find((existsF) => { - return existsF.fieldName === f.spec.name; - }); - }); - - // @todo: REMOVE - // Add a config for 'document count', identified by no field name if indexpattern is time based. - // if (currentIndexPattern.timeFieldName !== undefined) { - // configs.push({ - // type: JOB_FIELD_TYPES.NUMBER, - // existsInDocs: true, - // loading: true, - // aggregatable: true, - // }); - // } - - if (metricsLoaded === false) { - setMetricsLoaded(true); - return; - } - - let aggregatableFields: any[] = overallStats.aggregatableExistsFields; - if (allMetricFields.length !== metricExistsFields.length && metricsLoaded === true) { - aggregatableFields = aggregatableFields.concat(overallStats.aggregatableNotExistsFields); - } - - const metricFieldsToShow = - metricsLoaded === true && showEmptyFields === true ? allMetricFields : metricExistsFields; - - metricFieldsToShow.forEach((field) => { - const fieldData = aggregatableFields.find((f) => { - return f.fieldName === field.spec.name; - }); - - const metricConfig: FieldVisConfig = { - ...(fieldData ? fieldData : {}), - fieldFormat: currentIndexPattern.getFormatterForField(field), - type: JOB_FIELD_TYPES.NUMBER, - loading: true, - aggregatable: true, - deletable: field.runtimeField !== undefined, - }; - if (field.displayName !== metricConfig.fieldName) { - metricConfig.displayName = field.displayName; - } - - configs.push(metricConfig); - }); - - setMetricsStats({ - totalMetricFieldsCount: allMetricFields.length, - visibleMetricsCount: metricFieldsToShow.length, - }); - setMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - metricsLoaded, - overallStats, - showEmptyFields, - ]); - - const createNonMetricCards = useCallback(() => { - const allNonMetricFields = indexPatternFields.filter((f) => { - return ( - f.type !== KBN_FIELD_TYPES.NUMBER && - f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true - ); - }); - // Obtain the list of all non-metric fields which appear in documents - // (aggregatable or not aggregatable). - const populatedNonMetricFields: any[] = []; // Kibana index pattern non metric fields. - let nonMetricFieldData: any[] = []; // Basic non metric field data loaded from requesting overall stats. - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; - const nonAggregatableExistsFields: any[] = overallStats.nonAggregatableExistsFields || []; - - allNonMetricFields.forEach((f) => { - const checkAggregatableField = aggregatableExistsFields.find( - (existsField) => existsField.fieldName === f.spec.name - ); - - if (checkAggregatableField !== undefined) { - populatedNonMetricFields.push(f); - nonMetricFieldData.push(checkAggregatableField); - } else { - const checkNonAggregatableField = nonAggregatableExistsFields.find( - (existsField) => existsField.fieldName === f.spec.name - ); - - if (checkNonAggregatableField !== undefined) { - populatedNonMetricFields.push(f); - nonMetricFieldData.push(checkNonAggregatableField); - } - } - }); - - if (nonMetricsLoaded === false) { - setNonMetricsLoaded(true); - return; - } - - if (allNonMetricFields.length !== nonMetricFieldData.length && showEmptyFields === true) { - // Combine the field data obtained from Elasticsearch into a single array. - nonMetricFieldData = nonMetricFieldData.concat( - overallStats.aggregatableNotExistsFields, - overallStats.nonAggregatableNotExistsFields - ); - } - - const nonMetricFieldsToShow = showEmptyFields ? allNonMetricFields : populatedNonMetricFields; - - const configs: FieldVisConfig[] = []; - - nonMetricFieldsToShow.forEach((field) => { - const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); - - const nonMetricConfig = { - ...(fieldData ? fieldData : {}), - fieldFormat: currentIndexPattern.getFormatterForField(field), - aggregatable: field.aggregatable, - scripted: field.scripted, - loading: fieldData?.existsInDocs, - deletable: field.runtimeField !== undefined, - }; - - // Map the field type from the Kibana index pattern to the field type - // used in the data visualizer. - const dataVisualizerType = kbnTypeToJobType(field); - if (dataVisualizerType !== undefined) { - nonMetricConfig.type = dataVisualizerType; - } else { - // Add a flag to indicate that this is one of the 'other' Kibana - // field types that do not yet have a specific card type. - nonMetricConfig.type = field.type; - nonMetricConfig.isUnsupportedType = true; - } - - if (field.displayName !== nonMetricConfig.fieldName) { - nonMetricConfig.displayName = field.displayName; - } - - configs.push(nonMetricConfig); - }); - - setNonMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - nonMetricsLoaded, - overallStats, - showEmptyFields, - ]); - const wizardPanelWidth = '280px'; - const configs = useMemo(() => { - let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; - if (visibleFieldTypes && visibleFieldTypes.length > 0) { - combinedConfigs = combinedConfigs.filter( - (config) => visibleFieldTypes.findIndex((field) => field === config.type) > -1 - ); - } - if (visibleFieldNames && visibleFieldNames.length > 0) { - combinedConfigs = combinedConfigs.filter( - (config) => visibleFieldNames.findIndex((field) => field === config.fieldName) > -1 - ); - } - - return combinedConfigs; - }, [nonMetricConfigs, metricConfigs, visibleFieldTypes, visibleFieldNames]); - const fieldsCountStats: TotalFieldsStats | undefined = useMemo(() => { let _visibleFieldsCount = 0; let _totalFieldsCount = 0; @@ -926,7 +457,7 @@ export const IndexDataVisualizerView: FC = (dataVi {overallStats?.totalCount !== undefined && ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx new file mode 100644 index 0000000000000..5949668a82dfc --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx @@ -0,0 +1,984 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, Fragment, useEffect, useMemo, useState, useCallback, useRef } from 'react'; +import { merge } from 'rxjs'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiPage, + EuiPageBody, + EuiPageContentBody, + EuiPageContentHeader, + EuiPageContentHeaderSection, + EuiPanel, + EuiSpacer, + EuiTitle, +} from '@elastic/eui'; +import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { Required } from 'utility-types'; +import { i18n } from '@kbn/i18n'; +import { Filter } from '@kbn/es-query'; +import { + KBN_FIELD_TYPES, + UI_SETTINGS, + Query, + generateFilters, +} from '../../../../../../../../src/plugins/data/public'; +import { FullTimeRangeSelector } from '../full_time_range_selector'; +import { usePageUrlState, useUrlState } from '../../../common/util/url_state'; +import { + DataVisualizerTable, + ItemIdToExpandedRowMap, +} from '../../../common/components/stats_table'; +import { FieldVisConfig } from '../../../common/components/stats_table/types'; +import type { + MetricFieldsStats, + TotalFieldsStats, +} from '../../../common/components/stats_table/components/field_count_stats'; +import { OverallStats } from '../../types/overall_stats'; +import { getActions } from '../../../common/components/field_data_row/action_menu'; +import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; +import { DATA_VISUALIZER_INDEX_VIEWER } from '../../constants/index_data_visualizer_viewer'; +import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; +import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../../types/combined_query'; +import { + FieldRequestConfig, + JobFieldType, + SavedSearchSavedObject, +} from '../../../../../common/types'; +import { useDataVisualizerKibana } from '../../../kibana_context'; +import { FieldCountPanel } from '../../../common/components/field_count_panel'; +import { DocumentCountContent } from '../../../common/components/document_count_content'; +import { DataLoader } from '../../data_loader/data_loader'; +import { JOB_FIELD_TYPES, OMIT_FIELDS } from '../../../../../common'; +import { useTimefilter } from '../../hooks/use_time_filter'; +import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; +import { SearchPanel } from '../search_panel'; +import { ActionsPanel } from '../actions_panel'; +import { DatePickerWrapper } from '../../../common/components/date_picker_wrapper'; +import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; +import { HelpMenu } from '../../../common/components/help_menu'; +import { createMergedEsQuery, getEsQueryFromSavedSearch } from '../../utils/saved_search_utils'; +import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; +import { ResultLink } from '../../../common/components/results_links'; +import { extractErrorProperties } from '../../utils/error_utils'; +import { IndexPatternField, IndexPattern } from '../../../../../../../../src/plugins/data/common'; +import './_index.scss'; +import { TimeBuckets } from '../../../../../common/services/time_buckets'; + +interface DataVisualizerPageState { + overallStats: OverallStats; + metricConfigs: FieldVisConfig[]; + totalMetricFieldCount: number; + populatedMetricFieldCount: number; + metricsLoaded: boolean; + nonMetricConfigs: FieldVisConfig[]; + nonMetricsLoaded: boolean; + documentCountStats?: FieldVisConfig; +} + +const defaultSearchQuery = { + match_all: {}, +}; + +export function getDefaultPageState(): DataVisualizerPageState { + return { + overallStats: { + totalCount: 0, + aggregatableExistsFields: [], + aggregatableNotExistsFields: [], + nonAggregatableExistsFields: [], + nonAggregatableNotExistsFields: [], + }, + metricConfigs: [], + totalMetricFieldCount: 0, + populatedMetricFieldCount: 0, + metricsLoaded: false, + nonMetricConfigs: [], + nonMetricsLoaded: false, + documentCountStats: undefined, + }; +} +export const getDefaultDataVisualizerListState = ( + overrides?: Partial +): Required => ({ + pageIndex: 0, + pageSize: 25, + sortField: 'fieldName', + sortDirection: 'asc', + visibleFieldTypes: [], + visibleFieldNames: [], + samplerShardSize: 5000, + searchString: '', + searchQuery: defaultSearchQuery, + searchQueryLanguage: SEARCH_QUERY_LANGUAGE.KUERY, + filters: [], + showDistributions: true, + showAllFields: false, + showEmptyFields: false, + ...overrides, +}); + +export interface IndexDataVisualizerViewProps { + currentIndexPattern: IndexPattern; + currentSavedSearch: SavedSearchSavedObject | null; + additionalLinks?: ResultLink[]; +} +const restorableDefaults = getDefaultDataVisualizerListState(); + +export const IndexDataVisualizerView: FC = (dataVisualizerProps) => { + const { services } = useDataVisualizerKibana(); + const { docLinks, notifications, uiSettings, data } = services; + const { toasts } = notifications; + + const [dataVisualizerListState, setDataVisualizerListState] = usePageUrlState( + DATA_VISUALIZER_INDEX_VIEWER, + restorableDefaults + ); + const [globalState, setGlobalState] = useUrlState('_g'); + + const [currentSavedSearch, setCurrentSavedSearch] = useState( + dataVisualizerProps.currentSavedSearch + ); + + const { currentIndexPattern, additionalLinks } = dataVisualizerProps; + + useEffect(() => { + if (dataVisualizerProps?.currentSavedSearch !== undefined) { + setCurrentSavedSearch(dataVisualizerProps?.currentSavedSearch); + } + }, [dataVisualizerProps?.currentSavedSearch]); + + useEffect(() => { + return () => { + // When navigating away from the index pattern + // Reset all previously set filters + // to make sure new page doesn't have unrelated filters + data.query.filterManager.removeAll(); + }; + }, [currentIndexPattern.id, data.query.filterManager]); + + const getTimeBuckets = useCallback(() => { + return new TimeBuckets({ + [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), + [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), + dateFormat: uiSettings.get('dateFormat'), + 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), + }); + }, [uiSettings]); + + const timefilter = useTimefilter({ + timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, + autoRefreshSelector: true, + }); + + const dataLoader = useMemo( + () => new DataLoader(currentIndexPattern, toasts), + [currentIndexPattern, toasts] + ); + + useEffect(() => { + if (globalState?.time !== undefined) { + timefilter.setTime({ + from: globalState.time.from, + to: globalState.time.to, + }); + setLastRefresh(Date.now()); + } + }, [globalState, timefilter]); + + useEffect(() => { + if (globalState?.refreshInterval !== undefined) { + timefilter.setRefreshInterval(globalState.refreshInterval); + setLastRefresh(Date.now()); + } + }, [globalState, timefilter]); + + const [lastRefresh, setLastRefresh] = useState(0); + + useEffect(() => { + if (!currentIndexPattern.isTimeBased()) { + toasts.addWarning({ + title: i18n.translate( + 'xpack.dataVisualizer.index.indexPatternNotBasedOnTimeSeriesNotificationTitle', + { + defaultMessage: 'The index pattern {indexPatternTitle} is not based on a time series', + values: { indexPatternTitle: currentIndexPattern.title }, + } + ), + text: i18n.translate( + 'xpack.dataVisualizer.index.indexPatternNotBasedOnTimeSeriesNotificationDescription', + { + defaultMessage: 'Anomaly detection only runs over time-based indices', + } + ), + }); + } + }, [currentIndexPattern, toasts]); + + const indexPatternFields: IndexPatternField[] = currentIndexPattern.fields; + + const fieldTypes = useMemo(() => { + // Obtain the list of non metric field types which appear in the index pattern. + const indexedFieldTypes: JobFieldType[] = []; + indexPatternFields.forEach((field) => { + if (!OMIT_FIELDS.includes(field.name) && field.scripted !== true) { + const dataVisualizerType: JobFieldType | undefined = kbnTypeToJobType(field); + if (dataVisualizerType !== undefined && !indexedFieldTypes.includes(dataVisualizerType)) { + indexedFieldTypes.push(dataVisualizerType); + } + } + }); + return indexedFieldTypes.sort(); + }, [indexPatternFields]); + + const defaults = getDefaultPageState(); + + const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { + const searchData = getEsQueryFromSavedSearch({ + indexPattern: currentIndexPattern, + uiSettings, + savedSearch: currentSavedSearch, + filterManager: data.query.filterManager, + }); + + if (searchData === undefined || dataVisualizerListState.searchString !== '') { + if (dataVisualizerListState.filters) { + data.query.filterManager.setFilters(dataVisualizerListState.filters); + } + return { + searchQuery: dataVisualizerListState.searchQuery, + searchString: dataVisualizerListState.searchString, + searchQueryLanguage: dataVisualizerListState.searchQueryLanguage, + }; + } else { + return { + searchQuery: searchData.searchQuery, + searchString: searchData.searchString, + searchQueryLanguage: searchData.queryLanguage, + }; + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState, data.query]); + + const setSearchParams = useCallback( + (searchParams: { + searchQuery: Query['query']; + searchString: Query['query']; + queryLanguage: SearchQueryLanguage; + filters: Filter[]; + }) => { + // When the user loads saved search and then clear or modify the query + // we should remove the saved search and replace it with the index pattern id + if (currentSavedSearch !== null) { + setCurrentSavedSearch(null); + } + + setDataVisualizerListState({ + ...dataVisualizerListState, + searchQuery: searchParams.searchQuery, + searchString: searchParams.searchString, + searchQueryLanguage: searchParams.queryLanguage, + filters: searchParams.filters, + }); + }, + [currentSavedSearch, dataVisualizerListState, setDataVisualizerListState] + ); + + const samplerShardSize = + dataVisualizerListState.samplerShardSize ?? restorableDefaults.samplerShardSize; + const setSamplerShardSize = (value: number) => { + setDataVisualizerListState({ ...dataVisualizerListState, samplerShardSize: value }); + }; + + const visibleFieldTypes = + dataVisualizerListState.visibleFieldTypes ?? restorableDefaults.visibleFieldTypes; + const setVisibleFieldTypes = (values: string[]) => { + setDataVisualizerListState({ ...dataVisualizerListState, visibleFieldTypes: values }); + }; + + const visibleFieldNames = + dataVisualizerListState.visibleFieldNames ?? restorableDefaults.visibleFieldNames; + const setVisibleFieldNames = (values: string[]) => { + setDataVisualizerListState({ ...dataVisualizerListState, visibleFieldNames: values }); + }; + + const showEmptyFields = + dataVisualizerListState.showEmptyFields ?? restorableDefaults.showEmptyFields; + const toggleShowEmptyFields = () => { + setDataVisualizerListState({ + ...dataVisualizerListState, + showEmptyFields: !dataVisualizerListState.showEmptyFields, + }); + }; + + const [overallStats, setOverallStats] = useState(defaults.overallStats); + + const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); + + console.log('documentCountStats', documentCountStats); + const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); + const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); + const [metricsStats, setMetricsStats] = useState(); + + const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); + const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); + + const onAddFilter = useCallback( + (field: IndexPatternField | string, values: string, operation: '+' | '-') => { + const newFilters = generateFilters( + data.query.filterManager, + field, + values, + operation, + String(currentIndexPattern.id) + ); + if (newFilters) { + data.query.filterManager.addFilters(newFilters); + } + + // Merge current query with new filters + const mergedQuery = { + query: searchString || '', + language: searchQueryLanguage, + }; + + const combinedQuery = createMergedEsQuery( + { + query: searchString || '', + language: searchQueryLanguage, + }, + data.query.filterManager.getFilters() ?? [], + currentIndexPattern, + uiSettings + ); + + setSearchParams({ + searchQuery: combinedQuery, + searchString: mergedQuery.query, + queryLanguage: mergedQuery.language as SearchQueryLanguage, + filters: data.query.filterManager.getFilters(), + }); + }, + [ + currentIndexPattern, + data.query.filterManager, + searchQueryLanguage, + searchString, + setSearchParams, + uiSettings, + ] + ); + + useEffect(() => { + const timeUpdateSubscription = merge( + timefilter.getTimeUpdate$(), + dataVisualizerRefresh$ + ).subscribe(() => { + setGlobalState({ + time: timefilter.getTime(), + refreshInterval: timefilter.getRefreshInterval(), + }); + setLastRefresh(Date.now()); + }); + return () => { + timeUpdateSubscription.unsubscribe(); + }; + }); + + useEffect(() => { + loadOverallStats(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [searchQuery, samplerShardSize, lastRefresh]); + + useEffect(() => { + createMetricCards(); + createNonMetricCards(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [overallStats, showEmptyFields]); + + useEffect(() => { + loadMetricFieldStats(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [metricConfigs]); + + useEffect(() => { + loadNonMetricFieldStats(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [nonMetricConfigs]); + + useEffect(() => { + createMetricCards(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [metricsLoaded]); + + useEffect(() => { + createNonMetricCards(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [nonMetricsLoaded]); + + async function loadOverallStats() { + const tf = timefilter as any; + let earliest; + let latest; + + const activeBounds = tf.getActiveBounds(); + + if (currentIndexPattern.timeFieldName !== undefined && activeBounds === undefined) { + return; + } + + if (currentIndexPattern.timeFieldName !== undefined) { + earliest = activeBounds.min.valueOf(); + latest = activeBounds.max.valueOf(); + } + + try { + const allStats = await dataLoader.loadOverallData( + searchQuery, + samplerShardSize, + earliest, + latest + ); + // Because load overall stats perform queries in batches + // there could be multiple errors + if (Array.isArray(allStats.errors) && allStats.errors.length > 0) { + allStats.errors.forEach((err: any) => { + dataLoader.displayError(extractErrorProperties(err)); + }); + } + setOverallStats(allStats); + } catch (err) { + dataLoader.displayError(err.body ?? err); + } + } + + async function loadMetricFieldStats() { + // Only request data for fields that exist in documents. + if (metricConfigs.length === 0) { + return; + } + + const configsToLoad = metricConfigs.filter( + (config) => config.existsInDocs === true && config.loading === true + ); + if (configsToLoad.length === 0) { + return; + } + + // Pass the field name, type and cardinality in the request. + // Top values will be obtained on a sample if cardinality > 100000. + const existMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { + const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; + if (config.stats !== undefined && config.stats.cardinality !== undefined) { + props.cardinality = config.stats.cardinality; + } + return props; + }); + + // Obtain the interval to use for date histogram aggregations + // (such as the document count chart). Aim for 75 bars. + const buckets = getTimeBuckets(); + + const tf = timefilter as any; + let earliest: number | undefined; + let latest: number | undefined; + if (currentIndexPattern.timeFieldName !== undefined) { + earliest = tf.getActiveBounds().min.valueOf(); + latest = tf.getActiveBounds().max.valueOf(); + } + + const bounds = tf.getActiveBounds(); + const BAR_TARGET = 75; + buckets.setInterval('auto'); + buckets.setBounds(bounds); + buckets.setBarTarget(BAR_TARGET); + const aggInterval = buckets.getInterval(); + + try { + const metricFieldStats = await dataLoader.loadFieldStats( + searchQuery, + samplerShardSize, + earliest, + latest, + existMetricFields, + aggInterval.asMilliseconds() + ); + + // Add the metric stats to the existing stats in the corresponding config. + const configs: FieldVisConfig[] = []; + metricConfigs.forEach((config) => { + const configWithStats = { ...config }; + if (config.fieldName !== undefined) { + configWithStats.stats = { + ...configWithStats.stats, + ...metricFieldStats.find( + (fieldStats: any) => fieldStats.fieldName === config.fieldName + ), + }; + configWithStats.loading = false; + configs.push(configWithStats); + } else { + // Document count card. + configWithStats.stats = metricFieldStats.find( + (fieldStats: any) => fieldStats.fieldName === undefined + ); + + if (configWithStats.stats !== undefined) { + // Add earliest / latest of timefilter for setting x axis domain. + configWithStats.stats.timeRangeEarliest = earliest; + configWithStats.stats.timeRangeLatest = latest; + } + setDocumentCountStats(configWithStats); + } + }); + + setMetricConfigs(configs); + } catch (err) { + dataLoader.displayError(err); + } + } + + async function loadNonMetricFieldStats() { + // Only request data for fields that exist in documents. + if (nonMetricConfigs.length === 0) { + return; + } + + const configsToLoad = nonMetricConfigs.filter( + (config) => config.existsInDocs === true && config.loading === true + ); + if (configsToLoad.length === 0) { + return; + } + + // Pass the field name, type and cardinality in the request. + // Top values will be obtained on a sample if cardinality > 100000. + const existNonMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { + const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; + if (config.stats !== undefined && config.stats.cardinality !== undefined) { + props.cardinality = config.stats.cardinality; + } + return props; + }); + + const tf = timefilter as any; + let earliest; + let latest; + if (currentIndexPattern.timeFieldName !== undefined) { + earliest = tf.getActiveBounds().min.valueOf(); + latest = tf.getActiveBounds().max.valueOf(); + } + + try { + const nonMetricFieldStats = await dataLoader.loadFieldStats( + searchQuery, + samplerShardSize, + earliest, + latest, + existNonMetricFields + ); + + // Add the field stats to the existing stats in the corresponding config. + const configs: FieldVisConfig[] = []; + nonMetricConfigs.forEach((config) => { + const configWithStats = { ...config }; + if (config.fieldName !== undefined) { + configWithStats.stats = { + ...configWithStats.stats, + ...nonMetricFieldStats.find( + (fieldStats: any) => fieldStats.fieldName === config.fieldName + ), + }; + } + configWithStats.loading = false; + configs.push(configWithStats); + }); + + setNonMetricConfigs(configs); + } catch (err) { + dataLoader.displayError(err); + } + } + + const createMetricCards = useCallback(() => { + const configs: FieldVisConfig[] = []; + const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; + + const allMetricFields = indexPatternFields.filter((f) => { + return ( + f.type === KBN_FIELD_TYPES.NUMBER && + f.displayName !== undefined && + dataLoader.isDisplayField(f.displayName) === true + ); + }); + const metricExistsFields = allMetricFields.filter((f) => { + return aggregatableExistsFields.find((existsF) => { + return existsF.fieldName === f.spec.name; + }); + }); + + // @todo: REMOVE + // Add a config for 'document count', identified by no field name if indexpattern is time based. + // if (currentIndexPattern.timeFieldName !== undefined) { + // configs.push({ + // type: JOB_FIELD_TYPES.NUMBER, + // existsInDocs: true, + // loading: true, + // aggregatable: true, + // }); + // } + + if (metricsLoaded === false) { + setMetricsLoaded(true); + return; + } + + let aggregatableFields: any[] = overallStats.aggregatableExistsFields; + if (allMetricFields.length !== metricExistsFields.length && metricsLoaded === true) { + aggregatableFields = aggregatableFields.concat(overallStats.aggregatableNotExistsFields); + } + + const metricFieldsToShow = + metricsLoaded === true && showEmptyFields === true ? allMetricFields : metricExistsFields; + + metricFieldsToShow.forEach((field) => { + const fieldData = aggregatableFields.find((f) => { + return f.fieldName === field.spec.name; + }); + + const metricConfig: FieldVisConfig = { + ...(fieldData ? fieldData : {}), + fieldFormat: currentIndexPattern.getFormatterForField(field), + type: JOB_FIELD_TYPES.NUMBER, + loading: true, + aggregatable: true, + deletable: field.runtimeField !== undefined, + }; + if (field.displayName !== metricConfig.fieldName) { + metricConfig.displayName = field.displayName; + } + + configs.push(metricConfig); + }); + + setMetricsStats({ + totalMetricFieldsCount: allMetricFields.length, + visibleMetricsCount: metricFieldsToShow.length, + }); + setMetricConfigs(configs); + }, [ + currentIndexPattern, + dataLoader, + indexPatternFields, + metricsLoaded, + overallStats, + showEmptyFields, + ]); + + const createNonMetricCards = useCallback(() => { + const allNonMetricFields = indexPatternFields.filter((f) => { + return ( + f.type !== KBN_FIELD_TYPES.NUMBER && + f.displayName !== undefined && + dataLoader.isDisplayField(f.displayName) === true + ); + }); + // Obtain the list of all non-metric fields which appear in documents + // (aggregatable or not aggregatable). + const populatedNonMetricFields: any[] = []; // Kibana index pattern non metric fields. + let nonMetricFieldData: any[] = []; // Basic non metric field data loaded from requesting overall stats. + const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; + const nonAggregatableExistsFields: any[] = overallStats.nonAggregatableExistsFields || []; + + allNonMetricFields.forEach((f) => { + const checkAggregatableField = aggregatableExistsFields.find( + (existsField) => existsField.fieldName === f.spec.name + ); + + if (checkAggregatableField !== undefined) { + populatedNonMetricFields.push(f); + nonMetricFieldData.push(checkAggregatableField); + } else { + const checkNonAggregatableField = nonAggregatableExistsFields.find( + (existsField) => existsField.fieldName === f.spec.name + ); + + if (checkNonAggregatableField !== undefined) { + populatedNonMetricFields.push(f); + nonMetricFieldData.push(checkNonAggregatableField); + } + } + }); + + if (nonMetricsLoaded === false) { + setNonMetricsLoaded(true); + return; + } + + if (allNonMetricFields.length !== nonMetricFieldData.length && showEmptyFields === true) { + // Combine the field data obtained from Elasticsearch into a single array. + nonMetricFieldData = nonMetricFieldData.concat( + overallStats.aggregatableNotExistsFields, + overallStats.nonAggregatableNotExistsFields + ); + } + + const nonMetricFieldsToShow = showEmptyFields ? allNonMetricFields : populatedNonMetricFields; + + const configs: FieldVisConfig[] = []; + + nonMetricFieldsToShow.forEach((field) => { + const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); + + const nonMetricConfig = { + ...(fieldData ? fieldData : {}), + fieldFormat: currentIndexPattern.getFormatterForField(field), + aggregatable: field.aggregatable, + scripted: field.scripted, + loading: fieldData?.existsInDocs, + deletable: field.runtimeField !== undefined, + }; + + // Map the field type from the Kibana index pattern to the field type + // used in the data visualizer. + const dataVisualizerType = kbnTypeToJobType(field); + if (dataVisualizerType !== undefined) { + nonMetricConfig.type = dataVisualizerType; + } else { + // Add a flag to indicate that this is one of the 'other' Kibana + // field types that do not yet have a specific card type. + nonMetricConfig.type = field.type; + nonMetricConfig.isUnsupportedType = true; + } + + if (field.displayName !== nonMetricConfig.fieldName) { + nonMetricConfig.displayName = field.displayName; + } + + configs.push(nonMetricConfig); + }); + + setNonMetricConfigs(configs); + }, [ + currentIndexPattern, + dataLoader, + indexPatternFields, + nonMetricsLoaded, + overallStats, + showEmptyFields, + ]); + + const wizardPanelWidth = '280px'; + + const configs = useMemo(() => { + let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; + if (visibleFieldTypes && visibleFieldTypes.length > 0) { + combinedConfigs = combinedConfigs.filter( + (config) => visibleFieldTypes.findIndex((field) => field === config.type) > -1 + ); + } + if (visibleFieldNames && visibleFieldNames.length > 0) { + combinedConfigs = combinedConfigs.filter( + (config) => visibleFieldNames.findIndex((field) => field === config.fieldName) > -1 + ); + } + + return combinedConfigs; + }, [nonMetricConfigs, metricConfigs, visibleFieldTypes, visibleFieldNames]); + + const fieldsCountStats: TotalFieldsStats | undefined = useMemo(() => { + let _visibleFieldsCount = 0; + let _totalFieldsCount = 0; + Object.keys(overallStats).forEach((key) => { + const fieldsGroup = overallStats[key as keyof OverallStats]; + if (Array.isArray(fieldsGroup) && fieldsGroup.length > 0) { + _totalFieldsCount += fieldsGroup.length; + } + }); + + if (showEmptyFields === true) { + _visibleFieldsCount = _totalFieldsCount; + } else { + _visibleFieldsCount = + overallStats.aggregatableExistsFields.length + + overallStats.nonAggregatableExistsFields.length; + } + return { visibleFieldsCount: _visibleFieldsCount, totalFieldsCount: _totalFieldsCount }; + }, [overallStats, showEmptyFields]); + + const getItemIdToExpandedRowMap = useCallback( + function (itemIds: string[], items: FieldVisConfig[]): ItemIdToExpandedRowMap { + return itemIds.reduce((m: ItemIdToExpandedRowMap, fieldName: string) => { + const item = items.find((fieldVisConfig) => fieldVisConfig.fieldName === fieldName); + if (item !== undefined) { + m[fieldName] = ( + + ); + } + return m; + }, {} as ItemIdToExpandedRowMap); + }, + [currentIndexPattern, searchQueryLanguage, searchString, onAddFilter] + ); + + // Some actions open up fly-out or popup + // This variable is used to keep track of them and clean up when unmounting + const actionFlyoutRef = useRef<() => void | undefined>(); + useEffect(() => { + const ref = actionFlyoutRef; + return () => { + // Clean up any of the flyout/editor opened from the actions + if (ref.current) { + ref.current(); + } + }; + }, []); + + // Inject custom action column for the index based visualizer + // Hide the column completely if no access to any of the plugins + const extendedColumns = useMemo(() => { + const actions = getActions( + currentIndexPattern, + services, + { + searchQueryLanguage, + searchString, + }, + actionFlyoutRef + ); + if (!Array.isArray(actions) || actions.length < 1) return; + + const actionColumn: EuiTableActionsColumnType = { + name: ( + + ), + actions, + width: '100px', + }; + + return [actionColumn]; + }, [currentIndexPattern, services, searchQueryLanguage, searchString]); + + const helpLink = docLinks.links.ml.guide; + + return ( + + + + + + + +
+ +

{currentIndexPattern.title}

+ + +
+ + + + {currentIndexPattern.timeFieldName !== undefined && ( + + + + )} + + + + + + + + + + + + + {overallStats?.totalCount !== undefined && ( + + + + )} + + + + + + items={configs} + pageState={dataVisualizerListState} + updatePageState={setDataVisualizerListState} + getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} + extendedColumns={extendedColumns} + /> + + + + + + + + + + + + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index b19b5dd391559..6de7cb06faada 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -36,15 +36,15 @@ import { } from '../../../common/components/stats_table'; import { FieldVisConfig } from '../../../common/components/stats_table/types'; import { getDefaultDataVisualizerListState } from '../../components/index_data_visualizer_view/index_data_visualizer_view'; -import { DataVisualizerTableState } from '../../../../../common'; +import { DataVisualizerTableState, SavedSearchSavedObject } from '../../../../../common'; import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; import { useDataVisualizerGridData } from '../../hooks/use_data_visualizer_grid_data'; export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerStartDependencies]; -export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { +export interface DataVisualizerGridInput { indexPattern: IndexPattern; - savedSearch?: SavedSearch; + savedSearch?: SavedSearch | SavedSearchSavedObject | null; query?: Query; visibleFieldNames?: string[]; filters?: Filter[]; @@ -54,6 +54,7 @@ export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { */ onAddFilter?: (field: IndexPatternField | string, value: string, type: '+' | '-') => void; } +export type DataVisualizerGridEmbeddableInput = EmbeddableInput & DataVisualizerGridInput; export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput; export type IDataVisualizerGridEmbeddable = typeof DataVisualizerGridEmbeddable; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index d395b2b56d25f..491e80975e3f8 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -31,7 +31,10 @@ import { } from '../../../../common'; import { kbnTypeToJobType } from '../../common/util/field_types_utils'; import { getActions } from '../../common/components/field_data_row/action_menu'; -import { DataVisualizerGridEmbeddableInput } from '../embeddables/grid_embeddable/grid_embeddable'; +import { + DataVisualizerGridEmbeddableInput, + DataVisualizerGridInput, +} from '../embeddables/grid_embeddable/grid_embeddable'; import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; import { useFieldStatsSearchStrategy } from './use_field_stats'; import { useOverallStats } from './use_overall_stats'; @@ -44,8 +47,10 @@ function isDisplayField(fieldName: string): boolean { } export const useDataVisualizerGridData = ( - input: DataVisualizerGridEmbeddableInput, - dataVisualizerListState: Required + input: DataVisualizerGridInput, + dataVisualizerListState: Required, + globalState, + setGlobalState ) => { const { services } = useDataVisualizerKibana(); const { uiSettings, data } = services; @@ -248,6 +253,12 @@ export const useDataVisualizerGridData = ( timefilter.getTimeUpdate$(), dataVisualizerRefresh$ ).subscribe(() => { + if (setGlobalState) { + setGlobalState({ + time: timefilter.getTime(), + refreshInterval: timefilter.getRefreshInterval(), + }); + } setLastRefresh(Date.now()); }); return () => { @@ -506,5 +517,7 @@ export const useDataVisualizerGridData = ( extendedColumns, documentCountStats: overallStats.documentCountStats, metricsStats, + overallStats, + timefilter, }; }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 9a6d0abe558a1..742b58639cf83 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -182,8 +182,8 @@ export function useOverallStats { - if (!body || intervalMs === undefined) { - return { - documentCounts: { - interval: 0, - buckets: {}, - }, - }; + params: OverallStatsSearchStrategyParams +): DocumentCountStats | undefined => { + if ( + !body || + params.intervalMs === undefined || + params.earliest === undefined || + params.latest === undefined + ) { + return undefined; } const buckets: { [key: string]: number } = {}; const dataByTimeBucket: Array<{ key: string; doc_count: number }> = get( @@ -81,9 +81,9 @@ export const processDocumentCountStats = ( }); return { - documentCounts: { - interval: intervalMs, - buckets, - }, + interval: params.intervalMs, + buckets, + timeRangeEarliest: params.earliest, + timeRangeLatest: params.latest, }; }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts index 76574fcbb0a23..84a6142f012da 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/overall_stats.ts @@ -21,7 +21,7 @@ export type NonAggregatableField = Omit; export interface OverallStats { totalCount: number; - documentCountStats?: DocumentCountStats['documentCounts']; + documentCountStats?: DocumentCountStats; aggregatableExistsFields: AggregatableField[]; aggregatableNotExistsFields: AggregatableField[]; nonAggregatableExistsFields: NonAggregatableField[]; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index 80a2069aab1a8..b36cbed2d4ede 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -71,18 +71,20 @@ export function createMergedEsQuery( } const filterQuery = buildQueryFromFilters(filters, indexPattern); - if (Array.isArray(combinedQuery.bool.filter) === false) { - combinedQuery.bool.filter = - combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; + if (combinedQuery.bool) { + if (Array.isArray(combinedQuery.bool.filter) === false) { + combinedQuery.bool.filter = + combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; + } + + if (Array.isArray(combinedQuery.bool.must_not) === false) { + combinedQuery.bool.must_not = + combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; + } + + combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; + combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; } - - if (Array.isArray(combinedQuery.bool.must_not) === false) { - combinedQuery.bool.must_not = - combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; - } - - combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; - combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; } else { combinedQuery = buildEsQuery( indexPattern, From 2e9a369906579b8f58dc08744011c6d78079045f Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 15 Oct 2021 15:10:16 -0500 Subject: [PATCH 150/188] Merge to use hook completely --- .../common/types/field_stats.ts | 4 +- .../index_data_visualizer_view.tsx | 3 +- .../index_data_visualizer_view_temp.tsx | 2 - .../grid_embeddable/grid_embeddable.tsx | 2 +- .../hooks/use_data_visualizer_grid_data.ts | 141 +++++++++--------- .../hooks/use_field_stats.ts | 33 ++-- .../hooks/use_overall_stats.ts | 41 ++++- .../index_data_visualizer/progress_utils.ts | 21 +++ 8 files changed, 151 insertions(+), 96 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/progress_utils.ts diff --git a/x-pack/plugins/data_visualizer/common/types/field_stats.ts b/x-pack/plugins/data_visualizer/common/types/field_stats.ts index 22642074f703d..290a337a5781a 100644 --- a/x-pack/plugins/data_visualizer/common/types/field_stats.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_stats.ts @@ -220,13 +220,13 @@ export interface OverallStatsSearchStrategyParams { } export interface FieldStatsSearchStrategyReturnBase { - progress: FieldStatsSearchStrategyProgress; + progress: DataStatsFetchProgress; fieldStats: Map | undefined; startFetch: () => void; cancelFetch: () => void; } -export interface FieldStatsSearchStrategyProgress { +export interface DataStatsFetchProgress { error?: Error; isRunning: boolean; loaded: number; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 241ce196bff20..a63b7617634f0 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -43,7 +43,6 @@ import { useDataVisualizerKibana } from '../../../kibana_context'; import { FieldCountPanel } from '../../../common/components/field_count_panel'; import { DocumentCountContent } from '../../../common/components/document_count_content'; import { OMIT_FIELDS } from '../../../../../common'; -import { useTimefilter } from '../../hooks/use_time_filter'; import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; import { SearchPanel } from '../search_panel'; import { ActionsPanel } from '../actions_panel'; @@ -259,7 +258,7 @@ export const IndexDataVisualizerView: FC = (dataVi documentCountStats, metricsStats, timefilter, - } = useDataVisualizerGridData(input, dataVisualizerListState, globalState, setGlobalState); + } = useDataVisualizerGridData(input, dataVisualizerListState, setGlobalState); useEffect(() => { if (globalState?.time !== undefined) { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx index 5949668a82dfc..cde4914b56e46 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx @@ -321,8 +321,6 @@ export const IndexDataVisualizerView: FC = (dataVi const [overallStats, setOverallStats] = useState(defaults.overallStats); const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); - - console.log('documentCountStats', documentCountStats); const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); const [metricsStats, setMetricsStats] = useState(); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 6de7cb06faada..f70a37d54e028 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -104,7 +104,7 @@ export const EmbeddableWrapper = ({ return (
- + items={configs} pageState={dataVisualizerListState} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 491e80975e3f8..86e9c61ff199d 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -31,14 +31,12 @@ import { } from '../../../../common'; import { kbnTypeToJobType } from '../../common/util/field_types_utils'; import { getActions } from '../../common/components/field_data_row/action_menu'; -import { - DataVisualizerGridEmbeddableInput, - DataVisualizerGridInput, -} from '../embeddables/grid_embeddable/grid_embeddable'; +import { DataVisualizerGridInput } from '../embeddables/grid_embeddable/grid_embeddable'; import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; import { useFieldStatsSearchStrategy } from './use_field_stats'; import { useOverallStats } from './use_overall_stats'; import { OverallStatsSearchStrategyParams } from '../../../../common/types/field_stats'; +import { Dictionary } from '../../common/util/url_state'; const defaults = getDefaultPageState(); @@ -49,8 +47,7 @@ function isDisplayField(fieldName: string): boolean { export const useDataVisualizerGridData = ( input: DataVisualizerGridInput, dataVisualizerListState: Required, - globalState, - setGlobalState + onUpdate?: (params: string | Dictionary) => void ) => { const { services } = useDataVisualizerKibana(); const { uiSettings, data } = services; @@ -137,7 +134,6 @@ export const useDataVisualizerGridData = ( autoRefreshSelector: true, }); - const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); const [metricsStats, setMetricsStats] = useState(); @@ -146,66 +142,71 @@ export const useDataVisualizerGridData = ( const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); /** Search strategy **/ - const fieldStatsRequest: OverallStatsSearchStrategyParams | undefined = useMemo(() => { - // Obtain the interval to use for date histogram aggregations - // (such as the document count chart). Aim for 75 bars. - const buckets = _timeBuckets; - - const tf = timefilter as any; - - if (!buckets || !tf || !currentIndexPattern) return; - - const activeBounds = tf.getActiveBounds(); - let earliest: number | undefined; - let latest: number | undefined; - if (activeBounds !== undefined && currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } + const fieldStatsRequest: OverallStatsSearchStrategyParams | undefined = useMemo( + () => { + // Obtain the interval to use for date histogram aggregations + // (such as the document count chart). Aim for 75 bars. + const buckets = _timeBuckets; + + const tf = timefilter as any; + + if (!buckets || !tf || !currentIndexPattern) return; + + const activeBounds = tf.getActiveBounds(); + let earliest: number | undefined; + let latest: number | undefined; + if (activeBounds !== undefined && currentIndexPattern.timeFieldName !== undefined) { + earliest = tf.getActiveBounds().min.valueOf(); + latest = tf.getActiveBounds().max.valueOf(); + } - const bounds = tf.getActiveBounds(); - const BAR_TARGET = 75; - buckets.setInterval('auto'); - buckets.setBounds(bounds); - buckets.setBarTarget(BAR_TARGET); - const aggInterval = buckets.getInterval(); - - const aggregatableFields: string[] = []; - const nonAggregatableFields: string[] = []; - currentIndexPattern.fields.forEach((field) => { - const fieldName = field.displayName !== undefined ? field.displayName : field.name; - if (!OMIT_FIELDS.includes(fieldName)) { - if (field.aggregatable === true && !NON_AGGREGATABLE_FIELD_TYPES.has(field.type)) { - aggregatableFields.push(field.name); - } else { - nonAggregatableFields.push(field.name); + const bounds = tf.getActiveBounds(); + const BAR_TARGET = 75; + buckets.setInterval('auto'); + buckets.setBounds(bounds); + buckets.setBarTarget(BAR_TARGET); + const aggInterval = buckets.getInterval(); + + const aggregatableFields: string[] = []; + const nonAggregatableFields: string[] = []; + currentIndexPattern.fields.forEach((field) => { + const fieldName = field.displayName !== undefined ? field.displayName : field.name; + if (!OMIT_FIELDS.includes(fieldName)) { + if (field.aggregatable === true && !NON_AGGREGATABLE_FIELD_TYPES.has(field.type)) { + aggregatableFields.push(field.name); + } else { + nonAggregatableFields.push(field.name); + } } - } - }); - return { - earliest, - latest, - aggInterval, - intervalMs: aggInterval?.asMilliseconds(), - searchQuery, + }); + return { + earliest, + latest, + aggInterval, + intervalMs: aggInterval?.asMilliseconds(), + searchQuery, + samplerShardSize, + sessionId: searchSessionId, + index: currentIndexPattern.title, + timeFieldName: currentIndexPattern.timeFieldName, + runtimeFieldMap: currentIndexPattern.getComputedFields().runtimeFields, + aggregatableFields, + nonAggregatableFields, + lastRefresh, + }; + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [ + _timeBuckets, + timefilter, + currentIndexPattern.id, + // eslint-disable-next-line react-hooks/exhaustive-deps + JSON.stringify(searchQuery), samplerShardSize, - sessionId: searchSessionId, - index: currentIndexPattern.title, - timeFieldName: currentIndexPattern.timeFieldName, - runtimeFieldMap: currentIndexPattern.getComputedFields().runtimeFields, - aggregatableFields, - nonAggregatableFields, + searchSessionId, lastRefresh, - }; - }, [ - _timeBuckets, - timefilter, - currentIndexPattern.id, - JSON.stringify(searchQuery), - samplerShardSize, - searchSessionId, - lastRefresh, - ]); + ] + ); useEffect(() => console.log('_timeBuckets', _timeBuckets), [_timeBuckets]); useEffect(() => console.log('timefilter', timefilter), [timefilter]); @@ -241,20 +242,25 @@ export const useDataVisualizerGridData = ( return { metricConfigs: existMetricFields, nonMetricConfigs: existNonMetricFields }; }, [metricConfigs, nonMetricConfigs]); - const overallStats = useOverallStats(fieldStatsRequest); + const { overallStats, progress: overallStatsProgress } = useOverallStats(fieldStatsRequest); const strategyResponse = useFieldStatsSearchStrategy( fieldStatsRequest, configsWithoutStats, dataVisualizerListStateRef.current ); + const combinedProgress = useMemo( + () => overallStatsProgress.loaded * 0.2 + strategyResponse.progress.loaded * 0.8, + [overallStatsProgress.loaded, strategyResponse.progress.loaded] + ); + useEffect(() => { const timeUpdateSubscription = merge( timefilter.getTimeUpdate$(), dataVisualizerRefresh$ ).subscribe(() => { - if (setGlobalState) { - setGlobalState({ + if (onUpdate) { + onUpdate({ time: timefilter.getTime(), refreshInterval: timefilter.getRefreshInterval(), }); @@ -507,9 +513,8 @@ export const useDataVisualizerGridData = ( return [actionColumn]; }, [input.indexPattern, services, searchQueryLanguage, searchString]); - console.log('overallStats.documentCountStats', overallStats.documentCountStats); return { - progress: strategyResponse.progress, + progress: combinedProgress, configs, searchQueryLanguage, searchString, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index f276a6a9f224f..6b7ef471147b3 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -9,7 +9,7 @@ import { useCallback, useEffect, useReducer, useRef, useState } from 'react'; import { combineLatest, Observable, Subscription } from 'rxjs'; import { i18n } from '@kbn/i18n'; import type { - FieldStatsSearchStrategyProgress, + DataStatsFetchProgress, FieldStatsSearchStrategyReturnBase, OverallStatsSearchStrategyParams, FieldStatsCommonRequestParams, @@ -23,19 +23,8 @@ import { } from '../../../../common/utils/query_utils'; import { getFieldStats } from '../search_strategy/requests/get_field_stats'; import type { FieldStats, FieldStatsError } from '../../../../common/types/field_stats'; +import { getInitialProgress, getReducer } from '../progress_utils'; -const getInitialProgress = (): FieldStatsSearchStrategyProgress => ({ - isRunning: false, - loaded: 0, - total: 100, -}); - -const getReducer = - () => - (prev: T, update: Partial): T => ({ - ...prev, - ...update, - }); interface FieldStatsParams { metricConfigs: FieldRequestConfig[]; nonMetricConfigs: FieldRequestConfig[]; @@ -54,9 +43,8 @@ export function useFieldStatsSearchStrategy( } = useDataVisualizerKibana(); const [fieldStats, setFieldStats] = useState>(); - const [fetchState, setFetchState] = useReducer( - getReducer(), + getReducer(), getInitialProgress() ); @@ -146,6 +134,11 @@ export function useFieldStatsSearchStrategy( return map; }, new Map()); + setFetchState({ + loaded: (resp.length / sortedConfigs.length) * 100, + isRunning: true, + }); + setFieldStats(statsMap); } }, @@ -155,8 +148,16 @@ export function useFieldStatsSearchStrategy( defaultMessage: 'Error fetching field statistics', }), }); + setFetchState({ + isRunning: false, + error, + }); + }, + complete: () => { + setFetchState({ + isRunning: false, + }); }, - complete: () => console.log('useOverallStats'), }); }, [data, toasts, searchStrategyParams, fieldStatsParams, initialDataVisualizerListState]); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 742b58639cf83..22df1b44ab31e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { useCallback, useEffect, useState, useRef, useMemo } from 'react'; +import { useCallback, useEffect, useState, useRef, useMemo, useReducer } from 'react'; import { combineLatest, forkJoin, of, Subscription } from 'rxjs'; import { mergeMap, switchMap } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; @@ -24,11 +24,15 @@ import { import { OverallStats } from '../types/overall_stats'; import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; import { extractErrorProperties } from '../utils/error_utils'; -import { OverallStatsSearchStrategyParams } from '../../../../common/types/field_stats'; +import { + DataStatsFetchProgress, + OverallStatsSearchStrategyParams, +} from '../../../../common/types/field_stats'; import { getDocumentCountStatsRequest, processDocumentCountStats, } from '../search_strategy/requests/get_document_stats'; +import { getInitialProgress, getReducer } from '../progress_utils'; function displayError(toastNotifications: ToastsStart, indexPattern: string, err: any) { if (err.statusCode === 500) { @@ -58,7 +62,10 @@ function displayError(toastNotifications: ToastsStart, indexPattern: string, err export function useOverallStats( searchStrategyParams: TParams | undefined -): OverallStats { +): { + progress: DataStatsFetchProgress; + overallStats: OverallStats; +} { const { services: { data, @@ -67,6 +74,10 @@ export function useOverallStats(getDefaultPageState().overallStats); + const [fetchState, setFetchState] = useReducer( + getReducer(), + getInitialProgress() + ); const abortCtrl = useRef(new AbortController()); const searchSubscription$ = useRef(); @@ -77,6 +88,11 @@ export function useOverallStats { displayError(toasts, searchStrategyParams.index, extractErrorProperties(error)); + setFetchState({ + isRunning: false, + error, + }); + }, + complete: () => { + setFetchState({ + loaded: 100, + isRunning: false, + }); }, - complete: () => console.log('useOverallStats'), }); }, [data.search, searchStrategyParams, toasts]); @@ -216,5 +241,11 @@ export function useOverallStats stats, [stats]); + return useMemo( + () => ({ + progress: fetchState, + overallStats: stats, + }), + [stats, fetchState] + ); } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/progress_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/progress_utils.ts new file mode 100644 index 0000000000000..f329fe47e75b0 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/progress_utils.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataStatsFetchProgress } from '../../../common/types/field_stats'; + +export const getInitialProgress = (): DataStatsFetchProgress => ({ + isRunning: false, + loaded: 0, + total: 100, +}); + +export const getReducer = + () => + (prev: T, update: Partial): T => ({ + ...prev, + ...update, + }); From 0ae56a9d7f1e04be4b8299e4ab79ad202a86ed43 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 15 Oct 2021 15:56:35 -0500 Subject: [PATCH 151/188] Fix doc chart doesn't show up when page is first mounted --- .../index_data_visualizer_view.tsx | 9 ++- .../index_data_visualizer_view_temp.tsx | 74 +++++++++---------- .../hooks/use_data_visualizer_grid_data.ts | 15 +--- .../hooks/use_field_stats.ts | 1 + .../hooks/use_overall_stats.ts | 3 +- .../index_data_visualizer.tsx | 2 +- 6 files changed, 46 insertions(+), 58 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index a63b7617634f0..9a0478a4714e4 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -258,6 +258,7 @@ export const IndexDataVisualizerView: FC = (dataVi documentCountStats, metricsStats, timefilter, + setLastRefresh, } = useDataVisualizerGridData(input, dataVisualizerListState, setGlobalState); useEffect(() => { @@ -266,14 +267,18 @@ export const IndexDataVisualizerView: FC = (dataVi from: globalState.time.from, to: globalState.time.to, }); + setLastRefresh(Date.now()); } - }, [globalState, timefilter]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [JSON.stringify(globalState?.time), timefilter]); useEffect(() => { if (globalState?.refreshInterval !== undefined) { timefilter.setRefreshInterval(globalState.refreshInterval); + setLastRefresh(Date.now()); } - }, [globalState, timefilter]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [JSON.stringify(globalState?.refreshInterval), timefilter]); const onAddFilter = useCallback( (field: IndexPatternField | string, values: string, operation: '+' | '-') => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx index cde4914b56e46..45ec731b24bf2 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx @@ -183,22 +183,22 @@ export const IndexDataVisualizerView: FC = (dataVi [currentIndexPattern, toasts] ); - useEffect(() => { - if (globalState?.time !== undefined) { - timefilter.setTime({ - from: globalState.time.from, - to: globalState.time.to, - }); - setLastRefresh(Date.now()); - } - }, [globalState, timefilter]); - - useEffect(() => { - if (globalState?.refreshInterval !== undefined) { - timefilter.setRefreshInterval(globalState.refreshInterval); - setLastRefresh(Date.now()); - } - }, [globalState, timefilter]); + // useEffect(() => { + // if (globalState?.time !== undefined) { + // timefilter.setTime({ + // from: globalState.time.from, + // to: globalState.time.to, + // }); + // setLastRefresh(Date.now()); + // } + // }, [globalState, timefilter]); + + // useEffect(() => { + // if (globalState?.refreshInterval !== undefined) { + // timefilter.setRefreshInterval(globalState.refreshInterval); + // setLastRefresh(Date.now()); + // } + // }, [globalState, timefilter]); const [lastRefresh, setLastRefresh] = useState(0); @@ -374,21 +374,7 @@ export const IndexDataVisualizerView: FC = (dataVi ] ); - useEffect(() => { - const timeUpdateSubscription = merge( - timefilter.getTimeUpdate$(), - dataVisualizerRefresh$ - ).subscribe(() => { - setGlobalState({ - time: timefilter.getTime(), - refreshInterval: timefilter.getRefreshInterval(), - }); - setLastRefresh(Date.now()); - }); - return () => { - timeUpdateSubscription.unsubscribe(); - }; - }); + // useEffect(dataVisualizerRefresh$ useEffect(() => { loadOverallStats(); @@ -624,14 +610,14 @@ export const IndexDataVisualizerView: FC = (dataVi // @todo: REMOVE // Add a config for 'document count', identified by no field name if indexpattern is time based. - // if (currentIndexPattern.timeFieldName !== undefined) { - // configs.push({ - // type: JOB_FIELD_TYPES.NUMBER, - // existsInDocs: true, - // loading: true, - // aggregatable: true, - // }); - // } + if (currentIndexPattern.timeFieldName !== undefined) { + configs.push({ + type: JOB_FIELD_TYPES.NUMBER, + existsInDocs: true, + loading: true, + aggregatable: true, + }); + } if (metricsLoaded === false) { setMetricsLoaded(true); @@ -873,6 +859,7 @@ export const IndexDataVisualizerView: FC = (dataVi }, [currentIndexPattern, services, searchQueryLanguage, searchString]); const helpLink = docLinks.links.ml.guide; + console.log('documentCountStats', documentCountStats); return ( @@ -924,7 +911,14 @@ export const IndexDataVisualizerView: FC = (dataVi {overallStats?.totalCount !== undefined && ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 86e9c61ff199d..d1a2a9cee3d2b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -101,7 +101,6 @@ export const useDataVisualizerGridData = ( }, [ currentSavedSearch?.id, currentIndexPattern.id, - // dataVisualizerListState.searchQuery, dataVisualizerListState.searchString, dataVisualizerListState.searchQueryLanguage, JSON.stringify({ @@ -109,8 +108,6 @@ export const useDataVisualizerGridData = ( currentQuery, currentFilters, }), - // currentQuery, - // currentFilters, ]); useEffect(() => { @@ -294,17 +291,6 @@ export const useDataVisualizerGridData = ( }); }); - // @todo: remove - // Add a config for 'document count', identified by no field name if indexpattern is time based. - // if (currentIndexPattern.timeFieldName !== undefined) { - // configs.push({ - // type: JOB_FIELD_TYPES.NUMBER, - // existsInDocs: true, - // loading: true, - // aggregatable: true, - // }); - // } - if (metricsLoaded === false) { setMetricsLoaded(true); return; @@ -524,5 +510,6 @@ export const useDataVisualizerGridData = ( metricsStats, overallStats, timefilter, + setLastRefresh, }; }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index 6b7ef471147b3..744a19077c144 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -154,6 +154,7 @@ export function useFieldStatsSearchStrategy( }); }, complete: () => { + console.log('useFieldStats called'); setFetchState({ isRunning: false, }); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 22df1b44ab31e..5659da6491c84 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -164,7 +164,7 @@ export function useOverallStats 0 + timeFieldName !== undefined && intervalMs !== undefined && intervalMs > 0 ? data.search.search( { params: getDocumentCountStatsRequest(searchStrategyParams), @@ -221,6 +221,7 @@ export function useOverallStats { + console.log('useOverallStats called', intervalMs); setFetchState({ loaded: 100, isRunning: false, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx index b90814b753d1e..0d95bdc83d3a8 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx @@ -17,7 +17,7 @@ import { getCoreStart, getPluginsStart } from '../../kibana_services'; import { IndexDataVisualizerViewProps, IndexDataVisualizerView, -} from './components/index_data_visualizer_view'; +} from './components/index_data_visualizer_view/index_data_visualizer_view'; import { Accessor, Provider as UrlStateContextProvider, From b9ea1a8807fdea8acc278533fe07432a5ccec789 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 15 Oct 2021 16:34:48 -0500 Subject: [PATCH 152/188] Fix Discover auto refresh previously didn't update --- .../index_data_visualizer_view_temp.tsx | 3 --- .../hooks/use_data_visualizer_grid_data.ts | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx index 45ec731b24bf2..a7066bdb2156b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx @@ -374,8 +374,6 @@ export const IndexDataVisualizerView: FC = (dataVi ] ); - // useEffect(dataVisualizerRefresh$ - useEffect(() => { loadOverallStats(); // eslint-disable-next-line react-hooks/exhaustive-deps @@ -859,7 +857,6 @@ export const IndexDataVisualizerView: FC = (dataVi }, [currentIndexPattern, services, searchQueryLanguage, searchString]); const helpLink = docLinks.links.ml.guide; - console.log('documentCountStats', documentCountStats); return ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index d1a2a9cee3d2b..bed687d115358 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -254,6 +254,7 @@ export const useDataVisualizerGridData = ( useEffect(() => { const timeUpdateSubscription = merge( timefilter.getTimeUpdate$(), + timefilter.getAutoRefreshFetch$(), dataVisualizerRefresh$ ).subscribe(() => { if (onUpdate) { From b63dd07afb4219c900c0f764c2e730923a23f84e Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 19 Oct 2021 11:56:17 -0500 Subject: [PATCH 153/188] Fix query util to return search source's results right away. Fix texts. --- src/plugins/discover/public/plugin.tsx | 1 - src/plugins/discover/server/ui_settings.ts | 2 +- .../grid_embeddable/grid_embeddable.tsx | 2 +- .../utils/saved_search_utils.test.ts | 10 ++-- .../utils/saved_search_utils.ts | 58 +++++++++++++------ .../configuration_step/use_saved_search.ts | 4 +- .../jobs/new_job/utils/new_job_utils.ts | 4 +- .../ml/public/application/util/index_utils.ts | 2 +- 8 files changed, 52 insertions(+), 31 deletions(-) diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index 2cc0a3fee638f..c91bcf3897e14 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -351,7 +351,6 @@ export class DiscoverPlugin // FIXME: Temporarily hide overflow-y in Discover app when Field Stats table is shown // due to EUI bug https://github.com/elastic/eui/pull/5152 - // until EUI is bumped to 38.0.0 params.element.classList.add('dscAppWrapper'); const unmount = renderApp(params.element); diff --git a/src/plugins/discover/server/ui_settings.ts b/src/plugins/discover/server/ui_settings.ts index 2892fe5da8612..dc82891eb36fb 100644 --- a/src/plugins/discover/server/ui_settings.ts +++ b/src/plugins/discover/server/ui_settings.ts @@ -210,7 +210,7 @@ export const getUiSettings: () => Record = () => ({ description: i18n.translate( 'discover.advancedSettings.discover.showFieldStatisticsDescription', { - defaultMessage: `Enable Field statistics table in Discover.`, + defaultMessage: `Enable "Field statistics" table in Discover.`, } ), value: false, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index c5d4e4c30c2ac..f59225b1c019f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -170,7 +170,7 @@ export const IndexDataVisualizerViewWrapper = (props: {

} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts index 3dd81015393b4..c1bb20adc798d 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts @@ -6,7 +6,7 @@ */ import { - getQueryFromSavedSearch, + getQueryFromSavedSearchObject, createMergedEsQuery, getEsQueryFromSavedSearch, } from './saved_search_utils'; @@ -82,9 +82,9 @@ const kqlSavedSearch: SavedSearch = { }, }; -describe('getQueryFromSavedSearch()', () => { +describe('getQueryFromSavedSearchObject()', () => { it('should return parsed searchSourceJSON with query and filter', () => { - expect(getQueryFromSavedSearch(luceneSavedSearchObj)).toEqual({ + expect(getQueryFromSavedSearchObject(luceneSavedSearchObj)).toEqual({ filter: [ { $state: { store: 'appState' }, @@ -106,7 +106,7 @@ describe('getQueryFromSavedSearch()', () => { query: { language: 'lucene', query: 'responsetime:>50' }, version: true, }); - expect(getQueryFromSavedSearch(kqlSavedSearch)).toEqual({ + expect(getQueryFromSavedSearchObject(kqlSavedSearch)).toEqual({ filter: [ { $state: { store: 'appState' }, @@ -130,7 +130,7 @@ describe('getQueryFromSavedSearch()', () => { }); }); it('should return undefined if invalid searchSourceJSON', () => { - expect(getQueryFromSavedSearch(luceneInvalidSavedSearchObj)).toEqual(undefined); + expect(getQueryFromSavedSearchObject(luceneInvalidSavedSearchObj)).toEqual(undefined); }); }); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index e6288cde25fbd..1401b1038b8f2 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -16,17 +16,31 @@ import { Filter, } from '@kbn/es-query'; import { isSavedSearchSavedObject, SavedSearchSavedObject } from '../../../../common/types'; -import { IndexPattern } from '../../../../../../../src/plugins/data/common'; +import { IndexPattern, SearchSource } from '../../../../../../../src/plugins/data/common'; import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; import { SavedSearch } from '../../../../../../../src/plugins/discover/public'; import { getEsQueryConfig } from '../../../../../../../src/plugins/data/common'; import { FilterManager } from '../../../../../../../src/plugins/data/public'; +const DEFAULT_QUERY = { + bool: { + must: [ + { + match_all: {}, + }, + ], + }, +}; + +export function getDefaultQuery() { + return cloneDeep(DEFAULT_QUERY); +} + /** * Parse the stringified searchSourceJSON * from a saved search or saved search object */ -export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject | SavedSearch) { +export function getQueryFromSavedSearchObject(savedSearch: SavedSearchSavedObject | SavedSearch) { const search = isSavedSearchSavedObject(savedSearch) ? savedSearch?.attributes?.kibanaSavedObjectMeta : // @ts-expect-error kibanaSavedObjectMeta does exist @@ -117,10 +131,31 @@ export function getEsQueryFromSavedSearch({ }) { if (!indexPattern || !savedSearch) return; - const savedSearchData = getQueryFromSavedSearch(savedSearch); const userQuery = query; const userFilters = filters; + // If saved search has a search source with nested parent + // e.g. a search coming from Dashboard saved search embeddable + // which already combines both the saved search's original query/filters and the Dashboard's + // then no need to process any further + if ( + savedSearch && + 'searchSource' in savedSearch && + savedSearch?.searchSource instanceof SearchSource && + savedSearch.searchSource.getParent() !== undefined && + userQuery + ) { + return { + searchQuery: savedSearch.searchSource.getSearchRequestBody()?.query ?? getDefaultQuery(), + searchString: userQuery.query, + queryLanguage: userQuery.language as SearchQueryLanguage, + }; + } + + // If saved search is an json object with the original query and filter + // retrieve the parsed query and filter + const savedSearchData = getQueryFromSavedSearchObject(savedSearch); + // If no saved search available, use user's query and filters if (!savedSearchData && userQuery) { if (filterManager && userFilters) filterManager.setFilters(userFilters); @@ -139,7 +174,8 @@ export function getEsQueryFromSavedSearch({ }; } - // If saved search available, merge saved search with latest user query or filters differ from extracted saved search data + // If saved search available, merge saved search with latest user query or filters + // which might differ from extracted saved search data if (savedSearchData) { const currentQuery = userQuery ?? savedSearchData?.query; const currentFilters = userFilters ?? savedSearchData?.filter; @@ -160,17 +196,3 @@ export function getEsQueryFromSavedSearch({ }; } } - -const DEFAULT_QUERY = { - bool: { - must: [ - { - match_all: {}, - }, - ], - }, -}; - -export function getDefaultQuery() { - return cloneDeep(DEFAULT_QUERY); -} diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts index 324db0d6b2ad4..41973b5ec2d01 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts @@ -15,7 +15,7 @@ import { import { estypes } from '@elastic/elasticsearch'; import { useMlContext } from '../../../../../contexts/ml'; import { SEARCH_QUERY_LANGUAGE } from '../../../../../../../common/constants/search'; -import { getQueryFromSavedSearch } from '../../../../../util/index_utils'; +import { getQueryFromSavedSearchObject } from '../../../../../util/index_utils'; // `undefined` is used for a non-initialized state // `null` is set if no saved search is used @@ -40,7 +40,7 @@ export function useSavedSearch() { let qryString; if (currentSavedSearch !== null) { - const { query } = getQueryFromSavedSearch(currentSavedSearch); + const { query } = getQueryFromSavedSearchObject(currentSavedSearch); const queryLanguage = query.language; qryString = query.query; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts b/x-pack/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts index 5eae60900e09f..ebab3769fbe57 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts @@ -18,7 +18,7 @@ import { IUiSettingsClient } from 'kibana/public'; import { getEsQueryConfig } from '../../../../../../../../src/plugins/data/public'; import { SEARCH_QUERY_LANGUAGE } from '../../../../../common/constants/search'; import { SavedSearchSavedObject } from '../../../../../common/types/kibana'; -import { getQueryFromSavedSearch } from '../../../util/index_utils'; +import { getQueryFromSavedSearchObject } from '../../../util/index_utils'; // Provider for creating the items used for searching and job creation. @@ -52,7 +52,7 @@ export function createSearchItems( let combinedQuery: any = getDefaultDatafeedQuery(); if (savedSearch !== null) { - const data = getQueryFromSavedSearch(savedSearch); + const data = getQueryFromSavedSearchObject(savedSearch); query = data.query; const filter = data.filter; diff --git a/x-pack/plugins/ml/public/application/util/index_utils.ts b/x-pack/plugins/ml/public/application/util/index_utils.ts index b4f46d4df0cbb..c66e529ea7214 100644 --- a/x-pack/plugins/ml/public/application/util/index_utils.ts +++ b/x-pack/plugins/ml/public/application/util/index_utils.ts @@ -80,7 +80,7 @@ export async function getIndexPatternAndSavedSearch(savedSearchId: string) { return resp; } -export function getQueryFromSavedSearch(savedSearch: SavedSearchSavedObject) { +export function getQueryFromSavedSearchObject(savedSearch: SavedSearchSavedObject) { const search = savedSearch.attributes.kibanaSavedObjectMeta as { searchSourceJSON: string }; return JSON.parse(search.searchSourceJSON) as { query: Query; From 888b8342886abda284f8b78907bcb4ed585c2458 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 19 Oct 2021 12:46:43 -0500 Subject: [PATCH 154/188] Refactor documentStats --- .../components/stats_table/data_visualizer_stats_table.tsx | 3 +++ .../embeddables/grid_embeddable/grid_embeddable.tsx | 2 ++ .../hooks/use_data_visualizer_grid_data.ts | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index b4608006dfd83..1c8afe381f987 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -65,6 +65,7 @@ export const DataVisualizerTable = ({ extendedColumns, showPreviewByDefault, onChange, + progress, }: DataVisualizerTableProps) => { const [expandedRowItemIds, setExpandedRowItemIds] = useState([]); const [expandAll, setExpandAll] = useState(false); @@ -318,6 +319,8 @@ export const DataVisualizerTable = ({ return getItemIdToExpandedRowMap(itemIds, items); }, [items, expandedRowItemIds, getItemIdToExpandedRowMap]); + console.log('items', items); // Some actions open up fly-out or popup + return ( {(resizeRef) => ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 4902df4de0a91..ef000f1273a88 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -82,6 +82,7 @@ export const EmbeddableWrapper = ({ ); const { configs, searchQueryLanguage, searchString, extendedColumns, progress } = useDataVisualizerGridData(input, dataVisualizerListState); + const getItemIdToExpandedRowMap = useCallback( function (itemIds: string[], items: FieldVisConfig[]): ItemIdToExpandedRowMap { return itemIds.reduce((m: ItemIdToExpandedRowMap, fieldName: string) => { @@ -135,6 +136,7 @@ export const EmbeddableWrapper = ({ extendedColumns={extendedColumns} showPreviewByDefault={input?.showPreviewByDefault} onChange={onOutputChange} + progress={progress} />
); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index bed687d115358..dd403328dbdc1 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -442,7 +442,7 @@ export const useDataVisualizerGridData = ( // @todo // @ts-ignore combinedConfigs = combinedConfigs.map((c) => { - const loadedFullStats = fieldStats.get(c.fieldName); + const loadedFullStats = fieldStats.get(c.fieldName) ?? {}; return loadedFullStats ? { ...c, @@ -462,6 +462,7 @@ export const useDataVisualizerGridData = ( strategyResponse.fieldStats, ]); + useEffect(() => console.log('configs updated', configs), [configs]); // Some actions open up fly-out or popup // This variable is used to keep track of them and clean up when unmounting const actionFlyoutRef = useRef<() => void | undefined>(); From 79b574f22465c7e5afbbdb09a322daf40e8f4a91 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 19 Oct 2021 13:52:13 -0500 Subject: [PATCH 155/188] Fix doc stats not showing upon page mount --- .../data_visualizer_stats_table.tsx | 2 - .../full_time_range_selector.tsx | 1 - .../index_data_visualizer_view.tsx | 26 +++++------ .../index_data_visualizer_view_temp.tsx | 13 +++--- .../grid_embeddable/grid_embeddable.tsx | 1 - .../hooks/use_data_visualizer_grid_data.ts | 46 +++++++++---------- .../hooks/use_field_stats.ts | 1 - .../hooks/use_overall_stats.ts | 1 - .../index_data_visualizer.tsx | 2 +- ...ata_visualizer_index_pattern_management.ts | 34 +------------- 10 files changed, 42 insertions(+), 85 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 1c8afe381f987..7858d37bc6a85 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -319,8 +319,6 @@ export const DataVisualizerTable = ({ return getItemIdToExpandedRowMap(itemIds, items); }, [items, expandedRowItemIds, getItemIdToExpandedRowMap]); - console.log('items', items); // Some actions open up fly-out or popup - return ( {(resizeRef) => ( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/full_time_range_selector/full_time_range_selector.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/full_time_range_selector/full_time_range_selector.tsx index 59f00bf00c270..9b121f16132a5 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/full_time_range_selector/full_time_range_selector.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/full_time_range_selector/full_time_range_selector.tsx @@ -6,7 +6,6 @@ */ import React, { FC } from 'react'; - import { FormattedMessage } from '@kbn/i18n/react'; import { Query, IndexPattern, TimefilterContract } from 'src/plugins/data/public'; import { EuiButton } from '@elastic/eui'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 9a0478a4714e4..206ba9aeb766f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -139,20 +139,6 @@ export const IndexDataVisualizerView: FC = (dataVi } }, [dataVisualizerProps?.currentSavedSearch]); - useEffect(() => { - return () => { - // When navigating away from the index pattern - // Reset all previously set filters - // to make sure new page doesn't have unrelated filters - data.query.filterManager.removeAll(); - }; - }, [currentIndexPattern.id, data.query.filterManager]); - - // const timefilter = useTimefilter({ - // timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, - // autoRefreshSelector: true, - // }); - useEffect(() => { if (!currentIndexPattern.isTimeBased()) { toasts.addWarning({ @@ -261,6 +247,18 @@ export const IndexDataVisualizerView: FC = (dataVi setLastRefresh, } = useDataVisualizerGridData(input, dataVisualizerListState, setGlobalState); + useEffect(() => { + // Force refresh on index pattern change + setLastRefresh(Date.now()); + + return () => { + // When navigating away from the index pattern + // Reset all previously set filters + // to make sure new page doesn't have unrelated filters + data.query.filterManager.removeAll(); + }; + }, [currentIndexPattern.id, data.query.filterManager, setLastRefresh]); + useEffect(() => { if (globalState?.time !== undefined) { timefilter.setTime({ diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx index a7066bdb2156b..951bbb3d15101 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx @@ -5,6 +5,7 @@ * 2.0. */ +// @ts-nocheck import React, { FC, Fragment, useEffect, useMemo, useState, useCallback, useRef } from 'react'; import { merge } from 'rxjs'; import { @@ -885,12 +886,12 @@ export const IndexDataVisualizerView: FC = (dataVi > {currentIndexPattern.timeFieldName !== undefined && ( - + {/* */} )} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 33634fc38861f..76607f0f4f29f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -136,7 +136,6 @@ export const EmbeddableWrapper = ({ extendedColumns={extendedColumns} showPreviewByDefault={input?.showPreviewByDefault} onChange={onOutputChange} - progress={progress} />
); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index dd403328dbdc1..2625ed402155b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -37,6 +37,7 @@ import { useFieldStatsSearchStrategy } from './use_field_stats'; import { useOverallStats } from './use_overall_stats'; import { OverallStatsSearchStrategyParams } from '../../../../common/types/field_stats'; import { Dictionary } from '../../common/util/url_state'; +import { AggregatableField, NonAggregatableField } from '../types/overall_stats'; const defaults = getDefaultPageState(); @@ -103,6 +104,7 @@ export const useDataVisualizerGridData = ( currentIndexPattern.id, dataVisualizerListState.searchString, dataVisualizerListState.searchQueryLanguage, + // eslint-disable-next-line react-hooks/exhaustive-deps JSON.stringify({ searchQuery: dataVisualizerListState.searchQuery, currentQuery, @@ -145,23 +147,28 @@ export const useDataVisualizerGridData = ( // (such as the document count chart). Aim for 75 bars. const buckets = _timeBuckets; - const tf = timefilter as any; + const tf = timefilter; if (!buckets || !tf || !currentIndexPattern) return; const activeBounds = tf.getActiveBounds(); + let earliest: number | undefined; let latest: number | undefined; if (activeBounds !== undefined && currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); + earliest = activeBounds.min?.valueOf(); + latest = activeBounds.max?.valueOf(); } const bounds = tf.getActiveBounds(); const BAR_TARGET = 75; buckets.setInterval('auto'); - buckets.setBounds(bounds); - buckets.setBarTarget(BAR_TARGET); + + if (bounds) { + buckets.setBounds(bounds); + buckets.setBarTarget(BAR_TARGET); + } + const aggInterval = buckets.getInterval(); const aggregatableFields: string[] = []; @@ -205,19 +212,6 @@ export const useDataVisualizerGridData = ( ] ); - useEffect(() => console.log('_timeBuckets', _timeBuckets), [_timeBuckets]); - useEffect(() => console.log('timefilter', timefilter), [timefilter]); - useEffect( - () => console.log('currentIndexPattern', currentIndexPattern.id), - [currentIndexPattern.id] - ); - useEffect( - () => console.log('searchQuery', JSON.stringify(searchQuery)), - [JSON.stringify(searchQuery)] - ); - useEffect(() => console.log('samplerShardSize', samplerShardSize), [samplerShardSize]); - useEffect(() => console.log('searchSessionId', searchSessionId), [searchSessionId]); - const configsWithoutStats = useMemo(() => { const existMetricFields: FieldRequestConfig[] = metricConfigs.map((config) => { const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; @@ -277,7 +271,8 @@ export const useDataVisualizerGridData = ( const createMetricCards = useCallback(() => { const configs: FieldVisConfig[] = []; - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; + const aggregatableExistsFields: AggregatableField[] = + overallStats.aggregatableExistsFields || []; const allMetricFields = indexPatternFields.filter((f) => { return ( @@ -297,7 +292,7 @@ export const useDataVisualizerGridData = ( return; } - let aggregatableFields: any[] = overallStats.aggregatableExistsFields; + let aggregatableFields: AggregatableField[] = overallStats.aggregatableExistsFields; if (allMetricFields.length !== metricExistsFields.length && metricsLoaded === true) { aggregatableFields = aggregatableFields.concat(overallStats.aggregatableNotExistsFields); } @@ -342,10 +337,12 @@ export const useDataVisualizerGridData = ( }); // Obtain the list of all non-metric fields which appear in documents // (aggregatable or not aggregatable). - const populatedNonMetricFields: any[] = []; // Kibana index pattern non metric fields. - let nonMetricFieldData: any[] = []; // Basic non metric field data loaded from requesting overall stats. - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; - const nonAggregatableExistsFields: any[] = overallStats.nonAggregatableExistsFields || []; + const populatedNonMetricFields: [] = []; // Kibana index pattern non metric fields. + let nonMetricFieldData: AggregatableField[] = []; // Basic non metric field data loaded from requesting overall stats. + const aggregatableExistsFields: AggregatableField[] = + overallStats.aggregatableExistsFields || []; + const nonAggregatableExistsFields: NonAggregatableField[] = + overallStats.nonAggregatableExistsFields || []; allNonMetricFields.forEach((f) => { const checkAggregatableField = aggregatableExistsFields.find( @@ -462,7 +459,6 @@ export const useDataVisualizerGridData = ( strategyResponse.fieldStats, ]); - useEffect(() => console.log('configs updated', configs), [configs]); // Some actions open up fly-out or popup // This variable is used to keep track of them and clean up when unmounting const actionFlyoutRef = useRef<() => void | undefined>(); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index 744a19077c144..6b7ef471147b3 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -154,7 +154,6 @@ export function useFieldStatsSearchStrategy( }); }, complete: () => { - console.log('useFieldStats called'); setFetchState({ isRunning: false, }); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 5659da6491c84..0c45fe2a142ba 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -221,7 +221,6 @@ export function useOverallStats { - console.log('useOverallStats called', intervalMs); setFetchState({ loaded: 100, isRunning: false, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx index 0d95bdc83d3a8..b90814b753d1e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx @@ -17,7 +17,7 @@ import { getCoreStart, getPluginsStart } from '../../kibana_services'; import { IndexDataVisualizerViewProps, IndexDataVisualizerView, -} from './components/index_data_visualizer_view/index_data_visualizer_view'; +} from './components/index_data_visualizer_view'; import { Accessor, Provider as UrlStateContextProvider, diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_index_pattern_management.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_index_pattern_management.ts index 0d9163a872043..ea59c37e08cc6 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_index_pattern_management.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_index_pattern_management.ts @@ -7,39 +7,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; import { ML_JOB_FIELD_TYPES } from '../../../../../plugins/ml/common/constants/field_types'; -import { FieldVisConfig } from '../../../../../plugins/data_visualizer/public/application/common/components/stats_table/types'; - -interface MetricFieldVisConfig extends FieldVisConfig { - statsMaxDecimalPlaces: number; - docCountFormatted: string; - topValuesCount: number; - viewableInLens: boolean; - hasActionMenu: boolean; -} - -interface NonMetricFieldVisConfig extends FieldVisConfig { - docCountFormatted: string; - exampleCount: number; - viewableInLens: boolean; - hasActionMenu: boolean; -} - -interface TestData { - suiteTitle: string; - sourceIndexOrSavedSearch: string; - rowsPerPage?: 10 | 25 | 50; - newFields?: Array<{ fieldName: string; type: string; script: string }>; - fieldsToRename?: Array<{ originalName: string; newName: string }>; - expected: { - totalDocCountFormatted: string; - metricFields?: MetricFieldVisConfig[]; - nonMetricFields?: NonMetricFieldVisConfig[]; - visibleMetricFieldsCount: number; - totalMetricFieldsCount: number; - populatedFieldsCount: number; - totalFieldsCount: number; - }; -} +import { TestData, MetricFieldVisConfig } from './types'; export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); From db52aca9db5442f17f454bba27bccd80072cf803 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 19 Oct 2021 14:04:58 -0500 Subject: [PATCH 156/188] Fix types --- .../hooks/use_data_visualizer_grid_data.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 2625ed402155b..50ab4e660f1ac 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -26,6 +26,7 @@ import { FieldVisConfig } from '../../common/components/stats_table/types'; import { FieldRequestConfig, JOB_FIELD_TYPES, + JobFieldType, NON_AGGREGATABLE_FIELD_TYPES, OMIT_FIELDS, } from '../../../../common'; @@ -304,9 +305,10 @@ export const useDataVisualizerGridData = ( const fieldData = aggregatableFields.find((f) => { return f.fieldName === field.spec.name; }); + if (!fieldData) return; const metricConfig: FieldVisConfig = { - ...(fieldData ? fieldData : {}), + ...fieldData, fieldFormat: currentIndexPattern.getFormatterForField(field), type: JOB_FIELD_TYPES.NUMBER, loading: true, @@ -337,8 +339,8 @@ export const useDataVisualizerGridData = ( }); // Obtain the list of all non-metric fields which appear in documents // (aggregatable or not aggregatable). - const populatedNonMetricFields: [] = []; // Kibana index pattern non metric fields. - let nonMetricFieldData: AggregatableField[] = []; // Basic non metric field data loaded from requesting overall stats. + const populatedNonMetricFields: DataViewField[] = []; // Kibana index pattern non metric fields. + let nonMetricFieldData: Array = []; // Basic non metric field data loaded from requesting overall stats. const aggregatableExistsFields: AggregatableField[] = overallStats.aggregatableExistsFields || []; const nonAggregatableExistsFields: NonAggregatableField[] = @@ -384,12 +386,11 @@ export const useDataVisualizerGridData = ( nonMetricFieldsToShow.forEach((field) => { const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); - const nonMetricConfig = { + const nonMetricConfig: Partial = { ...(fieldData ? fieldData : {}), fieldFormat: currentIndexPattern.getFormatterForField(field), aggregatable: field.aggregatable, - scripted: field.scripted, - loading: fieldData?.existsInDocs, + loading: fieldData?.existsInDocs ?? true, deletable: field.runtimeField !== undefined, }; @@ -401,7 +402,7 @@ export const useDataVisualizerGridData = ( } else { // Add a flag to indicate that this is one of the 'other' Kibana // field types that do not yet have a specific card type. - nonMetricConfig.type = field.type; + nonMetricConfig.type = field.type as JobFieldType; nonMetricConfig.isUnsupportedType = true; } @@ -409,7 +410,7 @@ export const useDataVisualizerGridData = ( nonMetricConfig.displayName = field.displayName; } - configs.push(nonMetricConfig); + configs.push(nonMetricConfig as FieldVisConfig); }); setNonMetricConfigs(configs); @@ -436,8 +437,6 @@ export const useDataVisualizerGridData = ( } if (fieldStats) { - // @todo - // @ts-ignore combinedConfigs = combinedConfigs.map((c) => { const loadedFullStats = fieldStats.get(c.fieldName) ?? {}; return loadedFullStats From 92887361c5245bfdeede8baa69da32c9b5a246ec Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 19 Oct 2021 14:05:29 -0500 Subject: [PATCH 157/188] Delete old files --- .../index_data_visualizer_view_temp.tsx | 974 ------------------ 1 file changed, 974 deletions(-) delete mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx deleted file mode 100644 index 951bbb3d15101..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view_temp.tsx +++ /dev/null @@ -1,974 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// @ts-nocheck -import React, { FC, Fragment, useEffect, useMemo, useState, useCallback, useRef } from 'react'; -import { merge } from 'rxjs'; -import { - EuiFlexGroup, - EuiFlexItem, - EuiPage, - EuiPageBody, - EuiPageContentBody, - EuiPageContentHeader, - EuiPageContentHeaderSection, - EuiPanel, - EuiSpacer, - EuiTitle, -} from '@elastic/eui'; -import { EuiTableActionsColumnType } from '@elastic/eui/src/components/basic_table/table_types'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { Required } from 'utility-types'; -import { i18n } from '@kbn/i18n'; -import { Filter } from '@kbn/es-query'; -import { - KBN_FIELD_TYPES, - UI_SETTINGS, - Query, - generateFilters, -} from '../../../../../../../../src/plugins/data/public'; -import { FullTimeRangeSelector } from '../full_time_range_selector'; -import { usePageUrlState, useUrlState } from '../../../common/util/url_state'; -import { - DataVisualizerTable, - ItemIdToExpandedRowMap, -} from '../../../common/components/stats_table'; -import { FieldVisConfig } from '../../../common/components/stats_table/types'; -import type { - MetricFieldsStats, - TotalFieldsStats, -} from '../../../common/components/stats_table/components/field_count_stats'; -import { OverallStats } from '../../types/overall_stats'; -import { getActions } from '../../../common/components/field_data_row/action_menu'; -import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; -import { DATA_VISUALIZER_INDEX_VIEWER } from '../../constants/index_data_visualizer_viewer'; -import { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; -import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../../types/combined_query'; -import { - FieldRequestConfig, - JobFieldType, - SavedSearchSavedObject, -} from '../../../../../common/types'; -import { useDataVisualizerKibana } from '../../../kibana_context'; -import { FieldCountPanel } from '../../../common/components/field_count_panel'; -import { DocumentCountContent } from '../../../common/components/document_count_content'; -import { DataLoader } from '../../data_loader/data_loader'; -import { JOB_FIELD_TYPES, OMIT_FIELDS } from '../../../../../common'; -import { useTimefilter } from '../../hooks/use_time_filter'; -import { kbnTypeToJobType } from '../../../common/util/field_types_utils'; -import { SearchPanel } from '../search_panel'; -import { ActionsPanel } from '../actions_panel'; -import { DatePickerWrapper } from '../../../common/components/date_picker_wrapper'; -import { dataVisualizerRefresh$ } from '../../services/timefilter_refresh_service'; -import { HelpMenu } from '../../../common/components/help_menu'; -import { createMergedEsQuery, getEsQueryFromSavedSearch } from '../../utils/saved_search_utils'; -import { DataVisualizerIndexPatternManagement } from '../index_pattern_management'; -import { ResultLink } from '../../../common/components/results_links'; -import { extractErrorProperties } from '../../utils/error_utils'; -import { IndexPatternField, IndexPattern } from '../../../../../../../../src/plugins/data/common'; -import './_index.scss'; -import { TimeBuckets } from '../../../../../common/services/time_buckets'; - -interface DataVisualizerPageState { - overallStats: OverallStats; - metricConfigs: FieldVisConfig[]; - totalMetricFieldCount: number; - populatedMetricFieldCount: number; - metricsLoaded: boolean; - nonMetricConfigs: FieldVisConfig[]; - nonMetricsLoaded: boolean; - documentCountStats?: FieldVisConfig; -} - -const defaultSearchQuery = { - match_all: {}, -}; - -export function getDefaultPageState(): DataVisualizerPageState { - return { - overallStats: { - totalCount: 0, - aggregatableExistsFields: [], - aggregatableNotExistsFields: [], - nonAggregatableExistsFields: [], - nonAggregatableNotExistsFields: [], - }, - metricConfigs: [], - totalMetricFieldCount: 0, - populatedMetricFieldCount: 0, - metricsLoaded: false, - nonMetricConfigs: [], - nonMetricsLoaded: false, - documentCountStats: undefined, - }; -} -export const getDefaultDataVisualizerListState = ( - overrides?: Partial -): Required => ({ - pageIndex: 0, - pageSize: 25, - sortField: 'fieldName', - sortDirection: 'asc', - visibleFieldTypes: [], - visibleFieldNames: [], - samplerShardSize: 5000, - searchString: '', - searchQuery: defaultSearchQuery, - searchQueryLanguage: SEARCH_QUERY_LANGUAGE.KUERY, - filters: [], - showDistributions: true, - showAllFields: false, - showEmptyFields: false, - ...overrides, -}); - -export interface IndexDataVisualizerViewProps { - currentIndexPattern: IndexPattern; - currentSavedSearch: SavedSearchSavedObject | null; - additionalLinks?: ResultLink[]; -} -const restorableDefaults = getDefaultDataVisualizerListState(); - -export const IndexDataVisualizerView: FC = (dataVisualizerProps) => { - const { services } = useDataVisualizerKibana(); - const { docLinks, notifications, uiSettings, data } = services; - const { toasts } = notifications; - - const [dataVisualizerListState, setDataVisualizerListState] = usePageUrlState( - DATA_VISUALIZER_INDEX_VIEWER, - restorableDefaults - ); - const [globalState, setGlobalState] = useUrlState('_g'); - - const [currentSavedSearch, setCurrentSavedSearch] = useState( - dataVisualizerProps.currentSavedSearch - ); - - const { currentIndexPattern, additionalLinks } = dataVisualizerProps; - - useEffect(() => { - if (dataVisualizerProps?.currentSavedSearch !== undefined) { - setCurrentSavedSearch(dataVisualizerProps?.currentSavedSearch); - } - }, [dataVisualizerProps?.currentSavedSearch]); - - useEffect(() => { - return () => { - // When navigating away from the index pattern - // Reset all previously set filters - // to make sure new page doesn't have unrelated filters - data.query.filterManager.removeAll(); - }; - }, [currentIndexPattern.id, data.query.filterManager]); - - const getTimeBuckets = useCallback(() => { - return new TimeBuckets({ - [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), - [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), - dateFormat: uiSettings.get('dateFormat'), - 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), - }); - }, [uiSettings]); - - const timefilter = useTimefilter({ - timeRangeSelector: currentIndexPattern?.timeFieldName !== undefined, - autoRefreshSelector: true, - }); - - const dataLoader = useMemo( - () => new DataLoader(currentIndexPattern, toasts), - [currentIndexPattern, toasts] - ); - - // useEffect(() => { - // if (globalState?.time !== undefined) { - // timefilter.setTime({ - // from: globalState.time.from, - // to: globalState.time.to, - // }); - // setLastRefresh(Date.now()); - // } - // }, [globalState, timefilter]); - - // useEffect(() => { - // if (globalState?.refreshInterval !== undefined) { - // timefilter.setRefreshInterval(globalState.refreshInterval); - // setLastRefresh(Date.now()); - // } - // }, [globalState, timefilter]); - - const [lastRefresh, setLastRefresh] = useState(0); - - useEffect(() => { - if (!currentIndexPattern.isTimeBased()) { - toasts.addWarning({ - title: i18n.translate( - 'xpack.dataVisualizer.index.indexPatternNotBasedOnTimeSeriesNotificationTitle', - { - defaultMessage: 'The index pattern {indexPatternTitle} is not based on a time series', - values: { indexPatternTitle: currentIndexPattern.title }, - } - ), - text: i18n.translate( - 'xpack.dataVisualizer.index.indexPatternNotBasedOnTimeSeriesNotificationDescription', - { - defaultMessage: 'Anomaly detection only runs over time-based indices', - } - ), - }); - } - }, [currentIndexPattern, toasts]); - - const indexPatternFields: IndexPatternField[] = currentIndexPattern.fields; - - const fieldTypes = useMemo(() => { - // Obtain the list of non metric field types which appear in the index pattern. - const indexedFieldTypes: JobFieldType[] = []; - indexPatternFields.forEach((field) => { - if (!OMIT_FIELDS.includes(field.name) && field.scripted !== true) { - const dataVisualizerType: JobFieldType | undefined = kbnTypeToJobType(field); - if (dataVisualizerType !== undefined && !indexedFieldTypes.includes(dataVisualizerType)) { - indexedFieldTypes.push(dataVisualizerType); - } - } - }); - return indexedFieldTypes.sort(); - }, [indexPatternFields]); - - const defaults = getDefaultPageState(); - - const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { - const searchData = getEsQueryFromSavedSearch({ - indexPattern: currentIndexPattern, - uiSettings, - savedSearch: currentSavedSearch, - filterManager: data.query.filterManager, - }); - - if (searchData === undefined || dataVisualizerListState.searchString !== '') { - if (dataVisualizerListState.filters) { - data.query.filterManager.setFilters(dataVisualizerListState.filters); - } - return { - searchQuery: dataVisualizerListState.searchQuery, - searchString: dataVisualizerListState.searchString, - searchQueryLanguage: dataVisualizerListState.searchQueryLanguage, - }; - } else { - return { - searchQuery: searchData.searchQuery, - searchString: searchData.searchString, - searchQueryLanguage: searchData.queryLanguage, - }; - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [currentSavedSearch, currentIndexPattern, dataVisualizerListState, data.query]); - - const setSearchParams = useCallback( - (searchParams: { - searchQuery: Query['query']; - searchString: Query['query']; - queryLanguage: SearchQueryLanguage; - filters: Filter[]; - }) => { - // When the user loads saved search and then clear or modify the query - // we should remove the saved search and replace it with the index pattern id - if (currentSavedSearch !== null) { - setCurrentSavedSearch(null); - } - - setDataVisualizerListState({ - ...dataVisualizerListState, - searchQuery: searchParams.searchQuery, - searchString: searchParams.searchString, - searchQueryLanguage: searchParams.queryLanguage, - filters: searchParams.filters, - }); - }, - [currentSavedSearch, dataVisualizerListState, setDataVisualizerListState] - ); - - const samplerShardSize = - dataVisualizerListState.samplerShardSize ?? restorableDefaults.samplerShardSize; - const setSamplerShardSize = (value: number) => { - setDataVisualizerListState({ ...dataVisualizerListState, samplerShardSize: value }); - }; - - const visibleFieldTypes = - dataVisualizerListState.visibleFieldTypes ?? restorableDefaults.visibleFieldTypes; - const setVisibleFieldTypes = (values: string[]) => { - setDataVisualizerListState({ ...dataVisualizerListState, visibleFieldTypes: values }); - }; - - const visibleFieldNames = - dataVisualizerListState.visibleFieldNames ?? restorableDefaults.visibleFieldNames; - const setVisibleFieldNames = (values: string[]) => { - setDataVisualizerListState({ ...dataVisualizerListState, visibleFieldNames: values }); - }; - - const showEmptyFields = - dataVisualizerListState.showEmptyFields ?? restorableDefaults.showEmptyFields; - const toggleShowEmptyFields = () => { - setDataVisualizerListState({ - ...dataVisualizerListState, - showEmptyFields: !dataVisualizerListState.showEmptyFields, - }); - }; - - const [overallStats, setOverallStats] = useState(defaults.overallStats); - - const [documentCountStats, setDocumentCountStats] = useState(defaults.documentCountStats); - const [metricConfigs, setMetricConfigs] = useState(defaults.metricConfigs); - const [metricsLoaded, setMetricsLoaded] = useState(defaults.metricsLoaded); - const [metricsStats, setMetricsStats] = useState(); - - const [nonMetricConfigs, setNonMetricConfigs] = useState(defaults.nonMetricConfigs); - const [nonMetricsLoaded, setNonMetricsLoaded] = useState(defaults.nonMetricsLoaded); - - const onAddFilter = useCallback( - (field: IndexPatternField | string, values: string, operation: '+' | '-') => { - const newFilters = generateFilters( - data.query.filterManager, - field, - values, - operation, - String(currentIndexPattern.id) - ); - if (newFilters) { - data.query.filterManager.addFilters(newFilters); - } - - // Merge current query with new filters - const mergedQuery = { - query: searchString || '', - language: searchQueryLanguage, - }; - - const combinedQuery = createMergedEsQuery( - { - query: searchString || '', - language: searchQueryLanguage, - }, - data.query.filterManager.getFilters() ?? [], - currentIndexPattern, - uiSettings - ); - - setSearchParams({ - searchQuery: combinedQuery, - searchString: mergedQuery.query, - queryLanguage: mergedQuery.language as SearchQueryLanguage, - filters: data.query.filterManager.getFilters(), - }); - }, - [ - currentIndexPattern, - data.query.filterManager, - searchQueryLanguage, - searchString, - setSearchParams, - uiSettings, - ] - ); - - useEffect(() => { - loadOverallStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchQuery, samplerShardSize, lastRefresh]); - - useEffect(() => { - createMetricCards(); - createNonMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [overallStats, showEmptyFields]); - - useEffect(() => { - loadMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricConfigs]); - - useEffect(() => { - loadNonMetricFieldStats(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricConfigs]); - - useEffect(() => { - createMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [metricsLoaded]); - - useEffect(() => { - createNonMetricCards(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [nonMetricsLoaded]); - - async function loadOverallStats() { - const tf = timefilter as any; - let earliest; - let latest; - - const activeBounds = tf.getActiveBounds(); - - if (currentIndexPattern.timeFieldName !== undefined && activeBounds === undefined) { - return; - } - - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = activeBounds.min.valueOf(); - latest = activeBounds.max.valueOf(); - } - - try { - const allStats = await dataLoader.loadOverallData( - searchQuery, - samplerShardSize, - earliest, - latest - ); - // Because load overall stats perform queries in batches - // there could be multiple errors - if (Array.isArray(allStats.errors) && allStats.errors.length > 0) { - allStats.errors.forEach((err: any) => { - dataLoader.displayError(extractErrorProperties(err)); - }); - } - setOverallStats(allStats); - } catch (err) { - dataLoader.displayError(err.body ?? err); - } - } - - async function loadMetricFieldStats() { - // Only request data for fields that exist in documents. - if (metricConfigs.length === 0) { - return; - } - - const configsToLoad = metricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - // Obtain the interval to use for date histogram aggregations - // (such as the document count chart). Aim for 75 bars. - const buckets = getTimeBuckets(); - - const tf = timefilter as any; - let earliest: number | undefined; - let latest: number | undefined; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - const bounds = tf.getActiveBounds(); - const BAR_TARGET = 75; - buckets.setInterval('auto'); - buckets.setBounds(bounds); - buckets.setBarTarget(BAR_TARGET); - const aggInterval = buckets.getInterval(); - - try { - const metricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existMetricFields, - aggInterval.asMilliseconds() - ); - - // Add the metric stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - metricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - configWithStats.loading = false; - configs.push(configWithStats); - } else { - // Document count card. - configWithStats.stats = metricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === undefined - ); - - if (configWithStats.stats !== undefined) { - // Add earliest / latest of timefilter for setting x axis domain. - configWithStats.stats.timeRangeEarliest = earliest; - configWithStats.stats.timeRangeLatest = latest; - } - setDocumentCountStats(configWithStats); - } - }); - - setMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - async function loadNonMetricFieldStats() { - // Only request data for fields that exist in documents. - if (nonMetricConfigs.length === 0) { - return; - } - - const configsToLoad = nonMetricConfigs.filter( - (config) => config.existsInDocs === true && config.loading === true - ); - if (configsToLoad.length === 0) { - return; - } - - // Pass the field name, type and cardinality in the request. - // Top values will be obtained on a sample if cardinality > 100000. - const existNonMetricFields: FieldRequestConfig[] = configsToLoad.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); - - const tf = timefilter as any; - let earliest; - let latest; - if (currentIndexPattern.timeFieldName !== undefined) { - earliest = tf.getActiveBounds().min.valueOf(); - latest = tf.getActiveBounds().max.valueOf(); - } - - try { - const nonMetricFieldStats = await dataLoader.loadFieldStats( - searchQuery, - samplerShardSize, - earliest, - latest, - existNonMetricFields - ); - - // Add the field stats to the existing stats in the corresponding config. - const configs: FieldVisConfig[] = []; - nonMetricConfigs.forEach((config) => { - const configWithStats = { ...config }; - if (config.fieldName !== undefined) { - configWithStats.stats = { - ...configWithStats.stats, - ...nonMetricFieldStats.find( - (fieldStats: any) => fieldStats.fieldName === config.fieldName - ), - }; - } - configWithStats.loading = false; - configs.push(configWithStats); - }); - - setNonMetricConfigs(configs); - } catch (err) { - dataLoader.displayError(err); - } - } - - const createMetricCards = useCallback(() => { - const configs: FieldVisConfig[] = []; - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; - - const allMetricFields = indexPatternFields.filter((f) => { - return ( - f.type === KBN_FIELD_TYPES.NUMBER && - f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true - ); - }); - const metricExistsFields = allMetricFields.filter((f) => { - return aggregatableExistsFields.find((existsF) => { - return existsF.fieldName === f.spec.name; - }); - }); - - // @todo: REMOVE - // Add a config for 'document count', identified by no field name if indexpattern is time based. - if (currentIndexPattern.timeFieldName !== undefined) { - configs.push({ - type: JOB_FIELD_TYPES.NUMBER, - existsInDocs: true, - loading: true, - aggregatable: true, - }); - } - - if (metricsLoaded === false) { - setMetricsLoaded(true); - return; - } - - let aggregatableFields: any[] = overallStats.aggregatableExistsFields; - if (allMetricFields.length !== metricExistsFields.length && metricsLoaded === true) { - aggregatableFields = aggregatableFields.concat(overallStats.aggregatableNotExistsFields); - } - - const metricFieldsToShow = - metricsLoaded === true && showEmptyFields === true ? allMetricFields : metricExistsFields; - - metricFieldsToShow.forEach((field) => { - const fieldData = aggregatableFields.find((f) => { - return f.fieldName === field.spec.name; - }); - - const metricConfig: FieldVisConfig = { - ...(fieldData ? fieldData : {}), - fieldFormat: currentIndexPattern.getFormatterForField(field), - type: JOB_FIELD_TYPES.NUMBER, - loading: true, - aggregatable: true, - deletable: field.runtimeField !== undefined, - }; - if (field.displayName !== metricConfig.fieldName) { - metricConfig.displayName = field.displayName; - } - - configs.push(metricConfig); - }); - - setMetricsStats({ - totalMetricFieldsCount: allMetricFields.length, - visibleMetricsCount: metricFieldsToShow.length, - }); - setMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - metricsLoaded, - overallStats, - showEmptyFields, - ]); - - const createNonMetricCards = useCallback(() => { - const allNonMetricFields = indexPatternFields.filter((f) => { - return ( - f.type !== KBN_FIELD_TYPES.NUMBER && - f.displayName !== undefined && - dataLoader.isDisplayField(f.displayName) === true - ); - }); - // Obtain the list of all non-metric fields which appear in documents - // (aggregatable or not aggregatable). - const populatedNonMetricFields: any[] = []; // Kibana index pattern non metric fields. - let nonMetricFieldData: any[] = []; // Basic non metric field data loaded from requesting overall stats. - const aggregatableExistsFields: any[] = overallStats.aggregatableExistsFields || []; - const nonAggregatableExistsFields: any[] = overallStats.nonAggregatableExistsFields || []; - - allNonMetricFields.forEach((f) => { - const checkAggregatableField = aggregatableExistsFields.find( - (existsField) => existsField.fieldName === f.spec.name - ); - - if (checkAggregatableField !== undefined) { - populatedNonMetricFields.push(f); - nonMetricFieldData.push(checkAggregatableField); - } else { - const checkNonAggregatableField = nonAggregatableExistsFields.find( - (existsField) => existsField.fieldName === f.spec.name - ); - - if (checkNonAggregatableField !== undefined) { - populatedNonMetricFields.push(f); - nonMetricFieldData.push(checkNonAggregatableField); - } - } - }); - - if (nonMetricsLoaded === false) { - setNonMetricsLoaded(true); - return; - } - - if (allNonMetricFields.length !== nonMetricFieldData.length && showEmptyFields === true) { - // Combine the field data obtained from Elasticsearch into a single array. - nonMetricFieldData = nonMetricFieldData.concat( - overallStats.aggregatableNotExistsFields, - overallStats.nonAggregatableNotExistsFields - ); - } - - const nonMetricFieldsToShow = showEmptyFields ? allNonMetricFields : populatedNonMetricFields; - - const configs: FieldVisConfig[] = []; - - nonMetricFieldsToShow.forEach((field) => { - const fieldData = nonMetricFieldData.find((f) => f.fieldName === field.spec.name); - - const nonMetricConfig = { - ...(fieldData ? fieldData : {}), - fieldFormat: currentIndexPattern.getFormatterForField(field), - aggregatable: field.aggregatable, - scripted: field.scripted, - loading: fieldData?.existsInDocs, - deletable: field.runtimeField !== undefined, - }; - - // Map the field type from the Kibana index pattern to the field type - // used in the data visualizer. - const dataVisualizerType = kbnTypeToJobType(field); - if (dataVisualizerType !== undefined) { - nonMetricConfig.type = dataVisualizerType; - } else { - // Add a flag to indicate that this is one of the 'other' Kibana - // field types that do not yet have a specific card type. - nonMetricConfig.type = field.type; - nonMetricConfig.isUnsupportedType = true; - } - - if (field.displayName !== nonMetricConfig.fieldName) { - nonMetricConfig.displayName = field.displayName; - } - - configs.push(nonMetricConfig); - }); - - setNonMetricConfigs(configs); - }, [ - currentIndexPattern, - dataLoader, - indexPatternFields, - nonMetricsLoaded, - overallStats, - showEmptyFields, - ]); - - const wizardPanelWidth = '280px'; - - const configs = useMemo(() => { - let combinedConfigs = [...nonMetricConfigs, ...metricConfigs]; - if (visibleFieldTypes && visibleFieldTypes.length > 0) { - combinedConfigs = combinedConfigs.filter( - (config) => visibleFieldTypes.findIndex((field) => field === config.type) > -1 - ); - } - if (visibleFieldNames && visibleFieldNames.length > 0) { - combinedConfigs = combinedConfigs.filter( - (config) => visibleFieldNames.findIndex((field) => field === config.fieldName) > -1 - ); - } - - return combinedConfigs; - }, [nonMetricConfigs, metricConfigs, visibleFieldTypes, visibleFieldNames]); - - const fieldsCountStats: TotalFieldsStats | undefined = useMemo(() => { - let _visibleFieldsCount = 0; - let _totalFieldsCount = 0; - Object.keys(overallStats).forEach((key) => { - const fieldsGroup = overallStats[key as keyof OverallStats]; - if (Array.isArray(fieldsGroup) && fieldsGroup.length > 0) { - _totalFieldsCount += fieldsGroup.length; - } - }); - - if (showEmptyFields === true) { - _visibleFieldsCount = _totalFieldsCount; - } else { - _visibleFieldsCount = - overallStats.aggregatableExistsFields.length + - overallStats.nonAggregatableExistsFields.length; - } - return { visibleFieldsCount: _visibleFieldsCount, totalFieldsCount: _totalFieldsCount }; - }, [overallStats, showEmptyFields]); - - const getItemIdToExpandedRowMap = useCallback( - function (itemIds: string[], items: FieldVisConfig[]): ItemIdToExpandedRowMap { - return itemIds.reduce((m: ItemIdToExpandedRowMap, fieldName: string) => { - const item = items.find((fieldVisConfig) => fieldVisConfig.fieldName === fieldName); - if (item !== undefined) { - m[fieldName] = ( - - ); - } - return m; - }, {} as ItemIdToExpandedRowMap); - }, - [currentIndexPattern, searchQueryLanguage, searchString, onAddFilter] - ); - - // Some actions open up fly-out or popup - // This variable is used to keep track of them and clean up when unmounting - const actionFlyoutRef = useRef<() => void | undefined>(); - useEffect(() => { - const ref = actionFlyoutRef; - return () => { - // Clean up any of the flyout/editor opened from the actions - if (ref.current) { - ref.current(); - } - }; - }, []); - - // Inject custom action column for the index based visualizer - // Hide the column completely if no access to any of the plugins - const extendedColumns = useMemo(() => { - const actions = getActions( - currentIndexPattern, - services, - { - searchQueryLanguage, - searchString, - }, - actionFlyoutRef - ); - if (!Array.isArray(actions) || actions.length < 1) return; - - const actionColumn: EuiTableActionsColumnType = { - name: ( - - ), - actions, - width: '100px', - }; - - return [actionColumn]; - }, [currentIndexPattern, services, searchQueryLanguage, searchString]); - - const helpLink = docLinks.links.ml.guide; - - return ( - - - - - - - -
- -

{currentIndexPattern.title}

- - -
- - - - {currentIndexPattern.timeFieldName !== undefined && ( - - {/* */} - - )} - - - - - - - - - - - - - {overallStats?.totalCount !== undefined && ( - - - - )} - - - - - - items={configs} - pageState={dataVisualizerListState} - updatePageState={setDataVisualizerListState} - getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} - extendedColumns={extendedColumns} - /> - - - - - - - - - - - - - ); -}; From 9e6a342e496a52045898a63cd2a654cba2799fea Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 19 Oct 2021 14:30:43 -0500 Subject: [PATCH 158/188] Update tests & i18n --- .../data_visualizer_stats_table.tsx | 1 - .../models/data_visualizer/get_fields_stats.ts | 8 ++++---- ...data_visualizer_index_pattern_management.ts | 18 +++++++++++++++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 7858d37bc6a85..b4608006dfd83 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -65,7 +65,6 @@ export const DataVisualizerTable = ({ extendedColumns, showPreviewByDefault, onChange, - progress, }: DataVisualizerTableProps) => { const [expandedRowItemIds, setExpandedRowItemIds] = useState([]); const [expandAll, setExpandAll] = useState(false); diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts index ec5ff2f62df44..402e3de9c7185 100644 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts +++ b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts @@ -86,10 +86,10 @@ export const getDocumentCountStats = async ( }); return { - documentCounts: { - interval: intervalMs, - buckets, - }, + interval: intervalMs, + buckets, + timeRangeEarliest: earliestMs ?? 0, + timeRangeLatest: latestMs ?? 0, }; }; diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_index_pattern_management.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_index_pattern_management.ts index ea59c37e08cc6..a98840082d102 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_index_pattern_management.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_index_pattern_management.ts @@ -7,7 +7,23 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; import { ML_JOB_FIELD_TYPES } from '../../../../../plugins/ml/common/constants/field_types'; -import { TestData, MetricFieldVisConfig } from './types'; +import { MetricFieldVisConfig, NonMetricFieldVisConfig } from './types'; +interface TestData { + suiteTitle: string; + sourceIndexOrSavedSearch: string; + rowsPerPage?: 10 | 25 | 50; + newFields?: Array<{ fieldName: string; type: string; script: string }>; + fieldsToRename?: Array<{ originalName: string; newName: string }>; + expected: { + totalDocCountFormatted: string; + metricFields?: MetricFieldVisConfig[]; + nonMetricFields?: NonMetricFieldVisConfig[]; + visibleMetricFieldsCount: number; + totalMetricFieldsCount: number; + populatedFieldsCount: number; + totalFieldsCount: number; + }; +} export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); From 1e4a19e1a4a27719179f5cbe4857cd1f4d0dc8fe Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 20 Oct 2021 10:55:09 -0500 Subject: [PATCH 159/188] Fix examples, tests --- .../apm/public/hooks/use_search_strategy.ts | 1 - .../hooks/use_field_stats.ts | 2 ++ .../requests/get_field_examples.ts | 3 ++- .../apis/ml/data_visualizer/get_field_stats.ts | 18 +++++++++--------- .../apps/ml/data_visualizer/types.ts | 2 ++ 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/apm/public/hooks/use_search_strategy.ts b/x-pack/plugins/apm/public/hooks/use_search_strategy.ts index 72af0e145231b..275eddb68ae00 100644 --- a/x-pack/plugins/apm/public/hooks/use_search_strategy.ts +++ b/x-pack/plugins/apm/public/hooks/use_search_strategy.ts @@ -173,7 +173,6 @@ export function useSearchStrategy< error: response as unknown as Error, isRunning: false, }); - } else { } }, error: (error: Error) => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index 6b7ef471147b3..93e3d16886064 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -24,6 +24,7 @@ import { import { getFieldStats } from '../search_strategy/requests/get_field_stats'; import type { FieldStats, FieldStatsError } from '../../../../common/types/field_stats'; import { getInitialProgress, getReducer } from '../progress_utils'; +import { MAX_EXAMPLES_DEFAULT } from '../search_strategy/requests/constants'; interface FieldStatsParams { metricConfigs: FieldRequestConfig[]; @@ -103,6 +104,7 @@ export function useFieldStatsSearchStrategy( filter: filterCriteria, }, }, + maxExamples: MAX_EXAMPLES_DEFAULT, }; const searchOptions = { abortSignal: abortCtrl.current.signal, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts index 902cafb67a9c1..8c608b3775b3a 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts @@ -84,7 +84,8 @@ export const fetchFieldExamples = ( fieldName: field.fieldName, examples: [] as any[], } as FieldExamples; - if (body.hits.total.value > 0) { + + if (body.hits.total > 0) { const hits = body.hits.hits; for (let i = 0; i < hits.length; i++) { // Use lodash get() to support field names containing dots. diff --git a/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_stats.ts b/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_stats.ts index 65fd1c1ff0c85..9e4343ea29a1b 100644 --- a/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_stats.ts +++ b/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_stats.ts @@ -42,16 +42,16 @@ export default ({ getService }: FtrProviderContext) => { responseCode: 200, responseBody: [ { - documentCounts: { - interval: 86400000, - buckets: { - '1454803200000': 846, - '1454889600000': 846, - '1454976000000': 859, - '1455062400000': 851, - '1455148800000': 858, - }, + interval: 86400000, + buckets: { + '1454803200000': 846, + '1454889600000': 846, + '1454976000000': 859, + '1455062400000': 851, + '1455148800000': 858, }, + timeRangeEarliest: 0, + timeRangeLatest: 0, }, { // Cannot verify median and percentiles responses as the ES percentiles agg is non-deterministic. diff --git a/x-pack/test/functional/apps/ml/data_visualizer/types.ts b/x-pack/test/functional/apps/ml/data_visualizer/types.ts index 5c3f890dba561..ba0077fed3661 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/types.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/types.ts @@ -12,6 +12,7 @@ export interface MetricFieldVisConfig extends FieldVisConfig { docCountFormatted: string; topValuesCount: number; viewableInLens: boolean; + hasActionMenu?: boolean; } export interface NonMetricFieldVisConfig extends FieldVisConfig { @@ -19,6 +20,7 @@ export interface NonMetricFieldVisConfig extends FieldVisConfig { exampleCount: number; exampleContent?: string[]; viewableInLens: boolean; + hasActionMenu?: boolean; } export interface TestData { From 1a9469860d8629edeb265c21918dcd2e5e23b5b7 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 20 Oct 2021 12:22:41 -0500 Subject: [PATCH 160/188] Remove old files & routes --- .../data_loader/data_loader.ts | 143 ----- .../data_visualizer/check_fields_exist.ts | 185 ------- .../models/data_visualizer/constants.ts | 16 - .../models/data_visualizer/data_visualizer.ts | 489 ------------------ .../data_visualizer/get_field_examples.ts | 82 --- .../data_visualizer/get_fields_stats.ts | 479 ----------------- .../get_histogram_for_fields.ts | 191 ------- .../server/models/data_visualizer/index.ts | 8 - .../process_distribution_data.ts | 109 ---- .../plugins/data_visualizer/server/plugin.ts | 3 - .../data_visualizer/server/routes/index.ts | 8 - .../data_visualizer/server/routes/routes.ts | 263 ---------- .../server/routes/schemas/index.ts | 8 - .../schemas/index_data_visualizer_schemas.ts | 76 --- .../ml/data_visualizer/get_field_stats.ts | 234 --------- .../ml/data_visualizer/get_overall_stats.ts | 153 ------ 16 files changed, 2447 deletions(-) delete mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts delete mode 100644 x-pack/plugins/data_visualizer/server/models/data_visualizer/check_fields_exist.ts delete mode 100644 x-pack/plugins/data_visualizer/server/models/data_visualizer/constants.ts delete mode 100644 x-pack/plugins/data_visualizer/server/models/data_visualizer/data_visualizer.ts delete mode 100644 x-pack/plugins/data_visualizer/server/models/data_visualizer/get_field_examples.ts delete mode 100644 x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts delete mode 100644 x-pack/plugins/data_visualizer/server/models/data_visualizer/get_histogram_for_fields.ts delete mode 100644 x-pack/plugins/data_visualizer/server/models/data_visualizer/index.ts delete mode 100644 x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts delete mode 100644 x-pack/plugins/data_visualizer/server/routes/index.ts delete mode 100644 x-pack/plugins/data_visualizer/server/routes/routes.ts delete mode 100644 x-pack/plugins/data_visualizer/server/routes/schemas/index.ts delete mode 100644 x-pack/plugins/data_visualizer/server/routes/schemas/index_data_visualizer_schemas.ts delete mode 100644 x-pack/test/api_integration/apis/ml/data_visualizer/get_field_stats.ts delete mode 100644 x-pack/test/api_integration/apis/ml/data_visualizer/get_overall_stats.ts diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts deleted file mode 100644 index 816f058964de3..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/data_loader/data_loader.ts +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// Maximum number of examples to obtain for text type fields. -import { CoreSetup } from 'kibana/public'; -import { estypes } from '@elastic/elasticsearch'; -import { i18n } from '@kbn/i18n'; -import { IndexPattern } from '../../../../../../../src/plugins/data/common'; -import { NON_AGGREGATABLE_FIELD_TYPES, OMIT_FIELDS } from '../../../../common/constants'; -import { FieldRequestConfig } from '../../../../common/types'; -import { getVisualizerFieldStats, getVisualizerOverallStats } from '../services/visualizer_stats'; - -type IndexPatternTitle = string; -type SavedSearchQuery = Record | null | undefined; - -const MAX_EXAMPLES_DEFAULT: number = 10; - -// @TODO: remove -export class DataLoader { - private _indexPattern: IndexPattern; - private _runtimeMappings: estypes.MappingRuntimeFields; - private _indexPatternTitle: IndexPatternTitle = ''; - private _maxExamples: number = MAX_EXAMPLES_DEFAULT; - private _toastNotifications: CoreSetup['notifications']['toasts']; - - constructor( - indexPattern: IndexPattern, - toastNotifications: CoreSetup['notifications']['toasts'] - ) { - this._indexPattern = indexPattern; - this._runtimeMappings = this._indexPattern.getComputedFields() - .runtimeFields as estypes.MappingRuntimeFields; - this._indexPatternTitle = indexPattern.title; - this._toastNotifications = toastNotifications; - } - - async loadOverallData( - query: string | SavedSearchQuery, - samplerShardSize: number, - earliest: number | undefined, - latest: number | undefined - ): Promise { - const aggregatableFields: string[] = []; - const nonAggregatableFields: string[] = []; - this._indexPattern.fields.forEach((field) => { - const fieldName = field.displayName !== undefined ? field.displayName : field.name; - if (this.isDisplayField(fieldName) === true) { - if (field.aggregatable === true && !NON_AGGREGATABLE_FIELD_TYPES.has(field.type)) { - aggregatableFields.push(field.name); - } else { - nonAggregatableFields.push(field.name); - } - } - }); - - // Need to find: - // 1. List of aggregatable fields that do exist in docs - // 2. List of aggregatable fields that do not exist in docs - // 3. List of non-aggregatable fields that do exist in docs. - // 4. List of non-aggregatable fields that do not exist in docs. - const stats = await getVisualizerOverallStats({ - indexPatternTitle: this._indexPatternTitle, - query, - timeFieldName: this._indexPattern.timeFieldName, - samplerShardSize, - earliest, - latest, - aggregatableFields, - nonAggregatableFields, - runtimeMappings: this._runtimeMappings, - }); - - return stats; - } - - async loadFieldStats( - query: string | SavedSearchQuery, - samplerShardSize: number, - earliest: number | undefined, - latest: number | undefined, - fields: FieldRequestConfig[], - interval?: number - ): Promise { - const stats = await getVisualizerFieldStats({ - indexPatternTitle: this._indexPatternTitle, - query, - timeFieldName: this._indexPattern.timeFieldName, - earliest, - latest, - samplerShardSize, - interval, - fields, - maxExamples: this._maxExamples, - runtimeMappings: this._runtimeMappings, - }); - - return stats; - } - - displayError(err: any) { - if (err.statusCode === 500) { - this._toastNotifications.addError(err, { - title: i18n.translate('xpack.dataVisualizer.index.dataLoader.internalServerErrorMessage', { - defaultMessage: - 'Error loading data in index {index}. {message}. ' + - 'The request may have timed out. Try using a smaller sample size or narrowing the time range.', - values: { - index: this._indexPattern.title, - message: err.error ?? err.message, - }, - }), - }); - } else { - this._toastNotifications.addError(err, { - title: i18n.translate('xpack.dataVisualizer.index.errorLoadingDataMessage', { - defaultMessage: 'Error loading data in index {index}. {message}.', - values: { - index: this._indexPattern.title, - message: err.error ?? err.message, - }, - }), - }); - } - } - - public set maxExamples(max: number) { - this._maxExamples = max; - } - - public get maxExamples(): number { - return this._maxExamples; - } - - // Returns whether the field with the specified name should be displayed, - // as certain fields such as _id and _source should be omitted from the view. - public isDisplayField(fieldName: string): boolean { - return !OMIT_FIELDS.includes(fieldName); - } -} diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/check_fields_exist.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/check_fields_exist.ts deleted file mode 100644 index 02f62c3b3f296..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/check_fields_exist.ts +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { estypes } from '@elastic/elasticsearch'; -import { get } from 'lodash'; -import { IScopedClusterClient } from 'kibana/server'; -import { AggCardinality, Aggs, FieldData } from '../../../common/types/field_stats'; -import { - buildBaseFilterCriteria, - buildSamplerAggregation, - getSafeAggregationName, - getSamplerAggregationsResponsePath, -} from '../../../common/utils/query_utils'; -import { getDatafeedAggregations } from '../../../common/utils/datafeed_utils'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; - -export const checkAggregatableFieldsExist = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - aggregatableFields: string[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs?: number, - latestMs?: number, - datafeedConfig?: estypes.MlDatafeed, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - const datafeedAggregations = getDatafeedAggregations(datafeedConfig); - - // Value count aggregation faster way of checking if field exists than using - // filter aggregation with exists query. - const aggs: Aggs = datafeedAggregations !== undefined ? { ...datafeedAggregations } : {}; - - // Combine runtime fields from the index pattern as well as the datafeed - const combinedRuntimeMappings: estypes.MappingRuntimeFields = { - ...(isPopulatedObject(runtimeMappings) ? runtimeMappings : {}), - ...(isPopulatedObject(datafeedConfig) && isPopulatedObject(datafeedConfig.runtime_mappings) - ? datafeedConfig.runtime_mappings - : {}), - }; - - aggregatableFields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field, i); - aggs[`${safeFieldName}_count`] = { - filter: { exists: { field } }, - }; - - let cardinalityField: AggCardinality; - if (datafeedConfig?.script_fields?.hasOwnProperty(field)) { - cardinalityField = aggs[`${safeFieldName}_cardinality`] = { - cardinality: { script: datafeedConfig?.script_fields[field].script }, - }; - } else { - cardinalityField = { - cardinality: { field }, - }; - } - aggs[`${safeFieldName}_cardinality`] = cardinalityField; - }); - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - ...(isPopulatedObject(aggs) ? { aggs: buildSamplerAggregation(aggs, samplerShardSize) } : {}), - ...(isPopulatedObject(combinedRuntimeMappings) - ? { runtime_mappings: combinedRuntimeMappings } - : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - track_total_hits: true, - size, - body: searchBody, - }); - - const aggregations = body.aggregations; - // @ts-expect-error incorrect search response type - const totalCount = body.hits.total.value; - const stats = { - totalCount, - aggregatableExistsFields: [] as FieldData[], - aggregatableNotExistsFields: [] as FieldData[], - }; - - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const sampleCount = - samplerShardSize > 0 ? get(aggregations, ['sample', 'doc_count'], 0) : totalCount; - aggregatableFields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field, i); - const count = get(aggregations, [...aggsPath, `${safeFieldName}_count`, 'doc_count'], 0); - if (count > 0) { - const cardinality = get( - aggregations, - [...aggsPath, `${safeFieldName}_cardinality`, 'value'], - 0 - ); - stats.aggregatableExistsFields.push({ - fieldName: field, - existsInDocs: true, - stats: { - sampleCount, - count, - cardinality, - }, - }); - } else { - if ( - datafeedConfig?.script_fields?.hasOwnProperty(field) || - datafeedConfig?.runtime_mappings?.hasOwnProperty(field) - ) { - const cardinality = get( - aggregations, - [...aggsPath, `${safeFieldName}_cardinality`, 'value'], - 0 - ); - stats.aggregatableExistsFields.push({ - fieldName: field, - existsInDocs: true, - stats: { - sampleCount, - count, - cardinality, - }, - }); - } else { - stats.aggregatableNotExistsFields.push({ - fieldName: field, - existsInDocs: false, - }); - } - } - }); - - return stats; -}; - -export const checkNonAggregatableFieldExists = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - field: string, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - if (Array.isArray(filterCriteria)) { - filterCriteria.push({ exists: { field } }); - } - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - // @ts-expect-error incorrect search response type - return body.hits.total.value > 0; -}; diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/constants.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/constants.ts deleted file mode 100644 index 4dbe51e5c0838..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/constants.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const SAMPLER_TOP_TERMS_THRESHOLD = 100000; -export const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000; -export const AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE = 200; -export const FIELDS_REQUEST_BATCH_SIZE = 10; - -export const MAX_CHART_COLUMNS = 20; - -export const MAX_PERCENT = 100; -export const PERCENTILE_SPACING = 5; diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/data_visualizer.ts deleted file mode 100644 index 7c2ef2cd49551..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/data_visualizer.ts +++ /dev/null @@ -1,489 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IScopedClusterClient } from 'kibana/server'; -import { each, last } from 'lodash'; -import { estypes } from '@elastic/elasticsearch'; -import { JOB_FIELD_TYPES } from '../../../common'; -import type { - BatchStats, - FieldData, - HistogramField, - Field, - DocumentCountStats, - FieldExamples, -} from '../../../common/types/field_stats'; -import { getHistogramsForFields } from './get_histogram_for_fields'; -import { - checkAggregatableFieldsExist, - checkNonAggregatableFieldExists, -} from './check_fields_exist'; -import { AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE, FIELDS_REQUEST_BATCH_SIZE } from './constants'; -import { getFieldExamples } from './get_field_examples'; -import { - getBooleanFieldsStats, - getDateFieldsStats, - getDocumentCountStats, - getNumericFieldsStats, - getStringFieldsStats, -} from './get_fields_stats'; -import { wrapError } from '../../utils/error_wrapper'; - -export class DataVisualizer { - private _client: IScopedClusterClient; - - constructor(client: IScopedClusterClient) { - this._client = client; - } - - // Obtains overall stats on the fields in the supplied index pattern, returning an object - // containing the total document count, and four arrays showing which of the supplied - // aggregatable and non-aggregatable fields do or do not exist in documents. - // Sampling will be used if supplied samplerShardSize > 0. - async getOverallStats( - indexPatternTitle: string, - query: object, - aggregatableFields: string[], - nonAggregatableFields: string[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - const stats = { - totalCount: 0, - aggregatableExistsFields: [] as FieldData[], - aggregatableNotExistsFields: [] as FieldData[], - nonAggregatableExistsFields: [] as FieldData[], - nonAggregatableNotExistsFields: [] as FieldData[], - errors: [] as any[], - }; - - // To avoid checking for the existence of too many aggregatable fields in one request, - // split the check into multiple batches (max 200 fields per request). - const batches: string[][] = [[]]; - each(aggregatableFields, (field) => { - let lastArray: string[] = last(batches) as string[]; - if (lastArray.length === AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE) { - lastArray = []; - batches.push(lastArray); - } - lastArray.push(field); - }); - - await Promise.all( - batches.map(async (fields) => { - try { - const batchStats = await this.checkAggregatableFieldsExist( - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - undefined, - runtimeMappings - ); - - // Total count will be returned with each batch of fields. Just overwrite. - stats.totalCount = batchStats.totalCount; - - // Add to the lists of fields which do and do not exist. - stats.aggregatableExistsFields.push(...batchStats.aggregatableExistsFields); - stats.aggregatableNotExistsFields.push(...batchStats.aggregatableNotExistsFields); - } catch (e) { - // If index not found, no need to proceed with other batches - if (e.statusCode === 404) { - throw e; - } - stats.errors.push(wrapError(e)); - } - }) - ); - - await Promise.all( - nonAggregatableFields.map(async (field) => { - try { - const existsInDocs = await this.checkNonAggregatableFieldExists( - indexPatternTitle, - query, - field, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - - const fieldData: FieldData = { - fieldName: field, - existsInDocs, - stats: {}, - }; - - if (existsInDocs === true) { - stats.nonAggregatableExistsFields.push(fieldData); - } else { - stats.nonAggregatableNotExistsFields.push(fieldData); - } - } catch (e) { - stats.errors.push(wrapError(e)); - } - }) - ); - - return stats; - } - - // Obtains binned histograms for supplied list of fields. The statistics for each field in the - // returned array depend on the type of the field (keyword, number, date etc). - // Sampling will be used if supplied samplerShardSize > 0. - async getHistogramsForFields( - indexPatternTitle: string, - query: any, - fields: HistogramField[], - samplerShardSize: number, - runtimeMappings?: estypes.MappingRuntimeFields - ): Promise { - return await getHistogramsForFields( - this._client, - indexPatternTitle, - query, - fields, - samplerShardSize, - runtimeMappings - ); - } - - // Obtains statistics for supplied list of fields. The statistics for each field in the - // returned array depend on the type of the field (keyword, number, date etc). - // Sampling will be used if supplied samplerShardSize > 0. - async getStatsForFields( - indexPatternTitle: string, - query: any, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - intervalMs: number | undefined, - maxExamples: number, - runtimeMappings: estypes.MappingRuntimeFields - ): Promise { - // Batch up fields by type, getting stats for multiple fields at a time. - const batches: Field[][] = []; - const batchedFields: { [key: string]: Field[][] } = {}; - each(fields, (field) => { - if (field.fieldName === undefined) { - // undefined fieldName is used for a document count request. - // getDocumentCountStats requires timeField - don't add to batched requests if not defined - if (timeFieldName !== undefined) { - batches.push([field]); - } - } else { - const fieldType = field.type; - if (batchedFields[fieldType] === undefined) { - batchedFields[fieldType] = [[]]; - } - let lastArray: Field[] = last(batchedFields[fieldType]) as Field[]; - if (lastArray.length === FIELDS_REQUEST_BATCH_SIZE) { - lastArray = []; - batchedFields[fieldType].push(lastArray); - } - lastArray.push(field); - } - }); - - each(batchedFields, (lists) => { - batches.push(...lists); - }); - - let results: BatchStats[] = []; - await Promise.all( - batches.map(async (batch) => { - let batchStats: BatchStats[] = []; - const first = batch[0]; - switch (first.type) { - case JOB_FIELD_TYPES.NUMBER: - // undefined fieldName is used for a document count request. - if (first.fieldName !== undefined) { - batchStats = await this.getNumericFieldsStats( - indexPatternTitle, - query, - batch, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - } else { - // Will only ever be one document count card, - // so no value in batching up the single request. - if (intervalMs !== undefined) { - const stats = await this.getDocumentCountStats( - indexPatternTitle, - query, - timeFieldName, - earliestMs, - latestMs, - intervalMs, - runtimeMappings - ); - batchStats.push(stats); - } - } - break; - case JOB_FIELD_TYPES.KEYWORD: - case JOB_FIELD_TYPES.IP: - batchStats = await this.getStringFieldsStats( - indexPatternTitle, - query, - batch, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - break; - case JOB_FIELD_TYPES.DATE: - batchStats = await this.getDateFieldsStats( - indexPatternTitle, - query, - batch, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - break; - case JOB_FIELD_TYPES.BOOLEAN: - batchStats = await this.getBooleanFieldsStats( - indexPatternTitle, - query, - batch, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - break; - case JOB_FIELD_TYPES.TEXT: - default: - // Use an exists filter on the the field name to get - // examples of the field, so cannot batch up. - await Promise.all( - batch.map(async (field) => { - const stats = await this.getFieldExamples( - indexPatternTitle, - query, - field.fieldName, - timeFieldName, - earliestMs, - latestMs, - maxExamples, - runtimeMappings - ); - batchStats.push(stats); - }) - ); - break; - } - - results = [...results, ...batchStats]; - }) - ); - - return results; - } - - async checkAggregatableFieldsExist( - indexPatternTitle: string, - query: any, - aggregatableFields: string[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs?: number, - latestMs?: number, - datafeedConfig?: estypes.MlDatafeed, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - return await checkAggregatableFieldsExist( - this._client, - indexPatternTitle, - query, - aggregatableFields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - datafeedConfig, - runtimeMappings - ); - } - - async checkNonAggregatableFieldExists( - indexPatternTitle: string, - query: any, - field: string, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - return await checkNonAggregatableFieldExists( - this._client, - indexPatternTitle, - query, - field, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - } - - async getDocumentCountStats( - indexPatternTitle: string, - query: any, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - intervalMs: number, - runtimeMappings: estypes.MappingRuntimeFields - ): Promise { - return await getDocumentCountStats( - this._client, - indexPatternTitle, - query, - timeFieldName, - earliestMs, - latestMs, - intervalMs, - runtimeMappings - ); - } - - async getNumericFieldsStats( - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - return await getNumericFieldsStats( - this._client, - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - } - - async getStringFieldsStats( - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - return await getStringFieldsStats( - this._client, - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - } - - async getDateFieldsStats( - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - return await getDateFieldsStats( - this._client, - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - } - - async getBooleanFieldsStats( - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields - ) { - return await getBooleanFieldsStats( - this._client, - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); - } - - async getFieldExamples( - indexPatternTitle: string, - query: any, - field: string, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - maxExamples: number, - runtimeMappings?: estypes.MappingRuntimeFields - ): Promise { - return await getFieldExamples( - this._client, - indexPatternTitle, - query, - field, - timeFieldName, - earliestMs, - latestMs, - maxExamples, - runtimeMappings - ); - } -} diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_field_examples.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_field_examples.ts deleted file mode 100644 index 61722570c4a06..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_field_examples.ts +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { estypes } from '@elastic/elasticsearch'; -import { get } from 'lodash'; -import { IScopedClusterClient } from 'kibana/server'; -import { buildBaseFilterCriteria } from '../../../common/utils/query_utils'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; -import { FieldExamples } from '../../../common/types/field_stats'; - -export const getFieldExamples = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - field: string, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - maxExamples: number, - runtimeMappings?: estypes.MappingRuntimeFields -): Promise => { - const { asCurrentUser } = client; - - const index = indexPatternTitle; - - // Request at least 100 docs so that we have a chance of obtaining - // 'maxExamples' of the field. - const size = Math.max(100, maxExamples); - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - if (Array.isArray(filterCriteria)) { - // Use an exists filter to return examples of the field. - filterCriteria.push({ - exists: { field }, - }); - } - - const searchBody = { - fields: [field], - _source: false, - query: { - bool: { - filter: filterCriteria, - }, - }, - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - const stats = { - fieldName: field, - examples: [] as any[], - }; - // @ts-expect-error incorrect search response type - if (body.hits.total.value > 0) { - const hits = body.hits.hits; - for (let i = 0; i < hits.length; i++) { - // Use lodash get() to support field names containing dots. - const doc: object[] | undefined = get(hits[i].fields, field); - // the results from fields query is always an array - if (Array.isArray(doc) && doc.length > 0) { - const example = doc[0]; - if (example !== undefined && stats.examples.indexOf(example) === -1) { - stats.examples.push(example); - if (stats.examples.length === maxExamples) { - break; - } - } - } - } - } - - return stats; -}; diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts deleted file mode 100644 index 402e3de9c7185..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_fields_stats.ts +++ /dev/null @@ -1,479 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { estypes } from '@elastic/elasticsearch'; -import { each, find, get } from 'lodash'; -import { IScopedClusterClient } from 'kibana/server'; -import { AggregationsAggregationContainer } from '@elastic/elasticsearch/api/types'; -import { - Aggs, - BooleanFieldStats, - Bucket, - DateFieldStats, - DocumentCountStats, - Field, - NumericFieldStats, - StringFieldStats, -} from '../../../common/types/field_stats'; -import { - buildBaseFilterCriteria, - buildSamplerAggregation, - getSafeAggregationName, - getSamplerAggregationsResponsePath, -} from '../../../common/utils/query_utils'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; -import { processDistributionData } from './process_distribution_data'; -import { SAMPLER_TOP_TERMS_SHARD_SIZE, SAMPLER_TOP_TERMS_THRESHOLD } from './constants'; - -export const getDocumentCountStats = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - intervalMs: number, - runtimeMappings: estypes.MappingRuntimeFields -): Promise => { - const { asCurrentUser } = client; - - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - // Don't use the sampler aggregation as this can lead to some potentially - // confusing date histogram results depending on the date range of data amongst shards. - - const aggs = { - eventRate: { - date_histogram: { - field: timeFieldName, - fixed_interval: `${intervalMs}ms`, - min_doc_count: 1, - }, - }, - }; - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs, - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - - const buckets: { [key: string]: number } = {}; - const dataByTimeBucket: Array<{ key: string; doc_count: number }> = get( - body, - ['aggregations', 'eventRate', 'buckets'], - [] - ); - each(dataByTimeBucket, (dataForTime) => { - const time = dataForTime.key; - buckets[time] = dataForTime.doc_count; - }); - - return { - interval: intervalMs, - buckets, - timeRangeEarliest: earliestMs ?? 0, - timeRangeLatest: latestMs ?? 0, - }; -}; - -export const getNumericFieldsStats = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - // Build the percents parameter which defines the percentiles to query - // for the metric distribution data. - // Use a fixed percentile spacing of 5%. - const MAX_PERCENT = 100; - const PERCENTILE_SPACING = 5; - let count = 0; - const percents = Array.from( - Array(MAX_PERCENT / PERCENTILE_SPACING), - () => (count += PERCENTILE_SPACING) - ); - - const aggs: { [key: string]: any } = {}; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - aggs[`${safeFieldName}_field_stats`] = { - filter: { exists: { field: field.fieldName } }, - aggs: { - actual_stats: { - stats: { field: field.fieldName }, - }, - }, - }; - aggs[`${safeFieldName}_percentiles`] = { - percentiles: { - field: field.fieldName, - percents, - keyed: false, - }, - }; - - const top = { - terms: { - field: field.fieldName, - size: 10, - order: { - _count: 'desc', - }, - }, - }; - - // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation - // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - aggs[`${safeFieldName}_top`] = { - sampler: { - shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, - }, - aggs: { - top, - }, - }; - } else { - aggs[`${safeFieldName}_top`] = top; - } - }); - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const batchStats: NumericFieldStats[] = []; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - const docCount = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], - 0 - ); - const fieldStatsResp = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], - {} - ); - - const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } - - const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); - - const stats: NumericFieldStats = { - fieldName: field.fieldName, - count: docCount, - min: get(fieldStatsResp, 'min', 0), - max: get(fieldStatsResp, 'max', 0), - avg: get(fieldStatsResp, 'avg', 0), - isTopValuesSampled: field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, - topValues, - topValuesSampleSize: topValues.reduce( - (acc, curr) => acc + curr.doc_count, - get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) - ), - topValuesSamplerShardSize: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? SAMPLER_TOP_TERMS_SHARD_SIZE - : samplerShardSize, - }; - - if (stats.count > 0) { - const percentiles = get( - aggregations, - [...aggsPath, `${safeFieldName}_percentiles`, 'values'], - [] - ); - const medianPercentile: { value: number; key: number } | undefined = find(percentiles, { - key: 50, - }); - stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; - stats.distribution = processDistributionData(percentiles, PERCENTILE_SPACING, stats.min); - } - - batchStats.push(stats); - }); - - return batchStats; -}; - -export const getStringFieldsStats = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - const aggs: Aggs = {}; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - const top: AggregationsAggregationContainer = { - terms: { - field: field.fieldName, - size: 10, - order: { - _count: 'desc', - }, - }, - }; - - // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation - // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - aggs[`${safeFieldName}_top`] = { - sampler: { - shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, - }, - aggs: { - top, - }, - }; - } else { - aggs[`${safeFieldName}_top`] = top; - } - }); - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const batchStats: StringFieldStats[] = []; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - - const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } - - const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); - - const stats = { - fieldName: field.fieldName, - isTopValuesSampled: field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, - topValues, - topValuesSampleSize: topValues.reduce( - (acc, curr) => acc + curr.doc_count, - get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) - ), - topValuesSamplerShardSize: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? SAMPLER_TOP_TERMS_SHARD_SIZE - : samplerShardSize, - }; - - batchStats.push(stats); - }); - - return batchStats; -}; - -export const getDateFieldsStats = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - const aggs: Aggs = {}; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - aggs[`${safeFieldName}_field_stats`] = { - filter: { exists: { field: field.fieldName } }, - aggs: { - actual_stats: { - stats: { field: field.fieldName }, - }, - }, - }; - }); - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const batchStats: DateFieldStats[] = []; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - const docCount = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], - 0 - ); - const fieldStatsResp = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], - {} - ); - batchStats.push({ - fieldName: field.fieldName, - count: docCount, - earliest: get(fieldStatsResp, 'min', 0), - latest: get(fieldStatsResp, 'max', 0), - }); - }); - - return batchStats; -}; - -export const getBooleanFieldsStats = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: object, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - - const index = indexPatternTitle; - const size = 0; - const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - - const aggs: Aggs = {}; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - aggs[`${safeFieldName}_value_count`] = { - filter: { exists: { field: field.fieldName } }, - }; - aggs[`${safeFieldName}_values`] = { - terms: { - field: field.fieldName, - size: 2, - }, - }; - }); - - const searchBody = { - query: { - bool: { - filter: filterCriteria, - }, - }, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }; - - const { body } = await asCurrentUser.search({ - index, - size, - body: searchBody, - }); - const aggregations = body.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const batchStats: BooleanFieldStats[] = []; - fields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field.fieldName, i); - const stats: BooleanFieldStats = { - fieldName: field.fieldName, - count: get(aggregations, [...aggsPath, `${safeFieldName}_value_count`, 'doc_count'], 0), - trueCount: 0, - falseCount: 0, - }; - - const valueBuckets: Array<{ [key: string]: number }> = get( - aggregations, - [...aggsPath, `${safeFieldName}_values`, 'buckets'], - [] - ); - valueBuckets.forEach((bucket) => { - stats[`${bucket.key_as_string}Count`] = bucket.doc_count; - }); - - batchStats.push(stats); - }); - - return batchStats; -}; diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_histogram_for_fields.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_histogram_for_fields.ts deleted file mode 100644 index 8db886a9e86c2..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/get_histogram_for_fields.ts +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IScopedClusterClient } from 'kibana/server'; -import { estypes } from '@elastic/elasticsearch'; -import { get } from 'lodash'; -import { - ChartData, - ChartRequestAgg, - HistogramField, - NumericColumnStatsMap, -} from '../../../common/types/field_stats'; -import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/common'; -import { stringHash } from '../../../common/utils/string_utils'; -import { - buildSamplerAggregation, - getSamplerAggregationsResponsePath, -} from '../../../common/utils/query_utils'; -import { isPopulatedObject } from '../../../common/utils/object_utils'; -import { MAX_CHART_COLUMNS } from './constants'; - -export const getAggIntervals = async ( - { asCurrentUser }: IScopedClusterClient, - indexPatternTitle: string, - query: any, - fields: HistogramField[], - samplerShardSize: number, - runtimeMappings?: estypes.MappingRuntimeFields -): Promise => { - const numericColumns = fields.filter((field) => { - return field.type === KBN_FIELD_TYPES.NUMBER || field.type === KBN_FIELD_TYPES.DATE; - }); - - if (numericColumns.length === 0) { - return {}; - } - - const minMaxAggs = numericColumns.reduce((aggs, c) => { - const id = stringHash(c.fieldName); - aggs[id] = { - stats: { - field: c.fieldName, - }, - }; - return aggs; - }, {} as Record); - - const { body } = await asCurrentUser.search({ - index: indexPatternTitle, - size: 0, - body: { - query, - aggs: buildSamplerAggregation(minMaxAggs, samplerShardSize), - size: 0, - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }, - }); - - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const aggregations = aggsPath.length > 0 ? get(body.aggregations, aggsPath) : body.aggregations; - - return Object.keys(aggregations).reduce((p, aggName) => { - const stats = [aggregations[aggName].min, aggregations[aggName].max]; - if (!stats.includes(null)) { - const delta = aggregations[aggName].max - aggregations[aggName].min; - - let aggInterval = 1; - - if (delta > MAX_CHART_COLUMNS || delta <= 1) { - aggInterval = delta / (MAX_CHART_COLUMNS - 1); - } - - p[aggName] = { interval: aggInterval, min: stats[0], max: stats[1] }; - } - - return p; - }, {} as NumericColumnStatsMap); -}; - -export const getHistogramsForFields = async ( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - fields: HistogramField[], - samplerShardSize: number, - runtimeMappings?: estypes.MappingRuntimeFields -) => { - const { asCurrentUser } = client; - const aggIntervals = await getAggIntervals( - client, - indexPatternTitle, - query, - fields, - samplerShardSize, - runtimeMappings - ); - - const chartDataAggs = fields.reduce((aggs, field) => { - const fieldName = field.fieldName; - const fieldType = field.type; - const id = stringHash(fieldName); - if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { - if (aggIntervals[id] !== undefined) { - aggs[`${id}_histogram`] = { - histogram: { - field: fieldName, - interval: aggIntervals[id].interval !== 0 ? aggIntervals[id].interval : 1, - }, - }; - } - } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { - if (fieldType === KBN_FIELD_TYPES.STRING) { - aggs[`${id}_cardinality`] = { - cardinality: { - field: fieldName, - }, - }; - } - aggs[`${id}_terms`] = { - terms: { - field: fieldName, - size: MAX_CHART_COLUMNS, - }, - }; - } - return aggs; - }, {} as Record); - - if (Object.keys(chartDataAggs).length === 0) { - return []; - } - - const { body } = await asCurrentUser.search({ - index: indexPatternTitle, - size: 0, - body: { - query, - aggs: buildSamplerAggregation(chartDataAggs, samplerShardSize), - size: 0, - ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), - }, - }); - - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const aggregations = aggsPath.length > 0 ? get(body.aggregations, aggsPath) : body.aggregations; - - const chartsData: ChartData[] = fields.map((field): ChartData => { - const fieldName = field.fieldName; - const fieldType = field.type; - const id = stringHash(field.fieldName); - - if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { - if (aggIntervals[id] === undefined) { - return { - type: 'numeric', - data: [], - interval: 0, - stats: [0, 0], - id: fieldName, - }; - } - - return { - data: aggregations[`${id}_histogram`].buckets, - interval: aggIntervals[id].interval, - stats: [aggIntervals[id].min, aggIntervals[id].max], - type: 'numeric', - id: fieldName, - }; - } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { - return { - type: fieldType === KBN_FIELD_TYPES.STRING ? 'ordinal' : 'boolean', - cardinality: - fieldType === KBN_FIELD_TYPES.STRING ? aggregations[`${id}_cardinality`].value : 2, - data: aggregations[`${id}_terms`].buckets, - id: fieldName, - }; - } - - return { - type: 'unsupported', - id: fieldName, - }; - }); - - return chartsData; -}; diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/index.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/index.ts deleted file mode 100644 index a29957b159b7e..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './data_visualizer'; diff --git a/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts b/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts deleted file mode 100644 index dcbe6e06738cc..0000000000000 --- a/x-pack/plugins/data_visualizer/server/models/data_visualizer/process_distribution_data.ts +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { last } from 'lodash'; -import { Distribution } from '../../../common/types/field_stats'; - -// @todo: REMOVE -export const processDistributionData = ( - percentiles: Array<{ value: number }>, - percentileSpacing: number, - minValue: number -): Distribution => { - const distribution: Distribution = { percentiles: [], minPercentile: 0, maxPercentile: 100 }; - if (percentiles.length === 0) { - return distribution; - } - - let percentileBuckets: Array<{ value: number }> = []; - let lowerBound = minValue; - if (lowerBound >= 0) { - // By default return results for 0 - 90% percentiles. - distribution.minPercentile = 0; - distribution.maxPercentile = 90; - percentileBuckets = percentiles.slice(0, percentiles.length - 2); - - // Look ahead to the last percentiles and process these too if - // they don't add more than 50% to the value range. - const lastValue = (last(percentileBuckets) as any).value; - const upperBound = lowerBound + 1.5 * (lastValue - lowerBound); - const filteredLength = percentileBuckets.length; - for (let i = filteredLength; i < percentiles.length; i++) { - if (percentiles[i].value < upperBound) { - percentileBuckets.push(percentiles[i]); - distribution.maxPercentile += percentileSpacing; - } else { - break; - } - } - } else { - // By default return results for 5 - 95% percentiles. - const dataMin = lowerBound; - lowerBound = percentiles[0].value; - distribution.minPercentile = 5; - distribution.maxPercentile = 95; - percentileBuckets = percentiles.slice(1, percentiles.length - 1); - - // Add in 0-5 and 95-100% if they don't add more - // than 25% to the value range at either end. - const lastValue: number = (last(percentileBuckets) as any).value; - const maxDiff = 0.25 * (lastValue - lowerBound); - if (lowerBound - dataMin < maxDiff) { - percentileBuckets.splice(0, 0, percentiles[0]); - distribution.minPercentile = 0; - lowerBound = dataMin; - } - - if (percentiles[percentiles.length - 1].value - lastValue < maxDiff) { - percentileBuckets.push(percentiles[percentiles.length - 1]); - distribution.maxPercentile = 100; - } - } - - // Combine buckets with the same value. - const totalBuckets = percentileBuckets.length; - let lastBucketValue = lowerBound; - let numEqualValueBuckets = 0; - for (let i = 0; i < totalBuckets; i++) { - const bucket = percentileBuckets[i]; - - // Results from the percentiles aggregation can have precision rounding - // artifacts e.g returning 200 and 200.000000000123, so check for equality - // around double floating point precision i.e. 15 sig figs. - if (bucket.value.toPrecision(15) !== lastBucketValue.toPrecision(15)) { - // Create a bucket for any 'equal value' buckets which had a value <= last bucket - if (numEqualValueBuckets > 0) { - distribution.percentiles.push({ - percent: numEqualValueBuckets * percentileSpacing, - minValue: lastBucketValue, - maxValue: lastBucketValue, - }); - } - - distribution.percentiles.push({ - percent: percentileSpacing, - minValue: lastBucketValue, - maxValue: bucket.value, - }); - - lastBucketValue = bucket.value; - numEqualValueBuckets = 0; - } else { - numEqualValueBuckets++; - if (i === totalBuckets - 1) { - // If at the last bucket, create a final bucket for the equal value buckets. - distribution.percentiles.push({ - percent: numEqualValueBuckets * percentileSpacing, - minValue: lastBucketValue, - maxValue: lastBucketValue, - }); - } - } - } - - return distribution; -}; diff --git a/x-pack/plugins/data_visualizer/server/plugin.ts b/x-pack/plugins/data_visualizer/server/plugin.ts index e2e0637ef8f3f..9ef6ca5ae6a69 100644 --- a/x-pack/plugins/data_visualizer/server/plugin.ts +++ b/x-pack/plugins/data_visualizer/server/plugin.ts @@ -7,15 +7,12 @@ import { CoreSetup, CoreStart, Plugin } from 'src/core/server'; import { StartDeps, SetupDeps } from './types'; -import { dataVisualizerRoutes } from './routes'; import { registerWithCustomIntegrations } from './register_custom_integration'; export class DataVisualizerPlugin implements Plugin { constructor() {} setup(coreSetup: CoreSetup, plugins: SetupDeps) { - dataVisualizerRoutes(coreSetup); - // home-plugin required if (plugins.home && plugins.customIntegrations) { registerWithCustomIntegrations(plugins.customIntegrations); diff --git a/x-pack/plugins/data_visualizer/server/routes/index.ts b/x-pack/plugins/data_visualizer/server/routes/index.ts deleted file mode 100644 index 892f6cbd77361..0000000000000 --- a/x-pack/plugins/data_visualizer/server/routes/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { dataVisualizerRoutes } from './routes'; diff --git a/x-pack/plugins/data_visualizer/server/routes/routes.ts b/x-pack/plugins/data_visualizer/server/routes/routes.ts deleted file mode 100644 index 582dab1ad7d97..0000000000000 --- a/x-pack/plugins/data_visualizer/server/routes/routes.ts +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { CoreSetup, IScopedClusterClient } from 'kibana/server'; -import { estypes } from '@elastic/elasticsearch'; -import { - dataVisualizerFieldHistogramsSchema, - dataVisualizerFieldStatsSchema, - dataVisualizerOverallStatsSchema, - indexPatternTitleSchema, -} from './schemas'; -import type { StartDeps } from '../types'; -import type { Field, HistogramField } from '../../common/types/field_stats'; -import { DataVisualizer } from '../models/data_visualizer'; -import { wrapError } from '../utils/error_wrapper'; - -function getOverallStats( - client: IScopedClusterClient, - indexPatternTitle: string, - query: object, - aggregatableFields: string[], - nonAggregatableFields: string[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - runtimeMappings: estypes.MappingRuntimeFields -) { - const dv = new DataVisualizer(client); - return dv.getOverallStats( - indexPatternTitle, - query, - aggregatableFields, - nonAggregatableFields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - runtimeMappings - ); -} - -function getStatsForFields( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - fields: Field[], - samplerShardSize: number, - timeFieldName: string | undefined, - earliestMs: number | undefined, - latestMs: number | undefined, - interval: number | undefined, - maxExamples: number, - runtimeMappings: estypes.MappingRuntimeFields -) { - const dv = new DataVisualizer(client); - return dv.getStatsForFields( - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliestMs, - latestMs, - interval, - maxExamples, - runtimeMappings - ); -} - -function getHistogramsForFields( - client: IScopedClusterClient, - indexPatternTitle: string, - query: any, - fields: HistogramField[], - samplerShardSize: number, - runtimeMappings: estypes.MappingRuntimeFields -) { - const dv = new DataVisualizer(client); - return dv.getHistogramsForFields( - indexPatternTitle, - query, - fields, - samplerShardSize, - runtimeMappings - ); -} -/** - * Routes for the index data visualizer. - */ -export function dataVisualizerRoutes(coreSetup: CoreSetup) { - const router = coreSetup.http.createRouter(); - - /** - * @apiGroup DataVisualizer - * - * @api {post} /internal/data_visualizer/get_field_histograms/:indexPatternTitle Get histograms for fields - * @apiName GetHistogramsForFields - * @apiDescription Returns the histograms on a list fields in the specified index pattern. - * - * @apiSchema (params) indexPatternTitleSchema - * @apiSchema (body) dataVisualizerFieldHistogramsSchema - * - * @apiSuccess {Object} fieldName histograms by field, keyed on the name of the field. - */ - router.post( - { - path: '/internal/data_visualizer/get_field_histograms/{indexPatternTitle}', - validate: { - params: indexPatternTitleSchema, - body: dataVisualizerFieldHistogramsSchema, - }, - }, - async (context, request, response) => { - try { - const { - params: { indexPatternTitle }, - body: { query, fields, samplerShardSize, runtimeMappings }, - } = request; - - const results = await getHistogramsForFields( - context.core.elasticsearch.client, - indexPatternTitle, - query, - fields, - samplerShardSize, - runtimeMappings - ); - - return response.ok({ - body: results, - }); - } catch (e) { - return response.customError(wrapError(e)); - } - } - ); - - /** - * @apiGroup DataVisualizer - * - * @api {post} /internal/data_visualizer/get_field_stats/:indexPatternTitle Get stats for fields - * @apiName GetStatsForFields - * @apiDescription Returns the stats on individual fields in the specified index pattern. - * - * @apiSchema (params) indexPatternTitleSchema - * @apiSchema (body) dataVisualizerFieldStatsSchema - * - * @apiSuccess {Object} fieldName stats by field, keyed on the name of the field. - */ - router.post( - { - path: '/internal/data_visualizer/get_field_stats/{indexPatternTitle}', - validate: { - params: indexPatternTitleSchema, - body: dataVisualizerFieldStatsSchema, - }, - }, - async (context, request, response) => { - try { - const { - params: { indexPatternTitle }, - body: { - query, - fields, - samplerShardSize, - timeFieldName, - earliest, - latest, - interval, - maxExamples, - runtimeMappings, - }, - } = request; - const results = await getStatsForFields( - context.core.elasticsearch.client, - indexPatternTitle, - query, - fields, - samplerShardSize, - timeFieldName, - earliest, - latest, - interval, - maxExamples, - runtimeMappings - ); - - return response.ok({ - body: results, - }); - } catch (e) { - return response.customError(wrapError(e)); - } - } - ); - - /** - * @apiGroup DataVisualizer - * - * @api {post} /internal/data_visualizer/get_overall_stats/:indexPatternTitle Get overall stats - * @apiName GetOverallStats - * @apiDescription Returns the top level overall stats for the specified index pattern. - * - * @apiSchema (params) indexPatternTitleSchema - * @apiSchema (body) dataVisualizerOverallStatsSchema - * - * @apiSuccess {number} totalCount total count of documents. - * @apiSuccess {Object} aggregatableExistsFields stats on aggregatable fields that exist in documents. - * @apiSuccess {Object} aggregatableNotExistsFields stats on aggregatable fields that do not exist in documents. - * @apiSuccess {Object} nonAggregatableExistsFields stats on non-aggregatable fields that exist in documents. - * @apiSuccess {Object} nonAggregatableNotExistsFields stats on non-aggregatable fields that do not exist in documents. - */ - router.post( - { - path: '/internal/data_visualizer/get_overall_stats/{indexPatternTitle}', - validate: { - params: indexPatternTitleSchema, - body: dataVisualizerOverallStatsSchema, - }, - }, - async (context, request, response) => { - try { - const { - params: { indexPatternTitle }, - body: { - query, - aggregatableFields, - nonAggregatableFields, - samplerShardSize, - timeFieldName, - earliest, - latest, - runtimeMappings, - }, - } = request; - - const results = await getOverallStats( - context.core.elasticsearch.client, - indexPatternTitle, - query, - aggregatableFields, - nonAggregatableFields, - samplerShardSize, - timeFieldName, - earliest, - latest, - runtimeMappings - ); - - return response.ok({ - body: results, - }); - } catch (e) { - return response.customError(wrapError(e)); - } - } - ); -} diff --git a/x-pack/plugins/data_visualizer/server/routes/schemas/index.ts b/x-pack/plugins/data_visualizer/server/routes/schemas/index.ts deleted file mode 100644 index 156336feef29e..0000000000000 --- a/x-pack/plugins/data_visualizer/server/routes/schemas/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './index_data_visualizer_schemas'; diff --git a/x-pack/plugins/data_visualizer/server/routes/schemas/index_data_visualizer_schemas.ts b/x-pack/plugins/data_visualizer/server/routes/schemas/index_data_visualizer_schemas.ts deleted file mode 100644 index 0f145081d8cec..0000000000000 --- a/x-pack/plugins/data_visualizer/server/routes/schemas/index_data_visualizer_schemas.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { schema } from '@kbn/config-schema'; -import { isRuntimeField } from '../../../common/utils/runtime_field_utils'; - -export const runtimeMappingsSchema = schema.object( - {}, - { - unknowns: 'allow', - validate: (v: object) => { - if (Object.values(v).some((o) => !isRuntimeField(o))) { - return 'Invalid runtime field'; - } - }, - } -); - -export const indexPatternTitleSchema = schema.object({ - /** Title of the index pattern for which to return stats. */ - indexPatternTitle: schema.string(), -}); - -export const dataVisualizerFieldHistogramsSchema = schema.object({ - /** Query to match documents in the index. */ - query: schema.any(), - /** The fields to return histogram data. */ - fields: schema.arrayOf(schema.any()), - /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ - samplerShardSize: schema.number(), - /** Optional search time runtime fields */ - runtimeMappings: runtimeMappingsSchema, -}); - -export const dataVisualizerFieldStatsSchema = schema.object({ - /** Query to match documents in the index. */ - query: schema.any(), - fields: schema.arrayOf(schema.any()), - /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ - samplerShardSize: schema.number(), - /** Name of the time field in the index (optional). */ - timeFieldName: schema.maybe(schema.string()), - /** Earliest timestamp for search, as epoch ms (optional). */ - earliest: schema.maybe(schema.number()), - /** Latest timestamp for search, as epoch ms (optional). */ - latest: schema.maybe(schema.number()), - /** Aggregation interval, in milliseconds, to use for obtaining document counts over time (optional). */ - interval: schema.maybe(schema.number()), - /** Maximum number of examples to return for text type fields. */ - maxExamples: schema.number(), - /** Optional search time runtime fields */ - runtimeMappings: runtimeMappingsSchema, -}); - -export const dataVisualizerOverallStatsSchema = schema.object({ - /** Query to match documents in the index. */ - query: schema.any(), - /** Names of aggregatable fields for which to return stats. */ - aggregatableFields: schema.arrayOf(schema.string()), - /** Names of non-aggregatable fields for which to return stats. */ - nonAggregatableFields: schema.arrayOf(schema.string()), - /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ - samplerShardSize: schema.number(), - /** Name of the time field in the index (optional). */ - timeFieldName: schema.maybe(schema.string()), - /** Earliest timestamp for search, as epoch ms (optional). */ - earliest: schema.maybe(schema.number()), - /** Latest timestamp for search, as epoch ms (optional). */ - latest: schema.maybe(schema.number()), - /** Optional search time runtime fields */ - runtimeMappings: runtimeMappingsSchema, -}); diff --git a/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_stats.ts b/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_stats.ts deleted file mode 100644 index 9e4343ea29a1b..0000000000000 --- a/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_stats.ts +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { sortBy } from 'lodash'; - -import { FtrProviderContext } from '../../../ftr_provider_context'; -import { USER } from '../../../../functional/services/ml/security_common'; -import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api'; - -export default ({ getService }: FtrProviderContext) => { - const esArchiver = getService('esArchiver'); - const supertest = getService('supertestWithoutAuth'); - const ml = getService('ml'); - - const metricFieldsTestData = { - testTitle: 'returns stats for metric fields over all time', - index: 'ft_farequote', - user: USER.ML_POWERUSER, - requestBody: { - query: { - bool: { - must: { - term: { airline: 'JZA' }, // Only use one airline to ensure no sampling. - }, - }, - }, - fields: [ - { type: 'number', cardinality: 0 }, - { fieldName: 'responsetime', type: 'number', cardinality: 4249 }, - ], - samplerShardSize: -1, // No sampling, as otherwise counts could vary on each run. - timeFieldName: '@timestamp', - interval: 86400000, - maxExamples: 10, - }, - expected: { - responseCode: 200, - responseBody: [ - { - interval: 86400000, - buckets: { - '1454803200000': 846, - '1454889600000': 846, - '1454976000000': 859, - '1455062400000': 851, - '1455148800000': 858, - }, - timeRangeEarliest: 0, - timeRangeLatest: 0, - }, - { - // Cannot verify median and percentiles responses as the ES percentiles agg is non-deterministic. - fieldName: 'responsetime', - count: 4260, - min: 963.4293212890625, - max: 1042.13525390625, - avg: 1000.0378077547315, - isTopValuesSampled: false, - topValues: [ - { key: 980.0411987304688, doc_count: 2 }, - { key: 989.278076171875, doc_count: 2 }, - { key: 989.763916015625, doc_count: 2 }, - { key: 991.290771484375, doc_count: 2 }, - { key: 992.0765991210938, doc_count: 2 }, - { key: 993.8115844726562, doc_count: 2 }, - { key: 993.8973999023438, doc_count: 2 }, - { key: 994.0230102539062, doc_count: 2 }, - { key: 994.364990234375, doc_count: 2 }, - { key: 994.916015625, doc_count: 2 }, - ], - topValuesSampleSize: 4260, - topValuesSamplerShardSize: -1, - }, - ], - }, - }; - - const nonMetricFieldsTestData = { - testTitle: 'returns stats for non-metric fields specifying query and time range', - index: 'ft_farequote', - user: USER.ML_POWERUSER, - requestBody: { - query: { - bool: { - must: { - term: { airline: 'AAL' }, - }, - }, - }, - fields: [ - { fieldName: '@timestamp', type: 'date', cardinality: 4751 }, - { fieldName: '@version.keyword', type: 'keyword', cardinality: 1 }, - { fieldName: 'airline', type: 'keyword', cardinality: 19 }, - { fieldName: 'type', type: 'text', cardinality: 0 }, - { fieldName: 'type.keyword', type: 'keyword', cardinality: 1 }, - ], - samplerShardSize: -1, // No sampling, as otherwise counts would vary on each run. - timeFieldName: '@timestamp', - earliest: 1454889600000, // February 8, 2016 12:00:00 AM GMT - latest: 1454976000000, // February 9, 2016 12:00:00 AM GMT - maxExamples: 10, - }, - expected: { - responseCode: 200, - responseBody: [ - { fieldName: '@timestamp', count: 1733, earliest: 1454889602000, latest: 1454975948000 }, - { - fieldName: '@version.keyword', - isTopValuesSampled: false, - topValues: [{ key: '1', doc_count: 1733 }], - topValuesSampleSize: 1733, - topValuesSamplerShardSize: -1, - }, - { - fieldName: 'airline', - isTopValuesSampled: false, - topValues: [{ key: 'AAL', doc_count: 1733 }], - topValuesSampleSize: 1733, - topValuesSamplerShardSize: -1, - }, - { - fieldName: 'type.keyword', - isTopValuesSampled: false, - topValues: [{ key: 'farequote', doc_count: 1733 }], - topValuesSampleSize: 1733, - topValuesSamplerShardSize: -1, - }, - { fieldName: 'type', examples: ['farequote'] }, - ], - }, - }; - - const errorTestData = { - testTitle: 'returns error for index which does not exist', - index: 'ft_farequote_not_exists', - user: USER.ML_POWERUSER, - requestBody: { - query: { bool: { must: [{ match_all: {} }] } }, - fields: [ - { type: 'number', cardinality: 0 }, - { fieldName: 'responsetime', type: 'number', cardinality: 4249 }, - ], - samplerShardSize: -1, // No sampling, as otherwise counts could vary on each run. - timeFieldName: '@timestamp', - interval: 86400000, - maxExamples: 10, - }, - expected: { - responseCode: 404, - responseBody: { - statusCode: 404, - error: 'Not Found', - message: 'index_not_found_exception', - }, - }, - }; - - async function runGetFieldStatsRequest( - index: string, - user: USER, - requestBody: object, - expectedResponsecode: number - ): Promise { - const { body } = await supertest - .post(`/internal/data_visualizer/get_field_stats/${index}`) - .auth(user, ml.securityCommon.getPasswordForUser(user)) - .set(COMMON_REQUEST_HEADERS) - .send(requestBody) - .expect(expectedResponsecode); - - return body; - } - - // Move these tests to file_data_visualizer plugin - describe('get_field_stats', function () { - before(async () => { - await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); - await ml.testResources.setKibanaTimeZoneToUTC(); - }); - - it(`${metricFieldsTestData.testTitle}`, async () => { - const body = await runGetFieldStatsRequest( - metricFieldsTestData.index, - metricFieldsTestData.user, - metricFieldsTestData.requestBody, - metricFieldsTestData.expected.responseCode - ); - - // Cannot verify median and percentiles responses as the ES percentiles agg is non-deterministic. - const expected = metricFieldsTestData.expected; - expect(body).to.have.length(expected.responseBody.length); - - const actualDocCounts = body[0]; - const expectedDocCounts = expected.responseBody[0]; - expect(actualDocCounts).to.eql(expectedDocCounts); - - const actualFieldData = { ...body[1] }; - delete actualFieldData.median; - delete actualFieldData.distribution; - - expect(actualFieldData).to.eql(expected.responseBody[1]); - }); - - it(`${nonMetricFieldsTestData.testTitle}`, async () => { - const body = await runGetFieldStatsRequest( - nonMetricFieldsTestData.index, - nonMetricFieldsTestData.user, - nonMetricFieldsTestData.requestBody, - nonMetricFieldsTestData.expected.responseCode - ); - - const expectedRspFields = sortBy(nonMetricFieldsTestData.expected.responseBody, 'fieldName'); - const actualRspFields = sortBy(body, 'fieldName'); - expect(actualRspFields).to.eql(expectedRspFields); - }); - - it(`${errorTestData.testTitle}`, async () => { - const body = await runGetFieldStatsRequest( - errorTestData.index, - errorTestData.user, - errorTestData.requestBody, - errorTestData.expected.responseCode - ); - - expect(body.error).to.eql(errorTestData.expected.responseBody.error); - expect(body.message).to.contain(errorTestData.expected.responseBody.message); - }); - }); -}; diff --git a/x-pack/test/api_integration/apis/ml/data_visualizer/get_overall_stats.ts b/x-pack/test/api_integration/apis/ml/data_visualizer/get_overall_stats.ts deleted file mode 100644 index 7987875a75519..0000000000000 --- a/x-pack/test/api_integration/apis/ml/data_visualizer/get_overall_stats.ts +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; - -import { FtrProviderContext } from '../../../ftr_provider_context'; -import { USER } from '../../../../functional/services/ml/security_common'; -import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api'; - -export default ({ getService }: FtrProviderContext) => { - const esArchiver = getService('esArchiver'); - const supertest = getService('supertestWithoutAuth'); - const ml = getService('ml'); - - const testDataList = [ - { - testTitle: 'returns stats over all time', - index: 'ft_farequote', - user: USER.ML_POWERUSER, - requestBody: { - query: { bool: { must: [{ match_all: {} }] } }, - aggregatableFields: ['@timestamp', 'airline', 'responsetime', 'sourcetype'], - nonAggregatableFields: ['type'], - samplerShardSize: -1, // No sampling, as otherwise counts would vary on each run. - timeFieldName: '@timestamp', - }, - expected: { - responseCode: 200, - responseBody: { - totalCount: 86274, - aggregatableExistsFields: [ - { - fieldName: '@timestamp', - existsInDocs: true, - stats: { sampleCount: 86274, count: 86274, cardinality: 78580 }, - }, - { - fieldName: 'airline', - existsInDocs: true, - stats: { sampleCount: 86274, count: 86274, cardinality: 19 }, - }, - { - fieldName: 'responsetime', - existsInDocs: true, - stats: { sampleCount: 86274, count: 86274, cardinality: 83346 }, - }, - ], - aggregatableNotExistsFields: [{ fieldName: 'sourcetype', existsInDocs: false }], - nonAggregatableExistsFields: [{ fieldName: 'type', existsInDocs: true, stats: {} }], - nonAggregatableNotExistsFields: [], - errors: [], - }, - }, - }, - { - testTitle: 'returns stats when specifying query and time range', - index: 'ft_farequote', - user: USER.ML_POWERUSER, - requestBody: { - query: { - bool: { - must: { - term: { airline: 'AAL' }, - }, - }, - }, - aggregatableFields: ['@timestamp', 'airline', 'responsetime', 'sourcetype'], - nonAggregatableFields: ['type'], - samplerShardSize: -1, // No sampling, as otherwise counts would vary on each run. - timeFieldName: '@timestamp', - earliest: 1454889600000, // February 8, 2016 12:00:00 AM GMT - latest: 1454976000000, // February 9, 2016 12:00:00 AM GMT - }, - expected: { - responseCode: 200, - responseBody: { - totalCount: 1733, - aggregatableExistsFields: [ - { - fieldName: '@timestamp', - existsInDocs: true, - stats: { sampleCount: 1733, count: 1733, cardinality: 1713 }, - }, - { - fieldName: 'airline', - existsInDocs: true, - stats: { sampleCount: 1733, count: 1733, cardinality: 1 }, - }, - { - fieldName: 'responsetime', - existsInDocs: true, - stats: { sampleCount: 1733, count: 1733, cardinality: 1730 }, - }, - ], - aggregatableNotExistsFields: [{ fieldName: 'sourcetype', existsInDocs: false }], - nonAggregatableExistsFields: [{ fieldName: 'type', existsInDocs: true, stats: {} }], - nonAggregatableNotExistsFields: [], - errors: [], - }, - }, - }, - { - testTitle: 'returns error for index which does not exist', - index: 'ft_farequote_not_exist', - user: USER.ML_POWERUSER, - requestBody: { - query: { bool: { must: [{ match_all: {} }] } }, - aggregatableFields: ['@timestamp', 'airline', 'responsetime', 'sourcetype'], - nonAggregatableFields: ['@version', 'type'], - samplerShardSize: 1000, - timeFieldName: '@timestamp', - }, - expected: { - responseCode: 404, - responseBody: { - statusCode: 404, - error: 'Not Found', - message: 'index_not_found_exception', - }, - }, - }, - ]; - - // Move these tests to file_data_visualizer plugin - describe('get_overall_stats', function () { - before(async () => { - await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); - await ml.testResources.setKibanaTimeZoneToUTC(); - }); - - for (const testData of testDataList) { - it(`${testData.testTitle}`, async () => { - const { body } = await supertest - .post(`/internal/data_visualizer/get_overall_stats/${testData.index}`) - .auth(testData.user, ml.securityCommon.getPasswordForUser(testData.user)) - .set(COMMON_REQUEST_HEADERS) - .send(testData.requestBody) - .expect(testData.expected.responseCode); - - if (body.error === undefined) { - expect(body).to.eql(testData.expected.responseBody); - } else { - expect(body.error).to.eql(testData.expected.responseBody.error); - expect(body.message).to.contain(testData.expected.responseBody.message); - } - }); - } - }); -}; From 6e9fe98bb4bf713559c7acc21ee45cd62efaa92d Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 20 Oct 2021 12:58:50 -0500 Subject: [PATCH 161/188] Add telemetry, clean up, rename components for clarity --- .../components/layout/discover_layout.tsx | 19 ++++++++++-- .../components/field_stats_table/constants.ts | 12 +++++++ .../field_stats_table.tsx} | 31 +++++++++++++------ ...d_stats_table_saved_search_embeddable.tsx} | 9 ++---- .../index.ts | 3 +- .../embeddable/saved_search_embeddable.tsx | 4 +-- .../hooks/use_field_stats.ts | 15 ++++++--- .../hooks/use_overall_stats.ts | 1 + .../search_strategy/requests/overall_stats.ts | 4 ++- 9 files changed, 72 insertions(+), 26 deletions(-) create mode 100644 src/plugins/discover/public/application/components/field_stats_table/constants.ts rename src/plugins/discover/public/application/components/{data_visualizer_grid/data_visualizer_grid.tsx => field_stats_table/field_stats_table.tsx} (90%) rename src/plugins/discover/public/application/components/{data_visualizer_grid/field_stats_table_embeddable.tsx => field_stats_table/field_stats_table_saved_search_embeddable.tsx} (78%) rename src/plugins/discover/public/application/components/{data_visualizer_grid => field_stats_table}/index.ts (68%) diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index a0799777a3947..4a2e938fbdd62 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -43,8 +43,12 @@ import { SavedSearchURLConflictCallout, useSavedSearchAliasMatchRedirect, } from '../../../../../saved_searches'; -import { DiscoverDataVisualizerGrid } from '../../../../components/data_visualizer_grid'; +import { FieldStatisticsTable } from '../../../../components/field_stats_table'; import { VIEW_MODE } from '../view_mode_toggle'; +import { + DOCUMENTS_VIEW_CLICK, + FIELD_STATISTICS_VIEW_CLICK, +} from '../../../../components/field_stats_table/constants'; /** * Local storage key for sidebar persistence state @@ -54,7 +58,7 @@ export const SIDEBAR_CLOSED_KEY = 'discover:sidebarClosed'; const SidebarMemoized = React.memo(DiscoverSidebarResponsive); const TopNavMemoized = React.memo(DiscoverTopNav); const DiscoverChartMemoized = React.memo(DiscoverChart); -const DataVisualizerGridMemoized = React.memo(DiscoverDataVisualizerGrid); +const DataVisualizerGridMemoized = React.memo(FieldStatisticsTable); export function DiscoverLayout({ indexPattern, @@ -95,8 +99,16 @@ export function DiscoverLayout({ const setDiscoverViewMode = useCallback( (mode: VIEW_MODE) => { stateContainer.setAppState({ viewMode: mode }); + + if (trackUiMetric) { + if (mode === VIEW_MODE.AGGREGATED_LEVEL) { + trackUiMetric(METRIC_TYPE.CLICK, FIELD_STATISTICS_VIEW_CLICK); + } else { + trackUiMetric(METRIC_TYPE.CLICK, DOCUMENTS_VIEW_CLICK); + } + } }, - [stateContainer] + [trackUiMetric, stateContainer] ); const fetchCounter = useRef(0); @@ -324,6 +336,7 @@ export function DiscoverLayout({ columns={columns} stateContainer={stateContainer} onAddFilter={onAddFilter} + trackUiMetric={trackUiMetric} /> )} diff --git a/src/plugins/discover/public/application/components/field_stats_table/constants.ts b/src/plugins/discover/public/application/components/field_stats_table/constants.ts new file mode 100644 index 0000000000000..bf1a36da59ecf --- /dev/null +++ b/src/plugins/discover/public/application/components/field_stats_table/constants.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** Telemetry related to field statistics table **/ +export const FIELD_STATISTICS_LOADED = 'field_statistics_loaded'; +export const FIELD_STATISTICS_VIEW_CLICK = 'field_statistics_view_click'; +export const DOCUMENTS_VIEW_CLICK = 'documents_view_click'; diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx b/src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx similarity index 90% rename from src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx rename to src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx index 5492fac014b74..b8c7446437392 100644 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx +++ b/src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx @@ -8,6 +8,7 @@ import React, { useEffect, useMemo, useRef, useState } from 'react'; import { Filter } from '@kbn/es-query'; +import { METRIC_TYPE, UiCounterMetricType } from '@kbn/analytics'; import { IndexPatternField, IndexPattern, DataView, Query } from '../../../../../data/common'; import { DiscoverServices } from '../../../build_services'; import { @@ -19,6 +20,7 @@ import { } from '../../../../../embeddable/public'; import { SavedSearch } from '../../../saved_searches'; import { GetStateReturn } from '../../apps/main/services/discover_state'; +import { FIELD_STATISTICS_LOADED } from './constants'; export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput { indexPattern: IndexPattern; @@ -69,14 +71,23 @@ export interface DiscoverDataVisualizerGridProps { * Filters query to update the table content */ filters?: Filter[]; + /** + * State container with persisted settings + */ stateContainer?: GetStateReturn; /** * Callback to add a filter to filter bar */ onAddFilter?: (field: IndexPatternField | string, value: string, type: '+' | '-') => void; + /** + * Metric tracking function + * @param metricType + * @param eventName + */ + trackUiMetric?: (metricType: UiCounterMetricType, eventName: string | string[]) => void; } -export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { +export const FieldStatisticsTable = (props: DiscoverDataVisualizerGridProps) => { const { services, indexPattern, @@ -86,6 +97,7 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp filters, stateContainer, onAddFilter, + trackUiMetric, } = props; const { uiSettings } = services; @@ -135,17 +147,11 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp embeddable.updateInput({ showPreviewByDefault, }); + embeddable.reload(); } }, [showPreviewByDefault, uiSettings, embeddable]); - useEffect(() => { - return () => { - // Clean up embeddable upon unmounting - embeddable?.destroy(); - }; - }, [embeddable]); - useEffect(() => { let unmounted = false; const loadEmbeddable = async () => { @@ -181,8 +187,15 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp useEffect(() => { if (embeddableRoot.current && embeddable) { embeddable.render(embeddableRoot.current); + + trackUiMetric?.(METRIC_TYPE.LOADED, FIELD_STATISTICS_LOADED); } - }, [embeddable, embeddableRoot, uiSettings]); + + return () => { + // Clean up embeddable upon unmounting + embeddable?.destroy(); + }; + }, [embeddable, embeddableRoot, uiSettings, trackUiMetric]); return (
- & Partial & { @@ -391,7 +391,7 @@ export class SavedSearchEmbeddable Array.isArray(searchProps.columns) ) { ReactDOM.render( - { - const existsInDocs = results.find((r) => r.rawResponse.fieldName === fieldName) !== undefined; + const foundField = results.find((r) => r.rawResponse.fieldName === fieldName); + const existsInDocs = foundField !== undefined && foundField.rawResponse.hits.hits.length > 0; const fieldData: NonAggregatableField = { fieldName, existsInDocs, From ddffbf84b1d8aabf7f0fa0f1ea065c28ddff19a4 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 20 Oct 2021 13:12:17 -0500 Subject: [PATCH 162/188] Fix size of callout message --- .../components/field_data_expanded_row/text_content.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx index 6f946fc1025ed..c283ec9bfa4ed 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx @@ -30,8 +30,9 @@ export const TextContent: FC = ({ config }) => { {numExamples > 0 && } {numExamples === 0 && ( - + Date: Wed, 20 Oct 2021 13:25:28 -0500 Subject: [PATCH 163/188] Fix texts field --- .../components/field_data_expanded_row/text_content.tsx | 2 +- .../search_strategy/requests/overall_stats.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx index c283ec9bfa4ed..4fc73f0831dfc 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/text_content.tsx @@ -30,7 +30,7 @@ export const TextContent: FC = ({ config }) => { {numExamples > 0 && } {numExamples === 0 && ( - + { const foundField = results.find((r) => r.rawResponse.fieldName === fieldName); - const existsInDocs = foundField !== undefined && foundField.rawResponse.hits.hits.length > 0; + const existsInDocs = foundField !== undefined && foundField.rawResponse.hits.total > 0; const fieldData: NonAggregatableField = { fieldName, existsInDocs, From 011cc882c75cf3422b91c8d5b2a43d0018ef00b0 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 20 Oct 2021 13:55:01 -0500 Subject: [PATCH 164/188] Consolidate field type --- .../components/sidebar/discover_field.tsx | 8 +- .../sidebar/lib/get_field_type_name.ts | 9 ++ .../public/field_icon/field_icon.tsx | 3 + .../field_type_icon/field_type_icon.tsx | 95 +++---------------- .../common/util/field_types_utils.ts | 42 -------- 5 files changed, 29 insertions(+), 128 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx index ab62094215bb2..caea99169cb42 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx @@ -58,9 +58,11 @@ const FieldInfoIcon: React.FC = memo(() => ( )); -const DiscoverFieldTypeIcon: React.FC<{ field: IndexPatternField }> = memo(({ field }) => ( - -)); +const DiscoverFieldTypeIcon: React.FC<{ field: IndexPatternField }> = memo(({ field }) => { + // If it's a string type, we want to distinguish between keyword and text + const tempType = field.type === 'string' && field.esTypes ? field.esTypes[0] : field.type; + return ; +}); const FieldName: React.FC<{ field: IndexPatternField }> = memo(({ field }) => { const title = diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts b/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts index e2d4c2f7ddcf2..f68395593bd8b 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts @@ -51,6 +51,15 @@ export function getFieldTypeName(type: string) { return i18n.translate('discover.fieldNameIcons.stringFieldAriaLabel', { defaultMessage: 'String field', }); + case 'text': + return i18n.translate('discover.fieldNameIcons.textFieldAriaLabel', { + defaultMessage: 'Text field', + }); + case 'keyword': + return i18n.translate('discover.fieldNameIcons.keywordFieldAriaLabel', { + defaultMessage: 'Keyword field', + }); + case 'nested': return i18n.translate('discover.fieldNameIcons.nestedFieldAriaLabel', { defaultMessage: 'Nested field', diff --git a/src/plugins/kibana_react/public/field_icon/field_icon.tsx b/src/plugins/kibana_react/public/field_icon/field_icon.tsx index cd3c7ee9e7a99..59ba2cfe30999 100644 --- a/src/plugins/kibana_react/public/field_icon/field_icon.tsx +++ b/src/plugins/kibana_react/public/field_icon/field_icon.tsx @@ -48,8 +48,11 @@ export const typeToEuiIconMap: Partial> = { murmur3: { iconType: 'tokenFile' }, number: { iconType: 'tokenNumber' }, number_range: { iconType: 'tokenNumber' }, + histogram: { iconType: 'tokenHistogram' }, _source: { iconType: 'editorCodeBlock', color: 'gray' }, string: { iconType: 'tokenString' }, + text: { iconType: 'tokenString' }, + keyword: { iconType: 'tokenKeyword' }, nested: { iconType: 'tokenNested' }, }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx index d5ec7b45edb6d..4274ecad8cb8d 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx @@ -6,103 +6,32 @@ */ import React, { FC } from 'react'; -import { EuiToken, EuiToolTip } from '@elastic/eui'; +import { EuiToolTip } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { getJobTypeAriaLabel } from '../../util/field_types_utils'; import type { JobFieldType } from '../../../../../common'; import './_index.scss'; +import { FieldIcon } from '../../../../../../../../src/plugins/kibana_react/public'; interface FieldTypeIconProps { tooltipEnabled: boolean; type: JobFieldType; - needsAria: boolean; } -interface FieldTypeIconContainerProps { - ariaLabel: string | null; - iconType: string; - color?: string; - needsAria: boolean; - [key: string]: any; -} - -// defaultIcon => a unknown datatype -const defaultIcon = { iconType: 'questionInCircle', color: 'gray' }; - -// Extended & modified version of src/plugins/kibana_react/public/field_icon/field_icon.tsx -export const typeToEuiIconMap: Record = { - boolean: { iconType: 'tokenBoolean' }, - // icon for an index pattern mapping conflict in discover - conflict: { iconType: 'alert', color: 'euiColorVis9' }, - date: { iconType: 'tokenDate' }, - date_range: { iconType: 'tokenDate' }, - geo_point: { iconType: 'tokenGeo' }, - geo_shape: { iconType: 'tokenGeo' }, - ip: { iconType: 'tokenIP' }, - ip_range: { iconType: 'tokenIP' }, - // is a plugin's data type https://www.elastic.co/guide/en/elasticsearch/plugins/current/mapper-murmur3-usage.html - murmur3: { iconType: 'tokenFile' }, - number: { iconType: 'tokenNumber' }, - number_range: { iconType: 'tokenNumber' }, - histogram: { iconType: 'tokenHistogram' }, - _source: { iconType: 'editorCodeBlock', color: 'gray' }, - string: { iconType: 'tokenString' }, - text: { iconType: 'tokenString' }, - keyword: { iconType: 'tokenString' }, - nested: { iconType: 'tokenNested' }, -}; - -export const FieldTypeIcon: FC = ({ - tooltipEnabled = false, - type, - needsAria = true, -}) => { - const ariaLabel = getJobTypeAriaLabel(type); - const token = typeToEuiIconMap[type] || defaultIcon; - const containerProps = { ...token, ariaLabel, needsAria }; - +export const FieldTypeIcon: FC = ({ tooltipEnabled = false, type }) => { + const label = + getJobTypeAriaLabel(type) ?? + i18n.translate('xpack.dataVisualizer.fieldTypeIcon.fieldTypeTooltip', { + defaultMessage: '{type} type', + values: { type }, + }); if (tooltipEnabled === true) { return ( - - + + ); } - return ; -}; - -// If the tooltip is used, it will apply its events to its first inner child. -// To pass on its properties we apply `rest` to the outer `span` element. -const FieldTypeIconContainer: FC = ({ - ariaLabel, - iconType, - color, - needsAria, - ...rest -}) => { - const wrapperProps: { className: string; 'aria-label'?: string } = { - className: 'field-type-icon', - }; - if (needsAria && ariaLabel) { - wrapperProps['aria-label'] = ariaLabel; - } - return ( - - ); + return ; }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts index 3e459cd2b079b..ecc159d8509a5 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts @@ -46,48 +46,6 @@ export const jobTypeAriaLabels = { }), }; -export const jobTypeLabels = { - [JOB_FIELD_TYPES.BOOLEAN]: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.booleanTypeLabel', { - defaultMessage: 'Boolean', - }), - [JOB_FIELD_TYPES.DATE]: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.dateTypeLabel', { - defaultMessage: 'Date', - }), - [JOB_FIELD_TYPES.GEO_POINT]: i18n.translate( - 'xpack.dataVisualizer.fieldTypeIcon.geoPointTypeLabel', - { - defaultMessage: 'Geo point', - } - ), - [JOB_FIELD_TYPES.GEO_SHAPE]: i18n.translate( - 'xpack.dataVisualizer.fieldTypeIcon.geoShapeTypeLabel', - { - defaultMessage: 'Geo shape', - } - ), - [JOB_FIELD_TYPES.IP]: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.ipTypeLabel', { - defaultMessage: 'IP', - }), - [JOB_FIELD_TYPES.KEYWORD]: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.keywordTypeLabel', { - defaultMessage: 'Keyword', - }), - [JOB_FIELD_TYPES.NUMBER]: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.numberTypeLabel', { - defaultMessage: 'Number', - }), - [JOB_FIELD_TYPES.HISTOGRAM]: i18n.translate( - 'xpack.dataVisualizer.fieldTypeIcon.histogramTypeLabel', - { - defaultMessage: 'Histogram', - } - ), - [JOB_FIELD_TYPES.TEXT]: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.textTypeLabel', { - defaultMessage: 'Text', - }), - [JOB_FIELD_TYPES.UNKNOWN]: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.unknownTypeLabel', { - defaultMessage: 'Unknown', - }), -}; - export const getJobTypeAriaLabel = (type: string) => { const requestedFieldType = Object.keys(JOB_FIELD_TYPES).find( (k) => JOB_FIELD_TYPES[k as keyof typeof JOB_FIELD_TYPES] === type From 48f93bd3bc4caa64abc021b014b380e1632063ff Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 20 Oct 2021 13:55:01 -0500 Subject: [PATCH 165/188] Consolidate field type, add count to top values --- .../components/sidebar/discover_field.tsx | 8 +- .../sidebar/lib/get_field_type_name.ts | 9 ++ .../public/field_icon/field_icon.tsx | 3 + .../field_type_icon/field_type_icon.tsx | 97 +++---------------- .../components/top_values/top_values.tsx | 8 +- .../common/util/field_types_utils.test.ts | 10 +- .../common/util/field_types_utils.ts | 46 +-------- .../requests/get_field_stats.ts | 4 +- .../translations/translations/ja-JP.json | 8 -- .../translations/translations/zh-CN.json | 8 -- 10 files changed, 42 insertions(+), 159 deletions(-) diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx index ab62094215bb2..caea99169cb42 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx @@ -58,9 +58,11 @@ const FieldInfoIcon: React.FC = memo(() => ( )); -const DiscoverFieldTypeIcon: React.FC<{ field: IndexPatternField }> = memo(({ field }) => ( - -)); +const DiscoverFieldTypeIcon: React.FC<{ field: IndexPatternField }> = memo(({ field }) => { + // If it's a string type, we want to distinguish between keyword and text + const tempType = field.type === 'string' && field.esTypes ? field.esTypes[0] : field.type; + return ; +}); const FieldName: React.FC<{ field: IndexPatternField }> = memo(({ field }) => { const title = diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts b/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts index e2d4c2f7ddcf2..f68395593bd8b 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts @@ -51,6 +51,15 @@ export function getFieldTypeName(type: string) { return i18n.translate('discover.fieldNameIcons.stringFieldAriaLabel', { defaultMessage: 'String field', }); + case 'text': + return i18n.translate('discover.fieldNameIcons.textFieldAriaLabel', { + defaultMessage: 'Text field', + }); + case 'keyword': + return i18n.translate('discover.fieldNameIcons.keywordFieldAriaLabel', { + defaultMessage: 'Keyword field', + }); + case 'nested': return i18n.translate('discover.fieldNameIcons.nestedFieldAriaLabel', { defaultMessage: 'Nested field', diff --git a/src/plugins/kibana_react/public/field_icon/field_icon.tsx b/src/plugins/kibana_react/public/field_icon/field_icon.tsx index cd3c7ee9e7a99..59ba2cfe30999 100644 --- a/src/plugins/kibana_react/public/field_icon/field_icon.tsx +++ b/src/plugins/kibana_react/public/field_icon/field_icon.tsx @@ -48,8 +48,11 @@ export const typeToEuiIconMap: Partial> = { murmur3: { iconType: 'tokenFile' }, number: { iconType: 'tokenNumber' }, number_range: { iconType: 'tokenNumber' }, + histogram: { iconType: 'tokenHistogram' }, _source: { iconType: 'editorCodeBlock', color: 'gray' }, string: { iconType: 'tokenString' }, + text: { iconType: 'tokenString' }, + keyword: { iconType: 'tokenKeyword' }, nested: { iconType: 'tokenNested' }, }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx index d5ec7b45edb6d..4e84e538c0ab8 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx @@ -6,103 +6,32 @@ */ import React, { FC } from 'react'; -import { EuiToken, EuiToolTip } from '@elastic/eui'; +import { EuiToolTip } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { getJobTypeAriaLabel } from '../../util/field_types_utils'; +import { getJobTypeLabel } from '../../util/field_types_utils'; import type { JobFieldType } from '../../../../../common'; import './_index.scss'; +import { FieldIcon } from '../../../../../../../../src/plugins/kibana_react/public'; interface FieldTypeIconProps { tooltipEnabled: boolean; type: JobFieldType; - needsAria: boolean; } -interface FieldTypeIconContainerProps { - ariaLabel: string | null; - iconType: string; - color?: string; - needsAria: boolean; - [key: string]: any; -} - -// defaultIcon => a unknown datatype -const defaultIcon = { iconType: 'questionInCircle', color: 'gray' }; - -// Extended & modified version of src/plugins/kibana_react/public/field_icon/field_icon.tsx -export const typeToEuiIconMap: Record = { - boolean: { iconType: 'tokenBoolean' }, - // icon for an index pattern mapping conflict in discover - conflict: { iconType: 'alert', color: 'euiColorVis9' }, - date: { iconType: 'tokenDate' }, - date_range: { iconType: 'tokenDate' }, - geo_point: { iconType: 'tokenGeo' }, - geo_shape: { iconType: 'tokenGeo' }, - ip: { iconType: 'tokenIP' }, - ip_range: { iconType: 'tokenIP' }, - // is a plugin's data type https://www.elastic.co/guide/en/elasticsearch/plugins/current/mapper-murmur3-usage.html - murmur3: { iconType: 'tokenFile' }, - number: { iconType: 'tokenNumber' }, - number_range: { iconType: 'tokenNumber' }, - histogram: { iconType: 'tokenHistogram' }, - _source: { iconType: 'editorCodeBlock', color: 'gray' }, - string: { iconType: 'tokenString' }, - text: { iconType: 'tokenString' }, - keyword: { iconType: 'tokenString' }, - nested: { iconType: 'tokenNested' }, -}; - -export const FieldTypeIcon: FC = ({ - tooltipEnabled = false, - type, - needsAria = true, -}) => { - const ariaLabel = getJobTypeAriaLabel(type); - const token = typeToEuiIconMap[type] || defaultIcon; - const containerProps = { ...token, ariaLabel, needsAria }; - +export const FieldTypeIcon: FC = ({ tooltipEnabled = false, type }) => { + const label = + getJobTypeLabel(type) ?? + i18n.translate('xpack.dataVisualizer.fieldTypeIcon.fieldTypeTooltip', { + defaultMessage: '{type} type', + values: { type }, + }); if (tooltipEnabled === true) { return ( - - + + ); } - return ; -}; - -// If the tooltip is used, it will apply its events to its first inner child. -// To pass on its properties we apply `rest` to the outer `span` element. -const FieldTypeIconContainer: FC = ({ - ariaLabel, - iconType, - color, - needsAria, - ...rest -}) => { - const wrapperProps: { className: string; 'aria-label'?: string } = { - className: 'field-type-icon', - }; - if (needsAria && ariaLabel) { - wrapperProps['aria-label'] = ariaLabel; - } - return ( - - ); + return ; }; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index e128e1db24dd5..c9b4137a0106d 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -81,11 +81,11 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed, size="xs" label={kibanaFieldFormat(value.key, fieldFormat)} className={classNames('eui-textTruncate', 'topValuesValueLabelContainer')} - valueText={ + valueText={`${value.doc_count}${ progressBarMax !== undefined - ? getPercentLabel(value.doc_count, progressBarMax) - : undefined - } + ? ` (${getPercentLabel(value.doc_count, progressBarMax)})` + : '' + }`} /> {fieldName !== undefined && value.key !== undefined && onAddFilter !== undefined ? ( diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.test.ts b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.test.ts index 5c0867c7a0745..055f043fb3cd9 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.test.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.test.ts @@ -6,16 +6,16 @@ */ import { JOB_FIELD_TYPES } from '../../../../common'; -import { getJobTypeAriaLabel, jobTypeAriaLabels } from './field_types_utils'; +import { getJobTypeLabel, jobTypeLabels } from './field_types_utils'; describe('field type utils', () => { - describe('getJobTypeAriaLabel: Getting a field type aria label by passing what it is stored in constants', () => { + describe('getJobTypeLabel: Getting a field type aria label by passing what it is stored in constants', () => { test('should returns all JOB_FIELD_TYPES labels exactly as it is for each correct value', () => { const keys = Object.keys(JOB_FIELD_TYPES); const receivedLabels: Record = {}; - const testStorage = jobTypeAriaLabels; + const testStorage = jobTypeLabels; keys.forEach((constant) => { - receivedLabels[constant] = getJobTypeAriaLabel( + receivedLabels[constant] = getJobTypeLabel( JOB_FIELD_TYPES[constant as keyof typeof JOB_FIELD_TYPES] ); }); @@ -23,7 +23,7 @@ describe('field type utils', () => { expect(receivedLabels).toEqual(testStorage); }); test('should returns NULL as JOB_FIELD_TYPES does not contain such a keyword', () => { - expect(getJobTypeAriaLabel('JOB_FIELD_TYPES')).toBe(null); + expect(getJobTypeLabel('JOB_FIELD_TYPES')).toBe(null); }); }); }); diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts index 3e459cd2b079b..1fda7140dbab2 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts @@ -10,40 +10,8 @@ import { JOB_FIELD_TYPES } from '../../../../common'; import type { IndexPatternField } from '../../../../../../../src/plugins/data/common'; import { KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/common'; -export const jobTypeAriaLabels = { - BOOLEAN: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.booleanTypeAriaLabel', { - defaultMessage: 'boolean type', - }), - DATE: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.dateTypeAriaLabel', { - defaultMessage: 'date type', - }), - GEO_POINT: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.geoPointTypeAriaLabel', { - defaultMessage: '{geoPointParam} type', - values: { - geoPointParam: 'geo point', - }, - }), - GEO_SHAPE: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.geoShapeTypeAriaLabel', { - defaultMessage: 'geo shape type', - }), - IP: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.ipTypeAriaLabel', { - defaultMessage: 'ip type', - }), - KEYWORD: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.keywordTypeAriaLabel', { - defaultMessage: 'keyword type', - }), - NUMBER: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.numberTypeAriaLabel', { - defaultMessage: 'number type', - }), - HISTOGRAM: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.histogramTypeAriaLabel', { - defaultMessage: 'histogram type', - }), - TEXT: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.textTypeAriaLabel', { - defaultMessage: 'text type', - }), - UNKNOWN: i18n.translate('xpack.dataVisualizer.fieldTypeIcon.unknownTypeAriaLabel', { - defaultMessage: 'unknown type', - }), +export const getJobTypeLabel = (type: string) => { + return type in jobTypeLabels ? jobTypeLabels[type as keyof typeof jobTypeLabels] : null; }; export const jobTypeLabels = { @@ -88,16 +56,6 @@ export const jobTypeLabels = { }), }; -export const getJobTypeAriaLabel = (type: string) => { - const requestedFieldType = Object.keys(JOB_FIELD_TYPES).find( - (k) => JOB_FIELD_TYPES[k as keyof typeof JOB_FIELD_TYPES] === type - ); - if (requestedFieldType === undefined) { - return null; - } - return jobTypeAriaLabels[requestedFieldType as keyof typeof jobTypeAriaLabels]; -}; - // convert kibana types to ML Job types // this is needed because kibana types only have string and not text and keyword. // and we can't use ES_FIELD_TYPES because it has no NUMBER type diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts index 2483ef5b4624a..41c484ac1a4f2 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts @@ -39,7 +39,7 @@ export const getFieldStats = ( case JOB_FIELD_TYPES.NUMBER: return fetchNumericFieldStats(dataPlugin, params, field, options); case JOB_FIELD_TYPES.KEYWORD: - // case JOB_FIELD_TYPES.IP: + case JOB_FIELD_TYPES.IP: return fetchStringFieldStats(dataPlugin, params, field, options); case JOB_FIELD_TYPES.DATE: return fetchDateFieldStats(dataPlugin, params, field, options); @@ -47,8 +47,6 @@ export const getFieldStats = ( return fetchBooleanFieldStats(dataPlugin, params, field, options); case JOB_FIELD_TYPES.TEXT: return fetchFieldExamples(dataPlugin, params, field, options); - case JOB_FIELD_TYPES.IP: - return fetchStringFieldStats(dataPlugin, params, field, options); default: // Use an exists filter on the the field name to get // examples of the field, so cannot batch up. diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index f831f3b91eaa5..2697a88b329db 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -9074,15 +9074,7 @@ "xpack.dataVisualizer.fieldStats.maxTitle": "最高", "xpack.dataVisualizer.fieldStats.medianTitle": "中間", "xpack.dataVisualizer.fieldStats.minTitle": "分", - "xpack.dataVisualizer.fieldTypeIcon.booleanTypeAriaLabel": "ブールタイプ", - "xpack.dataVisualizer.fieldTypeIcon.dateTypeAriaLabel": "日付タイプ", "xpack.dataVisualizer.fieldTypeIcon.fieldTypeTooltip": "{type} タイプ", - "xpack.dataVisualizer.fieldTypeIcon.geoPointTypeAriaLabel": "{geoPointParam} タイプ", - "xpack.dataVisualizer.fieldTypeIcon.ipTypeAriaLabel": "IP タイプ", - "xpack.dataVisualizer.fieldTypeIcon.keywordTypeAriaLabel": "キーワードタイプ", - "xpack.dataVisualizer.fieldTypeIcon.numberTypeAriaLabel": "数字タイプ", - "xpack.dataVisualizer.fieldTypeIcon.textTypeAriaLabel": "テキストタイプ", - "xpack.dataVisualizer.fieldTypeIcon.unknownTypeAriaLabel": "不明なタイプ", "xpack.dataVisualizer.fieldTypeSelect": "フィールド型", "xpack.dataVisualizer.file.aboutPanel.analyzingDataTitle": "データを分析中", "xpack.dataVisualizer.file.aboutPanel.selectOrDragAndDropFileDescription": "ファイルを選択するかドラッグ & ドロップしてください", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 5fbcd26340be3..29141fd01e9b5 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -9160,15 +9160,7 @@ "xpack.dataVisualizer.fieldStats.maxTitle": "最大值", "xpack.dataVisualizer.fieldStats.medianTitle": "中值", "xpack.dataVisualizer.fieldStats.minTitle": "最小值", - "xpack.dataVisualizer.fieldTypeIcon.booleanTypeAriaLabel": "布尔类型", - "xpack.dataVisualizer.fieldTypeIcon.dateTypeAriaLabel": "日期类型", "xpack.dataVisualizer.fieldTypeIcon.fieldTypeTooltip": "{type} 类型", - "xpack.dataVisualizer.fieldTypeIcon.geoPointTypeAriaLabel": "{geoPointParam} 类型", - "xpack.dataVisualizer.fieldTypeIcon.ipTypeAriaLabel": "IP 类型", - "xpack.dataVisualizer.fieldTypeIcon.keywordTypeAriaLabel": "关键字类型", - "xpack.dataVisualizer.fieldTypeIcon.numberTypeAriaLabel": "数字类型", - "xpack.dataVisualizer.fieldTypeIcon.textTypeAriaLabel": "文本类型", - "xpack.dataVisualizer.fieldTypeIcon.unknownTypeAriaLabel": "未知类型", "xpack.dataVisualizer.fieldTypeSelect": "字段类型", "xpack.dataVisualizer.file.aboutPanel.analyzingDataTitle": "正在分析数据", "xpack.dataVisualizer.file.aboutPanel.selectOrDragAndDropFileDescription": "选择或拖放文件", From 3c3e412c6e75768a971c9ca2362616c1c70be74b Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 20 Oct 2021 15:18:16 -0500 Subject: [PATCH 166/188] Clean up --- .../field_stats_table/field_stats_table.tsx | 4 +- .../field_type_icon/field_type_icon.test.tsx | 4 +- .../field_types_filter/field_types_filter.tsx | 2 +- .../data_visualizer_stats_table.tsx | 2 +- .../search_panel/field_type_filter.tsx | 2 +- .../data_visualizer/get_field_histograms.ts | 121 ------------------ .../apis/ml/data_visualizer/index.ts | 15 --- x-pack/test/api_integration/apis/ml/index.ts | 1 - 8 files changed, 7 insertions(+), 144 deletions(-) delete mode 100644 x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts delete mode 100644 x-pack/test/api_integration/apis/ml/data_visualizer/index.ts diff --git a/src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx b/src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx index 02e7192b58e5b..0af7dd54a009b 100644 --- a/src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx +++ b/src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx @@ -86,7 +86,7 @@ export interface DiscoverDataVisualizerGridProps { * @param eventName */ trackUiMetric?: (metricType: UiCounterMetricType, eventName: string | string[]) => void; - savedSearchRefetch$: DataRefetch$; + savedSearchRefetch$?: DataRefetch$; } export const FieldStatisticsTable = (props: DiscoverDataVisualizerGridProps) => { @@ -123,7 +123,7 @@ export const FieldStatisticsTable = (props: DiscoverDataVisualizerGridProps) => } }); - const refetch = savedSearchRefetch$.subscribe(() => { + const refetch = savedSearchRefetch$?.subscribe(() => { if (embeddable && !isErrorEmbeddable(embeddable)) { embeddable.updateInput({ lastReloadRequestTime: Date.now() }); } diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.test.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.test.tsx index b6a5ff3e5dbed..0c036dd6c6d76 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.test.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.test.tsx @@ -14,7 +14,7 @@ import { JOB_FIELD_TYPES } from '../../../../../common'; describe('FieldTypeIcon', () => { test(`render component when type matches a field type`, () => { const typeIconComponent = shallow( - + ); expect(typeIconComponent).toMatchSnapshot(); }); @@ -24,7 +24,7 @@ describe('FieldTypeIcon', () => { jest.useFakeTimers(); const typeIconComponent = mount( - + ); expect(typeIconComponent.find('EuiToolTip').children()).toHaveLength(1); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx index 23360bd7a0c87..0fa860bc6f55e 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_types_filter/field_types_filter.tsx @@ -50,7 +50,7 @@ export const DataVisualizerFieldTypesFilter: FC = ({ {label} {type && ( - + )} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index b4608006dfd83..c00293937b5ca 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -181,7 +181,7 @@ export const DataVisualizerTable = ({ defaultMessage: 'Type', }), render: (fieldType: JobFieldType) => { - return ; + return ; }, width: dimensions.type, sortable: true, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/field_type_filter.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/field_type_filter.tsx index 7e86425c0a891..ee54683b08435 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/field_type_filter.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/field_type_filter.tsx @@ -29,7 +29,7 @@ export const DataVisualizerFieldTypeFilter: FC<{ {label} {indexedFieldName && ( - + )} diff --git a/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts b/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts deleted file mode 100644 index 488df74b46968..0000000000000 --- a/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; - -import { FtrProviderContext } from '../../../ftr_provider_context'; -import { USER } from '../../../../functional/services/ml/security_common'; -import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api'; - -export default ({ getService }: FtrProviderContext) => { - const esArchiver = getService('esArchiver'); - const supertest = getService('supertestWithoutAuth'); - const ml = getService('ml'); - - const fieldHistogramsTestData = { - testTitle: 'returns histogram data for fields', - index: 'ft_farequote', - user: USER.ML_POWERUSER, - requestBody: { - query: { bool: { should: [{ match_phrase: { airline: 'JZA' } }], minimum_should_match: 1 } }, - fields: [ - { fieldName: '@timestamp', type: 'date' }, - { fieldName: 'airline', type: 'string' }, - { fieldName: 'responsetime', type: 'number' }, - ], - samplerShardSize: -1, // No sampling, as otherwise counts could vary on each run. - }, - expected: { - responseCode: 200, - responseBody: [ - { - dataLength: 20, - type: 'numeric', - id: '@timestamp', - }, - { type: 'ordinal', dataLength: 1, id: 'airline' }, - { - dataLength: 20, - type: 'numeric', - id: 'responsetime', - }, - ], - }, - }; - - const errorTestData = { - testTitle: 'returns error for index which does not exist', - index: 'ft_farequote_not_exists', - user: USER.ML_POWERUSER, - requestBody: { - query: { bool: { must: [{ match_all: {} }] } }, - fields: [{ fieldName: 'responsetime', type: 'number' }], - samplerShardSize: -1, - }, - expected: { - responseCode: 404, - responseBody: { - statusCode: 404, - error: 'Not Found', - message: 'index_not_found_exception', - }, - }, - }; - - async function runGetFieldHistogramsRequest( - index: string, - user: USER, - requestBody: object, - expectedResponsecode: number - ): Promise { - const { body } = await supertest - .post(`/api/ml/data_visualizer/get_field_histograms/${index}`) - .auth(user, ml.securityCommon.getPasswordForUser(user)) - .set(COMMON_REQUEST_HEADERS) - .send(requestBody) - .expect(expectedResponsecode); - - return body; - } - - describe('get_field_histograms', function () { - before(async () => { - await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); - await ml.testResources.setKibanaTimeZoneToUTC(); - }); - - it(`${fieldHistogramsTestData.testTitle}`, async () => { - const body = await runGetFieldHistogramsRequest( - fieldHistogramsTestData.index, - fieldHistogramsTestData.user, - fieldHistogramsTestData.requestBody, - fieldHistogramsTestData.expected.responseCode - ); - - const expected = fieldHistogramsTestData.expected; - - const actual = body.map((b: any) => ({ - dataLength: b.data.length, - type: b.type, - id: b.id, - })); - expect(actual).to.eql(expected.responseBody); - }); - - it(`${errorTestData.testTitle}`, async () => { - const body = await runGetFieldHistogramsRequest( - errorTestData.index, - errorTestData.user, - errorTestData.requestBody, - errorTestData.expected.responseCode - ); - - expect(body.error).to.eql(errorTestData.expected.responseBody.error); - expect(body.message).to.contain(errorTestData.expected.responseBody.message); - }); - }); -}; diff --git a/x-pack/test/api_integration/apis/ml/data_visualizer/index.ts b/x-pack/test/api_integration/apis/ml/data_visualizer/index.ts deleted file mode 100644 index 3865247979a4a..0000000000000 --- a/x-pack/test/api_integration/apis/ml/data_visualizer/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ loadTestFile }: FtrProviderContext) { - describe('data visualizer', function () { - loadTestFile(require.resolve('./get_field_stats')); - loadTestFile(require.resolve('./get_overall_stats')); - }); -} diff --git a/x-pack/test/api_integration/apis/ml/index.ts b/x-pack/test/api_integration/apis/ml/index.ts index e44d0cd10e9f2..6879b8efef06c 100644 --- a/x-pack/test/api_integration/apis/ml/index.ts +++ b/x-pack/test/api_integration/apis/ml/index.ts @@ -72,7 +72,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./anomaly_detectors')); loadTestFile(require.resolve('./calendars')); loadTestFile(require.resolve('./data_frame_analytics')); - loadTestFile(require.resolve('./data_visualizer')); loadTestFile(require.resolve('./fields_service')); loadTestFile(require.resolve('./filters')); loadTestFile(require.resolve('./indices')); From a89be4fea4cbfb4d17644939833d20df816f76b4 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 20 Oct 2021 19:19:02 -0500 Subject: [PATCH 167/188] Update tests --- .../field_type_icon.test.tsx.snap | 8 ++--- .../common/util/field_types_utils.test.ts | 7 ++-- .../test/api_integration_basic/apis/index.ts | 1 - .../apis/ml/data_visualizer/index.ts | 15 -------- .../api_integration_basic/apis/ml/index.ts | 35 ------------------- 5 files changed, 6 insertions(+), 60 deletions(-) delete mode 100644 x-pack/test/api_integration_basic/apis/ml/data_visualizer/index.ts delete mode 100644 x-pack/test/api_integration_basic/apis/ml/index.ts diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/__snapshots__/field_type_icon.test.tsx.snap b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/__snapshots__/field_type_icon.test.tsx.snap index 398dc5dad2dc7..af4464cbc6b4e 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/__snapshots__/field_type_icon.test.tsx.snap +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/__snapshots__/field_type_icon.test.tsx.snap @@ -3,15 +3,13 @@ exports[`FieldTypeIcon render component when type matches a field type 1`] = ` - `; diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.test.ts b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.test.ts index 055f043fb3cd9..710ba12313f17 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.test.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.test.ts @@ -14,10 +14,9 @@ describe('field type utils', () => { const keys = Object.keys(JOB_FIELD_TYPES); const receivedLabels: Record = {}; const testStorage = jobTypeLabels; - keys.forEach((constant) => { - receivedLabels[constant] = getJobTypeLabel( - JOB_FIELD_TYPES[constant as keyof typeof JOB_FIELD_TYPES] - ); + keys.forEach((key) => { + const constant = key as keyof typeof JOB_FIELD_TYPES; + receivedLabels[JOB_FIELD_TYPES[constant]] = getJobTypeLabel(JOB_FIELD_TYPES[constant]); }); expect(receivedLabels).toEqual(testStorage); diff --git a/x-pack/test/api_integration_basic/apis/index.ts b/x-pack/test/api_integration_basic/apis/index.ts index 27869095bd792..9490d4c277675 100644 --- a/x-pack/test/api_integration_basic/apis/index.ts +++ b/x-pack/test/api_integration_basic/apis/index.ts @@ -11,7 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { describe('apis', function () { this.tags('ciGroup11'); - loadTestFile(require.resolve('./ml')); loadTestFile(require.resolve('./transform')); loadTestFile(require.resolve('./security_solution')); }); diff --git a/x-pack/test/api_integration_basic/apis/ml/data_visualizer/index.ts b/x-pack/test/api_integration_basic/apis/ml/data_visualizer/index.ts deleted file mode 100644 index 85b462a01760b..0000000000000 --- a/x-pack/test/api_integration_basic/apis/ml/data_visualizer/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ loadTestFile }: FtrProviderContext) { - describe('data visualizer', function () { - // The data visualizer APIs should work the same as with a trial license - loadTestFile(require.resolve('../../../../api_integration/apis/ml/data_visualizer')); - }); -} diff --git a/x-pack/test/api_integration_basic/apis/ml/index.ts b/x-pack/test/api_integration_basic/apis/ml/index.ts deleted file mode 100644 index 5ca70103f41eb..0000000000000 --- a/x-pack/test/api_integration_basic/apis/ml/index.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../ftr_provider_context'; - -export default function ({ getService, loadTestFile }: FtrProviderContext) { - const esArchiver = getService('esArchiver'); - const ml = getService('ml'); - - describe('machine learning basic license', function () { - this.tags(['mlqa']); - - before(async () => { - await ml.securityCommon.createMlRoles(); - await ml.securityCommon.createMlUsers(); - }); - - after(async () => { - await ml.securityCommon.cleanMlUsers(); - await ml.securityCommon.cleanMlRoles(); - - await ml.testResources.deleteIndexPatternByTitle('ft_farequote'); - - await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote'); - - await ml.testResources.resetKibanaTimeZone(); - }); - - loadTestFile(require.resolve('./data_visualizer')); - }); -} From d0903dff33e3d75c794a58cb42d57e44463ccf2a Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 21 Oct 2021 11:49:46 -0500 Subject: [PATCH 168/188] Remove progress on embedadble --- .../index_data_visualizer_view.tsx | 3 +++ .../grid_embeddable/grid_embeddable.tsx | 23 ++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 206ba9aeb766f..9c855e9a11b50 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -15,6 +15,7 @@ import { EuiPageContentHeader, EuiPageContentHeaderSection, EuiPanel, + EuiProgress, EuiSpacer, EuiTitle, } from '@elastic/eui'; @@ -245,6 +246,7 @@ export const IndexDataVisualizerView: FC = (dataVi metricsStats, timefilter, setLastRefresh, + progress, } = useDataVisualizerGridData(input, dataVisualizerListState, setGlobalState); useEffect(() => { @@ -489,6 +491,7 @@ export const IndexDataVisualizerView: FC = (dataVi metricsStats={metricsStats} /> + items={configs} pageState={dataVisualizerListState} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 36783a8e8b6be..a3681ae643e5c 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -10,7 +10,7 @@ import { CoreStart } from 'kibana/public'; import ReactDOM from 'react-dom'; import React, { Suspense, useCallback, useEffect, useState } from 'react'; import useObservable from 'react-use/lib/useObservable'; -import { EuiEmptyPrompt, EuiProgress, EuiIcon, EuiSpacer, EuiText } from '@elastic/eui'; +import { EuiEmptyPrompt, EuiIcon, EuiSpacer, EuiText } from '@elastic/eui'; import { Filter } from '@kbn/es-query'; import { Required } from 'utility-types'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -130,18 +130,15 @@ export const EmbeddableWrapper = ({ ); } return ( -
- - - items={configs} - pageState={dataVisualizerListState} - updatePageState={onTableChange} - getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} - extendedColumns={extendedColumns} - showPreviewByDefault={input?.showPreviewByDefault} - onChange={onOutputChange} - /> -
+ + items={configs} + pageState={dataVisualizerListState} + updatePageState={onTableChange} + getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} + extendedColumns={extendedColumns} + showPreviewByDefault={input?.showPreviewByDefault} + onChange={onOutputChange} + /> ); }; From 09df96ed5ccbd074e29ac9ed23308fe0f94c1c0c Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 21 Oct 2021 15:23:37 -0500 Subject: [PATCH 169/188] Update snapshot --- .../__snapshots__/field_icon.test.tsx.snap | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/plugins/kibana_react/public/field_icon/__snapshots__/field_icon.test.tsx.snap b/src/plugins/kibana_react/public/field_icon/__snapshots__/field_icon.test.tsx.snap index f6870a5209c1e..0e9ae4ee2aaaa 100644 --- a/src/plugins/kibana_react/public/field_icon/__snapshots__/field_icon.test.tsx.snap +++ b/src/plugins/kibana_react/public/field_icon/__snapshots__/field_icon.test.tsx.snap @@ -95,6 +95,16 @@ exports[`FieldIcon renders known field types geo_shape is rendered 1`] = ` /> `; +exports[`FieldIcon renders known field types histogram is rendered 1`] = ` + +`; + exports[`FieldIcon renders known field types ip is rendered 1`] = ` `; +exports[`FieldIcon renders known field types keyword is rendered 1`] = ` + +`; + exports[`FieldIcon renders known field types murmur3 is rendered 1`] = ` `; +exports[`FieldIcon renders known field types text is rendered 1`] = ` + +`; + exports[`FieldIcon renders with className if provided 1`] = ` Date: Mon, 25 Oct 2021 13:18:38 -0500 Subject: [PATCH 170/188] Clean up, consolidate searchOptions --- .../common/types/field_vis_config.ts | 1 - .../document_count_content.tsx | 1 - .../hooks/use_field_stats.ts | 3 +- .../hooks/use_overall_stats.ts | 28 ++++++++----------- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/data_visualizer/common/types/field_vis_config.ts b/x-pack/plugins/data_visualizer/common/types/field_vis_config.ts index 5340fe4f77d15..dcd7da74b85ef 100644 --- a/x-pack/plugins/data_visualizer/common/types/field_vis_config.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_vis_config.ts @@ -6,7 +6,6 @@ */ import type { Percentile, JobFieldType, FieldVisStats } from './index'; -// @todo: move this back? export interface MetricFieldVisStats { avg?: number; distribution?: { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx index 1f010dcba1b1b..832e18a12369f 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx @@ -15,7 +15,6 @@ export interface Props { totalCount: number; } -// @todo: update document count stats for file based export const DocumentCountContent: FC = ({ documentCountStats, totalCount }) => { if (documentCountStats === undefined) { return totalCount !== undefined ? : null; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index c9a22422cdb3e..4d8a1ad0cd7fd 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -25,6 +25,7 @@ import { getFieldStats } from '../search_strategy/requests/get_field_stats'; import type { FieldStats, FieldStatsError } from '../../../../common/types/field_stats'; import { getInitialProgress, getReducer } from '../progress_utils'; import { MAX_EXAMPLES_DEFAULT } from '../search_strategy/requests/constants'; +import type { ISearchOptions } from '../../../../../../../src/plugins/data/common'; interface FieldStatsParams { metricConfigs: FieldRequestConfig[]; @@ -113,7 +114,7 @@ export function useFieldStatsSearchStrategy( }, maxExamples: MAX_EXAMPLES_DEFAULT, }; - const searchOptions = { + const searchOptions: ISearchOptions = { abortSignal: abortCtrl.current.signal, sessionId: searchStrategyParams?.sessionId, }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index a3230a8c53599..3713457a33149 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -9,7 +9,7 @@ import { useCallback, useEffect, useState, useRef, useMemo, useReducer } from 'r import { combineLatest, forkJoin, of, Subscription } from 'rxjs'; import { mergeMap, switchMap } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; -import { ToastsStart } from 'kibana/public'; +import type { ToastsStart } from 'kibana/public'; import { useDataVisualizerKibana } from '../../kibana_context'; import { checkAggregatableFieldsExistRequest, @@ -17,14 +17,15 @@ import { processAggregatableFieldsExistResponse, processNonAggregatableFieldsExistResponse, } from '../search_strategy/requests/overall_stats'; -import { +import type { IKibanaSearchRequest, IKibanaSearchResponse, + ISearchOptions, } from '../../../../../../../src/plugins/data/common'; -import { OverallStats } from '../types/overall_stats'; +import type { OverallStats } from '../types/overall_stats'; import { getDefaultPageState } from '../components/index_data_visualizer_view/index_data_visualizer_view'; import { extractErrorProperties } from '../utils/error_utils'; -import { +import type { DataStatsFetchProgress, OverallStatsSearchStrategyParams, } from '../../../../common/types/field_stats'; @@ -106,6 +107,10 @@ export function useOverallStats 0 ? combineLatest( @@ -123,10 +128,7 @@ export function useOverallStats { @@ -156,10 +158,7 @@ export function useOverallStats Date: Tue, 26 Oct 2021 21:17:47 -0500 Subject: [PATCH 171/188] Fix new es client types --- .../components/layout/discover_layout.tsx | 4 +- .../data_visualizer_grid.tsx | 196 ------------------ .../field_stats_table_embeddable.tsx | 31 --- .../components/data_visualizer_grid/index.ts | 9 - .../field_stats_table/field_stats_table.tsx | 4 +- ...ld_stats_table_saved_search_embeddable.tsx | 4 +- .../common/types/field_stats.ts | 5 +- .../common/utils/query_utils.ts | 2 +- .../requests/get_boolean_field_stats.ts | 4 +- .../requests/get_date_field_stats.ts | 4 +- .../requests/get_document_stats.ts | 2 +- .../requests/get_field_examples.ts | 5 +- .../requests/get_numeric_field_stats.ts | 4 +- .../requests/get_string_field_stats.ts | 7 +- .../apps/ml/data_visualizer/types.ts | 4 +- 15 files changed, 24 insertions(+), 261 deletions(-) delete mode 100644 src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx delete mode 100644 src/plugins/discover/public/application/components/data_visualizer_grid/field_stats_table_embeddable.tsx delete mode 100644 src/plugins/discover/public/application/components/data_visualizer_grid/index.ts diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx index 0009002b012e9..f21a56b6d0581 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx @@ -58,7 +58,7 @@ export const SIDEBAR_CLOSED_KEY = 'discover:sidebarClosed'; const SidebarMemoized = React.memo(DiscoverSidebarResponsive); const TopNavMemoized = React.memo(DiscoverTopNav); const DiscoverChartMemoized = React.memo(DiscoverChart); -const DataVisualizerGridMemoized = React.memo(FieldStatisticsTable); +const FieldStatisticsTableMemoized = React.memo(FieldStatisticsTable); export function DiscoverLayout({ indexPattern, @@ -327,7 +327,7 @@ export function DiscoverLayout({ stateContainer={stateContainer} /> ) : ( - void; -} -export interface DataVisualizerGridEmbeddableOutput extends EmbeddableOutput { - showDistributions?: boolean; -} - -export interface DiscoverDataVisualizerGridProps { - /** - * Determines which columns are displayed - */ - columns: string[]; - /** - * The used index pattern - */ - indexPattern: DataView; - /** - * Saved search description - */ - searchDescription?: string; - /** - * Saved search title - */ - searchTitle?: string; - /** - * Discover plugin services - */ - services: DiscoverServices; - /** - * Optional saved search - */ - savedSearch?: SavedSearch; - /** - * Optional query to update the table content - */ - query?: Query; - /** - * Filters query to update the table content - */ - filters?: Filter[]; - stateContainer?: GetStateReturn; - /** - * Callback to add a filter to filter bar - */ - onAddFilter?: (field: IndexPatternField | string, value: string, type: '+' | '-') => void; -} - -export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => { - const { - services, - indexPattern, - savedSearch, - query, - columns, - filters, - stateContainer, - onAddFilter, - } = props; - const { uiSettings } = services; - - const [embeddable, setEmbeddable] = useState< - | ErrorEmbeddable - | IEmbeddable - | undefined - >(); - const embeddableRoot: React.RefObject = useRef(null); - - const showPreviewByDefault = useMemo( - () => - stateContainer ? !stateContainer.appStateContainer.getState().hideAggregatedPreview : true, - [stateContainer] - ); - - useEffect(() => { - const sub = embeddable?.getOutput$().subscribe((output: DataVisualizerGridEmbeddableOutput) => { - if (output.showDistributions !== undefined && stateContainer) { - stateContainer.setAppState({ hideAggregatedPreview: !output.showDistributions }); - } - }); - - return () => { - sub?.unsubscribe(); - }; - }, [embeddable, stateContainer]); - - useEffect(() => { - if (embeddable && !isErrorEmbeddable(embeddable)) { - // Update embeddable whenever one of the important input changes - embeddable.updateInput({ - indexPattern, - savedSearch, - query, - filters, - visibleFieldNames: columns, - onAddFilter, - }); - embeddable.reload(); - } - }, [embeddable, indexPattern, savedSearch, query, columns, filters, onAddFilter]); - - useEffect(() => { - if (showPreviewByDefault && embeddable && !isErrorEmbeddable(embeddable)) { - // Update embeddable whenever one of the important input changes - embeddable.updateInput({ - showPreviewByDefault, - }); - embeddable.reload(); - } - }, [showPreviewByDefault, uiSettings, embeddable]); - - useEffect(() => { - return () => { - // Clean up embeddable upon unmounting - embeddable?.destroy(); - }; - }, [embeddable]); - - useEffect(() => { - let unmounted = false; - const loadEmbeddable = async () => { - if (services.embeddable) { - const factory = services.embeddable.getEmbeddableFactory< - DataVisualizerGridEmbeddableInput, - DataVisualizerGridEmbeddableOutput - >('data_visualizer_grid'); - if (factory) { - // Initialize embeddable with information available at mount - const initializedEmbeddable = await factory.create({ - id: 'discover_data_visualizer_grid', - indexPattern, - savedSearch, - query, - showPreviewByDefault, - onAddFilter, - }); - if (!unmounted) { - setEmbeddable(initializedEmbeddable); - } - } - } - }; - loadEmbeddable(); - return () => { - unmounted = true; - }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [services.embeddable, showPreviewByDefault]); - - // We can only render after embeddable has already initialized - useEffect(() => { - if (embeddableRoot.current && embeddable) { - embeddable.render(embeddableRoot.current); - } - }, [embeddable, embeddableRoot, uiSettings]); - - return ( -
- ); -}; diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/field_stats_table_embeddable.tsx b/src/plugins/discover/public/application/components/data_visualizer_grid/field_stats_table_embeddable.tsx deleted file mode 100644 index 099f45bf988cc..0000000000000 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/field_stats_table_embeddable.tsx +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import React from 'react'; -import { I18nProvider } from '@kbn/i18n/react'; -import { - DiscoverDataVisualizerGrid, - DiscoverDataVisualizerGridProps, -} from './data_visualizer_grid'; - -export function FieldStatsTableEmbeddable(renderProps: DiscoverDataVisualizerGridProps) { - return ( - - - - ); -} diff --git a/src/plugins/discover/public/application/components/data_visualizer_grid/index.ts b/src/plugins/discover/public/application/components/data_visualizer_grid/index.ts deleted file mode 100644 index dc85495a7c2ec..0000000000000 --- a/src/plugins/discover/public/application/components/data_visualizer_grid/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -export { DiscoverDataVisualizerGrid } from './data_visualizer_grid'; diff --git a/src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx b/src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx index 0af7dd54a009b..cff7eed4307c7 100644 --- a/src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx +++ b/src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx @@ -39,7 +39,7 @@ export interface DataVisualizerGridEmbeddableOutput extends EmbeddableOutput { showDistributions?: boolean; } -export interface DiscoverDataVisualizerGridProps { +export interface FieldStatisticsTableProps { /** * Determines which columns are displayed */ @@ -89,7 +89,7 @@ export interface DiscoverDataVisualizerGridProps { savedSearchRefetch$?: DataRefetch$; } -export const FieldStatisticsTable = (props: DiscoverDataVisualizerGridProps) => { +export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => { const { services, indexPattern, diff --git a/src/plugins/discover/public/application/components/field_stats_table/field_stats_table_saved_search_embeddable.tsx b/src/plugins/discover/public/application/components/field_stats_table/field_stats_table_saved_search_embeddable.tsx index e1787715fb1a9..9c0c6f4e11609 100644 --- a/src/plugins/discover/public/application/components/field_stats_table/field_stats_table_saved_search_embeddable.tsx +++ b/src/plugins/discover/public/application/components/field_stats_table/field_stats_table_saved_search_embeddable.tsx @@ -8,9 +8,9 @@ import React from 'react'; import { I18nProvider } from '@kbn/i18n/react'; -import { FieldStatisticsTable, DiscoverDataVisualizerGridProps } from './field_stats_table'; +import { FieldStatisticsTable, FieldStatisticsTableProps } from './field_stats_table'; -export function FieldStatsTableSavedSearchEmbeddable(renderProps: DiscoverDataVisualizerGridProps) { +export function FieldStatsTableSavedSearchEmbeddable(renderProps: FieldStatisticsTableProps) { return ( ; export interface AggHistogram { - histogram: AggregationsHistogramAggregation; + histogram: estypes.AggregationsHistogramAggregation; } export interface AggTerms { diff --git a/x-pack/plugins/data_visualizer/common/utils/query_utils.ts b/x-pack/plugins/data_visualizer/common/utils/query_utils.ts index e037e1e07533d..69bfe01c9cf95 100644 --- a/x-pack/plugins/data_visualizer/common/utils/query_utils.ts +++ b/x-pack/plugins/data_visualizer/common/utils/query_utils.ts @@ -17,7 +17,7 @@ export function buildBaseFilterCriteria( earliestMs?: number, latestMs?: number, query?: object -): estypes.QueryDslQueryContainer { +): estypes.QueryDslQueryContainer[] { const filterCriteria = []; if (timeFieldName && earliestMs && latestMs) { filterCriteria.push({ diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts index ccf9f87df2bfb..ec8603ca3c7b7 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts @@ -5,7 +5,7 @@ * 2.0. */ import { get } from 'lodash'; -import type { SearchRequest } from '@elastic/elasticsearch/api/types'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { Observable, of } from 'rxjs'; import { catchError, switchMap } from 'rxjs/operators'; import { @@ -68,7 +68,7 @@ export const fetchBooleanFieldStats = ( options: ISearchOptions ): Observable => { const { samplerShardSize } = params; - const request: SearchRequest = getBooleanFieldStatsRequest(params, field); + const request: estypes.SearchRequest = getBooleanFieldStatsRequest(params, field); return data.search .search({ params: request }, options) .pipe( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts index d796083d3c004..995c9ec5c50d9 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { get } from 'lodash'; import { Observable, of } from 'rxjs'; import { catchError, switchMap } from 'rxjs/operators'; @@ -61,7 +61,7 @@ export const fetchDateFieldStats = ( ): Observable => { const { samplerShardSize } = params; - const request: SearchRequest = getDateFieldStatsRequest(params, field); + const request: estypes.SearchRequest = getDateFieldStatsRequest(params, field); return data.search .search({ params: request }, options) .pipe( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts index fe2df8492974b..cdd69f5d3a369 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_document_stats.ts @@ -6,7 +6,7 @@ */ import { each, get } from 'lodash'; -import { estypes } from '@elastic/elasticsearch'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { buildBaseFilterCriteria } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; import type { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts index 8c608b3775b3a..1f2c5cf5fd0a0 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts @@ -4,8 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { get } from 'lodash'; import { Observable, of } from 'rxjs'; import { catchError, switchMap } from 'rxjs/operators'; @@ -66,7 +65,7 @@ export const fetchFieldExamples = ( field: Field, options: ISearchOptions ): Observable => { - const request: SearchRequest = getFieldExamplesRequest(params, field); + const request: estypes.SearchRequest = getFieldExamplesRequest(params, field); const { maxExamples } = params; return data.search .search({ params: request }, options) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts index 9780f6adafc37..af5d9a27cfa4d 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { find, get } from 'lodash'; import { catchError, switchMap } from 'rxjs/operators'; import { Observable, of } from 'rxjs'; @@ -117,7 +117,7 @@ export const fetchNumericFieldStats = ( options: ISearchOptions ): Observable => { const { samplerShardSize } = params; - const request: SearchRequest = getNumericFieldStatsRequest(params, field); + const request: estypes.SearchRequest = getNumericFieldStatsRequest(params, field); return data.search .search({ params: request }, options) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts index 668bc9abd081e..7a78dc8c1f13d 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts @@ -4,8 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { AggregationsAggregationContainer, SearchRequest } from '@elastic/elasticsearch/api/types'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { get } from 'lodash'; import { Observable, of } from 'rxjs'; import { catchError, switchMap } from 'rxjs/operators'; @@ -39,7 +38,7 @@ export const getStringFieldStatsRequest = (params: FieldStatsCommonRequestParams const aggs: Aggs = {}; const safeFieldName = field.safeFieldName; - const top: AggregationsAggregationContainer = { + const top: estypes.AggregationsAggregationContainer = { terms: { field: field.fieldName, size: 10, @@ -84,7 +83,7 @@ export const fetchStringFieldStats = ( options: ISearchOptions ): Observable => { const { samplerShardSize } = params; - const request: SearchRequest = getStringFieldStatsRequest(params, field); + const request: estypes.SearchRequest = getStringFieldStatsRequest(params, field); return data.search .search({ params: request }, options) diff --git a/x-pack/test/functional/apps/ml/data_visualizer/types.ts b/x-pack/test/functional/apps/ml/data_visualizer/types.ts index ba0077fed3661..06fc1d7bc8c8c 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/types.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/types.ts @@ -5,9 +5,10 @@ * 2.0. */ -import { FieldVisConfig } from '../../../../../plugins/data_visualizer/public/application/common/components/stats_table/types'; +import type { FieldVisConfig } from '../../../../../plugins/data_visualizer/public/application/common/components/stats_table/types'; export interface MetricFieldVisConfig extends FieldVisConfig { + fieldName: string; statsMaxDecimalPlaces: number; docCountFormatted: string; topValuesCount: number; @@ -16,6 +17,7 @@ export interface MetricFieldVisConfig extends FieldVisConfig { } export interface NonMetricFieldVisConfig extends FieldVisConfig { + fieldName: string; docCountFormatted: string; exampleCount: number; exampleContent?: string[]; From 0f8d3d8cb3d47d53cdac25d391418f89fef5fd0a Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 27 Oct 2021 12:35:32 -0500 Subject: [PATCH 172/188] Fix types --- .../common/types/field_stats.ts | 28 +-------- .../components/search_panel/search_panel.tsx | 3 +- .../hooks/use_data_visualizer_grid_data.ts | 2 +- .../requests/get_field_examples.ts | 2 +- .../requests/get_numeric_field_stats.ts | 61 ++++++++++--------- .../utils/saved_search_utils.ts | 26 ++++---- 6 files changed, 53 insertions(+), 69 deletions(-) diff --git a/x-pack/plugins/data_visualizer/common/types/field_stats.ts b/x-pack/plugins/data_visualizer/common/types/field_stats.ts index 85812bb3190ec..a1f7e49ec365d 100644 --- a/x-pack/plugins/data_visualizer/common/types/field_stats.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_stats.ts @@ -37,7 +37,7 @@ export interface HistogramField { } export interface Distribution { - percentiles: any[]; + percentiles: Array<{ value?: number; percent: number; minValue: number; maxValue: number }>; minPercentile: number; maxPercentile: number; } @@ -101,7 +101,7 @@ export interface DocumentCountStats { export interface FieldExamples { fieldName: string; - examples: any[]; + examples: unknown[]; } export interface NumericColumnStats { @@ -154,17 +154,8 @@ export interface UnsupportedChartData { type: 'unsupported'; } -export interface FieldAggCardinality { - field: string; - percent?: any; -} - -export interface ScriptAggCardinality { - script: any; -} - export interface AggCardinality { - cardinality: FieldAggCardinality | ScriptAggCardinality; + cardinality: estypes.AggregationsCardinalityAggregation; } export type ChartRequestAgg = AggHistogram | AggCardinality | AggTerms; @@ -252,16 +243,3 @@ export interface Field { export interface Aggs { [key: string]: estypes.AggregationsAggregationContainer; } - -export interface FieldAggCardinality { - field: string; - percent?: any; -} - -export interface ScriptAggCardinality { - script: any; -} - -export interface AggCardinality { - cardinality: FieldAggCardinality | ScriptAggCardinality; -} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx index f55114ca36d78..25ed13121fc34 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx @@ -22,6 +22,7 @@ import { SearchQueryLanguage } from '../../types/combined_query'; import { useDataVisualizerKibana } from '../../../kibana_context'; import './_index.scss'; import { createMergedEsQuery } from '../../utils/saved_search_utils'; +import { OverallStats } from '../../types/overall_stats'; interface Props { indexPattern: IndexPattern; searchString: Query['query']; @@ -29,7 +30,7 @@ interface Props { searchQueryLanguage: SearchQueryLanguage; samplerShardSize: number; setSamplerShardSize(s: number): void; - overallStats: any; + overallStats: OverallStats; indexedFieldTypes: JobFieldType[]; setVisibleFieldTypes(q: string[]): void; visibleFieldTypes: string[]; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 50ab4e660f1ac..7353941eeb418 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -49,7 +49,7 @@ function isDisplayField(fieldName: string): boolean { export const useDataVisualizerGridData = ( input: DataVisualizerGridInput, dataVisualizerListState: Required, - onUpdate?: (params: string | Dictionary) => void + onUpdate?: (params: Dictionary) => void ) => { const { services } = useDataVisualizerKibana(); const { uiSettings, data } = services; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts index 1f2c5cf5fd0a0..27607191f5ef0 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts @@ -81,7 +81,7 @@ export const fetchFieldExamples = ( const body = resp.rawResponse; const stats = { fieldName: field.fieldName, - examples: [] as any[], + examples: [] as unknown[], } as FieldExamples; if (body.hits.total > 0) { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts index af5d9a27cfa4d..ac6a42e376d7e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts @@ -9,6 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { find, get } from 'lodash'; import { catchError, switchMap } from 'rxjs/operators'; import { Observable, of } from 'rxjs'; +import { AggregationsTermsAggregation } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { MAX_PERCENT, PERCENTILE_SPACING, @@ -20,7 +21,7 @@ import { getSamplerAggregationsResponsePath, } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; -import type { FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats'; +import type { Aggs, FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats'; import type { Field, NumericFieldStats, @@ -54,23 +55,7 @@ export const getNumericFieldStatsRequest = ( () => (count += PERCENTILE_SPACING) ); - const aggs: { [key: string]: any } = {}; const safeFieldName = field.safeFieldName; - aggs[`${safeFieldName}_field_stats`] = { - filter: { exists: { field: field.fieldName } }, - aggs: { - actual_stats: { - stats: { field: field.fieldName }, - }, - }, - }; - aggs[`${safeFieldName}_percentiles`] = { - percentiles: { - field: field.fieldName, - percents, - keyed: false, - }, - }; const top = { terms: { @@ -79,23 +64,39 @@ export const getNumericFieldStatsRequest = ( order: { _count: 'desc', }, - }, + } as AggregationsTermsAggregation, }; - // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation - // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - aggs[`${safeFieldName}_top`] = { - sampler: { - shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, - }, + const aggs: Aggs = { + [`${safeFieldName}_field_stats`]: { + filter: { exists: { field: field.fieldName } }, aggs: { - top, + actual_stats: { + stats: { field: field.fieldName }, + }, + }, + }, + [`${safeFieldName}_percentiles`]: { + percentiles: { + field: field.fieldName, + percents, + keyed: false, }, - }; - } else { - aggs[`${safeFieldName}_top`] = top; - } + }, + // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation + // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). + [`${safeFieldName}_top`]: + samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? { + sampler: { + shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, + }, + aggs: { + top, + }, + } + : top, + }; const searchBody = { query, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index 1401b1038b8f2..53070791fea2e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -4,7 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { cloneDeep } from 'lodash'; import { IUiSettingsClient } from 'kibana/public'; import { @@ -15,6 +14,7 @@ import { Query, Filter, } from '@kbn/es-query'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { isSavedSearchSavedObject, SavedSearchSavedObject } from '../../../../common/types'; import { IndexPattern, SearchSource } from '../../../../../../../src/plugins/data/common'; import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; @@ -22,7 +22,7 @@ import { SavedSearch } from '../../../../../../../src/plugins/discover/public'; import { getEsQueryConfig } from '../../../../../../../src/plugins/data/common'; import { FilterManager } from '../../../../../../../src/plugins/data/public'; -const DEFAULT_QUERY = { +const DEFAULT_QUERY: estypes.QueryDslQueryContainer = { bool: { must: [ { @@ -32,7 +32,7 @@ const DEFAULT_QUERY = { }, }; -export function getDefaultQuery() { +export function getDefaultQuery(): estypes.QueryDslQueryContainer { return cloneDeep(DEFAULT_QUERY); } @@ -75,8 +75,10 @@ export function createMergedEsQuery( filters?: Filter[], indexPattern?: IndexPattern, uiSettings?: IUiSettingsClient -) { - let combinedQuery: any = getDefaultQuery(); +): estypes.QueryDslQueryContainer { + let combinedQuery = getDefaultQuery(); + let boolFilters: estypes.QueryDslQueryContainer[] = []; + let mustNotFilters: estypes.QueryDslQueryContainer[] = []; if (query && query.language === SEARCH_QUERY_LANGUAGE.KUERY) { const ast = fromKueryExpression(query.query); @@ -87,17 +89,19 @@ export function createMergedEsQuery( const filterQuery = buildQueryFromFilters(filters, indexPattern); if (Array.isArray(combinedQuery.bool.filter) === false) { - combinedQuery.bool.filter = - combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; + boolFilters = ( + combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter] + ) as estypes.QueryDslQueryContainer[]; } if (Array.isArray(combinedQuery.bool.must_not) === false) { - combinedQuery.bool.must_not = - combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; + mustNotFilters = ( + combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not] + ) as estypes.QueryDslQueryContainer[]; } - combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; - combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; + combinedQuery.bool.filter = [...boolFilters, ...filterQuery.filter]; + combinedQuery.bool.must_not = [...mustNotFilters, ...filterQuery.must_not]; } } else { combinedQuery = buildEsQuery( From 93489c5c9e2d564a6c935cba80ea27f79ee2eac4 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 27 Oct 2021 12:56:24 -0500 Subject: [PATCH 173/188] Fix loading state in Discover --- .../components/stats_table/data_visualizer_stats_table.tsx | 3 +++ .../embeddables/grid_embeddable/grid_embeddable.tsx | 1 + .../index_data_visualizer/hooks/use_overall_stats.ts | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index c00293937b5ca..2020d99f59928 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -55,6 +55,7 @@ interface DataVisualizerTableProps { showPreviewByDefault?: boolean; /** Callback to receive any updates when table or page state is changed **/ onChange?: (update: Partial) => void; + loading?: boolean; } export const DataVisualizerTable = ({ @@ -65,6 +66,7 @@ export const DataVisualizerTable = ({ extendedColumns, showPreviewByDefault, onChange, + loading, }: DataVisualizerTableProps) => { const [expandedRowItemIds, setExpandedRowItemIds] = useState([]); const [expandAll, setExpandAll] = useState(false); @@ -323,6 +325,7 @@ export const DataVisualizerTable = ({ {(resizeRef) => (
+ loading={loading === true} className={'dvTable'} items={items} itemId={FIELD_NAME} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index a3681ae643e5c..0391d5ae5d5d5 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -138,6 +138,7 @@ export const EmbeddableWrapper = ({ extendedColumns={extendedColumns} showPreviewByDefault={input?.showPreviewByDefault} onChange={onOutputChange} + loading={progress < 100} /> ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 3713457a33149..dcca6d71e63bf 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -7,7 +7,7 @@ import { useCallback, useEffect, useState, useRef, useMemo, useReducer } from 'react'; import { combineLatest, forkJoin, of, Subscription } from 'rxjs'; -import { mergeMap, switchMap } from 'rxjs/operators'; +import { switchMap } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; import type { ToastsStart } from 'kibana/public'; import { useDataVisualizerKibana } from '../../kibana_context'; @@ -176,7 +176,7 @@ export function useOverallStats Date: Wed, 27 Oct 2021 13:29:21 -0500 Subject: [PATCH 174/188] Remove unused services, Change switchMap to map, mergeMap -> switchMap, update types --- .../common/services/time_buckets.d.ts | 2 +- .../common/types/field_stats.ts | 3 +- .../data_visualizer/common/types/index.ts | 1 - .../common/utils/query_utils.ts | 6 +- .../requests/get_boolean_field_stats.ts | 8 +- .../requests/get_date_field_stats.ts | 10 +- .../requests/get_field_examples.ts | 8 +- .../requests/get_numeric_field_stats.ts | 8 +- .../requests/get_string_field_stats.ts | 8 +- .../search_strategy/requests/overall_stats.ts | 5 +- .../services/visualizer_stats.ts | 98 ------------------- .../types/combined_query.ts | 4 +- .../utils/process_distribution_data.ts | 2 +- 13 files changed, 35 insertions(+), 128 deletions(-) delete mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/visualizer_stats.ts diff --git a/x-pack/plugins/data_visualizer/common/services/time_buckets.d.ts b/x-pack/plugins/data_visualizer/common/services/time_buckets.d.ts index 9a5410918a099..62a3187be47dc 100644 --- a/x-pack/plugins/data_visualizer/common/services/time_buckets.d.ts +++ b/x-pack/plugins/data_visualizer/common/services/time_buckets.d.ts @@ -31,7 +31,7 @@ export declare class TimeBuckets { public setMaxBars(maxBars: number): void; public setInterval(interval: string): void; public setBounds(bounds: TimeRangeBounds): void; - public getBounds(): { min: any; max: any }; + public getBounds(): { min: Moment; max: Moment }; public getInterval(): TimeBucketsInterval; public getScaledDateFormat(): string; } diff --git a/x-pack/plugins/data_visualizer/common/types/field_stats.ts b/x-pack/plugins/data_visualizer/common/types/field_stats.ts index a1f7e49ec365d..4a040024e5234 100644 --- a/x-pack/plugins/data_visualizer/common/types/field_stats.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_stats.ts @@ -6,6 +6,7 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { Query } from '@kbn/es-query'; import { isPopulatedObject } from '../utils/object_utils'; import { IKibanaSearchResponse } from '../../../../../src/plugins/data/common'; import { TimeBucketsInterval } from '../services/time_buckets'; @@ -200,7 +201,7 @@ export interface OverallStatsSearchStrategyParams { latest?: number; aggInterval: TimeBucketsInterval; intervalMs?: number; - searchQuery?: any; + searchQuery: Query['query']; samplerShardSize: number; index: string; timeFieldName?: string; diff --git a/x-pack/plugins/data_visualizer/common/types/index.ts b/x-pack/plugins/data_visualizer/common/types/index.ts index 1153b45e1cce2..381f7a556b18d 100644 --- a/x-pack/plugins/data_visualizer/common/types/index.ts +++ b/x-pack/plugins/data_visualizer/common/types/index.ts @@ -15,7 +15,6 @@ export type { FieldVisStats, Percentile, } from './field_request_config'; -export type InputData = any[]; export interface DataVisualizerTableState { pageSize: number; diff --git a/x-pack/plugins/data_visualizer/common/utils/query_utils.ts b/x-pack/plugins/data_visualizer/common/utils/query_utils.ts index 69bfe01c9cf95..dc21bbcae96c3 100644 --- a/x-pack/plugins/data_visualizer/common/utils/query_utils.ts +++ b/x-pack/plugins/data_visualizer/common/utils/query_utils.ts @@ -6,6 +6,8 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { Query } from '@kbn/es-query'; + /* * Contains utility functions for building and processing queries. */ @@ -16,7 +18,7 @@ export function buildBaseFilterCriteria( timeFieldName?: string, earliestMs?: number, latestMs?: number, - query?: object + query?: Query['query'] ): estypes.QueryDslQueryContainer[] { const filterCriteria = []; if (timeFieldName && earliestMs && latestMs) { @@ -31,7 +33,7 @@ export function buildBaseFilterCriteria( }); } - if (query) { + if (query && typeof query === 'object') { filterCriteria.push(query); } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts index ec8603ca3c7b7..06229f80585f1 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts @@ -7,7 +7,7 @@ import { get } from 'lodash'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { Observable, of } from 'rxjs'; -import { catchError, switchMap } from 'rxjs/operators'; +import { catchError, map } from 'rxjs/operators'; import { buildSamplerAggregation, getSamplerAggregationsResponsePath, @@ -78,8 +78,8 @@ export const fetchBooleanFieldStats = ( error: extractErrorProperties(e), } as FieldStatsError) ), - switchMap((resp) => { - if (!isIKibanaSearchResponse(resp)) return of(resp); + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; const aggregations = resp.rawResponse.aggregations; const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); @@ -100,7 +100,7 @@ export const fetchBooleanFieldStats = ( valueBuckets.forEach((bucket) => { stats[`${bucket.key_as_string}Count`] = bucket.doc_count; }); - return of(stats); + return stats; }) ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts index 995c9ec5c50d9..b4336bc205a8d 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts @@ -8,7 +8,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { get } from 'lodash'; import { Observable, of } from 'rxjs'; -import { catchError, switchMap } from 'rxjs/operators'; +import { catchError, map } from 'rxjs/operators'; import { buildSamplerAggregation, getSamplerAggregationsResponsePath, @@ -71,8 +71,8 @@ export const fetchDateFieldStats = ( error: extractErrorProperties(e), } as FieldStatsError) ), - switchMap((resp) => { - if (!isIKibanaSearchResponse(resp)) return of(resp); + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; const aggregations = resp.rawResponse.aggregations; const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); const safeFieldName = field.safeFieldName; @@ -86,12 +86,12 @@ export const fetchDateFieldStats = ( [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], {} ); - return of({ + return { fieldName: field.fieldName, count: docCount, earliest: get(fieldStatsResp, 'min', 0), latest: get(fieldStatsResp, 'max', 0), - }); + }; }) ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts index 27607191f5ef0..de29fa6e31589 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts @@ -7,7 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { get } from 'lodash'; import { Observable, of } from 'rxjs'; -import { catchError, switchMap } from 'rxjs/operators'; +import { catchError, map } from 'rxjs/operators'; import { buildBaseFilterCriteria } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; import type { @@ -76,8 +76,8 @@ export const fetchFieldExamples = ( error: extractErrorProperties(e), } as FieldStatsError) ), - switchMap((resp) => { - if (!isIKibanaSearchResponse(resp)) return of(resp); + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; const body = resp.rawResponse; const stats = { fieldName: field.fieldName, @@ -102,7 +102,7 @@ export const fetchFieldExamples = ( } } - return of(stats); + return stats; }) ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts index ac6a42e376d7e..0c799d161b091 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts @@ -7,7 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { find, get } from 'lodash'; -import { catchError, switchMap } from 'rxjs/operators'; +import { catchError, map } from 'rxjs/operators'; import { Observable, of } from 'rxjs'; import { AggregationsTermsAggregation } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { @@ -129,8 +129,8 @@ export const fetchNumericFieldStats = ( error: extractErrorProperties(e), } as FieldStatsError) ), - switchMap((resp) => { - if (!isIKibanaSearchResponse(resp)) return of(resp); + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; const aggregations = resp.rawResponse.aggregations; const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); @@ -185,7 +185,7 @@ export const fetchNumericFieldStats = ( stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; stats.distribution = processDistributionData(percentiles, PERCENTILE_SPACING, stats.min); } - return of(stats); + return stats; }) ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts index 7a78dc8c1f13d..1c9cb49460613 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts @@ -7,7 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { get } from 'lodash'; import { Observable, of } from 'rxjs'; -import { catchError, switchMap } from 'rxjs/operators'; +import { catchError, map } from 'rxjs/operators'; import { SAMPLER_TOP_TERMS_SHARD_SIZE, SAMPLER_TOP_TERMS_THRESHOLD } from './constants'; import { buildSamplerAggregation, @@ -94,8 +94,8 @@ export const fetchStringFieldStats = ( error: extractErrorProperties(e), } as FieldStatsError) ), - switchMap((resp) => { - if (!isIKibanaSearchResponse(resp)) return of(resp); + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; const aggregations = resp.rawResponse.aggregations; const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); @@ -123,7 +123,7 @@ export const fetchStringFieldStats = ( : samplerShardSize, }; - return of(stats); + return stats; }) ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts index 7efc977c1cc3e..dd3b84bb54e6d 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts @@ -7,6 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { get } from 'lodash'; +import { Query } from '@kbn/es-query'; import { buildBaseFilterCriteria, buildSamplerAggregation, @@ -21,7 +22,7 @@ import { AggCardinality, Aggs } from '../../../../../common/types/field_stats'; export const checkAggregatableFieldsExistRequest = ( indexPatternTitle: string, - query: any, + query: Query['query'], aggregatableFields: string[], samplerShardSize: number, timeFieldName: string | undefined, @@ -163,7 +164,7 @@ export const processAggregatableFieldsExistResponse = ( export const checkNonAggregatableFieldExistsRequest = ( indexPatternTitle: string, - query: any, + query: Query['query'], field: string, timeFieldName: string | undefined, earliestMs: number | undefined, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/visualizer_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/visualizer_stats.ts deleted file mode 100644 index 791f51fdee6cd..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/services/visualizer_stats.ts +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { lazyLoadModules } from '../../../lazy_load_bundle'; -import type { DocumentCounts, FieldRequestConfig, FieldVisStats } from '../../../../common/types'; -import { OverallStats } from '../types/overall_stats'; - -export function basePath() { - return '/internal/data_visualizer'; -} - -export async function getVisualizerOverallStats({ - indexPatternTitle, - query, - timeFieldName, - earliest, - latest, - samplerShardSize, - aggregatableFields, - nonAggregatableFields, - runtimeMappings, -}: { - indexPatternTitle: string; - query: any; - timeFieldName?: string; - earliest?: number; - latest?: number; - samplerShardSize?: number; - aggregatableFields: string[]; - nonAggregatableFields: string[]; - runtimeMappings?: estypes.MappingRuntimeFields; -}) { - const body = JSON.stringify({ - query, - timeFieldName, - earliest, - latest, - samplerShardSize, - aggregatableFields, - nonAggregatableFields, - runtimeMappings, - }); - - const dataVisualizerModules = await lazyLoadModules(); - return await dataVisualizerModules.getHttp().fetch({ - path: `${basePath()}/get_overall_stats/${indexPatternTitle}`, - method: 'POST', - body, - }); -} - -export async function getVisualizerFieldStats({ - indexPatternTitle, - query, - timeFieldName, - earliest, - latest, - samplerShardSize, - interval, - fields, - maxExamples, - runtimeMappings, -}: { - indexPatternTitle: string; - query: any; - timeFieldName?: string; - earliest?: number; - latest?: number; - samplerShardSize?: number; - interval?: number; - fields?: FieldRequestConfig[]; - maxExamples?: number; - runtimeMappings?: estypes.MappingRuntimeFields; -}) { - const body = JSON.stringify({ - query, - timeFieldName, - earliest, - latest, - samplerShardSize, - interval, - fields, - maxExamples, - runtimeMappings, - }); - - const dataVisualizerModules = await lazyLoadModules(); - return await dataVisualizerModules.getHttp().fetch<[DocumentCounts, FieldVisStats]>({ - path: `${basePath()}/get_field_stats/${indexPatternTitle}`, - method: 'POST', - body, - }); -} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts index 734a47d7f01b0..13590505a5d1a 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { Query } from '@kbn/es-query'; + export const SEARCH_QUERY_LANGUAGE = { KUERY: 'kuery', LUCENE: 'lucene', @@ -13,7 +15,7 @@ export const SEARCH_QUERY_LANGUAGE = { export type SearchQueryLanguage = typeof SEARCH_QUERY_LANGUAGE[keyof typeof SEARCH_QUERY_LANGUAGE]; export interface CombinedQuery { - searchString: string | { [key: string]: any }; + searchString: Query['query']; searchQueryLanguage: string; } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts index ef05d19b78758..46719c06e2264 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/process_distribution_data.ts @@ -49,7 +49,7 @@ export const processDistributionData = ( // Add in 0-5 and 95-100% if they don't add more // than 25% to the value range at either end. - const lastValue: number = (last(percentileBuckets) as any).value; + const lastValue: number = (last(percentileBuckets) as { value: number }).value; const maxDiff = 0.25 * (lastValue - lowerBound); if (lowerBound - dataMin < maxDiff) { percentileBuckets.splice(0, 0, percentiles[0]); From ae6d2655eef854da8bb36c72d276daac10f646a0 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 27 Oct 2021 18:39:40 -0500 Subject: [PATCH 175/188] Fix missing filters --- .../hooks/use_data_visualizer_grid_data.ts | 4 +++ .../utils/saved_search_utils.ts | 31 +++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 7353941eeb418..619a709d17c5b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -84,9 +84,13 @@ export const useDataVisualizerGridData = ( savedSearch: currentSavedSearch, query: currentQuery, filters: currentFilters, + filterManager: data.query.filterManager, }); if (searchData === undefined || dataVisualizerListState.searchString !== '') { + if (dataVisualizerListState.filters) { + data.query.filterManager.setFilters(dataVisualizerListState.filters); + } return { searchQuery: dataVisualizerListState.searchQuery, searchString: dataVisualizerListState.searchString, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index 53070791fea2e..e78861ce33313 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + import { cloneDeep } from 'lodash'; import { IUiSettingsClient } from 'kibana/public'; import { @@ -14,7 +15,7 @@ import { Query, Filter, } from '@kbn/es-query'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { isSavedSearchSavedObject, SavedSearchSavedObject } from '../../../../common/types'; import { IndexPattern, SearchSource } from '../../../../../../../src/plugins/data/common'; import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; @@ -22,7 +23,7 @@ import { SavedSearch } from '../../../../../../../src/plugins/discover/public'; import { getEsQueryConfig } from '../../../../../../../src/plugins/data/common'; import { FilterManager } from '../../../../../../../src/plugins/data/public'; -const DEFAULT_QUERY: estypes.QueryDslQueryContainer = { +const DEFAULT_QUERY = { bool: { must: [ { @@ -32,7 +33,7 @@ const DEFAULT_QUERY: estypes.QueryDslQueryContainer = { }, }; -export function getDefaultQuery(): estypes.QueryDslQueryContainer { +export function getDefaultQuery() { return cloneDeep(DEFAULT_QUERY); } @@ -75,10 +76,8 @@ export function createMergedEsQuery( filters?: Filter[], indexPattern?: IndexPattern, uiSettings?: IUiSettingsClient -): estypes.QueryDslQueryContainer { - let combinedQuery = getDefaultQuery(); - let boolFilters: estypes.QueryDslQueryContainer[] = []; - let mustNotFilters: estypes.QueryDslQueryContainer[] = []; +) { + let combinedQuery: QueryDslQueryContainer = getDefaultQuery(); if (query && query.language === SEARCH_QUERY_LANGUAGE.KUERY) { const ast = fromKueryExpression(query.query); @@ -88,20 +87,18 @@ export function createMergedEsQuery( if (combinedQuery.bool !== undefined) { const filterQuery = buildQueryFromFilters(filters, indexPattern); - if (Array.isArray(combinedQuery.bool.filter) === false) { - boolFilters = ( - combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter] - ) as estypes.QueryDslQueryContainer[]; + if (!Array.isArray(combinedQuery.bool.filter)) { + combinedQuery.bool.filter = + combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; } - if (Array.isArray(combinedQuery.bool.must_not) === false) { - mustNotFilters = ( - combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not] - ) as estypes.QueryDslQueryContainer[]; + if (!Array.isArray(combinedQuery.bool.must_not)) { + combinedQuery.bool.must_not = + combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; } - combinedQuery.bool.filter = [...boolFilters, ...filterQuery.filter]; - combinedQuery.bool.must_not = [...mustNotFilters, ...filterQuery.must_not]; + combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; + combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; } } else { combinedQuery = buildEsQuery( From 8da7712f47a85e611244a3baad931e97d9006c26 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 28 Oct 2021 11:27:08 -0500 Subject: [PATCH 176/188] Fix message of table to show searching instead of no items found --- .../components/stats_table/data_visualizer_stats_table.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 2020d99f59928..8f476acb897ae 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -325,7 +325,9 @@ export const DataVisualizerTable = ({ {(resizeRef) => (
- loading={loading === true} + message={i18n.translate('xpack.dataVisualizer.dataGrid.searchingMessage', { + defaultMessage: 'Searching', + })} className={'dvTable'} items={items} itemId={FIELD_NAME} From 30826dafa6015fc9697a4dec9944be7f4186386c Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 28 Oct 2021 15:19:59 -0500 Subject: [PATCH 177/188] Fix dashboard saved search source persisting time range --- .../hooks/use_data_visualizer_grid_data.ts | 1 + .../utils/saved_search_utils.ts | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 619a709d17c5b..b88c838cf5192 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -115,6 +115,7 @@ export const useDataVisualizerGridData = ( currentQuery, currentFilters, }), + lastRefresh, ]); useEffect(() => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index e78861ce33313..6b1961ceac852 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -146,8 +146,20 @@ export function getEsQueryFromSavedSearch({ savedSearch.searchSource.getParent() !== undefined && userQuery ) { + // Flattened query from search source may contain a clause that narrows the time range + // which might interfere with global time pickers so we need to remove + const savedQuery = + cloneDeep(savedSearch.searchSource.getSearchRequestBody()?.query) ?? getDefaultQuery(); + const timeField = savedSearch.searchSource.getField('index')?.timeFieldName; + + if (Array.isArray(savedQuery.bool.filter) && timeField !== undefined) { + savedQuery.bool.filter = savedQuery.bool.filter.filter( + (c: QueryDslQueryContainer) => + !(c.hasOwnProperty('range') && c.range?.hasOwnProperty(timeField)) + ); + } return { - searchQuery: savedSearch.searchSource.getSearchRequestBody()?.query ?? getDefaultQuery(), + searchQuery: savedQuery, searchString: userQuery.query, queryLanguage: userQuery.language as SearchQueryLanguage, }; From 241f3b45894ae9448685cec5ffea89de4a738813 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 2 Nov 2021 17:29:19 -0500 Subject: [PATCH 178/188] [ML] Fix table message state --- .../components/field_data_row/use_column_chart.tsx | 4 +--- .../stats_table/data_visualizer_stats_table.tsx | 10 +++++++--- .../index_data_visualizer_view.tsx | 1 + x-pack/plugins/translations/translations/ja-JP.json | 1 - x-pack/plugins/translations/translations/zh-CN.json | 1 - 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.tsx index 60e1595c64ece..827e4a7f44857 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.tsx @@ -83,9 +83,7 @@ export const getLegendText = (chartData: ChartData, maxChartColumns: number): Le } if (chartData.data.length === 0) { - return i18n.translate('xpack.dataVisualizer.dataGridChart.notEnoughData', { - defaultMessage: `0 documents contain field.`, - }); + return ''; } if (chartData.type === 'boolean') { diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx index 8f476acb897ae..976afc464a672 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx @@ -325,9 +325,13 @@ export const DataVisualizerTable = ({ {(resizeRef) => (
- message={i18n.translate('xpack.dataVisualizer.dataGrid.searchingMessage', { - defaultMessage: 'Searching', - })} + message={ + loading + ? i18n.translate('xpack.dataVisualizer.dataGrid.searchingMessage', { + defaultMessage: 'Searching', + }) + : undefined + } className={'dvTable'} items={items} itemId={FIELD_NAME} diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 9c855e9a11b50..bfa9f28cfa65b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -498,6 +498,7 @@ export const IndexDataVisualizerView: FC = (dataVi updatePageState={setDataVisualizerListState} getItemIdToExpandedRowMap={getItemIdToExpandedRowMap} extendedColumns={extendedColumns} + loading={progress < 100} /> diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 0f75fa264d0d9..f32e7019e111c 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -8985,7 +8985,6 @@ "xpack.dataVisualizer.dataGrid.showDistributionsTooltip": "分布を表示", "xpack.dataVisualizer.dataGrid.typeColumnName": "型", "xpack.dataVisualizer.dataGridChart.histogramNotAvailable": "グラフはサポートされていません。", - "xpack.dataVisualizer.dataGridChart.notEnoughData": "0個のドキュメントにフィールドが含まれます。", "xpack.dataVisualizer.dataGridChart.topCategoriesLegend": "上位 {maxChartColumns}/{cardinality} カテゴリ", "xpack.dataVisualizer.description": "CSV、NDJSON、またはログファイルをインポートします。", "xpack.dataVisualizer.fieldNameSelect": "フィールド名", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index b1bd97992af7e..218e754928f03 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -9068,7 +9068,6 @@ "xpack.dataVisualizer.dataGrid.showDistributionsTooltip": "显示分布", "xpack.dataVisualizer.dataGrid.typeColumnName": "类型", "xpack.dataVisualizer.dataGridChart.histogramNotAvailable": "不支持图表。", - "xpack.dataVisualizer.dataGridChart.notEnoughData": "0 个文档包含字段。", "xpack.dataVisualizer.dataGridChart.singleCategoryLegend": "{cardinality, plural, other {# 个类别}}", "xpack.dataVisualizer.dataGridChart.topCategoriesLegend": "{cardinality} 个类别中的排名前 {maxChartColumns} 个", "xpack.dataVisualizer.description": "导入您自己的 CSV、NDJSON 或日志文件。", From 107aece22f1a2aaf91a0f328aa6ca593cd2f6af8 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 3 Nov 2021 09:51:59 -0500 Subject: [PATCH 179/188] [ML] Fix to not fetch field stats if cardinality is 0 --- .../field_data_row/use_column_chart.test.tsx | 4 +- .../hooks/use_data_visualizer_grid_data.ts | 37 ++++++++++++------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.test.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.test.tsx index 72181436977e4..d20217f673449 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.test.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_row/use_column_chart.test.tsx @@ -106,7 +106,7 @@ describe('getLegendText()', () => { expect(getLegendText(validUnsupportedChartData, 20)).toBe('Chart not supported.'); }); it('should return the chart legend text for empty datasets', () => { - expect(getLegendText(validNumericChartData, 20)).toBe('0 documents contain field.'); + expect(getLegendText(validNumericChartData, 20)).toBe(''); }); it('should return the chart legend text for boolean chart types', () => { const { getByText } = render( @@ -186,7 +186,7 @@ describe('useColumnChart()', () => { ); expect(result.current.data).toStrictEqual([]); - expect(result.current.legendText).toBe('0 documents contain field.'); + expect(result.current.legendText).toBe(''); expect(result.current.xScaleType).toBe('linear'); }); }); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index b88c838cf5192..5802bef13cb04 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -219,23 +219,32 @@ export const useDataVisualizerGridData = ( ); const configsWithoutStats = useMemo(() => { - const existMetricFields: FieldRequestConfig[] = metricConfigs.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); + const existMetricFields = metricConfigs + .map((config) => { + if (config?.stats?.cardinality !== undefined) { + return { + fieldName: config.fieldName, + type: config.type, + cardinality: config.stats.cardinality, + }; + } + }) + .filter((c) => c !== undefined) as FieldRequestConfig[]; // Pass the field name, type and cardinality in the request. // Top values will be obtained on a sample if cardinality > 100000. - const existNonMetricFields: FieldRequestConfig[] = nonMetricConfigs.map((config) => { - const props = { fieldName: config.fieldName, type: config.type, cardinality: 0 }; - if (config.stats !== undefined && config.stats.cardinality !== undefined) { - props.cardinality = config.stats.cardinality; - } - return props; - }); + const existNonMetricFields = nonMetricConfigs + .map((config) => { + if (config?.stats?.cardinality !== undefined) { + return { + fieldName: config.fieldName, + type: config.type, + cardinality: config.stats.cardinality, + }; + } + }) + .filter((c) => c !== undefined) as FieldRequestConfig[]; + return { metricConfigs: existMetricFields, nonMetricConfigs: existNonMetricFields }; }, [metricConfigs, nonMetricConfigs]); From cd65afc778d97f2c9348f860152a37bbb5418f51 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 3 Nov 2021 11:15:33 -0500 Subject: [PATCH 180/188] [ML] Fix locator missing view mode --- src/plugins/discover/public/locator.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/plugins/discover/public/locator.ts b/src/plugins/discover/public/locator.ts index 40b62841f19d1..65e7fa0be15f2 100644 --- a/src/plugins/discover/public/locator.ts +++ b/src/plugins/discover/public/locator.ts @@ -11,6 +11,7 @@ import type { TimeRange, Filter, Query, QueryState, RefreshInterval } from '../. import type { LocatorDefinition, LocatorPublic } from '../../share/public'; import { esFilters } from '../../data/public'; import { setStateToKbnUrl } from '../../kibana_utils/public'; +import type { VIEW_MODE } from './application/apps/main/components/view_mode_toggle'; export const DISCOVER_APP_LOCATOR = 'DISCOVER_APP_LOCATOR'; @@ -75,6 +76,14 @@ export interface DiscoverAppLocatorParams extends SerializableRecord { * id of the used saved query */ savedQuery?: string; + /** + * Table view: Documents vs Field Statistics + */ + viewMode?: VIEW_MODE; + /** + * Hide mini distribution/preview charts when in Field Statistics mode + */ + hideAggregatedPreview?: boolean; } export type DiscoverAppLocator = LocatorPublic; @@ -102,6 +111,8 @@ export class DiscoverAppLocatorDefinition implements LocatorDefinition esFilters.isFilterPinned(f)); if (refreshInterval) queryState.refreshInterval = refreshInterval; + if (viewMode) appState.viewMode = viewMode; + if (hideAggregatedPreview) appState.hideAggregatedPreview = hideAggregatedPreview; let path = `#/${savedSearchPath}`; path = setStateToKbnUrl('_g', queryState, { useHash }, path); From 93016deebcacd831b8978eb9f6556de1895997d9 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 3 Nov 2021 11:37:33 -0500 Subject: [PATCH 181/188] [ML] Quit right away if field doesn't exist in docs --- .../hooks/use_data_visualizer_grid_data.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 5802bef13cb04..6ac8c6d5b3b74 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -221,6 +221,7 @@ export const useDataVisualizerGridData = ( const configsWithoutStats = useMemo(() => { const existMetricFields = metricConfigs .map((config) => { + if (config.existsInDocs === false) return; if (config?.stats?.cardinality !== undefined) { return { fieldName: config.fieldName, @@ -233,8 +234,9 @@ export const useDataVisualizerGridData = ( // Pass the field name, type and cardinality in the request. // Top values will be obtained on a sample if cardinality > 100000. - const existNonMetricFields = nonMetricConfigs + const existNonMetricFields: FieldRequestConfig[] = nonMetricConfigs .map((config) => { + if (config.existsInDocs === false) return; if (config?.stats?.cardinality !== undefined) { return { fieldName: config.fieldName, From 817fe2da6687923e5a532de3279786fc8bbd4e18 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 4 Nov 2021 10:41:54 -0500 Subject: [PATCH 182/188] [ML] Change to use batch and only retry with individual field if failed --- .../common/types/field_stats.ts | 4 +- .../hooks/use_field_stats.ts | 160 ++++++-- .../requests/get_fields_stats.ts | 347 ++++++++++++++++++ 3 files changed, 472 insertions(+), 39 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts diff --git a/x-pack/plugins/data_visualizer/common/types/field_stats.ts b/x-pack/plugins/data_visualizer/common/types/field_stats.ts index 4a040024e5234..8932a0641cbe6 100644 --- a/x-pack/plugins/data_visualizer/common/types/field_stats.ts +++ b/x-pack/plugins/data_visualizer/common/types/field_stats.ts @@ -28,6 +28,7 @@ export interface Field { safeFieldName: string; } +// @todo: check export function isValidField(arg: unknown): arg is Field { return isPopulatedObject(arg, ['fieldName', 'type']) && typeof arg.fieldName === 'string'; } @@ -48,7 +49,8 @@ export interface Bucket { } export interface FieldStatsError { - fieldName: string; + fieldName?: string; + fields?: Field[]; error: Error; } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index 4d8a1ad0cd7fd..8f8ab73263a91 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -6,13 +6,16 @@ */ import { useCallback, useEffect, useReducer, useRef, useState } from 'react'; -import { combineLatest, Observable, Subscription } from 'rxjs'; +import { combineLatest, Observable, Subject, Subscription } from 'rxjs'; import { i18n } from '@kbn/i18n'; +import { last, cloneDeep } from 'lodash'; +import { switchMap } from 'rxjs/operators'; import type { DataStatsFetchProgress, FieldStatsSearchStrategyReturnBase, OverallStatsSearchStrategyParams, FieldStatsCommonRequestParams, + Field, } from '../../../../common/types/field_stats'; import { useDataVisualizerKibana } from '../../kibana_context'; import type { FieldRequestConfig } from '../../../../common'; @@ -21,17 +24,40 @@ import { buildBaseFilterCriteria, getSafeAggregationName, } from '../../../../common/utils/query_utils'; -import { getFieldStats } from '../search_strategy/requests/get_field_stats'; import type { FieldStats, FieldStatsError } from '../../../../common/types/field_stats'; import { getInitialProgress, getReducer } from '../progress_utils'; import { MAX_EXAMPLES_DEFAULT } from '../search_strategy/requests/constants'; import type { ISearchOptions } from '../../../../../../../src/plugins/data/common'; - +import { getFieldsStats } from '../search_strategy/requests/get_fields_stats'; interface FieldStatsParams { metricConfigs: FieldRequestConfig[]; nonMetricConfigs: FieldRequestConfig[]; } +const createBatchedRequests = (fields: Field[], maxBatchSize = 10) => { + // Batch up fields by type, getting stats for multiple fields at a time. + const batches: Field[][] = []; + const batchedFields: { [key: string]: Field[][] } = {}; + + fields.forEach((field) => { + const fieldType = field.type; + if (batchedFields[fieldType] === undefined) { + batchedFields[fieldType] = [[]]; + } + let lastArray: Field[] = last(batchedFields[fieldType]) as Field[]; + if (lastArray.length === maxBatchSize) { + lastArray = []; + batchedFields[fieldType].push(lastArray); + } + lastArray.push(field); + }); + + Object.values(batchedFields).forEach((lists) => { + batches.push(...lists); + }); + return batches; +}; + export function useFieldStatsSearchStrategy( searchStrategyParams: OverallStatsSearchStrategyParams | undefined, fieldStatsParams: FieldStatsParams | undefined, @@ -52,9 +78,12 @@ export function useFieldStatsSearchStrategy( const abortCtrl = useRef(new AbortController()); const searchSubscription$ = useRef(); + const retries$ = useRef(); const startFetch = useCallback(() => { searchSubscription$.current?.unsubscribe(); + retries$.current?.unsubscribe(); + abortCtrl.current.abort(); abortCtrl.current = new AbortController(); setFetchState({ @@ -118,31 +147,63 @@ export function useFieldStatsSearchStrategy( abortSignal: abortCtrl.current.signal, sessionId: searchStrategyParams?.sessionId, }; - const sub = combineLatest( - sortedConfigs - .map((config, idx) => - getFieldStats( - data, - params, - { - fieldName: config.fieldName, - type: config.type, - cardinality: config.cardinality, - safeFieldName: getSafeAggregationName(config.fieldName, idx), - }, - searchOptions - ) - ) - .filter((obs) => obs !== undefined) as Array> + + const batches = createBatchedRequests( + sortedConfigs.map((config, idx) => ({ + fieldName: config.fieldName, + type: config.type, + cardinality: config.cardinality, + safeFieldName: getSafeAggregationName(config.fieldName, idx), + })), + 10 ); - searchSubscription$.current = sub.subscribe({ + const subTemp = combineLatest( + batches + .map((batch) => getFieldsStats(data, params, batch, searchOptions, true)) + .filter((obs) => obs !== undefined) as Array> + ); + + const statsMap$ = new Subject(); + const fieldsToRetry$ = new Subject(); + + const onError = (error: any) => { + toasts.addError(error, { + title: i18n.translate('xpack.dataVisualizer.index.errorFetchingFieldStatisticsMessage', { + defaultMessage: 'Error fetching field statistics', + }), + }); + setFetchState({ + isRunning: false, + error, + }); + }; + + const onComplete = () => { + setFetchState({ + isRunning: false, + }); + }; + + // First, attempt to fetch field stats in batches of 10 + searchSubscription$.current = subTemp.subscribe({ next: (resp) => { if (resp) { - const statsMap = resp.reduce((map, field) => { - map.set(field.fieldName, field); - return map; - }, new Map()); + const statsMap = new Map(); + const failedFields: Field[] = []; + resp.forEach((batchResponse) => { + if (Array.isArray(batchResponse)) { + batchResponse.forEach((f) => { + if (f.fieldName !== undefined) { + statsMap.set(f.fieldName, f); + } + }); + } else { + // If an error occurred during batch + // retry each field in the failed batch individually + failedFields.push(...(batchResponse.fields ?? [])); + } + }); setFetchState({ loaded: (resp.length / sortedConfigs.length) * 100, @@ -150,30 +211,53 @@ export function useFieldStatsSearchStrategy( }); setFieldStats(statsMap); + + statsMap$.next(statsMap); + fieldsToRetry$.next(failedFields); } }, - error: (error) => { - toasts.addError(error, { - title: i18n.translate('xpack.dataVisualizer.index.errorFetchingFieldStatisticsMessage', { - defaultMessage: 'Error fetching field statistics', - }), - }); - setFetchState({ - isRunning: false, - error, - }); - }, - complete: () => { - setFetchState({ - isRunning: false, + error: onError, + complete: onComplete, + }); + + // If any of batches failed, retry each of the failed field at least one time individually + retries$.current = combineLatest([ + statsMap$, + fieldsToRetry$.pipe( + switchMap((failedFields) => { + return combineLatest( + failedFields + .map((failedField) => + getFieldsStats(data, params, [failedField], searchOptions, false) + ) + .filter((obs) => obs !== undefined) + ); + }) + ), + ]).subscribe({ + next: (resp) => { + const statsMap = cloneDeep(resp[0]) as Map; + const fieldBatches = resp[1]; + + fieldBatches.forEach((f) => { + if (Array.isArray(f) && f.length === 1) { + statsMap.set(f[0].fieldName, f[0]); + } }); + setFieldStats(statsMap); }, + error: onError, + complete: onComplete, }); }, [data, toasts, searchStrategyParams, fieldStatsParams, initialDataVisualizerListState]); const cancelFetch = useCallback(() => { searchSubscription$.current?.unsubscribe(); searchSubscription$.current = undefined; + + retries$.current?.unsubscribe(); + retries$.current = undefined; + abortCtrl.current.abort(); setFetchState({ isRunning: false, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts new file mode 100644 index 0000000000000..4023cd376396c --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts @@ -0,0 +1,347 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { find, get } from 'lodash'; +import { catchError, map } from 'rxjs/operators'; +import { Observable, of } from 'rxjs'; +import { AggregationsTermsAggregation } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { + MAX_PERCENT, + PERCENTILE_SPACING, + SAMPLER_TOP_TERMS_SHARD_SIZE, + SAMPLER_TOP_TERMS_THRESHOLD, +} from './constants'; +import { + buildSamplerAggregation, + getSamplerAggregationsResponsePath, +} from '../../../../../common/utils/query_utils'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; +import type { Aggs, FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats'; +import type { + Field, + NumericFieldStats, + Bucket, + FieldStatsError, +} from '../../../../../common/types/field_stats'; +import { processDistributionData } from '../../utils/process_distribution_data'; +import { + IKibanaSearchRequest, + IKibanaSearchResponse, + ISearchOptions, +} from '../../../../../../../../src/plugins/data/common'; +import { DataPublicPluginStart } from '../../../../../../../../src/plugins/data/public'; +import { extractErrorProperties } from '../../utils/error_utils'; +import { + FieldStats, + isIKibanaSearchResponse, + StringFieldStats, +} from '../../../../../common/types/field_stats'; +import { JOB_FIELD_TYPES } from '../../../../../common'; + +export const getFieldsStats = ( + dataPlugin: DataPublicPluginStart, + params: FieldStatsCommonRequestParams, + fields: Array<{ + fieldName: string; + type: string; + cardinality: number; + safeFieldName: string; + }>, + options: ISearchOptions, + fakeFailure = false +): Observable | undefined => { + const fieldType = fields[0].type; + switch (fieldType) { + case JOB_FIELD_TYPES.NUMBER: + return fetchNumericFieldsStats(dataPlugin, params, fields, options); + case JOB_FIELD_TYPES.KEYWORD: + // case JOB_FIELD_TYPES.IP: + return fetchStringFieldsStats(dataPlugin, params, fields, options); + case JOB_FIELD_TYPES.IP: + return fetchStringFieldsStats(dataPlugin, params, fields, options, fakeFailure); + } +}; + +export const getNumericFieldsStatsRequest = ( + params: FieldStatsCommonRequestParams, + fields: Field[] +) => { + const { index, query, runtimeFieldMap, samplerShardSize } = params; + + const size = 0; + + // Build the percents parameter which defines the percentiles to query + // for the metric distribution data. + // Use a fixed percentile spacing of 5%. + let count = 0; + const percents = Array.from( + Array(MAX_PERCENT / PERCENTILE_SPACING), + () => (count += PERCENTILE_SPACING) + ); + + const aggs: Aggs = {}; + + fields.forEach((field, i) => { + const { safeFieldName } = field; + + aggs[`${safeFieldName}_field_stats`] = { + filter: { exists: { field: field.fieldName } }, + aggs: { + actual_stats: { + stats: { field: field.fieldName }, + }, + }, + }; + aggs[`${safeFieldName}_percentiles`] = { + percentiles: { + field: field.fieldName, + percents, + keyed: false, + }, + }; + + const top = { + terms: { + field: field.fieldName, + size: 10, + order: { + _count: 'desc', + }, + } as AggregationsTermsAggregation, + }; + + // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation + // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + aggs[`${safeFieldName}_top`] = { + sampler: { + shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, + }, + aggs: { + top, + }, + }; + } else { + aggs[`${safeFieldName}_top`] = top; + } + }); + + const searchBody = { + query, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchNumericFieldsStats = ( + data: DataPublicPluginStart, + params: FieldStatsCommonRequestParams, + fields: Field[], + options: ISearchOptions +): Observable => { + const { samplerShardSize } = params; + const request: estypes.SearchRequest = getNumericFieldsStatsRequest(params, fields); + + return data.search + .search({ params: request }, options) + .pipe( + catchError((e) => { + // @todo: kick off another requests individually + return of({ + fields, + error: extractErrorProperties(e), + } as FieldStatsError); + }), + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; + + const aggregations = resp.rawResponse.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + + const batchStats: NumericFieldStats[] = []; + + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; + const docCount = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], + 0 + ); + const fieldStatsResp = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], + {} + ); + + const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + topAggsPath.push('top'); + } + + const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); + + const stats: NumericFieldStats = { + fieldName: field.fieldName, + count: docCount, + min: get(fieldStatsResp, 'min', 0), + max: get(fieldStatsResp, 'max', 0), + avg: get(fieldStatsResp, 'avg', 0), + isTopValuesSampled: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, + topValues, + topValuesSampleSize: topValues.reduce( + (acc, curr) => acc + curr.doc_count, + get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) + ), + topValuesSamplerShardSize: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? SAMPLER_TOP_TERMS_SHARD_SIZE + : samplerShardSize, + }; + + if (stats.count > 0) { + const percentiles = get( + aggregations, + [...aggsPath, `${safeFieldName}_percentiles`, 'values'], + [] + ); + const medianPercentile: { value: number; key: number } | undefined = find(percentiles, { + key: 50, + }); + stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; + stats.distribution = processDistributionData( + percentiles, + PERCENTILE_SPACING, + stats.min + ); + } + + batchStats.push(stats); + }); + + return batchStats; + }) + ); +}; + +export const getStringFieldStatsRequest = ( + params: FieldStatsCommonRequestParams, + fields: Field[] +) => { + const { index, query, runtimeFieldMap, samplerShardSize } = params; + + const size = 0; + + const aggs: Aggs = {}; + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; + const top = { + terms: { + field: field.fieldName, + size: 10, + order: { + _count: 'desc', + }, + } as AggregationsTermsAggregation, + }; + + // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation + // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + aggs[`${safeFieldName}_top`] = { + sampler: { + shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, + }, + aggs: { + top, + }, + }; + } else { + aggs[`${safeFieldName}_top`] = top; + } + }); + + const searchBody = { + query, + aggs: buildSamplerAggregation(aggs, samplerShardSize), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + }; + + return { + index, + size, + body: searchBody, + }; +}; + +export const fetchStringFieldsStats = ( + data: DataPublicPluginStart, + params: FieldStatsCommonRequestParams, + unfakeFields: Field[], + options: ISearchOptions, + // @todo: clean up fakeFailure + fakeFailure?: boolean +): Observable => { + const fields = fakeFailure ? unfakeFields.map((f) => f.fieldName) : unfakeFields; + + const { samplerShardSize } = params; + const request: estypes.SearchRequest = getStringFieldStatsRequest(params, fields); + + return data.search + .search({ params: request }, options) + .pipe( + catchError((e) => + of({ + fields: unfakeFields, + error: extractErrorProperties(e), + } as FieldStatsError) + ), + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; + const aggregations = resp.rawResponse.aggregations; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const batchStats: StringFieldStats[] = []; + + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; + + const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + topAggsPath.push('top'); + } + + const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); + + const stats = { + fieldName: field.fieldName, + isTopValuesSampled: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, + topValues, + topValuesSampleSize: topValues.reduce( + (acc, curr) => acc + curr.doc_count, + get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) + ), + topValuesSamplerShardSize: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? SAMPLER_TOP_TERMS_SHARD_SIZE + : samplerShardSize, + }; + + batchStats.push(stats); + }); + + return batchStats; + }) + ); +}; From 41e7cb60807c170e0d95e0e59ef91cbe518a6e22 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 4 Nov 2021 11:28:21 -0500 Subject: [PATCH 183/188] [ML] Batch requests for speed and retry failures for resiliency --- .../hooks/use_data_visualizer_grid_data.ts | 24 +- .../hooks/use_field_stats.ts | 30 +- .../requests/get_boolean_field_stats.ts | 72 ++-- .../requests/get_date_field_stats.ts | 71 ++-- .../requests/get_field_examples.ts | 82 +++-- .../requests/get_field_stats.ts | 55 --- .../requests/get_fields_stats.ts | 341 ++---------------- .../requests/get_numeric_field_stats.ts | 190 +++++----- .../requests/get_string_field_stats.ts | 114 +++--- 9 files changed, 336 insertions(+), 643 deletions(-) delete mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 6ac8c6d5b3b74..dd2b7d7ca640b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -222,13 +222,11 @@ export const useDataVisualizerGridData = ( const existMetricFields = metricConfigs .map((config) => { if (config.existsInDocs === false) return; - if (config?.stats?.cardinality !== undefined) { - return { - fieldName: config.fieldName, - type: config.type, - cardinality: config.stats.cardinality, - }; - } + return { + fieldName: config.fieldName, + type: config.type, + cardinality: config.stats?.cardinality ?? 0, + }; }) .filter((c) => c !== undefined) as FieldRequestConfig[]; @@ -237,13 +235,11 @@ export const useDataVisualizerGridData = ( const existNonMetricFields: FieldRequestConfig[] = nonMetricConfigs .map((config) => { if (config.existsInDocs === false) return; - if (config?.stats?.cardinality !== undefined) { - return { - fieldName: config.fieldName, - type: config.type, - cardinality: config.stats.cardinality, - }; - } + return { + fieldName: config.fieldName, + type: config.type, + cardinality: config.stats?.cardinality ?? 0, + }; }) .filter((c) => c !== undefined) as FieldRequestConfig[]; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index 8f8ab73263a91..604cf28b76ab4 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -158,9 +158,9 @@ export function useFieldStatsSearchStrategy( 10 ); - const subTemp = combineLatest( + const fieldStatsSub = combineLatest( batches - .map((batch) => getFieldsStats(data, params, batch, searchOptions, true)) + .map((batch) => getFieldsStats(data, params, batch, searchOptions)) .filter((obs) => obs !== undefined) as Array> ); @@ -186,7 +186,7 @@ export function useFieldStatsSearchStrategy( }; // First, attempt to fetch field stats in batches of 10 - searchSubscription$.current = subTemp.subscribe({ + searchSubscription$.current = fieldStatsSub.subscribe({ next: (resp) => { if (resp) { const statsMap = new Map(); @@ -206,7 +206,7 @@ export function useFieldStatsSearchStrategy( }); setFetchState({ - loaded: (resp.length / sortedConfigs.length) * 100, + loaded: (statsMap.size / sortedConfigs.length) * 100, isRunning: true, }); @@ -227,9 +227,7 @@ export function useFieldStatsSearchStrategy( switchMap((failedFields) => { return combineLatest( failedFields - .map((failedField) => - getFieldsStats(data, params, [failedField], searchOptions, false) - ) + .map((failedField) => getFieldsStats(data, params, [failedField], searchOptions)) .filter((obs) => obs !== undefined) ); }) @@ -239,12 +237,18 @@ export function useFieldStatsSearchStrategy( const statsMap = cloneDeep(resp[0]) as Map; const fieldBatches = resp[1]; - fieldBatches.forEach((f) => { - if (Array.isArray(f) && f.length === 1) { - statsMap.set(f[0].fieldName, f[0]); - } - }); - setFieldStats(statsMap); + if (Array.isArray(fieldBatches)) { + fieldBatches.forEach((f) => { + if (Array.isArray(f) && f.length === 1) { + statsMap.set(f[0].fieldName, f[0]); + } + }); + setFieldStats(statsMap); + setFetchState({ + loaded: (statsMap.size / sortedConfigs.length) * 100, + isRunning: true, + }); + } }, error: onError, complete: onComplete, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts index 06229f80585f1..bc856f9bf69d2 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts @@ -28,26 +28,26 @@ import { } from '../../../../../../../../src/plugins/data/public'; import { extractErrorProperties } from '../../utils/error_utils'; -export const getBooleanFieldStatsRequest = ( +export const getBooleanFieldsStatsRequest = ( params: FieldStatsCommonRequestParams, - field: Field + fields: Field[] ) => { const { index, query, runtimeFieldMap, samplerShardSize } = params; const size = 0; const aggs: Aggs = {}; - - const safeFieldName = field.safeFieldName; - aggs[`${safeFieldName}_value_count`] = { - filter: { exists: { field: field.fieldName } }, - }; - aggs[`${safeFieldName}_values`] = { - terms: { - field: field.fieldName, - size: 2, - }, - }; - + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; + aggs[`${safeFieldName}_value_count`] = { + filter: { exists: { field: field.fieldName } }, + }; + aggs[`${safeFieldName}_values`] = { + terms: { + field: field.fieldName, + size: 2, + }, + }; + }); const searchBody = { query, aggs: buildSamplerAggregation(aggs, samplerShardSize), @@ -61,20 +61,20 @@ export const getBooleanFieldStatsRequest = ( }; }; -export const fetchBooleanFieldStats = ( +export const fetchBooleanFieldsStats = ( data: DataPublicPluginStart, params: FieldStatsCommonRequestParams, - field: Field, + fields: Field[], options: ISearchOptions -): Observable => { +): Observable => { const { samplerShardSize } = params; - const request: estypes.SearchRequest = getBooleanFieldStatsRequest(params, field); + const request: estypes.SearchRequest = getBooleanFieldsStatsRequest(params, fields); return data.search .search({ params: request }, options) .pipe( catchError((e) => of({ - fieldName: field.fieldName, + fields, error: extractErrorProperties(e), } as FieldStatsError) ), @@ -84,23 +84,27 @@ export const fetchBooleanFieldStats = ( const aggregations = resp.rawResponse.aggregations; const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const safeFieldName = field.safeFieldName; - const stats: BooleanFieldStats = { - fieldName: field.fieldName, - count: get(aggregations, [...aggsPath, `${safeFieldName}_value_count`, 'doc_count'], 0), - trueCount: 0, - falseCount: 0, - }; + const batchStats: BooleanFieldStats[] = fields.map((field, i) => { + const safeFieldName = field.fieldName; + const stats: BooleanFieldStats = { + fieldName: field.fieldName, + count: get(aggregations, [...aggsPath, `${safeFieldName}_value_count`, 'doc_count'], 0), + trueCount: 0, + falseCount: 0, + }; - const valueBuckets: Array<{ [key: string]: number }> = get( - aggregations, - [...aggsPath, `${safeFieldName}_values`, 'buckets'], - [] - ); - valueBuckets.forEach((bucket) => { - stats[`${bucket.key_as_string}Count`] = bucket.doc_count; + const valueBuckets: Array<{ [key: string]: number }> = get( + aggregations, + [...aggsPath, `${safeFieldName}_values`, 'buckets'], + [] + ); + valueBuckets.forEach((bucket) => { + stats[`${bucket.key_as_string}Count`] = bucket.doc_count; + }); + return stats; }); - return stats; + + return batchStats; }) ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts index b4336bc205a8d..d87c002e907c9 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts @@ -25,21 +25,26 @@ import { import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; import { extractErrorProperties } from '../../utils/error_utils'; -export const getDateFieldStatsRequest = (params: FieldStatsCommonRequestParams, field: Field) => { +export const getDateFieldsStatsRequest = ( + params: FieldStatsCommonRequestParams, + fields: Field[] +) => { const { index, query, runtimeFieldMap, samplerShardSize } = params; const size = 0; const aggs: Aggs = {}; - const safeFieldName = field.safeFieldName; - aggs[`${safeFieldName}_field_stats`] = { - filter: { exists: { field: field.fieldName } }, - aggs: { - actual_stats: { - stats: { field: field.fieldName }, + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; + aggs[`${safeFieldName}_field_stats`] = { + filter: { exists: { field: field.fieldName } }, + aggs: { + actual_stats: { + stats: { field: field.fieldName }, + }, }, - }, - }; + }; + }); const searchBody = { query, @@ -53,21 +58,21 @@ export const getDateFieldStatsRequest = (params: FieldStatsCommonRequestParams, }; }; -export const fetchDateFieldStats = ( +export const fetchDateFieldsStats = ( data: DataPublicPluginStart, params: FieldStatsCommonRequestParams, - field: Field, + fields: Field[], options: ISearchOptions -): Observable => { +): Observable => { const { samplerShardSize } = params; - const request: estypes.SearchRequest = getDateFieldStatsRequest(params, field); + const request: estypes.SearchRequest = getDateFieldsStatsRequest(params, fields); return data.search .search({ params: request }, options) .pipe( catchError((e) => of({ - fieldName: field.fieldName, + fields, error: extractErrorProperties(e), } as FieldStatsError) ), @@ -75,23 +80,27 @@ export const fetchDateFieldStats = ( if (!isIKibanaSearchResponse(resp)) return resp; const aggregations = resp.rawResponse.aggregations; const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const safeFieldName = field.safeFieldName; - const docCount = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], - 0 - ); - const fieldStatsResp = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], - {} - ); - return { - fieldName: field.fieldName, - count: docCount, - earliest: get(fieldStatsResp, 'min', 0), - latest: get(fieldStatsResp, 'max', 0), - }; + + const batchStats: DateFieldStats[] = fields.map((field, i) => { + const safeFieldName = field.safeFieldName; + const docCount = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], + 0 + ); + const fieldStatsResp = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], + {} + ); + return { + fieldName: field.fieldName, + count: docCount, + earliest: get(fieldStatsResp, 'min', 0), + latest: get(fieldStatsResp, 'max', 0), + } as DateFieldStats; + }); + return batchStats; }) ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts index de29fa6e31589..d85bab5a1a30a 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts @@ -6,7 +6,7 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { get } from 'lodash'; -import { Observable, of } from 'rxjs'; +import { combineLatest, of } from 'rxjs'; import { catchError, map } from 'rxjs/operators'; import { buildBaseFilterCriteria } from '../../../../../common/utils/query_utils'; import { isPopulatedObject } from '../../../../../common/utils/object_utils'; @@ -59,50 +59,56 @@ export const getFieldExamplesRequest = (params: FieldStatsCommonRequestParams, f }; }; -export const fetchFieldExamples = ( +export const fetchFieldsExamples = ( data: DataPublicPluginStart, params: FieldStatsCommonRequestParams, - field: Field, + fields: Field[], options: ISearchOptions -): Observable => { - const request: estypes.SearchRequest = getFieldExamplesRequest(params, field); +) => { const { maxExamples } = params; - return data.search - .search({ params: request }, options) - .pipe( - catchError((e) => - of({ - fieldName: field.fieldName, - error: extractErrorProperties(e), - } as FieldStatsError) - ), - map((resp) => { - if (!isIKibanaSearchResponse(resp)) return resp; - const body = resp.rawResponse; - const stats = { - fieldName: field.fieldName, - examples: [] as unknown[], - } as FieldExamples; + return combineLatest( + fields.map((field) => { + const request: estypes.SearchRequest = getFieldExamplesRequest(params, field); - if (body.hits.total > 0) { - const hits = body.hits.hits; - for (let i = 0; i < hits.length; i++) { - // Use lodash get() to support field names containing dots. - const doc: object[] | undefined = get(hits[i].fields, field.fieldName); - // the results from fields query is always an array - if (Array.isArray(doc) && doc.length > 0) { - const example = doc[0]; - if (example !== undefined && stats.examples.indexOf(example) === -1) { - stats.examples.push(example); - if (stats.examples.length === maxExamples) { - break; + return data.search + .search({ params: request }, options) + .pipe( + catchError((e) => + of({ + fieldName: field.fieldName, + fields, + error: extractErrorProperties(e), + } as FieldStatsError) + ), + map((resp) => { + if (!isIKibanaSearchResponse(resp)) return resp; + const body = resp.rawResponse; + const stats = { + fieldName: field.fieldName, + examples: [] as unknown[], + } as FieldExamples; + + if (body.hits.total > 0) { + const hits = body.hits.hits; + for (let i = 0; i < hits.length; i++) { + // Use lodash get() to support field names containing dots. + const doc: object[] | undefined = get(hits[i].fields, field.fieldName); + // the results from fields query is always an array + if (Array.isArray(doc) && doc.length > 0) { + const example = doc[0]; + if (example !== undefined && stats.examples.indexOf(example) === -1) { + stats.examples.push(example); + if (stats.examples.length === maxExamples) { + break; + } + } } } } - } - } - return stats; - }) - ); + return stats; + }) + ); + }) + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts deleted file mode 100644 index 41c484ac1a4f2..0000000000000 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_stats.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Observable } from 'rxjs'; -import { - FieldStats, - FieldStatsError, - isValidField, - FieldStatsCommonRequestParams, -} from '../../../../../common/types/field_stats'; -import { fetchNumericFieldStats } from './get_numeric_field_stats'; -import { fetchStringFieldStats } from './get_string_field_stats'; -import { fetchDateFieldStats } from './get_date_field_stats'; -import { fetchBooleanFieldStats } from './get_boolean_field_stats'; -import { fetchFieldExamples } from './get_field_examples'; -import { JOB_FIELD_TYPES } from '../../../../../common'; -import { ISearchOptions } from '../../../../../../../../src/plugins/data/common'; -import { DataPublicPluginStart } from '../../../../../../../../src/plugins/data/public'; - -export const getFieldStats = ( - dataPlugin: DataPublicPluginStart, - params: FieldStatsCommonRequestParams, - field: { - fieldName: string; - type: string; - cardinality: number; - safeFieldName: string; - }, - options: ISearchOptions -): Observable | undefined => { - // An invalid field with undefined fieldName is used for a document count request. - if (!isValidField(field)) return; - - switch (field.type) { - case JOB_FIELD_TYPES.NUMBER: - return fetchNumericFieldStats(dataPlugin, params, field, options); - case JOB_FIELD_TYPES.KEYWORD: - case JOB_FIELD_TYPES.IP: - return fetchStringFieldStats(dataPlugin, params, field, options); - case JOB_FIELD_TYPES.DATE: - return fetchDateFieldStats(dataPlugin, params, field, options); - case JOB_FIELD_TYPES.BOOLEAN: - return fetchBooleanFieldStats(dataPlugin, params, field, options); - case JOB_FIELD_TYPES.TEXT: - return fetchFieldExamples(dataPlugin, params, field, options); - default: - // Use an exists filter on the the field name to get - // examples of the field, so cannot batch up. - return fetchFieldExamples(dataPlugin, params, field, options); - } -}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts index 4023cd376396c..c8b4ae67f4c8c 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts @@ -5,43 +5,18 @@ * 2.0. */ -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { find, get } from 'lodash'; -import { catchError, map } from 'rxjs/operators'; -import { Observable, of } from 'rxjs'; -import { AggregationsTermsAggregation } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { - MAX_PERCENT, - PERCENTILE_SPACING, - SAMPLER_TOP_TERMS_SHARD_SIZE, - SAMPLER_TOP_TERMS_THRESHOLD, -} from './constants'; -import { - buildSamplerAggregation, - getSamplerAggregationsResponsePath, -} from '../../../../../common/utils/query_utils'; -import { isPopulatedObject } from '../../../../../common/utils/object_utils'; -import type { Aggs, FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats'; -import type { - Field, - NumericFieldStats, - Bucket, - FieldStatsError, -} from '../../../../../common/types/field_stats'; -import { processDistributionData } from '../../utils/process_distribution_data'; -import { - IKibanaSearchRequest, - IKibanaSearchResponse, - ISearchOptions, -} from '../../../../../../../../src/plugins/data/common'; +import { Observable } from 'rxjs'; +import type { FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats'; +import type { FieldStatsError } from '../../../../../common/types/field_stats'; +import { ISearchOptions } from '../../../../../../../../src/plugins/data/common'; import { DataPublicPluginStart } from '../../../../../../../../src/plugins/data/public'; -import { extractErrorProperties } from '../../utils/error_utils'; -import { - FieldStats, - isIKibanaSearchResponse, - StringFieldStats, -} from '../../../../../common/types/field_stats'; +import { FieldStats } from '../../../../../common/types/field_stats'; import { JOB_FIELD_TYPES } from '../../../../../common'; +import { fetchDateFieldsStats } from './get_date_field_stats'; +import { fetchBooleanFieldsStats } from './get_boolean_field_stats'; +import { fetchFieldsExamples } from './get_field_examples'; +import { fetchNumericFieldsStats } from './get_numeric_field_stats'; +import { fetchStringFieldsStats } from './get_string_field_stats'; export const getFieldsStats = ( dataPlugin: DataPublicPluginStart, @@ -52,296 +27,24 @@ export const getFieldsStats = ( cardinality: number; safeFieldName: string; }>, - options: ISearchOptions, - fakeFailure = false + options: ISearchOptions ): Observable | undefined => { const fieldType = fields[0].type; switch (fieldType) { case JOB_FIELD_TYPES.NUMBER: return fetchNumericFieldsStats(dataPlugin, params, fields, options); case JOB_FIELD_TYPES.KEYWORD: - // case JOB_FIELD_TYPES.IP: - return fetchStringFieldsStats(dataPlugin, params, fields, options); case JOB_FIELD_TYPES.IP: - return fetchStringFieldsStats(dataPlugin, params, fields, options, fakeFailure); + return fetchStringFieldsStats(dataPlugin, params, fields, options); + case JOB_FIELD_TYPES.DATE: + return fetchDateFieldsStats(dataPlugin, params, fields, options); + case JOB_FIELD_TYPES.BOOLEAN: + return fetchBooleanFieldsStats(dataPlugin, params, fields, options); + case JOB_FIELD_TYPES.TEXT: + return fetchFieldsExamples(dataPlugin, params, fields, options); + default: + // Use an exists filter on the the field name to get + // examples of the field, so cannot batch up. + return fetchFieldsExamples(dataPlugin, params, fields, options); } }; - -export const getNumericFieldsStatsRequest = ( - params: FieldStatsCommonRequestParams, - fields: Field[] -) => { - const { index, query, runtimeFieldMap, samplerShardSize } = params; - - const size = 0; - - // Build the percents parameter which defines the percentiles to query - // for the metric distribution data. - // Use a fixed percentile spacing of 5%. - let count = 0; - const percents = Array.from( - Array(MAX_PERCENT / PERCENTILE_SPACING), - () => (count += PERCENTILE_SPACING) - ); - - const aggs: Aggs = {}; - - fields.forEach((field, i) => { - const { safeFieldName } = field; - - aggs[`${safeFieldName}_field_stats`] = { - filter: { exists: { field: field.fieldName } }, - aggs: { - actual_stats: { - stats: { field: field.fieldName }, - }, - }, - }; - aggs[`${safeFieldName}_percentiles`] = { - percentiles: { - field: field.fieldName, - percents, - keyed: false, - }, - }; - - const top = { - terms: { - field: field.fieldName, - size: 10, - order: { - _count: 'desc', - }, - } as AggregationsTermsAggregation, - }; - - // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation - // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - aggs[`${safeFieldName}_top`] = { - sampler: { - shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, - }, - aggs: { - top, - }, - }; - } else { - aggs[`${safeFieldName}_top`] = top; - } - }); - - const searchBody = { - query, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), - }; - - return { - index, - size, - body: searchBody, - }; -}; - -export const fetchNumericFieldsStats = ( - data: DataPublicPluginStart, - params: FieldStatsCommonRequestParams, - fields: Field[], - options: ISearchOptions -): Observable => { - const { samplerShardSize } = params; - const request: estypes.SearchRequest = getNumericFieldsStatsRequest(params, fields); - - return data.search - .search({ params: request }, options) - .pipe( - catchError((e) => { - // @todo: kick off another requests individually - return of({ - fields, - error: extractErrorProperties(e), - } as FieldStatsError); - }), - map((resp) => { - if (!isIKibanaSearchResponse(resp)) return resp; - - const aggregations = resp.rawResponse.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - - const batchStats: NumericFieldStats[] = []; - - fields.forEach((field, i) => { - const safeFieldName = field.safeFieldName; - const docCount = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], - 0 - ); - const fieldStatsResp = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], - {} - ); - - const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } - - const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); - - const stats: NumericFieldStats = { - fieldName: field.fieldName, - count: docCount, - min: get(fieldStatsResp, 'min', 0), - max: get(fieldStatsResp, 'max', 0), - avg: get(fieldStatsResp, 'avg', 0), - isTopValuesSampled: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, - topValues, - topValuesSampleSize: topValues.reduce( - (acc, curr) => acc + curr.doc_count, - get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) - ), - topValuesSamplerShardSize: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? SAMPLER_TOP_TERMS_SHARD_SIZE - : samplerShardSize, - }; - - if (stats.count > 0) { - const percentiles = get( - aggregations, - [...aggsPath, `${safeFieldName}_percentiles`, 'values'], - [] - ); - const medianPercentile: { value: number; key: number } | undefined = find(percentiles, { - key: 50, - }); - stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; - stats.distribution = processDistributionData( - percentiles, - PERCENTILE_SPACING, - stats.min - ); - } - - batchStats.push(stats); - }); - - return batchStats; - }) - ); -}; - -export const getStringFieldStatsRequest = ( - params: FieldStatsCommonRequestParams, - fields: Field[] -) => { - const { index, query, runtimeFieldMap, samplerShardSize } = params; - - const size = 0; - - const aggs: Aggs = {}; - fields.forEach((field, i) => { - const safeFieldName = field.safeFieldName; - const top = { - terms: { - field: field.fieldName, - size: 10, - order: { - _count: 'desc', - }, - } as AggregationsTermsAggregation, - }; - - // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation - // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - aggs[`${safeFieldName}_top`] = { - sampler: { - shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, - }, - aggs: { - top, - }, - }; - } else { - aggs[`${safeFieldName}_top`] = top; - } - }); - - const searchBody = { - query, - aggs: buildSamplerAggregation(aggs, samplerShardSize), - ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), - }; - - return { - index, - size, - body: searchBody, - }; -}; - -export const fetchStringFieldsStats = ( - data: DataPublicPluginStart, - params: FieldStatsCommonRequestParams, - unfakeFields: Field[], - options: ISearchOptions, - // @todo: clean up fakeFailure - fakeFailure?: boolean -): Observable => { - const fields = fakeFailure ? unfakeFields.map((f) => f.fieldName) : unfakeFields; - - const { samplerShardSize } = params; - const request: estypes.SearchRequest = getStringFieldStatsRequest(params, fields); - - return data.search - .search({ params: request }, options) - .pipe( - catchError((e) => - of({ - fields: unfakeFields, - error: extractErrorProperties(e), - } as FieldStatsError) - ), - map((resp) => { - if (!isIKibanaSearchResponse(resp)) return resp; - const aggregations = resp.rawResponse.aggregations; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const batchStats: StringFieldStats[] = []; - - fields.forEach((field, i) => { - const safeFieldName = field.safeFieldName; - - const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } - - const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); - - const stats = { - fieldName: field.fieldName, - isTopValuesSampled: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, - topValues, - topValuesSampleSize: topValues.reduce( - (acc, curr) => acc + curr.doc_count, - get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) - ), - topValuesSamplerShardSize: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? SAMPLER_TOP_TERMS_SHARD_SIZE - : samplerShardSize, - }; - - batchStats.push(stats); - }); - - return batchStats; - }) - ); -}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts index 0c799d161b091..d958d50dac69e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts @@ -38,9 +38,9 @@ import { DataPublicPluginStart } from '../../../../../../../../src/plugins/data/ import { extractErrorProperties } from '../../utils/error_utils'; import { isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; -export const getNumericFieldStatsRequest = ( +export const getNumericFieldsStatsRequest = ( params: FieldStatsCommonRequestParams, - field: Field + fields: Field[] ) => { const { index, query, runtimeFieldMap, samplerShardSize } = params; @@ -55,48 +55,52 @@ export const getNumericFieldStatsRequest = ( () => (count += PERCENTILE_SPACING) ); - const safeFieldName = field.safeFieldName; + const aggs: Aggs = {}; - const top = { - terms: { - field: field.fieldName, - size: 10, - order: { - _count: 'desc', - }, - } as AggregationsTermsAggregation, - }; + fields.forEach((field, i) => { + const { safeFieldName } = field; - const aggs: Aggs = { - [`${safeFieldName}_field_stats`]: { + aggs[`${safeFieldName}_field_stats`] = { filter: { exists: { field: field.fieldName } }, aggs: { actual_stats: { stats: { field: field.fieldName }, }, }, - }, - [`${safeFieldName}_percentiles`]: { + }; + aggs[`${safeFieldName}_percentiles`] = { percentiles: { field: field.fieldName, percents, keyed: false, }, - }, + }; + + const top = { + terms: { + field: field.fieldName, + size: 10, + order: { + _count: 'desc', + }, + } as AggregationsTermsAggregation, + }; + // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). - [`${safeFieldName}_top`]: - samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? { - sampler: { - shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, - }, - aggs: { - top, - }, - } - : top, - }; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + aggs[`${safeFieldName}_top`] = { + sampler: { + shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, + }, + aggs: { + top, + }, + }; + } else { + aggs[`${safeFieldName}_top`] = top; + } + }); const searchBody = { query, @@ -111,81 +115,93 @@ export const getNumericFieldStatsRequest = ( }; }; -export const fetchNumericFieldStats = ( +export const fetchNumericFieldsStats = ( data: DataPublicPluginStart, params: FieldStatsCommonRequestParams, - field: Field, + fields: Field[], options: ISearchOptions -): Observable => { +): Observable => { const { samplerShardSize } = params; - const request: estypes.SearchRequest = getNumericFieldStatsRequest(params, field); + const request: estypes.SearchRequest = getNumericFieldsStatsRequest(params, fields); return data.search .search({ params: request }, options) .pipe( - catchError((e) => - of({ - fieldName: field.fieldName, + catchError((e) => { + // @todo: kick off another requests individually + return of({ + fields, error: extractErrorProperties(e), - } as FieldStatsError) - ), + } as FieldStatsError); + }), map((resp) => { if (!isIKibanaSearchResponse(resp)) return resp; const aggregations = resp.rawResponse.aggregations; const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const safeFieldName = field.safeFieldName; - const docCount = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], - 0 - ); - const fieldStatsResp = get( - aggregations, - [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], - {} - ); - - const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } - - const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); - - const stats: NumericFieldStats = { - fieldName: field.fieldName, - count: docCount, - min: get(fieldStatsResp, 'min', 0), - max: get(fieldStatsResp, 'max', 0), - avg: get(fieldStatsResp, 'avg', 0), - isTopValuesSampled: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, - topValues, - topValuesSampleSize: topValues.reduce( - (acc, curr) => acc + curr.doc_count, - get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) - ), - topValuesSamplerShardSize: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? SAMPLER_TOP_TERMS_SHARD_SIZE - : samplerShardSize, - }; - - if (stats.count > 0) { - const percentiles = get( + const batchStats: NumericFieldStats[] = []; + + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; + const docCount = get( + aggregations, + [...aggsPath, `${safeFieldName}_field_stats`, 'doc_count'], + 0 + ); + const fieldStatsResp = get( aggregations, - [...aggsPath, `${safeFieldName}_percentiles`, 'values'], - [] + [...aggsPath, `${safeFieldName}_field_stats`, 'actual_stats'], + {} ); - const medianPercentile: { value: number; key: number } | undefined = find(percentiles, { - key: 50, - }); - stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; - stats.distribution = processDistributionData(percentiles, PERCENTILE_SPACING, stats.min); - } - return stats; + + const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + topAggsPath.push('top'); + } + + const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); + + const stats: NumericFieldStats = { + fieldName: field.fieldName, + count: docCount, + min: get(fieldStatsResp, 'min', 0), + max: get(fieldStatsResp, 'max', 0), + avg: get(fieldStatsResp, 'avg', 0), + isTopValuesSampled: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, + topValues, + topValuesSampleSize: topValues.reduce( + (acc, curr) => acc + curr.doc_count, + get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) + ), + topValuesSamplerShardSize: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? SAMPLER_TOP_TERMS_SHARD_SIZE + : samplerShardSize, + }; + + if (stats.count > 0) { + const percentiles = get( + aggregations, + [...aggsPath, `${safeFieldName}_percentiles`, 'values'], + [] + ); + const medianPercentile: { value: number; key: number } | undefined = find(percentiles, { + key: 50, + }); + stats.median = medianPercentile !== undefined ? medianPercentile!.value : 0; + stats.distribution = processDistributionData( + percentiles, + PERCENTILE_SPACING, + stats.min + ); + } + + batchStats.push(stats); + }); + + return batchStats; }) ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts index 1c9cb49460613..d8803a4347a4d 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts @@ -8,6 +8,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { get } from 'lodash'; import { Observable, of } from 'rxjs'; import { catchError, map } from 'rxjs/operators'; +import { AggregationsTermsAggregation } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { SAMPLER_TOP_TERMS_SHARD_SIZE, SAMPLER_TOP_TERMS_THRESHOLD } from './constants'; import { buildSamplerAggregation, @@ -30,38 +31,42 @@ import { import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; import { extractErrorProperties } from '../../utils/error_utils'; -export const getStringFieldStatsRequest = (params: FieldStatsCommonRequestParams, field: Field) => { +export const getStringFieldStatsRequest = ( + params: FieldStatsCommonRequestParams, + fields: Field[] +) => { const { index, query, runtimeFieldMap, samplerShardSize } = params; const size = 0; const aggs: Aggs = {}; - - const safeFieldName = field.safeFieldName; - const top: estypes.AggregationsAggregationContainer = { - terms: { - field: field.fieldName, - size: 10, - order: { - _count: 'desc', - }, - }, - }; - - // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation - // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - aggs[`${safeFieldName}_top`] = { - sampler: { - shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, - }, - aggs: { - top, - }, + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; + const top = { + terms: { + field: field.fieldName, + size: 10, + order: { + _count: 'desc', + }, + } as AggregationsTermsAggregation, }; - } else { - aggs[`${safeFieldName}_top`] = top; - } + + // If cardinality >= SAMPLE_TOP_TERMS_THRESHOLD, run the top terms aggregation + // in a sampler aggregation, even if no sampling has been specified (samplerShardSize < 1). + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + aggs[`${safeFieldName}_top`] = { + sampler: { + shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, + }, + aggs: { + top, + }, + }; + } else { + aggs[`${safeFieldName}_top`] = top; + } + }); const searchBody = { query, @@ -76,21 +81,21 @@ export const getStringFieldStatsRequest = (params: FieldStatsCommonRequestParams }; }; -export const fetchStringFieldStats = ( +export const fetchStringFieldsStats = ( data: DataPublicPluginStart, params: FieldStatsCommonRequestParams, - field: Field, + fields: Field[], options: ISearchOptions -): Observable => { +): Observable => { const { samplerShardSize } = params; - const request: estypes.SearchRequest = getStringFieldStatsRequest(params, field); + const request: estypes.SearchRequest = getStringFieldStatsRequest(params, fields); return data.search .search({ params: request }, options) .pipe( catchError((e) => of({ - fieldName: field.fieldName, + fields, error: extractErrorProperties(e), } as FieldStatsError) ), @@ -98,32 +103,37 @@ export const fetchStringFieldStats = ( if (!isIKibanaSearchResponse(resp)) return resp; const aggregations = resp.rawResponse.aggregations; const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const batchStats: StringFieldStats[] = []; + + fields.forEach((field, i) => { + const safeFieldName = field.safeFieldName; - const safeFieldName = field.safeFieldName; + const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; + if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { + topAggsPath.push('top'); + } - const topAggsPath = [...aggsPath, `${safeFieldName}_top`]; - if (samplerShardSize < 1 && field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD) { - topAggsPath.push('top'); - } + const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); - const topValues: Bucket[] = get(aggregations, [...topAggsPath, 'buckets'], []); + const stats = { + fieldName: field.fieldName, + isTopValuesSampled: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, + topValues, + topValuesSampleSize: topValues.reduce( + (acc, curr) => acc + curr.doc_count, + get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) + ), + topValuesSamplerShardSize: + field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD + ? SAMPLER_TOP_TERMS_SHARD_SIZE + : samplerShardSize, + }; - const stats = { - fieldName: field.fieldName, - isTopValuesSampled: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0, - topValues, - topValuesSampleSize: topValues.reduce( - (acc, curr) => acc + curr.doc_count, - get(aggregations, [...topAggsPath, 'sum_other_doc_count'], 0) - ), - topValuesSamplerShardSize: - field.cardinality >= SAMPLER_TOP_TERMS_THRESHOLD - ? SAMPLER_TOP_TERMS_SHARD_SIZE - : samplerShardSize, - }; + batchStats.push(stats); + }); - return stats; + return batchStats; }) ); }; From ccb0469954d15d851ea67ca86785da269bf847e9 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 5 Nov 2021 12:49:38 -0500 Subject: [PATCH 184/188] No need to fetch field stats if overall stats haven't completed --- .../hooks/use_data_visualizer_grid_data.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index dd2b7d7ca640b..5c8f4e3a0b13c 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -218,7 +218,10 @@ export const useDataVisualizerGridData = ( ] ); + const { overallStats, progress: overallStatsProgress } = useOverallStats(fieldStatsRequest); + const configsWithoutStats = useMemo(() => { + if (overallStatsProgress.loaded < 100) return; const existMetricFields = metricConfigs .map((config) => { if (config.existsInDocs === false) return; @@ -244,9 +247,8 @@ export const useDataVisualizerGridData = ( .filter((c) => c !== undefined) as FieldRequestConfig[]; return { metricConfigs: existMetricFields, nonMetricConfigs: existNonMetricFields }; - }, [metricConfigs, nonMetricConfigs]); + }, [metricConfigs, nonMetricConfigs, overallStatsProgress.loaded]); - const { overallStats, progress: overallStatsProgress } = useOverallStats(fieldStatsRequest); const strategyResponse = useFieldStatsSearchStrategy( fieldStatsRequest, configsWithoutStats, From 8b3d4c3f67decc1d16a871ffd3d2b882c79c20a1 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 5 Nov 2021 13:23:52 -0500 Subject: [PATCH 185/188] Wait on overallStats to complete --- .../index_data_visualizer/hooks/use_overall_stats.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index dcca6d71e63bf..8eacc1dc9aa4e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -6,7 +6,7 @@ */ import { useCallback, useEffect, useState, useRef, useMemo, useReducer } from 'react'; -import { combineLatest, forkJoin, of, Subscription } from 'rxjs'; +import { forkJoin, of, Subscription } from 'rxjs'; import { switchMap } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; import type { ToastsStart } from 'kibana/public'; @@ -113,7 +113,7 @@ export function useOverallStats 0 - ? combineLatest( + ? forkJoin( nonAggregatableFields.map((fieldName: string) => data.search .search( From 3d2d8b87a895abe78977c55ba2aaf860eaf24ad6 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 5 Nov 2021 15:07:06 -0500 Subject: [PATCH 186/188] Fix types after merge --- .../common/components/stats_table/types/index.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts index 00f8ac0c74eb9..6a597b50c47b4 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts @@ -4,11 +4,13 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - export type { FieldDataRowProps } from './field_data_row'; export type { FieldVisConfig, FileBasedFieldVisConfig, MetricFieldVisStats, -} from './field_vis_config'; -export { isFileBasedFieldVisConfig, isIndexBasedFieldVisConfig } from './field_vis_config'; +} from '../../../../../../common/types/field_vis_config'; +export type { + isFileBasedFieldVisConfig, + isIndexBasedFieldVisConfig, +} from '../../../../../../common/types/field_vis_config'; From 4370b44caad5a87c409cb83a4a510a76cafc87f8 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 5 Nov 2021 16:45:12 -0500 Subject: [PATCH 187/188] Fix payload size too big 413, num of requests --- .../components/stats_table/types/index.ts | 2 +- .../index_data_visualizer_view.tsx | 12 +-- .../hooks/use_data_visualizer_grid_data.ts | 6 +- .../hooks/use_field_stats.ts | 22 +++--- .../hooks/use_overall_stats.ts | 61 +++++++++----- .../requests/get_boolean_field_stats.ts | 8 +- .../requests/get_date_field_stats.ts | 8 +- .../requests/get_field_examples.ts | 8 +- .../requests/get_fields_stats.ts | 22 +++--- .../requests/get_numeric_field_stats.ts | 6 +- .../requests/get_string_field_stats.ts | 8 +- .../search_strategy/requests/overall_stats.ts | 79 ++++++++++--------- 12 files changed, 136 insertions(+), 106 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts index 6a597b50c47b4..6d9f4d5b86d28 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts @@ -10,7 +10,7 @@ export type { FileBasedFieldVisConfig, MetricFieldVisStats, } from '../../../../../../common/types/field_vis_config'; -export type { +export { isFileBasedFieldVisConfig, isIndexBasedFieldVisConfig, } from '../../../../../../common/types/field_vis_config'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index bfa9f28cfa65b..f528d8378bcd2 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -250,16 +250,18 @@ export const IndexDataVisualizerView: FC = (dataVi } = useDataVisualizerGridData(input, dataVisualizerListState, setGlobalState); useEffect(() => { - // Force refresh on index pattern change - setLastRefresh(Date.now()); - return () => { // When navigating away from the index pattern // Reset all previously set filters // to make sure new page doesn't have unrelated filters data.query.filterManager.removeAll(); }; - }, [currentIndexPattern.id, data.query.filterManager, setLastRefresh]); + }, [currentIndexPattern.id, data.query.filterManager]); + + useEffect(() => { + // Force refresh on index pattern change + setLastRefresh(Date.now()); + }, [currentIndexPattern.id, setLastRefresh]); useEffect(() => { if (globalState?.time !== undefined) { @@ -267,7 +269,6 @@ export const IndexDataVisualizerView: FC = (dataVi from: globalState.time.from, to: globalState.time.to, }); - setLastRefresh(Date.now()); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [JSON.stringify(globalState?.time), timefilter]); @@ -275,7 +276,6 @@ export const IndexDataVisualizerView: FC = (dataVi useEffect(() => { if (globalState?.refreshInterval !== undefined) { timefilter.setRefreshInterval(globalState.refreshInterval); - setLastRefresh(Date.now()); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [JSON.stringify(globalState?.refreshInterval), timefilter]); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 5c8f4e3a0b13c..e6e7a96e0329f 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -202,7 +202,6 @@ export const useDataVisualizerGridData = ( runtimeFieldMap: currentIndexPattern.getComputedFields().runtimeFields, aggregatableFields, nonAggregatableFields, - lastRefresh, }; }, // eslint-disable-next-line react-hooks/exhaustive-deps @@ -218,7 +217,10 @@ export const useDataVisualizerGridData = ( ] ); - const { overallStats, progress: overallStatsProgress } = useOverallStats(fieldStatsRequest); + const { overallStats, progress: overallStatsProgress } = useOverallStats( + fieldStatsRequest, + lastRefresh + ); const configsWithoutStats = useMemo(() => { if (overallStatsProgress.loaded < 100) return; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts index 604cf28b76ab4..64654d56db05b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts @@ -158,15 +158,14 @@ export function useFieldStatsSearchStrategy( 10 ); + const statsMap$ = new Subject(); + const fieldsToRetry$ = new Subject(); + const fieldStatsSub = combineLatest( batches - .map((batch) => getFieldsStats(data, params, batch, searchOptions)) + .map((batch) => getFieldsStats(data.search, params, batch, searchOptions)) .filter((obs) => obs !== undefined) as Array> ); - - const statsMap$ = new Subject(); - const fieldsToRetry$ = new Subject(); - const onError = (error: any) => { toasts.addError(error, { title: i18n.translate('xpack.dataVisualizer.index.errorFetchingFieldStatisticsMessage', { @@ -212,8 +211,10 @@ export function useFieldStatsSearchStrategy( setFieldStats(statsMap); - statsMap$.next(statsMap); - fieldsToRetry$.next(failedFields); + if (failedFields.length > 0) { + statsMap$.next(statsMap); + fieldsToRetry$.next(failedFields); + } } }, error: onError, @@ -227,7 +228,9 @@ export function useFieldStatsSearchStrategy( switchMap((failedFields) => { return combineLatest( failedFields - .map((failedField) => getFieldsStats(data, params, [failedField], searchOptions)) + .map((failedField) => + getFieldsStats(data.search, params, [failedField], searchOptions) + ) .filter((obs) => obs !== undefined) ); }) @@ -253,7 +256,8 @@ export function useFieldStatsSearchStrategy( error: onError, complete: onComplete, }); - }, [data, toasts, searchStrategyParams, fieldStatsParams, initialDataVisualizerListState]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [data.search, toasts, fieldStatsParams, initialDataVisualizerListState]); const cancelFetch = useCallback(() => { searchSubscription$.current?.unsubscribe(); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 8eacc1dc9aa4e..92a95bfacea42 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -10,8 +10,10 @@ import { forkJoin, of, Subscription } from 'rxjs'; import { switchMap } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; import type { ToastsStart } from 'kibana/public'; +import { chunk } from 'lodash'; import { useDataVisualizerKibana } from '../../kibana_context'; import { + AggregatableFieldOverallStats, checkAggregatableFieldsExistRequest, checkNonAggregatableFieldExistsRequest, processAggregatableFieldsExistResponse, @@ -62,7 +64,8 @@ function displayError(toastNotifications: ToastsStart, indexPattern: string, err } export function useOverallStats( - searchStrategyParams: TParams | undefined + searchStrategyParams: TParams | undefined, + lastRefresh: number ): { progress: DataStatsFetchProgress; overallStats: OverallStats; @@ -87,7 +90,8 @@ export function useOverallStats 0 - ? data.search.search( - { - params: checkAggregatableFieldsExistRequest( - index, - searchQuery, - aggregatableFields, - samplerShardSize, - timeFieldName, - earliest, - latest, - undefined, - runtimeFieldMap - ), - }, - searchOptions + ? aggregatableFieldsChunks.map((aggregatableFieldsChunk) => + data.search + .search( + { + params: checkAggregatableFieldsExistRequest( + index, + searchQuery, + aggregatableFieldsChunk, + samplerShardSize, + timeFieldName, + earliest, + latest, + undefined, + runtimeFieldMap + ), + }, + searchOptions + ) + .pipe( + switchMap((resp) => { + return of({ + ...resp, + aggregatableFields: aggregatableFieldsChunk, + } as AggregatableFieldOverallStats); + }) + ) ) - : of(undefined); + : of(undefined) + ); const documentCountStats$ = timeFieldName !== undefined && intervalMs !== undefined && intervalMs > 0 @@ -183,7 +202,7 @@ export function useOverallStats { const aggregatableOverallStats = processAggregatableFieldsExistResponse( - aggregatableOverallStatsResp?.rawResponse, + aggregatableOverallStatsResp, aggregatableFields, samplerShardSize ); @@ -224,7 +243,7 @@ export function useOverallStats { searchSubscription$.current?.unsubscribe(); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts index bc856f9bf69d2..b5359915ef63e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts @@ -20,11 +20,11 @@ import type { FieldStatsCommonRequestParams, } from '../../../../../common/types/field_stats'; import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; -import { - DataPublicPluginStart, +import type { IKibanaSearchRequest, IKibanaSearchResponse, ISearchOptions, + ISearchStart, } from '../../../../../../../../src/plugins/data/public'; import { extractErrorProperties } from '../../utils/error_utils'; @@ -62,14 +62,14 @@ export const getBooleanFieldsStatsRequest = ( }; export const fetchBooleanFieldsStats = ( - data: DataPublicPluginStart, + dataSearch: ISearchStart, params: FieldStatsCommonRequestParams, fields: Field[], options: ISearchOptions ): Observable => { const { samplerShardSize } = params; const request: estypes.SearchRequest = getBooleanFieldsStatsRequest(params, fields); - return data.search + return dataSearch .search({ params: request }, options) .pipe( catchError((e) => diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts index d87c002e907c9..07bdc8c14301c 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts @@ -16,11 +16,11 @@ import { import { isPopulatedObject } from '../../../../../common/utils/object_utils'; import type { FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats'; import type { Field, DateFieldStats, Aggs } from '../../../../../common/types/field_stats'; -import { - DataPublicPluginStart, +import type { IKibanaSearchRequest, IKibanaSearchResponse, ISearchOptions, + ISearchStart, } from '../../../../../../../../src/plugins/data/public'; import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; import { extractErrorProperties } from '../../utils/error_utils'; @@ -59,7 +59,7 @@ export const getDateFieldsStatsRequest = ( }; export const fetchDateFieldsStats = ( - data: DataPublicPluginStart, + dataSearch: ISearchStart, params: FieldStatsCommonRequestParams, fields: Field[], options: ISearchOptions @@ -67,7 +67,7 @@ export const fetchDateFieldsStats = ( const { samplerShardSize } = params; const request: estypes.SearchRequest = getDateFieldsStatsRequest(params, fields); - return data.search + return dataSearch .search({ params: request }, options) .pipe( catchError((e) => diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts index d85bab5a1a30a..618e47bb97e1d 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts @@ -15,11 +15,11 @@ import type { FieldExamples, FieldStatsCommonRequestParams, } from '../../../../../common/types/field_stats'; -import { - DataPublicPluginStart, +import type { IKibanaSearchRequest, IKibanaSearchResponse, ISearchOptions, + ISearchStart, } from '../../../../../../../../src/plugins/data/public'; import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; import { extractErrorProperties } from '../../utils/error_utils'; @@ -60,7 +60,7 @@ export const getFieldExamplesRequest = (params: FieldStatsCommonRequestParams, f }; export const fetchFieldsExamples = ( - data: DataPublicPluginStart, + dataSearch: ISearchStart, params: FieldStatsCommonRequestParams, fields: Field[], options: ISearchOptions @@ -70,7 +70,7 @@ export const fetchFieldsExamples = ( fields.map((field) => { const request: estypes.SearchRequest = getFieldExamplesRequest(params, field); - return data.search + return dataSearch .search({ params: request }, options) .pipe( catchError((e) => diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts index c8b4ae67f4c8c..aa19aa9fbb495 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts @@ -5,12 +5,12 @@ * 2.0. */ -import { Observable } from 'rxjs'; +import type { Observable } from 'rxjs'; import type { FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats'; import type { FieldStatsError } from '../../../../../common/types/field_stats'; -import { ISearchOptions } from '../../../../../../../../src/plugins/data/common'; -import { DataPublicPluginStart } from '../../../../../../../../src/plugins/data/public'; -import { FieldStats } from '../../../../../common/types/field_stats'; +import type { ISearchOptions } from '../../../../../../../../src/plugins/data/common'; +import { ISearchStart } from '../../../../../../../../src/plugins/data/public'; +import type { FieldStats } from '../../../../../common/types/field_stats'; import { JOB_FIELD_TYPES } from '../../../../../common'; import { fetchDateFieldsStats } from './get_date_field_stats'; import { fetchBooleanFieldsStats } from './get_boolean_field_stats'; @@ -19,7 +19,7 @@ import { fetchNumericFieldsStats } from './get_numeric_field_stats'; import { fetchStringFieldsStats } from './get_string_field_stats'; export const getFieldsStats = ( - dataPlugin: DataPublicPluginStart, + dataSearch: ISearchStart, params: FieldStatsCommonRequestParams, fields: Array<{ fieldName: string; @@ -32,19 +32,19 @@ export const getFieldsStats = ( const fieldType = fields[0].type; switch (fieldType) { case JOB_FIELD_TYPES.NUMBER: - return fetchNumericFieldsStats(dataPlugin, params, fields, options); + return fetchNumericFieldsStats(dataSearch, params, fields, options); case JOB_FIELD_TYPES.KEYWORD: case JOB_FIELD_TYPES.IP: - return fetchStringFieldsStats(dataPlugin, params, fields, options); + return fetchStringFieldsStats(dataSearch, params, fields, options); case JOB_FIELD_TYPES.DATE: - return fetchDateFieldsStats(dataPlugin, params, fields, options); + return fetchDateFieldsStats(dataSearch, params, fields, options); case JOB_FIELD_TYPES.BOOLEAN: - return fetchBooleanFieldsStats(dataPlugin, params, fields, options); + return fetchBooleanFieldsStats(dataSearch, params, fields, options); case JOB_FIELD_TYPES.TEXT: - return fetchFieldsExamples(dataPlugin, params, fields, options); + return fetchFieldsExamples(dataSearch, params, fields, options); default: // Use an exists filter on the the field name to get // examples of the field, so cannot batch up. - return fetchFieldsExamples(dataPlugin, params, fields, options); + return fetchFieldsExamples(dataSearch, params, fields, options); } }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts index d958d50dac69e..89ae7598b30fd 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_numeric_field_stats.ts @@ -34,7 +34,7 @@ import { IKibanaSearchResponse, ISearchOptions, } from '../../../../../../../../src/plugins/data/common'; -import { DataPublicPluginStart } from '../../../../../../../../src/plugins/data/public'; +import type { ISearchStart } from '../../../../../../../../src/plugins/data/public'; import { extractErrorProperties } from '../../utils/error_utils'; import { isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; @@ -116,7 +116,7 @@ export const getNumericFieldsStatsRequest = ( }; export const fetchNumericFieldsStats = ( - data: DataPublicPluginStart, + dataSearch: ISearchStart, params: FieldStatsCommonRequestParams, fields: Field[], options: ISearchOptions @@ -124,7 +124,7 @@ export const fetchNumericFieldsStats = ( const { samplerShardSize } = params; const request: estypes.SearchRequest = getNumericFieldsStatsRequest(params, fields); - return data.search + return dataSearch .search({ params: request }, options) .pipe( catchError((e) => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts index d8803a4347a4d..024464c1947c8 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_string_field_stats.ts @@ -22,11 +22,11 @@ import type { FieldStatsCommonRequestParams, StringFieldStats, } from '../../../../../common/types/field_stats'; -import { - DataPublicPluginStart, +import type { IKibanaSearchRequest, IKibanaSearchResponse, ISearchOptions, + ISearchStart, } from '../../../../../../../../src/plugins/data/public'; import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats'; import { extractErrorProperties } from '../../utils/error_utils'; @@ -82,7 +82,7 @@ export const getStringFieldStatsRequest = ( }; export const fetchStringFieldsStats = ( - data: DataPublicPluginStart, + dataSearch: ISearchStart, params: FieldStatsCommonRequestParams, fields: Field[], options: ISearchOptions @@ -90,7 +90,7 @@ export const fetchStringFieldsStats = ( const { samplerShardSize } = params; const request: estypes.SearchRequest = getStringFieldStatsRequest(params, fields); - return data.search + return dataSearch .search({ params: request }, options) .pipe( catchError((e) => diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts index dd3b84bb54e6d..fb392cc17b05b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/overall_stats.ts @@ -87,8 +87,11 @@ export const checkAggregatableFieldsExistRequest = ( }; }; +export interface AggregatableFieldOverallStats extends IKibanaSearchResponse { + aggregatableFields: string[]; +} export const processAggregatableFieldsExistResponse = ( - body: estypes.SearchResponse | undefined, + responses: AggregatableFieldOverallStats[] | undefined, aggregatableFields: string[], samplerShardSize: number, datafeedConfig?: estypes.MlDatafeed @@ -99,38 +102,20 @@ export const processAggregatableFieldsExistResponse = ( aggregatableNotExistsFields: [] as AggregatableField[], }; - if (!body || aggregatableFields.length === 0) return stats; + if (!responses || aggregatableFields.length === 0) return stats; - const aggregations = body.aggregations; - const totalCount = (body.hits.total as estypes.SearchTotalHits).value ?? body.hits.total; - stats.totalCount = totalCount as number; + responses.forEach(({ rawResponse: body, aggregatableFields: aggregatableFieldsChunk }) => { + const aggregations = body.aggregations; + const totalCount = (body.hits.total as estypes.SearchTotalHits).value ?? body.hits.total; + stats.totalCount = totalCount as number; - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const sampleCount = - samplerShardSize > 0 ? get(aggregations, ['sample', 'doc_count'], 0) : totalCount; - aggregatableFields.forEach((field, i) => { - const safeFieldName = getSafeAggregationName(field, i); - const count = get(aggregations, [...aggsPath, `${safeFieldName}_count`, 'doc_count'], 0); - if (count > 0) { - const cardinality = get( - aggregations, - [...aggsPath, `${safeFieldName}_cardinality`, 'value'], - 0 - ); - stats.aggregatableExistsFields.push({ - fieldName: field, - existsInDocs: true, - stats: { - sampleCount, - count, - cardinality, - }, - }); - } else { - if ( - datafeedConfig?.script_fields?.hasOwnProperty(field) || - datafeedConfig?.runtime_mappings?.hasOwnProperty(field) - ) { + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const sampleCount = + samplerShardSize > 0 ? get(aggregations, ['sample', 'doc_count'], 0) : totalCount; + aggregatableFieldsChunk.forEach((field, i) => { + const safeFieldName = getSafeAggregationName(field, i); + const count = get(aggregations, [...aggsPath, `${safeFieldName}_count`, 'doc_count'], 0); + if (count > 0) { const cardinality = get( aggregations, [...aggsPath, `${safeFieldName}_cardinality`, 'value'], @@ -146,13 +131,33 @@ export const processAggregatableFieldsExistResponse = ( }, }); } else { - stats.aggregatableNotExistsFields.push({ - fieldName: field, - existsInDocs: false, - stats: {}, - }); + if ( + datafeedConfig?.script_fields?.hasOwnProperty(field) || + datafeedConfig?.runtime_mappings?.hasOwnProperty(field) + ) { + const cardinality = get( + aggregations, + [...aggsPath, `${safeFieldName}_cardinality`, 'value'], + 0 + ); + stats.aggregatableExistsFields.push({ + fieldName: field, + existsInDocs: true, + stats: { + sampleCount, + count, + cardinality, + }, + }); + } else { + stats.aggregatableNotExistsFields.push({ + fieldName: field, + existsInDocs: false, + stats: {}, + }); + } } - } + }); }); return stats as { From 9ea2537173b8a56f78bb44a0adec9bac089b5921 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 8 Nov 2021 11:39:31 -0600 Subject: [PATCH 188/188] Update field icon to using kbn/react-field package --- src/plugins/discover/public/url_generator.ts | 2 +- .../common/components/field_type_icon/field_type_icon.tsx | 2 +- .../index_data_visualizer/utils/saved_search_utils.test.ts | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plugins/discover/public/url_generator.ts b/src/plugins/discover/public/url_generator.ts index 53f6064231ab3..32e89691574df 100644 --- a/src/plugins/discover/public/url_generator.ts +++ b/src/plugins/discover/public/url_generator.ts @@ -10,7 +10,7 @@ import type { UrlGeneratorsDefinition } from '../../share/public'; import type { TimeRange, Filter, Query, QueryState, RefreshInterval } from '../../data/public'; import { esFilters } from '../../data/public'; import { setStateToKbnUrl } from '../../kibana_utils/public'; -import { VIEW_MODE } from './application/apps/main/components/view_mode_toggle'; +import { VIEW_MODE } from './components/view_mode_toggle'; export const DISCOVER_APP_URL_GENERATOR = 'DISCOVER_APP_URL_GENERATOR'; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx index 4e84e538c0ab8..2a9767ccd62b1 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/field_type_icon/field_type_icon.tsx @@ -8,10 +8,10 @@ import React, { FC } from 'react'; import { EuiToolTip } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { FieldIcon } from '@kbn/react-field/field_icon'; import { getJobTypeLabel } from '../../util/field_types_utils'; import type { JobFieldType } from '../../../../../common'; import './_index.scss'; -import { FieldIcon } from '../../../../../../../../src/plugins/kibana_react/public'; interface FieldTypeIconProps { tooltipEnabled: boolean; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts index 796b9c55abdd5..586f636a088e1 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts @@ -75,6 +75,7 @@ const kqlSavedSearch: SavedSearch = { title: 'farequote_filter_and_kuery', description: '', columns: ['_source'], + // @ts-expect-error kibanaSavedObjectMeta: { searchSourceJSON: '{"highlightAll":true,"version":true,"query":{"query":"responsetime > 49","language":"kuery"},"filter":[{"meta":{"index":"90a978e0-1c80-11ec-b1d7-f7e5cf21b9e0","negate":false,"disabled":false,"alias":null,"type":"phrase","key":"airline","value":"ASA","params":{"query":"ASA","type":"phrase"}},"query":{"match":{"airline":{"query":"ASA","type":"phrase"}}},"$state":{"store":"appState"}}],"indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index"}',