From 6a079d3eb61e9e229457eba7d26213b50ee7a7c6 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Wed, 24 Jul 2024 13:56:33 -0700 Subject: [PATCH] [Discover-next] data set picker (#7426) * update using query manager Signed-off-by: Kawika Avilla * aggs not working Signed-off-by: Kawika Avilla * almost there Signed-off-by: Kawika Avilla * stablish Signed-off-by: Kawika Avilla * thanks ashwin Signed-off-by: Kawika Avilla * update ref name Signed-off-by: Kawika Avilla * fix timefields Signed-off-by: Kawika Avilla * clean up some console logs Signed-off-by: Kawika Avilla * go safer route of setting language Signed-off-by: Kawika Avilla * its working again Signed-off-by: Kawika Avilla * fix names Signed-off-by: Kawika Avilla * restore code editor and indices Signed-off-by: Kawika Avilla * sql df Signed-off-by: Kawika Avilla * fix external datasources again Signed-off-by: Kawika Avilla --------- Signed-off-by: Kawika Avilla --- changelogs/fragments/7368.yml | 2 + package.json | 2 +- src/plugins/data/common/data_frames/utils.ts | 23 - .../utils => data/common/data_sets}/index.ts | 2 +- src/plugins/data/common/data_sets/types.ts | 38 + src/plugins/data/common/index.ts | 1 + .../index_patterns/index_patterns.ts | 6 +- .../common/search/opensearch_search/types.ts | 4 + .../search/search_source/search_source.ts | 11 +- src/plugins/data/common/types.ts | 1 + src/plugins/data/public/antlr/shared/utils.ts | 24 +- src/plugins/data/public/index.ts | 1 + .../dataset_manager/dataset_manager.mock.ts | 22 + .../dataset_manager/dataset_manager.test.ts | 33 + .../query/dataset_manager/dataset_manager.ts | 77 ++ .../public/query/dataset_manager/index.ts | 6 + src/plugins/data/public/query/index.tsx | 1 + .../data/public/query/query_service.ts | 17 +- .../state_sync/connect_to_query_state.ts | 65 +- .../create_global_query_observable.ts | 8 + .../query/state_sync/sync_state_with_url.ts | 12 +- .../data/public/query/state_sync/types.ts | 3 +- .../data/public/search/search_service.ts | 23 +- src/plugins/data/public/ui/_index.scss | 1 + .../dataset_navigator/_dataset_navigator.scss | 16 + .../public/ui/dataset_navigator/_index.scss | 1 + .../create_dataset_navigator.tsx | 26 + .../dataset_navigator/dataset_navigator.tsx | 736 ++++++++++++++++++ .../public/ui/dataset_navigator/index.tsx | 7 + .../lib/catalog_cache/cache_intercept.ts | 23 + .../lib/catalog_cache/cache_loader.tsx | 470 +++++++++++ .../lib/catalog_cache/cache_manager.ts | 416 ++++++++++ .../lib/catalog_cache/index.tsx | 8 + .../ui/dataset_navigator/lib/constants.ts | 101 +++ .../lib/hooks/direct_query_hook.tsx | 99 +++ .../ui/dataset_navigator/lib/hooks/index.tsx} | 2 +- .../public/ui/dataset_navigator/lib/index.tsx | 9 + .../dataset_navigator/lib/requests/index.tsx} | 2 +- .../ui/dataset_navigator/lib/requests/sql.ts | 60 ++ .../public/ui/dataset_navigator/lib/types.tsx | 330 ++++++++ .../lib/utils/fetch_catalog_cache_status.ts | 26 + .../lib/utils/fetch_data_sources.ts | 19 + .../lib/utils/fetch_external_data_sources.ts | 33 + .../lib/utils/fetch_index_patterns.ts | 34 + .../lib/utils/fetch_indices.ts | 46 ++ .../ui/dataset_navigator/lib/utils/index.ts | 13 + .../lib/utils/query_session_utils.ts | 16 + .../ui/dataset_navigator/lib/utils/shared.ts | 332 ++++++++ .../lib/utils/use_polling.ts | 137 ++++ src/plugins/data/public/ui/index.ts | 1 + .../ui/query_editor/_language_switcher.scss | 8 - .../public/ui/query_editor/_query_editor.scss | 6 + .../query_editor/language_selector.test.tsx | 1 - .../ui/query_editor/language_switcher.tsx | 102 --- .../public/ui/query_editor/query_editor.tsx | 86 +- .../ui/query_editor/query_editor_top_row.tsx | 13 +- .../ui/search_bar/create_search_bar.tsx | 19 +- .../ui/search_bar/lib/use_dataset_manager.ts | 39 + .../data/public/ui/search_bar/search_bar.tsx | 6 +- .../data/public/ui/settings/settings.ts | 4 +- src/plugins/data/public/ui/types.ts | 11 +- src/plugins/data/public/ui/ui_service.ts | 23 +- .../data/server/search/search_service.ts | 7 +- .../public/components/sidebar/index.tsx | 36 +- src/plugins/data_explorer/public/index.ts | 1 + .../utils/state_management/metadata_slice.ts | 7 +- .../public/utils/state_management/store.ts | 2 +- .../utils/state_management/index.ts | 3 +- .../utils/update_search_source.ts | 7 +- .../view_components/utils/use_search.ts | 4 +- .../opensearch_dashboards.json | 2 +- .../components/connections_bar.tsx | 94 --- .../public/data_source_connection/index.ts | 7 - .../utils/create_extension.tsx | 34 - .../query_enhancements/public/plugin.tsx | 87 +-- .../components/query_assist_bar.tsx | 4 +- .../query_assist/utils/create_extension.tsx | 4 +- .../query_enhancements/public/search/index.ts | 1 - .../public/search/ppl_search_interceptor.ts | 40 +- .../search/sql_async_search_interceptor.ts | 137 ---- .../public/search/sql_search_interceptor.ts | 115 ++- .../query_enhancements/public/services.ts | 11 - .../services/connections_service.ts | 4 +- .../public/services/index.ts | 13 + .../routes/data_source_connection/routes.ts | 5 +- .../search/sql_async_search_strategy.ts | 2 +- .../server/search/sql_search_strategy.ts | 2 +- .../query_enhancements/server/types.ts | 2 +- 88 files changed, 3583 insertions(+), 712 deletions(-) create mode 100644 changelogs/fragments/7368.yml rename src/plugins/{query_enhancements/public/data_source_connection/utils => data/common/data_sets}/index.ts (70%) create mode 100644 src/plugins/data/common/data_sets/types.ts create mode 100644 src/plugins/data/public/query/dataset_manager/dataset_manager.mock.ts create mode 100644 src/plugins/data/public/query/dataset_manager/dataset_manager.test.ts create mode 100644 src/plugins/data/public/query/dataset_manager/dataset_manager.ts create mode 100644 src/plugins/data/public/query/dataset_manager/index.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss create mode 100644 src/plugins/data/public/ui/dataset_navigator/_index.scss create mode 100644 src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/index.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/constants.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/hooks/direct_query_hook.tsx rename src/plugins/{query_enhancements/public/data_source_connection/components/index.ts => data/public/ui/dataset_navigator/lib/hooks/index.tsx} (61%) create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/index.tsx rename src/plugins/{query_enhancements/public/data_source_connection/services/index.ts => data/public/ui/dataset_navigator/lib/requests/index.tsx} (58%) create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/types.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts delete mode 100644 src/plugins/data/public/ui/query_editor/_language_switcher.scss delete mode 100644 src/plugins/data/public/ui/query_editor/language_switcher.tsx create mode 100644 src/plugins/data/public/ui/search_bar/lib/use_dataset_manager.ts delete mode 100644 src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx delete mode 100644 src/plugins/query_enhancements/public/data_source_connection/index.ts delete mode 100644 src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx delete mode 100644 src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts delete mode 100644 src/plugins/query_enhancements/public/services.ts rename src/plugins/query_enhancements/public/{data_source_connection => }/services/connections_service.ts (95%) create mode 100644 src/plugins/query_enhancements/public/services/index.ts diff --git a/changelogs/fragments/7368.yml b/changelogs/fragments/7368.yml new file mode 100644 index 000000000000..c8316dc939f0 --- /dev/null +++ b/changelogs/fragments/7368.yml @@ -0,0 +1,2 @@ +feat: +- [Discover] Adds a dataset selector for Discover ([#7368](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7368)) \ No newline at end of file diff --git a/package.json b/package.json index 43cdf376b155..b84348cdd6f7 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "start": "scripts/use_node scripts/opensearch_dashboards --dev", "start:docker": "scripts/use_node scripts/opensearch_dashboards --dev --opensearch.hosts=$OPENSEARCH_HOSTS --opensearch.ignoreVersionMismatch=true --server.host=$SERVER_HOST", "start:security": "scripts/use_node scripts/opensearch_dashboards --dev --security", - "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true", + "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true", "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev", "debug-break": "scripts/use_node --nolazy --inspect-brk scripts/opensearch_dashboards --dev", "lint": "yarn run lint:es && yarn run lint:style", diff --git a/src/plugins/data/common/data_frames/utils.ts b/src/plugins/data/common/data_frames/utils.ts index 31df2626a98a..0da02cda295f 100644 --- a/src/plugins/data/common/data_frames/utils.ts +++ b/src/plugins/data/common/data_frames/utils.ts @@ -45,29 +45,6 @@ export const getRawQueryString = ( ); }; -/** - * Parses a raw query string and extracts the query string and data source. - * @param rawQueryString - The raw query string to parse. - * @returns An object containing the parsed query string and data source (if found). - */ -export const parseRawQueryString = (rawQueryString: string) => { - const rawDataSource = rawQueryString.match(/::(.*?)::/); - return { - qs: rawQueryString.replace(/::.*?::/, ''), - formattedQs(key: string = '.'): string { - const parts = rawQueryString.split('::'); - if (parts.length > 1) { - return (parts.slice(0, 1).join('') + parts.slice(1).join(key)).replace( - new RegExp(key + '$'), - '' - ); - } - return rawQueryString; - }, - ...(rawDataSource && { dataSource: rawDataSource[1] }), - }; -}; - /** * Returns the raw aggregations from the search request. * diff --git a/src/plugins/query_enhancements/public/data_source_connection/utils/index.ts b/src/plugins/data/common/data_sets/index.ts similarity index 70% rename from src/plugins/query_enhancements/public/data_source_connection/utils/index.ts rename to src/plugins/data/common/data_sets/index.ts index 9eccc9e6f35a..9f269633f307 100644 --- a/src/plugins/query_enhancements/public/data_source_connection/utils/index.ts +++ b/src/plugins/data/common/data_sets/index.ts @@ -3,4 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export * from './create_extension'; +export * from './types'; diff --git a/src/plugins/data/common/data_sets/types.ts b/src/plugins/data/common/data_sets/types.ts new file mode 100644 index 000000000000..23ab74bed030 --- /dev/null +++ b/src/plugins/data/common/data_sets/types.ts @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** @public **/ +export enum SIMPLE_DATA_SOURCE_TYPES { + DEFAULT = 'data-source', + EXTERNAL = 'external-source', +} + +/** @public **/ +export enum SIMPLE_DATA_SET_TYPES { + INDEX_PATTERN = 'index-pattern', + TEMPORARY = 'temporary', + TEMPORARY_ASYNC = 'temporary-async', +} + +export interface SimpleObject { + id: string; + title?: string; + dataSourceRef?: SimpleDataSource; +} + +export interface SimpleDataSource { + id: string; + name: string; + indices?: SimpleObject[]; + tables?: SimpleObject[]; + type: SIMPLE_DATA_SOURCE_TYPES; +} + +export interface SimpleDataSet extends SimpleObject { + fields?: any[]; + timeFieldName?: string; + timeFields?: any[]; + type?: SIMPLE_DATA_SET_TYPES; +} diff --git a/src/plugins/data/common/index.ts b/src/plugins/data/common/index.ts index d7b7e56e2280..0250a6ec2e01 100644 --- a/src/plugins/data/common/index.ts +++ b/src/plugins/data/common/index.ts @@ -31,6 +31,7 @@ export * from './constants'; export * from './opensearch_query'; export * from './data_frames'; +export * from './data_sets'; export * from './field_formats'; export * from './field_mapping'; export * from './index_patterns'; diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts index 3d7bd8fbb4a2..3d0dbe15dab7 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts @@ -433,11 +433,13 @@ export class IndexPatternsService { /** * Get an index pattern by id. Cache optimized * @param id + * @param onlyCheckCache - Only check cache for index pattern if it doesn't exist it will not error out */ - get = async (id: string): Promise => { + get = async (id: string, onlyCheckCache: boolean = false): Promise => { const cache = indexPatternCache.get(id); - if (cache) { + + if (cache || onlyCheckCache) { return cache; } diff --git a/src/plugins/data/common/search/opensearch_search/types.ts b/src/plugins/data/common/search/opensearch_search/types.ts index f90a3f1de245..6d24e8c36dd3 100644 --- a/src/plugins/data/common/search/opensearch_search/types.ts +++ b/src/plugins/data/common/search/opensearch_search/types.ts @@ -48,6 +48,10 @@ export interface ISearchOptions { * Use this option to enable support for long numerals. */ withLongNumeralsSupport?: boolean; + /** + * Use this option to enable support for async. + */ + isAsync?: boolean; } export type ISearchRequestParams> = { diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index d9518e6a6cab..da00e73f54f0 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -94,7 +94,6 @@ import { convertResult, createDataFrame, getRawQueryString, - parseRawQueryString, } from '../../data_frames'; import { IOpenSearchSearchRequest, IOpenSearchSearchResponse, ISearchOptions } from '../..'; import { IOpenSearchDashboardsSearchRequest, IOpenSearchDashboardsSearchResponse } from '../types'; @@ -324,7 +323,12 @@ export class SearchSource { const dataFrame = createDataFrame({ name: searchRequest.index.title || searchRequest.index, fields: [], - ...(rawQueryString && { meta: { queryConfig: parseRawQueryString(rawQueryString) } }), + ...(rawQueryString && { + meta: { + queryConfig: { qs: rawQueryString }, + ...(searchRequest.dataSourceId && { dataSource: searchRequest.dataSourceId }), + }, + }), }); await this.setDataFrame(dataFrame); return this.getDataFrame(); @@ -426,7 +430,8 @@ export class SearchSource { private async fetchExternalSearch(searchRequest: SearchRequest, options: ISearchOptions) { const { search, getConfig, onResponse } = this.dependencies; - if (!this.getDataFrame()) { + const currentDataframe = this.getDataFrame(); + if (!currentDataframe || currentDataframe.name !== searchRequest.index?.id) { await this.createDataFrame(searchRequest); } diff --git a/src/plugins/data/common/types.ts b/src/plugins/data/common/types.ts index 6a1f6e5a99d3..1670fbf72d5d 100644 --- a/src/plugins/data/common/types.ts +++ b/src/plugins/data/common/types.ts @@ -35,6 +35,7 @@ export * from './query/types'; export * from './osd_field_types/types'; export * from './index_patterns/types'; export * from './data_frames/types'; +export * from './data_sets/types'; /** * If a service is being shared on both the client and the server, and diff --git a/src/plugins/data/public/antlr/shared/utils.ts b/src/plugins/data/public/antlr/shared/utils.ts index b2658b304e0f..8be6e6524fc5 100644 --- a/src/plugins/data/public/antlr/shared/utils.ts +++ b/src/plugins/data/public/antlr/shared/utils.ts @@ -12,7 +12,7 @@ export interface IDataSourceRequestHandlerParams { } export const getRawSuggestionData$ = ( - connectionsService, + connectionsService: any, dataSourceReuqstHandler: ({ dataSourceId, title, @@ -21,11 +21,11 @@ export const getRawSuggestionData$ = ( ) => connectionsService.getSelectedConnection$().pipe( distinctUntilChanged(), - switchMap((connection) => { + switchMap((connection: any) => { if (connection === undefined) { return from(defaultReuqstHandler()); } - const dataSourceId = connection?.id; + const dataSourceId = connection?.dataSource?.id; const title = connection?.attributes?.title; return from(dataSourceReuqstHandler({ dataSourceId, title })); }) @@ -34,8 +34,8 @@ export const getRawSuggestionData$ = ( export const fetchData = ( tables: string[], queryFormatter: (table: string, dataSourceId?: string, title?: string) => any, - api, - connectionService + api: any, + connectionService: any ): Promise => { return new Promise((resolve, reject) => { getRawSuggestionData$( @@ -65,8 +65,8 @@ export const fetchData = ( ); } ).subscribe({ - next: (dataFrames) => resolve(dataFrames), - error: (err) => { + next: (dataFrames: any) => resolve(dataFrames), + error: (err: any) => { // TODO: pipe error to UI reject(err); }, @@ -74,7 +74,11 @@ export const fetchData = ( }); }; -export const fetchTableSchemas = (tables: string[], api, connectionService): Promise => { +export const fetchTableSchemas = ( + tables: string[], + api: any, + connectionService: any +): Promise => { return fetchData( tables, (table, dataSourceId, title) => ({ @@ -96,8 +100,8 @@ export const fetchTableSchemas = (tables: string[], api, connectionService): Pro export const fetchColumnValues = ( tables: string[], column: string, - api, - connectionService + api: any, + connectionService: any ): Promise => { return fetchData( tables, diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index f1ac419e9ec1..208359352e4b 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -445,6 +445,7 @@ export { QueryEditorTopRow, // for BWC, keeping the old name IUiStart as DataPublicPluginStartUi, + DataSetNavigator, } from './ui'; /** diff --git a/src/plugins/data/public/query/dataset_manager/dataset_manager.mock.ts b/src/plugins/data/public/query/dataset_manager/dataset_manager.mock.ts new file mode 100644 index 000000000000..2f1f5144274c --- /dev/null +++ b/src/plugins/data/public/query/dataset_manager/dataset_manager.mock.ts @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DataSetContract } from '.'; + +const createSetupContractMock = () => { + const dataSetManagerMock: jest.Mocked = { + init: jest.fn(), + getDataSet: jest.fn(), + setDataSet: jest.fn(), + getUpdates$: jest.fn(), + getDefaultDataSet: jest.fn(), + }; + return dataSetManagerMock; +}; + +export const dataSetManagerMock = { + createSetupContract: createSetupContractMock, + createStartContract: createSetupContractMock, +}; diff --git a/src/plugins/data/public/query/dataset_manager/dataset_manager.test.ts b/src/plugins/data/public/query/dataset_manager/dataset_manager.test.ts new file mode 100644 index 000000000000..fcf91e6b8f89 --- /dev/null +++ b/src/plugins/data/public/query/dataset_manager/dataset_manager.test.ts @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DataSetManager } from './dataset_manager'; +import { coreMock } from '../../../../../core/public/mocks'; +import { SimpleDataSet } from '../../../common/data_sets'; + +describe('DataSetManager', () => { + let service: DataSetManager; + + beforeEach(() => { + service = new DataSetManager(coreMock.createSetup().uiSettings); + }); + + test('getUpdates$ is a cold emits only after query changes', () => { + const obs$ = service.getUpdates$(); + const emittedValues: SimpleDataSet[] = []; + obs$.subscribe((v) => { + emittedValues.push(v!); + }); + expect(emittedValues).toHaveLength(0); + + const newDataSet: SimpleDataSet = { id: 'test_dataset', title: 'Test Dataset' }; + service.setDataSet(newDataSet); + expect(emittedValues).toHaveLength(1); + expect(emittedValues[0]).toEqual(newDataSet); + + service.setDataSet({ ...newDataSet }); + expect(emittedValues).toHaveLength(2); + }); +}); diff --git a/src/plugins/data/public/query/dataset_manager/dataset_manager.ts b/src/plugins/data/public/query/dataset_manager/dataset_manager.ts new file mode 100644 index 000000000000..018eba50ad73 --- /dev/null +++ b/src/plugins/data/public/query/dataset_manager/dataset_manager.ts @@ -0,0 +1,77 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { BehaviorSubject } from 'rxjs'; +import { skip } from 'rxjs/operators'; +import { CoreStart } from 'opensearch-dashboards/public'; +import { + IndexPatternsService, + SIMPLE_DATA_SET_TYPES, + SimpleDataSet, + SimpleDataSource, +} from '../../../common'; + +export class DataSetManager { + private dataSet$: BehaviorSubject; + private indexPatterns?: IndexPatternsService; + + constructor(private readonly uiSettings: CoreStart['uiSettings']) { + this.dataSet$ = new BehaviorSubject(undefined); + } + + public init = (indexPatterns: IndexPatternsService) => { + this.indexPatterns = indexPatterns; + }; + + public getUpdates$ = () => { + return this.dataSet$.asObservable().pipe(skip(1)); + }; + + public getDataSet = () => { + return this.dataSet$.getValue(); + }; + + /** + * Updates the query. + * @param {Query} query + */ + public setDataSet = (dataSet: SimpleDataSet | undefined) => { + this.dataSet$.next(dataSet); + }; + + public getDefaultDataSet = async (): Promise => { + const defaultIndexPatternId = await this.uiSettings.get('defaultIndex'); + if (!defaultIndexPatternId) { + return undefined; + } + + const indexPattern = await this.indexPatterns?.get(defaultIndexPatternId); + if (!indexPattern) { + return undefined; + } + + if (!indexPattern.id) { + return undefined; + } + + return { + id: indexPattern.id, + title: indexPattern.title, + type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, + timeFieldName: indexPattern.timeFieldName, + ...(indexPattern.dataSourceRef + ? { + dataSourceRef: { + id: indexPattern.dataSourceRef?.id, + name: indexPattern.dataSourceRef?.name, + type: indexPattern.dataSourceRef?.type, + } as SimpleDataSource, + } + : {}), + }; + }; +} + +export type DataSetContract = PublicMethodsOf; diff --git a/src/plugins/data/public/query/dataset_manager/index.ts b/src/plugins/data/public/query/dataset_manager/index.ts new file mode 100644 index 000000000000..8a9a39b81127 --- /dev/null +++ b/src/plugins/data/public/query/dataset_manager/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DataSetContract, DataSetManager } from './dataset_manager'; diff --git a/src/plugins/data/public/query/index.tsx b/src/plugins/data/public/query/index.tsx index 505f095aeda7..42c6349bcc89 100644 --- a/src/plugins/data/public/query/index.tsx +++ b/src/plugins/data/public/query/index.tsx @@ -32,6 +32,7 @@ export * from './lib'; export * from './query_service'; export * from './filter_manager'; +export * from './dataset_manager'; export * from './timefilter'; export * from './saved_query'; export * from './persisted_log'; diff --git a/src/plugins/data/public/query/query_service.ts b/src/plugins/data/public/query/query_service.ts index 1b758d18bda3..7d3bdca2f9b8 100644 --- a/src/plugins/data/public/query/query_service.ts +++ b/src/plugins/data/public/query/query_service.ts @@ -37,7 +37,8 @@ import { TimefilterService, TimefilterSetup } from './timefilter'; import { createSavedQueryService } from './saved_query/saved_query_service'; import { createQueryStateObservable } from './state_sync/create_global_query_observable'; import { QueryStringManager, QueryStringContract } from './query_string'; -import { buildOpenSearchQuery, getOpenSearchQueryConfig } from '../../common'; +import { DataSetManager, DataSetContract } from './dataset_manager'; +import { buildOpenSearchQuery, getOpenSearchQueryConfig, IndexPatternsService } from '../../common'; import { getUiSettings } from '../services'; import { IndexPattern } from '..'; @@ -55,12 +56,14 @@ interface QueryServiceStartDependencies { savedObjectsClient: SavedObjectsClientContract; storage: IStorageWrapper; uiSettings: IUiSettingsClient; + indexPatterns: IndexPatternsService; } export class QueryService { filterManager!: FilterManager; timefilter!: TimefilterSetup; queryStringManager!: QueryStringContract; + dataSetManager!: DataSetContract; state$!: ReturnType; @@ -74,22 +77,31 @@ export class QueryService { }); this.queryStringManager = new QueryStringManager(storage, uiSettings); + this.dataSetManager = new DataSetManager(uiSettings); this.state$ = createQueryStateObservable({ filterManager: this.filterManager, timefilter: this.timefilter, queryString: this.queryStringManager, + dataSet: this.dataSetManager, }).pipe(share()); return { filterManager: this.filterManager, timefilter: this.timefilter, queryString: this.queryStringManager, + dataSet: this.dataSetManager, state$: this.state$, }; } - public start({ savedObjectsClient, storage, uiSettings }: QueryServiceStartDependencies) { + public start({ + savedObjectsClient, + storage, + uiSettings, + indexPatterns, + }: QueryServiceStartDependencies) { + this.dataSetManager.init(indexPatterns); return { addToQueryLog: createAddToQueryLog({ storage, @@ -97,6 +109,7 @@ export class QueryService { }), filterManager: this.filterManager, queryString: this.queryStringManager, + dataSet: this.dataSetManager, savedQueries: createSavedQueryService(savedObjectsClient), state$: this.state$, timefilter: this.timefilter, diff --git a/src/plugins/data/public/query/state_sync/connect_to_query_state.ts b/src/plugins/data/public/query/state_sync/connect_to_query_state.ts index 8b850b36eabc..5db81e597b80 100644 --- a/src/plugins/data/public/query/state_sync/connect_to_query_state.ts +++ b/src/plugins/data/public/query/state_sync/connect_to_query_state.ts @@ -48,16 +48,21 @@ import { validateTimeRange } from '../timefilter'; * @param OsdUrlStateStorage to use for syncing and store data * @param syncConfig app filter and query */ -export const connectStorageToQueryState = ( +export const connectStorageToQueryState = async ( { + dataSet, filterManager, queryString, state$, - }: Pick, + }: Pick< + QueryStart | QuerySetup, + 'timefilter' | 'filterManager' | 'queryString' | 'dataSet' | 'state$' + >, OsdUrlStateStorage: IOsdUrlStateStorage, syncConfig: { filters: FilterStateStore; query: boolean; + dataSet?: boolean; } ) => { try { @@ -68,10 +73,14 @@ export const connectStorageToQueryState = ( if (syncConfig.filters === FilterStateStore.APP_STATE) { syncKeys.push('appFilters'); } + if (syncConfig.dataSet) { + syncKeys.push('dataSet'); + } const initialStateFromURL: QueryState = OsdUrlStateStorage.get('_q') ?? { query: queryString.getDefaultQuery(), filters: filterManager.getAppFilters(), + dataSet: await dataSet.getDefaultDataSet(), }; // set up initial '_q' flag in the URL to sync query and filter changes @@ -87,6 +96,17 @@ export const connectStorageToQueryState = ( } } + if (syncConfig.dataSet && !_.isEqual(initialStateFromURL.dataSet, dataSet.getDataSet())) { + if (initialStateFromURL.dataSet) { + dataSet.setDataSet(_.cloneDeep(initialStateFromURL.dataSet)); + } else { + const defaultDataSet = await dataSet.getDefaultDataSet(); + if (defaultDataSet) { + dataSet.setDataSet(defaultDataSet); + } + } + } + if (syncConfig.filters === FilterStateStore.APP_STATE) { if ( !initialStateFromURL.filters || @@ -119,6 +139,10 @@ export const connectStorageToQueryState = ( newState.filters = filterManager.getAppFilters(); } + if (syncConfig.dataSet && changes.dataSet) { + newState.dataSet = dataSet.getDataSet(); + } + return newState; }) ) @@ -143,19 +167,24 @@ export const connectStorageToQueryState = ( * @param QueryService: either setup or start * @param stateContainer to use for syncing */ -export const connectToQueryState = ( +export const connectToQueryState = async ( { timefilter: { timefilter }, filterManager, queryString, + dataSet, state$, - }: Pick, + }: Pick< + QueryStart | QuerySetup, + 'timefilter' | 'filterManager' | 'dataSet' | 'queryString' | 'state$' + >, stateContainer: BaseStateContainer, syncConfig: { time?: boolean; refreshInterval?: boolean; filters?: FilterStateStore | boolean; query?: boolean; + dataSet?: boolean; } ) => { const syncKeys: Array = []; @@ -181,6 +210,9 @@ export const connectToQueryState = ( break; } } + if (syncConfig.dataSet) { + syncKeys.push('dataSet'); + } // initial syncing // TODO: @@ -235,6 +267,11 @@ export const connectToQueryState = ( } } + if (syncConfig.dataSet && !initialState.dataSet) { + initialState.dataSet = await dataSet.getDefaultDataSet(); + initialDirty = true; + } + if (initialDirty) { stateContainer.set({ ...stateContainer.get(), ...initialState }); } @@ -272,13 +309,16 @@ export const connectToQueryState = ( newState.filters = filterManager.getAppFilters(); } } + if (syncConfig.dataSet && changes.dataSet) { + newState.dataSet = dataSet.getDataSet(); + } return newState; }) ) .subscribe((newState) => { stateContainer.set({ ...stateContainer.get(), ...newState }); }), - stateContainer.state$.subscribe((state) => { + stateContainer.state$.subscribe(async (state) => { updateInProgress = true; // cloneDeep is required because services are mutating passed objects @@ -331,6 +371,21 @@ export const connectToQueryState = ( } } + if (syncConfig.dataSet) { + const currentDataSet = dataSet.getDataSet(); + if (!_.isEqual(state.dataSet, currentDataSet)) { + if (state.dataSet) { + dataSet.setDataSet(state.dataSet); + } else { + const defaultDataSet = await dataSet.getDefaultDataSet(); + if (defaultDataSet) { + dataSet.setDataSet(defaultDataSet); + stateContainer.set({ ...stateContainer.get(), dataSet: defaultDataSet }); + } + } + } + } + updateInProgress = false; }), ]; diff --git a/src/plugins/data/public/query/state_sync/create_global_query_observable.ts b/src/plugins/data/public/query/state_sync/create_global_query_observable.ts index 8abcb3ece18d..440ea836383e 100644 --- a/src/plugins/data/public/query/state_sync/create_global_query_observable.ts +++ b/src/plugins/data/public/query/state_sync/create_global_query_observable.ts @@ -36,15 +36,18 @@ import { QueryState, QueryStateChange } from './index'; import { createStateContainer } from '../../../../opensearch_dashboards_utils/public'; import { isFilterPinned, compareFilters, COMPARE_ALL_OPTIONS } from '../../../common'; import { QueryStringContract } from '../query_string'; +import { DataSetContract } from '../dataset_manager'; export function createQueryStateObservable({ timefilter: { timefilter }, filterManager, queryString, + dataSet, }: { timefilter: TimefilterSetup; filterManager: FilterManager; queryString: QueryStringContract; + dataSet: DataSetContract; }): Observable<{ changes: QueryStateChange; state: QueryState }> { return new Observable((subscriber) => { const state = createStateContainer({ @@ -52,6 +55,7 @@ export function createQueryStateObservable({ refreshInterval: timefilter.getRefreshInterval(), filters: filterManager.getFilters(), query: queryString.getQuery(), + dataSet: dataSet.getDataSet(), }); let currentChange: QueryStateChange = {}; @@ -60,6 +64,10 @@ export function createQueryStateObservable({ currentChange.query = true; state.set({ ...state.get(), query: queryString.getQuery() }); }), + dataSet.getUpdates$().subscribe(() => { + currentChange.dataSet = true; + state.set({ ...state.get(), dataSet: dataSet.getDataSet() }); + }), timefilter.getTimeUpdate$().subscribe(() => { currentChange.time = true; state.set({ ...state.get(), time: timefilter.getTime() }); diff --git a/src/plugins/data/public/query/state_sync/sync_state_with_url.ts b/src/plugins/data/public/query/state_sync/sync_state_with_url.ts index 67245fd693ab..3b92cff04d26 100644 --- a/src/plugins/data/public/query/state_sync/sync_state_with_url.ts +++ b/src/plugins/data/public/query/state_sync/sync_state_with_url.ts @@ -46,17 +46,22 @@ const GLOBAL_STATE_STORAGE_KEY = '_g'; * @param osdUrlStateStorage to use for syncing */ export const syncQueryStateWithUrl = ( - query: Pick, + query: Pick< + QueryStart | QuerySetup, + 'filterManager' | 'timefilter' | 'queryString' | 'dataSet' | 'state$' + >, osdUrlStateStorage: IOsdUrlStateStorage ) => { const { timefilter: { timefilter }, filterManager, + dataSet, } = query; const defaultState: QueryState = { time: timefilter.getTime(), refreshInterval: timefilter.getRefreshInterval(), filters: filterManager.getGlobalFilters(), + dataSet: dataSet.getDataSet(), }; // retrieve current state from `_g` url @@ -78,6 +83,7 @@ export const syncQueryStateWithUrl = ( refreshInterval: true, time: true, filters: FilterStateStore.GLOBAL_STATE, + dataSet: true, }); // if there weren't any initial state in url, @@ -107,8 +113,8 @@ export const syncQueryStateWithUrl = ( start(); return { - stop: () => { - stopSyncingWithStateContainer(); + stop: async () => { + (await stopSyncingWithStateContainer)(); stopSyncingWithUrl(); }, hasInheritedQueryFromUrl, diff --git a/src/plugins/data/public/query/state_sync/types.ts b/src/plugins/data/public/query/state_sync/types.ts index 0ee0ad1c463e..8134a7208f13 100644 --- a/src/plugins/data/public/query/state_sync/types.ts +++ b/src/plugins/data/public/query/state_sync/types.ts @@ -28,7 +28,7 @@ * under the License. */ -import { Filter, RefreshInterval, TimeRange, Query } from '../../../common'; +import { Filter, RefreshInterval, TimeRange, Query, SimpleDataSet } from '../../../common'; /** * All query state service state @@ -38,6 +38,7 @@ export interface QueryState { refreshInterval?: RefreshInterval; filters?: Filter[]; query?: Query; + dataSet?: SimpleDataSet; } type QueryStateChangePartial = { diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts index 712f437d2e21..d2a2dd67a6bd 100644 --- a/src/plugins/data/public/search/search_service.ts +++ b/src/plugins/data/public/search/search_service.ts @@ -64,6 +64,8 @@ import { createDataFrameCache, dataFrameToSpec, } from '../../common/data_frames'; +import { getQueryService, getUiService } from '../services'; +import { UI_SETTINGS } from '../../common'; /** @internal */ export interface SearchServiceSetupDependencies { @@ -133,7 +135,21 @@ export class SearchService implements Plugin { { fieldFormats, indexPatterns }: SearchServiceStartDependencies ): ISearchStart { const search = ((request, options) => { - return this.searchInterceptor.search(request, options); + const selectedLanguage = getQueryService().queryString.getQuery().language; + const uiService = getUiService(); + const enhancement = uiService.Settings.getQueryEnhancements(selectedLanguage); + uiService.Settings.setUiOverridesByUserQueryLanguage(selectedLanguage); + const isEnhancedEnabled = uiSettings.get(UI_SETTINGS.QUERY_ENHANCEMENTS_ENABLED); + + if (enhancement) { + if (!isEnhancedEnabled) { + notifications.toasts.addWarning( + `Query enhancements are disabled. Please enable to use: ${selectedLanguage}.` + ); + } + return enhancement.search.search(request, options); + } + return this.defaultSearchInterceptor.search(request, options); }) as ISearchGeneric; const loadingCount$ = new BehaviorSubject(0); @@ -156,7 +172,8 @@ export class SearchService implements Plugin { dataFrame.meta.queryConfig.dataSourceId = dataSource?.id; } this.dfCache.set(dataFrame); - const existingIndexPattern = indexPatterns.getByTitle(dataFrame.name!, true); + const dataSetName = `${dataFrame.meta?.queryConfig?.dataSourceId ?? ''}.${dataFrame.name}`; + const existingIndexPattern = await indexPatterns.get(dataSetName, true); const dataSet = await indexPatterns.create( dataFrameToSpec(dataFrame, existingIndexPattern?.id), !existingIndexPattern?.id @@ -166,8 +183,6 @@ export class SearchService implements Plugin { }, clear: () => { if (this.dfCache.get() === undefined) return; - // name because the id is not unique for temporary index pattern created - indexPatterns.clearCache(this.dfCache.get()!.name, false); this.dfCache.clear(); }, }; diff --git a/src/plugins/data/public/ui/_index.scss b/src/plugins/data/public/ui/_index.scss index f7c738b8d09f..4aa425041f58 100644 --- a/src/plugins/data/public/ui/_index.scss +++ b/src/plugins/data/public/ui/_index.scss @@ -2,5 +2,6 @@ @import "./typeahead/index"; @import "./saved_query_management/index"; @import "./query_string_input/index"; +@import "./dataset_navigator/index"; @import "./query_editor/index"; @import "./shard_failure_modal/shard_failure_modal"; diff --git a/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss b/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss new file mode 100644 index 000000000000..73a8c8719500 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +.datasetNavigator { + min-width: 350px; + border-bottom: $euiBorderThin !important; +} + +.dataSetNavigatorFormWrapper { + padding: $euiSizeS; +} + +.dataSetNavigator__loading { + padding: $euiSizeS; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/_index.scss b/src/plugins/data/public/ui/dataset_navigator/_index.scss new file mode 100644 index 000000000000..53acdffad43d --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/_index.scss @@ -0,0 +1 @@ +@import "./dataset_navigator"; diff --git a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx new file mode 100644 index 000000000000..c1ab4b3f846b --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { DataSetNavigator, DataSetNavigatorProps } from './'; +import { DataSetContract } from '../../query'; + +// Updated function signature to include additional dependencies +export function createDataSetNavigator( + savedObjectsClient: SavedObjectsClientContract, + http: HttpStart, + dataSet: DataSetContract +) { + // Return a function that takes props, omitting the dependencies from the props type + return (props: Omit) => ( + + ); +} diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx new file mode 100644 index 000000000000..ad4490ff134d --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -0,0 +1,736 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useCallback, useEffect, useState } from 'react'; +import { + EuiButton, + EuiButtonEmpty, + EuiContextMenu, + EuiForm, + EuiFormRow, + EuiLoadingSpinner, + EuiPanel, + EuiPopover, + EuiSelect, +} from '@elastic/eui'; +import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import _ from 'lodash'; +import { i18n } from '@osd/i18n'; +import { + SIMPLE_DATA_SET_TYPES, + SIMPLE_DATA_SOURCE_TYPES, + SimpleDataSet, + SimpleDataSource, + SimpleObject, +} from '../../../common'; +import { + useLoadDatabasesToCache, + useLoadExternalDataSourcesToCache, + useLoadTablesToCache, +} from './lib/catalog_cache/cache_loader'; +import { CatalogCacheManager } from './lib/catalog_cache/cache_manager'; +import { CachedDataSourceStatus, DirectQueryLoadingStatus } from './lib/types'; +import { + getIndexPatterns, + getNotifications, + getQueryService, + getSearchService, + getUiService, +} from '../../services'; +import { + fetchDataSources, + fetchIndexPatterns, + fetchIndices, + isCatalogCacheFetching, + fetchIfExternalDataSourcesEnabled, +} from './lib'; +import { useDataSetManager } from '../search_bar/lib/use_dataset_manager'; +import { DataSetContract } from '../../query'; + +export interface DataSetNavigatorProps { + savedObjectsClient?: SavedObjectsClientContract; + http?: HttpStart; + dataSet?: DataSetContract; +} + +interface DataSetNavigatorState { + isMounted: boolean; + isOpen: boolean; + isLoading: boolean; + isExternalDataSourcesEnabled: boolean; + indexPatterns: any[]; + dataSources: SimpleDataSource[]; + externalDataSources: SimpleDataSource[]; + cachedDatabases: any[]; + cachedTables: SimpleObject[]; +} + +interface SelectedDataSetState extends SimpleDataSet { + isExternal: boolean; + database?: any | undefined; +} + +export const DataSetNavigator = (props: DataSetNavigatorProps) => { + const { savedObjectsClient, http, dataSet: dataSetManager } = props; + const searchService = getSearchService(); + const queryService = getQueryService(); + const uiService = getUiService(); + const indexPatternsService = getIndexPatterns(); + const notifications = getNotifications(); + + const { dataSet } = useDataSetManager({ dataSetManager: dataSetManager! }); + + const [navigatorState, setNavigatorState] = useState({ + isOpen: false, + isLoading: false, + isMounted: false, + isExternalDataSourcesEnabled: false, + dataSources: [], + externalDataSources: [], + indexPatterns: [], + cachedDatabases: [], + cachedTables: [], + }); + + const [selectedDataSetState, setSelectedDataSetState] = useState({ + id: dataSet?.id ?? '', + title: dataSet?.title, + type: dataSet?.type, + isExternal: false, + dataSourceRef: dataSet?.dataSourceRef, + database: undefined, + timeFieldName: dataSet?.timeFieldName, + fields: dataSet?.fields, + }); + + const { + loadStatus: dataSourcesLoadStatus, + loadExternalDataSources: startLoadingDataSources, + } = useLoadExternalDataSourcesToCache(http!, notifications); + const { + loadStatus: databasesLoadStatus, + startLoading: startLoadingDatabases, + } = useLoadDatabasesToCache(http!, notifications); + const { loadStatus: tablesLoadStatus, startLoading: startLoadingTables } = useLoadTablesToCache( + http!, + notifications + ); + + const onClick = () => { + setNavigatorState((prevState) => ({ + ...prevState, + isOpen: !prevState.isOpen, + })); + }; + + const isLoading = (loading: boolean) => { + setNavigatorState((prevState) => ({ + ...prevState, + isLoading: loading, + })); + }; + + const closePopover = () => { + setNavigatorState((prevState) => ({ + ...prevState, + isOpen: false, + externalDataSources: [], + cachedDatabases: [], + cachedTables: [], + })); + }; + + const onRefresh = () => { + if (!isCatalogCacheFetching(dataSourcesLoadStatus) && navigatorState.dataSources.length > 0) { + startLoadingDataSources(navigatorState.dataSources.map((dataSource) => dataSource.id)); + } + }; + + useEffect(() => { + setNavigatorState((prevState) => ({ ...prevState, isMounted: true, isLoading: true })); + Promise.all([ + fetchIndexPatterns(savedObjectsClient!, ''), + fetchDataSources(savedObjectsClient!), + fetchIfExternalDataSourcesEnabled(http!), + ]) + .then(([indexPatterns, dataSources, isExternalDataSourcesEnabled]) => { + if (!navigatorState.isMounted) return; + setNavigatorState((prevState) => ({ + ...prevState, + isExternalDataSourcesEnabled, + indexPatterns, + dataSources, + })); + const selectedPattern = indexPatterns.find( + (pattern) => pattern.id === props.dataSet?.getDataSet()?.id + ); + if (selectedPattern) { + setSelectedDataSetState({ + id: selectedPattern.id, + title: selectedPattern.title, + type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, + timeFieldName: selectedPattern.timeFieldName, + fields: selectedPattern.fields, + ...(selectedPattern.dataSourceRef + ? { + dataSourceRef: { + id: selectedPattern.dataSourceRef.id, + name: selectedPattern.dataSourceRef.name, + type: selectedPattern.dataSourceRef.type, + }, + } + : { dataSource: undefined }), + database: undefined, + isExternal: false, + }); + } + }) + .finally(() => { + isLoading(false); + }); + return () => { + setNavigatorState((prevState) => ({ ...prevState, isMounted: false })); + }; + }, [savedObjectsClient, http, navigatorState.isMounted, props.dataSet]); + + useEffect(() => { + const status = dataSourcesLoadStatus.toLowerCase(); + const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); + if (status === DirectQueryLoadingStatus.SUCCESS) { + setNavigatorState((prevState) => ({ + ...prevState, + externalDataSources: externalDataSourcesCache.externalDataSources.map((ds) => ({ + id: ds.dataSourceRef, + name: ds.name, + type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, + })), + })); + } else if ( + status === DirectQueryLoadingStatus.CANCELED || + status === DirectQueryLoadingStatus.FAILED + ) { + setNavigatorState((prevState) => ({ ...prevState, failed: true })); + } + }, [dataSourcesLoadStatus]); + + useEffect(() => { + const status = databasesLoadStatus.toLowerCase(); + if (selectedDataSetState.isExternal && selectedDataSetState.dataSourceRef) { + const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( + selectedDataSetState.dataSourceRef.name, + selectedDataSetState.dataSourceRef.id + ); + if (status === DirectQueryLoadingStatus.SUCCESS) { + setNavigatorState((prevState) => ({ + ...prevState, + cachedDatabases: dataSourceCache.databases, + })); + } else if ( + status === DirectQueryLoadingStatus.CANCELED || + status === DirectQueryLoadingStatus.FAILED + ) { + setNavigatorState((prevState) => ({ ...prevState, failed: true })); + } + } + }, [databasesLoadStatus, selectedDataSetState.isExternal, selectedDataSetState.dataSourceRef]); + + const handleSelectExternalDataSource = useCallback( + async (dataSource) => { + if (selectedDataSetState.isExternal && dataSource) { + const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( + dataSource.name, + dataSource.id + ); + if ( + (dataSourceCache.status === CachedDataSourceStatus.Empty || + dataSourceCache.status === CachedDataSourceStatus.Failed) && + !isCatalogCacheFetching(databasesLoadStatus) + ) { + await startLoadingDatabases({ + dataSourceName: dataSource.name, + dataSourceMDSId: dataSource.id, + }); + } else if (dataSourceCache.status === CachedDataSourceStatus.Updated) { + setNavigatorState((prevState) => ({ + ...prevState, + cachedDatabases: dataSourceCache.databases, + })); + setSelectedDataSetState((prevState) => ({ + ...prevState, + dataSource, + isExternal: true, + })); + } + } + }, + [databasesLoadStatus, selectedDataSetState.isExternal, startLoadingDatabases] + ); + + // Start loading tables for selected database + const handleSelectExternalDatabase = useCallback( + (externalDatabase: SimpleDataSource) => { + if (selectedDataSetState.dataSourceRef && externalDatabase) { + let databaseCache; + try { + databaseCache = CatalogCacheManager.getDatabase( + selectedDataSetState.dataSourceRef.name, + externalDatabase.name, + selectedDataSetState.dataSourceRef.id + ); + } catch (error) { + return; + } + if ( + databaseCache.status === CachedDataSourceStatus.Empty || + (databaseCache.status === CachedDataSourceStatus.Failed && + !isCatalogCacheFetching(tablesLoadStatus)) + ) { + startLoadingTables({ + dataSourceName: selectedDataSetState.dataSourceRef.name, + databaseName: externalDatabase.name, + dataSourceMDSId: selectedDataSetState.dataSourceRef.id, + }); + setSelectedDataSetState((prevState) => ({ + ...prevState, + database: externalDatabase, + })); + } else if (databaseCache.status === CachedDataSourceStatus.Updated) { + setNavigatorState((prevState) => ({ + ...prevState, + cachedTables: databaseCache.tables, + })); + } + } + }, + [selectedDataSetState.dataSourceRef, tablesLoadStatus, startLoadingTables] + ); + + // Retrieve tables from cache upon success + useEffect(() => { + if ( + selectedDataSetState.dataSourceRef && + selectedDataSetState.isExternal && + selectedDataSetState.database + ) { + const tablesStatus = tablesLoadStatus.toLowerCase(); + let databaseCache; + try { + databaseCache = CatalogCacheManager.getDatabase( + selectedDataSetState.dataSourceRef.name, + selectedDataSetState.database, + selectedDataSetState.dataSourceRef.id + ); + } catch (error) { + return; + } + if (tablesStatus === DirectQueryLoadingStatus.SUCCESS) { + setNavigatorState((prevState) => ({ + ...prevState, + cachedTables: databaseCache.tables, + })); + } else if ( + tablesStatus === DirectQueryLoadingStatus.CANCELED || + tablesStatus === DirectQueryLoadingStatus.FAILED + ) { + notifications.toasts.addWarning('Error loading tables'); + } + } + }, [ + tablesLoadStatus, + selectedDataSetState.dataSourceRef, + selectedDataSetState.isExternal, + selectedDataSetState.database, + notifications.toasts, + ]); + + const handleSelectedDataSource = useCallback( + async (source: SimpleDataSource) => { + if (source) { + isLoading(true); + const indices = await fetchIndices(searchService, source.id); + setSelectedDataSetState((prevState) => ({ + ...prevState, + isExternal: false, + dataSourceRef: { + ...source, + indices: indices.map((indexName: string) => ({ + id: indexName, + title: indexName, + dataSourceRef: { + id: source.id, + name: source.name, + type: source.type, + }, + })), + }, + })); + isLoading(false); + } + }, + [searchService] + ); + + const handleSelectedObject = useCallback( + async (object) => { + isLoading(true); + if (object) { + const fields = await indexPatternsService.getFieldsForWildcard({ + pattern: object.title, + dataSourceId: object.dataSourceRef?.id, + }); + + const timeFields = fields.filter((field: any) => field.type === 'date'); + const timeFieldName = timeFields?.length > 0 ? timeFields[0].name : undefined; + setSelectedDataSetState((prevState) => ({ + ...prevState, + id: object.id, + title: object.title, + fields, + timeFields, + timeFieldName, + type: SIMPLE_DATA_SET_TYPES.TEMPORARY, + })); + isLoading(false); + } + }, + [indexPatternsService] + ); + + const handleSelectedDataSet = useCallback( + async (ds?: SimpleDataSet) => { + const selectedDataSet = ds ?? selectedDataSetState; + if (!selectedDataSet || !selectedDataSet.id) return; + + const language = uiService.Settings.getUserQueryLanguage(); + const queryEnhancements = uiService.Settings.getQueryEnhancements(language); + const initialInput = queryEnhancements?.searchBar?.queryStringInput?.initialValue; + + // Update query + const query = initialInput + ? initialInput.replace('', selectedDataSet.title!) + : ''; + uiService.Settings.setUserQueryString(query); + queryService.queryString.setQuery({ query, language }); + + // Update dataset + queryService.dataSet.setDataSet(selectedDataSet); + + // Add to recent datasets + CatalogCacheManager.addRecentDataSet({ + id: selectedDataSet.id, + title: selectedDataSet.title ?? selectedDataSet.id!, + dataSourceRef: selectedDataSet.dataSourceRef, + timeFieldName: selectedDataSet.timeFieldName, + type: selectedDataSet.type, + }); + + // Update data set manager + dataSetManager!.setDataSet({ + id: selectedDataSet.id, + title: selectedDataSet.title, + dataSourceRef: selectedDataSet.dataSourceRef, + timeFieldName: selectedDataSet.timeFieldName, + type: selectedDataSet.type, + }); + + closePopover(); + }, + [ + dataSetManager, + queryService.dataSet, + queryService.queryString, + selectedDataSetState, + uiService.Settings, + ] + ); + + const indexPatternsLabel = i18n.translate('data.query.dataSetNavigator.indexPatternsName', { + defaultMessage: 'Index patterns', + }); + const indicesLabel = i18n.translate('data.query.dataSetNavigator.indicesName', { + defaultMessage: 'Indexes', + }); + const S3DataSourcesLabel = i18n.translate('data.query.dataSetNavigator.S3DataSourcesLabel', { + defaultMessage: 'S3', + }); + + const createRefreshButton = () => ( + + ); + + const createLoadingSpinner = () => ( + + + + ); + + const createIndexPatternsPanel = () => ({ + id: 1, + title: indexPatternsLabel, + items: navigatorState.indexPatterns.map((indexPattern) => ({ + name: indexPattern.title, + onClick: async () => await handleSelectedDataSet(indexPattern), + })), + content:
{navigatorState.indexPatterns.length === 0 && createLoadingSpinner()}
, + }); + + const createIndexesPanel = () => ({ + id: 2, + title: indicesLabel, + items: [ + ...navigatorState.dataSources.map((dataSource) => ({ + name: dataSource.name, + panel: 3, + onClick: async () => await handleSelectedDataSource(dataSource), + })), + ], + content:
{navigatorState.isLoading && createLoadingSpinner()}
, + }); + + const createDataSourcesPanel = () => ({ + id: 3, + title: selectedDataSetState.dataSourceRef?.name ?? indicesLabel, + items: selectedDataSetState.dataSourceRef?.indices?.map((object) => ({ + name: object.title, + panel: 7, + onClick: async () => + await handleSelectedObject({ ...object, type: SIMPLE_DATA_SET_TYPES.TEMPORARY }), + })), + content:
{navigatorState.isLoading && createLoadingSpinner()}
, + }); + + const createS3DataSourcesPanel = () => ({ + id: 4, + title: ( +
+ {S3DataSourcesLabel} + {CatalogCacheManager.getExternalDataSourcesCache().status === + CachedDataSourceStatus.Updated && createRefreshButton()} +
+ ), + items: [ + ...navigatorState.externalDataSources.map((dataSource) => ({ + name: dataSource.name, + onClick: async () => await handleSelectExternalDataSource(dataSource), + panel: 5, + })), + ], + content:
{dataSourcesLoadStatus && createLoadingSpinner()}
, + }); + + const createDatabasesPanel = () => ({ + id: 5, + title: selectedDataSetState.dataSourceRef?.name + ? selectedDataSetState.dataSourceRef?.name + : 'Databases', + items: [ + ...navigatorState.externalDataSources.map((db) => ({ + name: db.name, + onClick: async () => { + await handleSelectExternalDatabase(db); + }, + panel: 6, + })), + ], + content:
{isCatalogCacheFetching(databasesLoadStatus) && createLoadingSpinner()}
, + }); + + return ( + + {dataSet?.dataSourceRef?.name + ? `${dataSet.dataSourceRef?.name}::${dataSet?.title}` + : dataSet?.title} + + } + isOpen={navigatorState.isOpen} + closePopover={closePopover} + anchorPosition="downLeft" + panelPaddingSize="none" + > + 0 + ? [ + { + name: 'Recently Used', + panel: 8, + }, + ] + : []), + { + name: indexPatternsLabel, + panel: 1, + }, + { + name: indicesLabel, + panel: 2, + }, + ...(navigatorState.isExternalDataSourcesEnabled + ? [ + { + name: S3DataSourcesLabel, + panel: 4, + onClick: async () => { + const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); + if ( + (externalDataSourcesCache.status === CachedDataSourceStatus.Empty || + externalDataSourcesCache.status === CachedDataSourceStatus.Failed) && + !isCatalogCacheFetching(dataSourcesLoadStatus) && + navigatorState.dataSources.length > 0 + ) { + startLoadingDataSources( + navigatorState.dataSources.map((dataSource) => dataSource.id) + ); + } else if ( + externalDataSourcesCache.status === CachedDataSourceStatus.Updated + ) { + setNavigatorState((prevState) => ({ + ...prevState, + externalDataSources: externalDataSourcesCache.externalDataSources.map( + (ds) => ({ + id: ds.name, + name: ds.name, + type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, + }) + ), + })); + } + }, + }, + ] + : []), + ], + }, + createIndexPatternsPanel(), + createIndexesPanel(), + createDataSourcesPanel(), + createS3DataSourcesPanel(), + createDatabasesPanel(), + { + id: 6, + title: selectedDataSetState.database ? selectedDataSetState.database : 'Tables', + items: [ + ...navigatorState.cachedTables.map((table) => ({ + name: table.title, + onClick: async () => { + setSelectedDataSetState((prevState) => ({ + ...prevState, + object: { + id: `${selectedDataSetState.dataSourceRef!.name}.${ + selectedDataSetState.database + }.${table.title}`, + title: `${selectedDataSetState.dataSourceRef!.name}.${ + selectedDataSetState.database + }.${table.title}`, + dataSource: { + id: selectedDataSetState.dataSourceRef!.id, + name: selectedDataSetState.dataSourceRef!.name, + type: selectedDataSetState.dataSourceRef!.type, + }, + type: SIMPLE_DATA_SET_TYPES.TEMPORARY_ASYNC, + }, + })); + }, + })), + ], + content: ( +
{isCatalogCacheFetching(tablesLoadStatus) && createLoadingSpinner()}
+ ), + }, + { + id: 7, + title: selectedDataSetState.title, + content: + navigatorState.isLoading || !selectedDataSetState.title ? ( +
{createLoadingSpinner()}
+ ) : ( + + + 0 + ? [ + ...selectedDataSetState.timeFields!.map((field: any) => ({ + value: field.name, + text: field.name, + })), + ] + : []), + { value: 'no-time-filter', text: "I don't want to use a time filter" }, + ]} + onChange={(event) => { + setSelectedDataSetState((prevState) => ({ + ...prevState, + timeFieldName: + event.target.value !== 'no-time-filter' + ? event.target.value + : undefined, + })); + }} + aria-label="Select a date field" + /> + + { + await handleSelectedDataSet(); + }} + > + Select + + + ), + }, + { + id: 8, + title: 'Recently Used', + items: CatalogCacheManager.getRecentDataSets().map((ds) => ({ + name: ds.title, + onClick: async () => { + setSelectedDataSetState({ + id: ds.id ?? ds.title, + title: ds.title, + dataSourceRef: ds.dataSourceRef, + database: undefined, + isExternal: !ds.dataSourceRef?.type?.startsWith('data-source'), + timeFieldName: ds.timeFieldName, + }); + await handleSelectedDataSet(); + }, + })), + }, + ]} + /> + + ); +}; + +// eslint-disable-next-line import/no-default-export +export default DataSetNavigator; diff --git a/src/plugins/data/public/ui/dataset_navigator/index.tsx b/src/plugins/data/public/ui/dataset_navigator/index.tsx new file mode 100644 index 000000000000..e98e52c8421f --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/index.tsx @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DataSetNavigator, DataSetNavigatorProps } from './dataset_navigator'; +export { createDataSetNavigator } from './create_dataset_navigator'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts new file mode 100644 index 000000000000..0526cfd51212 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { HttpFetchOptionsWithPath, IHttpInterceptController } from 'opensearch-dashboards/public'; +import { SECURITY_DASHBOARDS_LOGOUT_URL } from '../constants'; +import { CatalogCacheManager } from './cache_manager'; + +export function catalogRequestIntercept(): any { + return ( + fetchOptions: Readonly, + _controller: IHttpInterceptController + ) => { + if (fetchOptions.path.includes(SECURITY_DASHBOARDS_LOGOUT_URL)) { + // Clears all user catalog cache details + CatalogCacheManager.clearDataSourceCache(); + CatalogCacheManager.clearAccelerationsCache(); + CatalogCacheManager.clearExternalDataSourcesCache(); + CatalogCacheManager.clearRecentDataSetsCache(); + } + }; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx new file mode 100644 index 000000000000..9a42bfc74aa8 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx @@ -0,0 +1,470 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useEffect, useRef, useState } from 'react'; +import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; +import { ASYNC_POLLING_INTERVAL, SPARK_HIVE_TABLE_REGEX, SPARK_PARTITION_INFO } from '../constants'; +import { + AsyncPollingResult, + CachedColumn, + CachedDataSourceStatus, + CachedTable, + LoadCacheType, + StartLoadingParams, + DirectQueryLoadingStatus, + DirectQueryRequest, +} from '../types'; +import { getAsyncSessionId, setAsyncSessionId } from '../utils/query_session_utils'; +import { + addBackticksIfNeeded, + combineSchemaAndDatarows, + get as getObjValue, + formatError, +} from '../utils/shared'; +import { usePolling } from '../utils/use_polling'; +import { SQLService } from '../requests/sql'; +import { CatalogCacheManager } from './cache_manager'; +import { fetchExternalDataSources } from '../utils'; + +export const updateDatabasesToCache = ( + dataSourceName: string, + pollingResult: AsyncPollingResult, + dataSourceMDSId?: string +) => { + const cachedDataSource = CatalogCacheManager.getOrCreateDataSource( + dataSourceName, + dataSourceMDSId + ); + + const currentTime = new Date().toUTCString(); + + if (!pollingResult) { + CatalogCacheManager.addOrUpdateDataSource( + { + ...cachedDataSource, + databases: [], + lastUpdated: currentTime, + status: CachedDataSourceStatus.Failed, + ...(dataSourceMDSId && { dataSourceMDSId }), + }, + dataSourceMDSId + ); + return; + } + + const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); + const newDatabases = combinedData.map((row: any) => ({ + name: row.namespace, + tables: [], + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + })); + + CatalogCacheManager.addOrUpdateDataSource( + { + ...cachedDataSource, + databases: newDatabases, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + ...(dataSourceMDSId && { dataSourceMDSId }), + }, + dataSourceMDSId + ); +}; + +export const updateTablesToCache = ( + dataSourceName: string, + databaseName: string, + pollingResult: AsyncPollingResult, + dataSourceMDSId?: string +) => { + try { + const cachedDatabase = CatalogCacheManager.getDatabase( + dataSourceName, + databaseName, + dataSourceMDSId + ); + const currentTime = new Date().toUTCString(); + + if (!pollingResult) { + CatalogCacheManager.updateDatabase( + dataSourceName, + { + ...cachedDatabase, + tables: [], + lastUpdated: currentTime, + status: CachedDataSourceStatus.Failed, + }, + dataSourceMDSId + ); + return; + } + + const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); + const newTables = combinedData + .filter((row: any) => !SPARK_HIVE_TABLE_REGEX.test(row.information)) + .map((row: any) => ({ + name: row.tableName, + })); + + CatalogCacheManager.updateDatabase( + dataSourceName, + { + ...cachedDatabase, + tables: newTables, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + }, + dataSourceMDSId + ); + } catch (error) { + // eslint-disable-next-line no-console + console.error(error); + } +}; + +export const updateAccelerationsToCache = ( + dataSourceName: string, + pollingResult: AsyncPollingResult, + dataSourceMDSId?: string +) => { + const currentTime = new Date().toUTCString(); + + if (!pollingResult) { + CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ + name: dataSourceName, + accelerations: [], + lastUpdated: currentTime, + status: CachedDataSourceStatus.Failed, + ...(dataSourceMDSId && { dataSourceMDSId }), + }); + return; + } + + const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); + + const newAccelerations: any[] = combinedData.map((row: any) => ({ + flintIndexName: row.flint_index_name, + type: row.kind === 'mv' ? 'materialized' : row.kind, + database: row.database, + table: row.table, + indexName: row.index_name, + autoRefresh: row.auto_refresh, + status: row.status, + })); + + CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ + name: dataSourceName, + accelerations: newAccelerations, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + ...(dataSourceMDSId && { dataSourceMDSId }), + }); +}; + +export const updateTableColumnsToCache = ( + dataSourceName: string, + databaseName: string, + tableName: string, + pollingResult: AsyncPollingResult, + dataSourceMDSId?: string +) => { + try { + if (!pollingResult) { + return; + } + const cachedDatabase = CatalogCacheManager.getDatabase( + dataSourceName, + databaseName, + dataSourceMDSId + ); + const currentTime = new Date().toUTCString(); + + const combinedData: Array<{ col_name: string; data_type: string }> = combineSchemaAndDatarows( + pollingResult.schema, + pollingResult.datarows + ); + + const tableColumns: CachedColumn[] = []; + for (const row of combinedData) { + if (row.col_name === SPARK_PARTITION_INFO) { + break; + } + tableColumns.push({ + fieldName: row.col_name, + dataType: row.data_type, + }); + } + + const newTables: CachedTable[] = cachedDatabase.tables.map((ts) => + ts.name === tableName ? { ...ts, columns: tableColumns } : { ...ts } + ); + + if (cachedDatabase.status === CachedDataSourceStatus.Updated) { + CatalogCacheManager.updateDatabase( + dataSourceName, + { + ...cachedDatabase, + tables: newTables, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + }, + dataSourceMDSId + ); + } + } catch (error) { + // eslint-disable-next-line no-console + console.error(error); + } +}; + +export const updateToCache = ( + pollResults: any, + loadCacheType: LoadCacheType, + dataSourceName: string, + databaseName?: string, + tableName?: string, + dataSourceMDSId?: string +) => { + switch (loadCacheType) { + case 'databases': + updateDatabasesToCache(dataSourceName, pollResults, dataSourceMDSId); + break; + case 'tables': + updateTablesToCache(dataSourceName, databaseName!, pollResults, dataSourceMDSId); + break; + case 'accelerations': + updateAccelerationsToCache(dataSourceName, pollResults, dataSourceMDSId); + break; + case 'tableColumns': + updateTableColumnsToCache( + dataSourceName, + databaseName!, + tableName!, + pollResults, + dataSourceMDSId + ); + default: + break; + } +}; + +export const createLoadQuery = ( + loadCacheType: LoadCacheType, + dataSourceName: string, + databaseName?: string, + tableName?: string +) => { + let query; + switch (loadCacheType) { + case 'databases': + query = `SHOW SCHEMAS IN ${addBackticksIfNeeded(dataSourceName)}`; + break; + case 'tables': + query = `SHOW TABLE EXTENDED IN ${addBackticksIfNeeded( + dataSourceName + )}.${addBackticksIfNeeded(databaseName!)} LIKE '*'`; + break; + case 'accelerations': + query = `SHOW FLINT INDEX in ${addBackticksIfNeeded(dataSourceName)}`; + break; + case 'tableColumns': + query = `DESC ${addBackticksIfNeeded(dataSourceName)}.${addBackticksIfNeeded( + databaseName! + )}.${addBackticksIfNeeded(tableName!)}`; + break; + default: + query = ''; + break; + } + return query; +}; + +export const useLoadToCache = ( + loadCacheType: LoadCacheType, + http: HttpStart, + notifications: NotificationsStart +) => { + const sqlService = new SQLService(http); + const [currentDataSourceName, setCurrentDataSourceName] = useState(''); + const [currentDatabaseName, setCurrentDatabaseName] = useState(''); + const [currentTableName, setCurrentTableName] = useState(''); + const [loadStatus, setLoadStatus] = useState( + DirectQueryLoadingStatus.INITIAL + ); + const dataSourceMDSClientId = useRef(''); + + const { + data: pollingResult, + loading: _pollingLoading, + error: pollingError, + startPolling, + stopPolling: stopLoading, + } = usePolling((params) => { + return sqlService.fetchWithJobId(params, dataSourceMDSClientId.current); + }, ASYNC_POLLING_INTERVAL); + + const onLoadingFailed = () => { + setLoadStatus(DirectQueryLoadingStatus.FAILED); + updateToCache( + null, + loadCacheType, + currentDataSourceName, + currentDatabaseName, + currentTableName, + dataSourceMDSClientId.current + ); + }; + + const startLoading = async ({ + dataSourceName, + dataSourceMDSId, + databaseName, + tableName, + }: StartLoadingParams) => { + setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); + setCurrentDataSourceName(dataSourceName); + setCurrentDatabaseName(databaseName); + setCurrentTableName(tableName); + dataSourceMDSClientId.current = dataSourceMDSId || ''; + + let requestPayload: DirectQueryRequest = { + lang: 'sql', + query: createLoadQuery(loadCacheType, dataSourceName, databaseName, tableName), + datasource: dataSourceName, + }; + + const sessionId = getAsyncSessionId(dataSourceName); + if (sessionId) { + requestPayload = { ...requestPayload, sessionId }; + } + await sqlService + .fetch(requestPayload, dataSourceMDSId) + .then((result) => { + setAsyncSessionId(dataSourceName, getObjValue(result, 'sessionId', null)); + if (result.queryId) { + startPolling({ + queryId: result.queryId, + }); + } else { + // eslint-disable-next-line no-console + console.error('No query id found in response'); + onLoadingFailed(); + } + }) + .catch((e) => { + onLoadingFailed(); + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + e.body?.message + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + // eslint-disable-next-line no-console + console.error(e); + }); + }; + + useEffect(() => { + // cancel direct query + if (!pollingResult) return; + const { status: anyCaseStatus, datarows, error } = pollingResult; + const status = anyCaseStatus?.toLowerCase(); + + if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { + setLoadStatus(status); + stopLoading(); + updateToCache( + pollingResult, + loadCacheType, + currentDataSourceName, + currentDatabaseName, + currentTableName, + dataSourceMDSClientId.current + ); + } else if (status === DirectQueryLoadingStatus.FAILED) { + onLoadingFailed(); + stopLoading(); + + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + error + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + } else { + setLoadStatus(status); + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [pollingResult, pollingError]); + + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadDatabasesToCache = (http: HttpStart, notifications: NotificationsStart) => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache( + 'databases', + http, + notifications + ); + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadTablesToCache = (http: HttpStart, notifications: NotificationsStart) => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache('tables', http, notifications); + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadTableColumnsToCache = (http: HttpStart, notifications: NotificationsStart) => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache( + 'tableColumns', + http, + notifications + ); + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadAccelerationsToCache = (http: HttpStart, notifications: NotificationsStart) => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache( + 'accelerations', + http, + notifications + ); + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadExternalDataSourcesToCache = ( + http: HttpStart, + notifications: NotificationsStart +) => { + const [loadStatus, setLoadStatus] = useState( + DirectQueryLoadingStatus.INITIAL + ); + + const loadExternalDataSources = async (connectedClusters: string[]) => { + setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); + CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Empty); + + try { + const externalDataSources = await fetchExternalDataSources(http, connectedClusters); + CatalogCacheManager.updateExternalDataSources(externalDataSources); + setLoadStatus(DirectQueryLoadingStatus.SUCCESS); + CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Updated); + } catch (error) { + setLoadStatus(DirectQueryLoadingStatus.FAILED); + CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Failed); + notifications.toasts.addError(error, { + title: 'Failed to load external datasources', + }); + } + }; + + return { loadStatus, loadExternalDataSources }; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts new file mode 100644 index 000000000000..3d0a8e0c982d --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts @@ -0,0 +1,416 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE, + CATALOG_CACHE_VERSION, + RECENT_DATASET_OPTIONS_CACHE, +} from '../constants'; +import { ASYNC_QUERY_ACCELERATIONS_CACHE, ASYNC_QUERY_DATASOURCE_CACHE } from '../utils/shared'; +import { + AccelerationsCacheData, + CachedAccelerationByDataSource, + CachedDataSource, + CachedDataSourceStatus, + CachedDatabase, + DataSetOption, + DataSourceCacheData, + ExternalDataSource, + ExternalDataSourcesCacheData, + RecentDataSetOptionsCacheData, +} from '../types'; +import { SimpleDataSet, SimpleObject } from '../../../../../common'; + +/** + * Manages caching for catalog data including data sources and accelerations. + */ +export class CatalogCacheManager { + /** + * Key for the data source cache in local storage. + */ + private static readonly datasourceCacheKey = ASYNC_QUERY_DATASOURCE_CACHE; + + /** + * Key for the accelerations cache in local storage. + */ + private static readonly accelerationsCacheKey = ASYNC_QUERY_ACCELERATIONS_CACHE; + + /** + * Key for external datasources cache in local storage + */ + private static readonly externalDataSourcesCacheKey = ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE; + + /** + * Key for recently selected datasets in local storage + */ + private static readonly recentDataSetCacheKey = RECENT_DATASET_OPTIONS_CACHE; + + // TODO: make this an advanced setting + private static readonly maxRecentDataSet = 4; + + /** + * Saves data source cache to local storage. + * @param {DataSourceCacheData} cacheData - The data source cache data to save. + */ + static saveDataSourceCache(cacheData: DataSourceCacheData): void { + sessionStorage.setItem(this.datasourceCacheKey, JSON.stringify(cacheData)); + } + + /** + * Retrieves data source cache from local storage. + * @returns {DataSourceCacheData} The retrieved data source cache. + */ + static getDataSourceCache(): DataSourceCacheData { + const catalogData = sessionStorage.getItem(this.datasourceCacheKey); + + if (catalogData) { + return JSON.parse(catalogData); + } else { + const defaultCacheObject = { version: CATALOG_CACHE_VERSION, dataSources: [] }; + this.saveDataSourceCache(defaultCacheObject); + return defaultCacheObject; + } + } + + /** + * Saves accelerations cache to local storage. + * @param {AccelerationsCacheData} cacheData - The accelerations cache data to save. + */ + static saveAccelerationsCache(cacheData: AccelerationsCacheData): void { + sessionStorage.setItem(this.accelerationsCacheKey, JSON.stringify(cacheData)); + } + + /** + * Retrieves accelerations cache from local storage. + * @returns {AccelerationsCacheData} The retrieved accelerations cache. + */ + static getAccelerationsCache(): AccelerationsCacheData { + const accelerationCacheData = sessionStorage.getItem(this.accelerationsCacheKey); + + if (accelerationCacheData) { + return JSON.parse(accelerationCacheData); + } else { + const defaultCacheObject = { + version: CATALOG_CACHE_VERSION, + dataSources: [], + }; + this.saveAccelerationsCache(defaultCacheObject); + return defaultCacheObject; + } + } + + /** + * Adds or updates a data source in the accelerations cache. + * @param {CachedAccelerationByDataSource} dataSource - The data source to add or update. + */ + static addOrUpdateAccelerationsByDataSource( + dataSource: CachedAccelerationByDataSource, + dataSourceMDSId?: string + ): void { + let index = -1; + const accCacheData = this.getAccelerationsCache(); + if (dataSourceMDSId) { + index = accCacheData.dataSources.findIndex( + (ds: CachedAccelerationByDataSource) => + ds.name === dataSource.name && ds.dataSourceMDSId === dataSourceMDSId + ); + } else { + index = accCacheData.dataSources.findIndex( + (ds: CachedAccelerationByDataSource) => ds.name === dataSource.name + ); + } + if (index !== -1) { + accCacheData.dataSources[index] = dataSource; + } else { + accCacheData.dataSources.push(dataSource); + } + this.saveAccelerationsCache(accCacheData); + } + + /** + * Retrieves accelerations cache from local storage by the datasource name. + * @param {string} dataSourceName - The name of the data source. + * @returns {CachedAccelerationByDataSource} The retrieved accelerations by datasource in cache. + * @throws {Error} If the data source is not found. + */ + static getOrCreateAccelerationsByDataSource( + dataSourceName: string, + dataSourceMDSId?: string + ): CachedAccelerationByDataSource { + const accCacheData = this.getAccelerationsCache(); + let cachedDataSource; + if (dataSourceMDSId) { + cachedDataSource = accCacheData.dataSources.find( + (ds) => ds.name === dataSourceName && ds.dataSourceMDSId === dataSourceMDSId + ); + } else { + cachedDataSource = accCacheData.dataSources.find((ds) => ds.name === dataSourceName); + } + if (cachedDataSource) return cachedDataSource; + else { + let defaultDataSourceObject: CachedAccelerationByDataSource = { + name: dataSourceName, + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + accelerations: [], + }; + + if (dataSourceMDSId !== '' && dataSourceMDSId !== undefined) { + defaultDataSourceObject = { ...defaultDataSourceObject, dataSourceMDSId }; + } + this.addOrUpdateAccelerationsByDataSource(defaultDataSourceObject, dataSourceMDSId); + return defaultDataSourceObject; + } + } + + /** + * Adds or updates a data source in the cache. + * @param {CachedDataSource} dataSource - The data source to add or update. + */ + static addOrUpdateDataSource(dataSource: CachedDataSource, dataSourceMDSId?: string): void { + const cacheData = this.getDataSourceCache(); + let index; + if (dataSourceMDSId) { + index = cacheData.dataSources.findIndex( + (ds: CachedDataSource) => + ds.name === dataSource.name && ds.dataSourceMDSId === dataSourceMDSId + ); + } + index = cacheData.dataSources.findIndex((ds: CachedDataSource) => ds.name === dataSource.name); + if (index !== -1) { + cacheData.dataSources[index] = dataSource; + } else { + cacheData.dataSources.push(dataSource); + } + this.saveDataSourceCache(cacheData); + } + + /** + * Retrieves or creates a data source with the specified name. + * @param {string} dataSourceName - The name of the data source. + * @returns {CachedDataSource} The retrieved or created data source. + */ + static getOrCreateDataSource(dataSourceName: string, dataSourceMDSId?: string): CachedDataSource { + let cachedDataSource; + if (dataSourceMDSId) { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName + ); + } else { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.name === dataSourceName + ); + } + if (cachedDataSource) { + return cachedDataSource; + } else { + let defaultDataSourceObject: CachedDataSource = { + name: dataSourceName, + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + databases: [], + }; + if (dataSourceMDSId !== '' && dataSourceMDSId !== undefined) { + defaultDataSourceObject = { ...defaultDataSourceObject, dataSourceMDSId }; + } + this.addOrUpdateDataSource(defaultDataSourceObject, dataSourceMDSId); + return defaultDataSourceObject; + } + } + + /** + * Retrieves a database from the cache. + * @param {string} dataSourceName - The name of the data source containing the database. + * @param {string} databaseName - The name of the database. + * @returns {CachedDatabase} The retrieved database. + * @throws {Error} If the data source or database is not found. + */ + static getDatabase( + dataSourceName: string, + databaseName: string, + dataSourceMDSId?: string + ): CachedDatabase { + let cachedDataSource; + if (dataSourceMDSId) { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName + ); + } else { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.name === dataSourceName + ); + } + if (!cachedDataSource) { + throw new Error('DataSource not found exception: ' + dataSourceName); + } + + const cachedDatabase = cachedDataSource.databases.find((db) => db.name === databaseName); + if (!cachedDatabase) { + throw new Error('Database not found exception: ' + databaseName); + } + + return cachedDatabase; + } + + /** + * Retrieves a table from the cache. + * @param {string} dataSourceName - The name of the data source containing the database. + * @param {string} databaseName - The name of the database. + * @param {string} tableName - The name of the database. + * @returns {Cachedtable} The retrieved database. + * @throws {Error} If the data source, database or table is not found. + */ + static getTable( + dataSourceName: string, + databaseName: string, + tableName: string, + dataSourceMDSId?: string + ): SimpleObject { + const cachedDatabase = this.getDatabase(dataSourceName, databaseName, dataSourceMDSId); + + const cachedTable = cachedDatabase.tables!.find((table) => table.title === tableName); + if (!cachedTable) { + throw new Error('Table not found exception: ' + tableName); + } + return cachedTable; + } + + /** + * Updates a database in the cache. + * @param {string} dataSourceName - The name of the data source containing the database. + * @param {CachedDatabase} database - The database to be updated. + * @throws {Error} If the data source or database is not found. + */ + static updateDatabase( + dataSourceName: string, + database: CachedDatabase, + dataSourceMDSId?: string + ): void { + let cachedDataSource; + if (dataSourceMDSId) { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName + ); + } else { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.name === dataSourceName + ); + } + + if (!cachedDataSource) { + throw new Error('DataSource not found exception: ' + dataSourceName); + } + + const index = cachedDataSource.databases.findIndex((db) => db.name === database.name); + if (index !== -1) { + cachedDataSource.databases[index] = database; + this.addOrUpdateDataSource(cachedDataSource, dataSourceMDSId); + } else { + throw new Error('Database not found exception: ' + database.name); + } + } + + /** + * Clears the data source cache from local storage. + */ + static clearDataSourceCache(): void { + sessionStorage.removeItem(this.datasourceCacheKey); + this.clearExternalDataSourcesCache(); + } + + /** + * Clears the accelerations cache from local storage. + */ + static clearAccelerationsCache(): void { + sessionStorage.removeItem(this.accelerationsCacheKey); + } + + static saveExternalDataSourcesCache(cacheData: ExternalDataSourcesCacheData): void { + sessionStorage.setItem(this.externalDataSourcesCacheKey, JSON.stringify(cacheData)); + } + + static getExternalDataSourcesCache(): ExternalDataSourcesCacheData { + const externalDataSourcesData = sessionStorage.getItem(this.externalDataSourcesCacheKey); + + if (externalDataSourcesData) { + return JSON.parse(externalDataSourcesData); + } else { + const defaultCacheObject: ExternalDataSourcesCacheData = { + version: CATALOG_CACHE_VERSION, + externalDataSources: [], + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + }; + this.saveExternalDataSourcesCache(defaultCacheObject); + return defaultCacheObject; + } + } + + static updateExternalDataSources(externalDataSources: ExternalDataSource[]): void { + const currentTime = new Date().toUTCString(); + const cacheData = this.getExternalDataSourcesCache(); + cacheData.externalDataSources = externalDataSources; + cacheData.lastUpdated = currentTime; + cacheData.status = CachedDataSourceStatus.Updated; + this.saveExternalDataSourcesCache(cacheData); + } + + static getExternalDataSources(): ExternalDataSourcesCacheData { + return this.getExternalDataSourcesCache(); + } + + static clearExternalDataSourcesCache(): void { + sessionStorage.removeItem(this.externalDataSourcesCacheKey); + } + + static setExternalDataSourcesLoadingStatus(status: CachedDataSourceStatus): void { + const cacheData = this.getExternalDataSourcesCache(); + cacheData.status = status; + this.saveExternalDataSourcesCache(cacheData); + } + + static saveRecentDataSetsCache(cacheData: RecentDataSetOptionsCacheData): void { + sessionStorage.setItem(this.recentDataSetCacheKey, JSON.stringify(cacheData)); + } + + static getRecentDataSetsCache(): RecentDataSetOptionsCacheData { + const recentDataSetOptionsData = sessionStorage.getItem(this.recentDataSetCacheKey); + + if (recentDataSetOptionsData) { + return JSON.parse(recentDataSetOptionsData); + } else { + const defaultCacheObject: RecentDataSetOptionsCacheData = { + version: CATALOG_CACHE_VERSION, + recentDataSets: [], + }; + this.saveRecentDataSetsCache(defaultCacheObject); + return defaultCacheObject; + } + } + + static addRecentDataSet(dataSet: SimpleDataSet): void { + const cacheData = this.getRecentDataSetsCache(); + + cacheData.recentDataSets = cacheData.recentDataSets.filter( + (option) => option.id !== dataSet.id + ); + + cacheData.recentDataSets.push(dataSet); + + if (cacheData.recentDataSets.length > this.maxRecentDataSet) { + cacheData.recentDataSets.shift(); + } + + this.saveRecentDataSetsCache(cacheData); + } + + static getRecentDataSets(): SimpleDataSet[] { + return this.getRecentDataSetsCache().recentDataSets; + } + + static clearRecentDataSetsCache(): void { + sessionStorage.removeItem(this.recentDataSetCacheKey); + } +} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx new file mode 100644 index 000000000000..5449277b2bd8 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx @@ -0,0 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './cache_intercept'; +export * from './cache_loader'; +export * from './cache_manager'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts b/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts new file mode 100644 index 000000000000..e22da95ff4c6 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts @@ -0,0 +1,101 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; +export const ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE = 'async_query_external_datasources_cache'; +export const RECENT_DATASET_OPTIONS_CACHE = 'recent_dataset_options_cache'; + +export const DATA_SOURCE_NAME_URL_PARAM_KEY = 'datasourceName'; +export const DATA_SOURCE_TYPE_URL_PARAM_KEY = 'datasourceType'; +export const OLLY_QUESTION_URL_PARAM_KEY = 'olly_q'; +export const INDEX_URL_PARAM_KEY = 'indexPattern'; +export const DEFAULT_DATA_SOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; +export const DEFAULT_DATA_SOURCE_NAME = 'Default cluster'; +export const DEFAULT_DATA_SOURCE_OBSERVABILITY_DISPLAY_NAME = 'OpenSearch'; +export const DEFAULT_DATA_SOURCE_TYPE_NAME = 'Default Group'; +export const enum QUERY_LANGUAGE { + PPL = 'PPL', + SQL = 'SQL', + DQL = 'DQL', +} +export enum DATA_SOURCE_TYPES { + DEFAULT_CLUSTER_TYPE = DEFAULT_DATA_SOURCE_TYPE, + SPARK = 'spark', + S3Glue = 's3glue', +} +export const ASYNC_POLLING_INTERVAL = 2000; + +export const CATALOG_CACHE_VERSION = '1.0'; +export const ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME = 'skipping'; +export const ACCELERATION_TIME_INTERVAL = [ + { text: 'millisecond(s)', value: 'millisecond' }, + { text: 'second(s)', value: 'second' }, + { text: 'minutes(s)', value: 'minute' }, + { text: 'hour(s)', value: 'hour' }, + { text: 'day(s)', value: 'day' }, + { text: 'week(s)', value: 'week' }, +]; +export const ACCELERATION_REFRESH_TIME_INTERVAL = [ + { text: 'minutes(s)', value: 'minute' }, + { text: 'hour(s)', value: 'hour' }, + { text: 'day(s)', value: 'day' }, + { text: 'week(s)', value: 'week' }, +]; + +export const ACCELERATION_ADD_FIELDS_TEXT = '(add fields here)'; +export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z0-9_]+$/; +export const ACCELERATION_S3_URL_REGEX = /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/; +export const SPARK_HIVE_TABLE_REGEX = /Provider:\s*hive/; +export const SANITIZE_QUERY_REGEX = /\s+/g; +export const SPARK_TIMESTAMP_DATATYPE = 'timestamp'; +export const SPARK_STRING_DATATYPE = 'string'; + +export const ACCELERATION_INDEX_TYPES = [ + { label: 'Skipping Index', value: 'skipping' }, + { label: 'Covering Index', value: 'covering' }, + { label: 'Materialized View', value: 'materialized' }, +]; + +export const ACC_INDEX_TYPE_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md'; +export const ACC_CHECKPOINT_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md#create-index-options'; + +export const ACCELERATION_INDEX_NAME_INFO = `All OpenSearch acceleration indices have a naming format of pattern: \`prefix__suffix\`. They share a common prefix structure, which is \`flint____\`. Additionally, they may have a suffix that varies based on the index type. +##### Skipping Index +- For 'Skipping' indices, a fixed index name 'skipping' is used, and this name cannot be modified by the user. The suffix added to this type is \`_index\`. + - An example of a 'Skipping' index name would be: \`flint_mydatasource_mydb_mytable_skipping_index\`. +##### Covering Index +- 'Covering' indices allow users to specify their index name. The suffix added to this type is \`_index\`. + - For instance, a 'Covering' index name could be: \`flint_mydatasource_mydb_mytable_myindexname_index\`. +##### Materialized View Index +- 'Materialized View' indices also enable users to define their index name, but they do not have a suffix. + - An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`. +##### Note: +- All user given index names must be in lowercase letters, numbers and underscore. Spaces, commas, and characters -, :, ", *, +, /, \, |, ?, #, >, or < are not allowed. + `; + +export const SKIPPING_INDEX_ACCELERATION_METHODS = [ + { value: 'PARTITION', text: 'Partition' }, + { value: 'VALUE_SET', text: 'Value Set' }, + { value: 'MIN_MAX', text: 'Min Max' }, + { value: 'BLOOM_FILTER', text: 'Bloom Filter' }, +]; + +export const ACCELERATION_AGGREGRATION_FUNCTIONS = [ + { label: 'window.start' }, + { label: 'count' }, + { label: 'sum' }, + { label: 'avg' }, + { label: 'max' }, + { label: 'min' }, +]; + +export const SPARK_PARTITION_INFO = `# Partition Information`; +export const OBS_DEFAULT_CLUSTER = 'observability-default'; // prefix key for generating data source id for default cluster in data selector +export const OBS_S3_DATA_SOURCE = 'observability-s3'; // prefix key for generating data source id for s3 data sources in data selector +export const S3_DATA_SOURCE_GROUP_DISPLAY_NAME = 'Amazon S3'; // display group name for Amazon-managed-s3 data sources in data selector +export const S3_DATA_SOURCE_GROUP_SPARK_DISPLAY_NAME = 'Spark'; // display group name for OpenSearch-spark-s3 data sources in data selector +export const SECURITY_DASHBOARDS_LOGOUT_URL = '/logout'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/hooks/direct_query_hook.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/hooks/direct_query_hook.tsx new file mode 100644 index 000000000000..a2b05f47e9ee --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/hooks/direct_query_hook.tsx @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useEffect, useState } from 'react'; +import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; +import { ASYNC_POLLING_INTERVAL } from '../constants'; +import { DirectQueryLoadingStatus, DirectQueryRequest } from '../types'; +import { getAsyncSessionId, setAsyncSessionId } from '../utils/query_session_utils'; +import { get as getObjValue, formatError } from '../utils/shared'; +import { usePolling } from '../utils/use_polling'; +import { SQLService } from '../requests/sql'; + +export const useDirectQuery = ( + http: HttpStart, + notifications: NotificationsStart, + dataSourceMDSId?: string +) => { + const sqlService = new SQLService(http); + const [loadStatus, setLoadStatus] = useState( + DirectQueryLoadingStatus.SCHEDULED + ); + + const { + data: pollingResult, + loading: _pollingLoading, + error: pollingError, + startPolling, + stopPolling: stopLoading, + } = usePolling((params) => { + return sqlService.fetchWithJobId(params, dataSourceMDSId || ''); + }, ASYNC_POLLING_INTERVAL); + + const startLoading = (requestPayload: DirectQueryRequest) => { + setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); + + const sessionId = getAsyncSessionId(requestPayload.datasource); + if (sessionId) { + requestPayload = { ...requestPayload, sessionId }; + } + + sqlService + .fetch(requestPayload, dataSourceMDSId) + .then((result) => { + setAsyncSessionId(requestPayload.datasource, getObjValue(result, 'sessionId', null)); + if (result.queryId) { + startPolling({ + queryId: result.queryId, + }); + } else { + // eslint-disable-next-line no-console + console.error('No query id found in response'); + setLoadStatus(DirectQueryLoadingStatus.FAILED); + } + }) + .catch((e) => { + setLoadStatus(DirectQueryLoadingStatus.FAILED); + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + e.body?.message + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + // eslint-disable-next-line no-console + console.error(e); + }); + }; + + useEffect(() => { + // cancel direct query + if (!pollingResult) return; + const { status: anyCaseStatus, datarows, error } = pollingResult; + const status = anyCaseStatus?.toLowerCase(); + + if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { + setLoadStatus(status); + stopLoading(); + } else if (status === DirectQueryLoadingStatus.FAILED) { + setLoadStatus(status); + stopLoading(); + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + error + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + } else { + setLoadStatus(status); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [pollingResult, pollingError, stopLoading]); + + return { loadStatus, startLoading, stopLoading, pollingResult }; +}; diff --git a/src/plugins/query_enhancements/public/data_source_connection/components/index.ts b/src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx similarity index 61% rename from src/plugins/query_enhancements/public/data_source_connection/components/index.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx index 1ee969a1d079..88974a7c9420 100644 --- a/src/plugins/query_enhancements/public/data_source_connection/components/index.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx @@ -3,4 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export { ConnectionsBar } from './connections_bar'; +export * from './direct_query_hook'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx new file mode 100644 index 000000000000..771fbd6eef3a --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx @@ -0,0 +1,9 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './catalog_cache'; +export * from './hooks'; +export * from './requests'; +export * from './utils'; diff --git a/src/plugins/query_enhancements/public/data_source_connection/services/index.ts b/src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx similarity index 58% rename from src/plugins/query_enhancements/public/data_source_connection/services/index.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx index 08eeda5a7aa1..3918a896bd0b 100644 --- a/src/plugins/query_enhancements/public/data_source_connection/services/index.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx @@ -3,4 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export { ConnectionsService } from './connections_service'; +export * from './sql'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts b/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts new file mode 100644 index 000000000000..f2c9c30c79b9 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { HttpStart } from 'opensearch-dashboards/public'; +import { DirectQueryRequest } from '../types'; + +export class SQLService { + private http: HttpStart; + + constructor(http: HttpStart) { + this.http = http; + } + + fetch = async ( + params: DirectQueryRequest, + dataSourceMDSId?: string, + errorHandler?: (error: any) => void + ) => { + const query = { + dataSourceMDSId, + }; + return this.http + .post('/api/observability/query/jobs', { + body: JSON.stringify(params), + query, + }) + .catch((error) => { + // eslint-disable-next-line no-console + console.error('fetch error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; + + fetchWithJobId = async ( + params: { queryId: string }, + dataSourceMDSId?: string, + errorHandler?: (error: any) => void + ) => { + return this.http + .get(`/api/observability/query/jobs/${params.queryId}/${dataSourceMDSId ?? ''}`) + .catch((error) => { + // eslint-disable-next-line no-console + console.error('fetch error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; + + deleteWithJobId = async (params: { queryId: string }, errorHandler?: (error: any) => void) => { + return this.http.delete(`/api/observability/query/jobs/${params.queryId}`).catch((error) => { + // eslint-disable-next-line no-console + console.error('delete error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx new file mode 100644 index 000000000000..03d844ff5689 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx @@ -0,0 +1,330 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EuiComboBoxOptionOption } from '@elastic/eui'; +import { SimpleDataSet, SimpleObject } from '../../../../common'; + +export enum DirectQueryLoadingStatus { + SUCCESS = 'success', + FAILED = 'failed', + RUNNING = 'running', + SCHEDULED = 'scheduled', + CANCELED = 'canceled', + WAITING = 'waiting', + INITIAL = 'initial', +} + +export interface DirectQueryRequest { + query: string; + lang: string; + datasource: string; + sessionId?: string; +} + +export type AccelerationStatus = 'ACTIVE' | 'INACTIVE'; + +export interface PermissionsConfigurationProps { + roles: Role[]; + selectedRoles: Role[]; + setSelectedRoles: React.Dispatch>; + layout: 'horizontal' | 'vertical'; + hasSecurityAccess: boolean; +} + +export interface TableColumn { + name: string; + dataType: string; +} + +export interface Acceleration { + name: string; + status: AccelerationStatus; + type: string; + database: string; + table: string; + destination: string; + dateCreated: number; + dateUpdated: number; + index: string; + sql: string; +} + +export interface AssociatedObject { + tableName: string; + datasource: string; + id: string; + name: string; + database: string; + type: AssociatedObjectIndexType; + accelerations: CachedAcceleration[] | AssociatedObject; + columns?: CachedColumn[]; +} + +export type Role = EuiComboBoxOptionOption; + +export type DatasourceType = 'S3GLUE' | 'PROMETHEUS'; + +export interface S3GlueProperties { + 'glue.indexstore.opensearch.uri': string; + 'glue.indexstore.opensearch.region': string; +} + +export interface PrometheusProperties { + 'prometheus.uri': string; +} + +export type DatasourceStatus = 'ACTIVE' | 'DISABLED'; + +export interface DatasourceDetails { + allowedRoles: string[]; + name: string; + connector: DatasourceType; + description: string; + properties: S3GlueProperties | PrometheusProperties; + status: DatasourceStatus; +} + +interface AsyncApiDataResponse { + status: string; + schema?: Array<{ name: string; type: string }>; + datarows?: any; + total?: number; + size?: number; + error?: string; +} + +export interface AsyncApiResponse { + data: { + ok: boolean; + resp: AsyncApiDataResponse; + }; +} + +export type PollingCallback = (statusObj: AsyncApiResponse) => void; + +export type AssociatedObjectIndexType = AccelerationIndexType | 'table'; + +export type AccelerationIndexType = 'skipping' | 'covering' | 'materialized'; + +export type LoadCacheType = 'databases' | 'tables' | 'accelerations' | 'tableColumns'; + +export enum CachedDataSourceStatus { + Updated = 'Updated', + Failed = 'Failed', + Empty = 'Empty', +} + +export interface CachedColumn { + fieldName: string; + dataType: string; +} + +export interface CachedDatabase { + name: string; + tables: SimpleObject[]; + lastUpdated: string; // date string in UTC format + status: CachedDataSourceStatus; +} + +export interface CachedDataSource { + name: string; + lastUpdated: string; // date string in UTC format + status: CachedDataSourceStatus; + databases: CachedDatabase[]; + dataSourceMDSId?: string; +} + +export interface DataSourceCacheData { + version: string; + dataSources: CachedDataSource[]; +} + +export interface CachedAcceleration { + flintIndexName: string; + type: AccelerationIndexType; + database: string; + table: string; + indexName: string; + autoRefresh: boolean; + status: string; +} + +export interface CachedAccelerationByDataSource { + name: string; + accelerations: CachedAcceleration[]; + lastUpdated: string; // date string in UTC format + status: CachedDataSourceStatus; + dataSourceMDSId?: string; +} + +export interface AccelerationsCacheData { + version: string; + dataSources: CachedAccelerationByDataSource[]; +} + +export interface PollingSuccessResult { + schema: Array<{ name: string; type: string }>; + datarows: Array>; +} + +export type AsyncPollingResult = PollingSuccessResult | null; + +export type AggregationFunctionType = 'count' | 'sum' | 'avg' | 'max' | 'min' | 'window.start'; + +export interface MaterializedViewColumn { + id: string; + functionName: AggregationFunctionType; + functionParam?: string; + fieldAlias?: string; +} + +export type SkippingIndexAccMethodType = 'PARTITION' | 'VALUE_SET' | 'MIN_MAX' | 'BLOOM_FILTER'; + +export interface SkippingIndexRowType { + id: string; + fieldName: string; + dataType: string; + accelerationMethod: SkippingIndexAccMethodType; +} + +export interface DataTableFieldsType { + id: string; + fieldName: string; + dataType: string; +} + +export interface RefreshIntervalType { + refreshWindow: number; + refreshInterval: string; +} + +export interface WatermarkDelayType { + delayWindow: number; + delayInterval: string; +} + +export interface GroupByTumbleType { + timeField: string; + tumbleWindow: number; + tumbleInterval: string; +} + +export interface MaterializedViewQueryType { + columnsValues: MaterializedViewColumn[]; + groupByTumbleValue: GroupByTumbleType; +} + +export interface FormErrorsType { + dataSourceError: string[]; + databaseError: string[]; + dataTableError: string[]; + skippingIndexError: string[]; + coveringIndexError: string[]; + materializedViewError: string[]; + indexNameError: string[]; + primaryShardsError: string[]; + replicaShardsError: string[]; + refreshIntervalError: string[]; + checkpointLocationError: string[]; + watermarkDelayError: string[]; +} + +export type AccelerationRefreshType = 'autoInterval' | 'manual' | 'manualIncrement'; + +export interface CreateAccelerationForm { + dataSource: string; + database: string; + dataTable: string; + dataTableFields: DataTableFieldsType[]; + accelerationIndexType: AccelerationIndexType; + skippingIndexQueryData: SkippingIndexRowType[]; + coveringIndexQueryData: string[]; + materializedViewQueryData: MaterializedViewQueryType; + accelerationIndexName: string; + primaryShardsCount: number; + replicaShardsCount: number; + refreshType: AccelerationRefreshType; + checkpointLocation: string | undefined; + watermarkDelay: WatermarkDelayType; + refreshIntervalOptions: RefreshIntervalType; + formErrors: FormErrorsType; +} + +export interface LoadCachehookOutput { + loadStatus: DirectQueryLoadingStatus; + startLoading: (params: StartLoadingParams) => void; + stopLoading: () => void; +} + +export interface StartLoadingParams { + dataSourceName: string; + dataSourceMDSId?: string; + databaseName?: string; + tableName?: string; +} + +export interface RenderAccelerationFlyoutParams { + dataSourceName: string; + dataSourceMDSId?: string; + databaseName?: string; + tableName?: string; + handleRefresh?: () => void; +} + +export interface RenderAssociatedObjectsDetailsFlyoutParams { + tableDetail: AssociatedObject; + dataSourceName: string; + handleRefresh?: () => void; + dataSourceMDSId?: string; +} + +export interface RenderAccelerationDetailsFlyoutParams { + acceleration: CachedAcceleration; + dataSourceName: string; + handleRefresh?: () => void; + dataSourceMDSId?: string; +} + +export interface DataSetOption { + id?: string; + name: string; + dataSourceRef?: string; +} + +export interface RecentDataSetOptionsCacheData { + version: string; + recentDataSets: SimpleDataSet[]; +} + +export interface ExternalDataSource { + name: string; + status: string; + dataSourceRef: string; +} + +export interface ExternalDataSourcesCacheData { + version: string; + externalDataSources: ExternalDataSource[]; + lastUpdated: string; + status: CachedDataSourceStatus; +} + +interface DataSourceMeta { + // ref: string; // MDS ID + // dsName?: string; // flint datasource + id: string; + name: string; + type?: string; +} + +export interface DataSet { + id: string | undefined; // index pattern ID, index name, or flintdatasource.database.table + datasource?: DataSourceMeta; + meta?: { + timestampField: string; + mapping?: any; + }; + type?: 'dataSet' | 'temporary'; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts new file mode 100644 index 000000000000..697852fdd772 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export enum DirectQueryLoadingStatus { + SUCCESS = 'success', + FAILED = 'failed', + RUNNING = 'running', + SCHEDULED = 'scheduled', + CANCELED = 'canceled', + WAITING = 'waiting', + INITIAL = 'initial', +} + +const catalogCacheFetchingStatus = [ + DirectQueryLoadingStatus.RUNNING, + DirectQueryLoadingStatus.WAITING, + DirectQueryLoadingStatus.SCHEDULED, +]; + +export const isCatalogCacheFetching = (...statuses: DirectQueryLoadingStatus[]) => { + return statuses.some((status: DirectQueryLoadingStatus) => + catalogCacheFetchingStatus.includes(status) + ); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts new file mode 100644 index 000000000000..7a10d7badb58 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { SimpleDataSource } from '../../../../../common'; + +export const fetchDataSources = async (client: SavedObjectsClientContract) => { + const resp = await client.find({ + type: 'data-source', + perPage: 10000, + }); + return resp.savedObjects.map((savedObject) => ({ + id: savedObject.id, + name: savedObject.attributes.title, + type: 'data-source', + })) as SimpleDataSource[]; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts new file mode 100644 index 000000000000..a9272155e602 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { HttpStart } from 'opensearch-dashboards/public'; + +export const fetchIfExternalDataSourcesEnabled = async (http: HttpStart) => { + try { + await http.get('/api/dataconnections'); + return true; + } catch (e) { + return false; + } +}; + +export const fetchExternalDataSources = async (http: HttpStart, connectedClusters: string[]) => { + const results = await Promise.all( + connectedClusters.map(async (cluster) => { + const dataSources = await http.get(`/api/dataconnections/dataSourceMDSId=${cluster}`); + return dataSources + .filter((dataSource) => dataSource.connector === 'S3GLUE') + .map((dataSource) => ({ + name: dataSource.name, + status: dataSource.status, + dataSourceRef: cluster, + })); + }) + ); + + const flattenedResults = results.flat(); + return flattenedResults; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts new file mode 100644 index 000000000000..3f2cd230300e --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { IIndexPattern } from '../.././../..'; +import { SIMPLE_DATA_SOURCE_TYPES, SIMPLE_DATA_SET_TYPES } from '../../../../../common'; + +export const fetchIndexPatterns = async (client: SavedObjectsClientContract, search: string) => { + const resp = await client.find({ + type: 'index-pattern', + fields: ['title', 'timeFieldName', 'references', 'fields'], + search: `${search}*`, + searchFields: ['title'], + perPage: 100, + }); + return resp.savedObjects.map((savedObject) => ({ + id: savedObject.id, + title: savedObject.attributes.title, + timeFieldName: savedObject.attributes.timeFieldName, + fields: savedObject.attributes.fields, + type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, + ...(savedObject.references[0] + ? { + dataSourceRef: { + id: savedObject.references[0]?.id, + name: savedObject.references[0]?.name, + type: SIMPLE_DATA_SOURCE_TYPES.DEFAULT, + }, + } + : {}), + })); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts new file mode 100644 index 000000000000..ef10c72bc08c --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { map } from 'rxjs/operators'; +import { ISearchStart } from '../../../../search'; + +export const fetchIndices = async (search: ISearchStart, dataSourceId?: string) => { + const buildSearchRequest = () => { + const request = { + params: { + ignoreUnavailable: true, + expand_wildcards: 'all', + index: '*', + body: { + size: 0, // no hits + aggs: { + indices: { + terms: { + field: '_index', + size: 100, + }, + }, + }, + }, + }, + dataSourceId, + }; + + return request; + }; + + const searchResponseToArray = (response: any) => { + const { rawResponse } = response; + return rawResponse.aggregations + ? rawResponse.aggregations.indices.buckets.map((bucket: { key: any }) => bucket.key) + : []; + }; + + return search + .getDefaultSearchInterceptor() + .search(buildSearchRequest()) + .pipe(map(searchResponseToArray)) + .toPromise(); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts new file mode 100644 index 000000000000..7dbe7ec2d4f4 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './fetch_catalog_cache_status'; +export * from './fetch_data_sources'; +export * from './fetch_external_data_sources'; +export * from './fetch_index_patterns'; +export * from './fetch_indices'; +export * from './query_session_utils'; +export * from './shared'; +export * from './use_polling'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts new file mode 100644 index 000000000000..beabcb48c197 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { ASYNC_QUERY_SESSION_ID } from '../constants'; + +export const setAsyncSessionId = (dataSource: string, value: string | null) => { + if (value !== null) { + sessionStorage.setItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`, value); + } +}; + +export const getAsyncSessionId = (dataSource: string) => { + return sessionStorage.getItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts new file mode 100644 index 000000000000..3e4afc94e80b --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts @@ -0,0 +1,332 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * TODO making this method type-safe is nontrivial: if you just define + * `Nested = { [k: string]: Nested | T }` then you can't accumulate because `T` is not `Nested` + * There might be a way to define a recursive type that accumulates cleanly but it's probably not + * worth the effort. + */ + +export function get(obj: Record, path: string, defaultValue?: T): T { + return path.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || defaultValue; +} + +export function addBackticksIfNeeded(input: string): string { + if (input === undefined) { + return ''; + } + // Check if the string already has backticks + if (input.startsWith('`') && input.endsWith('`')) { + return input; // Return the string as it is + } else { + // Add backticks to the string + return '`' + input + '`'; + } +} + +export function combineSchemaAndDatarows( + schema: Array<{ name: string; type: string }>, + datarows: Array> +): object[] { + const combinedData: object[] = []; + + datarows.forEach((row) => { + const rowData: { [key: string]: string | number | boolean } = {}; + schema.forEach((field, index) => { + rowData[field.name] = row[index]; + }); + combinedData.push(rowData); + }); + + return combinedData; +} + +export const formatError = (name: string, message: string, details: string) => { + return { + name, + message, + body: { + attributes: { + error: { + caused_by: { + type: '', + reason: details, + }, + }, + }, + }, + }; +}; + +// TODO: relocate to a more appropriate location +// Client route +export const PPL_BASE = '/api/ppl'; +export const PPL_SEARCH = '/search'; +export const DSL_BASE = '/api/dsl'; +export const DSL_SEARCH = '/search'; +export const DSL_CAT = '/cat.indices'; +export const DSL_MAPPING = '/indices.getFieldMapping'; +export const DSL_SETTINGS = '/indices.getFieldSettings'; +export const OBSERVABILITY_BASE = '/api/observability'; +export const INTEGRATIONS_BASE = '/api/integrations'; +export const JOBS_BASE = '/query/jobs'; +export const DATACONNECTIONS_BASE = '/api/dataconnections'; +export const EDIT = '/edit'; +export const DATACONNECTIONS_UPDATE_STATUS = '/status'; +export const SECURITY_ROLES = '/api/v1/configuration/roles'; +export const EVENT_ANALYTICS = '/event_analytics'; +export const SAVED_OBJECTS = '/saved_objects'; +export const SAVED_QUERY = '/query'; +export const SAVED_VISUALIZATION = '/vis'; +export const CONSOLE_PROXY = '/api/console/proxy'; +export const SECURITY_PLUGIN_ACCOUNT_API = '/api/v1/configuration/account'; + +// Server route +export const PPL_ENDPOINT = '/_plugins/_ppl'; +export const SQL_ENDPOINT = '/_plugins/_sql'; +export const DSL_ENDPOINT = '/_plugins/_dsl'; +export const DATACONNECTIONS_ENDPOINT = '/_plugins/_query/_datasources'; +export const JOBS_ENDPOINT_BASE = '/_plugins/_async_query'; +export const JOB_RESULT_ENDPOINT = '/result'; + +export const observabilityID = 'observability-logs'; +export const observabilityTitle = 'Observability'; +export const observabilityPluginOrder = 1500; + +export const observabilityApplicationsID = 'observability-applications'; +export const observabilityApplicationsTitle = 'Applications'; +export const observabilityApplicationsPluginOrder = 5090; + +export const observabilityLogsID = 'observability-logs'; +export const observabilityLogsTitle = 'Logs'; +export const observabilityLogsPluginOrder = 5091; + +export const observabilityMetricsID = 'observability-metrics'; +export const observabilityMetricsTitle = 'Metrics'; +export const observabilityMetricsPluginOrder = 5092; + +export const observabilityTracesID = 'observability-traces'; +export const observabilityTracesTitle = 'Traces'; +export const observabilityTracesPluginOrder = 5093; + +export const observabilityNotebookID = 'observability-notebooks'; +export const observabilityNotebookTitle = 'Notebooks'; +export const observabilityNotebookPluginOrder = 5094; + +export const observabilityPanelsID = 'observability-dashboards'; +export const observabilityPanelsTitle = 'Dashboards'; +export const observabilityPanelsPluginOrder = 5095; + +export const observabilityIntegrationsID = 'integrations'; +export const observabilityIntegrationsTitle = 'Integrations'; +export const observabilityIntegrationsPluginOrder = 9020; + +export const observabilityDataConnectionsID = 'datasources'; +export const observabilityDataConnectionsTitle = 'Data sources'; +export const observabilityDataConnectionsPluginOrder = 9030; + +export const queryWorkbenchPluginID = 'opensearch-query-workbench'; +export const queryWorkbenchPluginCheck = 'plugin:queryWorkbenchDashboards'; + +// Shared Constants +export const SQL_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/search-plugins/sql/index/'; +export const PPL_DOCUMENTATION_URL = + 'https://opensearch.org/docs/latest/search-plugins/sql/ppl/index'; +export const PPL_PATTERNS_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/sql/blob/2.x/docs/user/ppl/cmd/patterns.rst#description'; +export const UI_DATE_FORMAT = 'MM/DD/YYYY hh:mm A'; +export const PPL_DATE_FORMAT = 'YYYY-MM-DD HH:mm:ss.SSSSSS'; +export const OTEL_DATE_FORMAT = 'YYYY-MM-DDTHH:mm:ss'; +export const SPAN_REGEX = /span/; + +export const PROMQL_METRIC_SUBTYPE = 'promqlmetric'; +export const OTEL_METRIC_SUBTYPE = 'openTelemetryMetric'; +export const PPL_METRIC_SUBTYPE = 'metric'; + +export const PPL_SPAN_REGEX = /by\s*span/i; +export const PPL_STATS_REGEX = /\|\s*stats/i; +export const PPL_INDEX_INSERT_POINT_REGEX = /(search source|source|index)\s*=\s*([^|\s]+)(.*)/i; +export const PPL_INDEX_REGEX = /(search source|source|index)\s*=\s*([^|\s]+)/i; +export const PPL_WHERE_CLAUSE_REGEX = /\s*where\s+/i; +export const PPL_NEWLINE_REGEX = /[\n\r]+/g; +export const PPL_DESCRIBE_INDEX_REGEX = /(describe)\s+([^|\s]+)/i; + +// Observability plugin URI +const BASE_OBSERVABILITY_URI = '/_plugins/_observability'; +const BASE_DATACONNECTIONS_URI = '/_plugins/_query/_datasources'; +export const OPENSEARCH_PANELS_API = { + OBJECT: `${BASE_OBSERVABILITY_URI}/object`, +}; +export const OPENSEARCH_DATACONNECTIONS_API = { + DATACONNECTION: `${BASE_DATACONNECTIONS_URI}`, +}; + +// Saved Objects +export const SAVED_OBJECT = '/object'; + +// Color Constants +export const PLOTLY_COLOR = [ + '#3CA1C7', + '#54B399', + '#DB748A', + '#F2BE4B', + '#68CCC2', + '#2A7866', + '#843769', + '#374FB8', + '#BD6F26', + '#4C636F', +]; + +export const LONG_CHART_COLOR = PLOTLY_COLOR[1]; + +export const pageStyles: CSS.Properties = { + float: 'left', + width: '100%', + maxWidth: '1130px', +}; + +export enum VIS_CHART_TYPES { + Bar = 'bar', + HorizontalBar = 'horizontal_bar', + Line = 'line', + Pie = 'pie', + HeatMap = 'heatmap', + Text = 'text', + Histogram = 'histogram', +} + +export const NUMERICAL_FIELDS = ['short', 'integer', 'long', 'float', 'double']; + +export const ENABLED_VIS_TYPES = [ + VIS_CHART_TYPES.Bar, + VIS_CHART_TYPES.HorizontalBar, + VIS_CHART_TYPES.Line, + VIS_CHART_TYPES.Pie, + VIS_CHART_TYPES.HeatMap, + VIS_CHART_TYPES.Text, +]; + +// Live tail constants +export const LIVE_OPTIONS = [ + { + label: '5s', + startTime: 'now-5s', + delayTime: 5000, + }, + { + label: '10s', + startTime: 'now-10s', + delayTime: 10000, + }, + { + label: '30s', + startTime: 'now-30s', + delayTime: 30000, + }, + { + label: '1m', + startTime: 'now-1m', + delayTime: 60000, + }, + { + label: '5m', + startTime: 'now-5m', + delayTime: 60000 * 5, + }, + { + label: '15m', + startTime: 'now-15m', + delayTime: 60000 * 15, + }, + { + label: '30m', + startTime: 'now-30m', + delayTime: 60000 * 30, + }, + { + label: '1h', + startTime: 'now-1h', + delayTime: 60000 * 60, + }, + { + label: '2h', + startTime: 'now-2h', + delayTime: 60000 * 120, + }, +]; + +export const LIVE_END_TIME = 'now'; + +export interface DefaultChartStylesProps { + DefaultModeLine: string; + Interpolation: string; + LineWidth: number; + FillOpacity: number; + MarkerSize: number; + ShowLegend: string; + LegendPosition: string; + LabelAngle: number; + DefaultSortSectors: string; + DefaultModeScatter: string; +} + +export const DEFAULT_CHART_STYLES: DefaultChartStylesProps = { + DefaultModeLine: 'lines+markers', + Interpolation: 'spline', + LineWidth: 0, + FillOpacity: 100, + MarkerSize: 25, + ShowLegend: 'show', + LegendPosition: 'v', + LabelAngle: 0, + DefaultSortSectors: 'largest_to_smallest', + DefaultModeScatter: 'markers', +}; + +export const FILLOPACITY_DIV_FACTOR = 200; +export const SLIDER_MIN_VALUE = 0; +export const SLIDER_MAX_VALUE = 100; +export const SLIDER_STEP = 1; +export const THRESHOLD_LINE_WIDTH = 3; +export const THRESHOLD_LINE_OPACITY = 0.7; +export const MAX_BUCKET_LENGTH = 16; + +export enum BarOrientation { + horizontal = 'h', + vertical = 'v', +} + +export const PLOT_MARGIN = { + l: 30, + r: 5, + b: 30, + t: 50, + pad: 4, +}; + +export const WAITING_TIME_ON_USER_ACTIONS = 300; + +export const VISUALIZATION_ERROR = { + NO_DATA: 'No data found.', + INVALID_DATA: 'Invalid visualization data', + NO_SERIES: 'Add a field to start', + NO_METRIC: 'Invalid Metric MetaData', +}; + +export const S3_DATA_SOURCE_TYPE = 's3glue'; + +export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; +export const ASYNC_QUERY_DATASOURCE_CACHE = 'async-query-catalog-cache'; +export const ASYNC_QUERY_ACCELERATIONS_CACHE = 'async-query-acclerations-cache'; + +export const DIRECT_DUMMY_QUERY = 'select 1'; + +export const DEFAULT_START_TIME = 'now-15m'; +export const QUERY_ASSIST_START_TIME = 'now-40y'; +export const QUERY_ASSIST_END_TIME = 'now'; + +export const TIMESTAMP_DATETIME_TYPES = ['date', 'date_nanos']; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts new file mode 100644 index 000000000000..74fedd6cf110 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts @@ -0,0 +1,137 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useEffect, useRef, useState } from 'react'; + +type FetchFunction = (params?: P) => Promise; + +export interface PollingConfigurations { + tabId: string; +} + +export class UsePolling { + public data: T | null = null; + public error: Error | null = null; + public loading: boolean = true; + private shouldPoll: boolean = false; + private intervalRef?: NodeJS.Timeout; + + constructor( + private fetchFunction: FetchFunction, + private interval: number = 5000, + private onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, + private onPollingError?: (error: Error) => boolean, + private configurations?: PollingConfigurations + ) {} + + async fetchData(params?: P) { + this.loading = true; + try { + const result = await this.fetchFunction(params); + this.data = result; + this.loading = false; + + if (this.onPollingSuccess && this.onPollingSuccess(result, this.configurations!)) { + this.stopPolling(); + } + } catch (err) { + this.error = err as Error; + this.loading = false; + + if (this.onPollingError && this.onPollingError(this.error)) { + this.stopPolling(); + } + } + } + + startPolling(params?: P) { + this.shouldPoll = true; + if (!this.intervalRef) { + this.intervalRef = setInterval(() => { + if (this.shouldPoll) { + this.fetchData(params); + } + }, this.interval); + } + } + + stopPolling() { + this.shouldPoll = false; + if (this.intervalRef) { + clearInterval(this.intervalRef); + this.intervalRef = undefined; + } + } +} + +interface UsePollingReturn { + data: T | null; + loading: boolean; + error: Error | null; + startPolling: (params?: any) => void; + stopPolling: () => void; +} + +export function usePolling( + fetchFunction: FetchFunction, + interval: number = 5000, + onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, + onPollingError?: (error: Error) => boolean, + configurations?: PollingConfigurations +): UsePollingReturn { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(true); + const intervalRef = useRef(undefined); + const unmounted = useRef(false); + + const shouldPoll = useRef(false); + + const startPolling = (params?: P) => { + shouldPoll.current = true; + const intervalId = setInterval(() => { + if (shouldPoll.current) { + fetchData(params); + } + }, interval); + intervalRef.current = intervalId; + if (unmounted.current) { + clearInterval(intervalId); + } + }; + + const stopPolling = () => { + shouldPoll.current = false; + clearInterval(intervalRef.current); + }; + + const fetchData = async (params?: P) => { + try { + const result = await fetchFunction(params); + setData(result); + // Check the success condition and stop polling if it's met + if (onPollingSuccess && onPollingSuccess(result, configurations)) { + stopPolling(); + } + } catch (err: unknown) { + setError(err as Error); + + // Check the error condition and stop polling if it's met + if (onPollingError && onPollingError(err as Error)) { + stopPolling(); + } + } finally { + setLoading(false); + } + }; + + useEffect(() => { + return () => { + unmounted.current = true; + }; + }, []); + + return { data, loading, error, startPolling, stopPolling }; +} diff --git a/src/plugins/data/public/ui/index.ts b/src/plugins/data/public/ui/index.ts index 5483b540d5bf..400887e51d57 100644 --- a/src/plugins/data/public/ui/index.ts +++ b/src/plugins/data/public/ui/index.ts @@ -49,3 +49,4 @@ export { } from './query_editor'; export { SearchBar, SearchBarProps, StatefulSearchBarProps } from './search_bar'; export { SuggestionsComponent } from './typeahead'; +export { DataSetNavigator } from './dataset_navigator'; diff --git a/src/plugins/data/public/ui/query_editor/_language_switcher.scss b/src/plugins/data/public/ui/query_editor/_language_switcher.scss deleted file mode 100644 index 176d072c102b..000000000000 --- a/src/plugins/data/public/ui/query_editor/_language_switcher.scss +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -.languageSelect { - max-width: 150px; - transform: translateY(-1px) translateX(-0.5px); -} diff --git a/src/plugins/data/public/ui/query_editor/_query_editor.scss b/src/plugins/data/public/ui/query_editor/_query_editor.scss index 8fc81308b533..ac411b38ab88 100644 --- a/src/plugins/data/public/ui/query_editor/_query_editor.scss +++ b/src/plugins/data/public/ui/query_editor/_query_editor.scss @@ -86,6 +86,12 @@ } } +.osdQueryEditor__dataSetNavigatorWrapper { + :first-child { + border-bottom: $euiBorderThin !important; + } +} + @include euiBreakpoint("xs", "s") { .osdQueryEditor--withDatePicker { > :first-child { diff --git a/src/plugins/data/public/ui/query_editor/language_selector.test.tsx b/src/plugins/data/public/ui/query_editor/language_selector.test.tsx index f61134211a40..62c4ebea288f 100644 --- a/src/plugins/data/public/ui/query_editor/language_selector.test.tsx +++ b/src/plugins/data/public/ui/query_editor/language_selector.test.tsx @@ -8,7 +8,6 @@ import { QueryLanguageSelector } from './language_selector'; import { OpenSearchDashboardsContextProvider } from 'src/plugins/opensearch_dashboards_react/public'; import { coreMock } from '../../../../../core/public/mocks'; import { mountWithIntl } from 'test_utils/enzyme_helpers'; -import { EuiCompressedComboBox } from '@elastic/eui'; import { QueryEnhancement } from '../types'; const startMock = coreMock.createStart(); diff --git a/src/plugins/data/public/ui/query_editor/language_switcher.tsx b/src/plugins/data/public/ui/query_editor/language_switcher.tsx deleted file mode 100644 index be22ebffd775..000000000000 --- a/src/plugins/data/public/ui/query_editor/language_switcher.tsx +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { EuiComboBox, EuiComboBoxOptionOption, PopoverAnchorPosition } from '@elastic/eui'; -import { i18n } from '@osd/i18n'; -import React from 'react'; -import { getSearchService, getUiService } from '../../services'; - -interface Props { - language: string; - onSelectLanguage: (newLanguage: string) => void; - anchorPosition?: PopoverAnchorPosition; - appName?: string; -} - -function mapExternalLanguageToOptions(language: string) { - return { - label: language, - value: language, - }; -} - -export function QueryLanguageSwitcher(props: Props) { - const dqlLabel = i18n.translate('data.query.queryBar.dqlLanguageName', { - defaultMessage: 'DQL', - }); - const luceneLabel = i18n.translate('data.query.queryBar.luceneLanguageName', { - defaultMessage: 'Lucene', - }); - - const languageOptions: EuiComboBoxOptionOption[] = [ - { - label: dqlLabel, - value: 'kuery', - }, - { - label: luceneLabel, - value: 'lucene', - }, - ]; - - const uiService = getUiService(); - const searchService = getSearchService(); - - const queryEnhancements = uiService.queryEnhancements; - if (uiService.isEnhancementsEnabled) { - queryEnhancements.forEach((enhancement) => { - if ( - enhancement.supportedAppNames && - props.appName && - !enhancement.supportedAppNames.includes(props.appName) - ) - return; - languageOptions.push(mapExternalLanguageToOptions(enhancement.language)); - }); - } - - const selectedLanguage = { - label: - (languageOptions.find( - (option) => (option.value as string).toLowerCase() === props.language.toLowerCase() - )?.label as string) ?? languageOptions[0].label, - }; - - const setSearchEnhance = (queryLanguage: string) => { - if (!uiService.isEnhancementsEnabled) return; - const queryEnhancement = queryEnhancements.get(queryLanguage); - searchService.__enhance({ - searchInterceptor: queryEnhancement - ? queryEnhancement.search - : searchService.getDefaultSearchInterceptor(), - }); - - if (!queryEnhancement) { - searchService.df.clear(); - } - uiService.Settings.setUiOverridesByUserQueryLanguage(queryLanguage); - }; - - const handleLanguageChange = (newLanguage: EuiComboBoxOptionOption[]) => { - const queryLanguage = newLanguage[0].value as string; - props.onSelectLanguage(queryLanguage); - setSearchEnhance(queryLanguage); - }; - - setSearchEnhance(props.language); - - return ( - - ); -} diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index 44d000de1e8f..db4984b637d4 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -3,14 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { - EuiFlexGroup, - EuiFlexItem, - EuiForm, - EuiFormRow, - htmlIdGenerator, - PopoverAnchorPosition, -} from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, htmlIdGenerator, PopoverAnchorPosition } from '@elastic/eui'; import classNames from 'classnames'; import { isEqual } from 'lodash'; import React, { Component, createRef, RefObject } from 'react'; @@ -44,10 +37,7 @@ export interface QueryEditorProps { indexPatterns: Array; dataSource?: DataSource; query: Query; - container?: HTMLDivElement; - dataSourceContainerRef?: React.RefCallback; - containerRef?: React.RefCallback; - languageSelectorContainerRef?: React.RefCallback; + dataSetContainerRef?: React.RefCallback; settings: Settings; disableAutoFocus?: boolean; screenTitle?: string; @@ -60,7 +50,7 @@ export interface QueryEditorProps { onChange?: (query: Query, dateRange?: TimeRange) => void; onChangeQueryEditorFocus?: (isFocused: boolean) => void; onSubmit?: (query: Query, dateRange?: TimeRange) => void; - getQueryStringInitialValue?: (language: string) => string; + getQueryStringInitialValue?: (language: string, dataSetName?: string) => string; dataTestSubj?: string; size?: SuggestionsListSize; className?: string; @@ -77,8 +67,6 @@ interface Props extends QueryEditorProps { } interface State { - isDataSourcesVisible: boolean; - isDataSetsVisible: boolean; isSuggestionsVisible: boolean; index: number | null; suggestions: QuerySuggestion[]; @@ -105,8 +93,6 @@ const KEY_CODES = { // eslint-disable-next-line import/no-default-export export default class QueryEditorUI extends Component { public state: State = { - isDataSourcesVisible: false, - isDataSetsVisible: true, isSuggestionsVisible: false, index: null, suggestions: [], @@ -121,7 +107,6 @@ export default class QueryEditorUI extends Component { private persistedLog: PersistedLog | undefined; private abortController?: AbortController; private services = this.props.opensearchDashboards.services; - private componentIsUnmounting = false; private headerRef: RefObject = createRef(); private bannerRef: RefObject = createRef(); private extensionMap = this.props.settings?.getQueryEditorExtensionMap(); @@ -250,10 +235,6 @@ export default class QueryEditorUI extends Component { : undefined; this.onChange(newQuery, dateRange); this.onSubmit(newQuery, dateRange); - this.setState({ - isDataSourcesVisible: enhancement?.searchBar?.showDataSourcesSelector ?? true, - isDataSetsVisible: enhancement?.searchBar?.showDataSetsSelector ?? true, - }); }; private initPersistedLog = () => { @@ -263,20 +244,6 @@ export default class QueryEditorUI extends Component { : getQueryLog(uiSettings, storage, appName, this.props.query.language); }; - private initDataSourcesVisibility = () => { - if (this.componentIsUnmounting) return; - - return this.props.settings.getQueryEnhancements(this.props.query.language)?.searchBar - ?.showDataSourcesSelector; - }; - - private initDataSetsVisibility = () => { - if (this.componentIsUnmounting) return; - - return this.props.settings.getQueryEnhancements(this.props.query.language)?.searchBar - ?.showDataSetsSelector; - }; - public onMouseEnterSuggestion = (index: number) => { this.setState({ index }); }; @@ -291,10 +258,6 @@ export default class QueryEditorUI extends Component { this.initPersistedLog(); // this.fetchIndexPatterns().then(this.updateSuggestions); - this.setState({ - isDataSourcesVisible: this.initDataSourcesVisibility() || true, - isDataSetsVisible: this.initDataSetsVisibility() || true, - }); } public componentDidUpdate(prevProps: Props) { @@ -308,7 +271,6 @@ export default class QueryEditorUI extends Component { public componentWillUnmount() { if (this.abortController) this.abortController.abort(); - this.componentIsUnmounting = true; } handleOnFocus = () => { @@ -431,6 +393,15 @@ export default class QueryEditorUI extends Component { const useQueryEditor = this.props.query.language !== 'kuery' && this.props.query.language !== 'lucene'; + const languageSelector = ( + + ); + return (
@@ -443,17 +414,9 @@ export default class QueryEditorUI extends Component { isCollapsed={!this.state.isCollapsed} /> - {this.state.isDataSourcesVisible && ( - -
- - )} - - {this.state.isDataSetsVisible && ( - -
- - )} + +
+ {(this.state.isCollapsed || !useQueryEditor) && ( @@ -496,14 +459,7 @@ export default class QueryEditorUI extends Component { )} {!useQueryEditor && ( -
- -
+
{languageSelector}
 )} @@ -557,15 +513,7 @@ export default class QueryEditorUI extends Component { } > - - - + {languageSelector} {this.state.lineCount} {this.state.lineCount === 1 ? 'line' : 'lines'} diff --git a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx index a482d7416418..971d13cfc050 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx @@ -39,8 +39,7 @@ const QueryEditor = withOpenSearchDashboards(QueryEditorUI); // @internal export interface QueryEditorTopRowProps { query?: Query; - dataSourceContainerRef?: React.RefCallback; - containerRef?: React.RefCallback; + dataSetContainerRef?: React.RefCallback; settings?: Settings; onSubmit: (payload: { dateRange: TimeRange; query?: Query }) => void; onChange: (payload: { dateRange: TimeRange; query?: Query }) => void; @@ -208,11 +207,10 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { ) return ''; - const defaultDataSource = indexPatterns[0]; - const dataSource = - typeof defaultDataSource === 'string' ? defaultDataSource : defaultDataSource.title; + const defaultDataSet = indexPatterns[0]; + const dataSet = typeof defaultDataSet === 'string' ? defaultDataSet : defaultDataSet.title; - return input.replace('', dataSource); + return input.replace('', dataSet); } function renderQueryEditor() { @@ -225,8 +223,7 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { dataSource={props.dataSource} prepend={props.prepend} query={parsedQuery} - dataSourceContainerRef={props.dataSourceContainerRef} - containerRef={props.containerRef} + dataSetContainerRef={props.dataSetContainerRef} settings={props.settings!} screenTitle={props.screenTitle} onChange={onQueryChange} diff --git a/src/plugins/data/public/ui/search_bar/create_search_bar.tsx b/src/plugins/data/public/ui/search_bar/create_search_bar.tsx index 31f3401dc76f..d722aeda510a 100644 --- a/src/plugins/data/public/ui/search_bar/create_search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/create_search_bar.tsx @@ -48,8 +48,7 @@ interface StatefulSearchBarDeps { data: Omit; storage: IStorageWrapper; settings: Settings; - setDataSourceContainerRef: (ref: HTMLDivElement | null) => void; - setContainerRef: (ref: HTMLDivElement | null) => void; + setDataSetContainerRef: (ref: HTMLDivElement | null) => void; } export type StatefulSearchBarProps = SearchBarOwnProps & { @@ -139,8 +138,7 @@ export function createSearchBar({ storage, data, settings, - setDataSourceContainerRef, - setContainerRef, + setDataSetContainerRef, }: StatefulSearchBarDeps) { // App name should come from the core application service. // Until it's available, we'll ask the user to provide it for the pre-wired component. @@ -176,15 +174,9 @@ export function createSearchBar({ notifications: core.notifications, }); - const dataSourceContainerRef = useCallback((node) => { + const dataSetContainerRef = useCallback((node) => { if (node) { - setDataSourceContainerRef(node); - } - }, []); - - const containerRef = useCallback((node) => { - if (node) { - setContainerRef(node); + setDataSetContainerRef(node); } }, []); @@ -228,8 +220,7 @@ export function createSearchBar({ filters={filters} query={query} settings={settings} - dataSourceContainerRef={dataSourceContainerRef} - containerRef={containerRef} + dataSetContainerRef={dataSetContainerRef} onFiltersUpdated={defaultFiltersUpdated(data.query)} onRefreshChange={defaultOnRefreshChange(data.query)} savedQuery={savedQuery} diff --git a/src/plugins/data/public/ui/search_bar/lib/use_dataset_manager.ts b/src/plugins/data/public/ui/search_bar/lib/use_dataset_manager.ts new file mode 100644 index 000000000000..7a92d03e9f33 --- /dev/null +++ b/src/plugins/data/public/ui/search_bar/lib/use_dataset_manager.ts @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useState, useEffect } from 'react'; +import { Subscription } from 'rxjs'; +import { SimpleDataSet } from '../../../../../data/common'; +import { DataSetContract } from '../../../query'; + +interface UseDataSetManagerProps { + dataSet?: SimpleDataSet; + dataSetManager: DataSetContract; +} + +export const useDataSetManager = (props: UseDataSetManagerProps) => { + const [dataSet, setDataSet] = useState( + props.dataSet || props.dataSetManager.getDataSet() + ); + + useEffect(() => { + const subscriptions = new Subscription(); + + subscriptions.add( + props.dataSetManager.getUpdates$().subscribe({ + next: () => { + const newDataSet = props.dataSetManager.getDataSet(); + setDataSet(newDataSet); + }, + }) + ); + + return () => { + subscriptions.unsubscribe(); + }; + }, [dataSet, props.dataSet, props.dataSetManager]); + + return { dataSet }; +}; diff --git a/src/plugins/data/public/ui/search_bar/search_bar.tsx b/src/plugins/data/public/ui/search_bar/search_bar.tsx index b2ff6766e81c..4dddba69ff91 100644 --- a/src/plugins/data/public/ui/search_bar/search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/search_bar.tsx @@ -81,8 +81,7 @@ export interface SearchBarOwnProps { // Query bar - should be in SearchBarInjectedDeps query?: Query; settings?: Settings; - dataSourceContainerRef?: React.RefCallback; - containerRef?: React.RefCallback; + dataSetContainerRef?: React.RefCallback; // Show when user has privileges to save showSaveQuery?: boolean; savedQuery?: SavedQuery; @@ -493,8 +492,7 @@ class SearchBarUI extends Component { queryEditor = ( ; + /** + * @experimental - Subject to change + */ + DataSetNavigator: React.ComponentType; SearchBar: React.ComponentType; SuggestionsComponent: React.ComponentType; + /** + * @experimental - Subject to change + */ Settings: Settings; - dataSourceContainer$: Observable; - container$: Observable; + dataSetContainer$: Observable; } diff --git a/src/plugins/data/public/ui/ui_service.ts b/src/plugins/data/public/ui/ui_service.ts index 1e0e6be8b78c..4f403597467b 100644 --- a/src/plugins/data/public/ui/ui_service.ts +++ b/src/plugins/data/public/ui/ui_service.ts @@ -8,6 +8,7 @@ import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from 'src/core import { IStorageWrapper } from '../../../opensearch_dashboards_utils/public'; import { ConfigSchema } from '../../config'; import { DataPublicPluginStart } from '../types'; +import { createDataSetNavigator } from './dataset_navigator'; import { createIndexPatternSelect } from './index_pattern_select'; import { QueryEditorExtensionConfig } from './query_editor'; import { createSearchBar } from './search_bar/create_search_bar'; @@ -29,8 +30,7 @@ export class UiService implements Plugin { enhancementsConfig: ConfigSchema['enhancements']; private queryEnhancements: Map = new Map(); private queryEditorExtensionMap: Record = {}; - private dataSourceContainer$ = new BehaviorSubject(null); - private container$ = new BehaviorSubject(null); + private dataSetContainer$ = new BehaviorSubject(null); constructor(initializerContext: PluginInitializerContext) { const { enhancements } = initializerContext.config.get(); @@ -62,12 +62,8 @@ export class UiService implements Plugin { queryEditorExtensionMap: this.queryEditorExtensionMap, }); - const setDataSourceContainerRef = (ref: HTMLDivElement | null) => { - this.dataSourceContainer$.next(ref); - }; - - const setContainerRef = (ref: HTMLDivElement | null) => { - this.container$.next(ref); + const setDataSetContainerRef = (ref: HTMLDivElement | null) => { + this.dataSetContainer$.next(ref); }; const SearchBar = createSearchBar({ @@ -75,17 +71,20 @@ export class UiService implements Plugin { data: dataServices, storage, settings: Settings, - setDataSourceContainerRef, - setContainerRef, + setDataSetContainerRef, }); return { IndexPatternSelect: createIndexPatternSelect(core.savedObjects.client), + DataSetNavigator: createDataSetNavigator( + core.savedObjects.client, + core.http, + dataServices.query.dataSet + ), SearchBar, SuggestionsComponent, Settings, - dataSourceContainer$: this.dataSourceContainer$, - container$: this.container$, + dataSetContainer$: this.dataSetContainer$, }; } diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts index ecc17dbfe71a..02ca0c30161c 100644 --- a/src/plugins/data/server/search/search_service.ts +++ b/src/plugins/data/server/search/search_service.ts @@ -229,7 +229,10 @@ export class SearchService implements Plugin { dataFrame.meta.queryConfig.dataSourceId = dataSource?.id; } this.dfCache.set(dataFrame); - const existingIndexPattern = scopedIndexPatterns.getByTitle(dataFrame.name!, true); + const dataSetName = `${dataFrame.meta?.queryConfig?.dataSourceId ?? ''}.${ + dataFrame.name + }`; + const existingIndexPattern = await scopedIndexPatterns.get(dataSetName, true); const dataSet = await scopedIndexPatterns.create( dataFrameToSpec(dataFrame, existingIndexPattern?.id), !existingIndexPattern?.id @@ -239,8 +242,6 @@ export class SearchService implements Plugin { }, clear: () => { if (this.dfCache.get() === undefined) return; - // name because the id is not unique for temporary index pattern created - scopedIndexPatterns.clearCache(this.dfCache.get()!.name, false); this.dfCache.clear(); }, }; diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index eea1860dc950..616be16e9f56 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -30,6 +30,8 @@ export const Sidebar: FC = ({ children }) => { }, } = useOpenSearchDashboards(); + const { DataSetNavigator } = ui; + useEffect(() => { const subscriptions = ui.Settings.getEnabledQueryEnhancementsUpdated$().subscribe( (enabledQueryEnhancements) => { @@ -48,17 +50,17 @@ export const Sidebar: FC = ({ children }) => { useEffect(() => { if (!isEnhancementsEnabled) return; - const subscriptions = ui.container$.subscribe((container) => { - if (container === null) return; + const subscriptions = ui.dataSetContainer$.subscribe((dataSetContainer) => { + if (dataSetContainer === null) return; if (containerRef.current) { - setContainerRef(container); + setContainerRef(dataSetContainer); } }); return () => { subscriptions.unsubscribe(); }; - }, [ui.container$, containerRef, setContainerRef, isEnhancementsEnabled]); + }, [ui.dataSetContainer$, containerRef, setContainerRef, isEnhancementsEnabled]); useEffect(() => { let isMounted = true; @@ -134,19 +136,6 @@ export const Sidebar: FC = ({ children }) => { dataSources.dataSourceService.reload(); }, [dataSources.dataSourceService]); - const dataSourceSelector = ( - - ); - return ( { containerRef.current = node; }} > - {dataSourceSelector} + )} {!isEnhancementsEnabled && ( @@ -171,7 +160,16 @@ export const Sidebar: FC = ({ children }) => { color="transparent" className="deSidebar_dataSource" > - {dataSourceSelector} + )} diff --git a/src/plugins/data_explorer/public/index.ts b/src/plugins/data_explorer/public/index.ts index f8adda434ced..6b0561261c16 100644 --- a/src/plugins/data_explorer/public/index.ts +++ b/src/plugins/data_explorer/public/index.ts @@ -18,4 +18,5 @@ export { useTypedSelector, useTypedDispatch, setIndexPattern, + setDataSet, } from './utils/state_management'; diff --git a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts index e9fe84713120..fa41a29259e3 100644 --- a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts +++ b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts @@ -5,11 +5,13 @@ import { createSlice, PayloadAction } from '@reduxjs/toolkit'; import { DataExplorerServices } from '../../types'; +import { SimpleDataSet } from '../../../../data/common'; export interface MetadataState { indexPattern?: string; originatingApp?: string; view?: string; + dataSet?: Omit; } const initialState: MetadataState = {}; @@ -40,6 +42,9 @@ export const slice = createSlice({ setIndexPattern: (state, action: PayloadAction) => { state.indexPattern = action.payload; }, + setDataSet: (state, action: PayloadAction>) => { + state.dataSet = action.payload; + }, setOriginatingApp: (state, action: PayloadAction) => { state.originatingApp = action.payload; }, @@ -53,4 +58,4 @@ export const slice = createSlice({ }); export const { reducer } = slice; -export const { setIndexPattern, setOriginatingApp, setView, setState } = slice.actions; +export const { setIndexPattern, setDataSet, setOriginatingApp, setView, setState } = slice.actions; diff --git a/src/plugins/data_explorer/public/utils/state_management/store.ts b/src/plugins/data_explorer/public/utils/state_management/store.ts index daf0b3d7e369..9d320de4b54b 100644 --- a/src/plugins/data_explorer/public/utils/state_management/store.ts +++ b/src/plugins/data_explorer/public/utils/state_management/store.ts @@ -116,4 +116,4 @@ export type RenderState = Omit; // Remaining state after export type Store = ReturnType; export type AppDispatch = Store['dispatch']; -export { MetadataState, setIndexPattern, setOriginatingApp } from './metadata_slice'; +export { MetadataState, setIndexPattern, setDataSet, setOriginatingApp } from './metadata_slice'; diff --git a/src/plugins/discover/public/application/utils/state_management/index.ts b/src/plugins/discover/public/application/utils/state_management/index.ts index 989b2662f0d4..e6df7e4774b8 100644 --- a/src/plugins/discover/public/application/utils/state_management/index.ts +++ b/src/plugins/discover/public/application/utils/state_management/index.ts @@ -7,6 +7,7 @@ import { TypedUseSelectorHook } from 'react-redux'; import { RootState, setIndexPattern as updateIndexPattern, + setDataSet as updateDataSet, useTypedDispatch, useTypedSelector, } from '../../../../../data_explorer/public'; @@ -20,4 +21,4 @@ export interface DiscoverRootState extends RootState { export const useSelector: TypedUseSelectorHook = useTypedSelector; export const useDispatch = useTypedDispatch; -export { updateIndexPattern }; +export { updateIndexPattern, updateDataSet }; diff --git a/src/plugins/discover/public/application/view_components/utils/update_search_source.ts b/src/plugins/discover/public/application/view_components/utils/update_search_source.ts index a8480fdad18a..05d4a2dbd8b4 100644 --- a/src/plugins/discover/public/application/view_components/utils/update_search_source.ts +++ b/src/plugins/discover/public/application/view_components/utils/update_search_source.ts @@ -30,7 +30,12 @@ export const updateSearchSource = async ({ histogramConfigs, }: Props) => { const { uiSettings, data } = services; - let dataSet = indexPattern; + const queryDataSet = data.query.dataSet.getDataSet(); + + let dataSet = + indexPattern.id === queryDataSet?.id + ? await data.indexPatterns.getByTitle(queryDataSet?.title!) + : indexPattern; const dataFrame = searchSource?.getDataFrame(); if ( searchSource && diff --git a/src/plugins/discover/public/application/view_components/utils/use_search.ts b/src/plugins/discover/public/application/view_components/utils/use_search.ts index 8c2ace81b048..1e40cf40a8a9 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_search.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_search.ts @@ -250,7 +250,8 @@ export const useSearch = (services: DiscoverViewServices) => { timefilter.getFetch$(), timefilter.getTimeUpdate$(), timefilter.getAutoRefreshFetch$(), - data.query.queryString.getUpdates$() + data.query.queryString.getUpdates$(), + data.query.dataSet.getUpdates$() ).pipe(debounceTime(100)); const subscription = fetch$.subscribe(() => { @@ -280,6 +281,7 @@ export const useSearch = (services: DiscoverViewServices) => { fetch, core.fatalErrors, shouldSearchOnPageLoad, + data.query.dataSet, ]); // Get savedSearch if it exists diff --git a/src/plugins/query_enhancements/opensearch_dashboards.json b/src/plugins/query_enhancements/opensearch_dashboards.json index b09494aab0ca..69d8fd3bd667 100644 --- a/src/plugins/query_enhancements/opensearch_dashboards.json +++ b/src/plugins/query_enhancements/opensearch_dashboards.json @@ -3,7 +3,7 @@ "version": "opensearchDashboards", "server": true, "ui": true, - "requiredPlugins": ["data", "opensearchDashboardsReact", "opensearchDashboardsUtils", "dataSourceManagement", "savedObjects", "uiActions"], + "requiredPlugins": ["data", "opensearchDashboardsReact", "opensearchDashboardsUtils", "savedObjects", "uiActions"], "optionalPlugins": ["dataSource"] } diff --git a/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx b/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx deleted file mode 100644 index 3fd592e50b31..000000000000 --- a/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { useEffect, useRef, useState } from 'react'; -import { EuiPortal } from '@elastic/eui'; -import { distinctUntilChanged } from 'rxjs/operators'; -import { ToastsSetup } from 'opensearch-dashboards/public'; -import { DataPublicPluginStart, QueryEditorExtensionDependencies } from '../../../../data/public'; -import { DataSourceSelector } from '../../../../data_source_management/public'; -import { ConnectionsService } from '../services'; - -interface ConnectionsProps { - dependencies: QueryEditorExtensionDependencies; - toasts: ToastsSetup; - connectionsService: ConnectionsService; -} - -export const ConnectionsBar: React.FC = ({ connectionsService, toasts }) => { - const [isDataSourceEnabled, setIsDataSourceEnabled] = useState(false); - const [uiService, setUiService] = useState(undefined); - const containerRef = useRef(null); - - useEffect(() => { - const uiServiceSubscription = connectionsService.getUiService().subscribe(setUiService); - const dataSourceEnabledSubscription = connectionsService - .getIsDataSourceEnabled$() - .subscribe(setIsDataSourceEnabled); - - return () => { - uiServiceSubscription.unsubscribe(); - dataSourceEnabledSubscription.unsubscribe(); - }; - }, [connectionsService]); - - useEffect(() => { - if (!uiService || !isDataSourceEnabled || !containerRef.current) return; - const subscriptions = uiService.dataSourceContainer$.subscribe((container) => { - if (container && containerRef.current) { - container.append(containerRef.current); - } - }); - - return () => subscriptions.unsubscribe(); - }, [uiService, isDataSourceEnabled]); - - useEffect(() => { - const selectedConnectionSubscription = connectionsService - .getSelectedConnection$() - .pipe(distinctUntilChanged()) - .subscribe((connection) => { - if (connection) { - // Assuming setSelectedConnection$ is meant to update some state or perform an action outside this component - connectionsService.setSelectedConnection$(connection); - } - }); - - return () => selectedConnectionSubscription.unsubscribe(); - }, [connectionsService]); - - const handleSelectedConnection = (id: string | undefined) => { - if (!id) { - connectionsService.setSelectedConnection$(undefined); - return; - } - connectionsService.getConnectionById(id).subscribe((connection) => { - connectionsService.setSelectedConnection$(connection); - }); - }; - - return ( - { - containerRef.current = node; - }} - > -
- - handleSelectedConnection(dataSource[0]?.id || undefined) - } - /> -
- - ); -}; diff --git a/src/plugins/query_enhancements/public/data_source_connection/index.ts b/src/plugins/query_enhancements/public/data_source_connection/index.ts deleted file mode 100644 index e334163d91d4..000000000000 --- a/src/plugins/query_enhancements/public/data_source_connection/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export { createDataSourceConnectionExtension } from './utils'; -export * from './services'; diff --git a/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx b/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx deleted file mode 100644 index e5822c4b378e..000000000000 --- a/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React from 'react'; -import { ToastsSetup } from 'opensearch-dashboards/public'; -import { QueryEditorExtensionConfig } from '../../../../data/public'; -import { ConfigSchema } from '../../../common/config'; -import { ConnectionsBar } from '../components'; -import { ConnectionsService } from '../services'; - -export const createDataSourceConnectionExtension = ( - connectionsService: ConnectionsService, - toasts: ToastsSetup, - config: ConfigSchema -): QueryEditorExtensionConfig => { - return { - id: 'data-source-connection', - order: 2000, - isEnabled$: (dependencies) => { - return connectionsService.getIsDataSourceEnabled$(); - }, - getComponent: (dependencies) => { - return ( - - ); - }, - }; -}; diff --git a/src/plugins/query_enhancements/public/plugin.tsx b/src/plugins/query_enhancements/public/plugin.tsx index d65676b70e78..b74c00ced7e0 100644 --- a/src/plugins/query_enhancements/public/plugin.tsx +++ b/src/plugins/query_enhancements/public/plugin.tsx @@ -7,10 +7,9 @@ import moment from 'moment'; import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '../../../core/public'; import { IStorageWrapper, Storage } from '../../opensearch_dashboards_utils/public'; import { ConfigSchema } from '../common/config'; -import { ConnectionsService, createDataSourceConnectionExtension } from './data_source_connection'; +import { ConnectionsService, setData, setStorage } from './services'; import { createQueryAssistExtension } from './query_assist'; -import { PPLSearchInterceptor, SQLAsyncSearchInterceptor, SQLSearchInterceptor } from './search'; -import { setData, setStorage } from './services'; +import { PPLSearchInterceptor, SQLSearchInterceptor } from './search'; import { QueryEnhancementsPluginSetup, QueryEnhancementsPluginSetupDependencies, @@ -44,38 +43,21 @@ export class QueryEnhancementsPlugin http: core.http, }); - const pplSearchInterceptor = new PPLSearchInterceptor( - { - toasts: core.notifications.toasts, - http: core.http, - uiSettings: core.uiSettings, - startServices: core.getStartServices(), - usageCollector: data.search.usageCollector, - }, - this.connectionsService - ); - - const sqlSearchInterceptor = new SQLSearchInterceptor( - { - toasts: core.notifications.toasts, - http: core.http, - uiSettings: core.uiSettings, - startServices: core.getStartServices(), - usageCollector: data.search.usageCollector, - }, - this.connectionsService - ); + const pplSearchInterceptor = new PPLSearchInterceptor({ + toasts: core.notifications.toasts, + http: core.http, + uiSettings: core.uiSettings, + startServices: core.getStartServices(), + usageCollector: data.search.usageCollector, + }); - const sqlAsyncSearchInterceptor = new SQLAsyncSearchInterceptor( - { - toasts: core.notifications.toasts, - http: core.http, - uiSettings: core.uiSettings, - startServices: core.getStartServices(), - usageCollector: data.search.usageCollector, - }, - this.connectionsService - ); + const sqlSearchInterceptor = new SQLSearchInterceptor({ + toasts: core.notifications.toasts, + http: core.http, + uiSettings: core.uiSettings, + startServices: core.getStartServices(), + usageCollector: data.search.usageCollector, + }); data.__enhance({ ui: { @@ -89,7 +71,7 @@ export class QueryEnhancementsPlugin initialTo: moment().add(2, 'days').toISOString(), }, showFilterBar: false, - showDataSetsSelector: false, + showDataSetsSelector: true, showDataSourcesSelector: true, }, fields: { @@ -110,7 +92,7 @@ export class QueryEnhancementsPlugin searchBar: { showDatePicker: false, showFilterBar: false, - showDataSetsSelector: false, + showDataSetsSelector: true, showDataSourcesSelector: true, queryStringInput: { initialValue: 'SELECT * FROM ' }, }, @@ -125,29 +107,6 @@ export class QueryEnhancementsPlugin }, }); - data.__enhance({ - ui: { - query: { - language: 'SQLAsync', - search: sqlAsyncSearchInterceptor, - searchBar: { - showDatePicker: false, - showFilterBar: false, - showDataSetsSelector: false, - showDataSourcesSelector: true, - queryStringInput: { initialValue: 'SHOW DATABASES IN ::mys3::' }, - }, - fields: { - filterable: false, - visualizable: false, - }, - showDocLinks: false, - supportedAppNames: ['discover'], - connectionService: this.connectionsService, - }, - }, - }); - data.__enhance({ ui: { queryEditorExtension: createQueryAssistExtension( @@ -158,16 +117,6 @@ export class QueryEnhancementsPlugin }, }); - data.__enhance({ - ui: { - queryEditorExtension: createDataSourceConnectionExtension( - this.connectionsService, - core.notifications.toasts, - this.config - ), - }, - }); - return {}; } diff --git a/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx b/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx index e87e74ce2998..c28c5cb8b0be 100644 --- a/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx +++ b/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx @@ -12,7 +12,7 @@ import { } from '../../../../data/public'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { QueryAssistParameters } from '../../../common/query_assist'; -import { ConnectionsService } from '../../data_source_connection'; +import { ConnectionsService } from '../../services'; import { getStorage } from '../../services'; import { useGenerateQuery } from '../hooks'; import { getPersistedLog, ProhibitedQueryError } from '../utils'; @@ -45,7 +45,7 @@ export const QueryAssistBar: React.FC = (props) => { const subscription = props.connectionsService .getSelectedConnection$() .subscribe((connection) => { - dataSourceIdRef.current = connection?.dataSource.id; + dataSourceIdRef.current = connection?.dataSource?.id; }); return () => subscription.unsubscribe(); }, [props.connectionsService]); diff --git a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx index e088457a0717..23611e39501e 100644 --- a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx +++ b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx @@ -13,7 +13,7 @@ import { } from '../../../../data/public'; import { API } from '../../../common'; import { ConfigSchema } from '../../../common/config'; -import { ConnectionsService } from '../../data_source_connection'; +import { ConnectionsService } from '../../services'; import { QueryAssistBar, QueryAssistBanner } from '../components'; /** @@ -28,7 +28,7 @@ const getAvailableLanguages$ = ( connectionsService.getSelectedConnection$().pipe( distinctUntilChanged(), switchMap(async (connection) => { - const dataSourceId = connection?.dataSource.id; + const dataSourceId = connection?.dataSource?.id; const cached = availableLanguagesByDataSource.get(dataSourceId); if (cached !== undefined) return cached; const languages = await http diff --git a/src/plugins/query_enhancements/public/search/index.ts b/src/plugins/query_enhancements/public/search/index.ts index 9835c1345f02..624e7cf6e7b5 100644 --- a/src/plugins/query_enhancements/public/search/index.ts +++ b/src/plugins/query_enhancements/public/search/index.ts @@ -5,4 +5,3 @@ export { PPLSearchInterceptor } from './ppl_search_interceptor'; export { SQLSearchInterceptor } from './sql_search_interceptor'; -export { SQLAsyncSearchInterceptor } from './sql_async_search_interceptor'; diff --git a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts index bca9961fea3b..13d6bc25874c 100644 --- a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts @@ -5,7 +5,6 @@ import { trimEnd } from 'lodash'; import { Observable, throwError } from 'rxjs'; -import { i18n } from '@osd/i18n'; import { concatMap } from 'rxjs/operators'; import { DataFrameAggConfig, @@ -34,16 +33,12 @@ import { fetchDataFrame, } from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; -import { ConnectionsService } from '../data_source_connection'; export class PPLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; protected aggsService!: DataPublicPluginStart['search']['aggs']; - constructor( - deps: SearchInterceptorDeps, - private readonly connectionsService: ConnectionsService - ) { + constructor(deps: SearchInterceptorDeps) { super(deps); deps.startServices.then(([coreStart, depsStart]) => { @@ -147,34 +142,23 @@ export class PPLSearchInterceptor extends SearchInterceptor { }; const dataFrame = getRawDataFrame(searchRequest); - if (!dataFrame) { - return throwError( - this.handleSearchError( - { - stack: 'DataFrame is not defined', - }, - request, - signal! - ) - ); - } let queryString = dataFrame.meta?.queryConfig?.qs ?? getRawQueryString(searchRequest) ?? ''; dataFrame.meta = { ...dataFrame.meta, + aggConfig: { + ...dataFrame.meta.aggConfig, + ...(this.aggsService.types.get.bind(this) && + getAggConfig(searchRequest, {}, this.aggsService.types.get.bind(this))), + }, queryConfig: { ...dataFrame.meta.queryConfig, - ...(this.connectionsService.getSelectedConnection() && { - dataSourceId: this.connectionsService.getSelectedConnection()?.id, + ...(this.queryService.dataSet.getDataSet() && { + dataSourceId: this.queryService.dataSet.getDataSet()?.dataSourceRef?.id, }), }, }; - const aggConfig = getAggConfig( - searchRequest, - {}, - this.aggsService.types.get.bind(this) - ) as DataFrameAggConfig; if (!dataFrame.schema) { return fetchDataFrame(dfContext, queryString, dataFrame).pipe( @@ -184,14 +168,14 @@ export class PPLSearchInterceptor extends SearchInterceptor { const jsError = new Error(df.error.response); return throwError(jsError); } - const timeField = getTimeField(df, aggConfig); + const timeField = getTimeField(df, dataFrame.meta?.aggConfig); if (timeField) { const timeFilter = getTimeFilter(timeField); const newQuery = insertTimeFilter(queryString, timeFilter); updateDataFrameMeta({ dataFrame: df, qs: newQuery, - aggConfig, + aggConfig: dataFrame.meta?.aggConfig, timeField, timeFilter, getAggQsFn: getAggQsFn.bind(this), @@ -204,14 +188,14 @@ export class PPLSearchInterceptor extends SearchInterceptor { } if (dataFrame.schema) { - const timeField = getTimeField(dataFrame, aggConfig); + const timeField = getTimeField(dataFrame, dataFrame.meta?.aggConfig); if (timeField) { const timeFilter = getTimeFilter(timeField); const newQuery = insertTimeFilter(queryString, timeFilter); updateDataFrameMeta({ dataFrame, qs: newQuery, - aggConfig, + aggConfig: dataFrame.meta?.aggConfig, timeField, timeFilter, getAggQsFn: getAggQsFn.bind(this), diff --git a/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts deleted file mode 100644 index 9232ef146cdb..000000000000 --- a/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { trimEnd } from 'lodash'; -import { BehaviorSubject, Observable, throwError } from 'rxjs'; -import { i18n } from '@osd/i18n'; -import { concatMap, map } from 'rxjs/operators'; -import { - DATA_FRAME_TYPES, - DataPublicPluginStart, - IOpenSearchDashboardsSearchRequest, - IOpenSearchDashboardsSearchResponse, - ISearchOptions, - SearchInterceptor, - SearchInterceptorDeps, -} from '../../../data/public'; -import { getRawDataFrame, getRawQueryString, IDataFrameResponse } from '../../../data/common'; -import { - API, - DataFramePolling, - FetchDataFrameContext, - SEARCH_STRATEGY, - fetchDataFrame, - fetchDataFramePolling, -} from '../../common'; -import { QueryEnhancementsPluginStartDependencies } from '../types'; -import { ConnectionsService } from '../data_source_connection'; - -export class SQLAsyncSearchInterceptor extends SearchInterceptor { - protected queryService!: DataPublicPluginStart['query']; - protected aggsService!: DataPublicPluginStart['search']['aggs']; - protected indexPatterns!: DataPublicPluginStart['indexPatterns']; - protected dataFrame$ = new BehaviorSubject(undefined); - - constructor( - deps: SearchInterceptorDeps, - private readonly connectionsService: ConnectionsService - ) { - super(deps); - - deps.startServices.then(([coreStart, depsStart]) => { - this.queryService = (depsStart as QueryEnhancementsPluginStartDependencies).data.query; - this.aggsService = (depsStart as QueryEnhancementsPluginStartDependencies).data.search.aggs; - }); - } - - protected runSearch( - request: IOpenSearchDashboardsSearchRequest, - signal?: AbortSignal, - strategy?: string - ): Observable { - const { id, ...searchRequest } = request; - const path = trimEnd(API.SQL_ASYNC_SEARCH); - const dfContext: FetchDataFrameContext = { - http: this.deps.http, - path, - signal, - }; - - const dataFrame = getRawDataFrame(searchRequest); - if (!dataFrame) { - return throwError(this.handleSearchError('DataFrame is not defined', request, signal!)); - } - - const queryString = - dataFrame.meta?.queryConfig?.formattedQs() ?? getRawQueryString(searchRequest) ?? ''; - - dataFrame.meta = { - ...dataFrame.meta, - queryConfig: { - ...dataFrame.meta.queryConfig, - ...(this.connectionsService.getSelectedConnection() && - this.connectionsService.getSelectedConnection()?.dataSource && { - dataSourceId: this.connectionsService.getSelectedConnection()?.dataSource.id, - }), - }, - }; - - const onPollingSuccess = (pollingResult: any) => { - if (pollingResult && pollingResult.body.meta.status === 'SUCCESS') { - return false; - } - if (pollingResult && pollingResult.body.meta.status === 'FAILED') { - const jsError = new Error(pollingResult.data.error.response); - this.deps.toasts.addError(jsError, { - title: i18n.translate('queryEnhancements.sqlQueryError', { - defaultMessage: 'Could not complete the SQL async query', - }), - toastMessage: pollingResult.data.error.response, - }); - return false; - } - - this.deps.toasts.addInfo({ - title: i18n.translate('queryEnhancements.sqlQueryPolling', { - defaultMessage: 'Polling query job results...', - }), - }); - - return true; - }; - - const onPollingError = (error: Error) => { - throw new Error(error.message); - }; - - this.deps.toasts.addInfo({ - title: i18n.translate('queryEnhancements.sqlQueryInfo', { - defaultMessage: 'Starting query job...', - }), - }); - return fetchDataFrame(dfContext, queryString, dataFrame).pipe( - concatMap((jobResponse) => { - const df = jobResponse.body; - const dataFramePolling = new DataFramePolling( - () => fetchDataFramePolling(dfContext, df), - 5000, - onPollingSuccess, - onPollingError - ); - return dataFramePolling.fetch().pipe( - map(() => { - const dfPolling = dataFramePolling.data; - dfPolling.type = DATA_FRAME_TYPES.DEFAULT; - return dfPolling; - }) - ); - }) - ); - } - - public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) { - return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.SQL_ASYNC); - } -} diff --git a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts index 5a3b8278c65a..de7fb5938d25 100644 --- a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts @@ -6,8 +6,13 @@ import { trimEnd } from 'lodash'; import { Observable, throwError } from 'rxjs'; import { i18n } from '@osd/i18n'; -import { concatMap } from 'rxjs/operators'; -import { getRawDataFrame, getRawQueryString } from '../../../data/common'; +import { concatMap, map } from 'rxjs/operators'; +import { + DATA_FRAME_TYPES, + getRawDataFrame, + getRawQueryString, + SIMPLE_DATA_SET_TYPES, +} from '../../../data/common'; import { DataPublicPluginStart, IOpenSearchDashboardsSearchRequest, @@ -16,18 +21,21 @@ import { SearchInterceptor, SearchInterceptorDeps, } from '../../../data/public'; -import { API, FetchDataFrameContext, SEARCH_STRATEGY, fetchDataFrame } from '../../common'; +import { + API, + DataFramePolling, + FetchDataFrameContext, + SEARCH_STRATEGY, + fetchDataFrame, + fetchDataFramePolling, +} from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; -import { ConnectionsService } from '../data_source_connection'; export class SQLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; protected aggsService!: DataPublicPluginStart['search']['aggs']; - constructor( - deps: SearchInterceptorDeps, - private readonly connectionsService: ConnectionsService - ) { + constructor(deps: SearchInterceptorDeps) { super(deps); deps.startServices.then(([coreStart, depsStart]) => { @@ -49,9 +57,6 @@ export class SQLSearchInterceptor extends SearchInterceptor { }; const dataFrame = getRawDataFrame(searchRequest); - if (!dataFrame) { - return throwError(this.handleSearchError('DataFrame is not defined', request, signal!)); - } const queryString = dataFrame.meta?.queryConfig?.qs ?? getRawQueryString(searchRequest) ?? ''; @@ -59,8 +64,8 @@ export class SQLSearchInterceptor extends SearchInterceptor { ...dataFrame.meta, queryConfig: { ...dataFrame.meta.queryConfig, - ...(this.connectionsService.getSelectedConnection() && { - dataSourceId: this.connectionsService.getSelectedConnection()?.id, + ...(this.queryService.dataSet.getDataSet() && { + dataSourceId: this.queryService.dataSet.getDataSet()?.dataSourceRef?.id, }), }, }; @@ -81,7 +86,91 @@ export class SQLSearchInterceptor extends SearchInterceptor { return fetchDataFrame(dfContext, queryString, dataFrame); } + protected runSearchAsync( + request: IOpenSearchDashboardsSearchRequest, + signal?: AbortSignal, + strategy?: string + ): Observable { + const { id, ...searchRequest } = request; + const path = trimEnd(API.SQL_ASYNC_SEARCH); + const dfContext: FetchDataFrameContext = { + http: this.deps.http, + path, + signal, + }; + + const dataFrame = getRawDataFrame(searchRequest); + if (!dataFrame) { + return throwError(this.handleSearchError('DataFrame is not defined', request, signal!)); + } + + const queryString = getRawQueryString(searchRequest) ?? ''; + + dataFrame.meta = { + ...dataFrame.meta, + queryConfig: { + ...dataFrame.meta.queryConfig, + }, + }; + + const onPollingSuccess = (pollingResult: any) => { + if (pollingResult && pollingResult.body.meta.status === 'SUCCESS') { + return false; + } + if (pollingResult && pollingResult.body.meta.status === 'FAILED') { + const jsError = new Error(pollingResult.data.error.response); + this.deps.toasts.addError(jsError, { + title: i18n.translate('queryEnhancements.sqlQueryError', { + defaultMessage: 'Could not complete the SQL async query', + }), + toastMessage: pollingResult.data.error.response, + }); + return false; + } + + this.deps.toasts.addInfo({ + title: i18n.translate('queryEnhancements.sqlQueryPolling', { + defaultMessage: 'Polling query job results...', + }), + }); + + return true; + }; + + const onPollingError = (error: Error) => { + throw new Error(error.message); + }; + + this.deps.toasts.addInfo({ + title: i18n.translate('queryEnhancements.sqlQueryInfo', { + defaultMessage: 'Starting query job...', + }), + }); + return fetchDataFrame(dfContext, queryString, dataFrame).pipe( + concatMap((jobResponse) => { + const df = jobResponse.body; + const dataFramePolling = new DataFramePolling( + () => fetchDataFramePolling(dfContext, df), + 5000, + onPollingSuccess, + onPollingError + ); + return dataFramePolling.fetch().pipe( + map(() => { + const dfPolling = dataFramePolling.data; + dfPolling.type = DATA_FRAME_TYPES.DEFAULT; + return dfPolling; + }) + ); + }) + ); + } + public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) { + const dataSet = this.queryService.dataSet.getDataSet(); + if (dataSet?.type === SIMPLE_DATA_SET_TYPES.TEMPORARY_ASYNC) { + return this.runSearchAsync(request, options.abortSignal, SEARCH_STRATEGY.SQL_ASYNC); + } return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.SQL); } } diff --git a/src/plugins/query_enhancements/public/services.ts b/src/plugins/query_enhancements/public/services.ts deleted file mode 100644 index d11233be2dca..000000000000 --- a/src/plugins/query_enhancements/public/services.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { createGetterSetter } from '../../opensearch_dashboards_utils/common'; -import { IStorageWrapper } from '../../opensearch_dashboards_utils/public'; -import { DataPublicPluginStart } from '../../data/public'; - -export const [getStorage, setStorage] = createGetterSetter('storage'); -export const [getData, setData] = createGetterSetter('data'); diff --git a/src/plugins/query_enhancements/public/data_source_connection/services/connections_service.ts b/src/plugins/query_enhancements/public/services/connections_service.ts similarity index 95% rename from src/plugins/query_enhancements/public/data_source_connection/services/connections_service.ts rename to src/plugins/query_enhancements/public/services/connections_service.ts index 6afec4b51a99..97a59c2cd94a 100644 --- a/src/plugins/query_enhancements/public/data_source_connection/services/connections_service.ts +++ b/src/plugins/query_enhancements/public/services/connections_service.ts @@ -6,8 +6,8 @@ import { BehaviorSubject, Observable, from } from 'rxjs'; import { DataPublicPluginStart } from 'src/plugins/data/public'; import { CoreStart } from 'opensearch-dashboards/public'; -import { API } from '../../../common'; -import { Connection, ConnectionsServiceDeps } from '../../types'; +import { API } from '../../common'; +import { Connection, ConnectionsServiceDeps } from '../types'; export class ConnectionsService { protected http!: ConnectionsServiceDeps['http']; diff --git a/src/plugins/query_enhancements/public/services/index.ts b/src/plugins/query_enhancements/public/services/index.ts new file mode 100644 index 000000000000..bb0284408faa --- /dev/null +++ b/src/plugins/query_enhancements/public/services/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createGetterSetter } from '../../../opensearch_dashboards_utils/common'; +import { IStorageWrapper } from '../../../opensearch_dashboards_utils/public'; +import { DataPublicPluginStart } from '../../../data/public'; + +export const [getStorage, setStorage] = createGetterSetter('storage'); +export const [getData, setData] = createGetterSetter('data'); + +export { ConnectionsService } from './connections_service'; diff --git a/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts b/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts index f4fe42779dae..162cc7e8f103 100644 --- a/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts +++ b/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts @@ -5,7 +5,6 @@ import { schema } from '@osd/config-schema'; import { IRouter } from 'opensearch-dashboards/server'; -import { DataSourceAttributes } from '../../../../data_source/common/data_sources'; import { API } from '../../../common'; export function registerDataSourceConnectionsRoutes(router: IRouter) { @@ -18,7 +17,7 @@ export function registerDataSourceConnectionsRoutes(router: IRouter) { }, async (context, request, response) => { const fields = ['id', 'title', 'auth.type']; - const resp = await context.core.savedObjects.client.find({ + const resp = await context.core.savedObjects.client.find({ type: 'data-source', fields, perPage: 10000, @@ -38,7 +37,7 @@ export function registerDataSourceConnectionsRoutes(router: IRouter) { }, }, async (context, request, response) => { - const resp = await context.core.savedObjects.client.get( + const resp = await context.core.savedObjects.client.get( 'data-source', request.params.dataSourceId ); diff --git a/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts b/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts index acd0027d0bc1..8cd5014335e0 100644 --- a/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts +++ b/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts @@ -55,7 +55,7 @@ export const sqlAsyncSearchStrategyProvider = ( const sessionId = rawResponse.data?.sessionId; const partial: PartialDataFrame = { - name: '', + ...request.body.df, fields: rawResponse?.data?.schema || [], }; const dataFrame = createDataFrame(partial); diff --git a/src/plugins/query_enhancements/server/search/sql_search_strategy.ts b/src/plugins/query_enhancements/server/search/sql_search_strategy.ts index c5ebb40f882b..0a4683567302 100644 --- a/src/plugins/query_enhancements/server/search/sql_search_strategy.ts +++ b/src/plugins/query_enhancements/server/search/sql_search_strategy.ts @@ -39,7 +39,7 @@ export const sqlSearchStrategyProvider = ( } const partial: PartialDataFrame = { - name: '', + ...request.body.df, fields: rawResponse.data?.schema || [], }; const dataFrame = createDataFrame(partial); diff --git a/src/plugins/query_enhancements/server/types.ts b/src/plugins/query_enhancements/server/types.ts index 1ad76c7bbf85..b6a03b672de9 100644 --- a/src/plugins/query_enhancements/server/types.ts +++ b/src/plugins/query_enhancements/server/types.ts @@ -4,7 +4,7 @@ */ import { PluginSetup } from 'src/plugins/data/server'; -import { DataSourcePluginSetup } from '../../data_source/server'; +import { DataSourcePluginSetup } from 'src/plugins/data_source/server'; import { Logger } from '../../../core/server'; import { ConfigSchema } from '../common/config';