From 4d14b609ad3083c6299de1aec1ba0e5494eee784 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 15:43:45 -0700 Subject: [PATCH] [Discover] Data selector enhancement (#6571) (#6691) * datasource and service refactoring * datasource enhancement and refactoring * datasource selectable consolidation and refactoring * add in memory cache with refresh * move refresh to right side * renaming * update default datasource tests * added more tests for default datasource * update selector tests * update changelog * fix data source service tests * add and update tests for datasource service * add more data source service tests * fix sidebar tests * add to change log yml * address comments along with more tests * add test subject * reference from correct type path * correct text * minor change - remove yet used displayOrder * remove from changelog as having fragments already * use expanded name * fix one test --------- (cherry picked from commit 42166d0c5669ac55b6b0936dfcbc5eb527a3d36d) Signed-off-by: Eric Signed-off-by: Eric Wei Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- changelogs/fragments/6571.yml | 2 + .../data/public/data_sources/constants.tsx | 46 +++++ .../data_sources/datasource/datasource.ts | 52 +++-- .../data_sources/datasource/factory.test.ts | 4 +- .../public/data_sources/datasource/factory.ts | 5 +- .../public/data_sources/datasource/index.ts | 7 +- .../public/data_sources/datasource/types.ts | 109 +++++++--- .../data_selector_refresher.test.tsx | 77 +++++++ .../data_selector_refresher.tsx | 48 +++++ .../datasource_selectable.test.tsx | 59 +++++- .../datasource_selectable.tsx | 189 ++++++++++-------- .../data_sources/datasource_selector/types.ts | 11 +- .../datasource_service.test.ts | 157 +++++++++++++-- .../datasource_services/datasource_service.ts | 103 ++++++++-- .../data_sources/datasource_services/index.ts | 2 - .../data_sources/datasource_services/types.ts | 27 +-- .../default_datasource.test.ts | 66 +++++- .../default_datasource/default_datasource.ts | 64 ++++-- .../register_default_datasource.ts | 42 ++-- src/plugins/data/public/index.ts | 8 +- src/plugins/data/public/plugin.ts | 13 +- .../public/components/sidebar/index.test.tsx | 36 ++-- .../public/components/sidebar/index.tsx | 17 +- .../public/__mock__/index.test.mock.ts | 29 +-- 24 files changed, 891 insertions(+), 282 deletions(-) create mode 100644 changelogs/fragments/6571.yml create mode 100644 src/plugins/data/public/data_sources/constants.tsx create mode 100644 src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.test.tsx create mode 100644 src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.tsx diff --git a/changelogs/fragments/6571.yml b/changelogs/fragments/6571.yml new file mode 100644 index 000000000000..a6e341fc15e4 --- /dev/null +++ b/changelogs/fragments/6571.yml @@ -0,0 +1,2 @@ +refactor: +- discover data selector enhancement and refactoring ([#6571](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6571)) \ No newline at end of file diff --git a/src/plugins/data/public/data_sources/constants.tsx b/src/plugins/data/public/data_sources/constants.tsx new file mode 100644 index 000000000000..eb0a0cfb0ac8 --- /dev/null +++ b/src/plugins/data/public/data_sources/constants.tsx @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { i18n } from '@osd/i18n'; +import { DataSourceUIGroupType } from './datasource/types'; + +export const S3_GLUE_DATA_SOURCE_DISPLAY_NAME = 'Amazon S3'; +export const S3_GLUE_DATA_SOURCE_TYPE = 's3glue'; +export const DEFAULT_DATA_SOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; +export const DEFAULT_DATA_SOURCE_NAME = i18n.translate('data.datasource.type.openSearchDefault', { + defaultMessage: 'OpenSearch Default', +}); +export const DEFAULT_DATA_SOURCE_DISPLAY_NAME = i18n.translate( + 'data.datasource.type.openSearchDefaultDisplayName', + { + defaultMessage: 'Index patterns', + } +); + +export const defaultDataSourceMetadata = { + ui: { + label: DEFAULT_DATA_SOURCE_DISPLAY_NAME, + typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME, + groupType: DataSourceUIGroupType.defaultOpenSearchDataSource, + selector: { + displayDatasetsAsSource: true, + }, + }, +}; + +export const s3DataSourceMetadata = { + ui: { + label: S3_GLUE_DATA_SOURCE_DISPLAY_NAME, + typeLabel: S3_GLUE_DATA_SOURCE_TYPE, + groupType: DataSourceUIGroupType.s3glue, + selector: { + displayDatasetsAsSource: false, + }, + }, +}; + +export const DATA_SELECTOR_REFRESHER_POPOVER_TEXT = 'Refresh data selector'; +export const DATA_SELECTOR_DEFAULT_PLACEHOLDER = 'Select a data source'; +export const DATA_SELECTOR_S3_DATA_SOURCE_GROUP_HINT_LABEL = ' - Opens in Log Explorer'; diff --git a/src/plugins/data/public/data_sources/datasource/datasource.ts b/src/plugins/data/public/data_sources/datasource/datasource.ts index a2159c562064..bcff4d0d6cc7 100644 --- a/src/plugins/data/public/data_sources/datasource/datasource.ts +++ b/src/plugins/data/public/data_sources/datasource/datasource.ts @@ -13,23 +13,41 @@ * DataSourceQueryResult: Represents the result from querying the data source. */ -import { ConnectionStatus } from './types'; +import { + DataSourceConnectionStatus, + IDataSetParams, + IDataSourceDataSet, + IDataSourceMetadata, + IDataSourceQueryParams, + IDataSourceQueryResponse, + IDataSourceSettings, +} from './types'; /** * @experimental this class is experimental and might change in future releases. */ export abstract class DataSource< - DataSourceMetaData, - DataSetParams, - SourceDataSet, - DataSourceQueryParams, - DataSourceQueryResult + TMetadata extends IDataSourceMetadata = IDataSourceMetadata, + TDataSetParams extends IDataSetParams = IDataSetParams, + TDataSet extends IDataSourceDataSet = IDataSourceDataSet, + TQueryParams extends IDataSourceQueryParams = IDataSourceQueryParams, + TQueryResult extends IDataSourceQueryResponse = IDataSourceQueryResponse > { - constructor( - private readonly name: string, - private readonly type: string, - private readonly metadata: DataSourceMetaData - ) {} + private readonly id: string; + private readonly name: string; + private readonly type: string; + private readonly metadata: TMetadata; + + constructor(settings: IDataSourceSettings) { + this.id = settings.id; + this.name = settings.name; + this.type = settings.type; + this.metadata = settings.metadata; + } + + getId() { + return this.id; + } getName() { return this.name; @@ -53,18 +71,18 @@ export abstract class DataSource< * patterns for OpenSearch data source * * @experimental This API is experimental and might change in future releases. - * @returns {SourceDataSet} Dataset associated with the data source. + * @returns {Promise} Dataset associated with the data source. */ - abstract getDataSet(dataSetParams?: DataSetParams): SourceDataSet; + abstract getDataSet(dataSetParams?: TDataSetParams): Promise; /** * Abstract method to run a query against the data source. * Implementing classes need to provide the specific implementation. * * @experimental This API is experimental and might change in future releases. - * @returns {DataSourceQueryResult} Result from querying the data source. + * @returns {Promise} Result from querying the data source. */ - abstract runQuery(queryParams: DataSourceQueryParams): DataSourceQueryResult; + abstract runQuery(queryParams?: TQueryParams): Promise; /** * Abstract method to test the connection to the data source. @@ -72,8 +90,8 @@ export abstract class DataSource< * the connection status, typically indicating success or failure. * * @experimental This API is experimental and might change in future releases. - * @returns {ConnectionStatus | Promise} Status of the connection test. + * @returns {Promise} Status of the connection test. * @experimental */ - abstract testConnection(): ConnectionStatus | Promise; + abstract testConnection(): Promise; } diff --git a/src/plugins/data/public/data_sources/datasource/factory.test.ts b/src/plugins/data/public/data_sources/datasource/factory.test.ts index 0f9ea016748f..e0d35b6ed0a6 100644 --- a/src/plugins/data/public/data_sources/datasource/factory.test.ts +++ b/src/plugins/data/public/data_sources/datasource/factory.test.ts @@ -11,17 +11,19 @@ class MockDataSource extends DataSource { private readonly indexPatterns; constructor({ + id, name, type, metadata, indexPatterns, }: { + id: string; name: string; type: string; metadata: any; indexPatterns: IndexPatternsService; }) { - super(name, type, metadata); + super({ id, name, type, metadata }); this.indexPatterns = indexPatterns; } diff --git a/src/plugins/data/public/data_sources/datasource/factory.ts b/src/plugins/data/public/data_sources/datasource/factory.ts index f0b4e36cfb82..ae61e2c566a1 100644 --- a/src/plugins/data/public/data_sources/datasource/factory.ts +++ b/src/plugins/data/public/data_sources/datasource/factory.ts @@ -8,7 +8,6 @@ * It serves as a registry for different data source types and provides a way to instantiate them. */ -import { DataSourceType } from '../datasource_services'; import { DataSource } from '../datasource'; type DataSourceClass< @@ -66,11 +65,11 @@ export class DataSourceFactory { * * @experimental This API is experimental and might change in future releases. * @param {string} type - The identifier for the data source type. - * @param {any} config - The configuration for the data source instance. + * @param {unknown} config - The configuration for the data source instance. * @returns {DataSourceType} An instance of the specified data source type. * @throws {Error} Throws an error if the data source type is not supported. */ - getDataSourceInstance(type: string, config: any): DataSourceType { + getDataSourceInstance(type: string, config: unknown): DataSource { const DataSourceClass = this.dataSourceClasses[type]; if (!DataSourceClass) { throw new Error('Unsupported data source type'); diff --git a/src/plugins/data/public/data_sources/datasource/index.ts b/src/plugins/data/public/data_sources/datasource/index.ts index 10af40fdcfa2..e45cd6dad22c 100644 --- a/src/plugins/data/public/data_sources/datasource/index.ts +++ b/src/plugins/data/public/data_sources/datasource/index.ts @@ -5,13 +5,12 @@ export { DataSource } from './datasource'; export { - IDataSourceMetaData, - ISourceDataSet, + IDataSourceMetadata, + DataSetWithDataSource, IDataSetParams, IDataSourceQueryParams, IDataSourceQueryResult, - ConnectionStatus, - DataSourceConfig, + DataSourceConnectionStatus, IndexPatternOption, } from './types'; export { DataSourceFactory } from './factory'; diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index bf77ef123a30..74a670573898 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -3,49 +3,112 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { DataSource } from './datasource'; + /** * @experimental These interfaces are experimental and might change in future releases. */ -import { IndexPatternsService } from '../../index_patterns'; -import { DataSourceType } from '../datasource_services'; - export interface IndexPatternOption { title: string; id: string; } -export interface IDataSourceMetaData { +export interface IDataSourceGroup { name: string; } -export interface IDataSourceGroup { - name: string; +export interface DataSetWithDataSource { + ds: DataSource; + list: T[]; } -export interface ISourceDataSet { - ds: DataSourceType; - data_sets: Array; +export interface IDataSetParams { + query: T; } -// to-dos: add common interfaces for datasource -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface IDataSetParams {} +export interface IDataSourceQueryParams { + query: T; +} -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface IDataSourceQueryParams {} +export interface IDataSourceQueryResult { + data: T; +} -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface IDataSourceQueryResult {} +export enum ConnectionStatus { + Connected = 'connected', + Disconnected = 'disconnected', + Error = 'error', +} -export interface ConnectionStatus { - success: boolean; - info: string; +export interface DataSourceConnectionStatus { + status: ConnectionStatus; + message: string; + error?: Error; } -export interface DataSourceConfig { - name: string; +export interface IDataSourceSettings { + id: string; type: string; - metadata: any; - indexPatterns: IndexPatternsService; + name: string; + metadata: T; +} + +export interface IDataSourceMetadata { + ui: IDataSourceUISettings; +} + +export interface IDataSourceUISelector { + displayDatasetsAsSource: boolean; +} + +/** + * Represents the UI settings for a data source. + */ +export interface IDataSourceUISettings { + /** + * Controls UI elements related to data source selector. + */ + selector: IDataSourceUISelector; + + /** + * The display name of the data source. + */ + label: string; + + /** + * The group to which the data source belongs. This is used to group data sources in the selector. + */ + groupType: DataSourceUIGroupType; + + /** + * The display name of the data source type. + */ + typeLabel: string; + + /** + * A short description of the data source. + * @optional + */ + description?: string; + + /** + * URI of the icon representing the data source. + * @optional + */ + icon?: string; +} + +export interface IDataSourceDataSet { + dataSets: T; +} + +export interface IDataSourceQueryResponse { + data: T; +} + +export enum DataSourceUIGroupType { + defaultOpenSearchDataSource = 'DEFAULT_INDEX_PATTERNS', + s3glue = 's3glue', + spark = 'spark', } diff --git a/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.test.tsx new file mode 100644 index 000000000000..1a793e45d9a3 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.test.tsx @@ -0,0 +1,77 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import '@testing-library/jest-dom/extend-expect'; +import { DataSelectorRefresher } from './data_selector_refresher'; // adjust the import path as necessary +import { DATA_SELECTOR_REFRESHER_POPOVER_TEXT } from '../constants'; +import { ToolTipDelay } from '@elastic/eui/src/components/tool_tip/tool_tip'; +import { EuiToolTipProps } from '@elastic/eui'; + +describe('DataSelectorRefresher', () => { + const tooltipText = DATA_SELECTOR_REFRESHER_POPOVER_TEXT; + const onRefreshMock = jest.fn(); + + it('renders correctly with given tooltip text', () => { + const container = render( + + ); + + const refreshButton = container.getByLabelText('sourceRefresh'); + fireEvent.mouseOver(refreshButton); + + waitFor(() => { + expect(container.getByText(tooltipText)).toBeInTheDocument(); + }); + }); + + it('calls onRefresh when button is clicked', () => { + const container = render( + + ); + + fireEvent.click(container.getByLabelText('sourceRefresh')); + expect(onRefreshMock).toHaveBeenCalledTimes(1); + }); + + it('applies additional button properties', () => { + const buttonProps = { + 'aria-label': 'Custom Aria Label', + }; + + render( + + ); + + const button = screen.getByTestId('sourceRefreshButton'); + expect(button).toHaveAttribute('aria-label', 'Custom Aria Label'); + }); + + it('applies additional tooltip properties', () => { + const toolTipProps: Partial = { + delay: 'long' as ToolTipDelay, + }; + + const container = render( + + ); + + const refreshButton = container.getByLabelText('sourceRefresh'); + fireEvent.mouseOver(refreshButton); + waitFor(() => { + const tooltip = screen.getByTestId('sourceRefreshButtonToolTip'); + expect(tooltip).toHaveAttribute('delay', 'long'); + }); + }); +}); diff --git a/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.tsx b/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.tsx new file mode 100644 index 000000000000..4aabecd3bd6d --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.tsx @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { i18n } from '@osd/i18n'; +import { + EuiButtonIcon, + EuiButtonIconProps, + EuiText, + EuiToolTip, + EuiToolTipProps, +} from '@elastic/eui'; + +interface IDataSelectorRefresherProps { + tooltipText: string; + onRefresh: () => void; + buttonProps?: Partial; + toolTipProps?: Partial; +} + +export const DataSelectorRefresher: React.FC = React.memo( + ({ tooltipText, onRefresh, buttonProps, toolTipProps }) => { + return ( + + + + + + ); + } +); diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx index 63c2437cc7b3..6cb701c3b132 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx @@ -6,16 +6,24 @@ import React from 'react'; import { render, act, screen, fireEvent } from '@testing-library/react'; import { DataSourceSelectable } from './datasource_selectable'; -import { DataSourceType, GenericDataSource } from '../datasource_services'; import { DataSourceGroup, DataSourceOption } from './types'; +import { DataSource } from '../datasource/datasource'; +import { + DEFAULT_DATA_SOURCE_DISPLAY_NAME, + S3_GLUE_DATA_SOURCE_DISPLAY_NAME, + DEFAULT_DATA_SOURCE_TYPE, + defaultDataSourceMetadata, + s3DataSourceMetadata, +} from '../constants'; describe('DataSourceSelectable', () => { - let dataSourcesMock: GenericDataSource[]; + let dataSourcesMock: DataSource[]; let dataSourceOptionListMock: DataSourceGroup[]; let selectedSourcesMock: DataSourceOption[]; let setSelectedSourcesMock: (sources: DataSourceOption[]) => void = jest.fn(); let setDataSourceOptionListMock: (sources: DataSourceGroup[]) => void = jest.fn(); let onFetchDataSetErrorMock: (error: Error) => void = jest.fn(); + const onRefresh: () => void = jest.fn(); beforeEach(() => { dataSourcesMock = [ @@ -23,7 +31,14 @@ describe('DataSourceSelectable', () => { getDataSet: jest.fn().mockResolvedValue([]), getType: jest.fn().mockReturnValue('DEFAULT_INDEX_PATTERNS'), getName: jest.fn().mockReturnValue('SomeName'), - } as unknown) as DataSourceType, + getMetadata: jest.fn().mockReturnValue(defaultDataSourceMetadata), + } as unknown) as DataSource, + ({ + getDataSet: jest.fn().mockResolvedValue([]), + getType: jest.fn().mockReturnValue('s3glue'), + getName: jest.fn().mockReturnValue('Amazon S3'), + getMetadata: jest.fn().mockReturnValue(s3DataSourceMetadata), + } as unknown) as DataSource, ]; dataSourceOptionListMock = []; @@ -42,6 +57,7 @@ describe('DataSourceSelectable', () => { onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onGetDataSetError={onFetchDataSetErrorMock} + onRefresh={onRefresh} /> ); }); @@ -56,6 +72,7 @@ describe('DataSourceSelectable', () => { onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onGetDataSetError={onFetchDataSetErrorMock} + onRefresh={onRefresh} /> ); }); @@ -75,6 +92,7 @@ describe('DataSourceSelectable', () => { onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onGetDataSetError={onFetchDataSetErrorMock} + onRefresh={onRefresh} /> ); }); @@ -104,13 +122,15 @@ describe('DataSourceSelectable', () => { getDataSet: jest.fn().mockResolvedValue([]), getType: jest.fn().mockReturnValue('DEFAULT_INDEX_PATTERNS'), getName: jest.fn().mockReturnValue('Index patterns'), - } as unknown) as DataSourceType, + getMetadata: jest.fn().mockReturnValue(defaultDataSourceMetadata), + } as unknown) as DataSource, ]} dataSourceOptionList={mockDataSourceOptionList} selectedSources={selectedSourcesMock} onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onGetDataSetError={onFetchDataSetErrorMock} + onRefresh={onRefresh} /> ); @@ -153,14 +173,16 @@ describe('DataSourceSelectable', () => { ({ getDataSet: jest.fn().mockResolvedValue([]), getType: jest.fn().mockReturnValue('s3glue'), - getName: jest.fn().mockReturnValue('Amazon S3'), - } as unknown) as DataSourceType, + getName: jest.fn().mockReturnValue(S3_GLUE_DATA_SOURCE_DISPLAY_NAME), + getMetadata: jest.fn().mockReturnValue(s3DataSourceMetadata), + } as unknown) as DataSource, ]} dataSourceOptionList={mockDataSourceOptionList} selectedSources={selectedSourcesMock} onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onGetDataSetError={onFetchDataSetErrorMock} + onRefresh={onRefresh} /> ); @@ -202,15 +224,17 @@ describe('DataSourceSelectable', () => { dataSources={[ ({ getDataSet: jest.fn().mockResolvedValue([]), - getType: jest.fn().mockReturnValue('DEFAULT_INDEX_PATTERNS'), - getName: jest.fn().mockReturnValue('Index patterns'), - } as unknown) as DataSourceType, + getType: jest.fn().mockReturnValue(DEFAULT_DATA_SOURCE_TYPE), + getName: jest.fn().mockReturnValue(DEFAULT_DATA_SOURCE_DISPLAY_NAME), + getMetadata: jest.fn().mockReturnValue(defaultDataSourceMetadata), + } as unknown) as DataSource, ]} dataSourceOptionList={mockDataSourceOptionListWithDuplicates} selectedSources={selectedSourcesMock} onDataSourceSelect={handleSelect} setDataSourceOptionList={setDataSourceOptionListMock} onGetDataSetError={onFetchDataSetErrorMock} + onRefresh={onRefresh} /> ); @@ -224,4 +248,21 @@ describe('DataSourceSelectable', () => { expect.objectContaining([{ key: 'unique-key-3', label: 'duplicate-index-pattern' }]) ); }); + + it('should trigger onRefresh when the refresh button is clicked', () => { + const { getByLabelText } = render( + + ); + const refreshButton = getByLabelText('sourceRefresh'); + fireEvent.click(refreshButton); + expect(onRefresh).toHaveBeenCalled(); + }); }); diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 2aaa023de324..d9cdccc948dc 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -6,110 +6,109 @@ import React, { useEffect, useCallback, useMemo } from 'react'; import { EuiComboBox } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import { ISourceDataSet, IndexPatternOption } from '../datasource'; -import { DataSourceType, GenericDataSource } from '../datasource_services'; -import { DataSourceGroup, DataSourceSelectableProps } from './types'; - -type DataSourceTypeKey = 'DEFAULT_INDEX_PATTERNS' | 's3glue' | 'spark'; - -// Mapping between datasource type and its display name. -// Temporary solution, will be removed along with refactoring of data source APIs -const DATASOURCE_TYPE_DISPLAY_NAME_MAP: Record = { - DEFAULT_INDEX_PATTERNS: i18n.translate('dataExplorer.dataSourceSelector.indexPatternGroupTitle', { - defaultMessage: 'Index patterns', - }), - s3glue: i18n.translate('dataExplorer.dataSourceSelector.amazonS3GroupTitle', { - defaultMessage: 'Amazon S3', - }), - spark: i18n.translate('dataExplorer.dataSourceSelector.sparkGroupTitle', { - defaultMessage: 'Spark', - }), -}; - -type DataSetType = ISourceDataSet['data_sets'][number]; - -// Get data sets for a given datasource and returns it along with the source. -const getDataSetWithSource = async (ds: GenericDataSource): Promise => { - const dataSet = await ds.getDataSet(); - return { - ds, - data_sets: dataSet, - }; +import { DataSource, DataSetWithDataSource, IndexPatternOption } from '../datasource'; +import { DataSourceGroup, DataSourceOption, DataSourceSelectableProps } from './types'; +import { DataSelectorRefresher } from './data_selector_refresher'; +import { + DATA_SELECTOR_DEFAULT_PLACEHOLDER, + DATA_SELECTOR_REFRESHER_POPOVER_TEXT, + DATA_SELECTOR_S3_DATA_SOURCE_GROUP_HINT_LABEL, +} from '../constants'; + +// Asynchronously retrieves and formats dataset from a given data source. +const getAndFormatDataSetFromDataSource = async ( + ds: DataSource +): Promise> => { + const { dataSets } = await ds.getDataSet(); + return { ds, list: dataSets } as DataSetWithDataSource; }; // Map through all data sources and get their respective data sets. -const getDataSets = (dataSources: GenericDataSource[]) => - dataSources.map((ds) => getDataSetWithSource(ds)); - -export const isIndexPatterns = (dataSet: DataSetType): dataSet is IndexPatternOption => { - if (typeof dataSet === 'string') return false; - - return !!(dataSet.title && dataSet.id); -}; - -// Get the option format for the combo box from the dataSource and dataSet. -export const getSourceOptions = (dataSource: DataSourceType, dataSet: DataSetType) => { - const optionContent = { +const getAllDataSets = (dataSources: DataSource[]) => + dataSources.map((ds) => getAndFormatDataSetFromDataSource(ds)); + +export const isIndexPatterns = (dataSet: unknown) => + typeof dataSet !== 'string' && + 'title' in (dataSet as any) && + 'id' in (dataSet as any) && + !!(dataSet as any).title && + !!(dataSet as any).id; + +// Mapping function to get the option format for the combo box from the dataSource and dataSet. +const mapToOption = ( + dataSource: DataSource, + dataSet: DataSetWithDataSource | undefined = undefined +): DataSourceOption => { + const baseOption = { type: dataSource.getType(), name: dataSource.getName(), ds: dataSource, }; - if (isIndexPatterns(dataSet)) { + if (dataSet && 'title' in dataSet && 'id' in dataSet && isIndexPatterns(dataSet)) { return { - ...optionContent, - label: dataSet.title, - value: dataSet.id, - key: dataSet.id, + ...baseOption, + label: dataSet.title as string, + value: dataSet.id as string, + key: dataSet.id as string, }; } return { - ...optionContent, + ...baseOption, label: dataSource.getName(), value: dataSource.getName(), + key: dataSource.getId(), }; }; -// Convert data sets into a structured format suitable for selector rendering. -const getSourceList = (allDataSets: ISourceDataSet[]) => { - const finalList = [] as DataSourceGroup[]; - allDataSets.forEach((curDataSet) => { - const typeKey = curDataSet.ds.getType() as DataSourceTypeKey; - let groupName = - DATASOURCE_TYPE_DISPLAY_NAME_MAP[typeKey] || - i18n.translate('dataExplorer.dataSourceSelector.defaultGroupTitle', { - defaultMessage: 'Default Group', - }); - - // add '- Opens in Log Explorer' to hint user that selecting these types of data sources - // will lead to redirection to log explorer - if (typeKey !== 'DEFAULT_INDEX_PATTERNS') { - groupName = `${groupName}${i18n.translate('dataExplorer.dataSourceSelector.redirectionHint', { - defaultMessage: ' - Opens in Log Explorer', - })}`; - } +// Function to add or update groups in a reduction process +const addOrUpdateGroup = ( + existingGroups: DataSourceGroup[], + dataSource: DataSource, + option: DataSourceOption +) => { + const metadata = dataSource.getMetadata(); + const groupType = metadata.ui.groupType; + let groupName = + metadata.ui.typeLabel || + i18n.translate('dataExplorer.dataSourceSelector.defaultGroupTitle', { + defaultMessage: 'Default Group', + }); - const existingGroup = finalList.find((item) => item.label === groupName); - const mappedOptions = curDataSet.data_sets.map((dataSet) => - getSourceOptions(curDataSet.ds, dataSet) - ); + if (dataSource.getType() !== 'DEFAULT_INDEX_PATTERNS') { + groupName += i18n.translate('dataExplorer.dataSourceSelector.redirectionHint', { + defaultMessage: DATA_SELECTOR_S3_DATA_SOURCE_GROUP_HINT_LABEL, + }); + } - // check if add new datasource group or add to existing one - if (existingGroup) { - // options deduplication - const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.label)); - const nonDuplicateOptions = mappedOptions.filter((opt) => !existingOptionIds.has(opt.label)); + const group = existingGroups.find((g: DataSourceGroup) => g.id === groupType); + if (group && !group.options.some((opt) => opt.key === option.key)) { + group.options.push(option); + } else { + existingGroups.push({ + groupType, + label: groupName, + options: [option], + id: metadata.ui.groupType, // id for each group + }); + } +}; - // 'existingGroup' directly references an item in the finalList - // pushing options to 'existingGroup' updates the corresponding item in finalList - existingGroup.options.push(...nonDuplicateOptions); +const consolidateDataSourceGroups = ( + dataSets: DataSetWithDataSource[], + dataSources: DataSource[] +) => { + return [...dataSets, ...dataSources].reduce((dsGroup, item) => { + if ('list' in item && item.ds) { + // Confirm item is a DataSet + const options = item.list.map((dataset) => mapToOption(item.ds, dataset)); + options.forEach((option) => addOrUpdateGroup(dsGroup, item.ds, option)); } else { - finalList.push({ - label: groupName, - options: mappedOptions, - }); + // Handle DataSource directly + const option = mapToOption(item as InstanceType); + addOrUpdateGroup(dsGroup, item as InstanceType, option); } - }); - return finalList; + return dsGroup; + }, []); }; /** @@ -123,13 +122,23 @@ export const DataSourceSelectable = ({ setDataSourceOptionList, onGetDataSetError, // onGetDataSetError, Callback for handling get data set errors. Ensure it's memoized. singleSelection = { asPlainText: true }, + onRefresh, ...comboBoxProps }: DataSourceSelectableProps) => { // This effect gets data sets and prepares the datasource list for UI rendering. useEffect(() => { - Promise.all(getDataSets(dataSources)) - .then((results) => { - setDataSourceOptionList(getSourceList(results)); + Promise.all( + getAllDataSets( + dataSources.filter((ds) => ds.getMetadata().ui.selector.displayDatasetsAsSource) + ) + ) + .then((dataSetResults) => { + setDataSourceOptionList( + consolidateDataSourceGroups( + dataSetResults as DataSetWithDataSource[], + dataSources.filter((ds) => !ds.getMetadata().ui.selector.displayDatasetsAsSource) + ) + ); }) .catch((e) => onGetDataSetError(e)); }, [dataSources, setDataSourceOptionList, onGetDataSetError]); @@ -155,13 +164,19 @@ export const DataSourceSelectable = ({ {...comboBoxProps} data-test-subj="dataExplorerDSSelect" placeholder={i18n.translate('data.datasource.selectADatasource', { - defaultMessage: 'Select a datasource', + defaultMessage: DATA_SELECTOR_DEFAULT_PLACEHOLDER, })} options={memorizedDataSourceOptionList as any} selectedOptions={selectedSources as any} onChange={handleSourceChange} singleSelection={singleSelection} isClearable={false} + append={ + + } /> ); }; diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index 0fa1bf21cb19..fe1e4360e961 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -8,26 +8,31 @@ */ import { EuiComboBoxProps, EuiComboBoxSingleSelectionShape } from '@elastic/eui'; -import { GenericDataSource } from '../datasource_services'; +import { DataSource } from '../datasource/datasource'; export interface DataSourceGroup { label: string; + id: string; options: DataSourceOption[]; + groupType: string; } export interface DataSourceOption { + key: string; + name: string; label: string; value: string; type: string; - ds: GenericDataSource; + ds: DataSource; } export interface DataSourceSelectableProps extends Pick, 'fullWidth'> { - dataSources: GenericDataSource[]; + dataSources: DataSource[]; onDataSourceSelect: (dataSourceOption: DataSourceOption[]) => void; singleSelection?: boolean | EuiComboBoxSingleSelectionShape; onGetDataSetError: (error: Error) => void; dataSourceOptionList: DataSourceGroup[]; selectedSources: DataSourceOption[]; setDataSourceOptionList: (dataSourceList: DataSourceGroup[]) => void; + onRefresh: () => void; } diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts index 2c8f393d7093..e1d2077f60e5 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts @@ -3,29 +3,58 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { waitFor } from '@testing-library/dom'; import { DataSource } from '../datasource'; import { IndexPatternsService } from '../../index_patterns'; import { DataSourceService } from '../datasource_services'; +import { + LocalDSDataSetParams, + LocalDSDataSetResponse, + LocalDSMetadata, + LocalDSQueryParams, + LocalDSQueryResponse, +} from '../default_datasource/default_datasource'; +import { DataSourceUIGroupType } from '../datasource/types'; +import { DEFAULT_DATA_SOURCE_DISPLAY_NAME } from '../register_default_datasource'; + +const defaultDataSourceMetadata = { + ui: { + label: DEFAULT_DATA_SOURCE_DISPLAY_NAME, + typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME, + groupType: DataSourceUIGroupType.defaultOpenSearchDataSource, + selector: { + displayDatasetsAsSource: true, + }, + }, +}; -class MockDataSource extends DataSource { +class MockDataSource extends DataSource< + LocalDSMetadata, + LocalDSDataSetParams, + LocalDSDataSetResponse, + LocalDSQueryParams, + LocalDSQueryResponse +> { private readonly indexPattern; constructor({ + id, name, type, metadata, indexPattern, }: { + id: string; name: string; type: string; metadata: any; indexPattern: IndexPatternsService; }) { - super(name, type, metadata); + super({ id, name, type, metadata }); this.indexPattern = indexPattern; } - async getDataSet(dataSetParams?: any) { + async getDataSet(dataSetParams?: LocalDSDataSetParams): Promise { await this.indexPattern.ensureDefaultIndexPattern(); return await this.indexPattern.getCache(); } @@ -34,24 +63,28 @@ class MockDataSource extends DataSource { return true; } - async runQuery(queryParams: any) { - return undefined; + async runQuery(queryParams: any): Promise { + return { + data: {}, + }; } } const mockIndexPattern = {} as IndexPatternsService; const mockConfig1 = { + id: 'test_datasource1', name: 'test_datasource1', type: 'mock1', - metadata: null, + metadata: defaultDataSourceMetadata, indexPattern: mockIndexPattern, }; const mockConfig2 = { + id: 'test_datasource2', name: 'test_datasource2', type: 'mock1', - metadata: null, + metadata: defaultDataSourceMetadata, indexPattern: mockIndexPattern, }; @@ -79,7 +112,7 @@ describe('DataSourceService', () => { const ds = new MockDataSource(mockConfig1); await service.registerDataSource(ds); await expect(service.registerDataSource(ds)).rejects.toThrow( - 'Unable to register datasource test_datasource1, error: datasource name exists.' + 'Unable to register data source test_datasource1, error: data source exists.' ); }); @@ -95,15 +128,31 @@ describe('DataSourceService', () => { }); }); + it('loads data sources successfully with fetchers', async () => { + const service = DataSourceService.getInstance(); + const fetcherMock = jest.fn(() => { + service.registerDataSource(new MockDataSource(mockConfig1)); // Simulates adding a new data source after fetching + }); + service.registerDataSourceFetchers([{ type: 'mock', registerDataSources: fetcherMock }]); + + await service.load(); + expect(fetcherMock).toHaveBeenCalledTimes(1); + service.getDataSources$().subscribe((dataSources) => { + expect(Object.keys(dataSources)).toContain('test_datasource1'); + }); + }); + it('retrieves registered data sources based on filters', () => { const service = DataSourceService.getInstance(); const ds1 = new MockDataSource(mockConfig1); const ds2 = new MockDataSource(mockConfig2); service.registerMultipleDataSources([ds1, ds2]); const filter = { names: ['test_datasource1'] }; - const retrievedDataSources = service.getDataSources(filter); - expect(retrievedDataSources).toHaveProperty('test_datasource1'); - expect(retrievedDataSources).not.toHaveProperty('test_datasource2'); + waitFor(() => { + const retrievedDataSources = service.getDataSources$(filter); + expect(retrievedDataSources).toHaveProperty('test_datasource1'); + expect(retrievedDataSources).not.toHaveProperty('test_datasource2'); + }); }); it('returns all data sources if no filters provided', () => { @@ -111,8 +160,88 @@ describe('DataSourceService', () => { const ds1 = new MockDataSource(mockConfig1); const ds2 = new MockDataSource(mockConfig2); service.registerMultipleDataSources([ds1, ds2]); - const retrievedDataSources = service.getDataSources(); - expect(retrievedDataSources).toHaveProperty('test_datasource1'); - expect(retrievedDataSources).toHaveProperty('test_datasource2'); + waitFor(() => { + const retrievedDataSources = service.getDataSources$(); + expect(retrievedDataSources).toHaveProperty('test_datasource1'); + expect(retrievedDataSources).toHaveProperty('test_datasource2'); + }); + }); + + it('should reset and load data sources using registered fetchers', async () => { + const service = DataSourceService.getInstance(); + const fetcherMock = jest.fn(); + service.registerDataSourceFetchers([{ type: 'mock', registerDataSources: fetcherMock }]); + + service.load(); + expect(fetcherMock).toHaveBeenCalled(); + expect( + service.getDataSources$().subscribe((dataSources) => { + expect(Object.keys(dataSources).length).toBe(0); + }) + ); + }); + + it('handles failures in data fetchers gracefully during load', async () => { + const service = DataSourceService.getInstance(); + const fetcherMock = jest.fn(() => { + throw new Error('Failed to fetch data sources'); + }); + service.registerDataSourceFetchers([{ type: 'mock', registerDataSources: fetcherMock }]); + + await service.load(); + expect(fetcherMock).toHaveBeenCalledTimes(1); + service.getDataSources$().subscribe((dataSources) => { + expect(Object.keys(dataSources).length).toBe(0); // Assuming reset clears everything regardless of fetcher success + }); + }); + + it('should call load method when reload is invoked', () => { + const service = DataSourceService.getInstance(); + const loadSpy = jest.spyOn(service, 'load'); + service.reload(); + expect(loadSpy).toHaveBeenCalled(); + }); + + it('should reset all registered data sources', () => { + const service = DataSourceService.getInstance(); + const ds = new MockDataSource(mockConfig1); + service.registerDataSource(ds); // Ensure there is at least one data source + + service.reset(); + expect( + service.getDataSources$().subscribe((dataSources) => { + expect(Object.keys(dataSources).length).toBe(0); + }) + ); + }); + + it('successfully reloads and updates data sources', async () => { + const service = DataSourceService.getInstance(); + const fetcherMock = jest.fn().mockResolvedValue('Data fetched successfully'); + + // Register fetchers and perform initial load + service.registerDataSourceFetchers([{ type: 'default', registerDataSources: fetcherMock }]); + await service.load(); + + // Reset mocks to clear previous calls and reload + fetcherMock.mockClear(); + service.reload(); + + expect(fetcherMock).toHaveBeenCalledTimes(1); + service.getDataSources$().subscribe((dataSources) => { + // Expect that new data has been loaded; specifics depend on your implementation + expect(dataSources).toEqual(expect.anything()); // Adjust expectation based on your data structure + }); + }); + + it('ensures complete clearance of data sources on reset', () => { + const service = DataSourceService.getInstance(); + service.registerDataSource(new MockDataSource(mockConfig1)); + service.registerDataSource(new MockDataSource(mockConfig2)); + + service.reset(); + service.getDataSources$().subscribe((dataSources) => { + expect(Object.keys(dataSources).length).toBe(0); + }); }); }); diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index 9cf674585366..9fa335e0754e 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -4,18 +4,24 @@ */ import { BehaviorSubject } from 'rxjs'; +import { map } from 'rxjs/operators'; import { DataSourceRegistrationError, - GenericDataSource, IDataSourceFilter, IDataSourceRegistrationResult, + DataSourceFetcher, } from './types'; +import { DataSource } from '../datasource/datasource'; export class DataSourceService { private static dataSourceService: DataSourceService; // A record to store all registered data sources, using the data source name as the key. - private dataSources: Record = {}; - private dataSourcesSubject: BehaviorSubject>; + private dataSources: Record = {}; + private dataSourcesSubject: BehaviorSubject>; + // A record to store all data source fetchers, using the data source type as the key. + // Once application starts, all the different types of data source supported with have their fetchers registered here. + // And it becomes the single source of truth for reloading data sources. + private dataSourceFetchers: Record = {}; private constructor() { this.dataSourcesSubject = new BehaviorSubject(this.dataSources); @@ -36,7 +42,7 @@ export class DataSourceService { * @returns An array of registration results, one for each data source. */ async registerMultipleDataSources( - datasources: GenericDataSource[] + datasources: DataSource[] ): Promise { return Promise.all(datasources.map((ds) => this.registerDataSource(ds))); } @@ -50,21 +56,24 @@ export class DataSourceService { * @returns A registration result indicating success or failure. * @throws {DataSourceRegistrationError} Throws an error if a data source with the same name already exists. */ - async registerDataSource(ds: GenericDataSource): Promise { - const dsName = ds.getName(); - if (dsName in this.dataSources) { + async registerDataSource(ds: DataSource): Promise { + const dataSourceId = ds.getId(); + if (dataSourceId in this.dataSources) { throw new DataSourceRegistrationError( - `Unable to register datasource ${dsName}, error: datasource name exists.` + `Unable to register data source ${ds.getName()}, error: data source exists.` ); } else { - this.dataSources[dsName] = ds; + this.dataSources[dataSourceId] = ds; this.dataSourcesSubject.next(this.dataSources); return { success: true, info: '' } as IDataSourceRegistrationResult; } } - public get dataSources$() { - return this.dataSourcesSubject.asObservable(); + private isFilterEmpty(filter: IDataSourceFilter): boolean { + // Check if all filter properties are either undefined or empty arrays + return Object.values(filter).every( + (value) => !value || (Array.isArray(value) && value.length === 0) + ); } /** @@ -74,15 +83,71 @@ export class DataSourceService { * @param filter - An optional object with filter criteria (e.g., names of data sources). * @returns A record of filtered data sources. */ - getDataSources(filter?: IDataSourceFilter): Record { - if (!filter || !Array.isArray(filter.names) || filter.names.length === 0) - return this.dataSources; + public getDataSources$(filter?: IDataSourceFilter) { + return this.dataSourcesSubject.asObservable().pipe( + map((dataSources) => { + // Check if the filter is provided and valid + if (!filter || this.isFilterEmpty(filter)) { + return dataSources; + } - return filter.names.reduce>((filteredDataSources, dsName) => { - if (dsName in this.dataSources) { - filteredDataSources[dsName] = this.dataSources[dsName]; + // Apply filter + return Object.entries(dataSources).reduce((acc, [id, dataSource]) => { + const matchesId = !filter.ids || filter.ids.includes(id); + const matchesName = !filter.names || filter.names.includes(dataSource.getName()); + const matchesType = !filter.types || filter.types.includes(dataSource.getType()); + + if (matchesId && matchesName && matchesType) { + acc[id] = dataSource; + } + + return acc; + }, {} as Record); + }) + ); + } + + /** + * Registers functions responsible for fetching data for each data source type. + * + * @param fetchers - An array of fetcher configurations, each specifying how to fetch data for a specific data source type. + */ + registerDataSourceFetchers(fetchers: DataSourceFetcher[]) { + return fetchers.forEach((fetcher) => { + if (!this.dataSourceFetchers[fetcher.type]) + this.dataSourceFetchers[fetcher.type] = fetcher.registerDataSources; + }); + } + + /** + * Calls all registered data fetching functions to update data sources. + * Typically used to initialize or refresh the data source configurations. + */ + load() { + this.reset(); + Object.values(this.dataSourceFetchers).forEach((fetch) => { + try { + fetch(); // Directly call the synchronous fetch function + } catch (error) { + // Handle fetch errors or take corrective actions here + // TO-DO: Add error handling, maybe collect errors and show them in UI } - return filteredDataSources; - }, {} as Record); + }); + } + + /** + * Reloads all data source configurations by re-invoking the load method. + * Used for refreshing the system to reflect changes such as new data source registrations. + */ + reload() { + this.load(); + } + + /** + * Resets all registered data sources. + */ + reset() { + this.dataSources = {}; + this.dataSourcesSubject.next(this.dataSources); } } diff --git a/src/plugins/data/public/data_sources/datasource_services/index.ts b/src/plugins/data/public/data_sources/datasource_services/index.ts index 14db278b47a5..a6467904ea8d 100644 --- a/src/plugins/data/public/data_sources/datasource_services/index.ts +++ b/src/plugins/data/public/data_sources/datasource_services/index.ts @@ -8,6 +8,4 @@ export { IDataSourceFilter, IDataSourceRegistrationResult, DataSourceRegistrationError, - DataSourceType, - GenericDataSource, } from './types'; diff --git a/src/plugins/data/public/data_sources/datasource_services/types.ts b/src/plugins/data/public/data_sources/datasource_services/types.ts index cb5bb31500b4..1e657802a0a8 100644 --- a/src/plugins/data/public/data_sources/datasource_services/types.ts +++ b/src/plugins/data/public/data_sources/datasource_services/types.ts @@ -8,19 +8,13 @@ * in future releases. */ -import { - DataSource, - DataSourceFactory, - IDataSetParams, - IDataSourceMetaData, - IDataSourceQueryParams, - IDataSourceQueryResult, - ISourceDataSet, -} from '../datasource'; +import { DataSourceFactory } from '../datasource'; import { DataSourceService } from './datasource_service'; export interface IDataSourceFilter { - names: string[]; + ids?: string[]; // Array of data source IDs to filter by + names?: string[]; // Array of data source names to filter by + types?: string[]; // Array of data source types to filter by } export interface IDataSourceRegistrationResult { @@ -43,12 +37,7 @@ export interface DataSourceStart { dataSourceFactory: DataSourceFactory; } -export type DataSourceType = DataSource< - IDataSourceMetaData, - IDataSetParams, - ISourceDataSet, - IDataSourceQueryParams, - IDataSourceQueryResult ->; - -export type GenericDataSource = DataSource; +export interface DataSourceFetcher { + type: string; + registerDataSources: () => void; +} diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts index aedc6cd3853a..d302378c0177 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts @@ -4,6 +4,7 @@ */ import { IndexPatternsService } from '../../index_patterns'; +import { defaultDataSourceMetadata } from '../constants'; import { DefaultDslDataSource } from './default_datasource'; describe('DefaultDslDataSource', () => { @@ -18,9 +19,10 @@ describe('DefaultDslDataSource', () => { it('should ensure default index pattern and get cache', async () => { const dataSource = new DefaultDslDataSource({ + id: 'testId', name: 'testName', type: 'testType', - metadata: {}, + metadata: defaultDataSourceMetadata, indexPatterns: indexPatternsMock, }); @@ -30,11 +32,60 @@ describe('DefaultDslDataSource', () => { expect(indexPatternsMock.getCache).toHaveBeenCalledTimes(1); }); - it('should throw an error', async () => { + it('should return an empty dataset if getCache returns an empty array', async () => { + indexPatternsMock.getCache.mockResolvedValue([]); const dataSource = new DefaultDslDataSource({ + id: 'testId', name: 'testName', type: 'testType', - metadata: {}, + metadata: defaultDataSourceMetadata, + indexPatterns: indexPatternsMock, + }); + + const result = await dataSource.getDataSet(); + expect(result.dataSets).toEqual([]); + }); + + it('should return a populated dataset if getCache returns non-empty array', async () => { + const mockSavedObjects = [ + { id: '1', attributes: { title: 'Index1' } }, + { id: '2', attributes: { title: 'Index2' } }, + ]; + indexPatternsMock.getCache.mockResolvedValue(mockSavedObjects); + const dataSource = new DefaultDslDataSource({ + id: 'testId', + name: 'testName', + type: 'testType', + metadata: defaultDataSourceMetadata, + indexPatterns: indexPatternsMock, + }); + + const result = await dataSource.getDataSet(); + expect(result.dataSets).toEqual([ + { id: '1', title: 'Index1' }, + { id: '2', title: 'Index2' }, + ]); + }); + + it('should handle errors thrown by getCache', async () => { + indexPatternsMock.getCache.mockRejectedValue(new Error('Cache fetch failed')); + const dataSource = new DefaultDslDataSource({ + id: 'testId', + name: 'testName', + type: 'testType', + metadata: defaultDataSourceMetadata, + indexPatterns: indexPatternsMock, + }); + + await expect(dataSource.getDataSet()).rejects.toThrow('Cache fetch failed'); + }); + + it('should return true on default data source connection', async () => { + const dataSource = new DefaultDslDataSource({ + id: 'testId', + name: 'testName', + type: 'testType', + metadata: defaultDataSourceMetadata, indexPatterns: indexPatternsMock, }); @@ -43,13 +94,16 @@ describe('DefaultDslDataSource', () => { it('should return null', async () => { const dataSource = new DefaultDslDataSource({ + id: 'testId', name: 'testName', type: 'testType', - metadata: {}, + metadata: defaultDataSourceMetadata, indexPatterns: indexPatternsMock, }); - const result = await dataSource.runQuery({}); - expect(result).toBeUndefined(); + const result = await dataSource.runQuery(); + expect(result).toStrictEqual({ + data: {}, + }); }); }); diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts index c3b5d2a4cf99..db65a7df903d 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts @@ -4,44 +4,68 @@ */ import { SavedObject } from '../../../../../core/public'; -import { IndexPatternSavedObjectAttrs } from '../../index_patterns/index_patterns'; -import { DataSource, DataSourceConfig, IndexPatternOption } from '../datasource'; +import { + IndexPatternSavedObjectAttrs, + IndexPatternsContract, +} from '../../index_patterns/index_patterns'; +import { DataSource, IndexPatternOption } from '../datasource'; +import { + IDataSetParams, + IDataSourceDataSet, + IDataSourceMetadata, + IDataSourceQueryParams, + IDataSourceQueryResponse, + IDataSourceSettings, +} from '../datasource/types'; + +export type LocalDSMetadata = IDataSourceMetadata; +export type LocalDSDataSetParams = IDataSetParams; +export type LocalDSDataSetResponse = IDataSourceDataSet; +export type LocalDSQueryParams = IDataSourceQueryParams; +export type LocalDSQueryResponse = IDataSourceQueryResponse; +export interface LocalDataSourceSettings extends IDataSourceSettings { + indexPatterns: IndexPatternsContract; +} export class DefaultDslDataSource extends DataSource< - any, - any, - Promise, - any, - any + LocalDSMetadata, + LocalDSDataSetParams, + LocalDSDataSetResponse, + LocalDSQueryParams, + LocalDSQueryResponse > { - private readonly indexPatterns; + private readonly indexPatterns: IndexPatternsContract; - constructor({ name, type, metadata, indexPatterns }: DataSourceConfig) { - super(name, type, metadata); + constructor({ id, name, type, metadata, indexPatterns }: LocalDataSourceSettings) { + super({ id, name, type, metadata }); this.indexPatterns = indexPatterns; } - async getDataSet(dataSetParams?: any) { + async getDataSet(): Promise { await this.indexPatterns.ensureDefaultIndexPattern(); const savedObjectLst = await this.indexPatterns.getCache(); if (!Array.isArray(savedObjectLst)) { - return undefined; + return { dataSets: [] }; } - return savedObjectLst.map((savedObject: SavedObject) => { - return { - id: savedObject.id, - title: savedObject.attributes.title, - }; - }); + return { + dataSets: savedObjectLst.map((savedObject: SavedObject) => { + return { + id: savedObject.id, + title: savedObject.attributes.title, + }; + }), + }; } async testConnection(): Promise { return true; } - async runQuery(queryParams: any) { - return undefined; + async runQuery(): Promise { + return { + data: {}, + }; } } diff --git a/src/plugins/data/public/data_sources/register_default_datasource.ts b/src/plugins/data/public/data_sources/register_default_datasource.ts index 8dece27e82eb..ce169955704b 100644 --- a/src/plugins/data/public/data_sources/register_default_datasource.ts +++ b/src/plugins/data/public/data_sources/register_default_datasource.ts @@ -3,24 +3,38 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { i18n } from '@osd/i18n'; +import { htmlIdGenerator } from '@elastic/eui'; import { DataPublicPluginStart } from '../types'; -import { DefaultDslDataSource } from './default_datasource'; +import { DataSourceUIGroupType } from './datasource/types'; +import { + DEFAULT_DATA_SOURCE_DISPLAY_NAME, + DEFAULT_DATA_SOURCE_NAME, + DEFAULT_DATA_SOURCE_TYPE, +} from './constants'; -export const DEFAULT_DATASOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; -export const DEFAULT_DATASOURCE_NAME = i18n.translate('data.datasource.type.openSearchDefault', { - defaultMessage: 'OpenSearch Default', -}); - -export const registerDefaultDatasource = (data: Omit) => { - // Datasources registrations for index patterns datasource +/** + * Registers the default data source with the provided data excluding 'ui'. + * This sets up the default cluster data source with predefined configurations using constants. + * @param data - Data necessary to configure the data source, except for 'ui'. + */ +export const registerDefaultDataSource = (data: Omit) => { + // Registrations of index patterns as default data source const { dataSourceService, dataSourceFactory } = data.dataSources; - dataSourceFactory.registerDataSourceType(DEFAULT_DATASOURCE_TYPE, DefaultDslDataSource); dataSourceService.registerDataSource( - dataSourceFactory.getDataSourceInstance(DEFAULT_DATASOURCE_TYPE, { - name: DEFAULT_DATASOURCE_NAME, - type: DEFAULT_DATASOURCE_TYPE, - metadata: null, + dataSourceFactory.getDataSourceInstance(DEFAULT_DATA_SOURCE_TYPE, { + id: htmlIdGenerator(DEFAULT_DATA_SOURCE_NAME)(DEFAULT_DATA_SOURCE_TYPE), + name: DEFAULT_DATA_SOURCE_NAME, + type: DEFAULT_DATA_SOURCE_TYPE, + metadata: { + ui: { + label: DEFAULT_DATA_SOURCE_DISPLAY_NAME, // display name of your data source, + typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME, // display name of your data source type, + groupType: DataSourceUIGroupType.defaultOpenSearchDataSource, + selector: { + displayDatasetsAsSource: true, // when true, selector UI will render data sets with source by calling getDataSets() + }, + }, + }, indexPatterns: data.indexPatterns, }) ); diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 3b559f9e6c63..127e3dc720da 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -500,19 +500,17 @@ export { DataPublicPlugin as Plugin }; // Export datasources export { DataSource, - IDataSourceMetaData, + IDataSourceMetadata, IDataSetParams, IDataSourceQueryParams, IDataSourceQueryResult, - ISourceDataSet, - ConnectionStatus, + SourceDataSet, + DataSourceConnectionStatus, DataSourceFactory, - DataSourceConfig, } from './data_sources/datasource'; export { DataSourceRegistrationError, DataSourceService, - DataSourceType, IDataSourceFilter, IDataSourceRegistrationResult, } from './data_sources/datasource_services'; diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index 179b6c0a8c83..4917eb9db9e2 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -90,7 +90,9 @@ import { SavedObjectsClientPublicToCommon } from './index_patterns'; import { indexPatternLoad } from './index_patterns/expressions/load_index_pattern'; import { DataSourceService } from './data_sources/datasource_services'; import { DataSourceFactory } from './data_sources/datasource'; -import { registerDefaultDatasource } from './data_sources/register_default_datasource'; +import { registerDefaultDataSource } from './data_sources/register_default_datasource'; +import { DefaultDslDataSource } from './data_sources/default_datasource'; +import { DEFAULT_DATA_SOURCE_TYPE } from './data_sources/constants'; declare module '../../ui_actions/public' { export interface ActionContextMapping { @@ -218,6 +220,13 @@ export class DataPublicPlugin // Create or fetch the singleton instance const dataSourceService = DataSourceService.getInstance(); const dataSourceFactory = DataSourceFactory.getInstance(); + dataSourceFactory.registerDataSourceType(DEFAULT_DATA_SOURCE_TYPE, DefaultDslDataSource); + dataSourceService.registerDataSourceFetchers([ + { + type: DEFAULT_DATA_SOURCE_TYPE, + registerDataSources: () => registerDefaultDataSource(dataServices), + }, + ]); const dataServices = { actions: { @@ -235,7 +244,7 @@ export class DataPublicPlugin }, }; - registerDefaultDatasource(dataServices); + registerDefaultDataSource(dataServices); const SearchBar = createSearchBar({ core, diff --git a/src/plugins/data_explorer/public/components/sidebar/index.test.tsx b/src/plugins/data_explorer/public/components/sidebar/index.test.tsx index eccb0ffa909e..540321be55bf 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.test.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.test.tsx @@ -5,12 +5,14 @@ import React from 'react'; import { render, fireEvent, waitFor } from '@testing-library/react'; -import { Sidebar } from './index'; // Adjust the import path as necessary +import { of } from 'rxjs/'; +import { createMemoryHistory } from 'history'; +import { Router } from 'react-router-dom'; import { Provider } from 'react-redux'; import configureMockStore from 'redux-mock-store'; +import { Sidebar } from './index'; // Adjust the import path as necessary import { MockS3DataSource } from '../../../../discover/public/__mock__/index.test.mock'; -import { createMemoryHistory } from 'history'; -import { Router } from 'react-router-dom'; +import { s3DataSourceMetadata } from 'src/plugins/data/public/data_sources/constants'; const mockStore = configureMockStore(); const initialState = { @@ -18,6 +20,12 @@ const initialState = { }; const store = mockStore(initialState); +export const createObservable = (data: any) => { + return of(data); +}; + +const getMetaData = () => ({ ...s3DataSourceMetadata }); + jest.mock('../../../../opensearch_dashboards_react/public', () => { return { toMountPoint: jest.fn().mockImplementation((component) => () => component), @@ -27,18 +35,16 @@ jest.mock('../../../../opensearch_dashboards_react/public', () => { indexPatterns: {}, dataSources: { dataSourceService: { - dataSources$: { - subscribe: jest.fn((callback) => { - callback({ - 's3-prod-mock': new MockS3DataSource({ - name: 's3-prod-mock', - type: 's3glue', - metadata: {}, - }), - }); - return { unsubscribe: jest.fn() }; - }), - }, + getDataSources$: jest.fn().mockImplementation(() => + createObservable({ + 's3-prod-mock': new MockS3DataSource({ + id: 's3-prod-mock', + name: 's3-prod-mock', + type: 's3glue', + metadata: getMetaData(), + }), + }) + ), }, }, }, diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index ff07a59ab4b7..b65bd43950cd 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -6,7 +6,7 @@ import React, { FC, useCallback, useEffect, useState } from 'react'; import { EuiPageSideBar, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import { DataSourceGroup, DataSourceSelectable, DataSourceType } from '../../../../data/public'; +import { DataSource, DataSourceGroup, DataSourceSelectable } from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { DataExplorerServices } from '../../types'; @@ -18,7 +18,7 @@ export const Sidebar: FC = ({ children }) => { const dispatch = useTypedDispatch(); const [selectedSources, setSelectedSources] = useState([]); const [dataSourceOptionList, setDataSourceOptionList] = useState([]); - const [activeDataSources, setActiveDataSources] = useState([]); + const [activeDataSources, setActiveDataSources] = useState([]); const { services: { @@ -30,13 +30,13 @@ export const Sidebar: FC = ({ children }) => { useEffect(() => { let isMounted = true; - const subscription = dataSources.dataSourceService.dataSources$.subscribe( - (currentDataSources) => { + const subscription = dataSources.dataSourceService + .getDataSources$() + .subscribe((currentDataSources) => { if (isMounted) { setActiveDataSources(Object.values(currentDataSources)); } - } - ); + }); return () => { subscription.unsubscribe(); @@ -98,6 +98,10 @@ export const Sidebar: FC = ({ children }) => { [toasts] ); + const memorizedReload = useCallback(() => { + dataSources.dataSourceService.reload(); + }, [dataSources.dataSourceService]); + return ( { onDataSourceSelect={handleSourceSelection} selectedSources={selectedSources} onGetDataSetError={handleGetDataSetError} + onRefresh={memorizedReload} fullWidth /> diff --git a/src/plugins/discover/public/__mock__/index.test.mock.ts b/src/plugins/discover/public/__mock__/index.test.mock.ts index 6b09d1d84253..d0b47502c301 100644 --- a/src/plugins/discover/public/__mock__/index.test.mock.ts +++ b/src/plugins/discover/public/__mock__/index.test.mock.ts @@ -3,26 +3,29 @@ * SPDX-License-Identifier: Apache-2.0 */ -export class MockS3DataSource { - protected name: string; - protected type: string; - protected metadata: any; +import { DataSource } from 'src/plugins/data/public'; - constructor({ name, type, metadata }: { name: string; type: string; metadata: any }) { - this.name = name; - this.type = type; - this.metadata = metadata; +interface DataSourceConfig { + name: string; + type: string; + metadata: any; + id: string; +} + +export class MockS3DataSource extends DataSource { + constructor({ id, name, type, metadata }: DataSourceConfig) { + super({ id, name, type, metadata }); } async getDataSet(dataSetParams?: any) { - return [this.name]; + return { dataSets: [this.getName()] }; } - getName() { - return this.name; + async testConnection(): Promise { + return true; } - getType() { - return this.type; + async runQuery(queryParams: any) { + return { data: {} }; } }