From 953e448e5a0dcd352646866bbf4ee28c8f6b37c4 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 12 Mar 2024 20:08:52 +0000 Subject: [PATCH 01/23] datasource and service refactoring Signed-off-by: Eric --- .../data_sources/datasource/datasource.ts | 52 ++++++++++----- .../public/data_sources/datasource/factory.ts | 5 +- .../public/data_sources/datasource/index.ts | 5 +- .../public/data_sources/datasource/types.ts | 63 ++++++++++++------- .../datasource_selectable.test.tsx | 12 ++-- .../datasource_selectable.tsx | 19 +++--- .../data_sources/datasource_selector/types.ts | 6 +- .../datasource_services/datasource_service.ts | 28 ++++----- .../data_sources/datasource_services/index.ts | 2 - .../data_sources/datasource_services/types.ts | 20 +----- .../default_datasource/default_datasource.ts | 62 ++++++++++++------ .../register_default_datasource.ts | 14 ++++- src/plugins/data/public/index.ts | 6 +- .../public/components/sidebar/index.tsx | 4 +- 14 files changed, 171 insertions(+), 127 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/datasource.ts b/src/plugins/data/public/data_sources/datasource/datasource.ts index a2159c562064..b37d03709f36 100644 --- a/src/plugins/data/public/data_sources/datasource/datasource.ts +++ b/src/plugins/data/public/data_sources/datasource/datasource.ts @@ -13,23 +13,41 @@ * DataSourceQueryResult: Represents the result from querying the data source. */ -import { ConnectionStatus } from './types'; +import { + ConnectionStatus, + IDataSetParams, + IDataSourceDataSet, + IDataSourceMetadata, + IDataSourceQueryParams, + IDataSourceQueryResponse, + IDataSourceSettings, +} from './types'; /** * @experimental this class is experimental and might change in future releases. */ export abstract class DataSource< - DataSourceMetaData, - DataSetParams, - SourceDataSet, - DataSourceQueryParams, - DataSourceQueryResult + TMetadata extends IDataSourceMetadata = IDataSourceMetadata, + TDataSetParams extends IDataSetParams = IDataSetParams, + TDataSet extends IDataSourceDataSet = IDataSourceDataSet, + TQueryParams extends IDataSourceQueryParams = IDataSourceQueryParams, + TQueryResult extends IDataSourceQueryResponse = IDataSourceQueryResponse > { - constructor( - private readonly name: string, - private readonly type: string, - private readonly metadata: DataSourceMetaData - ) {} + private readonly id: string; + private readonly name: string; + private readonly type: string; + private readonly metadata: TMetadata; + + constructor(settings: IDataSourceSettings) { + this.id = settings.id; + this.name = settings.name; + this.type = settings.type; + this.metadata = settings.metadata; + } + + getId() { + return this.id; + } getName() { return this.name; @@ -53,18 +71,18 @@ export abstract class DataSource< * patterns for OpenSearch data source * * @experimental This API is experimental and might change in future releases. - * @returns {SourceDataSet} Dataset associated with the data source. + * @returns {Promise} Dataset associated with the data source. */ - abstract getDataSet(dataSetParams?: DataSetParams): SourceDataSet; + abstract getDataSet(dataSetParams?: TDataSetParams): Promise; /** * Abstract method to run a query against the data source. * Implementing classes need to provide the specific implementation. * * @experimental This API is experimental and might change in future releases. - * @returns {DataSourceQueryResult} Result from querying the data source. + * @returns {Promise} Result from querying the data source. */ - abstract runQuery(queryParams: DataSourceQueryParams): DataSourceQueryResult; + abstract runQuery(queryParams?: TQueryParams): Promise; /** * Abstract method to test the connection to the data source. @@ -72,8 +90,8 @@ export abstract class DataSource< * the connection status, typically indicating success or failure. * * @experimental This API is experimental and might change in future releases. - * @returns {ConnectionStatus | Promise} Status of the connection test. + * @returns {Promise} Status of the connection test. * @experimental */ - abstract testConnection(): ConnectionStatus | Promise; + abstract testConnection(): Promise; } diff --git a/src/plugins/data/public/data_sources/datasource/factory.ts b/src/plugins/data/public/data_sources/datasource/factory.ts index f0b4e36cfb82..ae61e2c566a1 100644 --- a/src/plugins/data/public/data_sources/datasource/factory.ts +++ b/src/plugins/data/public/data_sources/datasource/factory.ts @@ -8,7 +8,6 @@ * It serves as a registry for different data source types and provides a way to instantiate them. */ -import { DataSourceType } from '../datasource_services'; import { DataSource } from '../datasource'; type DataSourceClass< @@ -66,11 +65,11 @@ export class DataSourceFactory { * * @experimental This API is experimental and might change in future releases. * @param {string} type - The identifier for the data source type. - * @param {any} config - The configuration for the data source instance. + * @param {unknown} config - The configuration for the data source instance. * @returns {DataSourceType} An instance of the specified data source type. * @throws {Error} Throws an error if the data source type is not supported. */ - getDataSourceInstance(type: string, config: any): DataSourceType { + getDataSourceInstance(type: string, config: unknown): DataSource { const DataSourceClass = this.dataSourceClasses[type]; if (!DataSourceClass) { throw new Error('Unsupported data source type'); diff --git a/src/plugins/data/public/data_sources/datasource/index.ts b/src/plugins/data/public/data_sources/datasource/index.ts index 10af40fdcfa2..a3bac5bbc6a4 100644 --- a/src/plugins/data/public/data_sources/datasource/index.ts +++ b/src/plugins/data/public/data_sources/datasource/index.ts @@ -5,13 +5,12 @@ export { DataSource } from './datasource'; export { - IDataSourceMetaData, - ISourceDataSet, + IDataSourceMetadata, + SourceDataSet, IDataSetParams, IDataSourceQueryParams, IDataSourceQueryResult, ConnectionStatus, - DataSourceConfig, IndexPatternOption, } from './types'; export { DataSourceFactory } from './factory'; diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index bf77ef123a30..7b17d0b7135d 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -3,49 +3,66 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { DataSource } from './datasource'; + /** * @experimental These interfaces are experimental and might change in future releases. */ -import { IndexPatternsService } from '../../index_patterns'; -import { DataSourceType } from '../datasource_services'; - export interface IndexPatternOption { title: string; id: string; } -export interface IDataSourceMetaData { - name: string; -} - export interface IDataSourceGroup { name: string; } -export interface ISourceDataSet { - ds: DataSourceType; - data_sets: Array; +export interface SourceDataSet { + ds: DataSource; + data_sets: IndexPatternOption[]; } -// to-dos: add common interfaces for datasource -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface IDataSetParams {} +export interface IDataSetParams { + query: T; +} -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface IDataSourceQueryParams {} +export interface IDataSourceQueryParams { + query: T; +} -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface IDataSourceQueryResult {} +export interface IDataSourceQueryResult { + data: T; +} export interface ConnectionStatus { - success: boolean; - info: string; + status: string; + message: string; + error?: Error; } -export interface DataSourceConfig { - name: string; +export interface IDataSourceSettings { + id: string; type: string; - metadata: any; - indexPatterns: IndexPatternsService; + name: string; + metadata: T; +} + +export interface IDataSourceMetadata { + ui: IDataSourceUISettings; +} + +export interface IDataSourceUISettings { + label: string; // the display name of data source + typeLabel: string; // the display name of data source type + description?: string; // short description of your database + icon?: string; // uri of the icon +} + +export interface IDataSourceDataSet { + data_sets: T; +} + +export interface IDataSourceQueryResponse { + data: T; } diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx index 63c2437cc7b3..0a4384516b00 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx @@ -6,11 +6,11 @@ import React from 'react'; import { render, act, screen, fireEvent } from '@testing-library/react'; import { DataSourceSelectable } from './datasource_selectable'; -import { DataSourceType, GenericDataSource } from '../datasource_services'; import { DataSourceGroup, DataSourceOption } from './types'; +import { DataSource } from '../datasource/datasource'; describe('DataSourceSelectable', () => { - let dataSourcesMock: GenericDataSource[]; + let dataSourcesMock: DataSource[]; let dataSourceOptionListMock: DataSourceGroup[]; let selectedSourcesMock: DataSourceOption[]; let setSelectedSourcesMock: (sources: DataSourceOption[]) => void = jest.fn(); @@ -23,7 +23,7 @@ describe('DataSourceSelectable', () => { getDataSet: jest.fn().mockResolvedValue([]), getType: jest.fn().mockReturnValue('DEFAULT_INDEX_PATTERNS'), getName: jest.fn().mockReturnValue('SomeName'), - } as unknown) as DataSourceType, + } as unknown) as DataSource, ]; dataSourceOptionListMock = []; @@ -104,7 +104,7 @@ describe('DataSourceSelectable', () => { getDataSet: jest.fn().mockResolvedValue([]), getType: jest.fn().mockReturnValue('DEFAULT_INDEX_PATTERNS'), getName: jest.fn().mockReturnValue('Index patterns'), - } as unknown) as DataSourceType, + } as unknown) as DataSource, ]} dataSourceOptionList={mockDataSourceOptionList} selectedSources={selectedSourcesMock} @@ -154,7 +154,7 @@ describe('DataSourceSelectable', () => { getDataSet: jest.fn().mockResolvedValue([]), getType: jest.fn().mockReturnValue('s3glue'), getName: jest.fn().mockReturnValue('Amazon S3'), - } as unknown) as DataSourceType, + } as unknown) as DataSource, ]} dataSourceOptionList={mockDataSourceOptionList} selectedSources={selectedSourcesMock} @@ -204,7 +204,7 @@ describe('DataSourceSelectable', () => { getDataSet: jest.fn().mockResolvedValue([]), getType: jest.fn().mockReturnValue('DEFAULT_INDEX_PATTERNS'), getName: jest.fn().mockReturnValue('Index patterns'), - } as unknown) as DataSourceType, + } as unknown) as DataSource, ]} dataSourceOptionList={mockDataSourceOptionListWithDuplicates} selectedSources={selectedSourcesMock} diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 2aaa023de324..b88fc2fd661e 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -6,8 +6,7 @@ import React, { useEffect, useCallback, useMemo } from 'react'; import { EuiComboBox } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import { ISourceDataSet, IndexPatternOption } from '../datasource'; -import { DataSourceType, GenericDataSource } from '../datasource_services'; +import { DataSource, SourceDataSet, IndexPatternOption } from '../datasource'; import { DataSourceGroup, DataSourceSelectableProps } from './types'; type DataSourceTypeKey = 'DEFAULT_INDEX_PATTERNS' | 's3glue' | 'spark'; @@ -26,19 +25,19 @@ const DATASOURCE_TYPE_DISPLAY_NAME_MAP: Record = { }), }; -type DataSetType = ISourceDataSet['data_sets'][number]; +type DataSetType = SourceDataSet['data_sets'][number]; -// Get data sets for a given datasource and returns it along with the source. -const getDataSetWithSource = async (ds: GenericDataSource): Promise => { - const dataSet = await ds.getDataSet(); +// Get Index patterns for local cluster. +const getDataSetWithSource = async (ds: DataSource): Promise => { + const dataSet = (await ds.getDataSet()).data_sets; return { ds, data_sets: dataSet, - }; + } as SourceDataSet; }; // Map through all data sources and get their respective data sets. -const getDataSets = (dataSources: GenericDataSource[]) => +const getDataSets = (dataSources: DataSource[]) => dataSources.map((ds) => getDataSetWithSource(ds)); export const isIndexPatterns = (dataSet: DataSetType): dataSet is IndexPatternOption => { @@ -48,7 +47,7 @@ export const isIndexPatterns = (dataSet: DataSetType): dataSet is IndexPatternOp }; // Get the option format for the combo box from the dataSource and dataSet. -export const getSourceOptions = (dataSource: DataSourceType, dataSet: DataSetType) => { +export const getSourceOptions = (dataSource: DataSource, dataSet: DataSetType) => { const optionContent = { type: dataSource.getType(), name: dataSource.getName(), @@ -70,7 +69,7 @@ export const getSourceOptions = (dataSource: DataSourceType, dataSet: DataSetTyp }; // Convert data sets into a structured format suitable for selector rendering. -const getSourceList = (allDataSets: ISourceDataSet[]) => { +const getSourceList = (allDataSets: SourceDataSet[]) => { const finalList = [] as DataSourceGroup[]; allDataSets.forEach((curDataSet) => { const typeKey = curDataSet.ds.getType() as DataSourceTypeKey; diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index 0fa1bf21cb19..c7d54ae3a6b2 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -8,7 +8,7 @@ */ import { EuiComboBoxProps, EuiComboBoxSingleSelectionShape } from '@elastic/eui'; -import { GenericDataSource } from '../datasource_services'; +import { DataSource } from '../datasource/datasource'; export interface DataSourceGroup { label: string; @@ -19,11 +19,11 @@ export interface DataSourceOption { label: string; value: string; type: string; - ds: GenericDataSource; + ds: DataSource; } export interface DataSourceSelectableProps extends Pick, 'fullWidth'> { - dataSources: GenericDataSource[]; + dataSources: DataSource[]; onDataSourceSelect: (dataSourceOption: DataSourceOption[]) => void; singleSelection?: boolean | EuiComboBoxSingleSelectionShape; onGetDataSetError: (error: Error) => void; diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index 9cf674585366..80ae619a679f 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -6,16 +6,16 @@ import { BehaviorSubject } from 'rxjs'; import { DataSourceRegistrationError, - GenericDataSource, IDataSourceFilter, IDataSourceRegistrationResult, } from './types'; +import { DataSource } from '../datasource/datasource'; export class DataSourceService { private static dataSourceService: DataSourceService; // A record to store all registered data sources, using the data source name as the key. - private dataSources: Record = {}; - private dataSourcesSubject: BehaviorSubject>; + private dataSources: Record = {}; + private dataSourcesSubject: BehaviorSubject>; private constructor() { this.dataSourcesSubject = new BehaviorSubject(this.dataSources); @@ -36,7 +36,7 @@ export class DataSourceService { * @returns An array of registration results, one for each data source. */ async registerMultipleDataSources( - datasources: GenericDataSource[] + datasources: DataSource[] ): Promise { return Promise.all(datasources.map((ds) => this.registerDataSource(ds))); } @@ -50,14 +50,14 @@ export class DataSourceService { * @returns A registration result indicating success or failure. * @throws {DataSourceRegistrationError} Throws an error if a data source with the same name already exists. */ - async registerDataSource(ds: GenericDataSource): Promise { - const dsName = ds.getName(); - if (dsName in this.dataSources) { + async registerDataSource(ds: DataSource): Promise { + const dsId = ds.getId(); + if (dsId in this.dataSources) { throw new DataSourceRegistrationError( - `Unable to register datasource ${dsName}, error: datasource name exists.` + `Unable to register data source ${dsId}, error: data source name exists.` ); } else { - this.dataSources[dsName] = ds; + this.dataSources[dsId] = ds; this.dataSourcesSubject.next(this.dataSources); return { success: true, info: '' } as IDataSourceRegistrationResult; } @@ -74,15 +74,15 @@ export class DataSourceService { * @param filter - An optional object with filter criteria (e.g., names of data sources). * @returns A record of filtered data sources. */ - getDataSources(filter?: IDataSourceFilter): Record { + getDataSources(filter?: IDataSourceFilter): Record { if (!filter || !Array.isArray(filter.names) || filter.names.length === 0) return this.dataSources; - return filter.names.reduce>((filteredDataSources, dsName) => { - if (dsName in this.dataSources) { - filteredDataSources[dsName] = this.dataSources[dsName]; + return filter.names.reduce>((filteredDataSources, dsId) => { + if (dsId in this.dataSources) { + filteredDataSources[dsId] = this.dataSources[dsId]; } return filteredDataSources; - }, {} as Record); + }, {} as Record); } } diff --git a/src/plugins/data/public/data_sources/datasource_services/index.ts b/src/plugins/data/public/data_sources/datasource_services/index.ts index 14db278b47a5..a6467904ea8d 100644 --- a/src/plugins/data/public/data_sources/datasource_services/index.ts +++ b/src/plugins/data/public/data_sources/datasource_services/index.ts @@ -8,6 +8,4 @@ export { IDataSourceFilter, IDataSourceRegistrationResult, DataSourceRegistrationError, - DataSourceType, - GenericDataSource, } from './types'; diff --git a/src/plugins/data/public/data_sources/datasource_services/types.ts b/src/plugins/data/public/data_sources/datasource_services/types.ts index cb5bb31500b4..9fdf99b70421 100644 --- a/src/plugins/data/public/data_sources/datasource_services/types.ts +++ b/src/plugins/data/public/data_sources/datasource_services/types.ts @@ -8,15 +8,7 @@ * in future releases. */ -import { - DataSource, - DataSourceFactory, - IDataSetParams, - IDataSourceMetaData, - IDataSourceQueryParams, - IDataSourceQueryResult, - ISourceDataSet, -} from '../datasource'; +import { DataSourceFactory } from '../datasource'; import { DataSourceService } from './datasource_service'; export interface IDataSourceFilter { @@ -42,13 +34,3 @@ export interface DataSourceStart { dataSourceService: DataSourceService; dataSourceFactory: DataSourceFactory; } - -export type DataSourceType = DataSource< - IDataSourceMetaData, - IDataSetParams, - ISourceDataSet, - IDataSourceQueryParams, - IDataSourceQueryResult ->; - -export type GenericDataSource = DataSource; diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts index c3b5d2a4cf99..1f5ee05ac1ae 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts @@ -4,44 +4,68 @@ */ import { SavedObject } from '../../../../../core/public'; -import { IndexPatternSavedObjectAttrs } from '../../index_patterns/index_patterns'; -import { DataSource, DataSourceConfig, IndexPatternOption } from '../datasource'; +import { + IndexPatternSavedObjectAttrs, + IndexPatternsContract, +} from '../../index_patterns/index_patterns'; +import { DataSource, IndexPatternOption } from '../datasource'; +import { + IDataSetParams, + IDataSourceDataSet, + IDataSourceMetadata, + IDataSourceQueryParams, + IDataSourceQueryResponse, + IDataSourceSettings, +} from '../datasource/types'; + +export type LocalDSMetadata = IDataSourceMetadata; +export type LocalDSDataSetParams = IDataSetParams; +export type LocalDSDataSetResponse = IDataSourceDataSet; +export type LocalDSQueryParams = IDataSourceQueryParams; +export type LocalDSQueryResponse = IDataSourceQueryResponse; +export interface LocalDataSourceSettings extends IDataSourceSettings { + indexPatterns: IndexPatternsContract; +} export class DefaultDslDataSource extends DataSource< - any, - any, - Promise, - any, - any + LocalDSMetadata, + LocalDSDataSetParams, + LocalDSDataSetResponse, + LocalDSQueryParams, + LocalDSQueryResponse > { private readonly indexPatterns; - constructor({ name, type, metadata, indexPatterns }: DataSourceConfig) { - super(name, type, metadata); + constructor({ id, name, type, metadata, indexPatterns }: LocalDataSourceSettings) { + super({ id, name, type, metadata }); this.indexPatterns = indexPatterns; } - async getDataSet(dataSetParams?: any) { + async getDataSet(): Promise { await this.indexPatterns.ensureDefaultIndexPattern(); const savedObjectLst = await this.indexPatterns.getCache(); if (!Array.isArray(savedObjectLst)) { - return undefined; + return { data_sets: [] }; } - return savedObjectLst.map((savedObject: SavedObject) => { - return { - id: savedObject.id, - title: savedObject.attributes.title, - }; - }); + return { + data_sets: savedObjectLst.map((savedObject: SavedObject) => { + return { + id: savedObject.id, + title: savedObject.attributes.title, + }; + }), + }; } async testConnection(): Promise { return true; } - async runQuery(queryParams: any) { - return undefined; + async runQuery(): Promise { + return { + data: {}, + }; } } diff --git a/src/plugins/data/public/data_sources/register_default_datasource.ts b/src/plugins/data/public/data_sources/register_default_datasource.ts index 8dece27e82eb..65f5028489c6 100644 --- a/src/plugins/data/public/data_sources/register_default_datasource.ts +++ b/src/plugins/data/public/data_sources/register_default_datasource.ts @@ -4,6 +4,7 @@ */ import { i18n } from '@osd/i18n'; +import { htmlIdGenerator } from '@elastic/eui'; import { DataPublicPluginStart } from '../types'; import { DefaultDslDataSource } from './default_datasource'; @@ -13,14 +14,23 @@ export const DEFAULT_DATASOURCE_NAME = i18n.translate('data.datasource.type.open }); export const registerDefaultDatasource = (data: Omit) => { - // Datasources registrations for index patterns datasource + // Registrations of index patterns as default data source const { dataSourceService, dataSourceFactory } = data.dataSources; dataSourceFactory.registerDataSourceType(DEFAULT_DATASOURCE_TYPE, DefaultDslDataSource); dataSourceService.registerDataSource( dataSourceFactory.getDataSourceInstance(DEFAULT_DATASOURCE_TYPE, { + id: htmlIdGenerator('local-cluster')('indices'), name: DEFAULT_DATASOURCE_NAME, type: DEFAULT_DATASOURCE_TYPE, - metadata: null, + metadata: { + ui: { + label: 'Index patterns', // display name of your data source, + typeLabel: 'OpenSearch default', // display name of your data source type, + selector: { + displayDatasetsWithSource: true, // when true, selector UI will render data sets with source by calling getDataSets() + }, + }, + }, indexPatterns: data.indexPatterns, }) ); diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 3b559f9e6c63..2792f1c2d420 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -500,19 +500,17 @@ export { DataPublicPlugin as Plugin }; // Export datasources export { DataSource, - IDataSourceMetaData, + IDataSourceMetadata, IDataSetParams, IDataSourceQueryParams, IDataSourceQueryResult, - ISourceDataSet, + SourceDataSet, ConnectionStatus, DataSourceFactory, - DataSourceConfig, } from './data_sources/datasource'; export { DataSourceRegistrationError, DataSourceService, - DataSourceType, IDataSourceFilter, IDataSourceRegistrationResult, } from './data_sources/datasource_services'; diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index ff07a59ab4b7..7e32819932e2 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -6,7 +6,7 @@ import React, { FC, useCallback, useEffect, useState } from 'react'; import { EuiPageSideBar, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import { DataSourceGroup, DataSourceSelectable, DataSourceType } from '../../../../data/public'; +import { DataSource, DataSourceGroup, DataSourceSelectable } from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { DataExplorerServices } from '../../types'; @@ -18,7 +18,7 @@ export const Sidebar: FC = ({ children }) => { const dispatch = useTypedDispatch(); const [selectedSources, setSelectedSources] = useState([]); const [dataSourceOptionList, setDataSourceOptionList] = useState([]); - const [activeDataSources, setActiveDataSources] = useState([]); + const [activeDataSources, setActiveDataSources] = useState([]); const { services: { From a21e887996beb392e7c86533ae23b8c921fac505 Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 21 Apr 2024 23:54:01 +0000 Subject: [PATCH 02/23] datasource enhancement and refactoring Signed-off-by: Eric --- .../data_sources/datasource/datasource.ts | 6 +- .../public/data_sources/datasource/index.ts | 4 +- .../public/data_sources/datasource/types.ts | 10 +- .../datasource_selectable.tsx | 143 +++++++++++++----- .../data_sources/datasource_selector/types.ts | 4 + .../datasource_services/datasource_service.ts | 29 ++++ .../data_sources/datasource_services/types.ts | 5 + .../default_datasource/default_datasource.ts | 4 +- .../register_default_datasource.ts | 3 +- src/plugins/data/public/index.ts | 2 +- .../public/components/sidebar/index.tsx | 2 + 11 files changed, 164 insertions(+), 48 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/datasource.ts b/src/plugins/data/public/data_sources/datasource/datasource.ts index b37d03709f36..bcff4d0d6cc7 100644 --- a/src/plugins/data/public/data_sources/datasource/datasource.ts +++ b/src/plugins/data/public/data_sources/datasource/datasource.ts @@ -14,7 +14,7 @@ */ import { - ConnectionStatus, + DataSourceConnectionStatus, IDataSetParams, IDataSourceDataSet, IDataSourceMetadata, @@ -90,8 +90,8 @@ export abstract class DataSource< * the connection status, typically indicating success or failure. * * @experimental This API is experimental and might change in future releases. - * @returns {Promise} Status of the connection test. + * @returns {Promise} Status of the connection test. * @experimental */ - abstract testConnection(): Promise; + abstract testConnection(): Promise; } diff --git a/src/plugins/data/public/data_sources/datasource/index.ts b/src/plugins/data/public/data_sources/datasource/index.ts index a3bac5bbc6a4..a93d57a8d80f 100644 --- a/src/plugins/data/public/data_sources/datasource/index.ts +++ b/src/plugins/data/public/data_sources/datasource/index.ts @@ -6,11 +6,11 @@ export { DataSource } from './datasource'; export { IDataSourceMetadata, - SourceDataSet, + DataSourceDataSet, IDataSetParams, IDataSourceQueryParams, IDataSourceQueryResult, - ConnectionStatus, + DataSourceConnectionStatus, IndexPatternOption, } from './types'; export { DataSourceFactory } from './factory'; diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index 7b17d0b7135d..1ca185c98ec8 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -18,9 +18,9 @@ export interface IDataSourceGroup { name: string; } -export interface SourceDataSet { +export interface DataSourceDataSet { ds: DataSource; - data_sets: IndexPatternOption[]; + list: T; } export interface IDataSetParams { @@ -35,7 +35,7 @@ export interface IDataSourceQueryResult { data: T; } -export interface ConnectionStatus { +export interface DataSourceConnectionStatus { status: string; message: string; error?: Error; @@ -54,13 +54,15 @@ export interface IDataSourceMetadata { export interface IDataSourceUISettings { label: string; // the display name of data source + typeGroup: string; // the group to which the data source belongs typeLabel: string; // the display name of data source type + displayOrder?: number; // the order in which the data source should be displayed in selector description?: string; // short description of your database icon?: string; // uri of the icon } export interface IDataSourceDataSet { - data_sets: T; + dataSets: T; } export interface IDataSourceQueryResponse { diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index b88fc2fd661e..37f6cf197dc3 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -6,48 +6,35 @@ import React, { useEffect, useCallback, useMemo } from 'react'; import { EuiComboBox } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import { DataSource, SourceDataSet, IndexPatternOption } from '../datasource'; +import { DataSource, DataSourceDataSet, IndexPatternOption } from '../datasource'; import { DataSourceGroup, DataSourceSelectableProps } from './types'; +import { IDataSourceDataSet } from '../datasource/types'; type DataSourceTypeKey = 'DEFAULT_INDEX_PATTERNS' | 's3glue' | 'spark'; -// Mapping between datasource type and its display name. -// Temporary solution, will be removed along with refactoring of data source APIs -const DATASOURCE_TYPE_DISPLAY_NAME_MAP: Record = { - DEFAULT_INDEX_PATTERNS: i18n.translate('dataExplorer.dataSourceSelector.indexPatternGroupTitle', { - defaultMessage: 'Index patterns', - }), - s3glue: i18n.translate('dataExplorer.dataSourceSelector.amazonS3GroupTitle', { - defaultMessage: 'Amazon S3', - }), - spark: i18n.translate('dataExplorer.dataSourceSelector.sparkGroupTitle', { - defaultMessage: 'Spark', - }), -}; - -type DataSetType = SourceDataSet['data_sets'][number]; - // Get Index patterns for local cluster. -const getDataSetWithSource = async (ds: DataSource): Promise => { - const dataSet = (await ds.getDataSet()).data_sets; +const getDataSetFromDataSource = async ( + ds: DataSource +): Promise> => { + const dataSets = (await ds.getDataSet()).dataSets; return { ds, - data_sets: dataSet, - } as SourceDataSet; + list: dataSets, + } as DataSourceDataSet; }; // Map through all data sources and get their respective data sets. const getDataSets = (dataSources: DataSource[]) => - dataSources.map((ds) => getDataSetWithSource(ds)); + dataSources.map((ds) => getDataSetFromDataSource(ds)); -export const isIndexPatterns = (dataSet: DataSetType): dataSet is IndexPatternOption => { +export const isIndexPatterns = (dataSet: IndexPatternOption) => { if (typeof dataSet === 'string') return false; return !!(dataSet.title && dataSet.id); }; // Get the option format for the combo box from the dataSource and dataSet. -export const getSourceOptions = (dataSource: DataSource, dataSet: DataSetType) => { +export const getSourceOptions = (dataSource: DataSource, dataSet: IDataSourceDataSet) => { const optionContent = { type: dataSource.getType(), name: dataSource.getName(), @@ -68,13 +55,17 @@ export const getSourceOptions = (dataSource: DataSource, dataSet: DataSetType) = }; }; -// Convert data sets into a structured format suitable for selector rendering. -const getSourceList = (allDataSets: SourceDataSet[]) => { +const getGroupListFromDataSetDS = ( + dsListWithDataSetAsDisplayedDataSources: DataSourceDataSet[], + groupList: DataSourceGroup[] +) => { const finalList = [] as DataSourceGroup[]; - allDataSets.forEach((curDataSet) => { - const typeKey = curDataSet.ds.getType() as DataSourceTypeKey; + dsListWithDataSetAsDisplayedDataSources.forEach((curDataSet) => { + const typeKey = curDataSet.ds.getType() as DataSourceTypeKey; // ds type key + const dsMetadata = curDataSet.ds.getMetadata(); + const groupType = dsMetadata.ui.typeGroup; let groupName = - DATASOURCE_TYPE_DISPLAY_NAME_MAP[typeKey] || + dsMetadata.ui.typeLabel || i18n.translate('dataExplorer.dataSourceSelector.defaultGroupTitle', { defaultMessage: 'Default Group', }); @@ -87,15 +78,15 @@ const getSourceList = (allDataSets: SourceDataSet[]) => { })}`; } - const existingGroup = finalList.find((item) => item.label === groupName); - const mappedOptions = curDataSet.data_sets.map((dataSet) => + const existingGroup = finalList.find((item) => item.typeGroup === groupType); + const mappedOptions = curDataSet.list.map((dataSet) => getSourceOptions(curDataSet.ds, dataSet) ); - // check if add new datasource group or add to existing one + // check if to add new data source group or add to existing one if (existingGroup) { // options deduplication - const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.label)); + const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.id)); const nonDuplicateOptions = mappedOptions.filter((opt) => !existingOptionIds.has(opt.label)); // 'existingGroup' directly references an item in the finalList @@ -103,14 +94,92 @@ const getSourceList = (allDataSets: SourceDataSet[]) => { existingGroup.options.push(...nonDuplicateOptions); } else { finalList.push({ + typeGroup: dsMetadata.ui.typeGroup, label: groupName, options: mappedOptions, }); } }); + + return [...finalList, ...groupList]; +}; + +const getGroupListFromDS = (dataSources: DataSource[], groupList: DataSourceGroup[]) => { + const finalList = [] as DataSourceGroup[]; + dataSources.forEach((ds) => { + const typeKey = ds.getType() as DataSourceTypeKey; // ds type key + const dsMetadata = ds.getMetadata(); + const typeGroup = dsMetadata.ui.typeGroup; + let groupName = + dsMetadata.ui.typeLabel || + i18n.translate('dataExplorer.dataSourceSelector.defaultGroupTitle', { + defaultMessage: 'Default Group', + }); + + // add '- Opens in Log Explorer' to hint user that selecting these types of data sources + // will lead to redirection to log explorer + if (typeKey !== 'DEFAULT_INDEX_PATTERNS') { + groupName = `${groupName}${i18n.translate('dataExplorer.dataSourceSelector.redirectionHint', { + defaultMessage: ' - Opens in Log Explorer', + })}`; + } + + const existingGroup = finalList.find((item) => item.typeGroup === typeGroup); + const dsOption = { + type: ds.getType(), + name: ds.getName(), + ds, + label: dsMetadata.ui.label, + value: dsMetadata.ui.label, + key: ds.getId(), + }; + // check if to add new data source group or add to existing one + if (existingGroup) { + // options deduplication + // const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.id)); + // const nonDuplicateOptions = mappedOptions.filter((opt) => !existingOptionIds.has(opt.label)); + + // 'existingGroup' directly references an item in the finalList + // pushing options to 'existingGroup' updates the corresponding item in finalList + existingGroup.options.push(dsOption); + } else { + finalList.push({ + id: dsMetadata.ui.typeGroup, + typeGroup: dsMetadata.ui.typeGroup, + label: groupName, + options: [ + { + ...dsOption, + }, + ], + }); + } + }); + + return [...groupList, ...finalList]; +}; + +// Convert data sets into a structured format suitable for selector rendering. +const getConsolidatedDataSourceGroups = ( + dsListWithDataSetAsDisplayedDataSources: DataSourceDataSet[], + dsListWithThemselvesAsDisplayedDataSources: DataSource[] +) => { + // const finalList = [] as DataSourceGroup[]; + const dataSetFinalList = getGroupListFromDataSetDS(dsListWithDataSetAsDisplayedDataSources, []); + const finalList = getGroupListFromDS( + dsListWithThemselvesAsDisplayedDataSources, + dataSetFinalList + ); + return finalList; }; +const getDataSourcesRequireDataSetFetching = (dataSources: DataSource[]): DataSource[] => + dataSources.filter((ds) => ds.getMetadata()?.ui?.selector?.displayDatasetsWithSource); + +const getDataSourcesRequireNoDataSetFetching = (dataSources: DataSource[]): DataSource[] => + dataSources.filter((ds) => !ds.getMetadata()?.ui?.selector?.displayDatasetsWithSource); + /** * @experimental This component is experimental and might change in future releases. */ @@ -126,9 +195,13 @@ export const DataSourceSelectable = ({ }: DataSourceSelectableProps) => { // This effect gets data sets and prepares the datasource list for UI rendering. useEffect(() => { - Promise.all(getDataSets(dataSources)) + Promise.all(getDataSets(getDataSourcesRequireDataSetFetching(dataSources))) .then((results) => { - setDataSourceOptionList(getSourceList(results)); + const groupList = getConsolidatedDataSourceGroups( + results, + getDataSourcesRequireNoDataSetFetching(dataSources) + ); + setDataSourceOptionList(groupList); }) .catch((e) => onGetDataSetError(e)); }, [dataSources, setDataSourceOptionList, onGetDataSetError]); diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index c7d54ae3a6b2..23804569b7ae 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -12,10 +12,14 @@ import { DataSource } from '../datasource/datasource'; export interface DataSourceGroup { label: string; + id: string; options: DataSourceOption[]; + typeGroup: string; } export interface DataSourceOption { + id: string; + name: string; label: string; value: string; type: string; diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index 80ae619a679f..8ead65633c7f 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -8,6 +8,7 @@ import { DataSourceRegistrationError, IDataSourceFilter, IDataSourceRegistrationResult, + DataSourceFetcher, } from './types'; import { DataSource } from '../datasource/datasource'; @@ -16,6 +17,7 @@ export class DataSourceService { // A record to store all registered data sources, using the data source name as the key. private dataSources: Record = {}; private dataSourcesSubject: BehaviorSubject>; + private dataSourceFetchers: Record = {}; private constructor() { this.dataSourcesSubject = new BehaviorSubject(this.dataSources); @@ -85,4 +87,31 @@ export class DataSourceService { return filteredDataSources; }, {} as Record); } + + /** + * Registers functions responsible for fetching data for each data source type. + * + * @param fetchers - An array of fetcher configurations, each specifying how to fetch data for a specific data source type. + */ + registerDataSourceFetchers(fetchers: DataSourceFetcher[]) { + fetchers.forEach((fetcher) => { + this.dataSourceFetchers[fetcher.type] = fetcher.registerDataSources; + }); + } + + /** + * Calls all registered data fetching functions to update data sources. + * Typically used to initialize or refresh the data source configurations. + */ + load() { + Object.values(this.dataSourceFetchers).forEach((fetch) => fetch()); + } + + /** + * Reloads all data source configurations by re-invoking the load method. + * Useful for refreshing the system to reflect changes such as new data source registrations. + */ + reload() { + this.load(); + } } diff --git a/src/plugins/data/public/data_sources/datasource_services/types.ts b/src/plugins/data/public/data_sources/datasource_services/types.ts index 9fdf99b70421..1cbdbeadea8c 100644 --- a/src/plugins/data/public/data_sources/datasource_services/types.ts +++ b/src/plugins/data/public/data_sources/datasource_services/types.ts @@ -34,3 +34,8 @@ export interface DataSourceStart { dataSourceService: DataSourceService; dataSourceFactory: DataSourceFactory; } + +export interface DataSourceFetcher { + type: string; + registerDataSources: () => void; +} diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts index 1f5ee05ac1ae..262a8ec9d93d 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts @@ -41,7 +41,7 @@ export class DefaultDslDataSource extends DataSource< this.indexPatterns = indexPatterns; } - async getDataSet(): Promise { + async getDataSet() { await this.indexPatterns.ensureDefaultIndexPattern(); const savedObjectLst = await this.indexPatterns.getCache(); @@ -50,7 +50,7 @@ export class DefaultDslDataSource extends DataSource< } return { - data_sets: savedObjectLst.map((savedObject: SavedObject) => { + dataSets: savedObjectLst.map((savedObject: SavedObject) => { return { id: savedObject.id, title: savedObject.attributes.title, diff --git a/src/plugins/data/public/data_sources/register_default_datasource.ts b/src/plugins/data/public/data_sources/register_default_datasource.ts index 65f5028489c6..7fd6e8b147cf 100644 --- a/src/plugins/data/public/data_sources/register_default_datasource.ts +++ b/src/plugins/data/public/data_sources/register_default_datasource.ts @@ -25,7 +25,8 @@ export const registerDefaultDatasource = (data: Omit { } ); + dataSources.dataSourceService.load(); + return () => { subscription.unsubscribe(); isMounted = false; From 564c8202b43f860e25812205ab11ff5ca5b543a9 Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 22 Apr 2024 16:31:42 +0000 Subject: [PATCH 03/23] datasource selectable consolidation and refactoring Signed-off-by: Eric --- .../public/data_sources/datasource/types.ts | 7 +- .../datasource_selectable.tsx | 218 ++++++------------ .../data_sources/datasource_selector/types.ts | 1 + .../register_default_datasource.ts | 2 +- 4 files changed, 80 insertions(+), 148 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index 1ca185c98ec8..565054a20cd7 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -18,7 +18,7 @@ export interface IDataSourceGroup { name: string; } -export interface DataSourceDataSet { +export interface DataSourceDataSet { ds: DataSource; list: T; } @@ -52,7 +52,12 @@ export interface IDataSourceMetadata { ui: IDataSourceUISettings; } +export interface IDataSourceUISelector { + displayDatasetsAsSource: boolean; +} + export interface IDataSourceUISettings { + selector: IDataSourceUISelector; label: string; // the display name of data source typeGroup: string; // the group to which the data source belongs typeLabel: string; // the display name of data source type diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 37f6cf197dc3..776e8f28cca8 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -8,178 +8,99 @@ import { EuiComboBox } from '@elastic/eui'; import { i18n } from '@osd/i18n'; import { DataSource, DataSourceDataSet, IndexPatternOption } from '../datasource'; import { DataSourceGroup, DataSourceSelectableProps } from './types'; -import { IDataSourceDataSet } from '../datasource/types'; - -type DataSourceTypeKey = 'DEFAULT_INDEX_PATTERNS' | 's3glue' | 'spark'; // Get Index patterns for local cluster. -const getDataSetFromDataSource = async ( +const getAndFormatDataSetFromDataSource = async ( ds: DataSource ): Promise> => { - const dataSets = (await ds.getDataSet()).dataSets; - return { - ds, - list: dataSets, - } as DataSourceDataSet; + const { dataSets } = await ds.getDataSet(); + return { ds, list: dataSets } as DataSourceDataSet; }; // Map through all data sources and get their respective data sets. -const getDataSets = (dataSources: DataSource[]) => - dataSources.map((ds) => getDataSetFromDataSource(ds)); - -export const isIndexPatterns = (dataSet: IndexPatternOption) => { - if (typeof dataSet === 'string') return false; - - return !!(dataSet.title && dataSet.id); -}; - -// Get the option format for the combo box from the dataSource and dataSet. -export const getSourceOptions = (dataSource: DataSource, dataSet: IDataSourceDataSet) => { - const optionContent = { +const getAllDataSets = (dataSources: DataSource[]) => + dataSources.map((ds) => getAndFormatDataSetFromDataSource(ds)); + +export const isIndexPatterns = (dataSet: unknown) => + typeof dataSet !== 'string' && + 'title' in (dataSet as any) && + 'id' in (dataSet as any) && + !!(dataSet as any).title && + !!(dataSet as any).id; + +// Mapping function for datasets to get the option format for the combo box from the dataSource and dataSet. +const mapToOption = ( + dataSource: DataSource, + dataSet: DataSourceDataSet | undefined = undefined +) => { + const baseOption = { type: dataSource.getType(), name: dataSource.getName(), ds: dataSource, }; - if (isIndexPatterns(dataSet)) { + if (dataSet && 'title' in dataSet && 'id' in dataSet && isIndexPatterns(dataSet)) { return { - ...optionContent, + ...baseOption, label: dataSet.title, value: dataSet.id, key: dataSet.id, }; } return { - ...optionContent, + ...baseOption, label: dataSource.getName(), value: dataSource.getName(), + key: dataSource.getId(), }; }; -const getGroupListFromDataSetDS = ( - dsListWithDataSetAsDisplayedDataSources: DataSourceDataSet[], - groupList: DataSourceGroup[] -) => { - const finalList = [] as DataSourceGroup[]; - dsListWithDataSetAsDisplayedDataSources.forEach((curDataSet) => { - const typeKey = curDataSet.ds.getType() as DataSourceTypeKey; // ds type key - const dsMetadata = curDataSet.ds.getMetadata(); - const groupType = dsMetadata.ui.typeGroup; - let groupName = - dsMetadata.ui.typeLabel || - i18n.translate('dataExplorer.dataSourceSelector.defaultGroupTitle', { - defaultMessage: 'Default Group', - }); - - // add '- Opens in Log Explorer' to hint user that selecting these types of data sources - // will lead to redirection to log explorer - if (typeKey !== 'DEFAULT_INDEX_PATTERNS') { - groupName = `${groupName}${i18n.translate('dataExplorer.dataSourceSelector.redirectionHint', { - defaultMessage: ' - Opens in Log Explorer', - })}`; - } - - const existingGroup = finalList.find((item) => item.typeGroup === groupType); - const mappedOptions = curDataSet.list.map((dataSet) => - getSourceOptions(curDataSet.ds, dataSet) - ); +// Function to add or update groups in a reduction process +const addOrUpdateGroup = (acc: DataSourceGroup[], dataSource: DataSource, option) => { + const metadata = dataSource.getMetadata(); + const groupType = metadata.ui.typeGroup; + let groupName = + metadata.ui.typeLabel || + i18n.translate('dataExplorer.dataSourceSelector.defaultGroupTitle', { + defaultMessage: 'Default Group', + }); - // check if to add new data source group or add to existing one - if (existingGroup) { - // options deduplication - const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.id)); - const nonDuplicateOptions = mappedOptions.filter((opt) => !existingOptionIds.has(opt.label)); + if (dataSource.getType() !== 'DEFAULT_INDEX_PATTERNS') { + groupName += i18n.translate('dataExplorer.dataSourceSelector.redirectionHint', { + defaultMessage: ' - Opens in Log Explorer', + }); + } - // 'existingGroup' directly references an item in the finalList - // pushing options to 'existingGroup' updates the corresponding item in finalList - existingGroup.options.push(...nonDuplicateOptions); - } else { - finalList.push({ - typeGroup: dsMetadata.ui.typeGroup, - label: groupName, - options: mappedOptions, - }); + const group = acc.find((g: DataSourceGroup) => g.typeGroup === groupType); + if (group) { + if (!group.options.some((opt) => opt.key === option.key)) { + group.options.push(option); } - }); - - return [...finalList, ...groupList]; + } else { + acc.push({ + typeGroup: groupType, + label: groupName, + options: [option], + id: metadata.ui.typeGroup, // id for each group + }); + } + return acc; }; -const getGroupListFromDS = (dataSources: DataSource[], groupList: DataSourceGroup[]) => { - const finalList = [] as DataSourceGroup[]; - dataSources.forEach((ds) => { - const typeKey = ds.getType() as DataSourceTypeKey; // ds type key - const dsMetadata = ds.getMetadata(); - const typeGroup = dsMetadata.ui.typeGroup; - let groupName = - dsMetadata.ui.typeLabel || - i18n.translate('dataExplorer.dataSourceSelector.defaultGroupTitle', { - defaultMessage: 'Default Group', - }); - - // add '- Opens in Log Explorer' to hint user that selecting these types of data sources - // will lead to redirection to log explorer - if (typeKey !== 'DEFAULT_INDEX_PATTERNS') { - groupName = `${groupName}${i18n.translate('dataExplorer.dataSourceSelector.redirectionHint', { - defaultMessage: ' - Opens in Log Explorer', - })}`; - } - - const existingGroup = finalList.find((item) => item.typeGroup === typeGroup); - const dsOption = { - type: ds.getType(), - name: ds.getName(), - ds, - label: dsMetadata.ui.label, - value: dsMetadata.ui.label, - key: ds.getId(), - }; - // check if to add new data source group or add to existing one - if (existingGroup) { - // options deduplication - // const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.id)); - // const nonDuplicateOptions = mappedOptions.filter((opt) => !existingOptionIds.has(opt.label)); - - // 'existingGroup' directly references an item in the finalList - // pushing options to 'existingGroup' updates the corresponding item in finalList - existingGroup.options.push(dsOption); +const consolidateDataSourceGroups = (dataSets: DataSourceDataSet[], dataSources: DataSource[]) => { + return [...dataSets, ...dataSources].reduce((acc, item) => { + if ('list' in item && item.ds) { + // Confirm item is a DataSet + const options = item.list.map((dataset) => mapToOption(item.ds, dataset)); + options.forEach((option) => addOrUpdateGroup(acc, item.ds, option)); } else { - finalList.push({ - id: dsMetadata.ui.typeGroup, - typeGroup: dsMetadata.ui.typeGroup, - label: groupName, - options: [ - { - ...dsOption, - }, - ], - }); + // Handle DataSource directly + const option = mapToOption(item as InstanceType); + addOrUpdateGroup(acc, item as InstanceType, option); } - }); - - return [...groupList, ...finalList]; + return acc; + }, []); }; -// Convert data sets into a structured format suitable for selector rendering. -const getConsolidatedDataSourceGroups = ( - dsListWithDataSetAsDisplayedDataSources: DataSourceDataSet[], - dsListWithThemselvesAsDisplayedDataSources: DataSource[] -) => { - // const finalList = [] as DataSourceGroup[]; - const dataSetFinalList = getGroupListFromDataSetDS(dsListWithDataSetAsDisplayedDataSources, []); - const finalList = getGroupListFromDS( - dsListWithThemselvesAsDisplayedDataSources, - dataSetFinalList - ); - - return finalList; -}; - -const getDataSourcesRequireDataSetFetching = (dataSources: DataSource[]): DataSource[] => - dataSources.filter((ds) => ds.getMetadata()?.ui?.selector?.displayDatasetsWithSource); - -const getDataSourcesRequireNoDataSetFetching = (dataSources: DataSource[]): DataSource[] => - dataSources.filter((ds) => !ds.getMetadata()?.ui?.selector?.displayDatasetsWithSource); - /** * @experimental This component is experimental and might change in future releases. */ @@ -195,13 +116,18 @@ export const DataSourceSelectable = ({ }: DataSourceSelectableProps) => { // This effect gets data sets and prepares the datasource list for UI rendering. useEffect(() => { - Promise.all(getDataSets(getDataSourcesRequireDataSetFetching(dataSources))) - .then((results) => { - const groupList = getConsolidatedDataSourceGroups( - results, - getDataSourcesRequireNoDataSetFetching(dataSources) + Promise.all( + getAllDataSets( + dataSources.filter((ds) => ds.getMetadata().ui?.selector.displayDatasetsAsSource) + ) + ) + .then((dataSetResults) => { + setDataSourceOptionList( + consolidateDataSourceGroups( + dataSetResults as DataSourceDataSet[], + dataSources.filter((ds) => !ds.getMetadata().ui?.selector.displayDatasetsAsSource) + ) ); - setDataSourceOptionList(groupList); }) .catch((e) => onGetDataSetError(e)); }, [dataSources, setDataSourceOptionList, onGetDataSetError]); diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index 23804569b7ae..0a049ea5eaec 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -18,6 +18,7 @@ export interface DataSourceGroup { } export interface DataSourceOption { + key: string; id: string; name: string; label: string; diff --git a/src/plugins/data/public/data_sources/register_default_datasource.ts b/src/plugins/data/public/data_sources/register_default_datasource.ts index 7fd6e8b147cf..a4198b4e8e3a 100644 --- a/src/plugins/data/public/data_sources/register_default_datasource.ts +++ b/src/plugins/data/public/data_sources/register_default_datasource.ts @@ -28,7 +28,7 @@ export const registerDefaultDatasource = (data: Omit Date: Fri, 26 Apr 2024 03:37:06 +0000 Subject: [PATCH 04/23] add in memory cache with refresh Signed-off-by: Eric --- .../data_sources/datasource/factory.test.ts | 4 ++- .../datasource_selectable.tsx | 25 ++++++++++++++++--- .../data_sources/datasource_selector/types.ts | 1 + .../datasource_services/datasource_service.ts | 11 +++++++- .../register_default_datasource.ts | 4 +-- src/plugins/data/public/plugin.ts | 15 +++++++++-- .../public/components/sidebar/index.tsx | 7 ++++-- 7 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/factory.test.ts b/src/plugins/data/public/data_sources/datasource/factory.test.ts index 0f9ea016748f..e0d35b6ed0a6 100644 --- a/src/plugins/data/public/data_sources/datasource/factory.test.ts +++ b/src/plugins/data/public/data_sources/datasource/factory.test.ts @@ -11,17 +11,19 @@ class MockDataSource extends DataSource { private readonly indexPatterns; constructor({ + id, name, type, metadata, indexPatterns, }: { + id: string; name: string; type: string; metadata: any; indexPatterns: IndexPatternsService; }) { - super(name, type, metadata); + super({ id, name, type, metadata }); this.indexPatterns = indexPatterns; } diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 776e8f28cca8..1d07c2e95041 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -4,7 +4,7 @@ */ import React, { useEffect, useCallback, useMemo } from 'react'; -import { EuiComboBox } from '@elastic/eui'; +import { EuiButtonIcon, EuiComboBox, EuiText, EuiToolTip } from '@elastic/eui'; import { i18n } from '@osd/i18n'; import { DataSource, DataSourceDataSet, IndexPatternOption } from '../datasource'; import { DataSourceGroup, DataSourceSelectableProps } from './types'; @@ -112,20 +112,21 @@ export const DataSourceSelectable = ({ setDataSourceOptionList, onGetDataSetError, // onGetDataSetError, Callback for handling get data set errors. Ensure it's memoized. singleSelection = { asPlainText: true }, + onRefresh, ...comboBoxProps }: DataSourceSelectableProps) => { // This effect gets data sets and prepares the datasource list for UI rendering. useEffect(() => { Promise.all( getAllDataSets( - dataSources.filter((ds) => ds.getMetadata().ui?.selector.displayDatasetsAsSource) + dataSources.filter((ds) => ds.getMetadata().ui.selector.displayDatasetsAsSource) ) ) .then((dataSetResults) => { setDataSourceOptionList( consolidateDataSourceGroups( dataSetResults as DataSourceDataSet[], - dataSources.filter((ds) => !ds.getMetadata().ui?.selector.displayDatasetsAsSource) + dataSources.filter((ds) => !ds.getMetadata().ui.selector.displayDatasetsAsSource) ) ); }) @@ -160,6 +161,24 @@ export const DataSourceSelectable = ({ onChange={handleSourceChange} singleSelection={singleSelection} isClearable={false} + prepend={ + + + + + + } /> ); }; diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index 0a049ea5eaec..a8c1ba6a9fa3 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -35,4 +35,5 @@ export interface DataSourceSelectableProps extends Pick void; + onRefresh: () => void; } diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index 8ead65633c7f..e4cef2ece4ca 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -104,14 +104,23 @@ export class DataSourceService { * Typically used to initialize or refresh the data source configurations. */ load() { + this.reset(); Object.values(this.dataSourceFetchers).forEach((fetch) => fetch()); } /** * Reloads all data source configurations by re-invoking the load method. - * Useful for refreshing the system to reflect changes such as new data source registrations. + * Used for refreshing the system to reflect changes such as new data source registrations. */ reload() { this.load(); } + + /** + * Resets all registered data sources. + */ + reset() { + this.dataSources = {}; + this.dataSourcesSubject.next(this.dataSources); + } } diff --git a/src/plugins/data/public/data_sources/register_default_datasource.ts b/src/plugins/data/public/data_sources/register_default_datasource.ts index a4198b4e8e3a..6fa6bc8aa23c 100644 --- a/src/plugins/data/public/data_sources/register_default_datasource.ts +++ b/src/plugins/data/public/data_sources/register_default_datasource.ts @@ -6,17 +6,15 @@ import { i18n } from '@osd/i18n'; import { htmlIdGenerator } from '@elastic/eui'; import { DataPublicPluginStart } from '../types'; -import { DefaultDslDataSource } from './default_datasource'; export const DEFAULT_DATASOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; export const DEFAULT_DATASOURCE_NAME = i18n.translate('data.datasource.type.openSearchDefault', { defaultMessage: 'OpenSearch Default', }); -export const registerDefaultDatasource = (data: Omit) => { +export const registerDefaultDataSource = (data: Omit) => { // Registrations of index patterns as default data source const { dataSourceService, dataSourceFactory } = data.dataSources; - dataSourceFactory.registerDataSourceType(DEFAULT_DATASOURCE_TYPE, DefaultDslDataSource); dataSourceService.registerDataSource( dataSourceFactory.getDataSourceInstance(DEFAULT_DATASOURCE_TYPE, { id: htmlIdGenerator('local-cluster')('indices'), diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index 179b6c0a8c83..bc757467bd98 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -90,7 +90,11 @@ import { SavedObjectsClientPublicToCommon } from './index_patterns'; import { indexPatternLoad } from './index_patterns/expressions/load_index_pattern'; import { DataSourceService } from './data_sources/datasource_services'; import { DataSourceFactory } from './data_sources/datasource'; -import { registerDefaultDatasource } from './data_sources/register_default_datasource'; +import { + DEFAULT_DATASOURCE_TYPE, + registerDefaultDataSource, +} from './data_sources/register_default_datasource'; +import { DefaultDslDataSource } from './data_sources/default_datasource'; declare module '../../ui_actions/public' { export interface ActionContextMapping { @@ -218,6 +222,13 @@ export class DataPublicPlugin // Create or fetch the singleton instance const dataSourceService = DataSourceService.getInstance(); const dataSourceFactory = DataSourceFactory.getInstance(); + dataSourceFactory.registerDataSourceType(DEFAULT_DATASOURCE_TYPE, DefaultDslDataSource); + dataSourceService.registerDataSourceFetchers([ + { + type: DEFAULT_DATASOURCE_TYPE, + registerDataSources: () => registerDefaultDataSource(dataServices), + }, + ]); const dataServices = { actions: { @@ -235,7 +246,7 @@ export class DataPublicPlugin }, }; - registerDefaultDatasource(dataServices); + registerDefaultDataSource(dataServices); const SearchBar = createSearchBar({ core, diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index acede22c2e51..21f8d20d609a 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -38,8 +38,6 @@ export const Sidebar: FC = ({ children }) => { } ); - dataSources.dataSourceService.load(); - return () => { subscription.unsubscribe(); isMounted = false; @@ -100,6 +98,10 @@ export const Sidebar: FC = ({ children }) => { [toasts] ); + const memorizedReload = useCallback(() => { + dataSources.dataSourceService.reload(); + }, [dataSources.dataSourceService]); + return ( { onDataSourceSelect={handleSourceSelection} selectedSources={selectedSources} onGetDataSetError={handleGetDataSetError} + onRefresh={memorizedReload} fullWidth /> From 46d0b35b1d422d926a26eb821d6bf30f83cfa960 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 26 Apr 2024 19:38:00 +0000 Subject: [PATCH 05/23] move refresh to right side Signed-off-by: Eric --- .../datasource_selector/datasource_selectable.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 1d07c2e95041..7fac5bfac7bc 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -161,12 +161,12 @@ export const DataSourceSelectable = ({ onChange={handleSourceChange} singleSelection={singleSelection} isClearable={false} - prepend={ + append={ From 9005834c89ae47c78c9a8db9cb3496d00320d05b Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 28 Apr 2024 21:00:00 +0000 Subject: [PATCH 06/23] renaming Signed-off-by: Eric --- .../public/data_sources/datasource/index.ts | 2 +- .../public/data_sources/datasource/types.ts | 4 +- .../datasource_selectable.tsx | 60 ++++++++++--------- .../data_sources/datasource_selector/types.ts | 3 +- .../register_default_datasource.ts | 2 +- 5 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/index.ts b/src/plugins/data/public/data_sources/datasource/index.ts index a93d57a8d80f..e45cd6dad22c 100644 --- a/src/plugins/data/public/data_sources/datasource/index.ts +++ b/src/plugins/data/public/data_sources/datasource/index.ts @@ -6,7 +6,7 @@ export { DataSource } from './datasource'; export { IDataSourceMetadata, - DataSourceDataSet, + DataSetWithDataSource, IDataSetParams, IDataSourceQueryParams, IDataSourceQueryResult, diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index 565054a20cd7..a53a99a4d773 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -18,7 +18,7 @@ export interface IDataSourceGroup { name: string; } -export interface DataSourceDataSet { +export interface DataSetWithDataSource { ds: DataSource; list: T; } @@ -59,7 +59,7 @@ export interface IDataSourceUISelector { export interface IDataSourceUISettings { selector: IDataSourceUISelector; label: string; // the display name of data source - typeGroup: string; // the group to which the data source belongs + groupType: string; // the group to which the data source belongs typeLabel: string; // the display name of data source type displayOrder?: number; // the order in which the data source should be displayed in selector description?: string; // short description of your database diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 7fac5bfac7bc..5ea25d75ff8c 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -6,15 +6,15 @@ import React, { useEffect, useCallback, useMemo } from 'react'; import { EuiButtonIcon, EuiComboBox, EuiText, EuiToolTip } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import { DataSource, DataSourceDataSet, IndexPatternOption } from '../datasource'; -import { DataSourceGroup, DataSourceSelectableProps } from './types'; +import { DataSource, DataSetWithDataSource, IndexPatternOption } from '../datasource'; +import { DataSourceGroup, DataSourceOption, DataSourceSelectableProps } from './types'; -// Get Index patterns for local cluster. +// Asynchronously retrieves and formats dataset from a given data source. const getAndFormatDataSetFromDataSource = async ( ds: DataSource -): Promise> => { +): Promise> => { const { dataSets } = await ds.getDataSet(); - return { ds, list: dataSets } as DataSourceDataSet; + return { ds, list: dataSets } as DataSetWithDataSource; }; // Map through all data sources and get their respective data sets. @@ -28,11 +28,11 @@ export const isIndexPatterns = (dataSet: unknown) => !!(dataSet as any).title && !!(dataSet as any).id; -// Mapping function for datasets to get the option format for the combo box from the dataSource and dataSet. +// Mapping function to get the option format for the combo box from the dataSource and dataSet. const mapToOption = ( dataSource: DataSource, - dataSet: DataSourceDataSet | undefined = undefined -) => { + dataSet: DataSetWithDataSource | undefined = undefined +): DataSourceOption => { const baseOption = { type: dataSource.getType(), name: dataSource.getName(), @@ -41,9 +41,9 @@ const mapToOption = ( if (dataSet && 'title' in dataSet && 'id' in dataSet && isIndexPatterns(dataSet)) { return { ...baseOption, - label: dataSet.title, - value: dataSet.id, - key: dataSet.id, + label: dataSet.title as string, + value: dataSet.id as string, + key: dataSet.id as string, }; } return { @@ -55,9 +55,13 @@ const mapToOption = ( }; // Function to add or update groups in a reduction process -const addOrUpdateGroup = (acc: DataSourceGroup[], dataSource: DataSource, option) => { +const addOrUpdateGroup = ( + existingGroups: DataSourceGroup[], + dataSource: DataSource, + option: DataSourceOption +) => { const metadata = dataSource.getMetadata(); - const groupType = metadata.ui.typeGroup; + const groupType = metadata.ui.groupType; let groupName = metadata.ui.typeLabel || i18n.translate('dataExplorer.dataSourceSelector.defaultGroupTitle', { @@ -70,34 +74,34 @@ const addOrUpdateGroup = (acc: DataSourceGroup[], dataSource: DataSource, option }); } - const group = acc.find((g: DataSourceGroup) => g.typeGroup === groupType); - if (group) { - if (!group.options.some((opt) => opt.key === option.key)) { - group.options.push(option); - } + const group = existingGroups.find((g: DataSourceGroup) => g.id === groupType); + if (group && !group.options.some((opt) => opt.key === option.key)) { + group.options.push(option); } else { - acc.push({ - typeGroup: groupType, + existingGroups.push({ + groupType, label: groupName, options: [option], - id: metadata.ui.typeGroup, // id for each group + id: metadata.ui.groupType, // id for each group }); } - return acc; }; -const consolidateDataSourceGroups = (dataSets: DataSourceDataSet[], dataSources: DataSource[]) => { - return [...dataSets, ...dataSources].reduce((acc, item) => { +const consolidateDataSourceGroups = ( + dataSets: DataSetWithDataSource[], + dataSources: DataSource[] +) => { + return [...dataSets, ...dataSources].reduce((dsGroup, item) => { if ('list' in item && item.ds) { // Confirm item is a DataSet const options = item.list.map((dataset) => mapToOption(item.ds, dataset)); - options.forEach((option) => addOrUpdateGroup(acc, item.ds, option)); + options.forEach((option) => addOrUpdateGroup(dsGroup, item.ds, option)); } else { // Handle DataSource directly const option = mapToOption(item as InstanceType); - addOrUpdateGroup(acc, item as InstanceType, option); + addOrUpdateGroup(dsGroup, item as InstanceType, option); } - return acc; + return dsGroup; }, []); }; @@ -125,7 +129,7 @@ export const DataSourceSelectable = ({ .then((dataSetResults) => { setDataSourceOptionList( consolidateDataSourceGroups( - dataSetResults as DataSourceDataSet[], + dataSetResults as DataSetWithDataSource[], dataSources.filter((ds) => !ds.getMetadata().ui.selector.displayDatasetsAsSource) ) ); diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index a8c1ba6a9fa3..fe1e4360e961 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -14,12 +14,11 @@ export interface DataSourceGroup { label: string; id: string; options: DataSourceOption[]; - typeGroup: string; + groupType: string; } export interface DataSourceOption { key: string; - id: string; name: string; label: string; value: string; diff --git a/src/plugins/data/public/data_sources/register_default_datasource.ts b/src/plugins/data/public/data_sources/register_default_datasource.ts index 6fa6bc8aa23c..a17695d1d2be 100644 --- a/src/plugins/data/public/data_sources/register_default_datasource.ts +++ b/src/plugins/data/public/data_sources/register_default_datasource.ts @@ -24,7 +24,7 @@ export const registerDefaultDataSource = (data: Omit Date: Sun, 28 Apr 2024 21:54:00 +0000 Subject: [PATCH 07/23] update default datasource tests Signed-off-by: Eric --- .../public/data_sources/datasource/types.ts | 6 +++- .../default_datasource.test.ts | 28 +++++++++++++---- .../default_datasource/default_datasource.ts | 6 ++-- .../register_default_datasource.ts | 30 +++++++++++++------ src/plugins/data/public/plugin.ts | 6 ++-- 5 files changed, 55 insertions(+), 21 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index a53a99a4d773..35582a085ee7 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -59,7 +59,7 @@ export interface IDataSourceUISelector { export interface IDataSourceUISettings { selector: IDataSourceUISelector; label: string; // the display name of data source - groupType: string; // the group to which the data source belongs + groupType: DataSourceUIGroupType; // the group to which the data source belongs typeLabel: string; // the display name of data source type displayOrder?: number; // the order in which the data source should be displayed in selector description?: string; // short description of your database @@ -73,3 +73,7 @@ export interface IDataSourceDataSet { export interface IDataSourceQueryResponse { data: T; } + +export enum DataSourceUIGroupType { + defaultOpenSearchDataSource = 'DEFAULT_INDEX_PATTERNS', +} diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts index aedc6cd3853a..becd8dee423e 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts @@ -4,8 +4,21 @@ */ import { IndexPatternsService } from '../../index_patterns'; +import { DataSourceUIGroupType } from '../datasource/types'; +import { DEFAULT_DATA_SOURCE_DISPLAY_NAME } from '../register_default_datasource'; import { DefaultDslDataSource } from './default_datasource'; +const defaultDataSourceMetadata = { + ui: { + label: DEFAULT_DATA_SOURCE_DISPLAY_NAME, + typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME, + groupType: DataSourceUIGroupType.defaultOpenSearchDataSource, + selector: { + displayDatasetsAsSource: true, + }, + }, +}; + describe('DefaultDslDataSource', () => { let indexPatternsMock: IndexPatternsService; @@ -18,9 +31,10 @@ describe('DefaultDslDataSource', () => { it('should ensure default index pattern and get cache', async () => { const dataSource = new DefaultDslDataSource({ + id: 'testId', name: 'testName', type: 'testType', - metadata: {}, + metadata: defaultDataSourceMetadata, indexPatterns: indexPatternsMock, }); @@ -32,9 +46,10 @@ describe('DefaultDslDataSource', () => { it('should throw an error', async () => { const dataSource = new DefaultDslDataSource({ + id: 'testId', name: 'testName', type: 'testType', - metadata: {}, + metadata: defaultDataSourceMetadata, indexPatterns: indexPatternsMock, }); @@ -43,13 +58,16 @@ describe('DefaultDslDataSource', () => { it('should return null', async () => { const dataSource = new DefaultDslDataSource({ + id: 'testId', name: 'testName', type: 'testType', - metadata: {}, + metadata: defaultDataSourceMetadata, indexPatterns: indexPatternsMock, }); - const result = await dataSource.runQuery({}); - expect(result).toBeUndefined(); + const result = await dataSource.runQuery(); + expect(result).toStrictEqual({ + data: {}, + }); }); }); diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts index 262a8ec9d93d..db65a7df903d 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts @@ -34,19 +34,19 @@ export class DefaultDslDataSource extends DataSource< LocalDSQueryParams, LocalDSQueryResponse > { - private readonly indexPatterns; + private readonly indexPatterns: IndexPatternsContract; constructor({ id, name, type, metadata, indexPatterns }: LocalDataSourceSettings) { super({ id, name, type, metadata }); this.indexPatterns = indexPatterns; } - async getDataSet() { + async getDataSet(): Promise { await this.indexPatterns.ensureDefaultIndexPattern(); const savedObjectLst = await this.indexPatterns.getCache(); if (!Array.isArray(savedObjectLst)) { - return { data_sets: [] }; + return { dataSets: [] }; } return { diff --git a/src/plugins/data/public/data_sources/register_default_datasource.ts b/src/plugins/data/public/data_sources/register_default_datasource.ts index a17695d1d2be..9dc84a73cbf0 100644 --- a/src/plugins/data/public/data_sources/register_default_datasource.ts +++ b/src/plugins/data/public/data_sources/register_default_datasource.ts @@ -6,25 +6,37 @@ import { i18n } from '@osd/i18n'; import { htmlIdGenerator } from '@elastic/eui'; import { DataPublicPluginStart } from '../types'; +import { DataSourceUIGroupType } from './datasource/types'; -export const DEFAULT_DATASOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; -export const DEFAULT_DATASOURCE_NAME = i18n.translate('data.datasource.type.openSearchDefault', { +export const DEFAULT_DATA_SOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; +export const DEFAULT_DATA_SOURCE_NAME = i18n.translate('data.datasource.type.openSearchDefault', { defaultMessage: 'OpenSearch Default', }); +export const DEFAULT_DATA_SOURCE_DISPLAY_NAME = i18n.translate( + 'data.datasource.type.openSearchDefaultDisplayName', + { + defaultMessage: 'Index patterns', + } +); +/** + * Registers the default data source with the provided data excluding 'ui'. + * This sets up the default cluster data source with predefined configurations using constants. + * @param data - Data necessary to configure the data source, except for 'ui'. + */ export const registerDefaultDataSource = (data: Omit) => { // Registrations of index patterns as default data source const { dataSourceService, dataSourceFactory } = data.dataSources; dataSourceService.registerDataSource( - dataSourceFactory.getDataSourceInstance(DEFAULT_DATASOURCE_TYPE, { - id: htmlIdGenerator('local-cluster')('indices'), - name: DEFAULT_DATASOURCE_NAME, - type: DEFAULT_DATASOURCE_TYPE, + dataSourceFactory.getDataSourceInstance(DEFAULT_DATA_SOURCE_TYPE, { + id: htmlIdGenerator(DEFAULT_DATA_SOURCE_NAME)(DEFAULT_DATA_SOURCE_TYPE), + name: DEFAULT_DATA_SOURCE_NAME, + type: DEFAULT_DATA_SOURCE_TYPE, metadata: { ui: { - label: 'Index patterns', // display name of your data source, - typeLabel: 'Index patterns', // display name of your data source type, - groupType: 'DEFAULT_INDEX_PATTERNS', + label: DEFAULT_DATA_SOURCE_DISPLAY_NAME, // display name of your data source, + typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME, // display name of your data source type, + groupType: DataSourceUIGroupType.defaultOpenSearchDataSource, selector: { displayDatasetsAsSource: true, // when true, selector UI will render data sets with source by calling getDataSets() }, diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index bc757467bd98..9aacc39d1992 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -91,7 +91,7 @@ import { indexPatternLoad } from './index_patterns/expressions/load_index_patter import { DataSourceService } from './data_sources/datasource_services'; import { DataSourceFactory } from './data_sources/datasource'; import { - DEFAULT_DATASOURCE_TYPE, + DEFAULT_DATA_SOURCE_TYPE, registerDefaultDataSource, } from './data_sources/register_default_datasource'; import { DefaultDslDataSource } from './data_sources/default_datasource'; @@ -222,10 +222,10 @@ export class DataPublicPlugin // Create or fetch the singleton instance const dataSourceService = DataSourceService.getInstance(); const dataSourceFactory = DataSourceFactory.getInstance(); - dataSourceFactory.registerDataSourceType(DEFAULT_DATASOURCE_TYPE, DefaultDslDataSource); + dataSourceFactory.registerDataSourceType(DEFAULT_DATA_SOURCE_TYPE, DefaultDslDataSource); dataSourceService.registerDataSourceFetchers([ { - type: DEFAULT_DATASOURCE_TYPE, + type: DEFAULT_DATA_SOURCE_TYPE, registerDataSources: () => registerDefaultDataSource(dataServices), }, ]); From 5e8448a95a3b5dfdcca4040c3faa3b6fecb54a5a Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 28 Apr 2024 22:04:38 +0000 Subject: [PATCH 08/23] added more tests for default datasource Signed-off-by: Eric --- .../default_datasource.test.ts | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts index becd8dee423e..6d0f8c4149ea 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts @@ -44,7 +44,55 @@ describe('DefaultDslDataSource', () => { expect(indexPatternsMock.getCache).toHaveBeenCalledTimes(1); }); - it('should throw an error', async () => { + it('should return an empty dataset if getCache returns an empty array', async () => { + indexPatternsMock.getCache.mockResolvedValue([]); + const dataSource = new DefaultDslDataSource({ + id: 'testId', + name: 'testName', + type: 'testType', + metadata: defaultDataSourceMetadata, + indexPatterns: indexPatternsMock, + }); + + const result = await dataSource.getDataSet(); + expect(result.dataSets).toEqual([]); + }); + + it('should return a populated dataset if getCache returns non-empty array', async () => { + const mockSavedObjects = [ + { id: '1', attributes: { title: 'Index1' } }, + { id: '2', attributes: { title: 'Index2' } }, + ]; + indexPatternsMock.getCache.mockResolvedValue(mockSavedObjects); + const dataSource = new DefaultDslDataSource({ + id: 'testId', + name: 'testName', + type: 'testType', + metadata: defaultDataSourceMetadata, + indexPatterns: indexPatternsMock, + }); + + const result = await dataSource.getDataSet(); + expect(result.dataSets).toEqual([ + { id: '1', title: 'Index1' }, + { id: '2', title: 'Index2' }, + ]); + }); + + it('should handle errors thrown by getCache', async () => { + indexPatternsMock.getCache.mockRejectedValue(new Error('Cache fetch failed')); + const dataSource = new DefaultDslDataSource({ + id: 'testId', + name: 'testName', + type: 'testType', + metadata: defaultDataSourceMetadata, + indexPatterns: indexPatternsMock, + }); + + await expect(dataSource.getDataSet()).rejects.toThrow('Cache fetch failed'); + }); + + it('should return true on default data source connection', async () => { const dataSource = new DefaultDslDataSource({ id: 'testId', name: 'testName', From ae2921a27ca750365b9d3f0f9b4d61cc305a309b Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 28 Apr 2024 22:22:19 +0000 Subject: [PATCH 09/23] update selector tests Signed-off-by: Eric --- .../public/data_sources/datasource/types.ts | 1 + .../datasource_selectable.test.tsx | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index 35582a085ee7..017d12eb45cf 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -76,4 +76,5 @@ export interface IDataSourceQueryResponse { export enum DataSourceUIGroupType { defaultOpenSearchDataSource = 'DEFAULT_INDEX_PATTERNS', + s3glue = 's3glue', } diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx index 0a4384516b00..b4b82a5b9ae0 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx @@ -8,6 +8,30 @@ import { render, act, screen, fireEvent } from '@testing-library/react'; import { DataSourceSelectable } from './datasource_selectable'; import { DataSourceGroup, DataSourceOption } from './types'; import { DataSource } from '../datasource/datasource'; +import { DEFAULT_DATA_SOURCE_DISPLAY_NAME } from '../register_default_datasource'; +import { DataSourceUIGroupType } from '../datasource/types'; + +const defaultDataSourceMetadata = { + ui: { + label: DEFAULT_DATA_SOURCE_DISPLAY_NAME, + typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME, + groupType: DataSourceUIGroupType.defaultOpenSearchDataSource, + selector: { + displayDatasetsAsSource: true, + }, + }, +}; + +const s3DataSourceMetadata = { + ui: { + label: 'Amazon S3', + typeLabel: 's3glue', + groupType: DataSourceUIGroupType.s3glue, + selector: { + displayDatasetsAsSource: false, + }, + }, +}; describe('DataSourceSelectable', () => { let dataSourcesMock: DataSource[]; @@ -16,6 +40,7 @@ describe('DataSourceSelectable', () => { let setSelectedSourcesMock: (sources: DataSourceOption[]) => void = jest.fn(); let setDataSourceOptionListMock: (sources: DataSourceGroup[]) => void = jest.fn(); let onFetchDataSetErrorMock: (error: Error) => void = jest.fn(); + const onRefresh: () => void = jest.fn(); beforeEach(() => { dataSourcesMock = [ @@ -23,6 +48,7 @@ describe('DataSourceSelectable', () => { getDataSet: jest.fn().mockResolvedValue([]), getType: jest.fn().mockReturnValue('DEFAULT_INDEX_PATTERNS'), getName: jest.fn().mockReturnValue('SomeName'), + getMetadata: jest.fn().mockReturnValue(defaultDataSourceMetadata), } as unknown) as DataSource, ]; @@ -42,6 +68,7 @@ describe('DataSourceSelectable', () => { onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onGetDataSetError={onFetchDataSetErrorMock} + onRefresh={onRefresh} /> ); }); @@ -56,6 +83,7 @@ describe('DataSourceSelectable', () => { onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onGetDataSetError={onFetchDataSetErrorMock} + onRefresh={onRefresh} /> ); }); @@ -75,6 +103,7 @@ describe('DataSourceSelectable', () => { onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onGetDataSetError={onFetchDataSetErrorMock} + onRefresh={onRefresh} /> ); }); @@ -104,6 +133,7 @@ describe('DataSourceSelectable', () => { getDataSet: jest.fn().mockResolvedValue([]), getType: jest.fn().mockReturnValue('DEFAULT_INDEX_PATTERNS'), getName: jest.fn().mockReturnValue('Index patterns'), + getMetadata: jest.fn().mockReturnValue(defaultDataSourceMetadata), } as unknown) as DataSource, ]} dataSourceOptionList={mockDataSourceOptionList} @@ -111,6 +141,7 @@ describe('DataSourceSelectable', () => { onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onGetDataSetError={onFetchDataSetErrorMock} + onRefresh={onRefresh} /> ); @@ -154,6 +185,7 @@ describe('DataSourceSelectable', () => { getDataSet: jest.fn().mockResolvedValue([]), getType: jest.fn().mockReturnValue('s3glue'), getName: jest.fn().mockReturnValue('Amazon S3'), + getMetadata: jest.fn().mockReturnValue(s3DataSourceMetadata), } as unknown) as DataSource, ]} dataSourceOptionList={mockDataSourceOptionList} @@ -161,6 +193,7 @@ describe('DataSourceSelectable', () => { onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onGetDataSetError={onFetchDataSetErrorMock} + onRefresh={onRefresh} /> ); @@ -204,6 +237,7 @@ describe('DataSourceSelectable', () => { getDataSet: jest.fn().mockResolvedValue([]), getType: jest.fn().mockReturnValue('DEFAULT_INDEX_PATTERNS'), getName: jest.fn().mockReturnValue('Index patterns'), + getMetadata: jest.fn().mockReturnValue(defaultDataSourceMetadata), } as unknown) as DataSource, ]} dataSourceOptionList={mockDataSourceOptionListWithDuplicates} @@ -211,6 +245,7 @@ describe('DataSourceSelectable', () => { onDataSourceSelect={handleSelect} setDataSourceOptionList={setDataSourceOptionListMock} onGetDataSetError={onFetchDataSetErrorMock} + onRefresh={onRefresh} /> ); From 6e14ca8d6b0fe7624fc72bb4a54bcfec3f0900b3 Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 28 Apr 2024 22:25:38 +0000 Subject: [PATCH 10/23] update changelog Signed-off-by: Eric --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a7101f6342f..86dc4a7bd4b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - [Multiple Datasource] Extract the button component for datasource picker to avoid duplicate code ([#6559](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6559)) - [Workspace] Add workspaces filter to saved objects page. ([#6458](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6458)) - [Multiple Datasource] Support multi data source in Region map ([#6654](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6654)) +- [Discover Data Selector] - Data selector enhancement and refactoring ([#6571](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6571)) ### 🐛 Bug Fixes From ddbb6f207850e06702000112d95ff7e38b8f5fab Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 28 Apr 2024 23:11:36 +0000 Subject: [PATCH 11/23] fix data source service tests Signed-off-by: Eric --- .../datasource_service.test.ts | 48 +++++++++++++++---- .../default_datasource.test.ts | 2 +- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts index 2c8f393d7093..7e9d7e3bca84 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts @@ -6,26 +6,54 @@ import { DataSource } from '../datasource'; import { IndexPatternsService } from '../../index_patterns'; import { DataSourceService } from '../datasource_services'; +import { + LocalDSDataSetParams, + LocalDSDataSetResponse, + LocalDSMetadata, + LocalDSQueryParams, + LocalDSQueryResponse, +} from '../default_datasource/default_datasource'; +import { DataSourceUIGroupType } from '../datasource/types'; +import { DEFAULT_DATA_SOURCE_DISPLAY_NAME } from '../register_default_datasource'; + +export const defaultDataSourceMetadata = { + ui: { + label: DEFAULT_DATA_SOURCE_DISPLAY_NAME, + typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME, + groupType: DataSourceUIGroupType.defaultOpenSearchDataSource, + selector: { + displayDatasetsAsSource: true, + }, + }, +}; -class MockDataSource extends DataSource { +class MockDataSource extends DataSource< + LocalDSMetadata, + LocalDSDataSetParams, + LocalDSDataSetResponse, + LocalDSQueryParams, + LocalDSQueryResponse +> { private readonly indexPattern; constructor({ + id, name, type, metadata, indexPattern, }: { + id: string; name: string; type: string; metadata: any; indexPattern: IndexPatternsService; }) { - super(name, type, metadata); + super({ id, name, type, metadata }); this.indexPattern = indexPattern; } - async getDataSet(dataSetParams?: any) { + async getDataSet(dataSetParams?: LocalDSDataSetParams): Promise { await this.indexPattern.ensureDefaultIndexPattern(); return await this.indexPattern.getCache(); } @@ -34,24 +62,28 @@ class MockDataSource extends DataSource { return true; } - async runQuery(queryParams: any) { - return undefined; + async runQuery(queryParams: any): Promise { + return { + data: {}, + }; } } const mockIndexPattern = {} as IndexPatternsService; const mockConfig1 = { + id: 'test_datasource1', name: 'test_datasource1', type: 'mock1', - metadata: null, + metadata: defaultDataSourceMetadata, indexPattern: mockIndexPattern, }; const mockConfig2 = { + id: 'test_datasource2', name: 'test_datasource2', type: 'mock1', - metadata: null, + metadata: defaultDataSourceMetadata, indexPattern: mockIndexPattern, }; @@ -79,7 +111,7 @@ describe('DataSourceService', () => { const ds = new MockDataSource(mockConfig1); await service.registerDataSource(ds); await expect(service.registerDataSource(ds)).rejects.toThrow( - 'Unable to register datasource test_datasource1, error: datasource name exists.' + 'Unable to register data source test_datasource1, error: data source name exists.' ); }); diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts index 6d0f8c4149ea..9758a7f11718 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts @@ -8,7 +8,7 @@ import { DataSourceUIGroupType } from '../datasource/types'; import { DEFAULT_DATA_SOURCE_DISPLAY_NAME } from '../register_default_datasource'; import { DefaultDslDataSource } from './default_datasource'; -const defaultDataSourceMetadata = { +export const defaultDataSourceMetadata = { ui: { label: DEFAULT_DATA_SOURCE_DISPLAY_NAME, typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME, From 535d558dea667769e930b93dc714e6af205f7396 Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 28 Apr 2024 23:57:31 +0000 Subject: [PATCH 12/23] add and update tests for datasource service Signed-off-by: Eric --- .../datasource_service.test.ts | 19 ++++++---- .../datasource_services/datasource_service.ts | 38 +++++++++++++------ .../data_sources/datasource_services/types.ts | 4 +- .../public/components/sidebar/index.tsx | 8 ++-- 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts index 7e9d7e3bca84..4fa00bf2ac79 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { waitFor } from '@testing-library/dom'; import { DataSource } from '../datasource'; import { IndexPatternsService } from '../../index_patterns'; import { DataSourceService } from '../datasource_services'; @@ -16,7 +17,7 @@ import { import { DataSourceUIGroupType } from '../datasource/types'; import { DEFAULT_DATA_SOURCE_DISPLAY_NAME } from '../register_default_datasource'; -export const defaultDataSourceMetadata = { +const defaultDataSourceMetadata = { ui: { label: DEFAULT_DATA_SOURCE_DISPLAY_NAME, typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME, @@ -133,9 +134,11 @@ describe('DataSourceService', () => { const ds2 = new MockDataSource(mockConfig2); service.registerMultipleDataSources([ds1, ds2]); const filter = { names: ['test_datasource1'] }; - const retrievedDataSources = service.getDataSources(filter); - expect(retrievedDataSources).toHaveProperty('test_datasource1'); - expect(retrievedDataSources).not.toHaveProperty('test_datasource2'); + waitFor(() => { + const retrievedDataSources = service.getDataSources$(filter); + expect(retrievedDataSources).toHaveProperty('test_datasource1'); + expect(retrievedDataSources).not.toHaveProperty('test_datasource2'); + }); }); it('returns all data sources if no filters provided', () => { @@ -143,8 +146,10 @@ describe('DataSourceService', () => { const ds1 = new MockDataSource(mockConfig1); const ds2 = new MockDataSource(mockConfig2); service.registerMultipleDataSources([ds1, ds2]); - const retrievedDataSources = service.getDataSources(); - expect(retrievedDataSources).toHaveProperty('test_datasource1'); - expect(retrievedDataSources).toHaveProperty('test_datasource2'); + waitFor(() => { + const retrievedDataSources = service.getDataSources$(); + expect(retrievedDataSources).toHaveProperty('test_datasource1'); + expect(retrievedDataSources).toHaveProperty('test_datasource2'); + }); }); }); diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index e4cef2ece4ca..3d9940550697 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -4,6 +4,7 @@ */ import { BehaviorSubject } from 'rxjs'; +import { map } from 'rxjs/operators'; import { DataSourceRegistrationError, IDataSourceFilter, @@ -65,8 +66,11 @@ export class DataSourceService { } } - public get dataSources$() { - return this.dataSourcesSubject.asObservable(); + private isFilterEmpty(filter: IDataSourceFilter): boolean { + // Check if all filter properties are either undefined or empty arrays + return Object.values(filter).every( + (value) => !value || (Array.isArray(value) && value.length === 0) + ); } /** @@ -76,16 +80,28 @@ export class DataSourceService { * @param filter - An optional object with filter criteria (e.g., names of data sources). * @returns A record of filtered data sources. */ - getDataSources(filter?: IDataSourceFilter): Record { - if (!filter || !Array.isArray(filter.names) || filter.names.length === 0) - return this.dataSources; + public getDataSources$(filter?: IDataSourceFilter) { + return this.dataSourcesSubject.asObservable().pipe( + map((dataSources) => { + // Check if the filter is provided and valid + if (!filter || this.isFilterEmpty(filter)) { + return dataSources; + } - return filter.names.reduce>((filteredDataSources, dsId) => { - if (dsId in this.dataSources) { - filteredDataSources[dsId] = this.dataSources[dsId]; - } - return filteredDataSources; - }, {} as Record); + // Apply filter + return Object.entries(dataSources).reduce((acc, [id, dataSource]) => { + const matchesId = !filter.ids || filter.ids.includes(id); + const matchesName = !filter.names || filter.names.includes(dataSource.getName()); + const matchesType = !filter.types || filter.types.includes(dataSource.getType()); + + if (matchesId && matchesName && matchesType) { + acc[id] = dataSource; + } + + return acc; + }, {} as Record); + }) + ); } /** diff --git a/src/plugins/data/public/data_sources/datasource_services/types.ts b/src/plugins/data/public/data_sources/datasource_services/types.ts index 1cbdbeadea8c..1e657802a0a8 100644 --- a/src/plugins/data/public/data_sources/datasource_services/types.ts +++ b/src/plugins/data/public/data_sources/datasource_services/types.ts @@ -12,7 +12,9 @@ import { DataSourceFactory } from '../datasource'; import { DataSourceService } from './datasource_service'; export interface IDataSourceFilter { - names: string[]; + ids?: string[]; // Array of data source IDs to filter by + names?: string[]; // Array of data source names to filter by + types?: string[]; // Array of data source types to filter by } export interface IDataSourceRegistrationResult { diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 21f8d20d609a..b65bd43950cd 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -30,13 +30,13 @@ export const Sidebar: FC = ({ children }) => { useEffect(() => { let isMounted = true; - const subscription = dataSources.dataSourceService.dataSources$.subscribe( - (currentDataSources) => { + const subscription = dataSources.dataSourceService + .getDataSources$() + .subscribe((currentDataSources) => { if (isMounted) { setActiveDataSources(Object.values(currentDataSources)); } - } - ); + }); return () => { subscription.unsubscribe(); From 7de39cfba11c3d832f1873489ae32a3b1ecee28b Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 29 Apr 2024 02:26:59 +0000 Subject: [PATCH 13/23] add more data source service tests Signed-off-by: Eric --- .../datasource_selectable.test.tsx | 23 +++++ .../datasource_service.test.ts | 92 +++++++++++++++++++ .../datasource_services/datasource_service.ts | 17 +++- 3 files changed, 129 insertions(+), 3 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx index b4b82a5b9ae0..29982316225a 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx @@ -50,6 +50,12 @@ describe('DataSourceSelectable', () => { getName: jest.fn().mockReturnValue('SomeName'), getMetadata: jest.fn().mockReturnValue(defaultDataSourceMetadata), } as unknown) as DataSource, + ({ + getDataSet: jest.fn().mockResolvedValue([]), + getType: jest.fn().mockReturnValue('s3glue'), + getName: jest.fn().mockReturnValue('Amazon S3'), + getMetadata: jest.fn().mockReturnValue(s3DataSourceMetadata), + } as unknown) as DataSource, ]; dataSourceOptionListMock = []; @@ -259,4 +265,21 @@ describe('DataSourceSelectable', () => { expect.objectContaining([{ key: 'unique-key-3', label: 'duplicate-index-pattern' }]) ); }); + + it('should trigger onRefresh when the refresh button is clicked', () => { + const { getByLabelText } = render( + + ); + const refreshButton = getByLabelText('sourceRefresh'); + fireEvent.click(refreshButton); + expect(onRefresh).toHaveBeenCalled(); + }); }); diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts index 4fa00bf2ac79..113dcadca9f1 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts @@ -128,6 +128,20 @@ describe('DataSourceService', () => { }); }); + it('loads data sources successfully with fetchers', async () => { + const service = DataSourceService.getInstance(); + const fetcherMock = jest.fn(() => { + service.registerDataSource(new MockDataSource(mockConfig1)); // Simulates adding a new data source after fetching + }); + service.registerDataSourceFetchers([{ type: 'mock', registerDataSources: fetcherMock }]); + + await service.load(); + expect(fetcherMock).toHaveBeenCalledTimes(1); + service.getDataSources$().subscribe((dataSources) => { + expect(Object.keys(dataSources)).toContain('test_datasource1'); + }); + }); + it('retrieves registered data sources based on filters', () => { const service = DataSourceService.getInstance(); const ds1 = new MockDataSource(mockConfig1); @@ -152,4 +166,82 @@ describe('DataSourceService', () => { expect(retrievedDataSources).toHaveProperty('test_datasource2'); }); }); + + it('should reset and load data sources using registered fetchers', async () => { + const service = DataSourceService.getInstance(); + const fetcherMock = jest.fn(); + service.registerDataSourceFetchers([{ type: 'mock', registerDataSources: fetcherMock }]); + + service.load(); + expect(fetcherMock).toHaveBeenCalled(); + expect( + service.getDataSources$().subscribe((dataSources) => { + expect(Object.keys(dataSources).length).toBe(0); + }) + ); + }); + + it('handles failures in data fetchers gracefully during load', async () => { + const service = DataSourceService.getInstance(); + const fetcherMock = jest.fn(() => { + throw new Error('Failed to fetch data sources'); + }); + service.registerDataSourceFetchers([{ type: 'mock', registerDataSources: fetcherMock }]); + + await service.load(); + expect(fetcherMock).toHaveBeenCalledTimes(1); + service.getDataSources$().subscribe((dataSources) => { + expect(Object.keys(dataSources).length).toBe(0); // Assuming reset clears everything regardless of fetcher success + }); + }); + + it('should call load method when reload is invoked', () => { + const service = DataSourceService.getInstance(); + const loadSpy = jest.spyOn(service, 'load'); + service.reload(); + expect(loadSpy).toHaveBeenCalled(); + }); + + it('should reset all registered data sources', () => { + const service = DataSourceService.getInstance(); + const ds = new MockDataSource(mockConfig1); + service.registerDataSource(ds); // Ensure there is at least one data source + + service.reset(); + expect( + service.getDataSources$().subscribe((dataSources) => { + expect(Object.keys(dataSources).length).toBe(0); + }) + ); + }); + + it('successfully reloads and updates data sources', async () => { + const service = DataSourceService.getInstance(); + const fetcherMock = jest.fn().mockResolvedValue('Data fetched successfully'); + + // Register fetchers and perform initial load + service.registerDataSourceFetchers([{ type: 'default', registerDataSources: fetcherMock }]); + await service.load(); + + // Reset mocks to clear previous calls and reload + fetcherMock.mockClear(); + service.reload(); + + expect(fetcherMock).toHaveBeenCalledTimes(1); + service.getDataSources$().subscribe((dataSources) => { + // Expect that new data has been loaded; specifics depend on your implementation + expect(dataSources).toEqual(expect.anything()); // Adjust expectation based on your data structure + }); + }); + + it('ensures complete clearance of data sources on reset', () => { + const service = DataSourceService.getInstance(); + service.registerDataSource(new MockDataSource(mockConfig1)); + service.registerDataSource(new MockDataSource(mockConfig2)); + + service.reset(); + service.getDataSources$().subscribe((dataSources) => { + expect(Object.keys(dataSources).length).toBe(0); + }); + }); }); diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index 3d9940550697..b5b1f912ee8b 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -18,6 +18,9 @@ export class DataSourceService { // A record to store all registered data sources, using the data source name as the key. private dataSources: Record = {}; private dataSourcesSubject: BehaviorSubject>; + // A record to store all data source fetchers, using the data source type as the key. + // Once application starts, all the different types of data source supported with have their fetchers registered here. + // And it becomes the single source of truth for reloading data sources. private dataSourceFetchers: Record = {}; private constructor() { @@ -110,8 +113,9 @@ export class DataSourceService { * @param fetchers - An array of fetcher configurations, each specifying how to fetch data for a specific data source type. */ registerDataSourceFetchers(fetchers: DataSourceFetcher[]) { - fetchers.forEach((fetcher) => { - this.dataSourceFetchers[fetcher.type] = fetcher.registerDataSources; + return fetchers.forEach((fetcher) => { + if (!this.dataSourceFetchers[fetcher.type]) + this.dataSourceFetchers[fetcher.type] = fetcher.registerDataSources; }); } @@ -121,7 +125,14 @@ export class DataSourceService { */ load() { this.reset(); - Object.values(this.dataSourceFetchers).forEach((fetch) => fetch()); + Object.values(this.dataSourceFetchers).forEach((fetch) => { + try { + fetch(); // Directly call the synchronous fetch function + } catch (error) { + // Handle fetch errors or take corrective actions here + // TO-DO: Add error handling, maybe collect errors and show them in UI + } + }); } /** From 2a9b5d7327b7078388b4ece4df3219458f9b5d2c Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 29 Apr 2024 16:55:59 +0000 Subject: [PATCH 14/23] fix sidebar tests Signed-off-by: Eric --- .../data/public/data_sources/constants.tsx | 29 +++++++++++++++ .../public/data_sources/datasource/types.ts | 1 + .../datasource_selectable.test.tsx | 25 +------------ .../public/components/sidebar/index.test.tsx | 36 +++++++++++-------- .../public/__mock__/index.test.mock.ts | 29 ++++++++------- 5 files changed, 68 insertions(+), 52 deletions(-) create mode 100644 src/plugins/data/public/data_sources/constants.tsx diff --git a/src/plugins/data/public/data_sources/constants.tsx b/src/plugins/data/public/data_sources/constants.tsx new file mode 100644 index 000000000000..59dd1830b319 --- /dev/null +++ b/src/plugins/data/public/data_sources/constants.tsx @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DataSourceUIGroupType } from './datasource/types'; +import { DEFAULT_DATA_SOURCE_DISPLAY_NAME } from './register_default_datasource'; + +export const defaultDataSourceMetadata = { + ui: { + label: DEFAULT_DATA_SOURCE_DISPLAY_NAME, + typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME, + groupType: DataSourceUIGroupType.defaultOpenSearchDataSource, + selector: { + displayDatasetsAsSource: true, + }, + }, +}; + +export const s3DataSourceMetadata = { + ui: { + label: 'Amazon S3', + typeLabel: 's3glue', + groupType: DataSourceUIGroupType.s3glue, + selector: { + displayDatasetsAsSource: false, + }, + }, +}; diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index 017d12eb45cf..315ee9b9709a 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -77,4 +77,5 @@ export interface IDataSourceQueryResponse { export enum DataSourceUIGroupType { defaultOpenSearchDataSource = 'DEFAULT_INDEX_PATTERNS', s3glue = 's3glue', + spark = 'spark', } diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx index 29982316225a..cdef0500a2a3 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx @@ -8,30 +8,7 @@ import { render, act, screen, fireEvent } from '@testing-library/react'; import { DataSourceSelectable } from './datasource_selectable'; import { DataSourceGroup, DataSourceOption } from './types'; import { DataSource } from '../datasource/datasource'; -import { DEFAULT_DATA_SOURCE_DISPLAY_NAME } from '../register_default_datasource'; -import { DataSourceUIGroupType } from '../datasource/types'; - -const defaultDataSourceMetadata = { - ui: { - label: DEFAULT_DATA_SOURCE_DISPLAY_NAME, - typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME, - groupType: DataSourceUIGroupType.defaultOpenSearchDataSource, - selector: { - displayDatasetsAsSource: true, - }, - }, -}; - -const s3DataSourceMetadata = { - ui: { - label: 'Amazon S3', - typeLabel: 's3glue', - groupType: DataSourceUIGroupType.s3glue, - selector: { - displayDatasetsAsSource: false, - }, - }, -}; +import { defaultDataSourceMetadata, s3DataSourceMetadata } from '../constants'; describe('DataSourceSelectable', () => { let dataSourcesMock: DataSource[]; diff --git a/src/plugins/data_explorer/public/components/sidebar/index.test.tsx b/src/plugins/data_explorer/public/components/sidebar/index.test.tsx index eccb0ffa909e..540321be55bf 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.test.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.test.tsx @@ -5,12 +5,14 @@ import React from 'react'; import { render, fireEvent, waitFor } from '@testing-library/react'; -import { Sidebar } from './index'; // Adjust the import path as necessary +import { of } from 'rxjs/'; +import { createMemoryHistory } from 'history'; +import { Router } from 'react-router-dom'; import { Provider } from 'react-redux'; import configureMockStore from 'redux-mock-store'; +import { Sidebar } from './index'; // Adjust the import path as necessary import { MockS3DataSource } from '../../../../discover/public/__mock__/index.test.mock'; -import { createMemoryHistory } from 'history'; -import { Router } from 'react-router-dom'; +import { s3DataSourceMetadata } from 'src/plugins/data/public/data_sources/constants'; const mockStore = configureMockStore(); const initialState = { @@ -18,6 +20,12 @@ const initialState = { }; const store = mockStore(initialState); +export const createObservable = (data: any) => { + return of(data); +}; + +const getMetaData = () => ({ ...s3DataSourceMetadata }); + jest.mock('../../../../opensearch_dashboards_react/public', () => { return { toMountPoint: jest.fn().mockImplementation((component) => () => component), @@ -27,18 +35,16 @@ jest.mock('../../../../opensearch_dashboards_react/public', () => { indexPatterns: {}, dataSources: { dataSourceService: { - dataSources$: { - subscribe: jest.fn((callback) => { - callback({ - 's3-prod-mock': new MockS3DataSource({ - name: 's3-prod-mock', - type: 's3glue', - metadata: {}, - }), - }); - return { unsubscribe: jest.fn() }; - }), - }, + getDataSources$: jest.fn().mockImplementation(() => + createObservable({ + 's3-prod-mock': new MockS3DataSource({ + id: 's3-prod-mock', + name: 's3-prod-mock', + type: 's3glue', + metadata: getMetaData(), + }), + }) + ), }, }, }, diff --git a/src/plugins/discover/public/__mock__/index.test.mock.ts b/src/plugins/discover/public/__mock__/index.test.mock.ts index 6b09d1d84253..d0b47502c301 100644 --- a/src/plugins/discover/public/__mock__/index.test.mock.ts +++ b/src/plugins/discover/public/__mock__/index.test.mock.ts @@ -3,26 +3,29 @@ * SPDX-License-Identifier: Apache-2.0 */ -export class MockS3DataSource { - protected name: string; - protected type: string; - protected metadata: any; +import { DataSource } from 'src/plugins/data/public'; - constructor({ name, type, metadata }: { name: string; type: string; metadata: any }) { - this.name = name; - this.type = type; - this.metadata = metadata; +interface DataSourceConfig { + name: string; + type: string; + metadata: any; + id: string; +} + +export class MockS3DataSource extends DataSource { + constructor({ id, name, type, metadata }: DataSourceConfig) { + super({ id, name, type, metadata }); } async getDataSet(dataSetParams?: any) { - return [this.name]; + return { dataSets: [this.getName()] }; } - getName() { - return this.name; + async testConnection(): Promise { + return true; } - getType() { - return this.type; + async runQuery(queryParams: any) { + return { data: {} }; } } From a41a88cfcdbe7a7a708a8034dd158781369fc4af Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 29 Apr 2024 17:03:23 +0000 Subject: [PATCH 15/23] add to change log yml Signed-off-by: Eric --- changelogs/fragments/6571.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/6571.yml diff --git a/changelogs/fragments/6571.yml b/changelogs/fragments/6571.yml new file mode 100644 index 000000000000..a6e341fc15e4 --- /dev/null +++ b/changelogs/fragments/6571.yml @@ -0,0 +1,2 @@ +refactor: +- discover data selector enhancement and refactoring ([#6571](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6571)) \ No newline at end of file From ee1aad34e1c7d3f48d5d67957e5f9f24d52651f2 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 30 Apr 2024 00:02:22 +0000 Subject: [PATCH 16/23] address comments along with more tests Signed-off-by: Eric --- .../data/public/data_sources/constants.tsx | 23 +++++- .../public/data_sources/datasource/types.ts | 57 +++++++++++--- .../data_selector_refresher.test.tsx | 77 +++++++++++++++++++ .../data_selector_refresher.tsx | 47 +++++++++++ .../datasource_selectable.test.tsx | 14 +++- .../datasource_selectable.tsx | 32 ++++---- .../default_datasource.test.ts | 14 +--- .../register_default_datasource.ts | 17 ++-- 8 files changed, 221 insertions(+), 60 deletions(-) create mode 100644 src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.test.tsx create mode 100644 src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.tsx diff --git a/src/plugins/data/public/data_sources/constants.tsx b/src/plugins/data/public/data_sources/constants.tsx index 59dd1830b319..325d93954af3 100644 --- a/src/plugins/data/public/data_sources/constants.tsx +++ b/src/plugins/data/public/data_sources/constants.tsx @@ -3,8 +3,21 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { i18n } from '@osd/i18n'; import { DataSourceUIGroupType } from './datasource/types'; -import { DEFAULT_DATA_SOURCE_DISPLAY_NAME } from './register_default_datasource'; + +export const S3_GLUE_DATA_SOURCE_DISPLAY_NAME = 'Amazon S3'; +export const S3_GLUE_DATA_SOURCE_TYPE = 's3glue'; +export const DEFAULT_DATA_SOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; +export const DEFAULT_DATA_SOURCE_NAME = i18n.translate('data.datasource.type.openSearchDefault', { + defaultMessage: 'OpenSearch Default', +}); +export const DEFAULT_DATA_SOURCE_DISPLAY_NAME = i18n.translate( + 'data.datasource.type.openSearchDefaultDisplayName', + { + defaultMessage: 'Index patterns', + } +); export const defaultDataSourceMetadata = { ui: { @@ -19,11 +32,15 @@ export const defaultDataSourceMetadata = { export const s3DataSourceMetadata = { ui: { - label: 'Amazon S3', - typeLabel: 's3glue', + label: S3_GLUE_DATA_SOURCE_DISPLAY_NAME, + typeLabel: S3_GLUE_DATA_SOURCE_TYPE, groupType: DataSourceUIGroupType.s3glue, selector: { displayDatasetsAsSource: false, }, }, }; + +export const DATA_SELECTOR_REFRESHER_POPOVER_TEXT = 'Refresh data sources'; +export const DATA_SELECTOR_DEFAULT_PLACEHOLDER = 'Select a data source'; +export const DATA_SELECTOR_S3_DATA_SOURCE_GROUP_HINT_LABEL = ' - Opens in Log Explorer'; diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index 315ee9b9709a..747c872ff513 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -18,9 +18,9 @@ export interface IDataSourceGroup { name: string; } -export interface DataSetWithDataSource { +export interface DataSetWithDataSource { ds: DataSource; - list: T; + list: T[]; } export interface IDataSetParams { @@ -35,8 +35,14 @@ export interface IDataSourceQueryResult { data: T; } +export enum ConnectionStatus { + Connected = 'connected', + Disconnected = 'disconnected', + Error = 'error', +} + export interface DataSourceConnectionStatus { - status: string; + status: ConnectionStatus; message: string; error?: Error; } @@ -56,14 +62,47 @@ export interface IDataSourceUISelector { displayDatasetsAsSource: boolean; } +/** + * Represents the UI settings for a data source. + */ export interface IDataSourceUISettings { + /** + * Controls UI elements related to data source selector. + */ selector: IDataSourceUISelector; - label: string; // the display name of data source - groupType: DataSourceUIGroupType; // the group to which the data source belongs - typeLabel: string; // the display name of data source type - displayOrder?: number; // the order in which the data source should be displayed in selector - description?: string; // short description of your database - icon?: string; // uri of the icon + + /** + * The display name of the data source. + */ + label: string; + + /** + * The group to which the data source belongs. This is used to group data sources in the selector. + */ + groupType: DataSourceUIGroupType; + + /** + * The display name of the data source type. + */ + typeLabel: string; + + /** + * The order in which the data source should be displayed in the selector. + * @optional + */ + displayOrder?: number; + + /** + * A short description of the data source. + * @optional + */ + description?: string; + + /** + * URI of the icon representing the data source. + * @optional + */ + icon?: string; } export interface IDataSourceDataSet { diff --git a/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.test.tsx new file mode 100644 index 000000000000..1a793e45d9a3 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.test.tsx @@ -0,0 +1,77 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import '@testing-library/jest-dom/extend-expect'; +import { DataSelectorRefresher } from './data_selector_refresher'; // adjust the import path as necessary +import { DATA_SELECTOR_REFRESHER_POPOVER_TEXT } from '../constants'; +import { ToolTipDelay } from '@elastic/eui/src/components/tool_tip/tool_tip'; +import { EuiToolTipProps } from '@elastic/eui'; + +describe('DataSelectorRefresher', () => { + const tooltipText = DATA_SELECTOR_REFRESHER_POPOVER_TEXT; + const onRefreshMock = jest.fn(); + + it('renders correctly with given tooltip text', () => { + const container = render( + + ); + + const refreshButton = container.getByLabelText('sourceRefresh'); + fireEvent.mouseOver(refreshButton); + + waitFor(() => { + expect(container.getByText(tooltipText)).toBeInTheDocument(); + }); + }); + + it('calls onRefresh when button is clicked', () => { + const container = render( + + ); + + fireEvent.click(container.getByLabelText('sourceRefresh')); + expect(onRefreshMock).toHaveBeenCalledTimes(1); + }); + + it('applies additional button properties', () => { + const buttonProps = { + 'aria-label': 'Custom Aria Label', + }; + + render( + + ); + + const button = screen.getByTestId('sourceRefreshButton'); + expect(button).toHaveAttribute('aria-label', 'Custom Aria Label'); + }); + + it('applies additional tooltip properties', () => { + const toolTipProps: Partial = { + delay: 'long' as ToolTipDelay, + }; + + const container = render( + + ); + + const refreshButton = container.getByLabelText('sourceRefresh'); + fireEvent.mouseOver(refreshButton); + waitFor(() => { + const tooltip = screen.getByTestId('sourceRefreshButtonToolTip'); + expect(tooltip).toHaveAttribute('delay', 'long'); + }); + }); +}); diff --git a/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.tsx b/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.tsx new file mode 100644 index 000000000000..08ec73fd2ab1 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.tsx @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { i18n } from '@osd/i18n'; +import { + EuiButtonIcon, + EuiButtonIconProps, + EuiText, + EuiToolTip, + EuiToolTipProps, +} from '@elastic/eui'; + +interface IDataSelectorRefresherProps { + tooltipText: string; + onRefresh: () => void; + buttonProps?: Partial; + toolTipProps?: Partial; +} + +export const DataSelectorRefresher: React.FC = React.memo( + ({ tooltipText, onRefresh, buttonProps, toolTipProps }) => { + return ( + + + + + + ); + } +); diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx index cdef0500a2a3..6cb701c3b132 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx @@ -8,7 +8,13 @@ import { render, act, screen, fireEvent } from '@testing-library/react'; import { DataSourceSelectable } from './datasource_selectable'; import { DataSourceGroup, DataSourceOption } from './types'; import { DataSource } from '../datasource/datasource'; -import { defaultDataSourceMetadata, s3DataSourceMetadata } from '../constants'; +import { + DEFAULT_DATA_SOURCE_DISPLAY_NAME, + S3_GLUE_DATA_SOURCE_DISPLAY_NAME, + DEFAULT_DATA_SOURCE_TYPE, + defaultDataSourceMetadata, + s3DataSourceMetadata, +} from '../constants'; describe('DataSourceSelectable', () => { let dataSourcesMock: DataSource[]; @@ -167,7 +173,7 @@ describe('DataSourceSelectable', () => { ({ getDataSet: jest.fn().mockResolvedValue([]), getType: jest.fn().mockReturnValue('s3glue'), - getName: jest.fn().mockReturnValue('Amazon S3'), + getName: jest.fn().mockReturnValue(S3_GLUE_DATA_SOURCE_DISPLAY_NAME), getMetadata: jest.fn().mockReturnValue(s3DataSourceMetadata), } as unknown) as DataSource, ]} @@ -218,8 +224,8 @@ describe('DataSourceSelectable', () => { dataSources={[ ({ getDataSet: jest.fn().mockResolvedValue([]), - getType: jest.fn().mockReturnValue('DEFAULT_INDEX_PATTERNS'), - getName: jest.fn().mockReturnValue('Index patterns'), + getType: jest.fn().mockReturnValue(DEFAULT_DATA_SOURCE_TYPE), + getName: jest.fn().mockReturnValue(DEFAULT_DATA_SOURCE_DISPLAY_NAME), getMetadata: jest.fn().mockReturnValue(defaultDataSourceMetadata), } as unknown) as DataSource, ]} diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 5ea25d75ff8c..d9cdccc948dc 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -4,10 +4,16 @@ */ import React, { useEffect, useCallback, useMemo } from 'react'; -import { EuiButtonIcon, EuiComboBox, EuiText, EuiToolTip } from '@elastic/eui'; +import { EuiComboBox } from '@elastic/eui'; import { i18n } from '@osd/i18n'; import { DataSource, DataSetWithDataSource, IndexPatternOption } from '../datasource'; import { DataSourceGroup, DataSourceOption, DataSourceSelectableProps } from './types'; +import { DataSelectorRefresher } from './data_selector_refresher'; +import { + DATA_SELECTOR_DEFAULT_PLACEHOLDER, + DATA_SELECTOR_REFRESHER_POPOVER_TEXT, + DATA_SELECTOR_S3_DATA_SOURCE_GROUP_HINT_LABEL, +} from '../constants'; // Asynchronously retrieves and formats dataset from a given data source. const getAndFormatDataSetFromDataSource = async ( @@ -70,7 +76,7 @@ const addOrUpdateGroup = ( if (dataSource.getType() !== 'DEFAULT_INDEX_PATTERNS') { groupName += i18n.translate('dataExplorer.dataSourceSelector.redirectionHint', { - defaultMessage: ' - Opens in Log Explorer', + defaultMessage: DATA_SELECTOR_S3_DATA_SOURCE_GROUP_HINT_LABEL, }); } @@ -158,7 +164,7 @@ export const DataSourceSelectable = ({ {...comboBoxProps} data-test-subj="dataExplorerDSSelect" placeholder={i18n.translate('data.datasource.selectADatasource', { - defaultMessage: 'Select a datasource', + defaultMessage: DATA_SELECTOR_DEFAULT_PLACEHOLDER, })} options={memorizedDataSourceOptionList as any} selectedOptions={selectedSources as any} @@ -166,22 +172,10 @@ export const DataSourceSelectable = ({ singleSelection={singleSelection} isClearable={false} append={ - - - - - + } /> ); diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts index 9758a7f11718..d302378c0177 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts @@ -4,21 +4,9 @@ */ import { IndexPatternsService } from '../../index_patterns'; -import { DataSourceUIGroupType } from '../datasource/types'; -import { DEFAULT_DATA_SOURCE_DISPLAY_NAME } from '../register_default_datasource'; +import { defaultDataSourceMetadata } from '../constants'; import { DefaultDslDataSource } from './default_datasource'; -export const defaultDataSourceMetadata = { - ui: { - label: DEFAULT_DATA_SOURCE_DISPLAY_NAME, - typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME, - groupType: DataSourceUIGroupType.defaultOpenSearchDataSource, - selector: { - displayDatasetsAsSource: true, - }, - }, -}; - describe('DefaultDslDataSource', () => { let indexPatternsMock: IndexPatternsService; diff --git a/src/plugins/data/public/data_sources/register_default_datasource.ts b/src/plugins/data/public/data_sources/register_default_datasource.ts index 9dc84a73cbf0..ce169955704b 100644 --- a/src/plugins/data/public/data_sources/register_default_datasource.ts +++ b/src/plugins/data/public/data_sources/register_default_datasource.ts @@ -3,21 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { i18n } from '@osd/i18n'; import { htmlIdGenerator } from '@elastic/eui'; import { DataPublicPluginStart } from '../types'; import { DataSourceUIGroupType } from './datasource/types'; - -export const DEFAULT_DATA_SOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; -export const DEFAULT_DATA_SOURCE_NAME = i18n.translate('data.datasource.type.openSearchDefault', { - defaultMessage: 'OpenSearch Default', -}); -export const DEFAULT_DATA_SOURCE_DISPLAY_NAME = i18n.translate( - 'data.datasource.type.openSearchDefaultDisplayName', - { - defaultMessage: 'Index patterns', - } -); +import { + DEFAULT_DATA_SOURCE_DISPLAY_NAME, + DEFAULT_DATA_SOURCE_NAME, + DEFAULT_DATA_SOURCE_TYPE, +} from './constants'; /** * Registers the default data source with the provided data excluding 'ui'. From 2e688e2fdd93f69930f1b0c61598c6db5b0fff1c Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 30 Apr 2024 00:53:41 +0000 Subject: [PATCH 17/23] add test subject Signed-off-by: Eric --- .../data_sources/datasource_selector/data_selector_refresher.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.tsx b/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.tsx index 08ec73fd2ab1..4aabecd3bd6d 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/data_selector_refresher.tsx @@ -30,6 +30,7 @@ export const DataSelectorRefresher: React.FC = Reac defaultMessage: tooltipText, })} display="block" + data-test-subj="sourceRefreshButtonToolTip" {...toolTipProps} > Date: Tue, 30 Apr 2024 00:59:19 +0000 Subject: [PATCH 18/23] reference from correct type path Signed-off-by: Eric --- src/plugins/data/public/plugin.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index 9aacc39d1992..4917eb9db9e2 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -90,11 +90,9 @@ import { SavedObjectsClientPublicToCommon } from './index_patterns'; import { indexPatternLoad } from './index_patterns/expressions/load_index_pattern'; import { DataSourceService } from './data_sources/datasource_services'; import { DataSourceFactory } from './data_sources/datasource'; -import { - DEFAULT_DATA_SOURCE_TYPE, - registerDefaultDataSource, -} from './data_sources/register_default_datasource'; +import { registerDefaultDataSource } from './data_sources/register_default_datasource'; import { DefaultDslDataSource } from './data_sources/default_datasource'; +import { DEFAULT_DATA_SOURCE_TYPE } from './data_sources/constants'; declare module '../../ui_actions/public' { export interface ActionContextMapping { From ce3d4050823b017d8c96ecef8e50c6cbb9e2db9a Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 30 Apr 2024 01:11:30 +0000 Subject: [PATCH 19/23] correct text Signed-off-by: Eric --- src/plugins/data/public/data_sources/constants.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data/public/data_sources/constants.tsx b/src/plugins/data/public/data_sources/constants.tsx index 325d93954af3..eb0a0cfb0ac8 100644 --- a/src/plugins/data/public/data_sources/constants.tsx +++ b/src/plugins/data/public/data_sources/constants.tsx @@ -41,6 +41,6 @@ export const s3DataSourceMetadata = { }, }; -export const DATA_SELECTOR_REFRESHER_POPOVER_TEXT = 'Refresh data sources'; +export const DATA_SELECTOR_REFRESHER_POPOVER_TEXT = 'Refresh data selector'; export const DATA_SELECTOR_DEFAULT_PLACEHOLDER = 'Select a data source'; export const DATA_SELECTOR_S3_DATA_SOURCE_GROUP_HINT_LABEL = ' - Opens in Log Explorer'; From 6514483c33e4dcb45bab3028e93fc6e0b0177c3b Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 30 Apr 2024 06:03:08 +0000 Subject: [PATCH 20/23] minor change - remove yet used displayOrder Signed-off-by: Eric --- src/plugins/data/public/data_sources/datasource/types.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index 747c872ff513..74a670573898 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -86,12 +86,6 @@ export interface IDataSourceUISettings { */ typeLabel: string; - /** - * The order in which the data source should be displayed in the selector. - * @optional - */ - displayOrder?: number; - /** * A short description of the data source. * @optional From 8e7b8e449a2720b765806e20a8f09a72a0162693 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 30 Apr 2024 15:09:31 +0000 Subject: [PATCH 21/23] remove from changelog as having fragments already Signed-off-by: Eric --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ce45f96bf12..56f04ea211fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,7 +108,6 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - [Workspace] Add workspaces filter to saved objects page. ([#6458](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6458)) - [Multiple Datasource] Support multi data source in Region map ([#6654](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6654)) - Add `rightNavigationButton` component in chrome service for applications to register and add dev tool to top right navigation. ([#6553](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6553)) -- [Discover Data Selector] - Data selector enhancement and refactoring ([#6571](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6571)) ### 🐛 Bug Fixes From 909bb92b583711f47e8bbcd8f15a5be5c7090ab7 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 30 Apr 2024 15:15:31 +0000 Subject: [PATCH 22/23] use expanded name Signed-off-by: Eric --- .../datasource_services/datasource_service.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index b5b1f912ee8b..9fa335e0754e 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -57,13 +57,13 @@ export class DataSourceService { * @throws {DataSourceRegistrationError} Throws an error if a data source with the same name already exists. */ async registerDataSource(ds: DataSource): Promise { - const dsId = ds.getId(); - if (dsId in this.dataSources) { + const dataSourceId = ds.getId(); + if (dataSourceId in this.dataSources) { throw new DataSourceRegistrationError( - `Unable to register data source ${dsId}, error: data source name exists.` + `Unable to register data source ${ds.getName()}, error: data source exists.` ); } else { - this.dataSources[dsId] = ds; + this.dataSources[dataSourceId] = ds; this.dataSourcesSubject.next(this.dataSources); return { success: true, info: '' } as IDataSourceRegistrationResult; } From c8753d9a14c0f4f9afdfdf32c5a92666015b5577 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 30 Apr 2024 16:24:49 +0000 Subject: [PATCH 23/23] fix one test Signed-off-by: Eric --- .../data_sources/datasource_services/datasource_service.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts index 113dcadca9f1..e1d2077f60e5 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts @@ -112,7 +112,7 @@ describe('DataSourceService', () => { const ds = new MockDataSource(mockConfig1); await service.registerDataSource(ds); await expect(service.registerDataSource(ds)).rejects.toThrow( - 'Unable to register data source test_datasource1, error: data source name exists.' + 'Unable to register data source test_datasource1, error: data source exists.' ); });