From 95715cfec5d804a399aee69af61312895682b91a Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 21 Apr 2024 23:54:01 +0000 Subject: [PATCH] datasource enhancement and refactoring Signed-off-by: Eric --- .../data_sources/datasource/datasource.ts | 6 +- .../public/data_sources/datasource/index.ts | 4 +- .../public/data_sources/datasource/types.ts | 10 +- .../datasource_selectable.tsx | 143 +++++++++++++----- .../data_sources/datasource_selector/types.ts | 4 + .../datasource_services/datasource_service.ts | 29 ++++ .../data_sources/datasource_services/types.ts | 5 + .../default_datasource/default_datasource.ts | 4 +- .../register_default_datasource.ts | 3 +- src/plugins/data/public/index.ts | 2 +- .../public/components/sidebar/index.tsx | 2 + 11 files changed, 164 insertions(+), 48 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/datasource.ts b/src/plugins/data/public/data_sources/datasource/datasource.ts index b37d03709f36..bcff4d0d6cc7 100644 --- a/src/plugins/data/public/data_sources/datasource/datasource.ts +++ b/src/plugins/data/public/data_sources/datasource/datasource.ts @@ -14,7 +14,7 @@ */ import { - ConnectionStatus, + DataSourceConnectionStatus, IDataSetParams, IDataSourceDataSet, IDataSourceMetadata, @@ -90,8 +90,8 @@ export abstract class DataSource< * the connection status, typically indicating success or failure. * * @experimental This API is experimental and might change in future releases. - * @returns {Promise} Status of the connection test. + * @returns {Promise} Status of the connection test. * @experimental */ - abstract testConnection(): Promise; + abstract testConnection(): Promise; } diff --git a/src/plugins/data/public/data_sources/datasource/index.ts b/src/plugins/data/public/data_sources/datasource/index.ts index a3bac5bbc6a4..a93d57a8d80f 100644 --- a/src/plugins/data/public/data_sources/datasource/index.ts +++ b/src/plugins/data/public/data_sources/datasource/index.ts @@ -6,11 +6,11 @@ export { DataSource } from './datasource'; export { IDataSourceMetadata, - SourceDataSet, + DataSourceDataSet, IDataSetParams, IDataSourceQueryParams, IDataSourceQueryResult, - ConnectionStatus, + DataSourceConnectionStatus, IndexPatternOption, } from './types'; export { DataSourceFactory } from './factory'; diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index 7b17d0b7135d..1ca185c98ec8 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -18,9 +18,9 @@ export interface IDataSourceGroup { name: string; } -export interface SourceDataSet { +export interface DataSourceDataSet { ds: DataSource; - data_sets: IndexPatternOption[]; + list: T; } export interface IDataSetParams { @@ -35,7 +35,7 @@ export interface IDataSourceQueryResult { data: T; } -export interface ConnectionStatus { +export interface DataSourceConnectionStatus { status: string; message: string; error?: Error; @@ -54,13 +54,15 @@ export interface IDataSourceMetadata { export interface IDataSourceUISettings { label: string; // the display name of data source + typeGroup: string; // the group to which the data source belongs typeLabel: string; // the display name of data source type + displayOrder?: number; // the order in which the data source should be displayed in selector description?: string; // short description of your database icon?: string; // uri of the icon } export interface IDataSourceDataSet { - data_sets: T; + dataSets: T; } export interface IDataSourceQueryResponse { diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index b88fc2fd661e..37f6cf197dc3 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -6,48 +6,35 @@ import React, { useEffect, useCallback, useMemo } from 'react'; import { EuiComboBox } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import { DataSource, SourceDataSet, IndexPatternOption } from '../datasource'; +import { DataSource, DataSourceDataSet, IndexPatternOption } from '../datasource'; import { DataSourceGroup, DataSourceSelectableProps } from './types'; +import { IDataSourceDataSet } from '../datasource/types'; type DataSourceTypeKey = 'DEFAULT_INDEX_PATTERNS' | 's3glue' | 'spark'; -// Mapping between datasource type and its display name. -// Temporary solution, will be removed along with refactoring of data source APIs -const DATASOURCE_TYPE_DISPLAY_NAME_MAP: Record = { - DEFAULT_INDEX_PATTERNS: i18n.translate('dataExplorer.dataSourceSelector.indexPatternGroupTitle', { - defaultMessage: 'Index patterns', - }), - s3glue: i18n.translate('dataExplorer.dataSourceSelector.amazonS3GroupTitle', { - defaultMessage: 'Amazon S3', - }), - spark: i18n.translate('dataExplorer.dataSourceSelector.sparkGroupTitle', { - defaultMessage: 'Spark', - }), -}; - -type DataSetType = SourceDataSet['data_sets'][number]; - // Get Index patterns for local cluster. -const getDataSetWithSource = async (ds: DataSource): Promise => { - const dataSet = (await ds.getDataSet()).data_sets; +const getDataSetFromDataSource = async ( + ds: DataSource +): Promise> => { + const dataSets = (await ds.getDataSet()).dataSets; return { ds, - data_sets: dataSet, - } as SourceDataSet; + list: dataSets, + } as DataSourceDataSet; }; // Map through all data sources and get their respective data sets. const getDataSets = (dataSources: DataSource[]) => - dataSources.map((ds) => getDataSetWithSource(ds)); + dataSources.map((ds) => getDataSetFromDataSource(ds)); -export const isIndexPatterns = (dataSet: DataSetType): dataSet is IndexPatternOption => { +export const isIndexPatterns = (dataSet: IndexPatternOption) => { if (typeof dataSet === 'string') return false; return !!(dataSet.title && dataSet.id); }; // Get the option format for the combo box from the dataSource and dataSet. -export const getSourceOptions = (dataSource: DataSource, dataSet: DataSetType) => { +export const getSourceOptions = (dataSource: DataSource, dataSet: IDataSourceDataSet) => { const optionContent = { type: dataSource.getType(), name: dataSource.getName(), @@ -68,13 +55,17 @@ export const getSourceOptions = (dataSource: DataSource, dataSet: DataSetType) = }; }; -// Convert data sets into a structured format suitable for selector rendering. -const getSourceList = (allDataSets: SourceDataSet[]) => { +const getGroupListFromDataSetDS = ( + dsListWithDataSetAsDisplayedDataSources: DataSourceDataSet[], + groupList: DataSourceGroup[] +) => { const finalList = [] as DataSourceGroup[]; - allDataSets.forEach((curDataSet) => { - const typeKey = curDataSet.ds.getType() as DataSourceTypeKey; + dsListWithDataSetAsDisplayedDataSources.forEach((curDataSet) => { + const typeKey = curDataSet.ds.getType() as DataSourceTypeKey; // ds type key + const dsMetadata = curDataSet.ds.getMetadata(); + const groupType = dsMetadata.ui.typeGroup; let groupName = - DATASOURCE_TYPE_DISPLAY_NAME_MAP[typeKey] || + dsMetadata.ui.typeLabel || i18n.translate('dataExplorer.dataSourceSelector.defaultGroupTitle', { defaultMessage: 'Default Group', }); @@ -87,15 +78,15 @@ const getSourceList = (allDataSets: SourceDataSet[]) => { })}`; } - const existingGroup = finalList.find((item) => item.label === groupName); - const mappedOptions = curDataSet.data_sets.map((dataSet) => + const existingGroup = finalList.find((item) => item.typeGroup === groupType); + const mappedOptions = curDataSet.list.map((dataSet) => getSourceOptions(curDataSet.ds, dataSet) ); - // check if add new datasource group or add to existing one + // check if to add new data source group or add to existing one if (existingGroup) { // options deduplication - const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.label)); + const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.id)); const nonDuplicateOptions = mappedOptions.filter((opt) => !existingOptionIds.has(opt.label)); // 'existingGroup' directly references an item in the finalList @@ -103,14 +94,92 @@ const getSourceList = (allDataSets: SourceDataSet[]) => { existingGroup.options.push(...nonDuplicateOptions); } else { finalList.push({ + typeGroup: dsMetadata.ui.typeGroup, label: groupName, options: mappedOptions, }); } }); + + return [...finalList, ...groupList]; +}; + +const getGroupListFromDS = (dataSources: DataSource[], groupList: DataSourceGroup[]) => { + const finalList = [] as DataSourceGroup[]; + dataSources.forEach((ds) => { + const typeKey = ds.getType() as DataSourceTypeKey; // ds type key + const dsMetadata = ds.getMetadata(); + const typeGroup = dsMetadata.ui.typeGroup; + let groupName = + dsMetadata.ui.typeLabel || + i18n.translate('dataExplorer.dataSourceSelector.defaultGroupTitle', { + defaultMessage: 'Default Group', + }); + + // add '- Opens in Log Explorer' to hint user that selecting these types of data sources + // will lead to redirection to log explorer + if (typeKey !== 'DEFAULT_INDEX_PATTERNS') { + groupName = `${groupName}${i18n.translate('dataExplorer.dataSourceSelector.redirectionHint', { + defaultMessage: ' - Opens in Log Explorer', + })}`; + } + + const existingGroup = finalList.find((item) => item.typeGroup === typeGroup); + const dsOption = { + type: ds.getType(), + name: ds.getName(), + ds, + label: dsMetadata.ui.label, + value: dsMetadata.ui.label, + key: ds.getId(), + }; + // check if to add new data source group or add to existing one + if (existingGroup) { + // options deduplication + // const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.id)); + // const nonDuplicateOptions = mappedOptions.filter((opt) => !existingOptionIds.has(opt.label)); + + // 'existingGroup' directly references an item in the finalList + // pushing options to 'existingGroup' updates the corresponding item in finalList + existingGroup.options.push(dsOption); + } else { + finalList.push({ + id: dsMetadata.ui.typeGroup, + typeGroup: dsMetadata.ui.typeGroup, + label: groupName, + options: [ + { + ...dsOption, + }, + ], + }); + } + }); + + return [...groupList, ...finalList]; +}; + +// Convert data sets into a structured format suitable for selector rendering. +const getConsolidatedDataSourceGroups = ( + dsListWithDataSetAsDisplayedDataSources: DataSourceDataSet[], + dsListWithThemselvesAsDisplayedDataSources: DataSource[] +) => { + // const finalList = [] as DataSourceGroup[]; + const dataSetFinalList = getGroupListFromDataSetDS(dsListWithDataSetAsDisplayedDataSources, []); + const finalList = getGroupListFromDS( + dsListWithThemselvesAsDisplayedDataSources, + dataSetFinalList + ); + return finalList; }; +const getDataSourcesRequireDataSetFetching = (dataSources: DataSource[]): DataSource[] => + dataSources.filter((ds) => ds.getMetadata()?.ui?.selector?.displayDatasetsWithSource); + +const getDataSourcesRequireNoDataSetFetching = (dataSources: DataSource[]): DataSource[] => + dataSources.filter((ds) => !ds.getMetadata()?.ui?.selector?.displayDatasetsWithSource); + /** * @experimental This component is experimental and might change in future releases. */ @@ -126,9 +195,13 @@ export const DataSourceSelectable = ({ }: DataSourceSelectableProps) => { // This effect gets data sets and prepares the datasource list for UI rendering. useEffect(() => { - Promise.all(getDataSets(dataSources)) + Promise.all(getDataSets(getDataSourcesRequireDataSetFetching(dataSources))) .then((results) => { - setDataSourceOptionList(getSourceList(results)); + const groupList = getConsolidatedDataSourceGroups( + results, + getDataSourcesRequireNoDataSetFetching(dataSources) + ); + setDataSourceOptionList(groupList); }) .catch((e) => onGetDataSetError(e)); }, [dataSources, setDataSourceOptionList, onGetDataSetError]); diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index c7d54ae3a6b2..23804569b7ae 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -12,10 +12,14 @@ import { DataSource } from '../datasource/datasource'; export interface DataSourceGroup { label: string; + id: string; options: DataSourceOption[]; + typeGroup: string; } export interface DataSourceOption { + id: string; + name: string; label: string; value: string; type: string; diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index 80ae619a679f..8ead65633c7f 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -8,6 +8,7 @@ import { DataSourceRegistrationError, IDataSourceFilter, IDataSourceRegistrationResult, + DataSourceFetcher, } from './types'; import { DataSource } from '../datasource/datasource'; @@ -16,6 +17,7 @@ export class DataSourceService { // A record to store all registered data sources, using the data source name as the key. private dataSources: Record = {}; private dataSourcesSubject: BehaviorSubject>; + private dataSourceFetchers: Record = {}; private constructor() { this.dataSourcesSubject = new BehaviorSubject(this.dataSources); @@ -85,4 +87,31 @@ export class DataSourceService { return filteredDataSources; }, {} as Record); } + + /** + * Registers functions responsible for fetching data for each data source type. + * + * @param fetchers - An array of fetcher configurations, each specifying how to fetch data for a specific data source type. + */ + registerDataSourceFetchers(fetchers: DataSourceFetcher[]) { + fetchers.forEach((fetcher) => { + this.dataSourceFetchers[fetcher.type] = fetcher.registerDataSources; + }); + } + + /** + * Calls all registered data fetching functions to update data sources. + * Typically used to initialize or refresh the data source configurations. + */ + load() { + Object.values(this.dataSourceFetchers).forEach((fetch) => fetch()); + } + + /** + * Reloads all data source configurations by re-invoking the load method. + * Useful for refreshing the system to reflect changes such as new data source registrations. + */ + reload() { + this.load(); + } } diff --git a/src/plugins/data/public/data_sources/datasource_services/types.ts b/src/plugins/data/public/data_sources/datasource_services/types.ts index 9fdf99b70421..1cbdbeadea8c 100644 --- a/src/plugins/data/public/data_sources/datasource_services/types.ts +++ b/src/plugins/data/public/data_sources/datasource_services/types.ts @@ -34,3 +34,8 @@ export interface DataSourceStart { dataSourceService: DataSourceService; dataSourceFactory: DataSourceFactory; } + +export interface DataSourceFetcher { + type: string; + registerDataSources: () => void; +} diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts index 1f5ee05ac1ae..262a8ec9d93d 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts @@ -41,7 +41,7 @@ export class DefaultDslDataSource extends DataSource< this.indexPatterns = indexPatterns; } - async getDataSet(): Promise { + async getDataSet() { await this.indexPatterns.ensureDefaultIndexPattern(); const savedObjectLst = await this.indexPatterns.getCache(); @@ -50,7 +50,7 @@ export class DefaultDslDataSource extends DataSource< } return { - data_sets: savedObjectLst.map((savedObject: SavedObject) => { + dataSets: savedObjectLst.map((savedObject: SavedObject) => { return { id: savedObject.id, title: savedObject.attributes.title, diff --git a/src/plugins/data/public/data_sources/register_default_datasource.ts b/src/plugins/data/public/data_sources/register_default_datasource.ts index 65f5028489c6..7fd6e8b147cf 100644 --- a/src/plugins/data/public/data_sources/register_default_datasource.ts +++ b/src/plugins/data/public/data_sources/register_default_datasource.ts @@ -25,7 +25,8 @@ export const registerDefaultDatasource = (data: Omit { } ); + dataSources.dataSourceService.load(); + return () => { subscription.unsubscribe(); isMounted = false;