datasource enhancement and refactoring

Signed-off-by: Eric <[email protected]>

src/plugins/data/public/data_sources/datasource/datasource.ts

@@ -14,7 +14,7 @@
  */
 
 import {
-  ConnectionStatus,
+  DataSourceConnectionStatus,
   IDataSetParams,
   IDataSourceDataSet,
   IDataSourceMetadata,
@@ -90,8 +90,8 @@ export abstract class DataSource<
    * the connection status, typically indicating success or failure.
    *
    * @experimental This API is experimental and might change in future releases.
-   * @returns {Promise<ConnectionStatus | boolean>} Status of the connection test.
+   * @returns {Promise<DataSourceConnectionStatus | boolean>} Status of the connection test.
    * @experimental
    */
-  abstract testConnection(): Promise<ConnectionStatus | boolean>;
+  abstract testConnection(): Promise<DataSourceConnectionStatus | boolean>;
 }

src/plugins/data/public/data_sources/datasource/index.ts

@@ -6,11 +6,11 @@
 export { DataSource } from './datasource';
 export {
   IDataSourceMetadata,
-  SourceDataSet,
+  DataSourceDataSet,
   IDataSetParams,
   IDataSourceQueryParams,
   IDataSourceQueryResult,
-  ConnectionStatus,
+  DataSourceConnectionStatus,
   IndexPatternOption,
 } from './types';
 export { DataSourceFactory } from './factory';

src/plugins/data/public/data_sources/datasource/types.ts

@@ -18,9 +18,9 @@ export interface IDataSourceGroup {
   name: string;
 }
 
-export interface SourceDataSet {
+export interface DataSourceDataSet<T = {}> {
   ds: DataSource;
-  data_sets: IndexPatternOption[];
+  list: T;
 }
 
 export interface IDataSetParams<T = {}> {
@@ -35,7 +35,7 @@ export interface IDataSourceQueryResult<T = {}> {
   data: T;
 }
 
-export interface ConnectionStatus {
+export interface DataSourceConnectionStatus {
   status: string;
   message: string;
   error?: Error;
@@ -54,13 +54,15 @@ export interface IDataSourceMetadata {
 
 export interface IDataSourceUISettings {
   label: string; // the display name of data source
+  typeGroup: string; // the group to which the data source belongs
   typeLabel: string; // the display name of data source type
+  displayOrder?: number; // the order in which the data source should be displayed in selector
   description?: string; // short description of your database
   icon?: string; // uri of the icon
 }
 
 export interface IDataSourceDataSet<T = {}> {
-  data_sets: T;
+  dataSets: T;
 }
 
 export interface IDataSourceQueryResponse<T = {}> {

src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx

@@ -6,48 +6,35 @@
 import React, { useEffect, useCallback, useMemo } from 'react';
 import { EuiComboBox } from '@elastic/eui';
 import { i18n } from '@osd/i18n';
-import { DataSource, SourceDataSet, IndexPatternOption } from '../datasource';
+import { DataSource, DataSourceDataSet, IndexPatternOption } from '../datasource';
 import { DataSourceGroup, DataSourceSelectableProps } from './types';
+import { IDataSourceDataSet } from '../datasource/types';
 
 type DataSourceTypeKey = 'DEFAULT_INDEX_PATTERNS' | 's3glue' | 'spark';
 
-// Mapping between datasource type and its display name.
-// Temporary solution, will be removed along with refactoring of data source APIs
-const DATASOURCE_TYPE_DISPLAY_NAME_MAP: Record<DataSourceTypeKey, string> = {
-  DEFAULT_INDEX_PATTERNS: i18n.translate('dataExplorer.dataSourceSelector.indexPatternGroupTitle', {
-    defaultMessage: 'Index patterns',
-  }),
-  s3glue: i18n.translate('dataExplorer.dataSourceSelector.amazonS3GroupTitle', {
-    defaultMessage: 'Amazon S3',
-  }),
-  spark: i18n.translate('dataExplorer.dataSourceSelector.sparkGroupTitle', {
-    defaultMessage: 'Spark',
-  }),
-};
-
-type DataSetType = SourceDataSet['data_sets'][number];
-
 // Get Index patterns for local cluster.
-const getDataSetWithSource = async (ds: DataSource): Promise<SourceDataSet> => {
-  const dataSet = (await ds.getDataSet()).data_sets;
+const getDataSetFromDataSource = async (
+  ds: DataSource
+): Promise<DataSourceDataSet<IndexPatternOption[]>> => {
+  const dataSets = (await ds.getDataSet()).dataSets;
   return {
     ds,
-    data_sets: dataSet,
-  } as SourceDataSet;
+    list: dataSets,
+  } as DataSourceDataSet<IndexPatternOption[]>;
 };
 
 // Map through all data sources and get their respective data sets.
 const getDataSets = (dataSources: DataSource[]) =>
-  dataSources.map((ds) => getDataSetWithSource(ds));
+  dataSources.map((ds) => getDataSetFromDataSource(ds));
 
-export const isIndexPatterns = (dataSet: DataSetType): dataSet is IndexPatternOption => {
+export const isIndexPatterns = (dataSet: IndexPatternOption) => {
   if (typeof dataSet === 'string') return false;
 
   return !!(dataSet.title && dataSet.id);
 };
 
 // Get the option format for the combo box from the dataSource and dataSet.
-export const getSourceOptions = (dataSource: DataSource, dataSet: DataSetType) => {
+export const getSourceOptions = (dataSource: DataSource, dataSet: IDataSourceDataSet) => {
   const optionContent = {
     type: dataSource.getType(),
     name: dataSource.getName(),
@@ -68,13 +55,17 @@ export const getSourceOptions = (dataSource: DataSource, dataSet: DataSetType) =
   };
 };
 
-// Convert data sets into a structured format suitable for selector rendering.
-const getSourceList = (allDataSets: SourceDataSet[]) => {
+const getGroupListFromDataSetDS = (
+  dsListWithDataSetAsDisplayedDataSources: DataSourceDataSet[],
+  groupList: DataSourceGroup[]
+) => {
   const finalList = [] as DataSourceGroup[];
-  allDataSets.forEach((curDataSet) => {
-    const typeKey = curDataSet.ds.getType() as DataSourceTypeKey;
+  dsListWithDataSetAsDisplayedDataSources.forEach((curDataSet) => {
+    const typeKey = curDataSet.ds.getType() as DataSourceTypeKey; // ds type key
+    const dsMetadata = curDataSet.ds.getMetadata();
+    const groupType = dsMetadata.ui.typeGroup;
     let groupName =
-      DATASOURCE_TYPE_DISPLAY_NAME_MAP[typeKey] ||
+      dsMetadata.ui.typeLabel ||
       i18n.translate('dataExplorer.dataSourceSelector.defaultGroupTitle', {
         defaultMessage: 'Default Group',
       });
@@ -87,30 +78,108 @@ const getSourceList = (allDataSets: SourceDataSet[]) => {
       })}`;
     }
 
-    const existingGroup = finalList.find((item) => item.label === groupName);
-    const mappedOptions = curDataSet.data_sets.map((dataSet) =>
+    const existingGroup = finalList.find((item) => item.typeGroup === groupType);
+    const mappedOptions = curDataSet.list.map((dataSet) =>
       getSourceOptions(curDataSet.ds, dataSet)
     );
 
-    // check if add new datasource group or add to existing one
+    // check if to add new data source group or add to existing one
     if (existingGroup) {
       // options deduplication
-      const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.label));
+      const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.id));
       const nonDuplicateOptions = mappedOptions.filter((opt) => !existingOptionIds.has(opt.label));
 
       // 'existingGroup' directly references an item in the finalList
       // pushing options to 'existingGroup' updates the corresponding item in finalList
       existingGroup.options.push(...nonDuplicateOptions);
     } else {
       finalList.push({
+        typeGroup: dsMetadata.ui.typeGroup,
         label: groupName,
         options: mappedOptions,
       });
     }
   });
+
+  return [...finalList, ...groupList];
+};
+
+const getGroupListFromDS = (dataSources: DataSource[], groupList: DataSourceGroup[]) => {
+  const finalList = [] as DataSourceGroup[];
+  dataSources.forEach((ds) => {
+    const typeKey = ds.getType() as DataSourceTypeKey; // ds type key
+    const dsMetadata = ds.getMetadata();
+    const typeGroup = dsMetadata.ui.typeGroup;
+    let groupName =
+      dsMetadata.ui.typeLabel ||
+      i18n.translate('dataExplorer.dataSourceSelector.defaultGroupTitle', {
+        defaultMessage: 'Default Group',
+      });
+
+    // add '- Opens in Log Explorer' to hint user that selecting these types of data sources
+    // will lead to redirection to log explorer
+    if (typeKey !== 'DEFAULT_INDEX_PATTERNS') {
+      groupName = `${groupName}${i18n.translate('dataExplorer.dataSourceSelector.redirectionHint', {
+        defaultMessage: ' - Opens in Log Explorer',
+      })}`;
+    }
+
+    const existingGroup = finalList.find((item) => item.typeGroup === typeGroup);
+    const dsOption = {
+      type: ds.getType(),
+      name: ds.getName(),
+      ds,
+      label: dsMetadata.ui.label,
+      value: dsMetadata.ui.label,
+      key: ds.getId(),
+    };
+    // check if to add new data source group or add to existing one
+    if (existingGroup) {
+      // options deduplication
+      // const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.id));
+      // const nonDuplicateOptions = mappedOptions.filter((opt) => !existingOptionIds.has(opt.label));
+
+      // 'existingGroup' directly references an item in the finalList
+      // pushing options to 'existingGroup' updates the corresponding item in finalList
+      existingGroup.options.push(dsOption);
+    } else {
+      finalList.push({
+        id: dsMetadata.ui.typeGroup,
+        typeGroup: dsMetadata.ui.typeGroup,
+        label: groupName,
+        options: [
+          {
+            ...dsOption,
+          },
+        ],
+      });
+    }
+  });
+
+  return [...groupList, ...finalList];
+};
+
+// Convert data sets into a structured format suitable for selector rendering.
+const getConsolidatedDataSourceGroups = (
+  dsListWithDataSetAsDisplayedDataSources: DataSourceDataSet[],
+  dsListWithThemselvesAsDisplayedDataSources: DataSource[]
+) => {
+  // const finalList = [] as DataSourceGroup[];
+  const dataSetFinalList = getGroupListFromDataSetDS(dsListWithDataSetAsDisplayedDataSources, []);
+  const finalList = getGroupListFromDS(
+    dsListWithThemselvesAsDisplayedDataSources,
+    dataSetFinalList
+  );
+
   return finalList;
 };
 
+const getDataSourcesRequireDataSetFetching = (dataSources: DataSource[]): DataSource[] =>
+  dataSources.filter((ds) => ds.getMetadata()?.ui?.selector?.displayDatasetsWithSource);
+
+const getDataSourcesRequireNoDataSetFetching = (dataSources: DataSource[]): DataSource[] =>
+  dataSources.filter((ds) => !ds.getMetadata()?.ui?.selector?.displayDatasetsWithSource);
+
 /**
  * @experimental This component is experimental and might change in future releases.
  */
@@ -126,9 +195,13 @@ export const DataSourceSelectable = ({
 }: DataSourceSelectableProps) => {
   // This effect gets data sets and prepares the datasource list for UI rendering.
   useEffect(() => {
-    Promise.all(getDataSets(dataSources))
+    Promise.all(getDataSets(getDataSourcesRequireDataSetFetching(dataSources)))
       .then((results) => {
-        setDataSourceOptionList(getSourceList(results));
+        const groupList = getConsolidatedDataSourceGroups(
+          results,
+          getDataSourcesRequireNoDataSetFetching(dataSources)
+        );
+        setDataSourceOptionList(groupList);
       })
       .catch((e) => onGetDataSetError(e));
   }, [dataSources, setDataSourceOptionList, onGetDataSetError]);

src/plugins/data/public/data_sources/datasource_selector/types.ts

@@ -12,10 +12,14 @@ import { DataSource } from '../datasource/datasource';
 
 export interface DataSourceGroup {
   label: string;
+  id: string;
   options: DataSourceOption[];
+  typeGroup: string;
 }
 
 export interface DataSourceOption {
+  id: string;
+  name: string;
   label: string;
   value: string;
   type: string;

src/plugins/data/public/data_sources/datasource_services/datasource_service.ts

@@ -8,6 +8,7 @@ import {
   DataSourceRegistrationError,
   IDataSourceFilter,
   IDataSourceRegistrationResult,
+  DataSourceFetcher,
 } from './types';
 import { DataSource } from '../datasource/datasource';
 
@@ -16,6 +17,7 @@ export class DataSourceService {
   // A record to store all registered data sources, using the data source name as the key.
   private dataSources: Record<string, DataSource> = {};
   private dataSourcesSubject: BehaviorSubject<Record<string, DataSource>>;
+  private dataSourceFetchers: Record<string, DataSourceFetcher['registerDataSources']> = {};
 
   private constructor() {
     this.dataSourcesSubject = new BehaviorSubject(this.dataSources);
@@ -85,4 +87,31 @@ export class DataSourceService {
       return filteredDataSources;
     }, {} as Record<string, DataSource>);
   }
+
+  /**
+   * Registers functions responsible for fetching data for each data source type.
+   *
+   * @param fetchers - An array of fetcher configurations, each specifying how to fetch data for a specific data source type.
+   */
+  registerDataSourceFetchers(fetchers: DataSourceFetcher[]) {
+    fetchers.forEach((fetcher) => {
+      this.dataSourceFetchers[fetcher.type] = fetcher.registerDataSources;
+    });
+  }
+
+  /**
+   * Calls all registered data fetching functions to update data sources.
+   * Typically used to initialize or refresh the data source configurations.
+   */
+  load() {
+    Object.values(this.dataSourceFetchers).forEach((fetch) => fetch());
+  }
+
+  /**
+   * Reloads all data source configurations by re-invoking the load method.
+   * Useful for refreshing the system to reflect changes such as new data source registrations.
+   */
+  reload() {
+    this.load();
+  }
 }

src/plugins/data/public/data_sources/datasource_services/types.ts

@@ -34,3 +34,8 @@ export interface DataSourceStart {
   dataSourceService: DataSourceService;
   dataSourceFactory: DataSourceFactory;
 }
+
+export interface DataSourceFetcher {
+  type: string;
+  registerDataSources: () => void;
+}

src/plugins/data/public/data_sources/default_datasource/default_datasource.ts

@@ -41,7 +41,7 @@ export class DefaultDslDataSource extends DataSource<
     this.indexPatterns = indexPatterns;
   }
 
-  async getDataSet(): Promise<LocalDSDataSetResponse> {
+  async getDataSet() {
     await this.indexPatterns.ensureDefaultIndexPattern();
     const savedObjectLst = await this.indexPatterns.getCache();
 
@@ -50,7 +50,7 @@ export class DefaultDslDataSource extends DataSource<
     }
 
     return {
-      data_sets: savedObjectLst.map((savedObject: SavedObject<IndexPatternSavedObjectAttrs>) => {
+      dataSets: savedObjectLst.map((savedObject: SavedObject<IndexPatternSavedObjectAttrs>) => {
         return {
           id: savedObject.id,
           title: savedObject.attributes.title,

src/plugins/data/public/data_sources/register_default_datasource.ts

@@ -25,7 +25,8 @@ export const registerDefaultDatasource = (data: Omit<DataPublicPluginStart, 'ui'
       metadata: {
         ui: {
           label: 'Index patterns', // display name of your data source,
-          typeLabel: 'OpenSearch default', // display name of your data source type,
+          typeLabel: 'Index patterns', // display name of your data source type,
+          typeGroup: 'DEFAULT_INDEX_PATTERNS',
           selector: {
             displayDatasetsWithSource: true, // when true, selector UI will render data sets with source by calling getDataSets()
           },

src/plugins/data/public/index.ts

@@ -505,7 +505,7 @@ export {
   IDataSourceQueryParams,
   IDataSourceQueryResult,
   SourceDataSet,
-  ConnectionStatus,
+  DataSourceConnectionStatus,
   DataSourceFactory,
 } from './data_sources/datasource';
 export {