[Discover] Data selector enhancement (opensearch-project#6571) (opens…

…earch-project#6691)

* datasource and service refactoring



* datasource enhancement and refactoring



* datasource selectable consolidation and refactoring



* add in memory cache with refresh



* move refresh to right side



* renaming



* update default datasource tests



* added more tests for default datasource



* update selector tests



* update changelog



* fix data source service tests



* add and update tests for datasource service



* add more data source service tests



* fix sidebar tests



* add to change log yml



* address comments along with more tests



* add test subject



* reference from correct type path



* correct text



* minor change - remove yet used displayOrder



* remove from changelog as having fragments already



* use expanded name



* fix one test



---------



(cherry picked from commit 42166d0)

Signed-off-by: Eric <[email protected]>
Signed-off-by: Eric Wei <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

changelogs/fragments/6571.yml

@@ -0,0 +1,2 @@
+refactor:
+- discover data selector enhancement and refactoring ([#6571](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6571))

src/plugins/data/public/data_sources/constants.tsx

@@ -0,0 +1,46 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { i18n } from '@osd/i18n';
+import { DataSourceUIGroupType } from './datasource/types';
+
+export const S3_GLUE_DATA_SOURCE_DISPLAY_NAME = 'Amazon S3';
+export const S3_GLUE_DATA_SOURCE_TYPE = 's3glue';
+export const DEFAULT_DATA_SOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS';
+export const DEFAULT_DATA_SOURCE_NAME = i18n.translate('data.datasource.type.openSearchDefault', {
+  defaultMessage: 'OpenSearch Default',
+});
+export const DEFAULT_DATA_SOURCE_DISPLAY_NAME = i18n.translate(
+  'data.datasource.type.openSearchDefaultDisplayName',
+  {
+    defaultMessage: 'Index patterns',
+  }
+);
+
+export const defaultDataSourceMetadata = {
+  ui: {
+    label: DEFAULT_DATA_SOURCE_DISPLAY_NAME,
+    typeLabel: DEFAULT_DATA_SOURCE_DISPLAY_NAME,
+    groupType: DataSourceUIGroupType.defaultOpenSearchDataSource,
+    selector: {
+      displayDatasetsAsSource: true,
+    },
+  },
+};
+
+export const s3DataSourceMetadata = {
+  ui: {
+    label: S3_GLUE_DATA_SOURCE_DISPLAY_NAME,
+    typeLabel: S3_GLUE_DATA_SOURCE_TYPE,
+    groupType: DataSourceUIGroupType.s3glue,
+    selector: {
+      displayDatasetsAsSource: false,
+    },
+  },
+};
+
+export const DATA_SELECTOR_REFRESHER_POPOVER_TEXT = 'Refresh data selector';
+export const DATA_SELECTOR_DEFAULT_PLACEHOLDER = 'Select a data source';
+export const DATA_SELECTOR_S3_DATA_SOURCE_GROUP_HINT_LABEL = ' - Opens in Log Explorer';

src/plugins/data/public/data_sources/datasource/datasource.ts

@@ -13,23 +13,41 @@
  * DataSourceQueryResult: Represents the result from querying the data source.
  */
 
-import { ConnectionStatus } from './types';
+import {
+  DataSourceConnectionStatus,
+  IDataSetParams,
+  IDataSourceDataSet,
+  IDataSourceMetadata,
+  IDataSourceQueryParams,
+  IDataSourceQueryResponse,
+  IDataSourceSettings,
+} from './types';
 
 /**
  * @experimental this class is experimental and might change in future releases.
  */
 export abstract class DataSource<
-  DataSourceMetaData,
-  DataSetParams,
-  SourceDataSet,
-  DataSourceQueryParams,
-  DataSourceQueryResult
+  TMetadata extends IDataSourceMetadata = IDataSourceMetadata,
+  TDataSetParams extends IDataSetParams = IDataSetParams,
+  TDataSet extends IDataSourceDataSet = IDataSourceDataSet,
+  TQueryParams extends IDataSourceQueryParams = IDataSourceQueryParams,
+  TQueryResult extends IDataSourceQueryResponse = IDataSourceQueryResponse
 > {
-  constructor(
-    private readonly name: string,
-    private readonly type: string,
-    private readonly metadata: DataSourceMetaData
-  ) {}
+  private readonly id: string;
+  private readonly name: string;
+  private readonly type: string;
+  private readonly metadata: TMetadata;
+
+  constructor(settings: IDataSourceSettings<TMetadata>) {
+    this.id = settings.id;
+    this.name = settings.name;
+    this.type = settings.type;
+    this.metadata = settings.metadata;
+  }
+
+  getId() {
+    return this.id;
+  }
 
   getName() {
     return this.name;
@@ -53,27 +71,27 @@ export abstract class DataSource<
    * patterns for OpenSearch data source
    *
    * @experimental This API is experimental and might change in future releases.
-   * @returns {SourceDataSet} Dataset associated with the data source.
+   * @returns {Promise<TDataSet>} Dataset associated with the data source.
    */
-  abstract getDataSet(dataSetParams?: DataSetParams): SourceDataSet;
+  abstract getDataSet(dataSetParams?: TDataSetParams): Promise<TDataSet>;
 
   /**
    * Abstract method to run a query against the data source.
    * Implementing classes need to provide the specific implementation.
    *
    * @experimental This API is experimental and might change in future releases.
-   * @returns {DataSourceQueryResult} Result from querying the data source.
+   * @returns {Promise<TQueryResult>} Result from querying the data source.
    */
-  abstract runQuery(queryParams: DataSourceQueryParams): DataSourceQueryResult;
+  abstract runQuery(queryParams?: TQueryParams): Promise<TQueryResult>;
 
   /**
    * Abstract method to test the connection to the data source.
    * Implementing classes should provide the specific logic to determine
    * the connection status, typically indicating success or failure.
    *
    * @experimental This API is experimental and might change in future releases.
-   * @returns {ConnectionStatus | Promise<void>} Status of the connection test.
+   * @returns {Promise<DataSourceConnectionStatus | boolean>} Status of the connection test.
    * @experimental
    */
-  abstract testConnection(): ConnectionStatus | Promise<boolean>;
+  abstract testConnection(): Promise<DataSourceConnectionStatus | boolean>;
 }

src/plugins/data/public/data_sources/datasource/factory.test.ts

@@ -11,17 +11,19 @@ class MockDataSource extends DataSource<any, any, any, any, any> {
   private readonly indexPatterns;
 
   constructor({
+    id,
     name,
     type,
     metadata,
     indexPatterns,
   }: {
+    id: string;
     name: string;
     type: string;
     metadata: any;
     indexPatterns: IndexPatternsService;
   }) {
-    super(name, type, metadata);
+    super({ id, name, type, metadata });
     this.indexPatterns = indexPatterns;
   }
 

src/plugins/data/public/data_sources/datasource/factory.ts

@@ -8,7 +8,6 @@
  * It serves as a registry for different data source types and provides a way to instantiate them.
  */
 
-import { DataSourceType } from '../datasource_services';
 import { DataSource } from '../datasource';
 
 type DataSourceClass<
@@ -66,11 +65,11 @@ export class DataSourceFactory {
    *
    * @experimental This API is experimental and might change in future releases.
    * @param {string} type - The identifier for the data source type.
-   * @param {any} config - The configuration for the data source instance.
+   * @param {unknown} config - The configuration for the data source instance.
    * @returns {DataSourceType} An instance of the specified data source type.
    * @throws {Error} Throws an error if the data source type is not supported.
    */
-  getDataSourceInstance(type: string, config: any): DataSourceType {
+  getDataSourceInstance(type: string, config: unknown): DataSource {
     const DataSourceClass = this.dataSourceClasses[type];
     if (!DataSourceClass) {
       throw new Error('Unsupported data source type');

src/plugins/data/public/data_sources/datasource/index.ts

@@ -5,13 +5,12 @@
 
 export { DataSource } from './datasource';
 export {
-  IDataSourceMetaData,
-  ISourceDataSet,
+  IDataSourceMetadata,
+  DataSetWithDataSource,
   IDataSetParams,
   IDataSourceQueryParams,
   IDataSourceQueryResult,
-  ConnectionStatus,
-  DataSourceConfig,
+  DataSourceConnectionStatus,
   IndexPatternOption,
 } from './types';
 export { DataSourceFactory } from './factory';

src/plugins/data/public/data_sources/datasource/types.ts

@@ -3,49 +3,112 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { DataSource } from './datasource';
+
 /**
  * @experimental These interfaces are experimental and might change in future releases.
  */
 
-import { IndexPatternsService } from '../../index_patterns';
-import { DataSourceType } from '../datasource_services';
-
 export interface IndexPatternOption {
   title: string;
   id: string;
 }
 
-export interface IDataSourceMetaData {
+export interface IDataSourceGroup {
   name: string;
 }
 
-export interface IDataSourceGroup {
-  name: string;
+export interface DataSetWithDataSource<T = unknown> {
+  ds: DataSource;
+  list: T[];
 }
 
-export interface ISourceDataSet {
-  ds: DataSourceType;
-  data_sets: Array<string | IndexPatternOption>;
+export interface IDataSetParams<T = {}> {
+  query: T;
 }
 
-// to-dos: add common interfaces for datasource
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface IDataSetParams {}
+export interface IDataSourceQueryParams<T = {}> {
+  query: T;
+}
 
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface IDataSourceQueryParams {}
+export interface IDataSourceQueryResult<T = {}> {
+  data: T;
+}
 
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface IDataSourceQueryResult {}
+export enum ConnectionStatus {
+  Connected = 'connected',
+  Disconnected = 'disconnected',
+  Error = 'error',
+}
 
-export interface ConnectionStatus {
-  success: boolean;
-  info: string;
+export interface DataSourceConnectionStatus {
+  status: ConnectionStatus;
+  message: string;
+  error?: Error;
 }
 
-export interface DataSourceConfig {
-  name: string;
+export interface IDataSourceSettings<T extends IDataSourceMetadata = IDataSourceMetadata> {
+  id: string;
   type: string;
-  metadata: any;
-  indexPatterns: IndexPatternsService;
+  name: string;
+  metadata: T;
+}
+
+export interface IDataSourceMetadata {
+  ui: IDataSourceUISettings;
+}
+
+export interface IDataSourceUISelector {
+  displayDatasetsAsSource: boolean;
+}
+
+/**
+ * Represents the UI settings for a data source.
+ */
+export interface IDataSourceUISettings {
+  /**
+   * Controls UI elements related to data source selector.
+   */
+  selector: IDataSourceUISelector;
+
+  /**
+   * The display name of the data source.
+   */
+  label: string;
+
+  /**
+   * The group to which the data source belongs. This is used to group data sources in the selector.
+   */
+  groupType: DataSourceUIGroupType;
+
+  /**
+   * The display name of the data source type.
+   */
+  typeLabel: string;
+
+  /**
+   * A short description of the data source.
+   * @optional
+   */
+  description?: string;
+
+  /**
+   * URI of the icon representing the data source.
+   * @optional
+   */
+  icon?: string;
+}
+
+export interface IDataSourceDataSet<T = {}> {
+  dataSets: T;
+}
+
+export interface IDataSourceQueryResponse<T = {}> {
+  data: T;
+}
+
+export enum DataSourceUIGroupType {
+  defaultOpenSearchDataSource = 'DEFAULT_INDEX_PATTERNS',
+  s3glue = 's3glue',
+  spark = 'spark',
 }