added fix for jobs and cache Support for workbench ,MDS support (#1739)

* cacahe updated to support MDS-client and added MDS support Signed-off-by: sumukhswamy <[email protected]> * addressed PR comments, added fix for jobs and cache Signed-off-by: sumukhswamy <[email protected]> * updated the tests, PR comments Signed-off-by: sumukhswamy <[email protected]> * addressed comments, fixed loading of flint datasources Signed-off-by: sumukhswamy <[email protected]> * added fix for cache and cahnged router Signed-off-by: sumukhswamy <[email protected]> * fixed linter Signed-off-by: sumukhswamy <[email protected]> --------- Signed-off-by: sumukhswamy <[email protected]> Signed-off-by: Sumukh Swamy <[email protected]> (cherry picked from commit 868ddcb) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · May 1, 2024 · edb76e7 · edb76e7
1 parent d0528e9
commit edb76e7
Show file tree

Hide file tree

Showing 26 changed files with 485 additions and 222 deletions.
diff --git a/common/types/data_connections.ts b/common/types/data_connections.ts
@@ -121,6 +121,7 @@ export interface CachedDataSource {
   lastUpdated: string; // date string in UTC format
   status: CachedDataSourceStatus;
   databases: CachedDatabase[];
+  dataSourceMDSId?: string;
 }
 
 export interface DataSourceCacheData {
@@ -143,6 +144,7 @@ export interface CachedAccelerationByDataSource {
   accelerations: CachedAcceleration[];
   lastUpdated: string; // date string in UTC format
   status: CachedDataSourceStatus;
+  dataSourceMDSId?: string;
 }
 
 export interface AccelerationsCacheData {
@@ -240,6 +242,13 @@ export interface CreateAccelerationForm {
 
 export interface LoadCachehookOutput {
   loadStatus: DirectQueryLoadingStatus;
-  startLoading: (dataSourceName: string, databaseName?: string, tableName?: string) => void;
+  startLoading: (params: StartLoadingParams) => void;
   stopLoading: () => void;
 }
+
+export interface StartLoadingParams {
+  dataSourceName: string;
+  dataSourceMDSId?: string;
+  databaseName?: string;
+  tableName?: string;
+}
diff --git a/common/utils/shared.ts b/common/utils/shared.ts
@@ -18,6 +18,9 @@ export function get<T = unknown>(obj: Record<string, any>, path: string, default
 }
 
 export function addBackticksIfNeeded(input: string): string {
+  if (input === undefined) {
+    return '';
+  }
   // Check if the string already has backticks
   if (input.startsWith('`') && input.endsWith('`')) {
     return input; // Return the string as it is

diff --git a/...ic/components/datasources/components/manage/accelerations/acceleration_details_flyout.tsx b/...ic/components/datasources/components/manage/accelerations/acceleration_details_flyout.tsx
@@ -16,24 +16,25 @@ import {
   EuiText,
 } from '@elastic/eui';
 import React, { useEffect, useState } from 'react';
+import { OpenSearchDashboardsResponse } from '../../../../../../../../src/core/server/http/router';
+import { CachedAcceleration } from '../../../../../../common/types/data_connections';
+import { coreRefs } from '../../../../../framework/core_refs';
+import { AccelerationActionOverlay } from './acceleration_action_overlay';
+import { useAccelerationOperation } from './acceleration_operation';
 import { AccelerationDetailsTab } from './flyout_modules/acceleration_details_tab';
 import { AccelerationSchemaTab } from './flyout_modules/accelerations_schema_tab';
 import {
-  onDiscoverIconClick,
   AccelerationActionType,
   getAccelerationName,
+  onDiscoverIconClick,
 } from './utils/acceleration_utils';
-import { coreRefs } from '../../../../../framework/core_refs';
-import { OpenSearchDashboardsResponse } from '../../../../../../../../src/core/server/http/router';
-import { CachedAcceleration } from '../../../../../../common/types/data_connections';
-import { useAccelerationOperation } from './acceleration_operation';
-import { AccelerationActionOverlay } from './acceleration_action_overlay';
 
 export interface AccelerationDetailsFlyoutProps {
   acceleration: CachedAcceleration;
   dataSourceName: string;
   resetFlyout: () => void;
   handleRefresh?: () => void;
+  dataSourceMDSId?: string;
 }
 
 const getMappings = (index: string): Promise<OpenSearchDashboardsResponse> | undefined => {

diff --git a/public/components/datasources/components/manage/accelerations/acceleration_table.tsx b/public/components/datasources/components/manage/accelerations/acceleration_table.tsx
@@ -18,27 +18,29 @@ import {
   EuiText,
 } from '@elastic/eui';
 import React, { useCallback, useEffect, useState } from 'react';
-import {
-  onDiscoverIconClick,
-  AccelerationStatus,
-  ACC_LOADING_MSG,
-  ACC_PANEL_TITLE,
-  ACC_PANEL_DESC,
-  getAccelerationName,
-  AccelerationActionType,
-  CreateAccelerationFlyoutButton,
-} from './utils/acceleration_utils';
-import { getRenderAccelerationDetailsFlyout } from '../../../../../plugin';
-import { CatalogCacheManager } from '../../../../../framework/catalog_cache/cache_manager';
 import {
   CachedAcceleration,
   CachedDataSourceStatus,
 } from '../../../../../../common/types/data_connections';
 import { DirectQueryLoadingStatus } from '../../../../../../common/types/explorer';
-import { AccelerationActionOverlay } from './acceleration_action_overlay';
+import { CatalogCacheManager } from '../../../../../framework/catalog_cache/cache_manager';
+import {
+  getRenderAccelerationDetailsFlyout,
+  getRenderCreateAccelerationFlyout,
+} from '../../../../../plugin';
 import { isCatalogCacheFetching } from '../associated_objects/utils/associated_objects_tab_utils';
-import { getRenderCreateAccelerationFlyout } from '../../../../../plugin';
+import { AccelerationActionOverlay } from './acceleration_action_overlay';
 import { useAccelerationOperation } from './acceleration_operation';
+import {
+  ACC_LOADING_MSG,
+  ACC_PANEL_DESC,
+  ACC_PANEL_TITLE,
+  AccelerationActionType,
+  AccelerationStatus,
+  CreateAccelerationFlyoutButton,
+  getAccelerationName,
+  onDiscoverIconClick,
+} from './utils/acceleration_utils';
 
 interface AccelerationTableProps {
   dataSourceName: string;
@@ -108,7 +110,7 @@ export const AccelerationTable = ({
       !isCatalogCacheFetching(accelerationsLoadStatus)
     ) {
       setIsRefreshing(true);
-      startLoadingAccelerations(dataSourceName);
+      startLoadingAccelerations({ dataSourceName });
     } else {
       setAccelerations(cachedDataSource.accelerations);
       setUpdatedTime(cachedDataSource.lastUpdated);

diff --git a/...omponents/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx b/...omponents/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx
@@ -45,6 +45,7 @@ export interface CreateAccelerationProps {
   resetFlyout: () => void;
   databaseName?: string;
   tableName?: string;
+  dataSourceMDSId?: string;
   refreshHandler?: () => void;
 }
 
@@ -53,6 +54,7 @@ export const CreateAcceleration = ({
   resetFlyout,
   databaseName,
   tableName,
+  dataSourceMDSId,
   refreshHandler,
 }: CreateAccelerationProps) => {
   const { setToast } = useToast();
@@ -141,12 +143,22 @@ export const CreateAcceleration = ({
     if (dataTable !== '') {
       setTableFieldsLoading(true);
       try {
-        const cachedTable = CatalogCacheManager.getTable(dataSource, database, dataTable);
+        const cachedTable = CatalogCacheManager.getTable(
+          dataSource,
+          database,
+          dataTable,
+          dataSourceMDSId
+        );
         if (cachedTable.columns) {
           loadColumnsToAccelerationForm(cachedTable);
           setTableFieldsLoading(false);
         } else {
-          startLoading(dataSource, database, dataTable);
+          startLoading({
+            dataSourceName: dataSource,
+            dataSourceMDSId,
+            databaseName: database,
+            tableName: dataTable,
+          });
         }
       } catch (error) {
         setToast('Your cache is outdated, refresh databases and tables', 'warning');
@@ -173,7 +185,8 @@ export const CreateAcceleration = ({
         cachedTable = CatalogCacheManager.getTable(
           accelerationFormData.dataSource,
           accelerationFormData.database,
-          accelerationFormData.dataTable
+          accelerationFormData.dataTable,
+          dataSourceMDSId
         );
       } catch (error) {
         setToast('Your cache is outdated, refresh databases and tables', 'warning');
@@ -218,6 +231,7 @@ export const CreateAcceleration = ({
               selectedDatasource={selectedDatasource}
               dataSourcesPreselected={dataSourcesPreselected}
               tableFieldsLoading={tableFieldsLoading}
+              dataSourceMDSId={dataSourceMDSId}
             />
             <EuiSpacer size="xxl" />
             <IndexTypeSelector
@@ -235,6 +249,7 @@ export const CreateAcceleration = ({
               accelerationFormData={accelerationFormData}
               setAccelerationFormData={setAccelerationFormData}
               tableFieldsLoading={tableFieldsLoading}
+              dataSourceMDSId={dataSourceMDSId}
             />
             <EuiSpacer size="xxl" />
             <IndexAdvancedSettings

diff --git a/...e/accelerations/create_accelerations_flyout/selectors/selector_helpers/load_databases.tsx b/...e/accelerations/create_accelerations_flyout/selectors/selector_helpers/load_databases.tsx
@@ -42,7 +42,7 @@ export const SelectorLoadDatabases = ({
 
   const onClickRefreshDatabases = () => {
     setIsLoading(true);
-    startDatabasesLoading(dataSourceName);
+    startDatabasesLoading({ dataSourceName });
   };
 
   useEffect(() => {

diff --git a/...age/accelerations/create_accelerations_flyout/selectors/selector_helpers/load_objects.tsx b/...age/accelerations/create_accelerations_flyout/selectors/selector_helpers/load_objects.tsx
@@ -65,8 +65,8 @@ export const SelectorLoadObjects = ({
       tableStatus: true,
       accelerationsStatus: true,
     });
-    startLoadingTables(dataSourceName, databaseName);
-    startLoadingAccelerations(dataSourceName);
+    startLoadingTables({ dataSourceName, databaseName });
+    startLoadingAccelerations({ dataSourceName });
   };
 
   useEffect(() => {

diff --git a/...components/manage/accelerations/create_accelerations_flyout/selectors/source_selector.tsx b/...components/manage/accelerations/create_accelerations_flyout/selectors/source_selector.tsx
@@ -20,13 +20,13 @@ import React, { useEffect, useState } from 'react';
 import { CoreStart } from '../../../../../../../../../../src/core/public';
 import { DATACONNECTIONS_BASE } from '../../../../../../../../common/constants/shared';
 import {
-  CachedDatabase,
   CachedDataSourceStatus,
+  CachedDatabase,
   CreateAccelerationForm,
 } from '../../../../../../../../common/types/data_connections';
 import { CatalogCacheManager } from '../../../../../../../framework/catalog_cache/cache_manager';
 import { useToast } from '../../../../../../common/toast';
-import { hasError, validateDatabase, validateDataTable } from '../create/utils';
+import { hasError, validateDataTable, validateDatabase } from '../create/utils';
 import { SelectorLoadDatabases } from './selector_helpers/load_databases';
 import { SelectorLoadObjects } from './selector_helpers/load_objects';
 
@@ -37,6 +37,7 @@ interface AccelerationDataSourceSelectorProps {
   selectedDatasource: string;
   dataSourcesPreselected: boolean;
   tableFieldsLoading: boolean;
+  dataSourceMDSId?: string;
 }
 
 export const AccelerationDataSourceSelector = ({
@@ -46,6 +47,7 @@ export const AccelerationDataSourceSelector = ({
   selectedDatasource,
   dataSourcesPreselected,
   tableFieldsLoading,
+  dataSourceMDSId,
 }: AccelerationDataSourceSelectorProps) => {
   const { setToast } = useToast();
   const [databases, setDatabases] = useState<Array<EuiComboBoxOptionOption<string>>>([]);
@@ -72,7 +74,7 @@ export const AccelerationDataSourceSelector = ({
   const loadDataSource = () => {
     setLoadingComboBoxes({ ...loadingComboBoxes, dataSource: true });
     http
-      .get(DATACONNECTIONS_BASE)
+      .get(DATACONNECTIONS_BASE + `/dataSourceMDSId=${dataSourceMDSId}`)
       .then((res) => {
         const isValidDataSource = res.some(
           (connection: any) =>

diff --git a/...s/manage/accelerations/create_accelerations_flyout/visual_editors/query_visual_editor.tsx b/...s/manage/accelerations/create_accelerations_flyout/visual_editors/query_visual_editor.tsx
@@ -14,12 +14,14 @@ interface QueryVisualEditorProps {
   accelerationFormData: CreateAccelerationForm;
   setAccelerationFormData: React.Dispatch<React.SetStateAction<CreateAccelerationForm>>;
   tableFieldsLoading: boolean;
+  dataSourceMDSId?: string;
 }
 
 export const QueryVisualEditor = ({
   accelerationFormData,
   setAccelerationFormData,
   tableFieldsLoading,
+  dataSourceMDSId,
 }: QueryVisualEditorProps) => {
   return tableFieldsLoading ? (
     <>
@@ -38,6 +40,7 @@ export const QueryVisualEditor = ({
         <SkippingIndexBuilder
           accelerationFormData={accelerationFormData}
           setAccelerationFormData={setAccelerationFormData}
+          dataSourceMDSId={dataSourceMDSId}
         />
       )}
       {accelerationFormData.accelerationIndexType === 'covering' && (

diff --git a/...celerations/create_accelerations_flyout/visual_editors/skipping_index/generate_fields.tsx b/...celerations/create_accelerations_flyout/visual_editors/skipping_index/generate_fields.tsx
@@ -26,13 +26,15 @@ interface GenerateFieldsProps {
   setAccelerationFormData: React.Dispatch<React.SetStateAction<CreateAccelerationForm>>;
   isSkippingtableLoading: boolean;
   setIsSkippingtableLoading: React.Dispatch<boolean>;
+  dataSourceMDSId?: string;
 }
 
 export const GenerateFields = ({
   accelerationFormData,
   setAccelerationFormData,
   isSkippingtableLoading,
   setIsSkippingtableLoading,
+  dataSourceMDSId,
 }: GenerateFieldsProps) => {
   const [isGenerateRun, setIsGenerateRun] = useState(false);
   const { loadStatus, startLoading, stopLoading: _stopLoading, pollingResult } = useDirectQuery();
@@ -84,7 +86,7 @@ export const GenerateFields = ({
       )}`,
       datasource: accelerationFormData.dataSource,
     };
-    startLoading(requestPayload);
+    startLoading(requestPayload, dataSourceMDSId);
     setIsSkippingtableLoading(true);
     setIsGenerateRun(true);
     setReplaceDefinitionModal(<></>);

diff --git a/...ions/create_accelerations_flyout/visual_editors/skipping_index/skipping_index_builder.tsx b/...ions/create_accelerations_flyout/visual_editors/skipping_index/skipping_index_builder.tsx
@@ -30,11 +30,13 @@ import { GenerateFields } from './generate_fields';
 interface SkippingIndexBuilderProps {
   accelerationFormData: CreateAccelerationForm;
   setAccelerationFormData: React.Dispatch<React.SetStateAction<CreateAccelerationForm>>;
+  dataSourceMDSId?: string;
 }
 
 export const SkippingIndexBuilder = ({
   accelerationFormData,
   setAccelerationFormData,
+  dataSourceMDSId,
 }: SkippingIndexBuilderProps) => {
   const [pageIndex, setPageIndex] = useState(0);
   const [pageSize, setPageSize] = useState(20);
@@ -200,6 +202,7 @@ export const SkippingIndexBuilder = ({
             setIsSkippingtableLoading={setIsSkippingtableLoading}
             accelerationFormData={accelerationFormData}
             setAccelerationFormData={setAccelerationFormData}
+            dataSourceMDSId={dataSourceMDSId}
           />
         </EuiFlexItem>
       </EuiFlexGroup>

diff --git a/public/components/datasources/components/manage/accelerations/utils/acceleration_utils.tsx b/public/components/datasources/components/manage/accelerations/utils/acceleration_utils.tsx
@@ -3,14 +3,14 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import React from 'react';
 import { EuiButton, EuiHealth } from '@elastic/eui';
+import React from 'react';
+import { DATA_SOURCE_TYPES } from '../../../../../../../common/constants/data_sources';
 import { CachedAcceleration } from '../../../../../../../common/types/data_connections';
 import {
   redirectToExplorerOSIdx,
   redirectToExplorerWithDataSrc,
 } from '../../associated_objects/utils/associated_objects_tab_utils';
-import { DATA_SOURCE_TYPES } from '../../../../../../../common/constants/data_sources';
 
 export const ACC_PANEL_TITLE = 'Accelerations';
 export const ACC_PANEL_DESC =
@@ -93,6 +93,7 @@ export const CreateAccelerationFlyoutButton = ({
   dataSourceName: string;
   renderCreateAccelerationFlyout: (
     dataSource: string,
+    dataSourceMDSId?: string,
     databaseName?: string,
     tableName?: string,
     handleRefresh?: () => void
@@ -103,7 +104,13 @@ export const CreateAccelerationFlyoutButton = ({
     <>
       <EuiButton
         onClick={() =>
-          renderCreateAccelerationFlyout(dataSourceName, undefined, undefined, handleRefresh)
+          renderCreateAccelerationFlyout(
+            dataSourceName,
+            undefined,
+            undefined,
+            undefined,
+            handleRefresh
+          )
         }
         fill
       >