From 5e853ab3f579f02c4c99d7b08fd1e60cdaaa0090 Mon Sep 17 00:00:00 2001 From: Kama Huang Date: Fri, 27 Dec 2024 16:42:14 -0800 Subject: [PATCH] address comments on 12/26 Signed-off-by: Kama Huang --- common/constants.ts | 2 + .../workflow_detail/workflow_detail.test.tsx | 26 +-- .../workflow_inputs/processors_list.tsx | 10 +- .../workflows/new_workflow/new_workflow.tsx | 14 +- public/pages/workflows/new_workflow/utils.ts | 150 ++++++++++-------- 5 files changed, 115 insertions(+), 87 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index f6e0db8b..e61988b8 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -121,6 +121,8 @@ export enum WORKFLOW_TYPE { } // If no datasource version is found, we default to 2.17.0 export const MIN_SUPPORTED_VERSION = '2.17.0'; +// Min version to support ML processors +export const MINIMUM_FULL_SUPPORTED_VERSION = '2.19.0'; // the names should be consistent with the underlying implementation. used when generating the // final ingest/search pipeline configurations. diff --git a/public/pages/workflow_detail/workflow_detail.test.tsx b/public/pages/workflow_detail/workflow_detail.test.tsx index adb43839..531de490 100644 --- a/public/pages/workflow_detail/workflow_detail.test.tsx +++ b/public/pages/workflow_detail/workflow_detail.test.tsx @@ -110,28 +110,28 @@ describe('WorkflowDetail Page Functionality (Custom Workflow)', () => { workflowName, WORKFLOW_TYPE.CUSTOM ); - + // Export button opens the export component userEvent.click(getByTestId('exportButton')); await waitFor(() => { expect(getByText(`Export '${workflowName}'`)).toBeInTheDocument(); }); // Close the export component userEvent.click(getByTestId('exportCloseButton')); - + // Check workspace button group exists (Visual and JSON) getByTestId('visualJSONToggleButtonGroup'); - + // Tools panel should collapse and expand the toggle const toolsPanel = container.querySelector('#tools_panel_id'); expect(toolsPanel).toBeVisible(); const toggleButton = toolsPanel?.querySelector('button[type="button"]'); expect(toggleButton).toBeInTheDocument(); userEvent.click(toggleButton!); - + // Tools panel after collapsing const collapsedToolsPanel = container.querySelector('#tools_panel_id'); await waitFor(() => { expect(collapsedToolsPanel).toHaveClass('euiResizablePanel-isCollapsed'); }); - + // Tools panel after expanding userEvent.click(toggleButton!); const expandedToolsPanel = container.querySelector('#tools_panel_id'); await waitFor(() => { @@ -147,7 +147,7 @@ describe('WorkflowDetail Page Functionality (Custom Workflow)', () => { workflowName, WORKFLOW_TYPE.CUSTOM ); - + // The WorkflowDetail Page Close button should navigate back to the workflows list userEvent.click(getByTestId('closeButton')); await waitFor(() => { expect(history.location.pathname).toBe('/workflows'); @@ -166,20 +166,20 @@ describe('WorkflowDetail Page with skip ingestion option (Hybrid Search Workflow workflowName, WORKFLOW_TYPE.HYBRID_SEARCH ); - + // Defining a new ingest pipeline & index is enabled by default const enabledCheckbox = getByTestId('switch-ingest.enabled'); - + // Skipping ingest pipeline and navigating to search userEvent.click(enabledCheckbox); await waitFor(() => {}); const searchPipelineButton = getByTestId('searchPipelineButton'); userEvent.click(searchPipelineButton); - + // Search pipeline await waitFor(() => { expect(getAllByText('Define search flow').length).toBeGreaterThan(0); }); expect(getAllByText('Configure query').length).toBeGreaterThan(0); - + // Edit Search Query const queryEditButton = getByTestId('queryEditButton'); expect(queryEditButton).toBeInTheDocument(); userEvent.click(queryEditButton); @@ -193,7 +193,7 @@ describe('WorkflowDetail Page with skip ingestion option (Hybrid Search Workflow const updateSearchQueryButton = getByTestId('updateSearchQueryButton'); expect(updateSearchQueryButton).toBeInTheDocument(); userEvent.click(updateSearchQueryButton); - + // Add request processor const addRequestProcessorButton = await waitFor( () => getAllByTestId('addProcessorButton')[0] ); @@ -203,14 +203,14 @@ describe('WorkflowDetail Page with skip ingestion option (Hybrid Search Workflow const popoverPanel = document.querySelector('.euiPopover__panel'); expect(popoverPanel).toBeTruthy(); }); - + // Add response processor const addResponseProcessorButton = getAllByTestId('addProcessorButton')[1]; userEvent.click(addResponseProcessorButton); await waitFor(() => { const popoverPanel = document.querySelector('.euiPopover__panel'); expect(popoverPanel).toBeTruthy(); }); - + // Build and Run query, Back buttons are present const searchPipelineBackButton = getByTestId('searchPipelineBackButton'); userEvent.click(searchPipelineBackButton); diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index ea029e7a..ead3617d 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -45,6 +45,10 @@ import { import { ProcessorInputs } from './processor_inputs'; import { useLocation } from 'react-router-dom'; import { getDataSourceEnabled } from '../../../../public/services'; +import { + MIN_SUPPORTED_VERSION, + MINIMUM_FULL_SUPPORTED_VERSION, +} from '../../../../common'; interface ProcessorsListProps { uiConfig: WorkflowConfig; @@ -78,7 +82,7 @@ export function ProcessorsList(props: ProcessorsListProps) { const enabled = getDataSourceEnabled().enabled; if (!enabled) { - setVersion('2.19.0'); + setVersion(MINIMUM_FULL_SUPPORTED_VERSION); return; } @@ -110,7 +114,8 @@ export function ProcessorsList(props: ProcessorsListProps) { const getMenuItems = () => { const isPreV219 = - semver.gte(version, '2.17.0') && semver.lt(version, '2.19.0'); + semver.gte(version, MIN_SUPPORTED_VERSION) && + semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); const ingestProcessors = [ ...(isPreV219 ? [ @@ -377,7 +382,6 @@ export function ProcessorsList(props: ProcessorsListProps) { title: getMenuItems().length > 0 ? 'PROCESSORS' : '', items: (() => { const items = getMenuItems(); - console.log('Menu items:', items); return items; })(), }, diff --git a/public/pages/workflows/new_workflow/new_workflow.tsx b/public/pages/workflows/new_workflow/new_workflow.tsx index 4005fd5a..520dcdf8 100644 --- a/public/pages/workflows/new_workflow/new_workflow.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.tsx @@ -35,6 +35,7 @@ import { getSavedObjectsClient } from '../../../../public/services'; import { WORKFLOW_TYPE, MIN_SUPPORTED_VERSION, + MINIMUM_FULL_SUPPORTED_VERSION, } from '../../../../common/constants'; interface NewWorkflowProps {} @@ -64,7 +65,6 @@ const filterPresetsByVersion = async ( workflows: WorkflowTemplate[], dataSourceId: string | undefined ): Promise => { - console.log('Initial workflows count:', workflows.length); // if MDS is disabled, skip the version check and assume it is version 2.19+ const dataSourceEnabled = getDataSourceEnabled().enabled; if (!dataSourceEnabled) { @@ -84,11 +84,14 @@ const filterPresetsByVersion = async ( try { const version = await getEffectiveVersion(dataSourceId); - if (semver.lt(version, '2.17.0')) { + if (semver.lt(version, MIN_SUPPORTED_VERSION)) { return []; } - if (semver.gte(version, '2.17.0') && semver.lt(version, '2.19.0')) { + if ( + semver.gte(version, MIN_SUPPORTED_VERSION) && + semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION) + ) { return workflows.filter((workflow) => { const workflowType = workflow.ui_metadata?.type ?? WORKFLOW_TYPE.UNKNOWN; @@ -154,7 +157,10 @@ export function NewWorkflow(props: NewWorkflowProps) { if (!dataSourceEnabled) { const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => - enrichPresetWorkflowWithUiMetadata(presetWorkflow, '2.19.0') + enrichPresetWorkflowWithUiMetadata( + presetWorkflow, + MINIMUM_FULL_SUPPORTED_VERSION + ) ); setAllWorkflows(enrichedWorkflows); setFilteredWorkflows(enrichedWorkflows); diff --git a/public/pages/workflows/new_workflow/utils.ts b/public/pages/workflows/new_workflow/utils.ts index 819ad56a..4f84a220 100644 --- a/public/pages/workflows/new_workflow/utils.ts +++ b/public/pages/workflows/new_workflow/utils.ts @@ -10,6 +10,8 @@ import { MLSearchRequestProcessor, MLSearchResponseProcessor, NormalizationProcessor, + TextEmbeddingIngestProcessor, + TextImageEmbeddingIngestProcessor, } from '../../../configs'; import { WorkflowTemplate, @@ -19,7 +21,6 @@ import { WORKFLOW_TYPE, FETCH_ALL_QUERY, customStringify, - TERM_QUERY_TEXT, MULTIMODAL_SEARCH_QUERY_BOOL, IProcessorConfig, VECTOR_TEMPLATE_PLACEHOLDER, @@ -28,10 +29,13 @@ import { HYBRID_SEARCH_QUERY_MATCH_KNN, WorkflowConfig, UI_METADATA_SCHEMA_VERSION, - PROCESSOR_TYPE, + SEMANTIC_SEARCH_QUERY_NEURAL, + MULTIMODAL_SEARCH_QUERY_NEURAL, + HYBRID_SEARCH_QUERY_MATCH_NEURAL, } from '../../../../common'; import { generateId } from '../../../utils'; import semver from 'semver'; +import { MINIMUM_FULL_SUPPORTED_VERSION } from '../../../../common'; // Fn to produce the complete preset template with all necessary UI metadata. export function enrichPresetWorkflowWithUiMetadata( @@ -41,19 +45,19 @@ export function enrichPresetWorkflowWithUiMetadata( let uiMetadata = {} as UIState; switch (presetWorkflow.ui_metadata?.type || WORKFLOW_TYPE.CUSTOM) { case WORKFLOW_TYPE.SEMANTIC_SEARCH: { - uiMetadata = fetchSemanticSearchMetadata(); + uiMetadata = fetchSemanticSearchMetadata(version); break; } case WORKFLOW_TYPE.MULTIMODAL_SEARCH: { - uiMetadata = fetchMultimodalSearchMetadata(); + uiMetadata = fetchMultimodalSearchMetadata(version); break; } case WORKFLOW_TYPE.HYBRID_SEARCH: { - uiMetadata = fetchHybridSearchMetadata(); + uiMetadata = fetchHybridSearchMetadata(version); break; } case WORKFLOW_TYPE.RAG: { - uiMetadata = fetchRAGMetadata(); + uiMetadata = fetchRAGMetadata(version); break; } default: { @@ -62,39 +66,6 @@ export function enrichPresetWorkflowWithUiMetadata( } } - if (semver.gte(version, '2.17.0') && semver.lt(version, '2.19.0')) { - uiMetadata.config.ingest.enrich.processors = uiMetadata.config.ingest.enrich.processors.filter( - (p) => p.type !== PROCESSOR_TYPE.ML - ); - uiMetadata.config.search.enrichRequest.processors = uiMetadata.config.search.enrichRequest.processors.filter( - (p) => p.type !== PROCESSOR_TYPE.ML - ); - uiMetadata.config.search.enrichResponse.processors = uiMetadata.config.search.enrichResponse.processors.filter( - (p) => p.type !== PROCESSOR_TYPE.ML - ); - } - - if (semver.eq(version, '2.19.0')) { - uiMetadata.config.ingest.enrich.processors = uiMetadata.config.ingest.enrich.processors.filter( - (p) => - p.type !== PROCESSOR_TYPE.TEXT_EMBEDDING && - p.type !== PROCESSOR_TYPE.TEXT_IMAGE_EMBEDDING && - p.type !== PROCESSOR_TYPE.NORMALIZATION - ); - uiMetadata.config.search.enrichRequest.processors = uiMetadata.config.search.enrichRequest.processors.filter( - (p) => - p.type !== PROCESSOR_TYPE.TEXT_EMBEDDING && - p.type !== PROCESSOR_TYPE.TEXT_IMAGE_EMBEDDING && - p.type !== PROCESSOR_TYPE.NORMALIZATION - ); - uiMetadata.config.search.enrichResponse.processors = uiMetadata.config.search.enrichResponse.processors.filter( - (p) => - p.type !== PROCESSOR_TYPE.TEXT_EMBEDDING && - p.type !== PROCESSOR_TYPE.TEXT_IMAGE_EMBEDDING && - p.type !== PROCESSOR_TYPE.NORMALIZATION - ); - } - return { ...presetWorkflow, ui_metadata: { @@ -174,68 +145,113 @@ export function fetchEmptyUIConfig(): WorkflowConfig { }; } -export function fetchSemanticSearchMetadata(): UIState { +export function fetchSemanticSearchMetadata(version: string): UIState { let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.SEMANTIC_SEARCH; - baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; + const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); + + baseState.config.ingest.enrich.processors = isPreV219 + ? [ + new TextEmbeddingIngestProcessor().toObj(), + new TextImageEmbeddingIngestProcessor().toObj(), + ] + : [new MLIngestProcessor().toObj()]; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); - baseState.config.search.request.value = customStringify(TERM_QUERY_TEXT); - baseState.config.search.enrichRequest.processors = [ - injectQueryTemplateInProcessor( - new MLSearchRequestProcessor().toObj(), - KNN_QUERY - ), - ]; + + baseState.config.search.request.value = customStringify( + isPreV219 ? SEMANTIC_SEARCH_QUERY_NEURAL : KNN_QUERY + ); + + baseState.config.search.enrichRequest.processors = isPreV219 + ? [] + : [ + injectQueryTemplateInProcessor( + new MLSearchRequestProcessor().toObj(), + KNN_QUERY + ), + ]; + return baseState; } -export function fetchMultimodalSearchMetadata(): UIState { +export function fetchMultimodalSearchMetadata(version: string): UIState { let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.MULTIMODAL_SEARCH; - baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; + + const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); + + baseState.config.ingest.enrich.processors = isPreV219 + ? [ + new TextEmbeddingIngestProcessor().toObj(), + new TextImageEmbeddingIngestProcessor().toObj(), + ] + : [new MLIngestProcessor().toObj()]; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); + baseState.config.search.request.value = customStringify( - MULTIMODAL_SEARCH_QUERY_BOOL + isPreV219 ? MULTIMODAL_SEARCH_QUERY_NEURAL : MULTIMODAL_SEARCH_QUERY_BOOL ); - baseState.config.search.enrichRequest.processors = [ - injectQueryTemplateInProcessor( - new MLSearchRequestProcessor().toObj(), - KNN_QUERY - ), - ]; + + baseState.config.search.enrichRequest.processors = isPreV219 + ? [] + : [ + injectQueryTemplateInProcessor( + new MLSearchRequestProcessor().toObj(), + KNN_QUERY + ), + ]; + return baseState; } -export function fetchHybridSearchMetadata(): UIState { +export function fetchHybridSearchMetadata(version: string): UIState { let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.HYBRID_SEARCH; - baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; + const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); + + baseState.config.ingest.enrich.processors = isPreV219 + ? [ + new TextEmbeddingIngestProcessor().toObj(), + new TextImageEmbeddingIngestProcessor().toObj(), + ] + : [new MLIngestProcessor().toObj()]; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); - baseState.config.search.request.value = customStringify(TERM_QUERY_TEXT); + + baseState.config.search.request.value = customStringify( + isPreV219 ? HYBRID_SEARCH_QUERY_MATCH_NEURAL : HYBRID_SEARCH_QUERY_MATCH_KNN + ); + baseState.config.search.enrichResponse.processors = [ injectDefaultWeightsInNormalizationProcessor( new NormalizationProcessor().toObj() ), ]; - baseState.config.search.enrichRequest.processors = [ - injectQueryTemplateInProcessor( - new MLSearchRequestProcessor().toObj(), - HYBRID_SEARCH_QUERY_MATCH_KNN - ), - ]; + + baseState.config.search.enrichRequest.processors = isPreV219 + ? [] + : [ + injectQueryTemplateInProcessor( + new MLSearchRequestProcessor().toObj(), + HYBRID_SEARCH_QUERY_MATCH_KNN + ), + ]; + return baseState; } -export function fetchRAGMetadata(): UIState { +export function fetchRAGMetadata(version: string): UIState { let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.RAG; baseState.config.ingest.index.name.value = generateId('my_index', 6);