From 8a6fe0863383592a4f735deaef7d17f4d970a8ed Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 09:08:56 -0700 Subject: [PATCH] Onboard hybrid search use case; add readonly search flows for all use cases (#143) (#144) Signed-off-by: Tyler Ohlsen (cherry picked from commit 73cbed44bbc094e2fd25552120f6dd57846ae08d) Co-authored-by: Tyler Ohlsen --- common/constants.ts | 10 - common/interfaces.ts | 53 +- public/component_types/base_component.tsx | 11 + public/component_types/indexer/indexer.ts | 19 +- public/component_types/other/index.ts | 2 + public/component_types/other/query/index.ts | 7 + .../other/query/match_query.tsx | 28 + .../other/query/neural_query.tsx | 28 + public/component_types/other/query/query.tsx | 25 + public/component_types/other/results.tsx | 30 ++ public/component_types/transformer/index.ts | 2 + .../transformer/normalization_transformer.ts | 21 + .../transformer/results_transformer.ts | 34 ++ .../transformer/sparse_encoder_transformer.ts | 10 + .../transformer/text_embedding_transformer.ts | 10 + .../workflow_detail/prototype/ingestor.tsx | 2 + .../prototype/query_executor.tsx | 58 ++- .../pages/workflow_detail/prototype/utils.ts | 4 +- .../workflow_detail/resources/columns.tsx | 10 +- .../utils/data_extractor_utils.ts | 29 +- .../utils/workflow_to_template_utils.ts | 113 ++-- .../workspace/reactflow-styles.scss | 6 +- .../workspace_component.tsx | 10 +- public/pages/workflows/new_workflow/utils.ts | 485 +++++++++++++++++- public/route_service.ts | 21 +- public/store/reducers/opensearch_reducer.ts | 10 +- public/utils/constants.ts | 6 + server/resources/templates/hybrid_search.json | 12 + server/routes/helpers.ts | 1 + server/routes/opensearch_routes_service.ts | 19 +- 30 files changed, 971 insertions(+), 105 deletions(-) create mode 100644 public/component_types/other/query/index.ts create mode 100644 public/component_types/other/query/match_query.tsx create mode 100644 public/component_types/other/query/neural_query.tsx create mode 100644 public/component_types/other/query/query.tsx create mode 100644 public/component_types/other/results.tsx create mode 100644 public/component_types/transformer/normalization_transformer.ts create mode 100644 public/component_types/transformer/results_transformer.ts create mode 100644 server/resources/templates/hybrid_search.json diff --git a/common/constants.ts b/common/constants.ts index 4a9ab001..762436fe 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -55,16 +55,6 @@ export const GET_PRESET_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH export const BASE_MODEL_NODE_API_PATH = `${BASE_NODE_API_PATH}/model`; export const SEARCH_MODELS_NODE_API_PATH = `${BASE_MODEL_NODE_API_PATH}/search`; -/** - * BACKEND INTERFACES - */ -export const CREATE_INGEST_PIPELINE_STEP_TYPE = 'create_ingest_pipeline'; -export const CREATE_INDEX_STEP_TYPE = 'create_index'; -export const REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE = - 'register_local_pretrained_model'; -export const REGISTER_LOCAL_SPARSE_ENCODING_MODEL_STEP_TYPE = - 'register_local_sparse_encoding_model'; - /** * ML PLUGIN PRETRAINED MODELS * (based off of https://opensearch.org/docs/latest/ml-commons-plugin/pretrained-models) diff --git a/common/interfaces.ts b/common/interfaces.ts index a2686d6e..e5bd4080 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -43,9 +43,11 @@ export type WorkspaceFlowState = { ********** USE CASE TEMPLATE TYPES/INTERFACES ********** */ -export type IngestProcessor = { - description?: string; -}; +export type IngestProcessor = {}; +export type SearchProcessor = {}; +export type SearchRequestProcessor = SearchProcessor & {}; +export type SearchResponseProcessor = SearchProcessor & {}; +export type SearchPhaseResultsProcessor = SearchProcessor & {}; export type TextEmbeddingProcessor = IngestProcessor & { text_embedding: { @@ -61,6 +63,18 @@ export type SparseEncodingProcessor = IngestProcessor & { }; }; +export type NormalizationProcessor = SearchProcessor & { + normalization: { + technique: string; + }; + combination: { + technique: string; + parameters: { + weights: number[]; + }; + }; +}; + export type IndexConfiguration = { settings: {}; mappings: IndexMappings; @@ -90,6 +104,18 @@ export type CreateIngestPipelineNode = TemplateNode & { }; }; +export type CreateSearchPipelineNode = TemplateNode & { + user_inputs: { + pipeline_id: string; + configurations: { + description?: string; + request_processors?: SearchRequestProcessor[]; + response_processors?: SearchResponseProcessor[]; + phase_results_processors?: SearchPhaseResultsProcessor[]; + }; + }; +}; + export type CreateIndexNode = TemplateNode & { previous_node_inputs?: { [ingest_pipeline_step_id: string]: string; @@ -157,6 +183,7 @@ export type Workflow = WorkflowTemplate & { export enum USE_CASE { SEMANTIC_SEARCH = 'SEMANTIC_SEARCH', NEURAL_SPARSE_SEARCH = 'NEURAL_SPARSE_SEARCH', + HYBRID_SEARCH = 'HYBRID_SEARCH', } /** @@ -264,6 +291,7 @@ export enum WORKFLOW_STATE { export type WorkflowResource = { id: string; + stepType: WORKFLOW_STEP_TYPE; type: WORKFLOW_RESOURCE_TYPE; }; @@ -276,6 +304,25 @@ export enum WORKFLOW_RESOURCE_TYPE { CONNECTOR_ID = 'Connector', } +export enum WORKFLOW_STEP_TYPE { + CREATE_INGEST_PIPELINE_STEP_TYPE = 'create_ingest_pipeline', + CREATE_SEARCH_PIPELINE_STEP_TYPE = 'create_search_pipeline', + CREATE_INDEX_STEP_TYPE = 'create_index', + REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE = 'register_local_pretrained_model', + REGISTER_LOCAL_SPARSE_ENCODING_MODEL_STEP_TYPE = 'register_local_sparse_encoding_model', +} + +// We cannot disambiguate ingest vs. search pipelines based on workflow resource type. To work around +// that, we maintain this map from workflow step type -> formatted type +export enum WORKFLOW_STEP_TO_RESOURCE_TYPE_MAP { + 'create_ingest_pipeline' = 'Ingest pipeline', + 'create_search_pipeline' = 'Search pipeline', + 'create_index' = 'Index', + 'register_local_pretrained_model' = 'Model', + 'register_local_sparse_encoding_model' = 'Model', + 'deploy_model' = 'Model', +} + export type WorkflowDict = { [workflowId: string]: Workflow; }; diff --git a/public/component_types/base_component.tsx b/public/component_types/base_component.tsx index c2a511bd..d4540d76 100644 --- a/public/component_types/base_component.tsx +++ b/public/component_types/base_component.tsx @@ -34,4 +34,15 @@ export abstract class BaseComponent implements IComponent { toObj() { return Object.assign({}, this); } + + // Helper fn to strip all fields for a component if we want to view it in the UI + // but not have it tied to any form/inputs. Example: showing an Index component in search, + // even if it is provisioned in ingest. + toPlaceholderObj() { + return { + ...Object.assign({}, this), + createFields: [], + fields: [], + }; + } } diff --git a/public/component_types/indexer/indexer.ts b/public/component_types/indexer/indexer.ts index 25959e02..c3323a23 100644 --- a/public/component_types/indexer/indexer.ts +++ b/public/component_types/indexer/indexer.ts @@ -25,6 +25,12 @@ export class Indexer extends BaseComponent { baseClass: COMPONENT_CLASS.DOCUMENT, acceptMultiple: false, }, + { + id: 'query', + label: 'Query', + baseClass: COMPONENT_CLASS.QUERY, + acceptMultiple: true, + }, ]; this.fields = [ { @@ -46,12 +52,11 @@ export class Indexer extends BaseComponent { // placeholder: 'Enter an index mappings JSON blob...', // }, ]; - // this.outputs = [ - // { - // label: this.label, - // baseClasses: this.baseClasses, - // }, - // ]; - this.outputs = []; + this.outputs = [ + { + label: 'Results', + baseClasses: [COMPONENT_CLASS.RESULTS], + }, + ]; } } diff --git a/public/component_types/other/index.ts b/public/component_types/other/index.ts index 3441a8ed..85840547 100644 --- a/public/component_types/other/index.ts +++ b/public/component_types/other/index.ts @@ -4,3 +4,5 @@ */ export * from './document'; +export * from './results'; +export * from './query'; diff --git a/public/component_types/other/query/index.ts b/public/component_types/other/query/index.ts new file mode 100644 index 00000000..d7567aa5 --- /dev/null +++ b/public/component_types/other/query/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './match_query'; +export * from './neural_query'; diff --git a/public/component_types/other/query/match_query.tsx b/public/component_types/other/query/match_query.tsx new file mode 100644 index 00000000..4bd08098 --- /dev/null +++ b/public/component_types/other/query/match_query.tsx @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { COMPONENT_CLASS } from '../../../utils'; +import { Query } from './query'; + +/** + * A basic match query placeholder UI component. + * Does not have any functionality. + */ +export class MatchQuery extends Query { + constructor() { + super(); + this.type = COMPONENT_CLASS.MATCH_QUERY; + this.label = 'Match Query'; + this.description = 'An OpenSearch match query'; + this.inputs = []; + this.baseClasses = [...this.baseClasses, this.type]; + this.outputs = [ + { + label: this.label, + baseClasses: this.baseClasses, + }, + ]; + } +} diff --git a/public/component_types/other/query/neural_query.tsx b/public/component_types/other/query/neural_query.tsx new file mode 100644 index 00000000..57ea301c --- /dev/null +++ b/public/component_types/other/query/neural_query.tsx @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { COMPONENT_CLASS } from '../../../utils'; +import { Query } from './query'; + +/** + * A basic neural query placeholder UI component. + * Does not have any functionality. + */ +export class NeuralQuery extends Query { + constructor() { + super(); + this.type = COMPONENT_CLASS.NEURAL_QUERY; + this.label = 'Neural query'; + this.description = 'An OpenSearch neural query'; + this.inputs = []; + this.baseClasses = [...this.baseClasses, this.type]; + this.outputs = [ + { + label: this.label, + baseClasses: this.baseClasses, + }, + ]; + } +} diff --git a/public/component_types/other/query/query.tsx b/public/component_types/other/query/query.tsx new file mode 100644 index 00000000..24611979 --- /dev/null +++ b/public/component_types/other/query/query.tsx @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../../../utils'; +import { BaseComponent } from '../../base_component'; + +/** + * A basic Query placeholder UI component. + * Does not have any functionality. + */ +export abstract class Query extends BaseComponent { + constructor() { + super(); + this.type = COMPONENT_CLASS.QUERY; + this.label = 'Query'; + this.description = 'An OpenSearch query'; + this.categories = [COMPONENT_CATEGORY.SEARCH]; + this.allowsCreation = false; + this.baseClasses = [this.type]; + this.inputs = []; + this.outputs = []; + } +} diff --git a/public/component_types/other/results.tsx b/public/component_types/other/results.tsx new file mode 100644 index 00000000..972a8b88 --- /dev/null +++ b/public/component_types/other/results.tsx @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../../utils'; +import { BaseComponent } from '../base_component'; + +/** + * A basic Results placeholder UI component. + * Does not have any functionality. + */ +export class Results extends BaseComponent { + constructor() { + super(); + this.type = COMPONENT_CLASS.RESULTS; + this.label = 'Results'; + this.description = 'OpenSearch results'; + this.categories = [COMPONENT_CATEGORY.SEARCH]; + this.allowsCreation = false; + this.baseClasses = [this.type]; + this.inputs = []; + this.outputs = [ + { + label: this.label, + baseClasses: this.baseClasses, + }, + ]; + } +} diff --git a/public/component_types/transformer/index.ts b/public/component_types/transformer/index.ts index 740503a0..ce3afc3e 100644 --- a/public/component_types/transformer/index.ts +++ b/public/component_types/transformer/index.ts @@ -6,3 +6,5 @@ export * from './ml_transformer'; export * from './text_embedding_transformer'; export * from './sparse_encoder_transformer'; +export * from './results_transformer'; +export * from './normalization_transformer'; diff --git a/public/component_types/transformer/normalization_transformer.ts b/public/component_types/transformer/normalization_transformer.ts new file mode 100644 index 00000000..d04cff1f --- /dev/null +++ b/public/component_types/transformer/normalization_transformer.ts @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../../utils'; +import { ResultsTransformer } from './results_transformer'; + +/** + * A normalization results transformer UI component + */ +export class NormalizationTransformer extends ResultsTransformer { + constructor() { + super(); + (this.type = COMPONENT_CLASS.NORMALIZATION_TRANSFORMER), + (this.label = 'Normalization Transformer'); + this.description = 'A transformer to normalize search results'; + this.baseClasses = [...this.baseClasses, this.type]; + this.categories = [COMPONENT_CATEGORY.SEARCH]; + } +} diff --git a/public/component_types/transformer/results_transformer.ts b/public/component_types/transformer/results_transformer.ts new file mode 100644 index 00000000..8908bb13 --- /dev/null +++ b/public/component_types/transformer/results_transformer.ts @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { COMPONENT_CLASS } from '../../utils'; +import { BaseTransformer } from './base_transformer'; + +/** + * A generic results transformer UI component + */ +export class ResultsTransformer extends BaseTransformer { + constructor() { + super(); + (this.type = COMPONENT_CLASS.RESULTS_TRANSFORMER), + (this.label = 'Results Transformer'); + this.description = 'A general results transformer'; + this.baseClasses = [...this.baseClasses, this.type]; + this.inputs = [ + { + id: 'results', + label: 'Results', + baseClass: COMPONENT_CLASS.RESULTS, + acceptMultiple: false, + }, + ]; + this.outputs = [ + { + label: 'Transformed Results', + baseClasses: [COMPONENT_CLASS.RESULTS], + }, + ]; + } +} diff --git a/public/component_types/transformer/sparse_encoder_transformer.ts b/public/component_types/transformer/sparse_encoder_transformer.ts index 01ad8a92..6a631935 100644 --- a/public/component_types/transformer/sparse_encoder_transformer.ts +++ b/public/component_types/transformer/sparse_encoder_transformer.ts @@ -25,6 +25,12 @@ export class SparseEncoderTransformer extends MLTransformer { baseClass: COMPONENT_CLASS.DOCUMENT, acceptMultiple: false, }, + { + id: 'query', + label: 'Query', + baseClass: COMPONENT_CLASS.QUERY, + acceptMultiple: false, + }, ]; this.createFields = [ { @@ -59,6 +65,10 @@ export class SparseEncoderTransformer extends MLTransformer { label: 'Transformed Document', baseClasses: [COMPONENT_CLASS.DOCUMENT], }, + { + label: 'Transformed Query', + baseClasses: [COMPONENT_CLASS.QUERY], + }, ]; } } diff --git a/public/component_types/transformer/text_embedding_transformer.ts b/public/component_types/transformer/text_embedding_transformer.ts index bf05674f..e20e9cf7 100644 --- a/public/component_types/transformer/text_embedding_transformer.ts +++ b/public/component_types/transformer/text_embedding_transformer.ts @@ -24,6 +24,12 @@ export class TextEmbeddingTransformer extends MLTransformer { baseClass: COMPONENT_CLASS.DOCUMENT, acceptMultiple: false, }, + { + id: 'query', + label: 'Query', + baseClass: COMPONENT_CLASS.QUERY, + acceptMultiple: false, + }, ]; this.createFields = [ { @@ -57,6 +63,10 @@ export class TextEmbeddingTransformer extends MLTransformer { label: 'Transformed Document', baseClasses: [COMPONENT_CLASS.DOCUMENT], }, + { + label: 'Transformed Query', + baseClasses: [COMPONENT_CLASS.QUERY], + }, ]; } } diff --git a/public/pages/workflow_detail/prototype/ingestor.tsx b/public/pages/workflow_detail/prototype/ingestor.tsx index 00cbedac..8cfcfec4 100644 --- a/public/pages/workflow_detail/prototype/ingestor.tsx +++ b/public/pages/workflow_detail/prototype/ingestor.tsx @@ -185,6 +185,7 @@ function getDocGeneratorFn(workflow: Workflow): DocGeneratorFn { switch (workflow.use_case) { case USE_CASE.SEMANTIC_SEARCH: case USE_CASE.NEURAL_SPARSE_SEARCH: + case USE_CASE.HYBRID_SEARCH: default: { fn = () => generateNeuralSearchDoc; } @@ -198,6 +199,7 @@ function getWorkflowValues(workflow: Workflow): WorkflowValues { switch (workflow.use_case) { case USE_CASE.SEMANTIC_SEARCH: case USE_CASE.NEURAL_SPARSE_SEARCH: + case USE_CASE.HYBRID_SEARCH: default: { values = getNeuralSearchValues(workflow); } diff --git a/public/pages/workflow_detail/prototype/query_executor.tsx b/public/pages/workflow_detail/prototype/query_executor.tsx index c7ce567f..cc5f6d68 100644 --- a/public/pages/workflow_detail/prototype/query_executor.tsx +++ b/public/pages/workflow_detail/prototype/query_executor.tsx @@ -21,6 +21,7 @@ import { import { searchIndex, useAppDispatch } from '../../../store'; import { getCore } from '../../../services'; import { + HybridSearchValues, NeuralSparseValues, SemanticSearchValues, WorkflowValues, @@ -83,7 +84,13 @@ export function QueryExecutor(props: QueryExecutorProps) { // function onExecuteSearch() { - dispatch(searchIndex({ index: indexName, body: queryObj })) + dispatch( + searchIndex({ + index: indexName, + body: queryObj, + searchPipeline: workflowValues?.searchPipelineId, + }) + ) .unwrap() .then(async (result) => { setResultHits(result.hits.hits); @@ -187,9 +194,16 @@ function getQueryGeneratorFn(workflow: Workflow): QueryGeneratorFn { fn = () => generateSemanticSearchQuery; break; } - case USE_CASE.NEURAL_SPARSE_SEARCH: - default: { + case USE_CASE.NEURAL_SPARSE_SEARCH: { fn = () => generateNeuralSparseQuery; + break; + } + case USE_CASE.HYBRID_SEARCH: { + fn = () => generateHybridSearchQuery; + break; + } + default: { + fn = () => () => {}; } } return fn; @@ -200,6 +214,8 @@ function getWorkflowValues(workflow: Workflow): WorkflowValues { let values; switch (workflow.use_case) { case USE_CASE.SEMANTIC_SEARCH: + case USE_CASE.NEURAL_SPARSE_SEARCH: + case USE_CASE.HYBRID_SEARCH: default: { values = getNeuralSearchValues(workflow); } @@ -251,6 +267,42 @@ function generateNeuralSparseQuery( }; } +// utility fn to generate a hybrid search query +function generateHybridSearchQuery( + queryText: string, + workflowValues: HybridSearchValues +): {} { + return { + // TODO: can make this configurable + _source: { + excludes: [`${workflowValues.vectorField}`], + }, + query: { + hybrid: { + queries: [ + { + match: { + [workflowValues.inputField]: { + query: queryText, + }, + }, + }, + { + neural: { + [workflowValues.vectorField]: { + query_text: queryText, + model_id: workflowValues.modelId, + // TODO: expose k as configurable + k: 5, + }, + }, + }, + ], + }, + }, + }; +} + function processHits(hits: any[]): {}[] { return hits.map((hit) => hit._source); } diff --git a/public/pages/workflow_detail/prototype/utils.ts b/public/pages/workflow_detail/prototype/utils.ts index 920d5d65..11d6fd9a 100644 --- a/public/pages/workflow_detail/prototype/utils.ts +++ b/public/pages/workflow_detail/prototype/utils.ts @@ -21,5 +21,7 @@ export type SemanticSearchValues = WorkflowValues & { inputField: string; vectorField: string; }; - export type NeuralSparseValues = SemanticSearchValues; +export type HybridSearchValues = SemanticSearchValues & { + searchPipelineId: string; +}; diff --git a/public/pages/workflow_detail/resources/columns.tsx b/public/pages/workflow_detail/resources/columns.tsx index 7e13b49c..a7ffbf55 100644 --- a/public/pages/workflow_detail/resources/columns.tsx +++ b/public/pages/workflow_detail/resources/columns.tsx @@ -3,6 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { + WORKFLOW_STEP_TO_RESOURCE_TYPE_MAP, + WORKFLOW_STEP_TYPE, +} from '../../../../common'; + export const columns = [ { field: 'id', @@ -10,8 +15,11 @@ export const columns = [ sortable: true, }, { - field: 'type', + field: 'stepType', name: 'Type', sortable: true, + render: (stepType: WORKFLOW_STEP_TYPE) => { + return WORKFLOW_STEP_TO_RESOURCE_TYPE_MAP[stepType]; + }, }, ]; diff --git a/public/pages/workflow_detail/utils/data_extractor_utils.ts b/public/pages/workflow_detail/utils/data_extractor_utils.ts index 3a49c840..be03b207 100644 --- a/public/pages/workflow_detail/utils/data_extractor_utils.ts +++ b/public/pages/workflow_detail/utils/data_extractor_utils.ts @@ -13,8 +13,10 @@ import { Workflow, WORKFLOW_RESOURCE_TYPE, WorkflowResource, + NODE_CATEGORY, + WORKFLOW_STEP_TYPE, } from '../../../../common'; -import { getIngestNodesAndEdges } from './workflow_to_template_utils'; +import { getNodesAndEdgesUnderParent } from './workflow_to_template_utils'; /** * Collection of utility fns to extract @@ -37,7 +39,12 @@ export function getIndexName(workflow: Workflow): string | undefined { // persist the same values to use during ingest and search, so we keep the naming general export function getNeuralSearchValues( workflow: Workflow -): { modelId: string; inputField: string; vectorField: string } { +): { + modelId: string; + inputField: string; + vectorField: string; + searchPipelineId?: string; +} { const modelId = getModelId(workflow) as string; const transformerComponent = getTransformerComponent( workflow @@ -45,7 +52,13 @@ export function getNeuralSearchValues( const { inputField, vectorField } = componentDataToFormik( transformerComponent.data ) as { inputField: string; vectorField: string }; - return { modelId, inputField, vectorField }; + + const searchPipelineId = workflow.resourcesCreated?.find( + (resource) => + resource.stepType === WORKFLOW_STEP_TYPE.CREATE_SEARCH_PIPELINE_STEP_TYPE + )?.id; + + return { modelId, inputField, vectorField, searchPipelineId }; } function getFormValues(workflow: Workflow): WorkspaceFormValues | undefined { @@ -72,8 +85,8 @@ function getModelId(workflow: Workflow): string | undefined { if (model.category === MODEL_CATEGORY.PRETRAINED) { const modelResource = workflow.resourcesCreated?.find( (resource) => resource.type === WORKFLOW_RESOURCE_TYPE.MODEL_ID - ) as WorkflowResource; - return modelResource.id; + ); + return modelResource?.id; } else { return model.id; } @@ -85,7 +98,8 @@ function getTransformerComponent( workflow: Workflow ): ReactFlowComponent | undefined { if (workflow?.ui_metadata?.workspace_flow) { - const { ingestNodes } = getIngestNodesAndEdges( + const { nodes: ingestNodes } = getNodesAndEdgesUnderParent( + NODE_CATEGORY.INGEST_GROUP, workflow?.ui_metadata?.workspace_flow?.nodes, workflow?.ui_metadata?.workspace_flow?.edges ); @@ -99,7 +113,8 @@ function getIndexerComponent( workflow: Workflow ): ReactFlowComponent | undefined { if (workflow?.ui_metadata?.workspace_flow) { - const { ingestNodes } = getIngestNodesAndEdges( + const { nodes: ingestNodes } = getNodesAndEdgesUnderParent( + NODE_CATEGORY.INGEST_GROUP, workflow?.ui_metadata?.workspace_flow?.nodes, workflow?.ui_metadata?.workspace_flow?.edges ); diff --git a/public/pages/workflow_detail/utils/workflow_to_template_utils.ts b/public/pages/workflow_detail/utils/workflow_to_template_utils.ts index 534151d5..ce356a3d 100644 --- a/public/pages/workflow_detail/utils/workflow_to_template_utils.ts +++ b/public/pages/workflow_detail/utils/workflow_to_template_utils.ts @@ -11,8 +11,6 @@ import { NODE_CATEGORY, TemplateNode, COMPONENT_CLASS, - CREATE_INGEST_PIPELINE_STEP_TYPE, - CREATE_INDEX_STEP_TYPE, CreateIngestPipelineNode, TextEmbeddingProcessor, componentDataToFormik, @@ -27,14 +25,14 @@ import { ROBERTA_SENTENCE_TRANSFORMER, MPNET_SENTENCE_TRANSFORMER, BERT_SENTENCE_TRANSFORMER, - REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE, generateId, NEURAL_SPARSE_TRANSFORMER, NEURAL_SPARSE_DOC_TRANSFORMER, NEURAL_SPARSE_TOKENIZER_TRANSFORMER, - REGISTER_LOCAL_SPARSE_ENCODING_MODEL_STEP_TYPE, SparseEncodingProcessor, IndexMappings, + CreateSearchPipelineNode, + WORKFLOW_STEP_TYPE, } from '../../../../common'; /** @@ -45,11 +43,7 @@ import { export function toTemplateFlows( workspaceFlow: WorkspaceFlowState ): TemplateFlows { - const { ingestNodes, ingestEdges } = getIngestNodesAndEdges( - workspaceFlow.nodes, - workspaceFlow.edges - ); - const provisionFlow = toProvisionTemplateFlow(ingestNodes, ingestEdges); + const provisionFlow = toProvisionTemplateFlow(workspaceFlow); // TODO: support beyond provision return { @@ -57,36 +51,51 @@ export function toTemplateFlows( }; } -export function getIngestNodesAndEdges( +export function getNodesAndEdgesUnderParent( + parentGroup: NODE_CATEGORY, allNodes: ReactFlowComponent[], allEdges: ReactFlowEdge[] -): { ingestNodes: ReactFlowComponent[]; ingestEdges: ReactFlowEdge[] } { - const ingestParentId = allNodes.find( - (node) => node.type === NODE_CATEGORY.INGEST_GROUP - )?.id as string; - const ingestNodes = allNodes.filter( - (node) => node.parentNode === ingestParentId - ); - const ingestIds = ingestNodes.map((node) => node.id); - const ingestEdges = allEdges.filter( - (edge) => ingestIds.includes(edge.source) || ingestIds.includes(edge.target) +): { nodes: ReactFlowComponent[]; edges: ReactFlowEdge[] } { + const parentId = allNodes.find((node) => node.type === parentGroup) + ?.id as string; + const nodes = allNodes.filter((node) => node.parentNode === parentId); + const nodeIds = nodes.map((node) => node.id); + const edges = allEdges.filter( + (edge) => nodeIds.includes(edge.source) || nodeIds.includes(edge.target) ); return { - ingestNodes, - ingestEdges, + nodes, + edges, }; } // Generates the end-to-end provision subflow, if applicable function toProvisionTemplateFlow( - nodes: ReactFlowComponent[], - edges: ReactFlowEdge[] + workspaceFlow: WorkspaceFlowState ): TemplateFlow { + const { + nodes: ingestNodes, + edges: ingestEdges, + } = getNodesAndEdgesUnderParent( + NODE_CATEGORY.INGEST_GROUP, + workspaceFlow.nodes, + workspaceFlow.edges + ); + const { + nodes: searchNodes, + edges: searchEdges, + } = getNodesAndEdgesUnderParent( + NODE_CATEGORY.SEARCH_GROUP, + workspaceFlow.nodes, + workspaceFlow.edges + ); + + // INGEST: iterate through nodes/edges and generate the valid template nodes const prevNodes = [] as ReactFlowComponent[]; const finalTemplateNodes = [] as TemplateNode[]; const templateEdges = [] as TemplateEdge[]; - nodes.forEach((node) => { - const templateNodes = toTemplateNodes(node, prevNodes, edges); + ingestNodes.forEach((node) => { + const templateNodes = toTemplateNodes(node, prevNodes, ingestEdges); // it may be undefined if the node is not convertible for some reason if (templateNodes) { finalTemplateNodes.push(...templateNodes); @@ -94,7 +103,7 @@ function toProvisionTemplateFlow( } }); - edges.forEach((edge) => { + ingestEdges.forEach((edge) => { // it may be undefined if the edge is not convertible // (e.g., connecting to some meta/other UI component, like "document" or "query") const templateEdge = toTemplateEdge(edge); @@ -103,6 +112,16 @@ function toProvisionTemplateFlow( } }); + // SEARCH: iterate through nodes/edges and generate the valid template nodes + // TODO: currently the scope is limited to only expecting a single search processor + // node, and hence logic is hardcoded to return a single CreateSearchPipelineNode + searchNodes.forEach((node) => { + if (node.data.baseClasses?.includes(COMPONENT_CLASS.RESULTS_TRANSFORMER)) { + const templateNode = resultsTransformerToTemplateNode(node); + finalTemplateNodes.push(templateNode); + } + }); + return { nodes: finalTemplateNodes, edges: templateEdges, @@ -157,8 +176,8 @@ function transformerToTemplateNodes( // register model workflow step type is different per use case const registerModelStepType = flowNode.data.type === COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER - ? REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE - : REGISTER_LOCAL_SPARSE_ENCODING_MODEL_STEP_TYPE; + ? WORKFLOW_STEP_TYPE.REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE + : WORKFLOW_STEP_TYPE.REGISTER_LOCAL_SPARSE_ENCODING_MODEL_STEP_TYPE; let registerModelStep = undefined as | RegisterPretrainedModelNode @@ -224,7 +243,7 @@ function transformerToTemplateNodes( const createIngestPipelineStep = { id: flowNode.data.id, - type: CREATE_INGEST_PIPELINE_STEP_TYPE, + type: WORKFLOW_STEP_TYPE.CREATE_INGEST_PIPELINE_STEP_TYPE, user_inputs: { pipeline_id: ingestPipelineName, model_id: finalModelId, @@ -307,7 +326,7 @@ function indexerToTemplateNode( return { id: flowNode.data.id, - type: CREATE_INDEX_STEP_TYPE, + type: WORKFLOW_STEP_TYPE.CREATE_INDEX_STEP_TYPE, previous_node_inputs: { [directlyConnectedNode.id]: 'pipeline_id', }, @@ -325,6 +344,38 @@ function indexerToTemplateNode( } } +// General fn to process all result transformer nodes. +// TODO: currently hardcoding to return a static configuration of a normalization +// phase results processor. Should make dynamic & generic +function resultsTransformerToTemplateNode( + flowNode: ReactFlowComponent +): CreateSearchPipelineNode { + return { + id: flowNode.data.id, + type: WORKFLOW_STEP_TYPE.CREATE_SEARCH_PIPELINE_STEP_TYPE, + user_inputs: { + pipeline_id: generateId('search_pipeline'), + configurations: { + phase_results_processors: [ + { + ['normalization-processor']: { + normalization: { + technique: 'min_max', + }, + combination: { + technique: 'arithmetic_mean', + parameters: { + weights: `[0.3, 0.7]`, + }, + }, + }, + }, + ], + }, + }, + } as CreateSearchPipelineNode; +} + // Fetch all directly connected predecessor nodes function getDirectlyConnectedNodes( node: ReactFlowComponent, diff --git a/public/pages/workflow_detail/workspace/reactflow-styles.scss b/public/pages/workflow_detail/workspace/reactflow-styles.scss index c4fc599d..d95b4eaa 100644 --- a/public/pages/workflow_detail/workspace/reactflow-styles.scss +++ b/public/pages/workflow_detail/workspace/reactflow-styles.scss @@ -19,7 +19,7 @@ $handle-color-invalid: $euiColorDanger; .reactflow-workspace .react-flow__node { width: 300px; - height: 250px; + min-height: 100px; } .reactflow__group-node { @@ -38,6 +38,10 @@ $handle-color-invalid: $euiColorDanger; // Overriding the styling for the reactflow node when it is selected. // We need to use important tag to override ReactFlow's wrapNode that sets the box-shadow. // Ref: https://github.com/wbkd/react-flow/blob/main/packages/core/src/components/Nodes/wrapNode.tsx#L187 +// TODO: when the node sizing is dynamic (e.g., 'min-height', it causes several issues: +// 1. the shadow only covers the min height instead of the node's final rendered height +// 2. the bounding edges of the parent node only fit with the 'min-height' amount, causing +// the node to look like it can be drug slightly out of the parent node .reactflow-workspace .react-flow__node-custom.selected { box-shadow: 0 0 2px 2px rgba(0, 0, 0, 0.5); border-radius: 5px; diff --git a/public/pages/workflow_detail/workspace/workspace_components/workspace_component.tsx b/public/pages/workflow_detail/workspace/workspace_components/workspace_component.tsx index d7a0d637..8f89562e 100644 --- a/public/pages/workflow_detail/workspace/workspace_components/workspace_component.tsx +++ b/public/pages/workflow_detail/workspace/workspace_components/workspace_component.tsx @@ -10,6 +10,7 @@ import { EuiCard, EuiText, EuiTitle, + EuiSpacer, } from '@elastic/eui'; import { setDirty, useAppDispatch } from '../../../../store'; import { IComponentData } from '../../../../component_types'; @@ -32,6 +33,9 @@ interface WorkspaceComponentProps { export function WorkspaceComponent(props: WorkspaceComponentProps) { const dispatch = useAppDispatch(); const component = props.data; + // TODO: remove hardcoded logic that only create fields are allowed + const containsFormFields = + props.data.createFields !== undefined && props.data.createFields.length > 0; const reactFlowInstance = useReactFlow(); // TODO: re-enable deletion @@ -50,9 +54,12 @@ export function WorkspaceComponent(props: WorkspaceComponentProps) { dispatch(setDirty()); }; + const backgroundColor = containsFormFields ? '#172430' : '#0A121A'; + return ( @@ -76,11 +83,12 @@ export function WorkspaceComponent(props: WorkspaceComponentProps) { } > - + {component.description} + {component.inputs?.map((input, index) => { return ( diff --git a/public/pages/workflows/new_workflow/utils.ts b/public/pages/workflows/new_workflow/utils.ts index 3c43526c..9fd7deb4 100644 --- a/public/pages/workflows/new_workflow/utils.ts +++ b/public/pages/workflows/new_workflow/utils.ts @@ -21,6 +21,9 @@ import { DEFAULT_NEW_WORKFLOW_NAME, Document, SparseEncoderTransformer, + NeuralQuery, + MatchQuery, + NormalizationTransformer, } from '../../../../common'; // Fn to produce the complete preset template with all necessary UI metadata. @@ -40,6 +43,10 @@ export function enrichPresetWorkflowWithUiMetadata( workspaceFlowState = fetchNeuralSparseSearchWorkspaceFlow(); break; } + case USE_CASE.HYBRID_SEARCH: { + workspaceFlowState = fetchHybridSearchWorkspaceFlow(); + break; + } default: { workspaceFlowState = fetchEmptyWorkspaceFlow(); break; @@ -67,9 +74,14 @@ function fetchSemanticSearchWorkspaceFlow(): WorkspaceFlowState { const ingestId1 = generateId(COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER); const ingestId2 = generateId(COMPONENT_CLASS.KNN_INDEXER); const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST); - // const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH); + const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH); + const searchId0 = generateId(COMPONENT_CLASS.NEURAL_QUERY); + const searchId1 = generateId(COMPONENT_CLASS.SPARSE_ENCODER_TRANSFORMER); + const searchId2 = generateId(COMPONENT_CLASS.KNN_INDEXER); const edgeId0 = generateId('edge'); const edgeId1 = generateId('edge'); + const edgeId2 = generateId('edge'); + const edgeId3 = generateId('edge'); const ingestNodes = [ { @@ -83,7 +95,7 @@ function fetchSemanticSearchWorkspaceFlow(): WorkspaceFlowState { }, className: 'reactflow__group-node__ingest', selectable: true, - draggable: false, + draggable: true, deletable: false, }, { @@ -93,7 +105,7 @@ function fetchSemanticSearchWorkspaceFlow(): WorkspaceFlowState { type: NODE_CATEGORY.CUSTOM, parentNode: ingestGroupId, extent: 'parent', - draggable: false, + draggable: true, deletable: false, }, { @@ -106,7 +118,7 @@ function fetchSemanticSearchWorkspaceFlow(): WorkspaceFlowState { type: NODE_CATEGORY.CUSTOM, parentNode: ingestGroupId, extent: 'parent', - draggable: false, + draggable: true, deletable: false, }, { @@ -116,28 +128,59 @@ function fetchSemanticSearchWorkspaceFlow(): WorkspaceFlowState { type: NODE_CATEGORY.CUSTOM, parentNode: ingestGroupId, extent: 'parent', - draggable: false, + draggable: true, + deletable: false, + }, + ] as ReactFlowComponent[]; + const searchNodes = [ + { + id: searchGroupId, + position: { x: 400, y: 1000 }, + type: NODE_CATEGORY.SEARCH_GROUP, + data: { label: COMPONENT_CATEGORY.SEARCH }, + style: { + width: 1300, + height: 400, + }, + className: 'reactflow__group-node__search', + selectable: true, + draggable: true, + deletable: false, + }, + { + id: searchId0, + position: { x: 100, y: 70 }, + data: initComponentData(new NeuralQuery().toObj(), searchId0), + type: NODE_CATEGORY.CUSTOM, + parentNode: searchGroupId, + extent: 'parent', + draggable: true, + deletable: false, + }, + { + id: searchId1, + position: { x: 500, y: 70 }, + data: initComponentData( + new TextEmbeddingTransformer().toPlaceholderObj(), + searchId1 + ), + type: NODE_CATEGORY.CUSTOM, + parentNode: searchGroupId, + extent: 'parent', + draggable: true, + deletable: false, + }, + { + id: searchId2, + position: { x: 900, y: 70 }, + data: initComponentData(new KnnIndexer().toPlaceholderObj(), searchId2), + type: NODE_CATEGORY.CUSTOM, + parentNode: searchGroupId, + extent: 'parent', + draggable: true, deletable: false, }, ] as ReactFlowComponent[]; - - // const searchNodes = [ - // { - // id: searchGroupId, - // position: { x: 400, y: 1000 }, - // type: NODE_CATEGORY.SEARCH_GROUP, - // data: { label: COMPONENT_CATEGORY.SEARCH }, - // style: { - // width: 900, - // height: 400, - // }, - // className: 'reactflow__group-node__search', - // selectable: true, - // draggable: false, - // deletable: false, - // }, - // ] as ReactFlowComponent[]; - const searchNodes = [] as ReactFlowComponent[]; return { nodes: [...ingestNodes, ...searchNodes], @@ -176,6 +219,44 @@ function fetchSemanticSearchWorkspaceFlow(): WorkspaceFlowState { zIndex: 2, deletable: false, }, + { + id: edgeId2, + key: edgeId2, + source: searchId0, + target: searchId1, + sourceClasses: ingestNodes.find((node) => node.id === searchId0)?.data + .baseClasses, + targetClasses: ingestNodes.find((node) => node.id === searchId1)?.data + .baseClasses, + sourceHandle: COMPONENT_CLASS.QUERY, + targetHandle: COMPONENT_CLASS.QUERY, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, + { + id: edgeId3, + key: edgeId3, + source: searchId1, + target: searchId2, + sourceClasses: ingestNodes.find((node) => node.id === searchId1)?.data + .baseClasses, + targetClasses: ingestNodes.find((node) => node.id === searchId2)?.data + .baseClasses, + sourceHandle: COMPONENT_CLASS.QUERY, + targetHandle: COMPONENT_CLASS.QUERY, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, ] as ReactFlowEdge[], }; } @@ -185,8 +266,14 @@ function fetchNeuralSparseSearchWorkspaceFlow(): WorkspaceFlowState { const ingestId1 = generateId(COMPONENT_CLASS.SPARSE_ENCODER_TRANSFORMER); const ingestId2 = generateId(COMPONENT_CLASS.KNN_INDEXER); const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST); + const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH); + const searchId0 = generateId(COMPONENT_CLASS.NEURAL_QUERY); + const searchId1 = generateId(COMPONENT_CLASS.SPARSE_ENCODER_TRANSFORMER); + const searchId2 = generateId(COMPONENT_CLASS.KNN_INDEXER); const edgeId0 = generateId('edge'); const edgeId1 = generateId('edge'); + const edgeId2 = generateId('edge'); + const edgeId3 = generateId('edge'); const ingestNodes = [ { @@ -200,7 +287,7 @@ function fetchNeuralSparseSearchWorkspaceFlow(): WorkspaceFlowState { }, className: 'reactflow__group-node__ingest', selectable: true, - draggable: false, + draggable: true, deletable: false, }, { @@ -210,7 +297,7 @@ function fetchNeuralSparseSearchWorkspaceFlow(): WorkspaceFlowState { type: NODE_CATEGORY.CUSTOM, parentNode: ingestGroupId, extent: 'parent', - draggable: false, + draggable: true, deletable: false, }, { @@ -223,7 +310,204 @@ function fetchNeuralSparseSearchWorkspaceFlow(): WorkspaceFlowState { type: NODE_CATEGORY.CUSTOM, parentNode: ingestGroupId, extent: 'parent', - draggable: false, + draggable: true, + deletable: false, + }, + { + id: ingestId2, + position: { x: 900, y: 70 }, + data: initComponentData(new KnnIndexer().toObj(), ingestId2), + type: NODE_CATEGORY.CUSTOM, + parentNode: ingestGroupId, + extent: 'parent', + draggable: true, + deletable: false, + }, + ] as ReactFlowComponent[]; + + const searchNodes = [ + { + id: searchGroupId, + position: { x: 400, y: 1000 }, + type: NODE_CATEGORY.SEARCH_GROUP, + data: { label: COMPONENT_CATEGORY.SEARCH }, + style: { + width: 1300, + height: 400, + }, + className: 'reactflow__group-node__search', + selectable: true, + draggable: true, + deletable: false, + }, + { + id: searchId0, + position: { x: 100, y: 70 }, + data: initComponentData(new NeuralQuery().toObj(), searchId0), + type: NODE_CATEGORY.CUSTOM, + parentNode: searchGroupId, + extent: 'parent', + draggable: true, + deletable: false, + }, + { + id: searchId1, + position: { x: 500, y: 70 }, + data: initComponentData( + new SparseEncoderTransformer().toPlaceholderObj(), + searchId1 + ), + type: NODE_CATEGORY.CUSTOM, + parentNode: searchGroupId, + extent: 'parent', + draggable: true, + deletable: false, + }, + { + id: searchId2, + position: { x: 900, y: 70 }, + data: initComponentData(new KnnIndexer().toPlaceholderObj(), searchId2), + type: NODE_CATEGORY.CUSTOM, + parentNode: searchGroupId, + extent: 'parent', + draggable: true, + deletable: false, + }, + ] as ReactFlowComponent[]; + + return { + nodes: [...ingestNodes, ...searchNodes], + edges: [ + { + id: edgeId0, + key: edgeId0, + source: ingestId0, + target: ingestId1, + sourceClasses: ingestNodes.find((node) => node.id === ingestId0)?.data + .baseClasses, + targetClasses: ingestNodes.find((node) => node.id === ingestId1)?.data + .baseClasses, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, + { + id: edgeId1, + key: edgeId1, + source: ingestId1, + target: ingestId2, + sourceClasses: ingestNodes.find((node) => node.id === ingestId1)?.data + .baseClasses, + targetClasses: ingestNodes.find((node) => node.id === ingestId2)?.data + .baseClasses, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, + { + id: edgeId2, + key: edgeId2, + source: searchId0, + target: searchId1, + sourceClasses: ingestNodes.find((node) => node.id === searchId0)?.data + .baseClasses, + targetClasses: ingestNodes.find((node) => node.id === searchId1)?.data + .baseClasses, + sourceHandle: COMPONENT_CLASS.QUERY, + targetHandle: COMPONENT_CLASS.QUERY, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, + { + id: edgeId3, + key: edgeId3, + source: searchId1, + target: searchId2, + sourceClasses: ingestNodes.find((node) => node.id === searchId1)?.data + .baseClasses, + targetClasses: ingestNodes.find((node) => node.id === searchId2)?.data + .baseClasses, + sourceHandle: COMPONENT_CLASS.QUERY, + targetHandle: COMPONENT_CLASS.QUERY, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, + ] as ReactFlowEdge[], + }; +} + +function fetchHybridSearchWorkspaceFlow(): WorkspaceFlowState { + const ingestId0 = generateId(COMPONENT_CLASS.DOCUMENT); + const ingestId1 = generateId(COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER); + const ingestId2 = generateId(COMPONENT_CLASS.KNN_INDEXER); + const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST); + const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH); + const searchId0 = generateId(COMPONENT_CLASS.MATCH_QUERY); + const searchId1 = generateId(COMPONENT_CLASS.NEURAL_QUERY); + const searchId2 = generateId(COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER); + const searchId3 = generateId(COMPONENT_CLASS.KNN_INDEXER); + const searchId4 = generateId(COMPONENT_CLASS.NORMALIZATION_TRANSFORMER); + const edgeId0 = generateId('edge'); + const edgeId1 = generateId('edge'); + const edgeId2 = generateId('edge'); + const edgeId3 = generateId('edge'); + const edgeId4 = generateId('edge'); + const edgeId5 = generateId('edge'); + + const ingestNodes = [ + { + id: ingestGroupId, + position: { x: 400, y: 400 }, + type: NODE_CATEGORY.INGEST_GROUP, + data: { label: COMPONENT_CATEGORY.INGEST }, + style: { + width: 1300, + height: 400, + }, + className: 'reactflow__group-node__ingest', + selectable: true, + draggable: true, + deletable: false, + }, + { + id: ingestId0, + position: { x: 100, y: 70 }, + data: initComponentData(new Document().toObj(), ingestId0), + type: NODE_CATEGORY.CUSTOM, + parentNode: ingestGroupId, + extent: 'parent', + draggable: true, + deletable: false, + }, + { + id: ingestId1, + position: { x: 500, y: 70 }, + data: initComponentData( + new TextEmbeddingTransformer().toObj(), + ingestId1 + ), + type: NODE_CATEGORY.CUSTOM, + parentNode: ingestGroupId, + extent: 'parent', + draggable: true, deletable: false, }, { @@ -233,12 +517,83 @@ function fetchNeuralSparseSearchWorkspaceFlow(): WorkspaceFlowState { type: NODE_CATEGORY.CUSTOM, parentNode: ingestGroupId, extent: 'parent', - draggable: false, + draggable: true, deletable: false, }, ] as ReactFlowComponent[]; - const searchNodes = [] as ReactFlowComponent[]; + const searchNodes = [ + { + id: searchGroupId, + position: { x: 400, y: 1000 }, + type: NODE_CATEGORY.SEARCH_GROUP, + data: { label: COMPONENT_CATEGORY.SEARCH }, + style: { + width: 1700, + height: 600, + }, + className: 'reactflow__group-node__search', + selectable: true, + draggable: true, + deletable: false, + }, + { + id: searchId0, + position: { x: 100, y: 70 }, + data: initComponentData(new NeuralQuery().toObj(), searchId0), + type: NODE_CATEGORY.CUSTOM, + parentNode: searchGroupId, + extent: 'parent', + draggable: true, + deletable: false, + }, + { + id: searchId1, + position: { x: 100, y: 370 }, + data: initComponentData(new MatchQuery().toObj(), searchId1), + type: NODE_CATEGORY.CUSTOM, + parentNode: searchGroupId, + extent: 'parent', + draggable: true, + deletable: false, + }, + { + id: searchId2, + position: { x: 500, y: 70 }, + data: initComponentData( + new TextEmbeddingTransformer().toPlaceholderObj(), + searchId2 + ), + type: NODE_CATEGORY.CUSTOM, + parentNode: searchGroupId, + extent: 'parent', + draggable: true, + deletable: false, + }, + { + id: searchId3, + position: { x: 900, y: 200 }, + data: initComponentData(new KnnIndexer().toPlaceholderObj(), searchId3), + type: NODE_CATEGORY.CUSTOM, + parentNode: searchGroupId, + extent: 'parent', + draggable: true, + deletable: false, + }, + { + id: searchId4, + position: { x: 1300, y: 200 }, + data: initComponentData( + new NormalizationTransformer().toObj(), + searchId4 + ), + type: NODE_CATEGORY.CUSTOM, + parentNode: searchGroupId, + extent: 'parent', + draggable: true, + deletable: false, + }, + ] as ReactFlowComponent[]; return { nodes: [...ingestNodes, ...searchNodes], @@ -269,6 +624,80 @@ function fetchNeuralSparseSearchWorkspaceFlow(): WorkspaceFlowState { .baseClasses, targetClasses: ingestNodes.find((node) => node.id === ingestId2)?.data .baseClasses, + targetHandle: COMPONENT_CLASS.DOCUMENT, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, + { + id: edgeId2, + key: edgeId2, + source: searchId0, + target: searchId2, + sourceClasses: searchNodes.find((node) => node.id === searchId0)?.data + .baseClasses, + targetClasses: searchNodes.find((node) => node.id === searchId2)?.data + .baseClasses, + targetHandle: COMPONENT_CLASS.QUERY, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, + { + id: edgeId3, + key: edgeId3, + source: searchId2, + target: searchId3, + sourceClasses: searchNodes.find((node) => node.id === searchId2)?.data + .baseClasses, + targetClasses: searchNodes.find((node) => node.id === searchId3)?.data + .baseClasses, + sourceHandle: COMPONENT_CLASS.QUERY, + targetHandle: COMPONENT_CLASS.QUERY, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, + { + id: edgeId4, + key: edgeId4, + source: searchId1, + target: searchId3, + sourceClasses: searchNodes.find((node) => node.id === searchId1)?.data + .baseClasses, + targetClasses: searchNodes.find((node) => node.id === searchId3)?.data + .baseClasses, + targetHandle: COMPONENT_CLASS.QUERY, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, + { + id: edgeId5, + key: edgeId5, + source: searchId3, + target: searchId4, + sourceClasses: searchNodes.find((node) => node.id === searchId3)?.data + .baseClasses, + targetClasses: searchNodes.find((node) => node.id === searchId4)?.data + .baseClasses, + targetHandle: COMPONENT_CLASS.RESULTS, markerEnd: { type: MarkerType.ArrowClosed, width: 20, diff --git a/public/route_service.ts b/public/route_service.ts index ce57599a..782de01f 100644 --- a/public/route_service.ts +++ b/public/route_service.ts @@ -42,7 +42,11 @@ export interface RouteService { deleteWorkflow: (workflowId: string) => Promise; getWorkflowPresets: () => Promise; catIndices: (pattern: string) => Promise; - searchIndex: (index: string, body: {}) => Promise; + searchIndex: ( + index: string, + body: {}, + searchPipeline?: string + ) => Promise; ingest: (index: string, doc: {}) => Promise; searchModels: (body: {}) => Promise; } @@ -161,14 +165,15 @@ export function configureRoutes(core: CoreStart): RouteService { return e as HttpFetchError; } }, - searchIndex: async (index: string, body: {}) => { + searchIndex: async (index: string, body: {}, searchPipeline?: string) => { try { - const response = await core.http.post<{ respString: string }>( - `${SEARCH_INDEX_NODE_API_PATH}/${index}`, - { - body: JSON.stringify(body), - } - ); + const basePath = `${SEARCH_INDEX_NODE_API_PATH}/${index}`; + const path = searchPipeline + ? `${basePath}/${searchPipeline}` + : basePath; + const response = await core.http.post<{ respString: string }>(path, { + body: JSON.stringify(body), + }); return response; } catch (e: any) { return e as HttpFetchError; diff --git a/public/store/reducers/opensearch_reducer.ts b/public/store/reducers/opensearch_reducer.ts index 10735776..a9f71edc 100644 --- a/public/store/reducers/opensearch_reducer.ts +++ b/public/store/reducers/opensearch_reducer.ts @@ -39,11 +39,15 @@ export const catIndices = createAsyncThunk( export const searchIndex = createAsyncThunk( SEARCH_INDEX_ACTION, - async (searchIndexInfo: { index: string; body: {} }, { rejectWithValue }) => { - const { index, body } = searchIndexInfo; + async ( + searchIndexInfo: { index: string; body: {}; searchPipeline?: string }, + { rejectWithValue } + ) => { + const { index, body, searchPipeline } = searchIndexInfo; const response: any | HttpFetchError = await getRouteService().searchIndex( index, - body + body, + searchPipeline ); if (response instanceof HttpFetchError) { return rejectWithValue('Error searching index: ' + response.body.message); diff --git a/public/utils/constants.ts b/public/utils/constants.ts index 8450fd2b..a248b33d 100644 --- a/public/utils/constants.ts +++ b/public/utils/constants.ts @@ -50,8 +50,14 @@ export enum COMPONENT_CLASS { ML_TRANSFORMER = 'ml_transformer', TEXT_EMBEDDING_TRANSFORMER = 'text_embedding_transformer', SPARSE_ENCODER_TRANSFORMER = 'sparse_encoder_transformer', + RESULTS_TRANSFORMER = 'results_transformer', + NORMALIZATION_TRANSFORMER = 'normalization_transformer', // Query-related classes QUERY = 'query', + MATCH_QUERY = 'match_query', + NEURAL_QUERY = 'neural_query', // Document-related classes DOCUMENT = 'document', + // Results-related classes + RESULTS = 'results', } diff --git a/server/resources/templates/hybrid_search.json b/server/resources/templates/hybrid_search.json new file mode 100644 index 00000000..15be8c5e --- /dev/null +++ b/server/resources/templates/hybrid_search.json @@ -0,0 +1,12 @@ +{ + "name": "Hybrid Search", + "description": "A basic workflow containing the ingest pipeline, search pipeline, and index configurations for performing hybrid search", + "use_case": "HYBRID_SEARCH", + "version": { + "template": "1.0.0", + "compatibility": [ + "2.13.0", + "3.0.0" + ] + } +} \ No newline at end of file diff --git a/server/routes/helpers.ts b/server/routes/helpers.ts index afb404f7..6498b568 100644 --- a/server/routes/helpers.ts +++ b/server/routes/helpers.ts @@ -128,6 +128,7 @@ export function getResourcesCreatedFromResponse( resourcesCreated.forEach((backendResource) => { finalResources.push({ id: backendResource.resource_id, + stepType: backendResource.workflow_step_name, type: // @ts-ignore WORKFLOW_RESOURCE_TYPE[ diff --git a/server/routes/opensearch_routes_service.ts b/server/routes/opensearch_routes_service.ts index aae7577a..b1440f97 100644 --- a/server/routes/opensearch_routes_service.ts +++ b/server/routes/opensearch_routes_service.ts @@ -50,6 +50,19 @@ export function registerOpenSearchRoutes( }, opensearchRoutesService.searchIndex ); + router.post( + { + path: `${SEARCH_INDEX_NODE_API_PATH}/{index}/{search_pipeline}`, + validate: { + params: schema.object({ + index: schema.string(), + search_pipeline: schema.string(), + }), + body: schema.any(), + }, + }, + opensearchRoutesService.searchIndex + ); router.put( { path: `${INGEST_NODE_API_PATH}/{index}`, @@ -103,7 +116,10 @@ export class OpenSearchRoutesService { req: OpenSearchDashboardsRequest, res: OpenSearchDashboardsResponseFactory ): Promise> => { - const { index } = req.params as { index: string }; + const { index, search_pipeline } = req.params as { + index: string; + search_pipeline: string | undefined; + }; const body = req.body; try { const response = await this.client @@ -111,6 +127,7 @@ export class OpenSearchRoutesService { .callAsCurrentUser('search', { index, body, + search_pipeline, }); return res.ok({ body: response });