Onboard hybrid search use case; add readonly search flows for all use…

… cases (#143) (#144)

Signed-off-by: Tyler Ohlsen <[email protected]>
(cherry picked from commit 73cbed4)

Co-authored-by: Tyler Ohlsen <[email protected]>

common/constants.ts

@@ -55,16 +55,6 @@ export const GET_PRESET_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH
 export const BASE_MODEL_NODE_API_PATH = `${BASE_NODE_API_PATH}/model`;
 export const SEARCH_MODELS_NODE_API_PATH = `${BASE_MODEL_NODE_API_PATH}/search`;
 
-/**
- * BACKEND INTERFACES
- */
-export const CREATE_INGEST_PIPELINE_STEP_TYPE = 'create_ingest_pipeline';
-export const CREATE_INDEX_STEP_TYPE = 'create_index';
-export const REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE =
-  'register_local_pretrained_model';
-export const REGISTER_LOCAL_SPARSE_ENCODING_MODEL_STEP_TYPE =
-  'register_local_sparse_encoding_model';
-
 /**
  * ML PLUGIN PRETRAINED MODELS
  * (based off of https://opensearch.org/docs/latest/ml-commons-plugin/pretrained-models)

common/interfaces.ts

@@ -43,9 +43,11 @@ export type WorkspaceFlowState = {
  ********** USE CASE TEMPLATE TYPES/INTERFACES **********
  */
 
-export type IngestProcessor = {
-  description?: string;
-};
+export type IngestProcessor = {};
+export type SearchProcessor = {};
+export type SearchRequestProcessor = SearchProcessor & {};
+export type SearchResponseProcessor = SearchProcessor & {};
+export type SearchPhaseResultsProcessor = SearchProcessor & {};
 
 export type TextEmbeddingProcessor = IngestProcessor & {
   text_embedding: {
@@ -61,6 +63,18 @@ export type SparseEncodingProcessor = IngestProcessor & {
   };
 };
 
+export type NormalizationProcessor = SearchProcessor & {
+  normalization: {
+    technique: string;
+  };
+  combination: {
+    technique: string;
+    parameters: {
+      weights: number[];
+    };
+  };
+};
+
 export type IndexConfiguration = {
   settings: {};
   mappings: IndexMappings;
@@ -90,6 +104,18 @@ export type CreateIngestPipelineNode = TemplateNode & {
   };
 };
 
+export type CreateSearchPipelineNode = TemplateNode & {
+  user_inputs: {
+    pipeline_id: string;
+    configurations: {
+      description?: string;
+      request_processors?: SearchRequestProcessor[];
+      response_processors?: SearchResponseProcessor[];
+      phase_results_processors?: SearchPhaseResultsProcessor[];
+    };
+  };
+};
+
 export type CreateIndexNode = TemplateNode & {
   previous_node_inputs?: {
     [ingest_pipeline_step_id: string]: string;
@@ -157,6 +183,7 @@ export type Workflow = WorkflowTemplate & {
 export enum USE_CASE {
   SEMANTIC_SEARCH = 'SEMANTIC_SEARCH',
   NEURAL_SPARSE_SEARCH = 'NEURAL_SPARSE_SEARCH',
+  HYBRID_SEARCH = 'HYBRID_SEARCH',
 }
 
 /**
@@ -264,6 +291,7 @@ export enum WORKFLOW_STATE {
 
 export type WorkflowResource = {
   id: string;
+  stepType: WORKFLOW_STEP_TYPE;
   type: WORKFLOW_RESOURCE_TYPE;
 };
 
@@ -276,6 +304,25 @@ export enum WORKFLOW_RESOURCE_TYPE {
   CONNECTOR_ID = 'Connector',
 }
 
+export enum WORKFLOW_STEP_TYPE {
+  CREATE_INGEST_PIPELINE_STEP_TYPE = 'create_ingest_pipeline',
+  CREATE_SEARCH_PIPELINE_STEP_TYPE = 'create_search_pipeline',
+  CREATE_INDEX_STEP_TYPE = 'create_index',
+  REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE = 'register_local_pretrained_model',
+  REGISTER_LOCAL_SPARSE_ENCODING_MODEL_STEP_TYPE = 'register_local_sparse_encoding_model',
+}
+
+// We cannot disambiguate ingest vs. search pipelines based on workflow resource type. To work around
+// that, we maintain this map from workflow step type -> formatted type
+export enum WORKFLOW_STEP_TO_RESOURCE_TYPE_MAP {
+  'create_ingest_pipeline' = 'Ingest pipeline',
+  'create_search_pipeline' = 'Search pipeline',
+  'create_index' = 'Index',
+  'register_local_pretrained_model' = 'Model',
+  'register_local_sparse_encoding_model' = 'Model',
+  'deploy_model' = 'Model',
+}
+
 export type WorkflowDict = {
   [workflowId: string]: Workflow;
 };

public/component_types/base_component.tsx

@@ -34,4 +34,15 @@ export abstract class BaseComponent implements IComponent {
   toObj() {
     return Object.assign({}, this);
   }
+
+  // Helper fn to strip all fields for a component if we want to view it in the UI
+  // but not have it tied to any form/inputs. Example: showing an Index component in search,
+  // even if it is provisioned in ingest.
+  toPlaceholderObj() {
+    return {
+      ...Object.assign({}, this),
+      createFields: [],
+      fields: [],
+    };
+  }
 }

public/component_types/indexer/indexer.ts

@@ -25,6 +25,12 @@ export class Indexer extends BaseComponent {
         baseClass: COMPONENT_CLASS.DOCUMENT,
         acceptMultiple: false,
       },
+      {
+        id: 'query',
+        label: 'Query',
+        baseClass: COMPONENT_CLASS.QUERY,
+        acceptMultiple: true,
+      },
     ];
     this.fields = [
       {
@@ -46,12 +52,11 @@ export class Indexer extends BaseComponent {
       //   placeholder: 'Enter an index mappings JSON blob...',
       // },
     ];
-    // this.outputs = [
-    //   {
-    //     label: this.label,
-    //     baseClasses: this.baseClasses,
-    //   },
-    // ];
-    this.outputs = [];
+    this.outputs = [
+      {
+        label: 'Results',
+        baseClasses: [COMPONENT_CLASS.RESULTS],
+      },
+    ];
   }
 }

public/component_types/other/index.ts

@@ -4,3 +4,5 @@
  */
 
 export * from './document';
+export * from './results';
+export * from './query';

public/component_types/other/query/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './match_query';
+export * from './neural_query';

public/component_types/other/query/match_query.tsx

@@ -0,0 +1,28 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { COMPONENT_CLASS } from '../../../utils';
+import { Query } from './query';
+
+/**
+ * A basic match query placeholder UI component.
+ * Does not have any functionality.
+ */
+export class MatchQuery extends Query {
+  constructor() {
+    super();
+    this.type = COMPONENT_CLASS.MATCH_QUERY;
+    this.label = 'Match Query';
+    this.description = 'An OpenSearch match query';
+    this.inputs = [];
+    this.baseClasses = [...this.baseClasses, this.type];
+    this.outputs = [
+      {
+        label: this.label,
+        baseClasses: this.baseClasses,
+      },
+    ];
+  }
+}

public/component_types/other/query/neural_query.tsx

@@ -0,0 +1,28 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { COMPONENT_CLASS } from '../../../utils';
+import { Query } from './query';
+
+/**
+ * A basic neural query placeholder UI component.
+ * Does not have any functionality.
+ */
+export class NeuralQuery extends Query {
+  constructor() {
+    super();
+    this.type = COMPONENT_CLASS.NEURAL_QUERY;
+    this.label = 'Neural query';
+    this.description = 'An OpenSearch neural query';
+    this.inputs = [];
+    this.baseClasses = [...this.baseClasses, this.type];
+    this.outputs = [
+      {
+        label: this.label,
+        baseClasses: this.baseClasses,
+      },
+    ];
+  }
+}

public/component_types/other/query/query.tsx

@@ -0,0 +1,25 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../../../utils';
+import { BaseComponent } from '../../base_component';
+
+/**
+ * A basic Query placeholder UI component.
+ * Does not have any functionality.
+ */
+export abstract class Query extends BaseComponent {
+  constructor() {
+    super();
+    this.type = COMPONENT_CLASS.QUERY;
+    this.label = 'Query';
+    this.description = 'An OpenSearch query';
+    this.categories = [COMPONENT_CATEGORY.SEARCH];
+    this.allowsCreation = false;
+    this.baseClasses = [this.type];
+    this.inputs = [];
+    this.outputs = [];
+  }
+}

public/component_types/other/results.tsx

@@ -0,0 +1,30 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../../utils';
+import { BaseComponent } from '../base_component';
+
+/**
+ * A basic Results placeholder UI component.
+ * Does not have any functionality.
+ */
+export class Results extends BaseComponent {
+  constructor() {
+    super();
+    this.type = COMPONENT_CLASS.RESULTS;
+    this.label = 'Results';
+    this.description = 'OpenSearch results';
+    this.categories = [COMPONENT_CATEGORY.SEARCH];
+    this.allowsCreation = false;
+    this.baseClasses = [this.type];
+    this.inputs = [];
+    this.outputs = [
+      {
+        label: this.label,
+        baseClasses: this.baseClasses,
+      },
+    ];
+  }
+}

public/component_types/transformer/index.ts

@@ -6,3 +6,5 @@
 export * from './ml_transformer';
 export * from './text_embedding_transformer';
 export * from './sparse_encoder_transformer';
+export * from './results_transformer';
+export * from './normalization_transformer';

public/component_types/transformer/normalization_transformer.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../../utils';
+import { ResultsTransformer } from './results_transformer';
+
+/**
+ * A normalization results transformer UI component
+ */
+export class NormalizationTransformer extends ResultsTransformer {
+  constructor() {
+    super();
+    (this.type = COMPONENT_CLASS.NORMALIZATION_TRANSFORMER),
+      (this.label = 'Normalization Transformer');
+    this.description = 'A transformer to normalize search results';
+    this.baseClasses = [...this.baseClasses, this.type];
+    this.categories = [COMPONENT_CATEGORY.SEARCH];
+  }
+}

public/component_types/transformer/results_transformer.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { COMPONENT_CLASS } from '../../utils';
+import { BaseTransformer } from './base_transformer';
+
+/**
+ * A generic results transformer UI component
+ */
+export class ResultsTransformer extends BaseTransformer {
+  constructor() {
+    super();
+    (this.type = COMPONENT_CLASS.RESULTS_TRANSFORMER),
+      (this.label = 'Results Transformer');
+    this.description = 'A general results transformer';
+    this.baseClasses = [...this.baseClasses, this.type];
+    this.inputs = [
+      {
+        id: 'results',
+        label: 'Results',
+        baseClass: COMPONENT_CLASS.RESULTS,
+        acceptMultiple: false,
+      },
+    ];
+    this.outputs = [
+      {
+        label: 'Transformed Results',
+        baseClasses: [COMPONENT_CLASS.RESULTS],
+      },
+    ];
+  }
+}

public/component_types/transformer/sparse_encoder_transformer.ts

@@ -25,6 +25,12 @@ export class SparseEncoderTransformer extends MLTransformer {
         baseClass: COMPONENT_CLASS.DOCUMENT,
         acceptMultiple: false,
       },
+      {
+        id: 'query',
+        label: 'Query',
+        baseClass: COMPONENT_CLASS.QUERY,
+        acceptMultiple: false,
+      },
     ];
     this.createFields = [
       {
@@ -59,6 +65,10 @@ export class SparseEncoderTransformer extends MLTransformer {
         label: 'Transformed Document',
         baseClasses: [COMPONENT_CLASS.DOCUMENT],
       },
+      {
+        label: 'Transformed Query',
+        baseClasses: [COMPONENT_CLASS.QUERY],
+      },
     ];
   }
 }

public/component_types/transformer/text_embedding_transformer.ts

@@ -24,6 +24,12 @@ export class TextEmbeddingTransformer extends MLTransformer {
         baseClass: COMPONENT_CLASS.DOCUMENT,
         acceptMultiple: false,
       },
+      {
+        id: 'query',
+        label: 'Query',
+        baseClass: COMPONENT_CLASS.QUERY,
+        acceptMultiple: false,
+      },
     ];
     this.createFields = [
       {
@@ -57,6 +63,10 @@ export class TextEmbeddingTransformer extends MLTransformer {
         label: 'Transformed Document',
         baseClasses: [COMPONENT_CLASS.DOCUMENT],
       },
+      {
+        label: 'Transformed Query',
+        baseClasses: [COMPONENT_CLASS.QUERY],
+      },
     ];
   }
 }