adding blue print doc for cohere multi-modal model (#3229)

* adding blue print doc Signed-off-by: Dhrubo Saha <[email protected]> * addressed comments Signed-off-by: Dhrubo Saha <[email protected]> * addressed comment Signed-off-by: Dhrubo Saha <[email protected]> --------- Signed-off-by: Dhrubo Saha <[email protected]>
opensearch-project · Nov 19, 2024 · de301b7 · de301b7
1 parent 622f73d
commit de301b7
Show file tree

Hide file tree

Showing 4 changed files with 332 additions and 8 deletions.
diff --git a/...ml/common/connector/functions/preprocess/CohereMultiModalEmbeddingPreProcessFunction.java b/...ml/common/connector/functions/preprocess/CohereMultiModalEmbeddingPreProcessFunction.java
@@ -25,23 +25,23 @@ public CohereMultiModalEmbeddingPreProcessFunction() {
     public void validate(MLInput mlInput) {
         validateTextDocsInput(mlInput);
         List<String> docs = ((TextDocsInputDataSet) mlInput.getInputDataset()).getDocs();
-        if (docs == null || docs.isEmpty() || (docs.size() == 1 && docs.get(0) == null)) {
+        if (docs == null || docs.isEmpty() || docs.get(0) == null) {
             throw new IllegalArgumentException("No image provided");
         }
     }
 
     @Override
     public RemoteInferenceInputDataSet process(MLInput mlInput) {
         TextDocsInputDataSet inputData = (TextDocsInputDataSet) mlInput.getInputDataset();
-        Map<String, String> parametersMap = new HashMap<>();
+        Map<String, Object> parametersMap = new HashMap<>();
 
         /**
          * Cohere multi-modal model expects either image or texts, not both.
          * For image, customer can use this pre-process function. For texts, customer can use
          * connector.pre_process.cohere.embedding
          * Cohere expects An array of image data URIs for the model to embed. Maximum number of images per call is 1.
          */
-        parametersMap.put("images", inputData.getDocs().get(0));
+        parametersMap.put("images", inputData.getDocs());
         return RemoteInferenceInputDataSet
             .builder()
             .parameters(convertScriptStringToJsonString(Map.of("parameters", parametersMap)))

diff --git a/...ommon/connector/functions/preprocess/CohereMultiModalEmbeddingPreProcessFunctionTest.java b/...ommon/connector/functions/preprocess/CohereMultiModalEmbeddingPreProcessFunctionTest.java
@@ -66,7 +66,7 @@ public void testProcess_whenCorrectInput_expectCorrectOutput() {
         MLInput mlInput = MLInput.builder().algorithm(FunctionName.TEXT_EMBEDDING).inputDataset(textDocsInputDataSet).build();
         RemoteInferenceInputDataSet dataSet = function.apply(mlInput);
         assertEquals(1, dataSet.getParameters().size());
-        assertEquals("imageString", dataSet.getParameters().get("images"));
+        assertEquals("[\"imageString\"]", dataSet.getParameters().get("images"));
 
     }
 

diff --git a/docs/remote_inference_blueprints/cohere_connector_embedding_blueprint.md b/docs/remote_inference_blueprints/cohere_connector_embedding_blueprint.md
@@ -1,6 +1,6 @@
 ### Cohere Embedding Connector Blueprint:
 
-This blueprint will show you how to connect a Cohere embedding model to your Opensearch cluster, including creating a k-nn index and your own Embedding pipeline. You will require a Cohere API key to create a connector.
+This blueprint will show you how to connect a Cohere embedding model to your OpenSearch cluster, including creating a k-nn index and your own Embedding pipeline. You will require a Cohere API key to create a connector.
 
 Cohere currently offers the following Embedding models (with model name and embedding dimensions). Note that only the following have been tested with the blueprint guide.
 
@@ -97,7 +97,7 @@ The last step is to deploy your model. Use the `model_id` returned by the regist
 POST /_plugins/_ml/models/<MODEL_ID>/_deploy
 ```
 
-This will once again spawn a task to deploy your Model, with a response that will look like:
+This will once again spawn a task to deploy your model, with a response that will look like:
 
 ```json
 {
@@ -113,11 +113,11 @@ You can run the GET tasks request again to verify the status.
 GET /_plugins/_ml/tasks/<TASK_ID>
 ```
 
-Once this is complete, your Model is deployed and ready!
+Once this is complete, your model is deployed and ready!
 
 ##### 1e. Test model
 
-You can try this request to test that the Model behaves correctly:
+You can try this request to test that the model behaves correctly:
 
 ```json
 POST /_plugins/_ml/models/<MODEL_ID_HERE>/_predict