Merge pull request #46 from MuleSoft-AI-Chain-Project/develop

Develop

README.md

@@ -22,7 +22,7 @@ MAC Vectors provides access to a broad number of external Vector Stores and Data
 <dependency>
    <groupId>cloud.anypoint</groupId>
    <artifactId>mule-vectors-connector</artifactId>
-   <version>0.1.25</version>
+   <version>0.2.0</version>
    <classifier>mule-plugin</classifier>
 </dependency>
 ```
@@ -34,9 +34,9 @@ Then add the following dependency to your application's `pom.xml`:
 
 ```xml
 <dependency>
-    <groupId>com.mule.mulechain</groupId>
-    <artifactId>mulechain-vectors</artifactId>
-    <version>0.1.25</version>
+    <groupId>com.mulesoft.connectors</groupId>
+    <artifactId>mule4-vectors-connector</artifactId>
+    <version>0.2.0</version>
     <classifier>mule-plugin</classifier>
 </dependency>
 ```

pom.xml

@@ -6,7 +6,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.mulesoft.connectors</groupId>
     <artifactId>mule4-vectors-connector</artifactId>
-    <version>0.1.117-SNAPSHOT</version>
+    <version>0.1.122-SNAPSHOT</version>
     <packaging>mule-extension</packaging>
     <name>MuleSoft Vectors Connector - Mule 4</name>
     <description>MuleSoft Vectors Connector provides access to a broad number of external Vector Stores.</description>

src/main/java/org/mule/extension/mulechain/vectors/api/metadata/DocumentResponseAttributes.java

@@ -1,14 +1,24 @@
 package org.mule.extension.mulechain.vectors.api.metadata;
 
+import org.mule.runtime.extension.api.annotation.param.MediaType;
+
 import java.io.Serializable;
 import java.util.HashMap;
+import java.util.Map;
+
+import static org.mule.runtime.extension.api.annotation.param.MediaType.APPLICATION_JSON;
 
 public class DocumentResponseAttributes implements Serializable {
 
-  private final HashMap<String, String> documentAttributes;
+  private final HashMap<String, Object> requestAttributes;
 
-  public DocumentResponseAttributes(HashMap<String, String> documentAttributes) {
+  public DocumentResponseAttributes(HashMap<String, Object> requestAttributes) {
+
+    this.requestAttributes = requestAttributes;
+  }
 
-    this.documentAttributes = documentAttributes;
+  @MediaType(value = APPLICATION_JSON, strict = false)
+  public Map<String, Object> getRequestAttributes() {
+    return requestAttributes;
   }
 }

src/main/java/org/mule/extension/mulechain/vectors/api/metadata/EmbeddingResponseAttributes.java

@@ -1,14 +1,24 @@
 package org.mule.extension.mulechain.vectors.api.metadata;
 
+import org.mule.runtime.extension.api.annotation.param.MediaType;
+
 import java.io.Serializable;
 import java.util.HashMap;
+import java.util.Map;
+
+import static org.mule.runtime.extension.api.annotation.param.MediaType.APPLICATION_JSON;
 
 public class EmbeddingResponseAttributes  implements Serializable {
 
-  private final HashMap<String, String> embeddingAttributes;
+  private final HashMap<String, Object> requestAttributes;
 
-  public EmbeddingResponseAttributes(HashMap<String, String> embeddingAttributes) {
+  public EmbeddingResponseAttributes(HashMap<String, Object> requestAttributes) {
+
+    this.requestAttributes = requestAttributes;
+  }
 
-    this.embeddingAttributes = embeddingAttributes;
+  @MediaType(value = APPLICATION_JSON, strict = false)
+  public Map<String, Object> getRequestAttributes() {
+    return requestAttributes;
   }
 }

src/main/java/org/mule/extension/mulechain/vectors/internal/constant/Constants.java

@@ -17,10 +17,10 @@ private Constants() {}
   public static final String STORAGE_TYPE_AZURE_BLOB = "AZURE_BLOB";
 
   public static final String EMBEDDING_MODEL_SERVICE_OPENAI = "OPENAI";
+  public static final String EMBEDDING_MODEL_SERVICE_AZURE_OPENAI = "AZURE_OPENAI";
   public static final String EMBEDDING_MODEL_SERVICE_MISTRAL_AI = "MISTRAL_AI";
   public static final String EMBEDDING_MODEL_SERVICE_NOMIC = "NOMIC";
   public static final String EMBEDDING_MODEL_SERVICE_HUGGING_FACE = "HUGGING_FACE";
-  public static final String EMBEDDING_MODEL_SERVICE_AZURE_OPENAI = "AZURE_OPENAI";
 
   public static final String EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_3_SMALL = "text-embedding-3-small";
   public static final String EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_3_LARGE = "text-embedding-3-large";
@@ -66,6 +66,24 @@ private Constants() {}
   public static final String EMBEDDING_OPERATION_TYPE_REMOVE_EMBEDDINGS = "REMOVE_EMBEDDINGS";
   public static final String EMBEDDING_OPERATION_TYPE_QUERY_ALL = "QUERY_ALL";
 
+  public static final String JSON_KEY_SOURCES = "sources";
+  public static final String JSON_KEY_SEGMENTS = "segments";
+  public static final String JSON_KEY_SEGMENT_COUNT = "segmentCount";
+  public static final String JSON_KEY_SOURCE_COUNT = "sourceCount";
+  public static final String JSON_KEY_STORE_NAME = "storeName";
+  public static final String JSON_KEY_TEXT = "text";
+  public static final String JSON_KEY_STATUS = "status";
+  public static final String JSON_KEY_EMBEDDING = "embedding";
+  public static final String JSON_KEY_DIMENSIONS = "dimensions";
+  public static final String JSON_KEY_RESPONSE = "response";
+  public static final String JSON_KEY_QUESTION = "question";
+  public static final String JSON_KEY_MAX_RESULTS = "maxResults";
+  public static final String JSON_KEY_MIN_SCORE = "minScore";
+  public static final String JSON_KEY_EMBEDDING_ID = "embeddingId";
+  public static final String JSON_KEY_SCORE = "score";
+  public static final String JSON_KEY_METADATA = "metadata";
+  public static final String JSON_KEY_INDEX = "index";
+
   public static final String OPERATION_STATUS_ADDED = "added";
   public static final String OPERATION_STATUS_UPDATED = "updated";
   public static final String OPERATION_STATUS_DELETED = "deleted";

src/main/java/org/mule/extension/mulechain/vectors/internal/error/MuleVectorsErrorType.java

@@ -7,5 +7,5 @@
  */
 public enum MuleVectorsErrorType implements ErrorTypeDefinition<MuleVectorsErrorType> {
 
-  DOCUMENT_OPERATIONS_FAILURE, EMBEDDING_OPERATIONS_FAILURE
+  DOCUMENT_OPERATIONS_FAILURE, EMBEDDING_OPERATIONS_FAILURE, AI_SERVICES_FAILURE
 }

src/main/java/org/mule/extension/mulechain/vectors/internal/error/provider/EmbeddingErrorTypeProvider.java

@@ -11,13 +11,16 @@
 
 import static java.util.Arrays.asList;
 import static java.util.Collections.unmodifiableSet;
+import static org.mule.extension.mulechain.vectors.internal.error.MuleVectorsErrorType.AI_SERVICES_FAILURE;
 import static org.mule.extension.mulechain.vectors.internal.error.MuleVectorsErrorType.EMBEDDING_OPERATIONS_FAILURE;
 
 public class EmbeddingErrorTypeProvider implements ErrorTypeProvider {
 
   @SuppressWarnings("rawtypes")
   @Override
   public Set<ErrorTypeDefinition> getErrorTypes() {
-    return unmodifiableSet(new HashSet<>(asList(EMBEDDING_OPERATIONS_FAILURE)));
+    return unmodifiableSet(new HashSet<>(asList(
+        EMBEDDING_OPERATIONS_FAILURE,
+        AI_SERVICES_FAILURE)));
   }
 }

src/main/java/org/mule/extension/mulechain/vectors/internal/helper/ResponseHelper.java

@@ -18,10 +18,10 @@ private ResponseHelper() {
 
   public static Result<InputStream, EmbeddingResponseAttributes> createEmbeddingResponse(
       String response,
-      Map<String, String> embeddingAttributes) {
+      Map<String, Object> embeddingAttributes) {
 
     return Result.<InputStream, EmbeddingResponseAttributes>builder()
-        .attributes(new EmbeddingResponseAttributes((HashMap<String, String>) embeddingAttributes))
+        .attributes(new EmbeddingResponseAttributes((HashMap<String, Object>) embeddingAttributes))
         .attributesMediaType(org.mule.runtime.api.metadata.MediaType.APPLICATION_JAVA)
         .output(toInputStream(response, StandardCharsets.UTF_8))
         .mediaType(org.mule.runtime.api.metadata.MediaType.APPLICATION_JSON)
@@ -30,10 +30,10 @@ public static Result<InputStream, EmbeddingResponseAttributes> createEmbeddingRe
 
   public static Result<InputStream, DocumentResponseAttributes> createDocumentResponse(
       String response,
-      Map<String, String> documentAttributes) {
+      Map<String, Object> documentAttributes) {
 
     return Result.<InputStream, DocumentResponseAttributes>builder()
-        .attributes(new DocumentResponseAttributes((HashMap<String, String>) documentAttributes))
+        .attributes(new DocumentResponseAttributes((HashMap<String, Object>) documentAttributes))
         .attributesMediaType(org.mule.runtime.api.metadata.MediaType.APPLICATION_JAVA)
         .output(toInputStream(response, StandardCharsets.UTF_8))
         .mediaType(org.mule.runtime.api.metadata.MediaType.APPLICATION_JSON)

src/main/java/org/mule/extension/mulechain/vectors/internal/model/BaseModel.java

@@ -3,12 +3,15 @@
 import dev.langchain4j.model.embedding.EmbeddingModel;
 import org.mule.extension.mulechain.vectors.internal.config.Configuration;
 import org.mule.extension.mulechain.vectors.internal.constant.Constants;
+import org.mule.extension.mulechain.vectors.internal.error.MuleVectorsErrorType;
 import org.mule.extension.mulechain.vectors.internal.helper.parameter.EmbeddingModelParameters;
 import org.mule.extension.mulechain.vectors.internal.model.azureopenai.AzureOpenAIModel;
 import org.mule.extension.mulechain.vectors.internal.model.huggingface.HuggingFaceModel;
 import org.mule.extension.mulechain.vectors.internal.model.mistralai.MistralAIModel;
 import org.mule.extension.mulechain.vectors.internal.model.nomic.NomicModel;
 import org.mule.extension.mulechain.vectors.internal.model.openai.OpenAIModel;
+import org.mule.runtime.extension.api.exception.ModuleException;
+import org.mule.runtime.module.extension.internal.runtime.operation.IllegalOperationException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -82,8 +85,9 @@ public BaseModel build() {
           break;
 
         default:
-          //throw new IllegalOperationException("Unsupported Vector Store: " + configuration.getVectorStore());
-          baseModel = null;
+          throw new ModuleException(
+              String.format("Error while initializing embedding model service. \"%s\" is not supported.", configuration.getEmbeddingModelService()),
+              MuleVectorsErrorType.AI_SERVICES_FAILURE);
       }
       return baseModel;
     }

src/main/java/org/mule/extension/mulechain/vectors/internal/model/azureopenai/AzureOpenAIModel.java

@@ -4,6 +4,7 @@
 import dev.langchain4j.model.embedding.EmbeddingModel;
 import org.json.JSONObject;
 import org.mule.extension.mulechain.vectors.internal.config.Configuration;
+import org.mule.extension.mulechain.vectors.internal.constant.Constants;
 import org.mule.extension.mulechain.vectors.internal.helper.parameter.EmbeddingModelParameters;
 import org.mule.extension.mulechain.vectors.internal.model.BaseModel;
 
@@ -19,7 +20,7 @@ public AzureOpenAIModel(Configuration configuration, EmbeddingModelParameters em
     super(configuration,embeddingModelParameters);
     JSONObject config = readConfigFile(configuration.getConfigFilePath());
     assert config != null;
-    JSONObject modelConfig = config.getJSONObject("AZURE_OPENAI");
+    JSONObject modelConfig = config.getJSONObject(Constants.EMBEDDING_MODEL_SERVICE_AZURE_OPENAI);
     this.apiKey = modelConfig.getString("AZURE_OPENAI_KEY");
     this.endpoint = modelConfig.getString("AZURE_OPENAI_ENDPOINT");
   }

src/main/java/org/mule/extension/mulechain/vectors/internal/operation/DocumentOperations.java

@@ -2,11 +2,7 @@
 
 import dev.langchain4j.data.document.Document;
 import dev.langchain4j.data.document.DocumentSplitter;
-import dev.langchain4j.data.document.loader.UrlDocumentLoader;
-import dev.langchain4j.data.document.parser.TextDocumentParser;
-import dev.langchain4j.data.document.parser.apache.tika.ApacheTikaDocumentParser;
 import dev.langchain4j.data.document.splitter.DocumentSplitters;
-import dev.langchain4j.data.document.transformer.jsoup.HtmlToTextDocumentTransformer;
 import dev.langchain4j.data.segment.TextSegment;
 import org.json.JSONArray;
 import org.json.JSONObject;
@@ -25,23 +21,15 @@
 import org.mule.runtime.extension.api.annotation.param.Config;
 import org.mule.runtime.extension.api.annotation.param.MediaType;
 import org.mule.runtime.extension.api.annotation.param.ParameterGroup;
-import org.mule.runtime.extension.api.annotation.param.display.DisplayName;
-import org.mule.runtime.extension.api.annotation.param.display.Summary;
 import org.mule.runtime.extension.api.exception.ModuleException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
-import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
-import static dev.langchain4j.data.document.loader.FileSystemDocumentLoader.loadDocument;
 import static org.mule.extension.mulechain.vectors.internal.helper.ResponseHelper.createDocumentResponse;
 import static org.mule.runtime.extension.api.annotation.param.MediaType.APPLICATION_JSON;
 
@@ -81,17 +69,20 @@ public class DocumentOperations {
       JSONArray jsonSegments = IntStream.range(0, segments.size())
           .mapToObj(i -> {
             JSONObject jsonObject = new JSONObject();
-            jsonObject.put("text", segments.get(i).text()); // Replace getText with the actual method
-            jsonObject.put("index", i);
+            jsonObject.put(Constants.JSON_KEY_TEXT, segments.get(i).text()); // Replace getText with the actual method
+            jsonObject.put(Constants.JSON_KEY_INDEX, i);
             return jsonObject;
           })
           .collect(JSONArray::new, JSONArray::put, JSONArray::putAll);
 
       JSONObject jsonObject = new JSONObject();
-      jsonObject.put("segments", jsonSegments);
+      jsonObject.put(Constants.JSON_KEY_SEGMENTS, jsonSegments);
 
       return createDocumentResponse(jsonObject.toString(), new HashMap<>());
 
+    } catch (ModuleException me) {
+      throw me;
+
     } catch (Exception e) {
 
       throw new ModuleException(
@@ -123,10 +114,13 @@ public class DocumentOperations {
       Document document = baseStorage.getSingleDocument();
 
       JSONObject jsonObject = new JSONObject();
-      jsonObject.put("text",document.text());
+      jsonObject.put(Constants.JSON_KEY_TEXT,document.text());
 
       return createDocumentResponse(jsonObject.toString(), new HashMap<>());
 
+    } catch (ModuleException me) {
+      throw me;
+
     } catch (Exception e) {
 
       throw new ModuleException(