diff --git a/pom.xml b/pom.xml index b26e9b3..3ac5f4d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 com.mulesoft.connectors mule4-vectors-connector - 0.2.0 + 0.2.50 mule-extension MuleSoft Vectors Connector - Mule 4 MuleSoft Vectors Connector provides access to a broad number of external Vector Stores. diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/extension/Connector.java b/src/main/java/org/mule/extension/mulechain/vectors/internal/extension/Connector.java deleted file mode 100644 index d368975..0000000 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/extension/Connector.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.mule.extension.mulechain.vectors.internal.extension; - -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.error.MuleVectorsErrorType; -import org.mule.runtime.api.meta.Category; -import org.mule.runtime.extension.api.annotation.Extension; -import org.mule.runtime.extension.api.annotation.Configurations; -import org.mule.runtime.extension.api.annotation.dsl.xml.Xml; -import org.mule.runtime.extension.api.annotation.error.ErrorTypes; -import org.mule.runtime.extension.api.annotation.license.RequiresEnterpriseLicense; -import org.mule.sdk.api.annotation.JavaVersionSupport; -import static org.mule.sdk.api.meta.JavaVersion.JAVA_11; -import static org.mule.sdk.api.meta.JavaVersion.JAVA_17; -import static org.mule.sdk.api.meta.JavaVersion.JAVA_8; - -/** - * This is the main class of an extension, is the entry point from which configurations, connection providers, operations - * and sources are going to be declared. - */ -@Xml(prefix = "ms-vectors") -@Extension(name = "MuleSoft Vectors Connector", category = Category.SELECT) -@Configurations(Configuration.class) -@RequiresEnterpriseLicense(allowEvaluationLicense = true) -@ErrorTypes(MuleVectorsErrorType.class) -@JavaVersionSupport({JAVA_8, JAVA_11, JAVA_17}) -public class Connector { - -} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/EmbeddingModelServiceProvider.java b/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/EmbeddingModelServiceProvider.java deleted file mode 100644 index 6dc944f..0000000 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/EmbeddingModelServiceProvider.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.mule.extension.mulechain.vectors.internal.helper.provider; - -import java.util.Set; - -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.runtime.api.value.Value; -import org.mule.runtime.extension.api.values.ValueBuilder; -import org.mule.runtime.extension.api.values.ValueProvider; -import org.mule.runtime.extension.api.values.ValueResolvingException; - -public class EmbeddingModelServiceProvider implements ValueProvider { - - @Override - public Set resolve() throws ValueResolvingException { - - return ValueBuilder.getValuesFor( - Constants.EMBEDDING_MODEL_SERVICE_OPENAI, - Constants.EMBEDDING_MODEL_SERVICE_MISTRAL_AI, - Constants.EMBEDDING_MODEL_SERVICE_NOMIC, - Constants.EMBEDDING_MODEL_SERVICE_HUGGING_FACE, - Constants.EMBEDDING_MODEL_SERVICE_AZURE_OPENAI); //"OLLAMA", "COHERE", "AZURE_OPENAI", "HUGGING_FACE"; -} - -} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/StorageTypeProvider.java b/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/StorageTypeProvider.java deleted file mode 100644 index d71df3b..0000000 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/StorageTypeProvider.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.mule.extension.mulechain.vectors.internal.helper.provider; - - -import java.util.Set; - -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.runtime.api.value.Value; -import org.mule.runtime.extension.api.values.ValueBuilder; -import org.mule.runtime.extension.api.values.ValueProvider; -import org.mule.runtime.extension.api.values.ValueResolvingException; - -public class StorageTypeProvider implements ValueProvider { - - @Override - public Set resolve() throws ValueResolvingException { - - return ValueBuilder.getValuesFor( - Constants.STORAGE_TYPE_LOCAL, - Constants.STORAGE_TYPE_S3, - Constants.STORAGE_TYPE_AZURE_BLOB); - } - -} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/VectorStoreProvider.java b/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/VectorStoreProvider.java deleted file mode 100644 index 042a109..0000000 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/VectorStoreProvider.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.mule.extension.mulechain.vectors.internal.helper.provider; - -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.runtime.api.value.Value; -import org.mule.runtime.extension.api.values.ValueBuilder; -import org.mule.runtime.extension.api.values.ValueProvider; -import org.mule.runtime.extension.api.values.ValueResolvingException; - -import java.util.Set; - -public class VectorStoreProvider implements ValueProvider { - - @Override - public Set resolve() throws ValueResolvingException { - - return ValueBuilder.getValuesFor( - Constants.VECTOR_STORE_PGVECTOR, - Constants.VECTOR_STORE_ELASTICSEARCH, - Constants.VECTOR_STORE_OPENSEARCH, - Constants.VECTOR_STORE_MILVUS, - Constants.VECTOR_STORE_CHROMA, - Constants.VECTOR_STORE_PINECONE, - Constants.VECTOR_STORE_AI_SEARCH, - Constants.VECTOR_STORE_OPENSEARCH, - Constants.VECTOR_STORE_QDRANT - ); // MuleChainVectorsConstants.VECTOR_STORE_NEO4J - } - -} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/model/azureopenai/AzureOpenAIModel.java b/src/main/java/org/mule/extension/mulechain/vectors/internal/model/azureopenai/AzureOpenAIModel.java deleted file mode 100644 index 2625110..0000000 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/model/azureopenai/AzureOpenAIModel.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.mule.extension.mulechain.vectors.internal.model.azureopenai; - -import dev.langchain4j.model.azure.AzureOpenAiEmbeddingModel; -import dev.langchain4j.model.embedding.EmbeddingModel; -import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.EmbeddingModelParameters; -import org.mule.extension.mulechain.vectors.internal.model.BaseModel; - -import static org.mule.extension.mulechain.vectors.internal.util.JsonUtils.readConfigFile; - -public class AzureOpenAIModel extends BaseModel { - - private final String apiKey; - private final String endpoint; - - public AzureOpenAIModel(Configuration configuration, EmbeddingModelParameters embeddingModelParameters) { - - super(configuration,embeddingModelParameters); - JSONObject config = readConfigFile(configuration.getConfigFilePath()); - assert config != null; - JSONObject modelConfig = config.getJSONObject(Constants.EMBEDDING_MODEL_SERVICE_AZURE_OPENAI); - this.apiKey = modelConfig.getString("AZURE_OPENAI_KEY"); - this.endpoint = modelConfig.getString("AZURE_OPENAI_ENDPOINT"); - } - - public EmbeddingModel buildEmbeddingModel() { - - return AzureOpenAiEmbeddingModel.builder() - .apiKey(apiKey) - .endpoint(endpoint) - .deploymentName(embeddingModelParameters.getEmbeddingModelName()) - .build(); - } -} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/model/huggingface/HuggingFaceModel.java b/src/main/java/org/mule/extension/mulechain/vectors/internal/model/huggingface/HuggingFaceModel.java deleted file mode 100644 index 7223847..0000000 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/model/huggingface/HuggingFaceModel.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.mule.extension.mulechain.vectors.internal.model.huggingface; - -import dev.langchain4j.model.embedding.EmbeddingModel; -import dev.langchain4j.model.huggingface.HuggingFaceEmbeddingModel; -import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.EmbeddingModelParameters; -import org.mule.extension.mulechain.vectors.internal.model.BaseModel; - -import static org.mule.extension.mulechain.vectors.internal.util.JsonUtils.readConfigFile; - -public class HuggingFaceModel extends BaseModel { - - private final String apiKey; - - public HuggingFaceModel(Configuration configuration, EmbeddingModelParameters embeddingModelParameters) { - - super(configuration,embeddingModelParameters); - JSONObject config = readConfigFile(configuration.getConfigFilePath()); - assert config != null; - JSONObject modelConfig = config.getJSONObject("HUGGING_FACE"); - this.apiKey = modelConfig.getString("HUGGING_FACE_API_KEY"); - } - - public EmbeddingModel buildEmbeddingModel() { - - return HuggingFaceEmbeddingModel.builder() - .accessToken(apiKey) - .modelId(embeddingModelParameters.getEmbeddingModelName()) - .build(); - } -} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/model/mistralai/MistralAIModel.java b/src/main/java/org/mule/extension/mulechain/vectors/internal/model/mistralai/MistralAIModel.java deleted file mode 100644 index 7089f2f..0000000 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/model/mistralai/MistralAIModel.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.mule.extension.mulechain.vectors.internal.model.mistralai; - -import dev.langchain4j.model.embedding.EmbeddingModel; -import dev.langchain4j.model.mistralai.MistralAiEmbeddingModel; -import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.EmbeddingModelParameters; -import org.mule.extension.mulechain.vectors.internal.model.BaseModel; - -import static org.mule.extension.mulechain.vectors.internal.util.JsonUtils.readConfigFile; - -public class MistralAIModel extends BaseModel { - - private final String apiKey; - - public MistralAIModel(Configuration configuration, EmbeddingModelParameters embeddingModelParameters) { - - super(configuration,embeddingModelParameters); - JSONObject config = readConfigFile(configuration.getConfigFilePath()); - assert config != null; - JSONObject modelConfig = config.getJSONObject("MISTRAL_AI"); - this.apiKey = modelConfig.getString("MISTRAL_AI_API_KEY"); - } - - public EmbeddingModel buildEmbeddingModel() { - - return MistralAiEmbeddingModel.builder() - .apiKey(apiKey) - .modelName(embeddingModelParameters.getEmbeddingModelName()) - .build(); - } -} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/model/openai/OpenAIModel.java b/src/main/java/org/mule/extension/mulechain/vectors/internal/model/openai/OpenAIModel.java deleted file mode 100644 index 43d10e9..0000000 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/model/openai/OpenAIModel.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.mule.extension.mulechain.vectors.internal.model.openai; - -import dev.langchain4j.model.embedding.EmbeddingModel; -import dev.langchain4j.model.openai.OpenAiEmbeddingModel; -import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.EmbeddingModelParameters; -import org.mule.extension.mulechain.vectors.internal.model.BaseModel; - -import static org.mule.extension.mulechain.vectors.internal.util.JsonUtils.readConfigFile; - -public class OpenAIModel extends BaseModel { - - private final String apiKey; - - public OpenAIModel(Configuration configuration, EmbeddingModelParameters embeddingModelParameters) { - - super(configuration,embeddingModelParameters); - JSONObject config = readConfigFile(configuration.getConfigFilePath()); - assert config != null; - JSONObject modelConfig = config.getJSONObject("OPENAI"); - this.apiKey = modelConfig.getString("OPENAI_API_KEY"); - } - - public EmbeddingModel buildEmbeddingModel() { - - return OpenAiEmbeddingModel.builder() - .apiKey(apiKey) - .modelName(embeddingModelParameters.getEmbeddingModelName()) - .build(); - } -} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/storage/BaseStorage.java b/src/main/java/org/mule/extension/mulechain/vectors/internal/storage/BaseStorage.java deleted file mode 100644 index ce72838..0000000 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/storage/BaseStorage.java +++ /dev/null @@ -1,131 +0,0 @@ -package org.mule.extension.mulechain.vectors.internal.storage; - -import dev.langchain4j.data.document.Document; -import dev.langchain4j.data.document.DocumentParser; -import dev.langchain4j.data.document.parser.TextDocumentParser; -import dev.langchain4j.data.document.parser.apache.tika.ApacheTikaDocumentParser; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.storage.azureblob.AzureBlobStorage; -import org.mule.extension.mulechain.vectors.internal.storage.local.LocalStorage; -import org.mule.extension.mulechain.vectors.internal.storage.s3.AWSS3Storage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Iterator; - -public abstract class BaseStorage implements Iterator { - - private static final Logger LOGGER = LoggerFactory.getLogger(BaseStorage.class); - - protected Configuration configuration; - protected String contextPath; - protected String fileType; - protected DocumentParser documentParser; - - public BaseStorage(Configuration configuration, String contextPath, String fileType) { - - this.configuration = configuration; - this.contextPath = contextPath; - this.fileType = fileType; - this.documentParser = getDocumentParser(fileType); - } - - @Override - public boolean hasNext() { - throw new UnsupportedOperationException("This method should be overridden by subclasses"); - } - - @Override - public Document next() { - throw new UnsupportedOperationException("This method should be overridden by subclasses"); - } - - public Document getSingleDocument() { - throw new UnsupportedOperationException("This method should be overridden by subclasses"); - } - - protected DocumentParser getDocumentParser(String fileType) { - - DocumentParser documentParser = null; - switch (fileType){ - - case Constants.FILE_TYPE_TEXT: - case Constants.FILE_TYPE_CRAWL: - case Constants.FILE_TYPE_URL: - documentParser = new TextDocumentParser(); - break; - case Constants.FILE_TYPE_ANY: - documentParser = new ApacheTikaDocumentParser(); - break; - default: - throw new IllegalArgumentException("Unsupported File Type: " + fileType); - } - return documentParser; - } - - public static BaseStorage.Builder builder() { - - return new BaseStorage.Builder(); - } - - public static class Builder { - - private Configuration configuration; - private String storageType; - private String contextPath; - private String fileType; - - public Builder() { - - } - - public BaseStorage.Builder configuration(Configuration configuration) { - this.configuration = configuration; - return this; - } - - public BaseStorage.Builder storageType(String storageType) { - this.storageType = storageType; - return this; - } - - public BaseStorage.Builder contextPath(String contextPath) { - this.contextPath = contextPath; - return this; - } - - public BaseStorage.Builder fileType(String fileType) { - this.fileType = fileType; - return this; - } - - public BaseStorage build() { - - BaseStorage baseStorage; - - LOGGER.debug("Storage Type: " + storageType); - switch (storageType) { - - case Constants.STORAGE_TYPE_LOCAL: - - baseStorage = new LocalStorage(configuration, contextPath, fileType); - break; - case Constants.STORAGE_TYPE_S3: - - baseStorage = new AWSS3Storage(configuration, contextPath, fileType); - break; - - case Constants.STORAGE_TYPE_AZURE_BLOB: - - baseStorage = new AzureBlobStorage(configuration, contextPath, fileType); - break; - - default: - //throw new IllegalOperationException("Unsupported Vector Store: " + configuration.getVectorStore()); - baseStorage = null; - } - return baseStorage; - } - } -} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/elasticsearch/ElasticsearchStore.java b/src/main/java/org/mule/extension/mulechain/vectors/internal/store/elasticsearch/ElasticsearchStore.java deleted file mode 100644 index f021777..0000000 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/elasticsearch/ElasticsearchStore.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.mule.extension.mulechain.vectors.internal.store.elasticsearch; - -import dev.langchain4j.data.segment.TextSegment; -import dev.langchain4j.model.embedding.EmbeddingModel; -import dev.langchain4j.store.embedding.EmbeddingStore; -import dev.langchain4j.store.embedding.elasticsearch.ElasticsearchEmbeddingStore; -import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.QueryParameters; -import org.mule.extension.mulechain.vectors.internal.store.BaseStore; -import org.mule.extension.mulechain.vectors.internal.util.JsonUtils; - -public class ElasticsearchStore extends BaseStore { - - private String url; - private String userName; - private String password; - - public ElasticsearchStore(String storeName, Configuration configuration, QueryParameters queryParams, int dimension) { - - super(storeName, configuration, queryParams, dimension); - - JSONObject config = JsonUtils.readConfigFile(configuration.getConfigFilePath()); - JSONObject vectorStoreConfig = config.getJSONObject(Constants.VECTOR_STORE_ELASTICSEARCH); - this.url = vectorStoreConfig.getString("ELASTICSEARCH_URL"); - this.userName = vectorStoreConfig.getString("ELASTICSEARCH_USER"); - this.password = vectorStoreConfig.getString("ELASTICSEARCH_PASSWORD"); - } - - public EmbeddingStore buildEmbeddingStore() { - - return ElasticsearchEmbeddingStore.builder() - .serverUrl(url) - .userName(userName) - .password(password) - .indexName(storeName) - .dimension(dimension) - .build(); - } -} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/opensearch/OpenSearchStore.java b/src/main/java/org/mule/extension/mulechain/vectors/internal/store/opensearch/OpenSearchStore.java deleted file mode 100644 index cc34640..0000000 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/opensearch/OpenSearchStore.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.mule.extension.mulechain.vectors.internal.store.opensearch; - -import dev.langchain4j.data.segment.TextSegment; -import dev.langchain4j.model.embedding.EmbeddingModel; -import dev.langchain4j.store.embedding.EmbeddingStore; -import dev.langchain4j.store.embedding.opensearch.OpenSearchEmbeddingStore; -import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.QueryParameters; -import org.mule.extension.mulechain.vectors.internal.store.BaseStore; -import org.mule.extension.mulechain.vectors.internal.util.JsonUtils; - -public class OpenSearchStore extends BaseStore { - - private String url; - private String userName; - private String password; - - public OpenSearchStore(String storeName, Configuration configuration, QueryParameters queryParams, int dimension) { - - super(storeName, configuration, queryParams, dimension); - - JSONObject config = JsonUtils.readConfigFile(configuration.getConfigFilePath()); - JSONObject vectorStoreConfig = config.getJSONObject(Constants.VECTOR_STORE_OPENSEARCH); - this.url = vectorStoreConfig.getString("OPENSEARCH_URL"); - this.userName = vectorStoreConfig.getString("OPENSEARCH_USER"); - this.password = vectorStoreConfig.getString("OPENSEARCH_PASSWORD"); - } - - public EmbeddingStore buildEmbeddingStore() { - - return OpenSearchEmbeddingStore.builder() - .serverUrl(url) - .userName(userName) - .password(password) - .indexName(storeName) - .build(); - } -} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/api/metadata/DocumentResponseAttributes.java b/src/main/java/org/mule/extension/vectors/api/metadata/DocumentResponseAttributes.java similarity index 91% rename from src/main/java/org/mule/extension/mulechain/vectors/api/metadata/DocumentResponseAttributes.java rename to src/main/java/org/mule/extension/vectors/api/metadata/DocumentResponseAttributes.java index b121253..e9ce6fe 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/api/metadata/DocumentResponseAttributes.java +++ b/src/main/java/org/mule/extension/vectors/api/metadata/DocumentResponseAttributes.java @@ -1,4 +1,4 @@ -package org.mule.extension.mulechain.vectors.api.metadata; +package org.mule.extension.vectors.api.metadata; import org.mule.runtime.extension.api.annotation.param.MediaType; diff --git a/src/main/java/org/mule/extension/mulechain/vectors/api/metadata/EmbeddingResponseAttributes.java b/src/main/java/org/mule/extension/vectors/api/metadata/EmbeddingResponseAttributes.java similarity index 91% rename from src/main/java/org/mule/extension/mulechain/vectors/api/metadata/EmbeddingResponseAttributes.java rename to src/main/java/org/mule/extension/vectors/api/metadata/EmbeddingResponseAttributes.java index 0074086..b1b8e23 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/api/metadata/EmbeddingResponseAttributes.java +++ b/src/main/java/org/mule/extension/vectors/api/metadata/EmbeddingResponseAttributes.java @@ -1,4 +1,4 @@ -package org.mule.extension.mulechain.vectors.api.metadata; +package org.mule.extension.vectors.api.metadata; import org.mule.runtime.extension.api.annotation.param.MediaType; diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/config/Configuration.java b/src/main/java/org/mule/extension/vectors/internal/config/Configuration.java similarity index 51% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/config/Configuration.java rename to src/main/java/org/mule/extension/vectors/internal/config/Configuration.java index 3fc6718..f74f439 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/config/Configuration.java +++ b/src/main/java/org/mule/extension/vectors/internal/config/Configuration.java @@ -1,16 +1,19 @@ -package org.mule.extension.mulechain.vectors.internal.config; - -import org.mule.extension.mulechain.vectors.internal.helper.provider.EmbeddingModelServiceProvider; -import org.mule.extension.mulechain.vectors.internal.helper.provider.VectorStoreProvider; -import org.mule.extension.mulechain.vectors.internal.operation.DocumentOperations; -import org.mule.extension.mulechain.vectors.internal.operation.EmbeddingOperations; +package org.mule.extension.vectors.internal.config; + +import org.mule.extension.vectors.internal.model.BaseModelConfiguration; +import org.mule.extension.vectors.internal.operation.DocumentOperations; +import org.mule.extension.vectors.internal.operation.EmbeddingOperations; +import org.mule.extension.vectors.internal.storage.BaseStorageConfiguration; +import org.mule.extension.vectors.internal.store.BaseStoreConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; import org.mule.runtime.extension.api.annotation.Operations; +import org.mule.runtime.extension.api.annotation.param.Optional; import org.mule.runtime.extension.api.annotation.param.Parameter; import org.mule.runtime.extension.api.annotation.param.display.DisplayName; import org.mule.runtime.extension.api.annotation.param.display.Placement; import org.mule.runtime.extension.api.annotation.param.display.Summary; -import org.mule.runtime.extension.api.annotation.values.OfValues; /** * This class represents an extension configuration, values set in this class are commonly used across multiple @@ -25,34 +28,32 @@ public class Configuration { @DisplayName("Embedding Model Service") @Summary("The embedding model service.") @Placement(order = 1, tab = Placement.DEFAULT_TAB) - @OfValues(EmbeddingModelServiceProvider.class) - private String embeddingModelService; + private BaseModelConfiguration modelConfiguration; @Parameter @Alias("vectorStore") @DisplayName("Vector Store") @Summary("The vector store.") @Placement(order = 2, tab = Placement.DEFAULT_TAB) - @OfValues(VectorStoreProvider.class) - private String vectorStore; + private BaseStoreConfiguration storeConfiguration; @Parameter - @Alias("configFilePath") - @DisplayName("Configuration File Path") - @Summary("The configuration file path.") + @Alias("storage") + @DisplayName("Storage") + @Summary("The storage type.") + @Optional @Placement(order = 3, tab = Placement.DEFAULT_TAB) - private String configFilePath; + private BaseStorageConfiguration storageConfiguration; - public String getEmbeddingModelService() { - return embeddingModelService; + public BaseModelConfiguration getModelConfiguration() { + return modelConfiguration; } - public String getVectorStore() { - return vectorStore; + public BaseStoreConfiguration getStoreConfiguration() { + return storeConfiguration; } - public String getConfigFilePath() { - return configFilePath; + public BaseStorageConfiguration getStorageConfiguration() { + return storageConfiguration; } - } diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/constant/Constants.java b/src/main/java/org/mule/extension/vectors/internal/constant/Constants.java similarity index 87% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/constant/Constants.java rename to src/main/java/org/mule/extension/vectors/internal/constant/Constants.java index 036e16f..a70fa7e 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/constant/Constants.java +++ b/src/main/java/org/mule/extension/vectors/internal/constant/Constants.java @@ -1,7 +1,7 @@ /** * (c) 2003-2024 MuleSoft, Inc. The software in this package is published under the terms of the Commercial Free Software license V.1 a copy of which has been included with this distribution in the LICENSE.md file. */ -package org.mule.extension.mulechain.vectors.internal.constant; +package org.mule.extension.vectors.internal.constant; public class Constants { @@ -13,7 +13,7 @@ private Constants() {} public static final String FILE_TYPE_URL = "url"; public static final String STORAGE_TYPE_LOCAL = "Local"; - public static final String STORAGE_TYPE_S3 = "S3"; + public static final String STORAGE_TYPE_AWS_S3 = "S3"; public static final String STORAGE_TYPE_AZURE_BLOB = "AZURE_BLOB"; public static final String EMBEDDING_MODEL_SERVICE_OPENAI = "OPENAI"; @@ -21,6 +21,7 @@ private Constants() {} public static final String EMBEDDING_MODEL_SERVICE_MISTRAL_AI = "MISTRAL_AI"; public static final String EMBEDDING_MODEL_SERVICE_NOMIC = "NOMIC"; public static final String EMBEDDING_MODEL_SERVICE_HUGGING_FACE = "HUGGING_FACE"; + public static final String EMBEDDING_MODEL_SERVICE_EINSTEIN = "EINSTEIN"; public static final String EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_3_SMALL = "text-embedding-3-small"; public static final String EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_3_LARGE = "text-embedding-3-large"; @@ -29,6 +30,9 @@ private Constants() {} public static final String EMBEDDING_MODEL_NAME_NOMIC_EMBED_TEXT = "nomic-embed-text"; public static final String EMBEDDING_MODEL_NAME_FALCON_7B_INSTRUCT = "tiiuae/falcon-7b-instruct"; public static final String EMBEDDING_MODEL_NAME_MINI_LM_L6_V2 = "sentence-transformers/all-MiniLM-L6-v2"; + public static final String EMBEDDING_MODEL_NAME_SFDC_TEXT_EMBEDDING_ADA_002 = "sfdc_ai__DefaultTextEmbeddingAda_002"; + public static final String EMBEDDING_MODEL_NAME_SFDC_AZURE_TEXT_EMBEDDING_ADA_002 = "sfdc_ai__DefaultAzureOpenAITextEmbeddingAda_002"; + public static final String EMBEDDING_MODEL_NAME_SFDC_OPENAI_TEXT_EMBEDDING_ADA_002 = "sfdc_ai__DefaultOpenAITextEmbeddingAda_002"; public static final String VECTOR_STORE_PGVECTOR = "PGVECTOR"; public static final String VECTOR_STORE_ELASTICSEARCH = "ELASTICSEARCH"; @@ -84,7 +88,8 @@ private Constants() {} public static final String JSON_KEY_METADATA = "metadata"; public static final String JSON_KEY_INDEX = "index"; - public static final String OPERATION_STATUS_ADDED = "added"; public static final String OPERATION_STATUS_UPDATED = "updated"; public static final String OPERATION_STATUS_DELETED = "deleted"; + + public static final String PARAM_DISPLAY_NAME_STORAGE_OVERRIDE = "Storage (Override Module Configuration)"; } diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/error/MuleVectorsErrorType.java b/src/main/java/org/mule/extension/vectors/internal/error/MuleVectorsErrorType.java similarity index 79% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/error/MuleVectorsErrorType.java rename to src/main/java/org/mule/extension/vectors/internal/error/MuleVectorsErrorType.java index b31fa55..d134ab1 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/error/MuleVectorsErrorType.java +++ b/src/main/java/org/mule/extension/vectors/internal/error/MuleVectorsErrorType.java @@ -1,4 +1,4 @@ -package org.mule.extension.mulechain.vectors.internal.error; +package org.mule.extension.vectors.internal.error; import org.mule.runtime.extension.api.error.ErrorTypeDefinition; @@ -7,5 +7,5 @@ */ public enum MuleVectorsErrorType implements ErrorTypeDefinition { - DOCUMENT_OPERATIONS_FAILURE, EMBEDDING_OPERATIONS_FAILURE, AI_SERVICES_FAILURE + DOCUMENT_OPERATIONS_FAILURE, EMBEDDING_OPERATIONS_FAILURE, AI_SERVICES_FAILURE, STORE_SERVICES_FAILURE, STORAGE_SERVICES_FAILURE } diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/error/provider/DocumentErrorTypeProvider.java b/src/main/java/org/mule/extension/vectors/internal/error/provider/DocumentErrorTypeProvider.java similarity index 70% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/error/provider/DocumentErrorTypeProvider.java rename to src/main/java/org/mule/extension/vectors/internal/error/provider/DocumentErrorTypeProvider.java index e250d5e..4631979 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/error/provider/DocumentErrorTypeProvider.java +++ b/src/main/java/org/mule/extension/vectors/internal/error/provider/DocumentErrorTypeProvider.java @@ -1,7 +1,7 @@ /** * (c) 2003-2024 MuleSoft, Inc. The software in this package is published under the terms of the Commercial Free Software license V.1 a copy of which has been included with this distribution in the LICENSE.md file. */ -package org.mule.extension.mulechain.vectors.internal.error.provider; +package org.mule.extension.vectors.internal.error.provider; import org.mule.runtime.extension.api.annotation.error.ErrorTypeProvider; import org.mule.runtime.extension.api.error.ErrorTypeDefinition; @@ -11,13 +11,14 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableSet; -import static org.mule.extension.mulechain.vectors.internal.error.MuleVectorsErrorType.DOCUMENT_OPERATIONS_FAILURE; +import static org.mule.extension.vectors.internal.error.MuleVectorsErrorType.DOCUMENT_OPERATIONS_FAILURE; +import static org.mule.extension.vectors.internal.error.MuleVectorsErrorType.STORAGE_SERVICES_FAILURE; public class DocumentErrorTypeProvider implements ErrorTypeProvider { @SuppressWarnings("rawtypes") @Override public Set getErrorTypes() { - return unmodifiableSet(new HashSet<>(asList(DOCUMENT_OPERATIONS_FAILURE))); + return unmodifiableSet(new HashSet<>(asList(DOCUMENT_OPERATIONS_FAILURE, STORAGE_SERVICES_FAILURE))); } } diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/error/provider/EmbeddingErrorTypeProvider.java b/src/main/java/org/mule/extension/vectors/internal/error/provider/EmbeddingErrorTypeProvider.java similarity index 70% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/error/provider/EmbeddingErrorTypeProvider.java rename to src/main/java/org/mule/extension/vectors/internal/error/provider/EmbeddingErrorTypeProvider.java index 947d5c5..1bb360f 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/error/provider/EmbeddingErrorTypeProvider.java +++ b/src/main/java/org/mule/extension/vectors/internal/error/provider/EmbeddingErrorTypeProvider.java @@ -1,7 +1,7 @@ /** * (c) 2003-2024 MuleSoft, Inc. The software in this package is published under the terms of the Commercial Free Software license V.1 a copy of which has been included with this distribution in the LICENSE.md file. */ -package org.mule.extension.mulechain.vectors.internal.error.provider; +package org.mule.extension.vectors.internal.error.provider; import org.mule.runtime.extension.api.annotation.error.ErrorTypeProvider; import org.mule.runtime.extension.api.error.ErrorTypeDefinition; @@ -11,8 +11,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableSet; -import static org.mule.extension.mulechain.vectors.internal.error.MuleVectorsErrorType.AI_SERVICES_FAILURE; -import static org.mule.extension.mulechain.vectors.internal.error.MuleVectorsErrorType.EMBEDDING_OPERATIONS_FAILURE; +import static org.mule.extension.vectors.internal.error.MuleVectorsErrorType.*; public class EmbeddingErrorTypeProvider implements ErrorTypeProvider { @@ -21,6 +20,8 @@ public class EmbeddingErrorTypeProvider implements ErrorTypeProvider { public Set getErrorTypes() { return unmodifiableSet(new HashSet<>(asList( EMBEDDING_OPERATIONS_FAILURE, - AI_SERVICES_FAILURE))); + AI_SERVICES_FAILURE, + STORE_SERVICES_FAILURE, + STORAGE_SERVICES_FAILURE))); } } diff --git a/src/main/java/org/mule/extension/vectors/internal/extension/Connector.java b/src/main/java/org/mule/extension/vectors/internal/extension/Connector.java new file mode 100644 index 0000000..d576f57 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/extension/Connector.java @@ -0,0 +1,72 @@ +package org.mule.extension.vectors.internal.extension; + +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.error.MuleVectorsErrorType; +import org.mule.extension.vectors.internal.model.BaseModelConfiguration; +import org.mule.extension.vectors.internal.model.azureopenai.AzureOpenAIModelConfiguration; +import org.mule.extension.vectors.internal.model.einstein.EinsteinModelConfiguration; +import org.mule.extension.vectors.internal.model.huggingface.HuggingFaceModelConfiguration; +import org.mule.extension.vectors.internal.model.mistralai.MistralAIModelConfiguration; +import org.mule.extension.vectors.internal.model.nomic.NomicModelConfiguration; +import org.mule.extension.vectors.internal.model.openai.OpenAIModelConfiguration; +import org.mule.extension.vectors.internal.storage.BaseStorageConfiguration; +import org.mule.extension.vectors.internal.storage.azureblob.AzureBlobStorageConfiguration; +import org.mule.extension.vectors.internal.storage.local.LocalStorageConfiguration; +import org.mule.extension.vectors.internal.storage.s3.AWSS3StorageConfiguration; +import org.mule.extension.vectors.internal.store.BaseStoreConfiguration; +import org.mule.extension.vectors.internal.store.aisearch.AISearchStoreConfiguration; +import org.mule.extension.vectors.internal.store.chroma.ChromaStoreConfiguration; +import org.mule.extension.vectors.internal.store.elasticsearch.ElasticsearchStoreConfiguration; +import org.mule.extension.vectors.internal.store.milvus.MilvusStoreConfiguration; +import org.mule.extension.vectors.internal.store.opensearch.OpenSearchStoreConfiguration; +import org.mule.extension.vectors.internal.store.pgvector.PGVectorStoreConfiguration; +import org.mule.extension.vectors.internal.store.pinecone.PineconeStoreConfiguration; +import org.mule.extension.vectors.internal.store.qdrant.QdrantStoreConfiguration; +import org.mule.runtime.api.meta.Category; +import org.mule.runtime.extension.api.annotation.Extension; +import org.mule.runtime.extension.api.annotation.Configurations; +import org.mule.runtime.extension.api.annotation.SubTypeMapping; +import org.mule.runtime.extension.api.annotation.dsl.xml.Xml; +import org.mule.runtime.extension.api.annotation.error.ErrorTypes; +import org.mule.runtime.extension.api.annotation.license.RequiresEnterpriseLicense; +import org.mule.sdk.api.annotation.JavaVersionSupport; +import static org.mule.sdk.api.meta.JavaVersion.JAVA_11; +import static org.mule.sdk.api.meta.JavaVersion.JAVA_17; +import static org.mule.sdk.api.meta.JavaVersion.JAVA_8; + +/** + * This is the main class of an extension, is the entry point from which configurations, connection providers, operations + * and sources are going to be declared. + */ +@Xml(prefix = "ms-vectors") +@Extension(name = "MuleSoft Vectors Connector", category = Category.SELECT) +@Configurations(Configuration.class) +@SubTypeMapping(baseType = BaseStoreConfiguration.class, + subTypes = { + AISearchStoreConfiguration.class, + ChromaStoreConfiguration.class, + ElasticsearchStoreConfiguration.class, + MilvusStoreConfiguration.class, + OpenSearchStoreConfiguration.class, + PGVectorStoreConfiguration.class, + PineconeStoreConfiguration.class, + QdrantStoreConfiguration.class}) +@SubTypeMapping(baseType = BaseModelConfiguration.class, + subTypes = { + AzureOpenAIModelConfiguration.class, + EinsteinModelConfiguration.class, + HuggingFaceModelConfiguration.class, + MistralAIModelConfiguration.class, + NomicModelConfiguration.class, + OpenAIModelConfiguration.class}) +@SubTypeMapping(baseType = BaseStorageConfiguration.class, + subTypes = { + LocalStorageConfiguration.class, + AWSS3StorageConfiguration.class, + AzureBlobStorageConfiguration.class}) +@RequiresEnterpriseLicense(allowEvaluationLicense = true) +@ErrorTypes(MuleVectorsErrorType.class) +@JavaVersionSupport({JAVA_8, JAVA_11, JAVA_17}) +public class Connector { + +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/EmbeddingOperationValidator.java b/src/main/java/org/mule/extension/vectors/internal/helper/EmbeddingOperationValidator.java similarity index 97% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/helper/EmbeddingOperationValidator.java rename to src/main/java/org/mule/extension/vectors/internal/helper/EmbeddingOperationValidator.java index c2e250b..16ef3dc 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/EmbeddingOperationValidator.java +++ b/src/main/java/org/mule/extension/vectors/internal/helper/EmbeddingOperationValidator.java @@ -1,6 +1,6 @@ -package org.mule.extension.mulechain.vectors.internal.helper; +package org.mule.extension.vectors.internal.helper; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.constant.Constants; import java.util.*; diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/ResponseHelper.java b/src/main/java/org/mule/extension/vectors/internal/helper/ResponseHelper.java similarity index 86% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/helper/ResponseHelper.java rename to src/main/java/org/mule/extension/vectors/internal/helper/ResponseHelper.java index e8dfc08..e00d7b5 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/ResponseHelper.java +++ b/src/main/java/org/mule/extension/vectors/internal/helper/ResponseHelper.java @@ -1,7 +1,7 @@ -package org.mule.extension.mulechain.vectors.internal.helper; +package org.mule.extension.vectors.internal.helper; -import org.mule.extension.mulechain.vectors.api.metadata.DocumentResponseAttributes; -import org.mule.extension.mulechain.vectors.api.metadata.EmbeddingResponseAttributes; +import org.mule.extension.vectors.api.metadata.DocumentResponseAttributes; +import org.mule.extension.vectors.api.metadata.EmbeddingResponseAttributes; import org.mule.runtime.extension.api.runtime.operation.Result; import java.io.InputStream; diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/DocumentParameters.java b/src/main/java/org/mule/extension/vectors/internal/helper/parameter/DocumentParameters.java similarity index 66% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/DocumentParameters.java rename to src/main/java/org/mule/extension/vectors/internal/helper/parameter/DocumentParameters.java index e29f237..8833316 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/DocumentParameters.java +++ b/src/main/java/org/mule/extension/vectors/internal/helper/parameter/DocumentParameters.java @@ -1,8 +1,7 @@ -package org.mule.extension.mulechain.vectors.internal.helper.parameter; +package org.mule.extension.vectors.internal.helper.parameter; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.helper.provider.FileTypeEmbeddingProvider; -import org.mule.extension.mulechain.vectors.internal.helper.provider.StorageTypeProvider; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.helper.provider.FileTypeEmbeddingProvider; import org.mule.runtime.api.meta.ExpressionSupport; import org.mule.runtime.extension.api.annotation.Alias; import org.mule.runtime.extension.api.annotation.Expression; @@ -15,21 +14,11 @@ public class DocumentParameters { - @Parameter - @Alias("storageType") - @DisplayName("Storage Type") - @Summary("The supported storage types.") - @Placement(order = 1) - @Expression(ExpressionSupport.SUPPORTED) - @OfValues(StorageTypeProvider.class) - @Optional(defaultValue = Constants.STORAGE_TYPE_LOCAL) - private String storageType; - @Parameter @Alias("fileType") @DisplayName("File Type") @Summary("The supported types of file.") - @Placement(order = 2) + @Placement(order = 1) @Expression(ExpressionSupport.SUPPORTED) @OfValues(FileTypeEmbeddingProvider.class) @Optional(defaultValue = Constants.FILE_TYPE_TEXT) @@ -39,7 +28,7 @@ public class DocumentParameters { @Alias("contextPath") @DisplayName("Context Path") @Summary("The context path.") - @Placement(order = 3) + @Placement(order = 2) @Expression(ExpressionSupport.SUPPORTED) private String contextPath; @@ -56,10 +45,6 @@ public String getFileType() { return fileType; } - public String getStorageType() { - return storageType; - } - public String getContextPath() { return contextPath; } diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/EmbeddingModelParameters.java b/src/main/java/org/mule/extension/vectors/internal/helper/parameter/EmbeddingModelParameters.java similarity index 84% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/EmbeddingModelParameters.java rename to src/main/java/org/mule/extension/vectors/internal/helper/parameter/EmbeddingModelParameters.java index b5fef76..e78e9a5 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/EmbeddingModelParameters.java +++ b/src/main/java/org/mule/extension/vectors/internal/helper/parameter/EmbeddingModelParameters.java @@ -1,6 +1,6 @@ -package org.mule.extension.mulechain.vectors.internal.helper.parameter; +package org.mule.extension.vectors.internal.helper.parameter; -import org.mule.extension.mulechain.vectors.internal.helper.provider.EmbeddingModelNameProvider; +import org.mule.extension.vectors.internal.helper.provider.EmbeddingModelNameProvider; import org.mule.runtime.api.meta.ExpressionSupport; import org.mule.runtime.extension.api.annotation.Alias; import org.mule.runtime.extension.api.annotation.Expression; diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/FileTypeParameters.java b/src/main/java/org/mule/extension/vectors/internal/helper/parameter/FileTypeParameters.java similarity index 70% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/FileTypeParameters.java rename to src/main/java/org/mule/extension/vectors/internal/helper/parameter/FileTypeParameters.java index 5c75c39..f3d0b0a 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/FileTypeParameters.java +++ b/src/main/java/org/mule/extension/vectors/internal/helper/parameter/FileTypeParameters.java @@ -1,7 +1,7 @@ -package org.mule.extension.mulechain.vectors.internal.helper.parameter; +package org.mule.extension.vectors.internal.helper.parameter; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.helper.provider.FileTypeEmbeddingProvider; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.helper.provider.FileTypeEmbeddingProvider; import org.mule.runtime.api.meta.ExpressionSupport; import org.mule.runtime.extension.api.annotation.Expression; import org.mule.runtime.extension.api.annotation.param.Optional; diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/MetadataFilterParameters.java b/src/main/java/org/mule/extension/vectors/internal/helper/parameter/MetadataFilterParameters.java similarity index 95% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/MetadataFilterParameters.java rename to src/main/java/org/mule/extension/vectors/internal/helper/parameter/MetadataFilterParameters.java index f881fd9..8df095d 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/MetadataFilterParameters.java +++ b/src/main/java/org/mule/extension/vectors/internal/helper/parameter/MetadataFilterParameters.java @@ -1,12 +1,12 @@ -package org.mule.extension.mulechain.vectors.internal.helper.parameter; +package org.mule.extension.vectors.internal.helper.parameter; import dev.langchain4j.store.embedding.filter.MetadataFilterBuilder; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.helper.provider.MetadataFilterMethodProvider; -import org.mule.extension.mulechain.vectors.internal.helper.provider.MetadataKeyProvider; -import org.mule.extension.mulechain.vectors.internal.util.Utils; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.helper.provider.MetadataFilterMethodProvider; +import org.mule.extension.vectors.internal.helper.provider.MetadataKeyProvider; +import org.mule.extension.vectors.internal.util.Utils; import org.mule.runtime.api.meta.ExpressionSupport; import org.mule.runtime.extension.api.annotation.Expression; @@ -22,7 +22,6 @@ import java.lang.reflect.Method; import java.util.Arrays; -import java.util.UUID; /** * Abstract class that defines filter parameters for MuleChain Vectors. diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/QueryParameters.java b/src/main/java/org/mule/extension/vectors/internal/helper/parameter/QueryParameters.java similarity index 94% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/QueryParameters.java rename to src/main/java/org/mule/extension/vectors/internal/helper/parameter/QueryParameters.java index 12f6371..556e4aa 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/QueryParameters.java +++ b/src/main/java/org/mule/extension/vectors/internal/helper/parameter/QueryParameters.java @@ -1,4 +1,4 @@ -package org.mule.extension.mulechain.vectors.internal.helper.parameter; +package org.mule.extension.vectors.internal.helper.parameter; import org.mule.runtime.api.meta.ExpressionSupport; import org.mule.runtime.extension.api.annotation.Expression; diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/SegmentationParameters.java b/src/main/java/org/mule/extension/vectors/internal/helper/parameter/SegmentationParameters.java similarity index 94% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/SegmentationParameters.java rename to src/main/java/org/mule/extension/vectors/internal/helper/parameter/SegmentationParameters.java index b5b70f1..101a874 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/parameter/SegmentationParameters.java +++ b/src/main/java/org/mule/extension/vectors/internal/helper/parameter/SegmentationParameters.java @@ -1,4 +1,4 @@ -package org.mule.extension.mulechain.vectors.internal.helper.parameter; +package org.mule.extension.vectors.internal.helper.parameter; import org.mule.runtime.api.meta.ExpressionSupport; import org.mule.runtime.extension.api.annotation.Alias; diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/EmbeddingModelNameProvider.java b/src/main/java/org/mule/extension/vectors/internal/helper/provider/EmbeddingModelNameProvider.java similarity index 55% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/EmbeddingModelNameProvider.java rename to src/main/java/org/mule/extension/vectors/internal/helper/provider/EmbeddingModelNameProvider.java index 546df30..38fefe0 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/EmbeddingModelNameProvider.java +++ b/src/main/java/org/mule/extension/vectors/internal/helper/provider/EmbeddingModelNameProvider.java @@ -1,10 +1,10 @@ -package org.mule.extension.mulechain.vectors.internal.helper.provider; +package org.mule.extension.vectors.internal.helper.provider; import java.util.Collections; import java.util.Set; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; import org.mule.runtime.api.value.Value; import org.mule.runtime.extension.api.annotation.param.Config; import org.mule.runtime.extension.api.values.ValueBuilder; @@ -17,34 +17,40 @@ public class EmbeddingModelNameProvider implements ValueProvider { private Configuration configuration; private static final Set VALUES_FOR_AZURE_OPENAI = ValueBuilder.getValuesFor( - Constants.EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_3_SMALL, - Constants.EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_3_LARGE, - Constants.EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_ADA_002 + Constants.EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_3_SMALL, + Constants.EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_3_LARGE, + Constants.EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_ADA_002 ); private static final Set VALUES_FOR_OPENAI = ValueBuilder.getValuesFor( - Constants.EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_3_SMALL, - Constants.EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_3_LARGE, - Constants.EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_ADA_002 + Constants.EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_3_SMALL, + Constants.EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_3_LARGE, + Constants.EMBEDDING_MODEL_NAME_TEXT_EMBEDDING_ADA_002 ); private static final Set VALUES_FOR_MISTRAL_AI = ValueBuilder.getValuesFor( - Constants.EMBEDDING_MODEL_NAME_MISTRAL_EMBED + Constants.EMBEDDING_MODEL_NAME_MISTRAL_EMBED ); private static final Set VALUES_FOR_NOMIC = ValueBuilder.getValuesFor( - Constants.EMBEDDING_MODEL_NAME_NOMIC_EMBED_TEXT + Constants.EMBEDDING_MODEL_NAME_NOMIC_EMBED_TEXT ); private static final Set VALUES_FOR_HUGGING_FACE = ValueBuilder.getValuesFor( - Constants.EMBEDDING_MODEL_NAME_FALCON_7B_INSTRUCT, - Constants.EMBEDDING_MODEL_NAME_MINI_LM_L6_V2 + Constants.EMBEDDING_MODEL_NAME_FALCON_7B_INSTRUCT, + Constants.EMBEDDING_MODEL_NAME_MINI_LM_L6_V2 + ); + + private static final Set VALUES_FOR_EINSTEIN = ValueBuilder.getValuesFor( + Constants.EMBEDDING_MODEL_NAME_SFDC_TEXT_EMBEDDING_ADA_002, + Constants.EMBEDDING_MODEL_NAME_SFDC_AZURE_TEXT_EMBEDDING_ADA_002, + Constants.EMBEDDING_MODEL_NAME_SFDC_OPENAI_TEXT_EMBEDDING_ADA_002 ); @Override public Set resolve() throws ValueResolvingException { - String embeddingModelService = configuration.getEmbeddingModelService(); + String embeddingModelService = configuration.getModelConfiguration().getEmbeddingModelService(); switch (embeddingModelService) { case Constants.EMBEDDING_MODEL_SERVICE_OPENAI: return VALUES_FOR_OPENAI; @@ -56,6 +62,8 @@ public Set resolve() throws ValueResolvingException { return VALUES_FOR_NOMIC; case Constants.EMBEDDING_MODEL_SERVICE_HUGGING_FACE: return VALUES_FOR_HUGGING_FACE; + case Constants.EMBEDDING_MODEL_SERVICE_EINSTEIN: + return VALUES_FOR_EINSTEIN; default: return Collections.emptySet(); } diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/FileTypeEmbeddingProvider.java b/src/main/java/org/mule/extension/vectors/internal/helper/provider/FileTypeEmbeddingProvider.java similarity index 80% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/FileTypeEmbeddingProvider.java rename to src/main/java/org/mule/extension/vectors/internal/helper/provider/FileTypeEmbeddingProvider.java index 8e17b18..97b1301 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/FileTypeEmbeddingProvider.java +++ b/src/main/java/org/mule/extension/vectors/internal/helper/provider/FileTypeEmbeddingProvider.java @@ -1,9 +1,9 @@ -package org.mule.extension.mulechain.vectors.internal.helper.provider; +package org.mule.extension.vectors.internal.helper.provider; import java.util.Set; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.constant.Constants; import org.mule.runtime.api.value.Value; import org.mule.runtime.extension.api.values.ValueBuilder; import org.mule.runtime.extension.api.values.ValueProvider; diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/MetadataFilterMethodProvider.java b/src/main/java/org/mule/extension/vectors/internal/helper/provider/MetadataFilterMethodProvider.java similarity index 70% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/MetadataFilterMethodProvider.java rename to src/main/java/org/mule/extension/vectors/internal/helper/provider/MetadataFilterMethodProvider.java index dcdfb78..95e9b8e 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/MetadataFilterMethodProvider.java +++ b/src/main/java/org/mule/extension/vectors/internal/helper/provider/MetadataFilterMethodProvider.java @@ -1,6 +1,6 @@ -package org.mule.extension.mulechain.vectors.internal.helper.provider; +package org.mule.extension.vectors.internal.helper.provider; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.constant.Constants; import org.mule.runtime.api.value.Value; import org.mule.runtime.extension.api.values.ValueBuilder; import org.mule.runtime.extension.api.values.ValueProvider; @@ -26,10 +26,10 @@ public class MetadataFilterMethodProvider implements ValueProvider { public Set resolve() throws ValueResolvingException { return ValueBuilder.getValuesFor( - Constants.METADATA_FILTER_METHOD_IS_EQUAL_TO, - Constants.METADATA_FILTER_METHOD_IS_NOT_EQUAL_TO, - Constants.METADATA_FILTER_METHOD_IS_GREATER_THAN, - Constants.METADATA_FILTER_METHOD_IS_LESS_THAN); // Additional methods can be added as needed + Constants.METADATA_FILTER_METHOD_IS_EQUAL_TO, + Constants.METADATA_FILTER_METHOD_IS_NOT_EQUAL_TO, + Constants.METADATA_FILTER_METHOD_IS_GREATER_THAN, + Constants.METADATA_FILTER_METHOD_IS_LESS_THAN); // Additional methods can be added as needed } } diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/MetadataKeyProvider.java b/src/main/java/org/mule/extension/vectors/internal/helper/provider/MetadataKeyProvider.java similarity index 85% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/MetadataKeyProvider.java rename to src/main/java/org/mule/extension/vectors/internal/helper/provider/MetadataKeyProvider.java index f504fcd..b2bfc22 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/helper/provider/MetadataKeyProvider.java +++ b/src/main/java/org/mule/extension/vectors/internal/helper/provider/MetadataKeyProvider.java @@ -1,6 +1,6 @@ -package org.mule.extension.mulechain.vectors.internal.helper.provider; +package org.mule.extension.vectors.internal.helper.provider; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.constant.Constants; import org.mule.runtime.api.value.Value; import org.mule.runtime.extension.api.values.ValueBuilder; import org.mule.runtime.extension.api.values.ValueProvider; diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/model/BaseModel.java b/src/main/java/org/mule/extension/vectors/internal/model/BaseModel.java similarity index 67% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/model/BaseModel.java rename to src/main/java/org/mule/extension/vectors/internal/model/BaseModel.java index e1bb16f..d039826 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/model/BaseModel.java +++ b/src/main/java/org/mule/extension/vectors/internal/model/BaseModel.java @@ -1,17 +1,17 @@ -package org.mule.extension.mulechain.vectors.internal.model; +package org.mule.extension.vectors.internal.model; import dev.langchain4j.model.embedding.EmbeddingModel; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.error.MuleVectorsErrorType; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.EmbeddingModelParameters; -import org.mule.extension.mulechain.vectors.internal.model.azureopenai.AzureOpenAIModel; -import org.mule.extension.mulechain.vectors.internal.model.huggingface.HuggingFaceModel; -import org.mule.extension.mulechain.vectors.internal.model.mistralai.MistralAIModel; -import org.mule.extension.mulechain.vectors.internal.model.nomic.NomicModel; -import org.mule.extension.mulechain.vectors.internal.model.openai.OpenAIModel; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.error.MuleVectorsErrorType; +import org.mule.extension.vectors.internal.helper.parameter.EmbeddingModelParameters; +import org.mule.extension.vectors.internal.model.azureopenai.AzureOpenAIModel; +import org.mule.extension.vectors.internal.model.einstein.EinsteinModel; +import org.mule.extension.vectors.internal.model.huggingface.HuggingFaceModel; +import org.mule.extension.vectors.internal.model.mistralai.MistralAIModel; +import org.mule.extension.vectors.internal.model.nomic.NomicModel; +import org.mule.extension.vectors.internal.model.openai.OpenAIModel; import org.mule.runtime.extension.api.exception.ModuleException; -import org.mule.runtime.module.extension.internal.runtime.operation.IllegalOperationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,8 +61,8 @@ public BaseModel build() { BaseModel baseModel; - LOGGER.debug("Embedding Model Service: " + configuration.getEmbeddingModelService()); - switch (configuration.getEmbeddingModelService()) { + LOGGER.debug("Embedding Model Service: " + configuration.getModelConfiguration().getEmbeddingModelService()); + switch (configuration.getModelConfiguration().getEmbeddingModelService()) { case Constants.EMBEDDING_MODEL_SERVICE_AZURE_OPENAI: baseModel = new AzureOpenAIModel(configuration, embeddingModelParameters); @@ -84,9 +84,13 @@ public BaseModel build() { baseModel = new HuggingFaceModel(configuration, embeddingModelParameters); break; + case Constants.EMBEDDING_MODEL_SERVICE_EINSTEIN: + baseModel = new EinsteinModel(configuration, embeddingModelParameters); + break; + default: throw new ModuleException( - String.format("Error while initializing embedding model service. \"%s\" is not supported.", configuration.getEmbeddingModelService()), + String.format("Error while initializing embedding model service. \"%s\" is not supported.", configuration.getModelConfiguration().getEmbeddingModelService()), MuleVectorsErrorType.AI_SERVICES_FAILURE); } return baseModel; diff --git a/src/main/java/org/mule/extension/vectors/internal/model/BaseModelConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/model/BaseModelConfiguration.java new file mode 100644 index 0000000..db5ad7c --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/BaseModelConfiguration.java @@ -0,0 +1,6 @@ +package org.mule.extension.vectors.internal.model; + +public interface BaseModelConfiguration { + + String getEmbeddingModelService(); +} diff --git a/src/main/java/org/mule/extension/vectors/internal/model/azureopenai/AzureOpenAIModel.java b/src/main/java/org/mule/extension/vectors/internal/model/azureopenai/AzureOpenAIModel.java new file mode 100644 index 0000000..e6e34ef --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/azureopenai/AzureOpenAIModel.java @@ -0,0 +1,35 @@ +package org.mule.extension.vectors.internal.model.azureopenai; + +import dev.langchain4j.model.azure.AzureOpenAiEmbeddingModel; +import dev.langchain4j.model.embedding.EmbeddingModel; +import org.json.JSONObject; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.helper.parameter.EmbeddingModelParameters; +import org.mule.extension.vectors.internal.model.BaseModel; + +import static org.mule.extension.vectors.internal.util.JsonUtils.readConfigFile; + +public class AzureOpenAIModel extends BaseModel { + + private final String endpoint; + private final String apiKey; + + public AzureOpenAIModel(Configuration configuration, EmbeddingModelParameters embeddingModelParameters) { + + super(configuration,embeddingModelParameters); + + AzureOpenAIModelConfiguration azureOpenAIModelConfiguration = (AzureOpenAIModelConfiguration) configuration.getModelConfiguration(); + this.endpoint = azureOpenAIModelConfiguration.getEndpoint(); + this.apiKey = azureOpenAIModelConfiguration.getApiKey(); + } + + public EmbeddingModel buildEmbeddingModel() { + + return AzureOpenAiEmbeddingModel.builder() + .apiKey(apiKey) + .endpoint(endpoint) + .deploymentName(embeddingModelParameters.getEmbeddingModelName()) + .build(); + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/model/azureopenai/AzureOpenAIModelConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/model/azureopenai/AzureOpenAIModelConfiguration.java new file mode 100644 index 0000000..058fe22 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/azureopenai/AzureOpenAIModelConfiguration.java @@ -0,0 +1,41 @@ +package org.mule.extension.vectors.internal.model.azureopenai; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.model.BaseModelConfiguration; +import org.mule.extension.vectors.internal.store.BaseStoreConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("azureOpenAI") +@DisplayName("Azure OpenAI") +public class AzureOpenAIModelConfiguration implements BaseModelConfiguration { + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String endpoint; + + @Parameter + @Password + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 2) + private String apiKey; + + @Override + public String getEmbeddingModelService() { + return Constants.EMBEDDING_MODEL_SERVICE_AZURE_OPENAI; + } + + public String getEndpoint() { + return endpoint; + } + + public String getApiKey() { + return apiKey; + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/model/einstein/EinsteinEmbeddingModel.java b/src/main/java/org/mule/extension/vectors/internal/model/einstein/EinsteinEmbeddingModel.java new file mode 100644 index 0000000..c7968eb --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/einstein/EinsteinEmbeddingModel.java @@ -0,0 +1,352 @@ +package org.mule.extension.vectors.internal.model.einstein; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.internal.Utils; +import dev.langchain4j.model.Tokenizer; +import dev.langchain4j.model.embedding.DimensionAwareEmbeddingModel; +import dev.langchain4j.model.output.Response; +import dev.langchain4j.model.output.TokenUsage; +import org.json.JSONArray; +import org.json.JSONObject; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.error.MuleVectorsErrorType; +import org.mule.runtime.extension.api.exception.ModuleException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.stream.Collectors; + +/** + * Implementation of Einstein AI's embedding model service that extends DimensionAwareEmbeddingModel. + * This class handles the generation of text embeddings using Salesforce Einstein API. + */ +public class EinsteinEmbeddingModel extends DimensionAwareEmbeddingModel { + + private static final Logger LOGGER = LoggerFactory.getLogger(EinsteinEmbeddingModel.class); + + private static final String URL_BASE = "https://api.salesforce.com/einstein/platform/v1/models/"; + + private final String modelName; + private final Integer dimensions; + private final String accessToken; + + /** + * Private constructor used by the builder pattern. + * Initializes the embedding model with Salesforce credentials and model configuration. + * + * @param salesforceOrgUrl The Salesforce organization identifier + * @param clientId OAuth client ID for authentication + * @param clientSecret OAuth client secret for authentication + * @param modelName Name of the Einstein embedding model to use + * @param dimensions Number of dimensions for the embeddings + */ + private EinsteinEmbeddingModel(String salesforceOrgUrl, String clientId, String clientSecret, String modelName, Integer dimensions) { + // Default to SFDC text embedding model if none specified + this.modelName = Utils.getOrDefault(modelName, Constants.EMBEDDING_MODEL_NAME_SFDC_TEXT_EMBEDDING_ADA_002); + this.dimensions = dimensions; + this.accessToken = getAccessToken(salesforceOrgUrl, clientId, clientSecret); + } + + /** + * Returns the known dimension of the embedding model. + * + * @return The dimension size of the embeddings + */ + protected Integer knownDimension() { + return this.dimensions != null ? this.dimensions : EinsteinEmbeddingModelName.knownDimension(this.modelName()); + } + + /** + * Returns the name of the current embedding model. + * + * @return The model name + */ + public String modelName() { + return this.modelName; + } + + /** + * Generates embeddings for a list of text segments. + * + * @param textSegments List of text segments to embed + * @return Response containing list of embeddings and token usage information + */ + public Response> embedAll(List textSegments) { + // Convert TextSegments to plain strings + List texts = textSegments.stream().map(TextSegment::text).collect(Collectors.toList()); + return this.embedTexts(texts); + } + + /** + * Internal method to process text strings and generate embeddings. + * Handles batching of requests to the Einstein API. + * + * @param texts List of text strings to embed + * @return Response containing embeddings and token usage + */ + private Response> embedTexts(List texts) { + List embeddings = new ArrayList<>(); + int tokenUsage = 0; + + // Process texts in batches of 16 (Einstein API limit) + for(int x = 0; x < texts.size(); x += 16) { + // Extract current batch + List batch = texts.subList(x, Math.min(x + 16, texts.size())); + + // Generate embeddings for current batch + String response = generateEmbeddings(buildPayload(batch)); + JSONObject jsonResponse = new JSONObject(response); + + // Accumulate token usage + tokenUsage += jsonResponse.getJSONObject("parameters") + .getJSONObject("usage") + .getInt("total_tokens"); + + // Parse embeddings from response + JSONArray embeddingsArray = jsonResponse.getJSONArray("embeddings"); + + // Process each embedding in the response + for (int i = 0; i < embeddingsArray.length(); i++) { + JSONObject embeddingObject = embeddingsArray.getJSONObject(i); + JSONArray embeddingArray = embeddingObject.getJSONArray("embedding"); + + // Convert JSON array to float array + float[] vector = new float[embeddingArray.length()]; + for (int y = 0; y < embeddingArray.length(); y++) { + vector[y] = (float) embeddingArray.getDouble(y); + } + + embeddings.add(Embedding.from(vector)); + } + } + + return Response.from(embeddings, new TokenUsage(tokenUsage)); + } + + /** + * Authenticates with Salesforce and obtains an access token. + * + * @param salesforceOrg Salesforce organization identifier + * @param clientId OAuth client ID + * @param clientSecret OAuth client secret + * @return Access token for API calls + * @throws ModuleException if authentication fails + */ + private String getAccessToken(String salesforceOrg, String clientId, String clientSecret) { + String urlString = salesforceOrg + "/services/oauth2/token"; + String params = "grant_type=client_credentials&client_id=" + clientId + "&client_secret=" + clientSecret; + + try { + URL url = new URL(urlString); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + + // Configure connection for OAuth token request + conn.setDoOutput(true); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + + // Write parameters to request body + try (OutputStream os = conn.getOutputStream()) { + byte[] input = params.getBytes(StandardCharsets.UTF_8); + os.write(input, 0, input.length); + } + + int responseCode = conn.getResponseCode(); + if (responseCode == HttpURLConnection.HTTP_OK) { + // Read and parse response + try (java.io.BufferedReader br = new java.io.BufferedReader( + new java.io.InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) { + StringBuilder response = new StringBuilder(); + String responseLine; + while ((responseLine = br.readLine()) != null) { + response.append(responseLine.trim()); + } + return new JSONObject(response.toString()).getString("access_token"); + } + } else { + throw new ModuleException( + "Error while getting access token for \"EINSTEIN\" embedding model service. Response code: " + responseCode, + MuleVectorsErrorType.AI_SERVICES_FAILURE); + } + } catch (Exception e) { + throw new ModuleException( + "Error while getting access token for \"EINSTEIN\" embedding model service.", + MuleVectorsErrorType.AI_SERVICES_FAILURE, + e); + } + } + + /** + * Creates and configures an HTTP connection for Einstein API requests. + * + * @param url The endpoint URL + * @param accessToken OAuth access token + * @return Configured HttpURLConnection + * @throws IOException if connection setup fails + */ + private HttpURLConnection getConnectionObject(URL url, String accessToken) throws IOException { + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setDoOutput(true); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Authorization", "Bearer " + accessToken); + conn.setRequestProperty("x-sfdc-app-context", "EinsteinGPT"); + conn.setRequestProperty("x-client-feature-id", "ai-platform-models-connected-app"); + conn.setRequestProperty("Content-Type", "application/json;charset=utf-8"); + return conn; + } + + /** + * Builds JSON payload for single text embedding request. + * + * @param text Text to embed + * @return JSON string payload + */ + private static String buildPayload(String text) { + JSONArray input = new JSONArray(); + input.put(text); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("input", input); + return jsonObject.toString(); + } + + /** + * Builds JSON payload for batch text embedding request. + * + * @param texts List of texts to embed + * @return JSON string payload + */ + private static String buildPayload(List texts) { + JSONArray input = new JSONArray(texts); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("input", input); + return jsonObject.toString(); + } + + /** + * Makes the API call to Einstein to generate embeddings. + * + * @param payload JSON payload for the request + * @return JSON response string + * @throws ModuleException if the API call fails + */ + private String generateEmbeddings(String payload) { + try { + // Prepare connection + String urlString = URL_BASE + this.modelName + "/embeddings"; + HttpURLConnection connection; + try { + URL url = new URL(urlString); + connection = getConnectionObject(url, this.accessToken); + } catch (Exception e) { + throw new ModuleException( + "Error while connecting to \"EINSTEIN\" embedding model service.", + MuleVectorsErrorType.AI_SERVICES_FAILURE, + e); + } + + // Send request + try (OutputStream os = connection.getOutputStream()) { + byte[] input = payload.getBytes(StandardCharsets.UTF_8); + os.write(input, 0, input.length); + } + + int responseCode = connection.getResponseCode(); + + if (responseCode == HttpURLConnection.HTTP_OK) { + // Read response + try (java.io.BufferedReader br = new java.io.BufferedReader( + new java.io.InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8))) { + StringBuilder response = new StringBuilder(); + String responseLine; + while ((responseLine = br.readLine()) != null) { + response.append(responseLine.trim()); + } + return response.toString(); + } + } else { + throw new ModuleException( + "Error while generating embeddings with \"EINSTEIN\" embedding model service. Response code: " + responseCode, + MuleVectorsErrorType.AI_SERVICES_FAILURE); + } + } catch (ModuleException e) { + throw e; + } catch (Exception e) { + throw new ModuleException( + "Error while generating embeddings with \"EINSTEIN\" embedding model service.", + MuleVectorsErrorType.AI_SERVICES_FAILURE, + e); + } + } + + /** + * Creates a new builder instance for EinsteinEmbeddingModel. + * + * @return A new builder instance + */ + public static EinsteinEmbeddingModel.EinsteinEmbeddingModelBuilder builder() { + return new EinsteinEmbeddingModel.EinsteinEmbeddingModelBuilder(); + } + + /** + * Builder class for EinsteinEmbeddingModel. + * Implements the Builder pattern for constructing EinsteinEmbeddingModel instances. + */ + public static class EinsteinEmbeddingModelBuilder { + private String salesforceOrgUrl; + private String clientId; + private String clientSecret; + private String modelName; + private Integer dimensions; + private Tokenizer tokenizer; + private HttpURLConnection connection; + + public EinsteinEmbeddingModelBuilder() { + } + + public EinsteinEmbeddingModel.EinsteinEmbeddingModelBuilder salesforceOrgUrl(String salesforceOrg) { + this.salesforceOrgUrl = salesforceOrg; + return this; + } + + public EinsteinEmbeddingModel.EinsteinEmbeddingModelBuilder clientId(String clientId) { + this.clientId = clientId; + return this; + } + + public EinsteinEmbeddingModel.EinsteinEmbeddingModelBuilder clientSecret(String clientSecret) { + this.clientSecret = clientSecret; + return this; + } + + public EinsteinEmbeddingModel.EinsteinEmbeddingModelBuilder modelName(String modelName) { + this.modelName = modelName; + return this; + } + + public EinsteinEmbeddingModel.EinsteinEmbeddingModelBuilder modelName(EinsteinEmbeddingModelName modelName) { + this.modelName = modelName.toString(); + return this; + } + + public EinsteinEmbeddingModel.EinsteinEmbeddingModelBuilder dimensions(Integer dimensions) { + this.dimensions = dimensions; + return this; + } + + /** + * Builds and returns a new EinsteinEmbeddingModel instance. + * + * @return A new EinsteinEmbeddingModel configured with the builder's parameters + */ + public EinsteinEmbeddingModel build() { + return new EinsteinEmbeddingModel(this.salesforceOrgUrl, this.clientId, this.clientSecret, this.modelName, this.dimensions); + } + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/model/einstein/EinsteinEmbeddingModelName.java b/src/main/java/org/mule/extension/vectors/internal/model/einstein/EinsteinEmbeddingModelName.java new file mode 100644 index 0000000..19e0f08 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/einstein/EinsteinEmbeddingModelName.java @@ -0,0 +1,44 @@ +package org.mule.extension.vectors.internal.model.einstein; + +import org.mule.extension.vectors.internal.constant.Constants; + +import java.util.HashMap; +import java.util.Map; + +public enum EinsteinEmbeddingModelName { + + DEFAULT_ADA_002(Constants.EMBEDDING_MODEL_NAME_SFDC_TEXT_EMBEDDING_ADA_002, 1536), + AZURE_OPENAI_ADA_002(Constants.EMBEDDING_MODEL_NAME_SFDC_AZURE_TEXT_EMBEDDING_ADA_002, 1536), + OPENAI_ADA_002(Constants.EMBEDDING_MODEL_NAME_SFDC_OPENAI_TEXT_EMBEDDING_ADA_002, 1536); + + private final String stringValue; + private final Integer dimension; + private static final Map KNOWN_DIMENSION = new HashMap(values().length); + + private EinsteinEmbeddingModelName(String stringValue, Integer dimension) { + this.stringValue = stringValue; + this.dimension = dimension; + } + + public String toString() { + return this.stringValue; + } + + public Integer dimension() { + return this.dimension; + } + + public static Integer knownDimension(String modelName) { + return (Integer)KNOWN_DIMENSION.get(modelName); + } + + static { + EinsteinEmbeddingModelName[] var0 = values(); + int var1 = var0.length; + + for(int var2 = 0; var2 < var1; ++var2) { + EinsteinEmbeddingModelName embeddingModelName = var0[var2]; + KNOWN_DIMENSION.put(embeddingModelName.toString(), embeddingModelName.dimension()); + } + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/model/einstein/EinsteinModel.java b/src/main/java/org/mule/extension/vectors/internal/model/einstein/EinsteinModel.java new file mode 100644 index 0000000..adf34c1 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/einstein/EinsteinModel.java @@ -0,0 +1,39 @@ +package org.mule.extension.vectors.internal.model.einstein; + +import dev.langchain4j.model.embedding.EmbeddingModel; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.helper.parameter.EmbeddingModelParameters; +import org.mule.extension.vectors.internal.model.BaseModel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EinsteinModel extends BaseModel { + + private static final Logger LOGGER = LoggerFactory.getLogger(EinsteinModel.class); + + private final String salesforceOrgUrl; + private final String clientId; + private final String clientSecret; + private final String modelName; + + public EinsteinModel(Configuration configuration, EmbeddingModelParameters embeddingModelParameters) { + + super(configuration,embeddingModelParameters); + + EinsteinModelConfiguration einsteinModelConfiguration = (EinsteinModelConfiguration) configuration.getModelConfiguration(); + this.salesforceOrgUrl = einsteinModelConfiguration.getSalesforceOrgUrl(); + this.clientId = einsteinModelConfiguration.getClientId(); + this.clientSecret = einsteinModelConfiguration.getClientSecret(); + this.modelName = embeddingModelParameters.getEmbeddingModelName(); + } + + public EmbeddingModel buildEmbeddingModel() { + + return EinsteinEmbeddingModel.builder() + .salesforceOrgUrl(salesforceOrgUrl) + .clientId(clientId) + .clientSecret(clientSecret) + .modelName(modelName) + .build(); + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/model/einstein/EinsteinModelConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/model/einstein/EinsteinModelConfiguration.java new file mode 100644 index 0000000..43f8b2f --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/einstein/EinsteinModelConfiguration.java @@ -0,0 +1,54 @@ +package org.mule.extension.vectors.internal.model.einstein; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.model.BaseModelConfiguration; +import org.mule.extension.vectors.internal.store.BaseStoreConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; +import org.mule.runtime.extension.api.annotation.param.display.Summary; + +@Alias("einstein") +@DisplayName("Einstein") +public class EinsteinModelConfiguration implements BaseModelConfiguration { + + @Parameter + @Alias("salesforceOrgUrl") + @DisplayName("Salesforce Org URL") + @Summary("The salesforce org base URL.") + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String salesforceOrgUrl; + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 2) + private String clientId; + + @Parameter + @Password + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 3) + private String clientSecret; + + @Override + public String getEmbeddingModelService() { + return Constants.EMBEDDING_MODEL_SERVICE_EINSTEIN; + } + + public String getSalesforceOrgUrl() { + return salesforceOrgUrl; + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/model/huggingface/HuggingFaceModel.java b/src/main/java/org/mule/extension/vectors/internal/model/huggingface/HuggingFaceModel.java new file mode 100644 index 0000000..22bc031 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/huggingface/HuggingFaceModel.java @@ -0,0 +1,31 @@ +package org.mule.extension.vectors.internal.model.huggingface; + +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.huggingface.HuggingFaceEmbeddingModel; +import org.json.JSONObject; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.helper.parameter.EmbeddingModelParameters; +import org.mule.extension.vectors.internal.model.BaseModel; + +import static org.mule.extension.vectors.internal.util.JsonUtils.readConfigFile; + +public class HuggingFaceModel extends BaseModel { + + private final String apiKey; + + public HuggingFaceModel(Configuration configuration, EmbeddingModelParameters embeddingModelParameters) { + + super(configuration,embeddingModelParameters); + + HuggingFaceModelConfiguration huggingFaceModelConfiguration =(HuggingFaceModelConfiguration) configuration.getModelConfiguration(); + this.apiKey = huggingFaceModelConfiguration.getApiKey(); + } + + public EmbeddingModel buildEmbeddingModel() { + + return HuggingFaceEmbeddingModel.builder() + .accessToken(apiKey) + .modelId(embeddingModelParameters.getEmbeddingModelName()) + .build(); + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/model/huggingface/HuggingFaceModelConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/model/huggingface/HuggingFaceModelConfiguration.java new file mode 100644 index 0000000..dbc5a94 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/huggingface/HuggingFaceModelConfiguration.java @@ -0,0 +1,31 @@ +package org.mule.extension.vectors.internal.model.huggingface; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.model.BaseModelConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("huggingFace") +@DisplayName("Hugging Face") +public class HuggingFaceModelConfiguration implements BaseModelConfiguration { + + @Parameter + @Password + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String apiKey; + + @Override + public String getEmbeddingModelService() { + return Constants.EMBEDDING_MODEL_SERVICE_HUGGING_FACE; + } + + public String getApiKey() { + return apiKey; + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/model/mistralai/MistralAIModel.java b/src/main/java/org/mule/extension/vectors/internal/model/mistralai/MistralAIModel.java new file mode 100644 index 0000000..c7307f5 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/mistralai/MistralAIModel.java @@ -0,0 +1,31 @@ +package org.mule.extension.vectors.internal.model.mistralai; + +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.mistralai.MistralAiEmbeddingModel; +import org.json.JSONObject; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.helper.parameter.EmbeddingModelParameters; +import org.mule.extension.vectors.internal.model.BaseModel; + +import static org.mule.extension.vectors.internal.util.JsonUtils.readConfigFile; + +public class MistralAIModel extends BaseModel { + + private final String apiKey; + + public MistralAIModel(Configuration configuration, EmbeddingModelParameters embeddingModelParameters) { + + super(configuration,embeddingModelParameters); + + MistralAIModelConfiguration mistralAIModelConfiguration = (MistralAIModelConfiguration) configuration.getModelConfiguration(); + this.apiKey = mistralAIModelConfiguration.getApiKey(); + } + + public EmbeddingModel buildEmbeddingModel() { + + return MistralAiEmbeddingModel.builder() + .apiKey(apiKey) + .modelName(embeddingModelParameters.getEmbeddingModelName()) + .build(); + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/model/mistralai/MistralAIModelConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/model/mistralai/MistralAIModelConfiguration.java new file mode 100644 index 0000000..db9552e --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/mistralai/MistralAIModelConfiguration.java @@ -0,0 +1,31 @@ +package org.mule.extension.vectors.internal.model.mistralai; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.model.BaseModelConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("mistralAI") +@DisplayName("Mistral AI") +public class MistralAIModelConfiguration implements BaseModelConfiguration { + + @Parameter + @Password + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String apiKey; + + @Override + public String getEmbeddingModelService() { + return Constants.EMBEDDING_MODEL_SERVICE_MISTRAL_AI; + } + + public String getApiKey() { + return apiKey; + } +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/model/nomic/NomicModel.java b/src/main/java/org/mule/extension/vectors/internal/model/nomic/NomicModel.java similarity index 54% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/model/nomic/NomicModel.java rename to src/main/java/org/mule/extension/vectors/internal/model/nomic/NomicModel.java index b00d61b..0873868 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/model/nomic/NomicModel.java +++ b/src/main/java/org/mule/extension/vectors/internal/model/nomic/NomicModel.java @@ -1,13 +1,13 @@ -package org.mule.extension.mulechain.vectors.internal.model.nomic; +package org.mule.extension.vectors.internal.model.nomic; import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.model.nomic.NomicEmbeddingModel; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.EmbeddingModelParameters; -import org.mule.extension.mulechain.vectors.internal.model.BaseModel; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.helper.parameter.EmbeddingModelParameters; +import org.mule.extension.vectors.internal.model.BaseModel; -import static org.mule.extension.mulechain.vectors.internal.util.JsonUtils.readConfigFile; +import static org.mule.extension.vectors.internal.util.JsonUtils.readConfigFile; public class NomicModel extends BaseModel { @@ -16,10 +16,9 @@ public class NomicModel extends BaseModel { public NomicModel(Configuration configuration, EmbeddingModelParameters embeddingModelParameters) { super(configuration,embeddingModelParameters); - JSONObject config = readConfigFile(configuration.getConfigFilePath()); - assert config != null; - JSONObject modelConfig = config.getJSONObject("NOMIC"); - this.apiKey = modelConfig.getString("NOMIC_API_KEY"); + + NomicModelConfiguration nomicModelConfiguration =(NomicModelConfiguration) configuration.getModelConfiguration(); + this.apiKey = nomicModelConfiguration.getApiKey(); } public EmbeddingModel buildEmbeddingModel() { diff --git a/src/main/java/org/mule/extension/vectors/internal/model/nomic/NomicModelConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/model/nomic/NomicModelConfiguration.java new file mode 100644 index 0000000..b006d3c --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/nomic/NomicModelConfiguration.java @@ -0,0 +1,31 @@ +package org.mule.extension.vectors.internal.model.nomic; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.model.BaseModelConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("nomic") +@DisplayName("Nomic") +public class NomicModelConfiguration implements BaseModelConfiguration { + + @Parameter + @Password + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String apiKey; + + @Override + public String getEmbeddingModelService() { + return Constants.EMBEDDING_MODEL_SERVICE_NOMIC; + } + + public String getApiKey() { + return apiKey; + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/model/openai/OpenAIModel.java b/src/main/java/org/mule/extension/vectors/internal/model/openai/OpenAIModel.java new file mode 100644 index 0000000..c28bde9 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/openai/OpenAIModel.java @@ -0,0 +1,31 @@ +package org.mule.extension.vectors.internal.model.openai; + +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.openai.OpenAiEmbeddingModel; +import org.json.JSONObject; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.helper.parameter.EmbeddingModelParameters; +import org.mule.extension.vectors.internal.model.BaseModel; + +import static org.mule.extension.vectors.internal.util.JsonUtils.readConfigFile; + +public class OpenAIModel extends BaseModel { + + private final String apiKey; + + public OpenAIModel(Configuration configuration, EmbeddingModelParameters embeddingModelParameters) { + + super(configuration,embeddingModelParameters); + + OpenAIModelConfiguration openAIModelConfiguration = (OpenAIModelConfiguration) configuration.getModelConfiguration(); + this.apiKey = openAIModelConfiguration.getApiKey(); + } + + public EmbeddingModel buildEmbeddingModel() { + + return OpenAiEmbeddingModel.builder() + .apiKey(apiKey) + .modelName(embeddingModelParameters.getEmbeddingModelName()) + .build(); + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/model/openai/OpenAIModelConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/model/openai/OpenAIModelConfiguration.java new file mode 100644 index 0000000..38a1d62 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/model/openai/OpenAIModelConfiguration.java @@ -0,0 +1,31 @@ +package org.mule.extension.vectors.internal.model.openai; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.model.BaseModelConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("openAI") +@DisplayName("OpenAI") +public class OpenAIModelConfiguration implements BaseModelConfiguration { + + @Parameter + @Password + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String apiKey; + + @Override + public String getEmbeddingModelService() { + return Constants.EMBEDDING_MODEL_SERVICE_OPENAI; + } + + public String getApiKey() { + return apiKey; + } +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/operation/DocumentOperations.java b/src/main/java/org/mule/extension/vectors/internal/operation/DocumentOperations.java similarity index 67% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/operation/DocumentOperations.java rename to src/main/java/org/mule/extension/vectors/internal/operation/DocumentOperations.java index 3187a7d..69e1556 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/operation/DocumentOperations.java +++ b/src/main/java/org/mule/extension/vectors/internal/operation/DocumentOperations.java @@ -1,4 +1,4 @@ -package org.mule.extension.mulechain.vectors.internal.operation; +package org.mule.extension.vectors.internal.operation; import dev.langchain4j.data.document.Document; import dev.langchain4j.data.document.DocumentSplitter; @@ -6,21 +6,22 @@ import dev.langchain4j.data.segment.TextSegment; import org.json.JSONArray; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.api.metadata.DocumentResponseAttributes; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.error.MuleVectorsErrorType; -import org.mule.extension.mulechain.vectors.internal.error.provider.DocumentErrorTypeProvider; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.DocumentParameters; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.FileTypeParameters; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.SegmentationParameters; -import org.mule.extension.mulechain.vectors.internal.storage.BaseStorage; +import org.mule.extension.vectors.api.metadata.DocumentResponseAttributes; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.error.MuleVectorsErrorType; +import org.mule.extension.vectors.internal.error.provider.DocumentErrorTypeProvider; +import org.mule.extension.vectors.internal.helper.parameter.DocumentParameters; +import org.mule.extension.vectors.internal.helper.parameter.SegmentationParameters; +import org.mule.extension.vectors.internal.storage.BaseStorage; +import org.mule.extension.vectors.internal.storage.BaseStorageConfiguration; import org.mule.runtime.extension.api.annotation.Alias; import org.mule.runtime.extension.api.annotation.error.Throws; import org.mule.runtime.extension.api.annotation.metadata.fixed.OutputJsonType; -import org.mule.runtime.extension.api.annotation.param.Config; -import org.mule.runtime.extension.api.annotation.param.MediaType; -import org.mule.runtime.extension.api.annotation.param.ParameterGroup; +import org.mule.runtime.extension.api.annotation.param.*; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Placement; +import org.mule.runtime.extension.api.annotation.param.display.Summary; import org.mule.runtime.extension.api.exception.ModuleException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,8 +31,9 @@ import java.util.List; import java.util.stream.IntStream; -import static org.mule.extension.mulechain.vectors.internal.helper.ResponseHelper.createDocumentResponse; +import static org.mule.extension.vectors.internal.helper.ResponseHelper.createDocumentResponse; import static org.mule.runtime.extension.api.annotation.param.MediaType.APPLICATION_JSON; +import static org.mule.runtime.extension.api.annotation.param.display.Placement.ADVANCED_TAB; public class DocumentOperations { @@ -45,15 +47,17 @@ public class DocumentOperations { @Throws(DocumentErrorTypeProvider.class) @OutputJsonType(schema = "api/response/DocumentSplitResponse.json") public org.mule.runtime.extension.api.runtime.operation.Result - documentSplitter(@Config Configuration configuration, - @ParameterGroup(name = "Document") DocumentParameters documentParameters, - @ParameterGroup(name = "Segmentation") SegmentationParameters segmentationParameters){ + documentSplitter( @Config Configuration configuration, + @ParameterGroup(name = "Document") DocumentParameters documentParameters, + @ConfigOverride @Alias("storage") @DisplayName(Constants.PARAM_DISPLAY_NAME_STORAGE_OVERRIDE) + BaseStorageConfiguration storageConfiguration, + @ParameterGroup(name = "Segmentation") SegmentationParameters segmentationParameters + ){ try { BaseStorage baseStorage = BaseStorage.builder() - .configuration(configuration) - .storageType(documentParameters.getStorageType()) + .storageConfiguration(storageConfiguration) .contextPath(documentParameters.getContextPath()) .fileType(documentParameters.getFileType()) .build(); @@ -101,13 +105,15 @@ public class DocumentOperations { @OutputJsonType(schema = "api/response/DocumentParseResponse.json") public org.mule.runtime.extension.api.runtime.operation.Result documentParser( @Config Configuration configuration, - @ParameterGroup(name = "Document") DocumentParameters documentParameters){ + @ConfigOverride @Alias("storage") @DisplayName(Constants.PARAM_DISPLAY_NAME_STORAGE_OVERRIDE) + BaseStorageConfiguration storageConfiguration, + @ParameterGroup(name = "Document") DocumentParameters documentParameters + ){ try { BaseStorage baseStorage = BaseStorage.builder() - .configuration(configuration) - .storageType(documentParameters.getStorageType()) + .storageConfiguration(storageConfiguration) .contextPath(documentParameters.getContextPath()) .fileType(documentParameters.getFileType()) .build(); diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/operation/EmbeddingOperations.java b/src/main/java/org/mule/extension/vectors/internal/operation/EmbeddingOperations.java similarity index 91% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/operation/EmbeddingOperations.java rename to src/main/java/org/mule/extension/vectors/internal/operation/EmbeddingOperations.java index 25bc74d..fead0a1 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/operation/EmbeddingOperations.java +++ b/src/main/java/org/mule/extension/vectors/internal/operation/EmbeddingOperations.java @@ -1,6 +1,6 @@ -package org.mule.extension.mulechain.vectors.internal.operation; +package org.mule.extension.vectors.internal.operation; -import static org.mule.extension.mulechain.vectors.internal.helper.ResponseHelper.createEmbeddingResponse; +import static org.mule.extension.vectors.internal.helper.ResponseHelper.createEmbeddingResponse; import static org.mule.runtime.extension.api.annotation.param.MediaType.APPLICATION_JSON; import java.io.InputStream; @@ -11,22 +11,23 @@ import dev.langchain4j.data.document.Document; import dev.langchain4j.data.document.DocumentSplitter; -import org.mule.extension.mulechain.vectors.api.metadata.EmbeddingResponseAttributes; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.error.MuleVectorsErrorType; -import org.mule.extension.mulechain.vectors.internal.error.provider.EmbeddingErrorTypeProvider; -import org.mule.extension.mulechain.vectors.internal.helper.EmbeddingOperationValidator; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.*; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; +import org.mule.extension.vectors.api.metadata.EmbeddingResponseAttributes; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.error.MuleVectorsErrorType; +import org.mule.extension.vectors.internal.error.provider.EmbeddingErrorTypeProvider; +import org.mule.extension.vectors.internal.helper.EmbeddingOperationValidator; +import org.mule.extension.vectors.internal.helper.parameter.*; +import org.mule.extension.vectors.internal.config.Configuration; import dev.langchain4j.store.embedding.*; import dev.langchain4j.store.embedding.filter.Filter; import org.json.JSONArray; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.model.BaseModel; -import org.mule.extension.mulechain.vectors.internal.storage.BaseStorage; -import org.mule.extension.mulechain.vectors.internal.store.BaseStore; -import org.mule.extension.mulechain.vectors.internal.util.MetadatatUtils; -import org.mule.extension.mulechain.vectors.internal.util.JsonUtils; +import org.mule.extension.vectors.internal.model.BaseModel; +import org.mule.extension.vectors.internal.storage.BaseStorage; +import org.mule.extension.vectors.internal.storage.BaseStorageConfiguration; +import org.mule.extension.vectors.internal.store.BaseStore; +import org.mule.extension.vectors.internal.util.MetadatatUtils; +import org.mule.extension.vectors.internal.util.JsonUtils; import org.mule.runtime.extension.api.annotation.Alias; import org.mule.runtime.extension.api.annotation.error.Throws; import org.mule.runtime.extension.api.annotation.metadata.fixed.OutputJsonType; @@ -40,6 +41,7 @@ import dev.langchain4j.model.embedding.EmbeddingModel; import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Placement; import org.mule.runtime.extension.api.exception.ModuleException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -192,14 +194,16 @@ public class EmbeddingOperations { public org.mule.runtime.extension.api.runtime.operation.Result addFolderToStore( @Config Configuration configuration, @Alias("storeName") @DisplayName("Store Name") String storeName, - @ParameterGroup(name = "Documents") DocumentParameters documentParameters, + @ConfigOverride @Alias("storage") @DisplayName(Constants.PARAM_DISPLAY_NAME_STORAGE_OVERRIDE) + BaseStorageConfiguration storageConfiguration, + @ParameterGroup(name = "Documents") DocumentParameters documentParameters, @ParameterGroup(name = "Segmentation") SegmentationParameters segmentationParameters, @ParameterGroup(name = "Embedding Model") EmbeddingModelParameters embeddingModelParameters){ try { EmbeddingOperationValidator.validateOperationType( - Constants.EMBEDDING_OPERATION_TYPE_STORE_METADATA,configuration.getVectorStore()); + Constants.EMBEDDING_OPERATION_TYPE_STORE_METADATA,configuration.getStoreConfiguration().getVectorStore()); BaseModel baseModel = BaseModel.builder() .configuration(configuration) @@ -223,8 +227,7 @@ public class EmbeddingOperations { .build(); BaseStorage baseStorage = BaseStorage.builder() - .configuration(configuration) - .storageType(documentParameters.getStorageType()) + .storageConfiguration(storageConfiguration) .contextPath(documentParameters.getContextPath()) .fileType(documentParameters.getFileType()) .build(); @@ -244,7 +247,7 @@ public class EmbeddingOperations { new HashMap() {{ put("documentCount", finalDocumentNumber); put("storeName", storeName); - put("storageType", documentParameters.getStorageType()); + put("storageType", baseStorage.getStorageType()); put("fileType", documentParameters.getFileType()); put("contextPath", documentParameters.getContextPath()); }}); @@ -272,6 +275,8 @@ public class EmbeddingOperations { public org.mule.runtime.extension.api.runtime.operation.Result addFileEmbedding( @Config Configuration configuration, @Alias("storeName") @DisplayName("Store Name") String storeName, + @ConfigOverride @Alias("storage") @DisplayName(Constants.PARAM_DISPLAY_NAME_STORAGE_OVERRIDE) + BaseStorageConfiguration storageConfiguration, @ParameterGroup(name = "Document") DocumentParameters documentParameters, @ParameterGroup(name = "Segmentation") SegmentationParameters segmentationParameters, @ParameterGroup(name = "Embedding Model") EmbeddingModelParameters embeddingModelParameters) { @@ -279,7 +284,7 @@ public class EmbeddingOperations { try { EmbeddingOperationValidator.validateOperationType( - Constants.EMBEDDING_OPERATION_TYPE_STORE_METADATA,configuration.getVectorStore()); + Constants.EMBEDDING_OPERATION_TYPE_STORE_METADATA,configuration.getStoreConfiguration().getVectorStore()); BaseModel baseModel = BaseModel.builder() .configuration(configuration) @@ -303,8 +308,7 @@ public class EmbeddingOperations { .build(); BaseStorage baseStorage = BaseStorage.builder() - .configuration(configuration) - .storageType(documentParameters.getStorageType()) + .storageConfiguration(storageConfiguration) .contextPath(documentParameters.getContextPath()) .fileType(documentParameters.getFileType()) .build(); @@ -318,7 +322,7 @@ public class EmbeddingOperations { jsonObject.toString(), new HashMap() {{ put("storeName", storeName); - put("storageType", documentParameters.getStorageType()); + put("storageType", baseStorage.getStorageType()); put("fileType", documentParameters.getFileType()); put("contextPath", documentParameters.getContextPath()); }}); @@ -451,7 +455,7 @@ public class EmbeddingOperations { try { EmbeddingOperationValidator.validateOperationType( - Constants.EMBEDDING_OPERATION_TYPE_FILTER_BY_METADATA,configuration.getVectorStore()); + Constants.EMBEDDING_OPERATION_TYPE_FILTER_BY_METADATA,configuration.getStoreConfiguration().getVectorStore()); int maximumResults = (int) maxResults; if (minScore == null) { //|| minScore == 0) { @@ -570,9 +574,9 @@ public class EmbeddingOperations { try { EmbeddingOperationValidator.validateOperationType( - Constants.EMBEDDING_OPERATION_TYPE_QUERY_ALL,configuration.getVectorStore()); + Constants.EMBEDDING_OPERATION_TYPE_QUERY_ALL,configuration.getStoreConfiguration().getVectorStore()); EmbeddingOperationValidator.validateOperationType( - Constants.EMBEDDING_OPERATION_TYPE_FILTER_BY_METADATA,configuration.getVectorStore()); + Constants.EMBEDDING_OPERATION_TYPE_FILTER_BY_METADATA,configuration.getStoreConfiguration().getVectorStore()); BaseStore baseStore = BaseStore.builder() .storeName(storeName) @@ -616,9 +620,9 @@ public class EmbeddingOperations { try { EmbeddingOperationValidator.validateOperationType( - Constants.EMBEDDING_OPERATION_TYPE_REMOVE_EMBEDDINGS,configuration.getVectorStore()); + Constants.EMBEDDING_OPERATION_TYPE_REMOVE_EMBEDDINGS,configuration.getStoreConfiguration().getVectorStore()); EmbeddingOperationValidator.validateOperationType( - Constants.EMBEDDING_OPERATION_TYPE_FILTER_BY_METADATA,configuration.getVectorStore()); + Constants.EMBEDDING_OPERATION_TYPE_FILTER_BY_METADATA,configuration.getStoreConfiguration().getVectorStore()); BaseModel baseModel = BaseModel.builder() .configuration(configuration) diff --git a/src/main/java/org/mule/extension/vectors/internal/storage/BaseStorage.java b/src/main/java/org/mule/extension/vectors/internal/storage/BaseStorage.java new file mode 100644 index 0000000..375b6e6 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/storage/BaseStorage.java @@ -0,0 +1,153 @@ +package org.mule.extension.vectors.internal.storage; + +import dev.langchain4j.data.document.Document; +import dev.langchain4j.data.document.DocumentParser; +import dev.langchain4j.data.document.parser.TextDocumentParser; +import dev.langchain4j.data.document.parser.apache.tika.ApacheTikaDocumentParser; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.error.MuleVectorsErrorType; +import org.mule.extension.vectors.internal.storage.azureblob.AzureBlobStorage; +import org.mule.extension.vectors.internal.storage.azureblob.AzureBlobStorageConfiguration; +import org.mule.extension.vectors.internal.storage.local.LocalStorage; +import org.mule.extension.vectors.internal.storage.local.LocalStorageConfiguration; +import org.mule.extension.vectors.internal.storage.s3.AWSS3Storage; +import org.mule.extension.vectors.internal.storage.s3.AWSS3StorageConfiguration; +import org.mule.runtime.extension.api.exception.ModuleException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Iterator; + +public abstract class BaseStorage implements Iterator { + + private static final Logger LOGGER = LoggerFactory.getLogger(BaseStorage.class); + + protected BaseStorageConfiguration storageConfiguration; + protected String contextPath; + protected String fileType; + protected DocumentParser documentParser; + + public BaseStorage(BaseStorageConfiguration storageConfiguration, String contextPath, String fileType) { + + this.storageConfiguration = storageConfiguration; + this.contextPath = contextPath; + this.fileType = fileType; + this.documentParser = getDocumentParser(fileType); + } + + @Override + public boolean hasNext() { + throw new UnsupportedOperationException("This method should be overridden by subclasses"); + } + + @Override + public Document next() { + throw new UnsupportedOperationException("This method should be overridden by subclasses"); + } + + public Document getSingleDocument() { + throw new UnsupportedOperationException("This method should be overridden by subclasses"); + } + + public String getStorageType() { + + return storageConfiguration == null ? Constants.STORAGE_TYPE_LOCAL : storageConfiguration.getStorageType(); + } + + protected DocumentParser getDocumentParser(String fileType) { + + DocumentParser documentParser = null; + switch (fileType){ + + case Constants.FILE_TYPE_TEXT: + case Constants.FILE_TYPE_CRAWL: + case Constants.FILE_TYPE_URL: + documentParser = new TextDocumentParser(); + break; + case Constants.FILE_TYPE_ANY: + documentParser = new ApacheTikaDocumentParser(); + break; + default: + throw new IllegalArgumentException("Unsupported File Type: " + fileType); + } + return documentParser; + } + + public static BaseStorage.Builder builder() { + + return new BaseStorage.Builder(); + } + + public static class Builder { + + private BaseStorageConfiguration storageConfiguration; + private String contextPath; + private String fileType; + + public Builder() { + + } + + public BaseStorage.Builder storageConfiguration(BaseStorageConfiguration storageConfiguration) { + this.storageConfiguration = storageConfiguration; + return this; + } + + public BaseStorage.Builder contextPath(String contextPath) { + this.contextPath = contextPath; + return this; + } + + public BaseStorage.Builder fileType(String fileType) { + this.fileType = fileType; + return this; + } + + public BaseStorage build() { + + BaseStorage baseStorage; + + String storageType = storageConfiguration == null ? Constants.STORAGE_TYPE_LOCAL : storageConfiguration.getStorageType(); + + try { + + LOGGER.debug("Storage Type: " + storageConfiguration.getStorageType()); + switch (storageType) { + + case Constants.STORAGE_TYPE_LOCAL: + + baseStorage = new LocalStorage((LocalStorageConfiguration) storageConfiguration, contextPath, fileType); + break; + + case Constants.STORAGE_TYPE_AWS_S3: + + baseStorage = new AWSS3Storage((AWSS3StorageConfiguration) storageConfiguration, contextPath, fileType); + break; + + case Constants.STORAGE_TYPE_AZURE_BLOB: + + baseStorage = new AzureBlobStorage((AzureBlobStorageConfiguration) storageConfiguration, contextPath, fileType); + break; + + default: + + throw new ModuleException( + String.format("Error while initializing storage. Type \"%s\" is not supported.", storageType), + MuleVectorsErrorType.STORAGE_SERVICES_FAILURE); + } + + } catch (ModuleException e) { + + throw e; + + } catch (Exception e) { + + throw new ModuleException( + String.format("Error while initializing storage type \"%s\".", storageType), + MuleVectorsErrorType.STORAGE_SERVICES_FAILURE, + e); + } + return baseStorage; + } + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/storage/BaseStorageConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/storage/BaseStorageConfiguration.java new file mode 100644 index 0000000..7b1d519 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/storage/BaseStorageConfiguration.java @@ -0,0 +1,6 @@ +package org.mule.extension.vectors.internal.storage; + +public interface BaseStorageConfiguration { + + String getStorageType(); +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/storage/azureblob/AzureBlobStorage.java b/src/main/java/org/mule/extension/vectors/internal/storage/azureblob/AzureBlobStorage.java similarity index 77% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/storage/azureblob/AzureBlobStorage.java rename to src/main/java/org/mule/extension/vectors/internal/storage/azureblob/AzureBlobStorage.java index 40583e9..1b84e78 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/storage/azureblob/AzureBlobStorage.java +++ b/src/main/java/org/mule/extension/vectors/internal/storage/azureblob/AzureBlobStorage.java @@ -1,4 +1,4 @@ -package org.mule.extension.mulechain.vectors.internal.storage.azureblob; +package org.mule.extension.vectors.internal.storage.azureblob; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; @@ -10,15 +10,14 @@ import java.util.Iterator; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.config.Configuration; import dev.langchain4j.data.document.Document; -import org.mule.extension.mulechain.vectors.internal.storage.BaseStorage; -import org.mule.extension.mulechain.vectors.internal.util.MetadatatUtils; +import org.mule.extension.vectors.internal.storage.BaseStorage; +import org.mule.extension.vectors.internal.util.MetadatatUtils; +import org.mule.extension.vectors.internal.util.JsonUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.mule.extension.mulechain.vectors.internal.util.JsonUtils.readConfigFile; - public class AzureBlobStorage extends BaseStorage { private static final Logger LOGGER = LoggerFactory.getLogger(AzureBlobStorage.class); @@ -71,14 +70,11 @@ private Iterator getBlobIterator() { return blobIterator; } - public AzureBlobStorage(Configuration configuration, String contextPath, String fileType) { + public AzureBlobStorage(AzureBlobStorageConfiguration azureBlobStorageDocumentLoader, String contextPath, String fileType) { - super(configuration, contextPath, fileType); - JSONObject config = readConfigFile(configuration.getConfigFilePath()); - assert config != null; - JSONObject storageConfig = config.getJSONObject("AZURE_BLOB"); - this.azureName = storageConfig.getString("AZURE_BLOB_ACCOUNT_NAME"); - this.azureKey = storageConfig.getString("AZURE_BLOB_ACCOUNT_KEY"); + super(azureBlobStorageDocumentLoader, contextPath, fileType); + this.azureName = azureBlobStorageDocumentLoader.getAzureName(); + this.azureKey = azureBlobStorageDocumentLoader.getAzureKey(); } @Override diff --git a/src/main/java/org/mule/extension/vectors/internal/storage/azureblob/AzureBlobStorageConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/storage/azureblob/AzureBlobStorageConfiguration.java new file mode 100644 index 0000000..cdf4c28 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/storage/azureblob/AzureBlobStorageConfiguration.java @@ -0,0 +1,45 @@ +package org.mule.extension.vectors.internal.storage.azureblob; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.storage.BaseStorageConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("azureBlob") +@DisplayName("Azure Blob") +public class AzureBlobStorageConfiguration implements BaseStorageConfiguration { + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String azureName; + + @Parameter + @Password + @Placement(order = 2) + private String azureKey; + + @Override + public String getStorageType() { return Constants.STORAGE_TYPE_AZURE_BLOB; } + + public String getAzureName() { + return azureName; + } + + public String getAzureKey() { + return azureKey; + } + + public void setAzureName(String azureName) { + this.azureName = azureName; + } + + public void setAzureKey(String azureKey) { + this.azureKey = azureKey; + } +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/storage/local/LocalStorage.java b/src/main/java/org/mule/extension/vectors/internal/storage/local/LocalStorage.java similarity index 74% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/storage/local/LocalStorage.java rename to src/main/java/org/mule/extension/vectors/internal/storage/local/LocalStorage.java index 43401a3..40eda39 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/storage/local/LocalStorage.java +++ b/src/main/java/org/mule/extension/vectors/internal/storage/local/LocalStorage.java @@ -1,14 +1,17 @@ -package org.mule.extension.mulechain.vectors.internal.storage.local; +package org.mule.extension.vectors.internal.storage.local; import dev.langchain4j.data.document.Document; import dev.langchain4j.data.document.DocumentParser; import dev.langchain4j.data.document.loader.UrlDocumentLoader; import dev.langchain4j.data.document.transformer.jsoup.HtmlToTextDocumentTransformer; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.storage.BaseStorage; -import org.mule.extension.mulechain.vectors.internal.util.MetadatatUtils; -import org.mule.extension.mulechain.vectors.internal.util.Utils; +import okhttp3.internal.Util; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.storage.BaseStorage; +import org.mule.extension.vectors.internal.util.MetadatatUtils; +import org.mule.extension.vectors.internal.util.Utils; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,12 +32,14 @@ public class LocalStorage extends BaseStorage { private static final Logger LOGGER = LoggerFactory.getLogger(LocalStorage.class); + private final String fullPath; + private List pathList; private Iterator pathIterator; private Iterator getPathIterator() { if (pathList == null) { // Only load files if not already loaded - try (Stream paths = Files.walk(Paths.get(contextPath))) { + try (Stream paths = Files.walk(Paths.get(fullPath))) { // Collect all files as a list pathList = paths.filter(Files::isRegularFile).collect(Collectors.toList()); // Create an iterator for the list of files @@ -46,9 +51,17 @@ private Iterator getPathIterator() { return pathIterator; } - public LocalStorage(Configuration configuration, String contextPath, String fileType) { + public LocalStorage(LocalStorageConfiguration localStorageConfiguration, String contextPath, String fileType) { + + super(localStorageConfiguration, contextPath, fileType); + this.fullPath = dev.langchain4j.internal.Utils.getOrDefault(localStorageConfiguration.getWorkingDirectory(), "") + + contextPath; + } + + public LocalStorage(String contextPath, String fileType) { - super(configuration, contextPath, fileType); + super(null, contextPath, fileType); + this.fullPath = contextPath; } // Override hasNext to check if there are files left to process @@ -72,7 +85,7 @@ public Document next() { public Document getSingleDocument() { - Path path = Paths.get(contextPath); + Path path = Paths.get(fullPath); DocumentParser documentParser = getDocumentParser(fileType); @@ -82,7 +95,7 @@ public Document getSingleDocument() { case Constants.FILE_TYPE_TEXT: case Constants.FILE_TYPE_ANY: document = loadDocument(path.toString(), documentParser); - MetadatatUtils.addMetadataToDocument(document, fileType, Utils.getFileNameFromPath(contextPath)); + MetadatatUtils.addMetadataToDocument(document, fileType, Utils.getFileNameFromPath(fullPath)); break; case Constants.FILE_TYPE_URL: document = loadUrlDocument(contextPath); diff --git a/src/main/java/org/mule/extension/vectors/internal/storage/local/LocalStorageConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/storage/local/LocalStorageConfiguration.java new file mode 100644 index 0000000..a7a6ca2 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/storage/local/LocalStorageConfiguration.java @@ -0,0 +1,37 @@ +package org.mule.extension.vectors.internal.storage.local; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.storage.BaseStorageConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Optional; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Placement; +import org.mule.runtime.extension.api.annotation.param.display.Summary; + +@Alias("local") +@DisplayName("Local") +public class LocalStorageConfiguration implements BaseStorageConfiguration { + + @Parameter + @Alias("workingDirecotry") + @DisplayName("Working Directory") + @Summary("The working directory. It represents the root for context paths.") + @Placement(order = 1) + @Expression(ExpressionSupport.SUPPORTED) + @Optional + private String workingDirectory; + + public String getWorkingDirectory() { + return workingDirectory; + } + + @Override + public String getStorageType() { return Constants.STORAGE_TYPE_LOCAL; } + + public void setWorkingDirectory(String workingDirectory) { + this.workingDirectory = workingDirectory; + } +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/storage/s3/AWSS3Storage.java b/src/main/java/org/mule/extension/vectors/internal/storage/s3/AWSS3Storage.java similarity index 64% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/storage/s3/AWSS3Storage.java rename to src/main/java/org/mule/extension/vectors/internal/storage/s3/AWSS3Storage.java index 430c615..26599bb 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/storage/s3/AWSS3Storage.java +++ b/src/main/java/org/mule/extension/vectors/internal/storage/s3/AWSS3Storage.java @@ -1,5 +1,9 @@ -package org.mule.extension.mulechain.vectors.internal.storage.s3; +package org.mule.extension.vectors.internal.storage.s3; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.storage.BaseStorage; +import org.mule.extension.vectors.internal.util.JsonUtils; +import org.mule.extension.vectors.internal.util.MetadatatUtils; import software.amazon.awssdk.regions.Region; import dev.langchain4j.data.document.loader.amazon.s3.AmazonS3DocumentLoader; import dev.langchain4j.data.document.loader.amazon.s3.AwsCredentials; @@ -7,10 +11,7 @@ import java.util.Iterator; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; import dev.langchain4j.data.document.Document; -import org.mule.extension.mulechain.vectors.internal.storage.BaseStorage; -import org.mule.extension.mulechain.vectors.internal.util.MetadatatUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; @@ -20,8 +21,6 @@ import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.S3Object; -import static org.mule.extension.mulechain.vectors.internal.util.JsonUtils.readConfigFile; - public class AWSS3Storage extends BaseStorage { private static final Logger LOGGER = LoggerFactory.getLogger(AWSS3Storage.class); @@ -29,7 +28,6 @@ public class AWSS3Storage extends BaseStorage { private final String awsAccessKeyId; private final String awsSecretAccessKey; private final String awsRegion; - private final String awsS3Bucket; private String continuationToken = null; @@ -82,7 +80,7 @@ private Iterator getS3ObjectIterator() { // Build the request ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder() - .bucket(awsS3Bucket); + .bucket(getAWSS3Bucket()); if (continuationToken != null) { requestBuilder.continuationToken(continuationToken); // Set continuation token } @@ -96,16 +94,12 @@ private Iterator getS3ObjectIterator() { return s3ObjectIterator; } - public AWSS3Storage(Configuration configuration, String contextPath, String fileType) { + public AWSS3Storage(AWSS3StorageConfiguration awsS3StorageConfiguration, String contextPath, String fileType) { - super(configuration, contextPath, fileType); - JSONObject config = readConfigFile(configuration.getConfigFilePath()); - assert config != null; - JSONObject storageConfig = config.getJSONObject("S3"); - this.awsAccessKeyId = storageConfig.getString("AWS_ACCESS_KEY_ID"); - this.awsSecretAccessKey = storageConfig.getString("AWS_SECRET_ACCESS_KEY"); - this.awsRegion = storageConfig.getString("AWS_DEFAULT_REGION"); - this.awsS3Bucket = storageConfig.getString("AWS_S3_BUCKET"); + super(awsS3StorageConfiguration, contextPath, fileType); + this.awsAccessKeyId = awsS3StorageConfiguration.getAwsAccessKeyId(); + this.awsSecretAccessKey = awsS3StorageConfiguration.getAwsSecretAccessKey(); + this.awsRegion = awsS3StorageConfiguration.getAwsRegion(); } @Override @@ -118,17 +112,42 @@ public boolean hasNext() { public Document next() { S3Object object = getS3ObjectIterator().next(); - LOGGER.debug("AWS S3 Key: " + object.key()); - Document document = getLoader().loadDocument(awsS3Bucket, object.key(), documentParser); + LOGGER.debug("AWS S3 Object Key: " + object.key()); + Document document = getLoader().loadDocument(getAWSS3Bucket(), object.key(), documentParser); MetadatatUtils.addMetadataToDocument(document, fileType, object.key()); return document; } public Document getSingleDocument() { - LOGGER.debug("AWS S3 Key: " + contextPath); - Document document = getLoader().loadDocument(awsS3Bucket, contextPath, documentParser); - MetadatatUtils.addMetadataToDocument(document, fileType, contextPath); + LOGGER.debug("S3 URL: " + contextPath); + Document document = getLoader().loadDocument(getAWSS3Bucket(), getAWSS3ObjectKey(), documentParser); + MetadatatUtils.addMetadataToDocument(document, fileType, getAWSS3ObjectKey()); return document; } + + private String getAWSS3Bucket() { + + String s3Url = this.contextPath; + // Remove the "s3://" prefix + if (s3Url.startsWith("s3://") || s3Url.startsWith("S3://")) { + s3Url = s3Url.substring(5); + } + // Extract the bucket name + String bucket = s3Url.contains("/") ? s3Url.substring(0, s3Url.indexOf("/")) : s3Url; + return bucket; + } + + private String getAWSS3ObjectKey() { + + String s3Url = this.contextPath; + // Remove the "s3://" prefix + if (s3Url.startsWith("s3://") || s3Url.startsWith("S3://")) { + s3Url = s3Url.substring(5); + } + // Extract the bucket name and object key + int slashIndex = s3Url.indexOf("/"); + String objectKey = slashIndex != -1 ? s3Url.substring(slashIndex + 1) : ""; + return objectKey; + } } diff --git a/src/main/java/org/mule/extension/vectors/internal/storage/s3/AWSS3StorageConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/storage/s3/AWSS3StorageConfiguration.java new file mode 100644 index 0000000..b8de1ae --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/storage/s3/AWSS3StorageConfiguration.java @@ -0,0 +1,60 @@ +package org.mule.extension.vectors.internal.storage.s3; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.storage.BaseStorageConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("awsS3") +@DisplayName("AWS S3") +public class AWSS3StorageConfiguration implements BaseStorageConfiguration { + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String awsRegion; + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 2) + private String awsAccessKeyId; + + @Parameter + @Password + @Placement(order = 3) + private String awsSecretAccessKey; + + @Override + public String getStorageType() { + return Constants.STORAGE_TYPE_AWS_S3; + } + + public String getAwsRegion() { + return awsRegion; + } + + public String getAwsAccessKeyId() { + return awsAccessKeyId; + } + + public String getAwsSecretAccessKey() { + return awsSecretAccessKey; + } + + public void setAwsRegion(String awsRegion) { + this.awsRegion = awsRegion; + } + + public void setAwsAccessKeyId(String awsAccessKeyId) { + this.awsAccessKeyId = awsAccessKeyId; + } + + public void setAwsSecretAccessKey(String awsSecretAccessKey) { + this.awsSecretAccessKey = awsSecretAccessKey; + } +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/BaseStore.java b/src/main/java/org/mule/extension/vectors/internal/store/BaseStore.java similarity index 90% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/store/BaseStore.java rename to src/main/java/org/mule/extension/vectors/internal/store/BaseStore.java index 44ff8c8..c6069ed 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/BaseStore.java +++ b/src/main/java/org/mule/extension/vectors/internal/store/BaseStore.java @@ -1,21 +1,22 @@ -package org.mule.extension.mulechain.vectors.internal.store; +package org.mule.extension.vectors.internal.store; import dev.langchain4j.data.embedding.Embedding; import dev.langchain4j.data.segment.TextSegment; -import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.store.embedding.EmbeddingStore; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.QueryParameters; -import org.mule.extension.mulechain.vectors.internal.store.aisearch.AISearchStore; -import org.mule.extension.mulechain.vectors.internal.store.chroma.ChromaStore; -import org.mule.extension.mulechain.vectors.internal.store.elasticsearch.ElasticsearchStore; -import org.mule.extension.mulechain.vectors.internal.store.milvus.MilvusStore; -import org.mule.extension.mulechain.vectors.internal.store.opensearch.OpenSearchStore; -import org.mule.extension.mulechain.vectors.internal.store.pgvector.PGVectorStore; -import org.mule.extension.mulechain.vectors.internal.store.pinecone.PineconeStore; -import org.mule.extension.mulechain.vectors.internal.store.qdrant.QdrantStore; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.error.MuleVectorsErrorType; +import org.mule.extension.vectors.internal.helper.parameter.QueryParameters; +import org.mule.extension.vectors.internal.store.aisearch.AISearchStore; +import org.mule.extension.vectors.internal.store.chroma.ChromaStore; +import org.mule.extension.vectors.internal.store.elasticsearch.ElasticsearchStore; +import org.mule.extension.vectors.internal.store.milvus.MilvusStore; +import org.mule.extension.vectors.internal.store.opensearch.OpenSearchStore; +import org.mule.extension.vectors.internal.store.pgvector.PGVectorStore; +import org.mule.extension.vectors.internal.store.pinecone.PineconeStore; +import org.mule.extension.vectors.internal.store.qdrant.QdrantStore; +import org.mule.runtime.extension.api.exception.ModuleException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -257,7 +258,7 @@ public BaseStore build() { BaseStore baseStore; - switch (configuration.getVectorStore()) { + switch (configuration.getStoreConfiguration().getVectorStore()) { case Constants.VECTOR_STORE_MILVUS: @@ -300,8 +301,9 @@ public BaseStore build() { break; default: - //throw new IllegalOperationException("Unsupported Vector Store: " + configuration.getVectorStore()); - baseStore = null; + throw new ModuleException( + String.format("Error while initializing embedding store. \"%s\" not supported.", configuration.getStoreConfiguration().getVectorStore()), + MuleVectorsErrorType.STORE_SERVICES_FAILURE); } return baseStore; } diff --git a/src/main/java/org/mule/extension/vectors/internal/store/BaseStoreConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/store/BaseStoreConfiguration.java new file mode 100644 index 0000000..7d175e4 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/store/BaseStoreConfiguration.java @@ -0,0 +1,6 @@ +package org.mule.extension.vectors.internal.store; + +public interface BaseStoreConfiguration { + + String getVectorStore(); +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/aisearch/AISearchStore.java b/src/main/java/org/mule/extension/vectors/internal/store/aisearch/AISearchStore.java similarity index 86% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/store/aisearch/AISearchStore.java rename to src/main/java/org/mule/extension/vectors/internal/store/aisearch/AISearchStore.java index fd3f1eb..18b559c 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/aisearch/AISearchStore.java +++ b/src/main/java/org/mule/extension/vectors/internal/store/aisearch/AISearchStore.java @@ -1,15 +1,15 @@ -package org.mule.extension.mulechain.vectors.internal.store.aisearch; +package org.mule.extension.vectors.internal.store.aisearch; import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.store.embedding.EmbeddingStore; import dev.langchain4j.store.embedding.azure.search.AzureAiSearchEmbeddingStore; import org.json.JSONArray; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.QueryParameters; -import org.mule.extension.mulechain.vectors.internal.store.BaseStore; -import org.mule.extension.mulechain.vectors.internal.util.JsonUtils; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.helper.parameter.QueryParameters; +import org.mule.extension.vectors.internal.store.BaseStore; +import org.mule.extension.vectors.internal.util.JsonUtils; import java.io.BufferedReader; import java.io.InputStreamReader; @@ -17,23 +17,22 @@ import java.net.URL; import java.util.HashMap; -import static org.mule.extension.mulechain.vectors.internal.util.JsonUtils.readConfigFile; +import static org.mule.extension.vectors.internal.util.JsonUtils.readConfigFile; public class AISearchStore extends BaseStore { private static final String API_VERSION = "2024-07-01"; - private String apiKey; - private String url; + private final String apiKey; + private final String url; public AISearchStore(String storeName, Configuration configuration, QueryParameters queryParams, int dimension) { super(storeName, configuration, queryParams, dimension); - JSONObject config = readConfigFile(configuration.getConfigFilePath()); - JSONObject vectorStoreConfig = config.getJSONObject(Constants.VECTOR_STORE_AI_SEARCH); - this.apiKey = vectorStoreConfig.getString("AI_SEARCH_KEY"); - this.url = vectorStoreConfig.getString("AI_SEARCH_URL"); + AISearchStoreConfiguration aiSearchStoreConfiguration = (AISearchStoreConfiguration) configuration.getStoreConfiguration(); + this.url = aiSearchStoreConfiguration.getUrl(); + this.apiKey = aiSearchStoreConfiguration.getApiKey(); } public EmbeddingStore buildEmbeddingStore() { diff --git a/src/main/java/org/mule/extension/vectors/internal/store/aisearch/AISearchStoreConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/store/aisearch/AISearchStoreConfiguration.java new file mode 100644 index 0000000..83a75a1 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/store/aisearch/AISearchStoreConfiguration.java @@ -0,0 +1,42 @@ +package org.mule.extension.vectors.internal.store.aisearch; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.store.BaseStoreConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +import javax.annotation.processing.SupportedOptions; + +@Alias("aiSearch") +@DisplayName("AI Search") +public class AISearchStoreConfiguration implements BaseStoreConfiguration { + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String url; + + @Parameter + @Password + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 2) + private String apiKey; + + @Override + public String getVectorStore() { + return Constants.VECTOR_STORE_AI_SEARCH; + } + + public String getUrl() { + return url; + } + + public String getApiKey() { + return apiKey; + } +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/chroma/ChromaStore.java b/src/main/java/org/mule/extension/vectors/internal/store/chroma/ChromaStore.java similarity index 92% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/store/chroma/ChromaStore.java rename to src/main/java/org/mule/extension/vectors/internal/store/chroma/ChromaStore.java index 720281a..6e91602 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/chroma/ChromaStore.java +++ b/src/main/java/org/mule/extension/vectors/internal/store/chroma/ChromaStore.java @@ -1,15 +1,16 @@ -package org.mule.extension.mulechain.vectors.internal.store.chroma; +package org.mule.extension.vectors.internal.store.chroma; import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.store.embedding.EmbeddingStore; import dev.langchain4j.store.embedding.chroma.ChromaEmbeddingStore; import org.json.JSONArray; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.QueryParameters; -import org.mule.extension.mulechain.vectors.internal.store.BaseStore; -import org.mule.extension.mulechain.vectors.internal.util.JsonUtils; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.helper.parameter.QueryParameters; +import org.mule.extension.vectors.internal.store.BaseStore; +import org.mule.extension.vectors.internal.store.aisearch.AISearchStoreConfiguration; +import org.mule.extension.vectors.internal.util.JsonUtils; import java.io.BufferedReader; import java.io.InputStreamReader; @@ -24,7 +25,7 @@ */ public class ChromaStore extends BaseStore { - private String url; + private final String url; /** @@ -38,9 +39,8 @@ public ChromaStore(String storeName, Configuration configuration, QueryParameter super(storeName, configuration, queryParams, dimension); - JSONObject config = JsonUtils.readConfigFile(configuration.getConfigFilePath()); - JSONObject vectorStoreConfig = config.getJSONObject(Constants.VECTOR_STORE_CHROMA); - this.url = vectorStoreConfig.getString("CHROMA_URL"); + ChromaStoreConfiguration chromaStoreConfiguration = (ChromaStoreConfiguration) configuration.getStoreConfiguration(); + this.url = chromaStoreConfiguration.getUrl(); } public EmbeddingStore buildEmbeddingStore() { diff --git a/src/main/java/org/mule/extension/vectors/internal/store/chroma/ChromaStoreConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/store/chroma/ChromaStoreConfiguration.java new file mode 100644 index 0000000..54aafae --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/store/chroma/ChromaStoreConfiguration.java @@ -0,0 +1,29 @@ +package org.mule.extension.vectors.internal.store.chroma; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.store.BaseStoreConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("chroma") +@DisplayName("Chroma") +public class ChromaStoreConfiguration implements BaseStoreConfiguration { + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String url; + + @Override + public String getVectorStore() { + return Constants.VECTOR_STORE_CHROMA; + } + + public String getUrl() { + return url; + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/store/elasticsearch/ElasticsearchStore.java b/src/main/java/org/mule/extension/vectors/internal/store/elasticsearch/ElasticsearchStore.java new file mode 100644 index 0000000..416e2f5 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/store/elasticsearch/ElasticsearchStore.java @@ -0,0 +1,40 @@ +package org.mule.extension.vectors.internal.store.elasticsearch; + +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.store.embedding.EmbeddingStore; +import dev.langchain4j.store.embedding.elasticsearch.ElasticsearchConfiguration; +import dev.langchain4j.store.embedding.elasticsearch.ElasticsearchEmbeddingStore; +import org.json.JSONObject; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.helper.parameter.QueryParameters; +import org.mule.extension.vectors.internal.store.BaseStore; +import org.mule.extension.vectors.internal.util.JsonUtils; + +public class ElasticsearchStore extends BaseStore { + + private final String url; + private final String userName; + private final String password; + + public ElasticsearchStore(String storeName, Configuration configuration, QueryParameters queryParams, int dimension) { + + super(storeName, configuration, queryParams, dimension); + + ElasticsearchStoreConfiguration elasticsearchStoreConfiguration = (ElasticsearchStoreConfiguration) configuration.getStoreConfiguration(); + this.url = elasticsearchStoreConfiguration.getUrl(); + this.userName = elasticsearchStoreConfiguration.getUserName(); + this.password = elasticsearchStoreConfiguration.getPassword(); + } + + public EmbeddingStore buildEmbeddingStore() { + + return ElasticsearchEmbeddingStore.builder() + .serverUrl(url) + .userName(userName) + .password(password) + .indexName(storeName) + .dimension(dimension) + .build(); + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/store/elasticsearch/ElasticsearchStoreConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/store/elasticsearch/ElasticsearchStoreConfiguration.java new file mode 100644 index 0000000..1ae7311 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/store/elasticsearch/ElasticsearchStoreConfiguration.java @@ -0,0 +1,49 @@ +package org.mule.extension.vectors.internal.store.elasticsearch; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.store.BaseStoreConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("elasticsearch") +@DisplayName("Elasticsearch") +public class ElasticsearchStoreConfiguration implements BaseStoreConfiguration { + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String url; + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 2) + private String userName; + + @Parameter + @Password + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 3) + private String password; + + @Override + public String getVectorStore() { + return Constants.VECTOR_STORE_ELASTICSEARCH; + } + + public String getUrl() { + return url; + } + + public String getUserName() { + return userName; + } + + public String getPassword() { + return password; + } +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/milvus/MilvusStore.java b/src/main/java/org/mule/extension/vectors/internal/store/milvus/MilvusStore.java similarity index 80% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/store/milvus/MilvusStore.java rename to src/main/java/org/mule/extension/vectors/internal/store/milvus/MilvusStore.java index d4b29a0..a0a52a1 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/milvus/MilvusStore.java +++ b/src/main/java/org/mule/extension/vectors/internal/store/milvus/MilvusStore.java @@ -1,4 +1,4 @@ -package org.mule.extension.mulechain.vectors.internal.store.milvus; +package org.mule.extension.vectors.internal.store.milvus; import com.google.gson.JsonObject; import dev.langchain4j.data.segment.TextSegment; @@ -11,30 +11,29 @@ import io.milvus.param.R; import io.milvus.response.QueryResultsWrapper; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.QueryParameters; -import org.mule.extension.mulechain.vectors.internal.store.BaseStore; -import org.mule.extension.mulechain.vectors.internal.util.JsonUtils; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.helper.parameter.QueryParameters; +import org.mule.extension.vectors.internal.store.BaseStore; +import org.mule.extension.vectors.internal.util.JsonUtils; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; -import static org.mule.extension.mulechain.vectors.internal.util.JsonUtils.readConfigFile; +import static org.mule.extension.vectors.internal.util.JsonUtils.readConfigFile; public class MilvusStore extends BaseStore { - private String uri; + private final String uri; public MilvusStore(String storeName, Configuration configuration, QueryParameters queryParams, int dimension) { super(storeName, configuration, queryParams, dimension); - JSONObject config = readConfigFile(configuration.getConfigFilePath()); - JSONObject vectorStoreConfig = config.getJSONObject(Constants.VECTOR_STORE_MILVUS); - this.uri = vectorStoreConfig.getString("MILVUS_URL"); + MilvusStoreConfiguration milvusStoreConfiguration = (MilvusStoreConfiguration) configuration.getStoreConfiguration(); + this.uri = milvusStoreConfiguration.getUrl(); } public EmbeddingStore buildEmbeddingStore() { diff --git a/src/main/java/org/mule/extension/vectors/internal/store/milvus/MilvusStoreConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/store/milvus/MilvusStoreConfiguration.java new file mode 100644 index 0000000..39927c5 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/store/milvus/MilvusStoreConfiguration.java @@ -0,0 +1,29 @@ +package org.mule.extension.vectors.internal.store.milvus; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.store.BaseStoreConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("milvus") +@DisplayName("Milvus") +public class MilvusStoreConfiguration implements BaseStoreConfiguration { + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String url; + + @Override + public String getVectorStore() { + return Constants.VECTOR_STORE_MILVUS; + } + + public String getUrl() { + return url; + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/store/opensearch/OpenSearchStore.java b/src/main/java/org/mule/extension/vectors/internal/store/opensearch/OpenSearchStore.java new file mode 100644 index 0000000..267d025 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/store/opensearch/OpenSearchStore.java @@ -0,0 +1,38 @@ +package org.mule.extension.vectors.internal.store.opensearch; + +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.store.embedding.EmbeddingStore; +import dev.langchain4j.store.embedding.opensearch.OpenSearchEmbeddingStore; +import org.json.JSONObject; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.helper.parameter.QueryParameters; +import org.mule.extension.vectors.internal.store.BaseStore; +import org.mule.extension.vectors.internal.util.JsonUtils; + +public class OpenSearchStore extends BaseStore { + + private final String url; + private final String userName; + private final String password; + + public OpenSearchStore(String storeName, Configuration configuration, QueryParameters queryParams, int dimension) { + + super(storeName, configuration, queryParams, dimension); + + OpenSearchStoreConfiguration openSearchStoreConfiguration = (OpenSearchStoreConfiguration) configuration.getStoreConfiguration(); + this.url = openSearchStoreConfiguration.getUrl(); + this.userName = openSearchStoreConfiguration.getUserName(); + this.password = openSearchStoreConfiguration.getPassword(); + } + + public EmbeddingStore buildEmbeddingStore() { + + return OpenSearchEmbeddingStore.builder() + .serverUrl(url) + .userName(userName) + .password(password) + .indexName(storeName) + .build(); + } +} diff --git a/src/main/java/org/mule/extension/vectors/internal/store/opensearch/OpenSearchStoreConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/store/opensearch/OpenSearchStoreConfiguration.java new file mode 100644 index 0000000..4a5b2af --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/store/opensearch/OpenSearchStoreConfiguration.java @@ -0,0 +1,49 @@ +package org.mule.extension.vectors.internal.store.opensearch; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.store.BaseStoreConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("openSearch") +@DisplayName("OpenSearch") +public class OpenSearchStoreConfiguration implements BaseStoreConfiguration { + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String url; + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 2) + private String userName; + + @Parameter + @Password + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 3) + private String password; + + @Override + public String getVectorStore() { + return Constants.VECTOR_STORE_OPENSEARCH; + } + + public String getUrl() { + return url; + } + + public String getUserName() { + return userName; + } + + public String getPassword() { + return password; + } +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/pgvector/PGVectorStore.java b/src/main/java/org/mule/extension/vectors/internal/store/pgvector/PGVectorStore.java similarity index 86% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/store/pgvector/PGVectorStore.java rename to src/main/java/org/mule/extension/vectors/internal/store/pgvector/PGVectorStore.java index 438aa9a..e766ce3 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/pgvector/PGVectorStore.java +++ b/src/main/java/org/mule/extension/vectors/internal/store/pgvector/PGVectorStore.java @@ -1,14 +1,14 @@ -package org.mule.extension.mulechain.vectors.internal.store.pgvector; +package org.mule.extension.vectors.internal.store.pgvector; import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.store.embedding.EmbeddingStore; import dev.langchain4j.store.embedding.pgvector.PgVectorEmbeddingStore; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.QueryParameters; -import org.mule.extension.mulechain.vectors.internal.store.BaseStore; -import org.mule.extension.mulechain.vectors.internal.util.JsonUtils; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.helper.parameter.QueryParameters; +import org.mule.extension.vectors.internal.store.BaseStore; +import org.mule.extension.vectors.internal.util.JsonUtils; import org.postgresql.ds.PGSimpleDataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +21,7 @@ import java.util.Iterator; import java.util.NoSuchElementException; -import static org.mule.extension.mulechain.vectors.internal.util.JsonUtils.readConfigFile; +import static org.mule.extension.vectors.internal.util.JsonUtils.readConfigFile; /** * Represents a store for vector data using PostgreSQL with PGVector extension. @@ -48,13 +48,12 @@ public PGVectorStore(String storeName, Configuration configuration, QueryParamet super(storeName, configuration, queryParams, dimension); - JSONObject config = readConfigFile(configuration.getConfigFilePath()); - JSONObject vectorStoreConfig = config.getJSONObject(Constants.VECTOR_STORE_PGVECTOR); - this.host = vectorStoreConfig.getString("POSTGRES_HOST"); - this.port = vectorStoreConfig.getInt("POSTGRES_PORT"); - this.database = vectorStoreConfig.getString("POSTGRES_DATABASE"); - this.userName = vectorStoreConfig.getString("POSTGRES_USER"); - this.password = vectorStoreConfig.getString("POSTGRES_PASSWORD"); + PGVectorStoreConfiguration pgVectorStoreConfiguration = (PGVectorStoreConfiguration)configuration.getStoreConfiguration(); + this.host = pgVectorStoreConfiguration.getHost(); + this.port = pgVectorStoreConfiguration.getPort(); + this.database = pgVectorStoreConfiguration.getDatabase(); + this.userName = pgVectorStoreConfiguration.getUserName(); + this.password = pgVectorStoreConfiguration.getPassword(); } public EmbeddingStore buildEmbeddingStore() { diff --git a/src/main/java/org/mule/extension/vectors/internal/store/pgvector/PGVectorStoreConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/store/pgvector/PGVectorStoreConfiguration.java new file mode 100644 index 0000000..64cdcae --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/store/pgvector/PGVectorStoreConfiguration.java @@ -0,0 +1,67 @@ +package org.mule.extension.vectors.internal.store.pgvector; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.store.BaseStoreConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("pgVector") +@DisplayName("PGVector") +public class PGVectorStoreConfiguration implements BaseStoreConfiguration { + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String host; + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 2) + private int port; + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 3) + private String database; + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 4) + private String userName; + + @Parameter + @Password + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 5) + private String password; + + @Override + public String getVectorStore() { + return Constants.VECTOR_STORE_PGVECTOR; + } + + public String getHost() { + return host; + } + + public int getPort() { + return port; + } + + public String getDatabase() { + return database; + } + + public String getUserName() { + return userName; + } + + public String getPassword() { + return password; + } +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/pinecone/PineconeStore.java b/src/main/java/org/mule/extension/vectors/internal/store/pinecone/PineconeStore.java similarity index 55% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/store/pinecone/PineconeStore.java rename to src/main/java/org/mule/extension/vectors/internal/store/pinecone/PineconeStore.java index 29ca64a..b13bd84 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/pinecone/PineconeStore.java +++ b/src/main/java/org/mule/extension/vectors/internal/store/pinecone/PineconeStore.java @@ -1,15 +1,15 @@ -package org.mule.extension.mulechain.vectors.internal.store.pinecone; +package org.mule.extension.vectors.internal.store.pinecone; import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.store.embedding.EmbeddingStore; import dev.langchain4j.store.embedding.pinecone.PineconeEmbeddingStore; import dev.langchain4j.store.embedding.pinecone.PineconeServerlessIndexConfig; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.QueryParameters; -import org.mule.extension.mulechain.vectors.internal.store.BaseStore; -import org.mule.extension.mulechain.vectors.internal.util.JsonUtils; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.helper.parameter.QueryParameters; +import org.mule.extension.vectors.internal.store.BaseStore; +import org.mule.extension.vectors.internal.util.JsonUtils; public class PineconeStore extends BaseStore { @@ -21,11 +21,10 @@ public PineconeStore(String storeName, Configuration configuration, QueryParamet super(storeName, configuration, queryParams, dimension); - JSONObject config = JsonUtils.readConfigFile(configuration.getConfigFilePath()); - JSONObject vectorStoreConfig = config.getJSONObject(Constants.VECTOR_STORE_PINECONE); - this.apiKey = vectorStoreConfig.getString("PINECONE_APIKEY"); - this.cloud = vectorStoreConfig.getString("PINECONE_SERVERLESS_CLOUD"); - this.region = vectorStoreConfig.getString("PINECONE_SERVERLESS_REGION"); + PineconeStoreConfiguration pineconeStoreConfiguration = (PineconeStoreConfiguration) configuration.getStoreConfiguration(); + this.apiKey = pineconeStoreConfiguration.getApiKey(); + this.cloud = pineconeStoreConfiguration.getCloud(); + this.region = pineconeStoreConfiguration.getRegion(); } public EmbeddingStore buildEmbeddingStore() { diff --git a/src/main/java/org/mule/extension/vectors/internal/store/pinecone/PineconeStoreConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/store/pinecone/PineconeStoreConfiguration.java new file mode 100644 index 0000000..b1f6e5b --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/store/pinecone/PineconeStoreConfiguration.java @@ -0,0 +1,49 @@ +package org.mule.extension.vectors.internal.store.pinecone; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.store.BaseStoreConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("pinecone") +@DisplayName("Pinecone") +public class PineconeStoreConfiguration implements BaseStoreConfiguration { + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String cloud; + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 2) + private String region; + + @Parameter + @Password + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 3) + private String apiKey; + + @Override + public String getVectorStore() { + return Constants.VECTOR_STORE_PINECONE; + } + + public String getCloud() { + return cloud; + } + + public String getRegion() { + return region; + } + + public String getApiKey() { + return apiKey; + } +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/qdrant/QdrantStore.java b/src/main/java/org/mule/extension/vectors/internal/store/qdrant/QdrantStore.java similarity index 80% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/store/qdrant/QdrantStore.java rename to src/main/java/org/mule/extension/vectors/internal/store/qdrant/QdrantStore.java index 65472ae..35bcb37 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/store/qdrant/QdrantStore.java +++ b/src/main/java/org/mule/extension/vectors/internal/store/qdrant/QdrantStore.java @@ -1,4 +1,4 @@ -package org.mule.extension.mulechain.vectors.internal.store.qdrant; +package org.mule.extension.vectors.internal.store.qdrant; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Struct; @@ -13,11 +13,13 @@ import io.qdrant.client.grpc.JsonWithInt; import io.qdrant.client.grpc.Points; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.config.Configuration; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; -import org.mule.extension.mulechain.vectors.internal.helper.parameter.QueryParameters; -import org.mule.extension.mulechain.vectors.internal.store.BaseStore; -import org.mule.extension.mulechain.vectors.internal.util.JsonUtils; +import org.mule.extension.vectors.internal.config.Configuration; +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.error.MuleVectorsErrorType; +import org.mule.extension.vectors.internal.helper.parameter.QueryParameters; +import org.mule.extension.vectors.internal.store.BaseStore; +import org.mule.extension.vectors.internal.util.JsonUtils; +import org.mule.runtime.extension.api.exception.ModuleException; import java.util.*; import java.util.concurrent.ExecutionException; @@ -31,24 +33,27 @@ public QdrantStore(String storeName, Configuration configuration, QueryParameter super(storeName, configuration, queryParams, dimension); - JSONObject config = JsonUtils.readConfigFile(configuration.getConfigFilePath()); - JSONObject vectorStoreConfig = config.getJSONObject(Constants.VECTOR_STORE_QDRANT); - String host = vectorStoreConfig.getString("QDRANT_HOST"); - String apiKey = vectorStoreConfig.getString("QDRANT_API_KEY"); - int port = vectorStoreConfig.getInt("QDRANT_GRPC_PORT"); - boolean useTls = vectorStoreConfig.getBoolean("QDRANT_USE_TLS"); - this.client = new QdrantClient(QdrantGrpcClient.newBuilder(host, port, useTls).withApiKey(apiKey).build()); - this.payloadTextKey = vectorStoreConfig.getString("QDRANT_TEXT_KEY"); - try { + + QdrantStoreConfiguration qdrantStoreConfiguration = (QdrantStoreConfiguration)configuration.getStoreConfiguration(); + String host = qdrantStoreConfiguration.getHost(); + String apiKey = qdrantStoreConfiguration.getApiKey(); + int port = qdrantStoreConfiguration.getGprcPort(); + boolean useTls = qdrantStoreConfiguration.isUseTLS(); + this.client = new QdrantClient(QdrantGrpcClient.newBuilder(host, port, useTls).withApiKey(apiKey).build()); + this.payloadTextKey = qdrantStoreConfiguration.getTextSegmentKey(); + if (!this.client.collectionExistsAsync(this.storeName).get() && dimension > 0) { this.client.createCollectionAsync(storeName, Collections.VectorParams.newBuilder().setDistance(Collections.Distance.Cosine) .setSize(dimension).build()) .get(); } - } catch (ExecutionException | InterruptedException e) { - throw new RuntimeException(e); + } catch (Exception e) { + + throw new ModuleException( + String.format("Error while initializing embedding store \"%s\".", configuration.getStoreConfiguration().getVectorStore()), + MuleVectorsErrorType.STORE_SERVICES_FAILURE); } } diff --git a/src/main/java/org/mule/extension/vectors/internal/store/qdrant/QdrantStoreConfiguration.java b/src/main/java/org/mule/extension/vectors/internal/store/qdrant/QdrantStoreConfiguration.java new file mode 100644 index 0000000..8fc47f6 --- /dev/null +++ b/src/main/java/org/mule/extension/vectors/internal/store/qdrant/QdrantStoreConfiguration.java @@ -0,0 +1,70 @@ +package org.mule.extension.vectors.internal.store.qdrant; + +import org.mule.extension.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.store.BaseStoreConfiguration; +import org.mule.runtime.api.meta.ExpressionSupport; +import org.mule.runtime.extension.api.annotation.Alias; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Password; +import org.mule.runtime.extension.api.annotation.param.display.Placement; + +@Alias("qdrant") +@DisplayName("Qdrant") +public class QdrantStoreConfiguration implements BaseStoreConfiguration { + + private final String vectorStore = Constants.VECTOR_STORE_OPENSEARCH; + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 1) + private String host; + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @DisplayName("GPRC Port") + @Placement(order = 2) + private int gprcPort; + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 3) + private boolean useTLS; + + @Parameter + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 4) + private String textSegmentKey; + + @Parameter + @Password + @Expression(ExpressionSupport.SUPPORTED) + @Placement(order = 5) + private String apiKey; + + @Override + public String getVectorStore() { + return Constants.VECTOR_STORE_QDRANT; + } + + public String getHost() { + return host; + } + + public int getGprcPort() { + return gprcPort; + } + + public boolean isUseTLS() { + return useTLS; + } + + public String getTextSegmentKey() { + return textSegmentKey; + } + + public String getApiKey() { + return apiKey; + } +} diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/util/JsonUtils.java b/src/main/java/org/mule/extension/vectors/internal/util/JsonUtils.java similarity index 89% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/util/JsonUtils.java rename to src/main/java/org/mule/extension/vectors/internal/util/JsonUtils.java index e544385..8e1bf88 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/util/JsonUtils.java +++ b/src/main/java/org/mule/extension/vectors/internal/util/JsonUtils.java @@ -1,16 +1,15 @@ /** * (c) 2003-2024 MuleSoft, Inc. The software in this package is published under the terms of the Commercial Free Software license V.1 a copy of which has been included with this distribution in the LICENSE.md file. */ -package org.mule.extension.mulechain.vectors.internal.util; +package org.mule.extension.vectors.internal.util; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import opennlp.tools.parser.Cons; import org.json.JSONArray; import org.json.JSONObject; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.constant.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,7 +61,6 @@ public static JSONArray jsonObjectCollectionToJsonArray(Collection j public static JSONObject createIngestionStatusObject(String storeName) { JSONObject jsonObject = new JSONObject(); - jsonObject.put(Constants.JSON_KEY_STORE_NAME, storeName); jsonObject.put(Constants.JSON_KEY_STATUS, Constants.OPERATION_STATUS_UPDATED); return jsonObject; } diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/util/MetadatatUtils.java b/src/main/java/org/mule/extension/vectors/internal/util/MetadatatUtils.java similarity index 91% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/util/MetadatatUtils.java rename to src/main/java/org/mule/extension/vectors/internal/util/MetadatatUtils.java index af4312a..aefabd9 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/util/MetadatatUtils.java +++ b/src/main/java/org/mule/extension/vectors/internal/util/MetadatatUtils.java @@ -1,9 +1,8 @@ -package org.mule.extension.mulechain.vectors.internal.util; +package org.mule.extension.vectors.internal.util; import com.fasterxml.jackson.databind.JsonNode; import dev.langchain4j.data.document.Document; -import dev.langchain4j.data.document.Metadata; -import org.mule.extension.mulechain.vectors.internal.constant.Constants; +import org.mule.extension.vectors.internal.constant.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/mule/extension/mulechain/vectors/internal/util/Utils.java b/src/main/java/org/mule/extension/vectors/internal/util/Utils.java similarity index 98% rename from src/main/java/org/mule/extension/mulechain/vectors/internal/util/Utils.java rename to src/main/java/org/mule/extension/vectors/internal/util/Utils.java index 9a777a0..97bf363 100644 --- a/src/main/java/org/mule/extension/mulechain/vectors/internal/util/Utils.java +++ b/src/main/java/org/mule/extension/vectors/internal/util/Utils.java @@ -1,4 +1,4 @@ -package org.mule.extension.mulechain.vectors.internal.util; +package org.mule.extension.vectors.internal.util; import java.io.File; import java.time.Instant; diff --git a/src/test/java/org/mule/extension/mulechain/vectors/EmbeddingOperationsTest.java b/src/test/java/org/mule/extension/mulechain/vectors/EmbeddingOperationsTest.java index 55b0efa..8133199 100644 --- a/src/test/java/org/mule/extension/mulechain/vectors/EmbeddingOperationsTest.java +++ b/src/test/java/org/mule/extension/mulechain/vectors/EmbeddingOperationsTest.java @@ -1,4 +1,4 @@ -package org.mule.extension.mulechain.vectors; +package org.mule.extension.vectors; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is;