[DERCBOT-1138] PGVector (#1735)

* [DERCBOT-1138] PGVector - A new vector store

* [DERCBOT-1138] PGVector - Reviews

* [DERCBOT-1138] PGVector - Update dependencies

bot/admin/server/src/main/kotlin/BotAdminVerticle.kt

@@ -57,6 +57,7 @@ import ai.tock.translator.I18nLabel
 import ai.tock.translator.Translator
 import ai.tock.translator.Translator.initTranslator
 import ai.tock.translator.TranslatorEngine
+import ch.tutteli.kbox.isNotNullAndNotBlank
 import com.fasterxml.jackson.module.kotlin.readValue
 import com.github.salomonbrys.kodein.instance
 import io.vertx.core.http.HttpMethod.GET
@@ -444,19 +445,17 @@ open class BotAdminVerticle : AdminVerticle() {
             }
         }
 
-        blockingJsonPost("/configuration/bots/:botId/rag", admin) { context, configuration: BotRAGConfigurationDTO  ->
-            if (context.organization == configuration.namespace) {
-                BotRAGConfigurationDTO(RAGService.saveRag(configuration))
+        blockingJsonPost("/configuration/bots/:botId/rag", admin) { context, request: BotRAGConfigurationDTO  ->
+            if (context.organization == request.namespace) {
+                BotRAGConfigurationDTO(RAGService.saveRag(request))
             } else {
                 unauthorized()
             }
         }
 
         blockingJsonGet("/configuration/bots/:botId/rag", admin) { context  ->
             RAGService.getRAGConfiguration(context.organization, context.path("botId"))
-                ?.let {
-                    BotRAGConfigurationDTO(it)
-                }
+                ?.let { BotRAGConfigurationDTO(it) }
         }
 
         blockingDelete("/configuration/bots/:botId/rag", admin) { context  ->

bot/admin/server/src/main/kotlin/model/BotObservabilityConfigurationDTO.kt

@@ -21,6 +21,7 @@ import ai.tock.bot.admin.bot.observability.BotObservabilityConfiguration
 import ai.tock.genai.orchestratorcore.mappers.ObservabilitySettingMapper
 import ai.tock.genai.orchestratorcore.models.Constants
 import ai.tock.genai.orchestratorcore.models.observability.ObservabilitySettingDTO
+import ai.tock.genai.orchestratorcore.models.observability.toDTO
 import ai.tock.genai.orchestratorcore.utils.SecurityUtils
 import org.litote.kmongo.newId
 import org.litote.kmongo.toId
@@ -33,24 +34,24 @@ data class BotObservabilityConfigurationDTO(
     val setting: ObservabilitySettingDTO,
 ) {
     constructor(configuration: BotObservabilityConfiguration) : this(
-        configuration._id.toString(),
-        configuration.namespace,
-        configuration.botId,
-        configuration.enabled,
-        ObservabilitySettingMapper.toDTO(configuration.setting),
+        id = configuration._id.toString(),
+        namespace = configuration.namespace,
+        botId = configuration.botId,
+        enabled = configuration.enabled,
+        setting = configuration.setting.toDTO(),
     )
 
     fun toBotObservabilityConfiguration(): BotObservabilityConfiguration =
         BotObservabilityConfiguration(
-            id?.toId() ?: newId(),
-            namespace,
-            botId,
-            enabled,
-            ObservabilitySettingMapper.toEntity(
-                namespace,
-                botId,
-                Constants.GEN_AI_OBSERVABILITY,
-                setting
+            _id = id?.toId() ?: newId(),
+            namespace = namespace,
+            botId = botId,
+            enabled = enabled,
+            setting = ObservabilitySettingMapper.toEntity(
+                namespace = namespace,
+                botId = botId,
+                feature = Constants.GEN_AI_OBSERVABILITY,
+                dto = setting
             ),
         )
 }

bot/admin/server/src/main/kotlin/model/BotRAGConfigurationDTO.kt

@@ -17,12 +17,14 @@
 package ai.tock.bot.admin.model
 
 import ai.tock.bot.admin.bot.rag.BotRAGConfiguration
+import ai.tock.bot.admin.service.VectorStoreService
 import ai.tock.genai.orchestratorcore.mappers.EMSettingMapper
 import ai.tock.genai.orchestratorcore.mappers.LLMSettingMapper
 import ai.tock.genai.orchestratorcore.models.Constants
 import ai.tock.genai.orchestratorcore.models.em.EMSettingDTO
+import ai.tock.genai.orchestratorcore.models.em.toDTO
 import ai.tock.genai.orchestratorcore.models.llm.LLMSettingDTO
-import ai.tock.genai.orchestratorcore.utils.SecurityUtils
+import ai.tock.genai.orchestratorcore.models.llm.toDTO
 import org.litote.kmongo.newId
 import org.litote.kmongo.toId
 
@@ -34,44 +36,54 @@ data class BotRAGConfigurationDTO(
     val llmSetting: LLMSettingDTO,
     val emSetting: EMSettingDTO,
     val indexSessionId: String? = null,
+    val indexName: String? = null,
     val noAnswerSentence: String,
     val noAnswerStoryId: String? = null,
 ) {
     constructor(configuration: BotRAGConfiguration) : this(
-        configuration._id.toString(),
-        configuration.namespace,
-        configuration.botId,
-        configuration.enabled,
-        LLMSettingMapper.toDTO(configuration.llmSetting),
-        EMSettingMapper.toDTO(configuration.emSetting),
-        configuration.indexSessionId,
-        configuration.noAnswerSentence,
-        configuration.noAnswerStoryId
+        id = configuration._id.toString(),
+        namespace = configuration.namespace,
+        botId = configuration.botId,
+        enabled = configuration.enabled,
+        llmSetting = configuration.llmSetting.toDTO(),
+        emSetting = configuration.emSetting.toDTO(),
+        indexSessionId = configuration.indexSessionId,
+        indexName = configuration.generateIndexName(),
+        noAnswerSentence = configuration.noAnswerSentence,
+        noAnswerStoryId = configuration.noAnswerStoryId
     )
 
     fun toBotRAGConfiguration(): BotRAGConfiguration =
         BotRAGConfiguration(
-            id?.toId() ?: newId(),
-            namespace,
-            botId,
-            enabled,
-            LLMSettingMapper.toEntity(
-                namespace,
-                botId,
-                Constants.GEN_AI_RAG_QUESTION_ANSWERING,
-                llmSetting
+            _id = id?.toId() ?: newId(),
+            namespace = namespace,
+            botId = botId,
+            enabled = enabled,
+            llmSetting = LLMSettingMapper.toEntity(
+                namespace = namespace,
+                botId = botId,
+                feature = Constants.GEN_AI_RAG_QUESTION_ANSWERING,
+                dto = llmSetting
             ),
-            EMSettingMapper.toEntity(
-                namespace,
-                botId,
-                Constants.GEN_AI_RAG_EMBEDDING_QUESTION,
-                emSetting
+            emSetting = EMSettingMapper.toEntity(
+                namespace = namespace,
+                botId = botId,
+                feature = Constants.GEN_AI_RAG_EMBEDDING_QUESTION,
+                dto = emSetting
             ),
             indexSessionId = indexSessionId,
             noAnswerSentence = noAnswerSentence,
             noAnswerStoryId = noAnswerStoryId
         )
 }
 
+private fun BotRAGConfiguration.generateIndexName(): String? {
+    return indexSessionId?.takeIf { it.isNotBlank() }?.let {
+        VectorStoreService.getVectorStoreConfiguration(namespace, botId, enabled = true)
+            ?.setting
+            ?.normalizeDocumentIndexName(namespace, botId, it)
+    }
+}
+
 
 

bot/admin/server/src/main/kotlin/model/BotSentenceGenerationConfigurationDTO.kt

@@ -20,6 +20,7 @@ import ai.tock.bot.admin.bot.sentencegeneration.BotSentenceGenerationConfigurati
 import ai.tock.genai.orchestratorcore.mappers.LLMSettingMapper
 import ai.tock.genai.orchestratorcore.models.Constants
 import ai.tock.genai.orchestratorcore.models.llm.LLMSettingDTO
+import ai.tock.genai.orchestratorcore.models.llm.toDTO
 import ai.tock.genai.orchestratorcore.utils.SecurityUtils
 import org.litote.kmongo.newId
 import org.litote.kmongo.toId
@@ -33,26 +34,26 @@ data class BotSentenceGenerationConfigurationDTO(
     val llmSetting: LLMSettingDTO,
 ) {
     constructor(configuration: BotSentenceGenerationConfiguration) : this(
-        configuration._id.toString(),
-        configuration.namespace,
-        configuration.botId,
-        configuration.enabled,
-        configuration.nbSentences,
-        LLMSettingMapper.toDTO(configuration.llmSetting),
+        id = configuration._id.toString(),
+        namespace = configuration.namespace,
+        botId = configuration.botId,
+        enabled = configuration.enabled,
+        nbSentences = configuration.nbSentences,
+        llmSetting = configuration.llmSetting.toDTO(),
     )
 
     fun toSentenceGenerationConfiguration(): BotSentenceGenerationConfiguration =
         BotSentenceGenerationConfiguration(
-            id?.toId() ?: newId(),
-            namespace,
-            botId,
-            enabled,
-            nbSentences,
-            LLMSettingMapper.toEntity(
-                namespace,
-                botId,
-                Constants.GEN_AI_COMPLETION_SENTENCE_GENERATION,
-                llmSetting
+            _id = id?.toId() ?: newId(),
+            namespace = namespace,
+            botId = botId,
+            enabled = enabled,
+            nbSentences = nbSentences,
+            llmSetting = LLMSettingMapper.toEntity(
+                namespace = namespace,
+                botId = botId,
+                feature = Constants.GEN_AI_COMPLETION_SENTENCE_GENERATION,
+                dto = llmSetting
             )
         )
 }

bot/admin/server/src/main/kotlin/model/BotVectorStoreConfigurationDTO.kt

@@ -21,6 +21,7 @@ import ai.tock.genai.orchestratorcore.mappers.LLMSettingMapper
 import ai.tock.genai.orchestratorcore.mappers.VectorStoreSettingMapper
 import ai.tock.genai.orchestratorcore.models.Constants
 import ai.tock.genai.orchestratorcore.models.vectorstore.VectorStoreSettingDTO
+import ai.tock.genai.orchestratorcore.models.vectorstore.toDTO
 import ai.tock.genai.orchestratorcore.utils.SecurityUtils
 import org.litote.kmongo.newId
 import org.litote.kmongo.toId
@@ -33,24 +34,24 @@ data class BotVectorStoreConfigurationDTO(
     val setting: VectorStoreSettingDTO,
 ) {
     constructor(configuration: BotVectorStoreConfiguration) : this(
-        configuration._id.toString(),
-        configuration.namespace,
-        configuration.botId,
-        configuration.enabled,
-        VectorStoreSettingMapper.toDTO(configuration.setting),
+        id = configuration._id.toString(),
+        namespace = configuration.namespace,
+        botId = configuration.botId,
+        enabled = configuration.enabled,
+        setting = configuration.setting.toDTO(),
     )
 
     fun toBotVectorStoreConfiguration(): BotVectorStoreConfiguration =
         BotVectorStoreConfiguration(
-            id?.toId() ?: newId(),
-            namespace,
-            botId,
-            enabled,
-            VectorStoreSettingMapper.toEntity(
-                namespace,
-                botId,
-                Constants.GEN_AI_VECTOR_STORE,
-                setting
+            _id = id?.toId() ?: newId(),
+            namespace = namespace,
+            botId = botId,
+            enabled = enabled,
+            setting = VectorStoreSettingMapper.toEntity(
+                namespace = namespace,
+                botId = botId,
+                feature = Constants.GEN_AI_VECTOR_STORE,
+                dto = setting
             )
         )
 }

bot/admin/server/src/main/kotlin/service/ObservabilityService.kt

@@ -44,6 +44,16 @@ object ObservabilityService {
         return observabilityConfigurationDAO.findByNamespaceAndBotId(namespace, botId)
     }
 
+    /**
+     * Get the Observability configuration
+     * @param namespace: the namespace
+     * @param botId: the botId
+     * @param enabled: the observability activation (enabled or not)
+     */
+    fun getObservabilityConfiguration(namespace: String, botId: String, enabled: Boolean): BotObservabilityConfiguration? {
+        return observabilityConfigurationDAO.findByNamespaceAndBotIdAndEnabled(namespace, botId, enabled)
+    }
+
     /**
      * Deleting the Observability Configuration
      * @param namespace: the namespace
@@ -56,16 +66,6 @@ object ObservabilityService {
         return observabilityConfigurationDAO.delete(observabilityConfig._id)
     }
 
-    /**
-     * Get the Observability configuration
-     * @param namespace: the namespace
-     * @param botId: the botId
-     * @param enabled: the observability activation (enabled or not)
-     */
-    fun getObservabilityConfiguration(namespace: String, botId: String, enabled: Boolean): BotObservabilityConfiguration? {
-        return observabilityConfigurationDAO.findByNamespaceAndBotIdAndEnabled(namespace, botId, enabled)
-    }
-
     /**
      * Save Observability configuration and filter errors
      * @param observabilityConfig : the observability configuration to create or update

bot/admin/server/src/main/kotlin/service/RAGValidationService.kt

@@ -19,9 +19,12 @@ package ai.tock.bot.admin.service
 import ai.tock.bot.admin.bot.rag.BotRAGConfiguration
 import ai.tock.genai.orchestratorclient.requests.EMProviderSettingStatusQuery
 import ai.tock.genai.orchestratorclient.requests.LLMProviderSettingStatusQuery
+import ai.tock.genai.orchestratorclient.requests.VectorStoreProviderSettingStatusQuery
 import ai.tock.genai.orchestratorclient.responses.ProviderSettingStatusResponse
 import ai.tock.genai.orchestratorclient.services.EMProviderService
 import ai.tock.genai.orchestratorclient.services.LLMProviderService
+import ai.tock.genai.orchestratorclient.services.VectorStoreProviderService
+import ai.tock.genai.orchestratorcore.utils.VectorStoreUtils
 import ai.tock.shared.exception.error.ErrorMessage
 import ai.tock.shared.injector
 import ai.tock.shared.provide
@@ -31,29 +34,44 @@ object RAGValidationService {
 
     private val llmProviderService: LLMProviderService get() = injector.provide()
     private val emProviderService: EMProviderService get() = injector.provide()
+    private val vectorStoreProviderService: VectorStoreProviderService get() = injector.provide()
 
     fun validate(ragConfig: BotRAGConfiguration): Set<ErrorMessage> {
         return mutableSetOf<ErrorMessage>().apply {
-            addAll(
-                llmProviderService
-                    .checkSetting(
-                        LLMProviderSettingStatusQuery(
-                            ragConfig.llmSetting,
-                            ObservabilityService.getObservabilityConfiguration(
-                                ragConfig.namespace,
-                                ragConfig.botId,
-                                enabled = true
-                            )?.setting
-                        )
+            val llmErrors = llmProviderService.checkSetting(
+                LLMProviderSettingStatusQuery(
+                    ragConfig.llmSetting,
+                    ObservabilityService.getObservabilityConfiguration(
+                        ragConfig.namespace, ragConfig.botId, enabled = true
+                    )?.setting
+                )
+            ).getErrors("LLM setting check failed")
+
+            val embeddingErrors = emProviderService.checkSetting(
+                EMProviderSettingStatusQuery(ragConfig.emSetting)
+            ).getErrors("Embedding Model setting check failed")
+
+            val indexSessionIdErrors = validateIndexSessionId(ragConfig)
+
+            val vectorStoreErrors = (indexSessionIdErrors + embeddingErrors).takeIf { it.isEmpty() }?.let {
+                val vectorStoreSetting = VectorStoreService.getVectorStoreConfiguration(
+                    ragConfig.namespace, ragConfig.botId, enabled = true
+                )?.setting
+
+                val (_, indexName) = VectorStoreUtils.getVectorStoreElements(
+                    ragConfig.namespace, ragConfig.botId, ragConfig.indexSessionId!!, vectorStoreSetting
+                )
+
+                vectorStoreProviderService.checkSetting(
+                    VectorStoreProviderSettingStatusQuery(
+                        vectorStoreSetting = vectorStoreSetting,
+                        emSetting = ragConfig.emSetting,
+                        documentIndexName = indexName
                     )
-                    .getErrors("LLM setting check failed")
-            )
-            addAll(
-                emProviderService
-                    .checkSetting(EMProviderSettingStatusQuery(ragConfig.emSetting))
-                    .getErrors("Embedding Model setting check failed")
-            )
-            addAll(validateIndexSessionId(ragConfig))
+                ).getErrors("Vector store setting check failed")
+            } ?: emptySet()
+
+            addAll(llmErrors + embeddingErrors + indexSessionIdErrors + vectorStoreErrors)
         }
     }