From 67e01f9363c509183ba5d3f29af662062c0e7406 Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:04:33 -0400 Subject: [PATCH 01/55] Examples Update + Code Refactor (#25) * Removed StorageManager * Added examples for OpenAI, Bedrock, Anthropic, and VertexAI * Updating old examples (1/2) * Updating old examples (2/2) --- examples/ack_test.yaml | 7 ++ examples/anthropic_bedrock.yaml | 33 +++--- examples/chat_model_with_history.yaml | 9 +- examples/error_handler.yaml | 46 ++++++-- examples/llm/anthropic_chat.yaml | 99 +++++++++++++++++ examples/llm/bedrock_anthropic_chat.yaml | 14 ++- .../langchain_openai_with_history_chat.yaml | 98 +++++++++++++++++ examples/llm/openai_chat.yaml | 70 ++++++------ examples/llm/vertexai_chat.yaml | 100 +++++++++++++++++ examples/milvus_store.yaml | 49 +++++---- examples/request_reply.yaml | 7 ++ examples/vector_store_search.yaml | 63 ++++++++--- src/solace_ai_connector/common/utils.py | 1 + .../components/__init__.py | 2 - .../components/component_base.py | 2 +- .../general/for_testing/storage_tester.py | 51 --------- .../general/langchain/langchain_chat_model.py | 3 + .../langchain_vector_store_embedding_base.py | 44 ++++++-- .../general/openai/openai_chat_model_base.py | 2 +- src/solace_ai_connector/flow/flow.py | 3 - .../solace_ai_connector.py | 3 - src/solace_ai_connector/storage/storage.py | 28 ----- .../storage/storage_file.py | 53 --------- .../storage/storage_manager.py | 42 ------- .../storage/storage_memory.py | 27 ----- src/solace_ai_connector/storage/storage_s3.py | 31 ------ tests/test_storage.py | 103 ------------------ 27 files changed, 534 insertions(+), 456 deletions(-) create mode 100644 examples/llm/anthropic_chat.yaml create mode 100644 examples/llm/langchain_openai_with_history_chat.yaml create mode 100644 examples/llm/vertexai_chat.yaml delete mode 100644 src/solace_ai_connector/components/general/for_testing/storage_tester.py delete mode 100644 src/solace_ai_connector/storage/storage.py delete mode 100644 src/solace_ai_connector/storage/storage_file.py delete mode 100644 src/solace_ai_connector/storage/storage_manager.py delete mode 100644 src/solace_ai_connector/storage/storage_memory.py delete mode 100644 src/solace_ai_connector/storage/storage_s3.py delete mode 100644 tests/test_storage.py diff --git a/examples/ack_test.yaml b/examples/ack_test.yaml index 13b8dee..278f9f0 100644 --- a/examples/ack_test.yaml +++ b/examples/ack_test.yaml @@ -1,6 +1,13 @@ --- # Simple loopback flow # Solace -> Pass Through -> Solace +# +# required ENV variables: +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN + log: stdout_log_level: DEBUG log_file_level: DEBUG diff --git a/examples/anthropic_bedrock.yaml b/examples/anthropic_bedrock.yaml index 517d3f6..1f3c60a 100644 --- a/examples/anthropic_bedrock.yaml +++ b/examples/anthropic_bedrock.yaml @@ -4,6 +4,22 @@ # sends the response back to the Solace broker # It will ask the model to write a dry joke about the input # message. It takes the entire payload of the input message +# +# Dependencies: +# pip install langchain_aws langchain_community +# +# Dependencies: +# - langchain_aws +# pip install langchain_aws +# +# required ENV variables: +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN +# - AWS_BEDROCK_ANTHROPIC_CLAUDE_MODEL_ID + + instance_name: LLM log: stdout_log_level: DEBUG @@ -19,21 +35,6 @@ shared_config: broker_password: ${SOLACE_BROKER_PASSWORD} broker_vpn: ${SOLACE_BROKER_VPN} -# Storage -storage: - - storage_name: default - storage_type: file - storage_config: - path: app/data.json - - storage_name: backup - storage_type: aws_s3 - storage_config: - aws_access_key_id: ${AWS_ACCESS_KEY_ID} - aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} - aws_region_name: ${AWS_REGION_NAME} - bucket_name: ${AWS_BUCKET_NAME} - path: app/data.json - # List of flows flows: - name: test_flow @@ -90,7 +91,7 @@ flows: payload_format: text input_transforms: - type: copy - source_expression: user_data.temp + source_expression: previous dest_expression: user_data.output:payload - type: copy source_expression: template:response/{{text://input.topic}} diff --git a/examples/chat_model_with_history.yaml b/examples/chat_model_with_history.yaml index 080236b..c889ea5 100644 --- a/examples/chat_model_with_history.yaml +++ b/examples/chat_model_with_history.yaml @@ -1,5 +1,12 @@ --- -# Example uses goes from STDIN to STDOUT with a chat model with history +# Example uses goes from STDIN to STDOUT with a chat model with history hosted on AWS Bedrock + +# Dependencies: +# pip install langchain_aws + +# required ENV variables: +# - AWS_DEFAULT_REGION + log: stdout_log_level: INFO log_file_level: DEBUG diff --git a/examples/error_handler.yaml b/examples/error_handler.yaml index 9e47856..416d1bc 100644 --- a/examples/error_handler.yaml +++ b/examples/error_handler.yaml @@ -1,8 +1,21 @@ --- # This is an example configuration file that contains an # error handler flow and a test flow. The error handler flow -# will log any error messages locally and will also -# send them to a Solace broker. +# will log any error messages locally to a file and will also +# send them to a Solace broker. +# +# It will subscribe to `my/topic1` and expect an event with the payload: +# { +# "value": +# } +# If value is not a number, the error will be caught, logged to file and send back to the Solace broker. +# +# required ENV variables: +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN + instance: name: solace_ai_connector1 log: @@ -27,12 +40,19 @@ flows: component_module: error_input component_config: - component_name: error_logger - component_module: logger - component_config: - log_level: ERROR - max_log_line_size: 1000 + component_module: file_output + input_transforms: + - type: copy + source_expression: input.payload + dest_expression: user_data.log:content + - type: copy + source_value: a + dest_expression: user_data.log:mode + - type: copy + source_value: error_log.log + dest_expression: user_data.log:file_path component_input: - source_expression: input.payload + source_expression: user_data.log - component_name: solace_sw_broker component_module: broker_output component_config: @@ -66,7 +86,7 @@ flows: - topic: my/topic1 qos: 1 payload_encoding: utf-8 - payload_format: text + payload_format: json - component_name: pass_through component_module: pass_through @@ -89,6 +109,16 @@ flows: - type: copy source_expression: input.payload dest_expression: user_data.output:payload.original_payload + - type: copy + source_expression: + invoke: + module: invoke_functions + function: power + params: + positional: + - source_expression(input.payload:value) # This will throw an error if value is not a number + - 2 + dest_expression: user_data.output:payload.valueSquared - type: copy source_expression: input.user_properties dest_expression: user_data.output:payload.user_properties diff --git a/examples/llm/anthropic_chat.yaml b/examples/llm/anthropic_chat.yaml new file mode 100644 index 0000000..5bc1c16 --- /dev/null +++ b/examples/llm/anthropic_chat.yaml @@ -0,0 +1,99 @@ +# This will create a flow like this: +# Solace -> Anthropic -> Solace +# +# It will subscribe to `demo/question` and expect an event with the payload: +# +# The input message has the following schema: +# { +# "text": "" +# } +# +# It will then send an event back to Solace with the topic: `demo/question/response` +# +# Dependencies: +# pip install -U langchain-anthropic +# +# required ENV variables: +# - ANTHROPIC_API_KEY +# - ANTHROPIC_API_ENDPOINT +# - MODEL_NAME +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN + +--- +log: + stdout_log_level: INFO + log_file_level: DEBUG + log_file: solace_ai_connector.log + +shared_config: + - broker_config: &broker_connection + broker_type: solace + broker_url: ${SOLACE_BROKER_URL} + broker_username: ${SOLACE_BROKER_USERNAME} + broker_password: ${SOLACE_BROKER_PASSWORD} + broker_vpn: ${SOLACE_BROKER_VPN} + +# Take from Slack and publish to Solace +flows: + # Slack chat input processing + - name: Simple template to LLM + components: + # Input from a Solace broker + - component_name: solace_sw_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_queue_name: demo_question + broker_subscriptions: + - topic: demo/question + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # + # Do an LLM request + # + - component_name: llm_request + component_module: langchain_chat_model + component_config: + langchain_module: langchain_anthropic + langchain_class: ChatAnthropic + langchain_component_config: + api_key: ${ANTHROPIC_API_KEY} + base_url: ${ANTHROPIC_API_ENDPOINT} + model: ${MODEL_NAME} + temperature: 0.01 + input_transforms: + - type: copy + source_expression: | + template:You are a helpful AI assistant. Please help with the user's request below: + + {{text://input.payload:text}} + + dest_expression: user_data.llm_input:messages.0.content + - type: copy + source_expression: static:user + dest_expression: user_data.llm_input:messages.0.role + component_input: + source_expression: user_data.llm_input + + # Send response back to broker + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.output:payload + - type: copy + source_expression: template:{{text://input.topic}}/response + dest_expression: user_data.output:topic + component_input: + source_expression: user_data.output diff --git a/examples/llm/bedrock_anthropic_chat.yaml b/examples/llm/bedrock_anthropic_chat.yaml index 3064a0e..879bb03 100644 --- a/examples/llm/bedrock_anthropic_chat.yaml +++ b/examples/llm/bedrock_anthropic_chat.yaml @@ -7,8 +7,16 @@ # # The input message has the following schema: # { -# "text": "", +# "text": "" # } +# +# Dependencies: +# pip install langchain_aws +# +# required ENV variables: +# - AWS_BEDROCK_ANTHROPIC_CLAUDE_MODEL_ID +# - AWS_BEDROCK_ANTHROPIC_CLAUDE_REGION + --- log: stdout_log_level: DEBUG @@ -35,8 +43,8 @@ flows: - component_name: llm_request component_module: langchain_chat_model component_config: - langchain_module: langchain_community.chat_models - langchain_class: BedrockChat + langchain_module: langchain_aws + langchain_class: ChatBedrock langchain_component_config: model_id: ${AWS_BEDROCK_ANTHROPIC_CLAUDE_MODEL_ID} region_name: ${AWS_BEDROCK_ANTHROPIC_CLAUDE_REGION} diff --git a/examples/llm/langchain_openai_with_history_chat.yaml b/examples/llm/langchain_openai_with_history_chat.yaml new file mode 100644 index 0000000..1440b33 --- /dev/null +++ b/examples/llm/langchain_openai_with_history_chat.yaml @@ -0,0 +1,98 @@ +# This will create a flow like this: +# Solace -> OpenAI -> Solace +# +# It will subscribe to `demo/joke/subject` and expect an event with the payload: +# +# { +# "joke": { +# "subject": "" +# } +# } +# +# It will then send an event back to Solace with the topic: `demo/joke/subject/response` +# +# Dependencies: +# pip install -U langchain_openai openai +# +# required ENV variables: +# - OPENAI_API_KEY +# - OPENAI_API_ENDPOINT - optional +# - MODEL_NAME +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN + +--- +log: + stdout_log_level: INFO + log_file_level: DEBUG + log_file: solace_ai_connector.log + +shared_config: + - broker_config: &broker_connection + broker_type: solace + broker_url: ${SOLACE_BROKER_URL} + broker_username: ${SOLACE_BROKER_USERNAME} + broker_password: ${SOLACE_BROKER_PASSWORD} + broker_vpn: ${SOLACE_BROKER_VPN} + +# Take from Slack and publish to Solace +flows: + # Slack chat input processing + - name: Simple template to LLM + components: + # Input from a Solace broker + - component_name: solace_sw_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_queue_name: ed_demo_joke + broker_subscriptions: + - topic: demo/joke/subject + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # Go to the LLM and keep history + - component_name: chat_request_llm + component_module: langchain_chat_model_with_history + component_config: + langchain_module: langchain_openai + langchain_class: ChatOpenAI + langchain_component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${MODEL_NAME} + temperature: 0.01 + history_module: langchain_core.chat_history + history_class: InMemoryChatMessageHistory + history_max_turns: 20 + history_max_length: 6000 + input_transforms: + - type: copy + source_expression: template:Write a joke about {{text://input.payload:joke.subject}} + dest_expression: user_data.input:messages.0.content + - type: copy + source_value: user + dest_expression: user_data.input:messages.0.role + component_input: + source_expression: user_data.input + + # Send response back to broker + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.output:payload + - type: copy + source_expression: template:{{text://input.topic}}/response + dest_expression: user_data.output:topic + component_input: + source_expression: user_data.output diff --git a/examples/llm/openai_chat.yaml b/examples/llm/openai_chat.yaml index 15fc937..65e9897 100644 --- a/examples/llm/openai_chat.yaml +++ b/examples/llm/openai_chat.yaml @@ -1,15 +1,26 @@ # This will create a flow like this: # Solace -> OpenAI -> Solace # -# It will subscribe to `demo/joke/subject` and expect an event with the payload: +# It will subscribe to `demo/question` and expect an event with the payload: # +# The input message has the following schema: # { -# "joke": { -# "subject": "" -# } +# "text": "" # } # -# It will then send an event back to Solace with the topic: `demo/joke/subject/response` +# It will then send an event back to Solace with the topic: `demo/question/response` +# +# Dependencies: +# pip install -U langchain_openai openai +# +# required ENV variables: +# - OPENAI_API_KEY +# - OPENAI_API_ENDPOINT +# - MODEL_NAME +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN --- log: @@ -27,49 +38,44 @@ shared_config: # Take from Slack and publish to Solace flows: - - # Slack chat input processing - name: Simple template to LLM components: - # Input from a Solace broker - component_name: solace_sw_broker component_module: broker_input - component_config: + component_config: <<: *broker_connection - broker_queue_name: ed_demo_joke + broker_queue_name: demo_question broker_subscriptions: - - topic: demo/joke/subject + - topic: demo/question qos: 1 payload_encoding: utf-8 payload_format: json - # Go to the LLM and keep history - - component_name: chat_request_llm - component_module: langchain_chat_model_with_history + # + # Do an LLM request + # + - component_name: llm_request + component_module: openai_chat_model component_config: - langchain_module: langchain_openai - langchain_class: ChatOpenAI - langchain_component_config: - openai_api_key: ${OPENAI_API_KEY} - openai_api_base: ${OPENAI_API_ENDPOINT} - model: ${MODEL_NAME} - temperature: 0.01 - history_module: langchain_core.chat_history - history_class: InMemoryChatMessageHistory - history_max_turns: 20 - history_max_length: 6000 + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${MODEL_NAME} + temperature: 0.01 input_transforms: - type: copy - source_expression: template:Write a joke about {{text://input.payload:joke.subject}} - dest_expression: user_data.input:messages.0.content + source_expression: | + template:You are a helpful AI assistant. Please help with the user's request below: + + {{text://input.payload:text}} + + dest_expression: user_data.llm_input:messages.0.content - type: copy - source_value: user - dest_expression: user_data.input:messages.0.role + source_expression: static:user + dest_expression: user_data.llm_input:messages.0.role component_input: - source_expression: user_data.input - + source_expression: user_data.llm_input # Send response back to broker - component_name: send_response @@ -88,5 +94,3 @@ flows: dest_expression: user_data.output:topic component_input: source_expression: user_data.output - - diff --git a/examples/llm/vertexai_chat.yaml b/examples/llm/vertexai_chat.yaml new file mode 100644 index 0000000..16201b8 --- /dev/null +++ b/examples/llm/vertexai_chat.yaml @@ -0,0 +1,100 @@ +# This will create a flow like this: +# Solace -> Google Vertex AI -> Solace +# +# It will subscribe to `demo/question` and expect an event with the payload: +# +# The input message has the following schema: +# { +# "text": "" +# } +# +# It will then send an event back to Solace with the topic: `demo/question/response` +# +# Dependencies: +# pip install -U langchain-google-vertexai +# +# required ENV variables: +# - GOOGLE_APPLICATION_CREDENTIALS: the path to a service account JSON file +# - VERTEX_REGION +# - VERTEX_API_ENDPOINT - optional +# - MODEL_NAME +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN + +--- +log: + stdout_log_level: INFO + log_file_level: DEBUG + log_file: solace_ai_connector.log + +shared_config: + - broker_config: &broker_connection + broker_type: solace + broker_url: ${SOLACE_BROKER_URL} + broker_username: ${SOLACE_BROKER_USERNAME} + broker_password: ${SOLACE_BROKER_PASSWORD} + broker_vpn: ${SOLACE_BROKER_VPN} + +# Take from Slack and publish to Solace +flows: + # Slack chat input processing + - name: Simple template to LLM + components: + # Input from a Solace broker + - component_name: solace_sw_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_queue_name: demo_question + broker_subscriptions: + - topic: demo/question + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # + # Do an LLM request + # + - component_name: llm_request + component_module: langchain_chat_model + component_config: + langchain_module: langchain_google_vertexai + langchain_class: ChatVertexAI + langchain_component_config: + base_url: ${VERTEX_API_ENDPOINT} + location: ${VERTEX_REGION} + model: ${MODEL_NAME} + temperature: 0.01 + input_transforms: + - type: copy + source_expression: | + template:You are a helpful AI assistant. Please help with the user's request below: + + {{text://input.payload:text}} + + dest_expression: user_data.llm_input:messages.0.content + - type: copy + source_expression: static:user + dest_expression: user_data.llm_input:messages.0.role + component_input: + source_expression: user_data.llm_input + + # Send response back to broker + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.output:payload + - type: copy + source_expression: template:{{text://input.topic}}/response + dest_expression: user_data.output:topic + component_input: + source_expression: user_data.output diff --git a/examples/milvus_store.yaml b/examples/milvus_store.yaml index 6cc8d10..4c6211b 100644 --- a/examples/milvus_store.yaml +++ b/examples/milvus_store.yaml @@ -1,21 +1,24 @@ --- # Example configuration file for adding a Milvus vector store and a Cohere embedding model # The input comes from STDIN and goes to STDOUT +# +# Dependencies: +# pip install langchain_milvus pymilvus +# +# required ENV variables: +# - MILVUS_HOST +# - MILVUS_PORT +# - MILVUS_COLLECTION_NAME +# - ACCESS_KEY: AWS access key +# - SECRET_KEY: AWS secret key +# - AWS_BEDROCK_COHERE_EMBED_MODEL_ID +# - AWS_BEDROCK_COHERE_EMBED_REGION + log: stdout_log_level: DEBUG log_file_level: DEBUG log_file: solace_ai_connector.log -shared_config: - - broker_config: &broker_connection - broker_connection_share: ${SOLACE_BROKER_URL} - broker_type: solace - broker_url: ${SOLACE_BROKER_URL} - broker_username: ${SOLACE_BROKER_USERNAME} - broker_password: ${SOLACE_BROKER_PASSWORD} - broker_vpn: ${SOLACE_BROKER_VPN} - - # List of flows flows: - name: test_flow @@ -24,25 +27,24 @@ flows: # Test input from STDIN - component_name: stdin component_module: stdin_input - component_config: - component_name: milvus_cohere_embed component_module: langchain_vector_store_embedding_index component_config: - vector_store_component_path: langchain_community.vectorstores + vector_store_component_path: langchain_milvus vector_store_component_name: Milvus vector_store_component_config: - collection_name: collection_2 + auto_id: true + collection_name: ${MILVUS_COLLECTION_NAME} connection_args: host: ${MILVUS_HOST} port: ${MILVUS_PORT} - # vector_store_index_name: solace-index-3 - embedding_component_path: langchain_community.embeddings + embedding_component_path: langchain_aws embedding_component_name: BedrockEmbeddings embedding_component_config: model_id: ${AWS_BEDROCK_COHERE_EMBED_MODEL_ID} region_name: ${AWS_BEDROCK_COHERE_EMBED_REGION} - credentials_profile_name: default + credentials_profile_name: default # Profile name in ~/.aws/credentials input_transforms: - type: copy source_value: @@ -51,31 +53,30 @@ flows: function: system dest_expression: user_data.vector_input:metadata.system - type: copy - source_value: efunneko + source_value: username dest_expression: user_data.vector_input:metadata.user - type: copy - source_value: input.payload - dest_expression: user_data.vector_input:text + source_expression: input.payload:text + dest_expression: user_data.vector_input:texts component_input: source_expression: user_data.vector_input - component_name: milvus_cohere_embed_search component_module: langchain_vector_store_embedding_search component_config: - vector_store_component_path: langchain_community.vectorstores + vector_store_component_path: langchain_milvus vector_store_component_name: Milvus vector_store_component_config: - collection_name: collection_1 + collection_name: ${MILVUS_COLLECTION_NAME} connection_args: host: ${MILVUS_HOST} port: ${MILVUS_PORT} - # vector_store_index_name: solace-index-3 - embedding_component_path: langchain_community.embeddings + embedding_component_path: langchain_aws embedding_component_name: BedrockEmbeddings embedding_component_config: model_id: ${AWS_BEDROCK_COHERE_EMBED_MODEL_ID} region_name: ${AWS_BEDROCK_COHERE_EMBED_REGION} - credentials_profile_name: default + credentials_profile_name: default # Profile name in ~/.aws/credentials max_results: 5 component_input: source_expression: input.payload diff --git a/examples/request_reply.yaml b/examples/request_reply.yaml index 87d775d..9755203 100644 --- a/examples/request_reply.yaml +++ b/examples/request_reply.yaml @@ -2,6 +2,13 @@ # Example for a request-reply flow # Flow 1: stdin -> broker_request_reply -> stdout # Flow 2: broker_input -> pass_through -> broker_output +# +# required ENV variables: +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN + log: stdout_log_level: INFO log_file_level: INFO diff --git a/examples/vector_store_search.yaml b/examples/vector_store_search.yaml index e427617..ca4b3f8 100644 --- a/examples/vector_store_search.yaml +++ b/examples/vector_store_search.yaml @@ -1,28 +1,40 @@ --- -# Example that uses Cohere embeddings and OpenSearch for vector store +# Example that uses Cohere embeddings and Amazon OpenSearch Service Serverless for vector store # This also shows how to use AWS credentials and AWS4Auth for OpenSearch # which involves using 'invoke' to create the required auth objects +# +# +# Follow Boto3 documentation for AWS credentials: +# https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html#configuration +# https://python.langchain.com/v0.2/docs/integrations/vectorstores/opensearch/#using-aoss-amazon-opensearch-service-serverless +# +# Dependencies: +# pip install -U langchain_community opensearch-py requests_aws4auth +# +# required ENV variables: +# - AWS_BEDROCK_COHERE_EMBED_MODEL_ID +# - AWS_BEDROCK_COHERE_EMBED_REGION +# - AWS_OPENSEARCH_INDEX_NAME +# - AWS_OPENSEARCH_ENDPOINT + log: stdout_log_level: DEBUG log_file_level: DEBUG log_file: solace_ai_connector.log shared_config: - - broker_config: &broker_connection - broker_connection_share: ${SOLACE_BROKER_URL} - broker_type: solace - broker_url: ${SOLACE_BROKER_URL} - broker_username: ${SOLACE_BROKER_USERNAME} - broker_password: ${SOLACE_BROKER_PASSWORD} - broker_vpn: ${SOLACE_BROKER_VPN} - - # Get AWS credentials object + # Get AWS credentials object from .aws credentials + # You can pass the ACCESS/SECRET/SESSION keys directly as ENV variables as well + # eg: aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} - aws_credentials: &aws_credentials invoke: object: invoke: module: boto3 function: Session + params: + keyword: + profile_name: default # The profile to choose from .aws/credentials function: get_credentials # Get AWS4Auth object @@ -47,42 +59,57 @@ shared_config: object: *aws_credentials attribute: token + # Create a bedrock client for use with AWS components + - bedrock_client_config: &bedrock_client_config + invoke: + module: boto3 + function: client + params: + keyword: + service_name: bedrock-runtime + region_name: ${AWS_BEDROCK_COHERE_EMBED_REGION} + aws_access_key_id: + invoke: + object: *aws_credentials + attribute: access_key + aws_secret_access_key: + invoke: + object: *aws_credentials + attribute: secret_key + # List of flows flows: - name: test_flow trace_level: DEBUG components: - # Input from a Solace broker + # Input from a standard in - component_name: stdin component_module: stdin_input - component_config: - component_name: opensearch_cohere_embed component_module: langchain_vector_store_embedding_search component_config: vector_store_component_path: langchain_community.vectorstores vector_store_component_name: OpenSearchVectorSearch + vector_store_index_name: ${AWS_OPENSEARCH_INDEX_NAME} vector_store_component_config: - index_name: ${AWS_OPENSEARCH_JIRA_INDEX_NAME} - opensearch_url: ${AWS_OPENSEARCH_JIRA_ENDPOINT} + opensearch_url: ${AWS_OPENSEARCH_ENDPOINT} connection_class: invoke: module: opensearchpy attribute: RequestsHttpConnection http_auth: *aws_4_auth_aoss timeout: 300 - vector_store_index_name: solace-index-3 - embedding_component_path: langchain_community.embeddings + embedding_component_path: langchain_aws embedding_component_name: BedrockEmbeddings embedding_component_config: + client: *bedrock_client_config model_id: ${AWS_BEDROCK_COHERE_EMBED_MODEL_ID} region_name: ${AWS_BEDROCK_COHERE_EMBED_REGION} - credentials_profile_name: default max_results: 7 component_input: source_expression: input.payload - - component_name: stdout component_module: stdout_output diff --git a/src/solace_ai_connector/common/utils.py b/src/solace_ai_connector/common/utils.py index 4377047..c04a0ca 100644 --- a/src/solace_ai_connector/common/utils.py +++ b/src/solace_ai_connector/common/utils.py @@ -118,6 +118,7 @@ def import_module(name, base_path=None, component_package=None): ".components.general", ".components.general.for_testing", ".components.general.langchain", + ".components.general.openai", ".components.inputs_outputs", ".transforms", ".common", diff --git a/src/solace_ai_connector/components/__init__.py b/src/solace_ai_connector/components/__init__.py index a7b8043..f373125 100644 --- a/src/solace_ai_connector/components/__init__.py +++ b/src/solace_ai_connector/components/__init__.py @@ -22,7 +22,6 @@ need_ack_input, fail, give_ack_output, - storage_tester, ) from .general.langchain import ( @@ -46,7 +45,6 @@ from .general.for_testing.need_ack_input import NeedAckInput from .general.for_testing.fail import Fail from .general.for_testing.give_ack_output import GiveAckOutput -from .general.for_testing.storage_tester import MemoryTester from .general.pass_through import PassThrough from .general.delay import Delay from .general.iterate import Iterate diff --git a/src/solace_ai_connector/components/component_base.py b/src/solace_ai_connector/components/component_base.py index f7365b5..82203ee 100644 --- a/src/solace_ai_connector/components/component_base.py +++ b/src/solace_ai_connector/components/component_base.py @@ -28,7 +28,6 @@ def __init__(self, module_info, **kwargs): self.component_index = kwargs.pop("component_index", None) self.error_queue = kwargs.pop("error_queue", None) self.instance_name = kwargs.pop("instance_name", None) - self.storage_manager = kwargs.pop("storage_manager", None) self.trace_queue = kwargs.pop("trace_queue", False) self.connector = kwargs.pop("connector", None) self.timer_manager = kwargs.pop("timer_manager", None) @@ -311,6 +310,7 @@ def handle_error(self, exception, event): "component_index": self.component_index, }, } + message = None if event and event.event_type == EventType.MESSAGE: message = event.data if message: diff --git a/src/solace_ai_connector/components/general/for_testing/storage_tester.py b/src/solace_ai_connector/components/general/for_testing/storage_tester.py deleted file mode 100644 index 97f1a68..0000000 --- a/src/solace_ai_connector/components/general/for_testing/storage_tester.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Simple memory tester component""" - -from ...component_base import ComponentBase - - -info = { - "class_name": "MemoryTester", - "description": ("A component that will exchange a value from the memory storage"), - "config_parameters": [ - { - "name": "storage_name", - "required": True, - "description": "The name of the storage to use", - "type": "string", - }, - ], - "input_schema": { - "type": "object", - "properties": { - "test_value": { - "type": "string", - "description": "The value to store in the memory storage", - }, - }, - "required": ["test_value"], - }, - "output_schema": { - "type": "object", - "properties": {}, - }, -} - - -class MemoryTester(ComponentBase): - def __init__(self, **kwargs): - super().__init__(info, **kwargs) - - def invoke(self, message, data): - storage = self.storage_manager.get_storage_handler( - self.get_config("storage_name") - ) - storage_data = storage.get("test") - - if storage_data is None: - storage_data = {"test": "initial_value"} - - old_value = storage_data.get("test") - new_value = data.get("test_value") - - storage.put("test", {"test": new_value}) - return {"test_value": old_value} diff --git a/src/solace_ai_connector/components/general/langchain/langchain_chat_model.py b/src/solace_ai_connector/components/general/langchain/langchain_chat_model.py index a025b51..85cd194 100644 --- a/src/solace_ai_connector/components/general/langchain/langchain_chat_model.py +++ b/src/solace_ai_connector/components/general/langchain/langchain_chat_model.py @@ -13,6 +13,9 @@ class LangChainChatModel(LangChainChatModelBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + def invoke_model( self, input_message, messages, session_id=None, clear_history=False ): diff --git a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_base.py b/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_base.py index aaa6ad1..f5543cc 100644 --- a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_base.py +++ b/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_base.py @@ -1,5 +1,5 @@ # This is the base class for vector store embedding classes - +import inspect from .langchain_base import ( LangChainBase, ) @@ -36,14 +36,42 @@ def init(self): self.vector_store_info["path"], self.vector_store_info["name"] ) - if "index" not in self.vector_store_info["config"]: - self.vector_store_info["config"]["index"] = self.vector_store_info["index"] - self.vector_store_info["config"]["embeddings"] = self.embedding - self.vector_store_info["config"]["embedding_function"] = self.embedding + # Get the expected parameter names of the vector store class + class_init_signature = inspect.signature(vector_store_class.__init__) + class_param_names = [ + param.name + for param in class_init_signature.parameters.values() + if param.name != "self" + ] + + # index is optional - not using it if "index" or "index_name" is provided in the config + if self.vector_store_info["index"] and ( + "index" not in self.vector_store_info["config"] + or "index_name" not in self.vector_store_info["config"] + ): + # Checking if the class expects 'index' or 'index_name' as a parameter + if "index" in class_param_names: + self.vector_store_info["config"]["index"] = self.vector_store_info[ + "index" + ] + elif "index_name" in class_param_names: + self.vector_store_info["config"]["index_name"] = self.vector_store_info[ + "index" + ] + else: + # If not defined, used "index" as a parameter + self.vector_store_info["config"]["index"] = self.vector_store_info[ + "index" + ] - # index is optional - remove it from the config if it is None - if self.vector_store_info["config"]["index"] is None: - del self.vector_store_info["config"]["index"] + # Checking if the vector store uses "embedding_function" or "embeddings" as a parameter + if "embedding_function" in class_param_names: + self.vector_store_info["config"]["embedding_function"] = self.embedding + elif "embeddings" in class_param_names: + self.vector_store_info["config"]["embeddings"] = self.embedding + else: + # If not defined, used "embeddings" as a parameter + self.vector_store_info["config"]["embeddings"] = self.embedding try: self.vector_store = self.create_component( diff --git a/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py b/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py index 1ae3e98..6a0884b 100644 --- a/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py +++ b/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py @@ -119,7 +119,7 @@ def invoke(self, message, data): response = client.chat.completions.create( messages=messages, model=self.model, temperature=self.temperature ) - return {"content": response.choices[0].message["content"]} + return {"content": response.choices[0].message.content} def invoke_stream(self, client, message, messages): response_uuid = str(uuid.uuid4()) diff --git a/src/solace_ai_connector/flow/flow.py b/src/solace_ai_connector/flow/flow.py index caa934e..c13ea71 100644 --- a/src/solace_ai_connector/flow/flow.py +++ b/src/solace_ai_connector/flow/flow.py @@ -42,7 +42,6 @@ def __init__( stop_signal, error_queue=None, instance_name=None, - storage_manager=None, trace_queue=None, flow_instance_index=0, connector=None, @@ -55,7 +54,6 @@ def __init__( self.stop_signal = stop_signal self.error_queue = error_queue self.instance_name = instance_name - self.storage_manager = storage_manager self.trace_queue = trace_queue self.flow_instance_index = flow_instance_index self.connector = connector @@ -120,7 +118,6 @@ def create_component_group(self, component, index): component_index=component_index, error_queue=self.error_queue, instance_name=self.instance_name, - storage_manager=self.storage_manager, trace_queue=self.trace_queue, connector=self.connector, timer_manager=self.connector.timer_manager, diff --git a/src/solace_ai_connector/solace_ai_connector.py b/src/solace_ai_connector/solace_ai_connector.py index 4952f62..83c3c2f 100644 --- a/src/solace_ai_connector/solace_ai_connector.py +++ b/src/solace_ai_connector/solace_ai_connector.py @@ -9,7 +9,6 @@ from .common.utils import resolve_config_values from .flow.flow import Flow from .flow.timer_manager import TimerManager -from .storage.storage_manager import StorageManager from .common.event import Event, EventType from .services.cache_service import CacheService, create_storage_backend @@ -31,7 +30,6 @@ def __init__(self, config, event_handlers=None, error_queue=None): resolve_config_values(self.config) self.validate_config() self.instance_name = self.config.get("instance_name", "solace_ai_connector") - self.storage_manager = StorageManager(self.config.get("storage", [])) self.timer_manager = TimerManager(self.stop_signal) self.cache_service = self.setup_cache_service() @@ -75,7 +73,6 @@ def create_flow(self, flow: dict, index: int, flow_instance_index: int): stop_signal=self.stop_signal, error_queue=self.error_queue, instance_name=self.instance_name, - storage_manager=self.storage_manager, trace_queue=self.trace_queue, connector=self, ) diff --git a/src/solace_ai_connector/storage/storage.py b/src/solace_ai_connector/storage/storage.py deleted file mode 100644 index 835313c..0000000 --- a/src/solace_ai_connector/storage/storage.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Top level storage module for the Solace AI Event Connector. This abstracts the -actual storage implementation and provides a common interface for the rest of -the application to use.""" - -from abc import abstractmethod - - -class Storage: - """Abstract storage class for the Solace AI Event Connector.""" - - def __init__(self, config: dict): - """Initialize the storage class.""" - - @abstractmethod - def put(self, key: str, value: dict): - """Put a value into the storage.""" - - @abstractmethod - def get(self, key: str) -> dict: - """Get a value from the storage.""" - - @abstractmethod - def delete(self, key: str): - """Delete a value from the storage.""" - - @abstractmethod - def list(self) -> list: - """List all keys in the storage.""" diff --git a/src/solace_ai_connector/storage/storage_file.py b/src/solace_ai_connector/storage/storage_file.py deleted file mode 100644 index 0ffb0c5..0000000 --- a/src/solace_ai_connector/storage/storage_file.py +++ /dev/null @@ -1,53 +0,0 @@ -"""File storage implementation for the storage interface.""" - -import os -import json -from .storage import Storage - -info = { - "class_name": "StorageFile", - "description": ("File storage class for the Solace AI Event Connector."), - "config_parameters": [ - { - "name": "file", - "required": True, - "description": "The file to use for storage", - "type": "string", - }, - ], -} - - -class StorageFile(Storage): - """File storage class for the Solace AI Event Connector.""" - - def __init__(self, config: dict): - """Initialize the file storage class.""" - self.storage_file = config["file"] - self.storage = {} - if os.path.exists(self.storage_file): - with open(self.storage_file, "r", encoding="utf-8") as file: - self.storage = json.load(file) - else: - with open(self.storage_file, "w", encoding="utf-8") as file: - json.dump(self.storage, file, ensure_ascii=False) - - def put(self, key: str, value: dict): - """Put a value into the file storage as a JSON object.""" - self.storage[key] = value - with open(self.storage_file, "w", encoding="utf-8") as file: - json.dump(self.storage, file, ensure_ascii=False) - - def get(self, key: str) -> dict: - """Get a value from the file storage""" - return self.storage.get(key, None) - - def delete(self, key: str): - """Delete a value from the file storage.""" - del self.storage[key] - with open(self.storage_file, "w", encoding="utf-8") as file: - json.dump(self.storage, file, ensure_ascii=False) - - def list(self) -> list: - """List all keys in the file storage.""" - return list(self.storage.keys()) diff --git a/src/solace_ai_connector/storage/storage_manager.py b/src/solace_ai_connector/storage/storage_manager.py deleted file mode 100644 index 8d47b19..0000000 --- a/src/solace_ai_connector/storage/storage_manager.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Create and hold the storage handlers""" - -from .storage import Storage -from .storage_file import StorageFile -from .storage_memory import StorageMemory -from .storage_s3 import StorageS3 - - -class StorageManager: - """Storage manager class for the Solace AI Event Connector.""" - - def __init__(self, storage_config: dict): - """Initialize the storage manager class.""" - self.storage_handlers = {} - self.create_storage_handlers(storage_config) - - def create_storage_handlers(self, storage_configs: list): - """Create the storage handlers""" - for storage in storage_configs: - storage_handler = self.create_storage_handler(storage) - self.storage_handlers[storage["name"]] = storage_handler - - def create_storage_handler(self, storage_config: dict): - """Create the storage handler""" - storage_handler = self.create_storage(storage_config) - return storage_handler - - def get_storage_handler(self, storage_name: str): - """Get the storage handler""" - return self.storage_handlers.get(storage_name) - - def create_storage(self, config: dict) -> "Storage": - """Static factory method to create a storage object of the correct type.""" - storage_config = config.get("storage_config", {}) - if config["storage_type"] == "file": - return StorageFile(storage_config) - elif config["storage_type"] == "memory": - return StorageMemory(storage_config) - elif config["storage_type"] == "aws_s3": - return StorageS3(storage_config) - else: - raise ValueError(f"Unsupported storage type: {config['storage_type']}") diff --git a/src/solace_ai_connector/storage/storage_memory.py b/src/solace_ai_connector/storage/storage_memory.py deleted file mode 100644 index de8cad6..0000000 --- a/src/solace_ai_connector/storage/storage_memory.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Memory storage class""" - -from .storage import Storage - - -class StorageMemory(Storage): - """Memory storage class for the Solace AI Event Connector.""" - - def __init__(self, config: dict): - """Initialize the memory storage class.""" - self.storage = {} - - def put(self, key: str, value: str): - """Put a value into the memory storage.""" - self.storage[key] = value - - def get(self, key: str) -> str: - """Get a value from the memory storage.""" - return self.storage.get(key, None) - - def delete(self, key: str): - """Delete a value from the memory storage.""" - del self.storage[key] - - def list(self) -> list: - """List all keys in the memory storage.""" - return list(self.storage.keys()) diff --git a/src/solace_ai_connector/storage/storage_s3.py b/src/solace_ai_connector/storage/storage_s3.py deleted file mode 100644 index 2b2a582..0000000 --- a/src/solace_ai_connector/storage/storage_s3.py +++ /dev/null @@ -1,31 +0,0 @@ -"""AWS S3 storage implementation for the storage interface.""" - -import boto3 -from .storage import Storage - - -class StorageS3(Storage): - """AWS S3 storage class for the Solace AI Event Connector. The data is stored as JSON""" - - def __init__(self, config: dict): - """Initialize the AWS S3 storage class.""" - self.bucket_name = config["bucket_name"] - self.s3 = boto3.resource("s3") - - def put(self, key: str, value: dict): - """Put a value into the AWS S3 storage as a JSON object.""" - self.s3.Object(self.bucket_name, key).put(Body=value) - - def get(self, key: str) -> dict: - """Get a value from the AWS S3 storage""" - return ( - self.s3.Object(self.bucket_name, key).get()["Body"].read().decode("utf-8") - ) - - def delete(self, key: str): - """Delete a value from the AWS S3 storage.""" - self.s3.Object(self.bucket_name, key).delete() - - def list(self) -> list: - """List all keys in the AWS S3 storage.""" - return [obj.key for obj in self.s3.Bucket(self.bucket_name).objects.all()] diff --git a/tests/test_storage.py b/tests/test_storage.py deleted file mode 100644 index 174c011..0000000 --- a/tests/test_storage.py +++ /dev/null @@ -1,103 +0,0 @@ -"""This file contains tests for for memory and file storage""" - -import sys -import os - -# import queue - -sys.path.append("src") - -from utils_for_test_files import ( # pylint: disable=wrong-import-position - create_test_flows, - # create_and_run_component, - dispose_connector, - send_message_to_flow, - get_message_from_flow, -) -from solace_ai_connector.common.message import ( # pylint: disable=wrong-import-position - Message, -) - - -def test_memory_storage(): - """Test the memory storage""" - # Create a simple configuration - config_yaml = """ -instance_name: test_instance -log: - log_file_level: DEBUG - log_file: solace_ai_connector.log -storage: - - name: memory - storage_type: memory -flows: - # This will fail with the specified error - - name: flow - components: - - component_name: storage_tester - component_module: storage_tester - component_config: - storage_name: memory - component_input: - source_expression: input.payload - -""" - connector, flows = create_test_flows(config_yaml) - flow = flows[0] - - # Send a message to the input flow - send_message_to_flow(flow, Message(payload={"test_value": "second_value"})) - output_message = get_message_from_flow(flow) - assert output_message.get_data("previous") == {"test_value": "initial_value"} - - send_message_to_flow(flow, Message(payload={"test_value": "third_value"})) - output_message = get_message_from_flow(flow) - assert output_message.get_data("previous") == {"test_value": "second_value"} - - dispose_connector(connector) - - -def test_file_storage(): - """Test the file storage""" - # Create a simple configuration - config_yaml = """ -instance_name: test_instance -log: - log_file_level: DEBUG - log_file: solace_ai_connector.log -storage: - - name: file - storage_type: file - storage_config: - file: test_storage.json -flows: - # This will fail with the specified error - - name: flow - components: - - component_name: storage_tester - component_module: storage_tester - component_config: - storage_name: file - component_input: - source_expression: input.payload - -""" - # If the file exists, delete it - if os.path.exists("test_storage.json"): - os.remove("test_storage.json") - - connector, flows = create_test_flows(config_yaml) - flow = flows[0] - - # Send a message to the input flow - send_message_to_flow(flow, Message(payload={"test_value": "second_value"})) - output_message = get_message_from_flow(flow) - assert output_message.get_data("previous") == {"test_value": "initial_value"} - - send_message_to_flow(flow, Message(payload={"test_value": "third_value"})) - output_message = get_message_from_flow(flow) - assert output_message.get_data("previous") == {"test_value": "second_value"} - - dispose_connector(connector) - - os.remove("test_storage.json") From 31adf198e8d1a3b4138ee0eaa2b77c509fb5c7d7 Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Wed, 28 Aug 2024 08:05:10 -0400 Subject: [PATCH 02/55] Added support for temporary queue + UUID queue name (#26) --- .../components/inputs_outputs/broker_base.py | 4 ++++ .../components/inputs_outputs/broker_input.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_base.py b/src/solace_ai_connector/components/inputs_outputs/broker_base.py index df4bb5e..157177b 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_base.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_base.py @@ -11,6 +11,7 @@ from ..component_base import ComponentBase from ...common.message import Message from ...common.messaging.messaging_builder import MessagingServiceBuilder +import uuid # TBD - at the moment, there is no connection sharing supported. It should be possible # to share a connection between multiple components and even flows. The changes @@ -138,3 +139,6 @@ def get_acknowledgement_callback(self): def start(self): pass + + def generate_uuid(self): + return str(uuid.uuid4()) diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_input.py b/src/solace_ai_connector/components/inputs_outputs/broker_input.py index 507702a..841c8ee 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_input.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_input.py @@ -38,13 +38,13 @@ }, { "name": "broker_queue_name", - "required": True, - "description": "Queue name for broker", + "required": False, + "description": "Queue name for broker, if not provided it will use a temporary queue", }, { "name": "temporary_queue", "required": False, - "description": "Whether to create a temporary queue that will be deleted after disconnection", + "description": "Whether to create a temporary queue that will be deleted after disconnection, defaulted to True if broker_queue_name is not provided", "default": False, }, { @@ -91,7 +91,12 @@ def __init__(self, **kwargs): super().__init__(info, **kwargs) self.need_acknowledgement = True self.temporary_queue = self.get_config("temporary_queue", False) - self.broker_properties["temporary_queue"] = self.temporary_queue + # If broker_queue_name is not provided, use temporary queue + if not self.get_config("broker_queue_name"): + self.temporary_queue = True + self.broker_properties["temporary_queue"] = True + # Generating a UUID for the queue name + self.broker_properties["queue_name"] = self.generate_uuid() self.connect() def invoke(self, message, data): From 3ede57a88d5eb7a215e104853640b162562c550e Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Thu, 29 Aug 2024 11:39:11 -0400 Subject: [PATCH 03/55] Add assembly component and auto-generated documents (#27) * Added the assembly component * Auto-generated documents * Added type check * Update the cache service expiry logic + Update the assembly component to use cache expiry for timeout * Moved assembly to the correct place --- docs/components/assembly.md | 50 ++++++++ docs/components/broker_input.md | 4 +- docs/components/index.md | 3 + .../langchain_chat_model_with_history.md | 2 + docs/components/openai_chat_model.md | 62 +++++++++ .../openai_chat_model_with_history.md | 68 ++++++++++ examples/assembly_inputs.yaml | 74 +++++++++++ .../langchain_openai_with_history_chat.yaml | 4 +- examples/llm/openai_chat.yaml | 4 +- .../components/component_base.py | 6 + .../components/general/assembly.py | 119 ++++++++++++++++++ .../services/cache_service.py | 38 ++++-- 12 files changed, 417 insertions(+), 17 deletions(-) create mode 100644 docs/components/assembly.md create mode 100644 docs/components/openai_chat_model.md create mode 100644 docs/components/openai_chat_model_with_history.md create mode 100644 examples/assembly_inputs.yaml create mode 100644 src/solace_ai_connector/components/general/assembly.py diff --git a/docs/components/assembly.md b/docs/components/assembly.md new file mode 100644 index 0000000..5d65192 --- /dev/null +++ b/docs/components/assembly.md @@ -0,0 +1,50 @@ +# Assembly + +Assembles messages till criteria is met, the output will be the assembled message + +## Configuration Parameters + +```yaml +component_name: +component_module: assembly +component_config: + assemble_key: + max_items: + max_time_ms: +``` + +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| assemble_key | True | | The key from input message that would cluster the similar messages together | +| max_items | False | 10 | Maximum number of messages to assemble. Once this value is reached, the messages would be flushed to the output | +| max_time_ms | False | 10000 | The timeout in seconds to wait for the messages to assemble. If timeout is reached before the max size is reached, the messages would be flushed to the output | + + +## Component Input Schema + +``` +{ + +} +``` + + +## Component Output Schema + +``` +[ + { + payload: , + topic: , + user_properties: { + + } + }, + ... +] +``` +| Field | Required | Description | +| --- | --- | --- | +| [].payload | False | | +| [].topic | False | | +| [].user_properties | False | | diff --git a/docs/components/broker_input.md b/docs/components/broker_input.md index 51c02d0..ece65fb 100644 --- a/docs/components/broker_input.md +++ b/docs/components/broker_input.md @@ -27,8 +27,8 @@ component_config: | broker_username | True | | Client username for broker | | broker_password | True | | Client password for broker | | broker_vpn | True | | Client VPN for broker | -| broker_queue_name | True | | Queue name for broker | -| temporary_queue | False | False | Whether to create a temporary queue that will be deleted after disconnection | +| broker_queue_name | False | | Queue name for broker, if not provided it will use a temporary queue | +| temporary_queue | False | False | Whether to create a temporary queue that will be deleted after disconnection, defaulted to True if broker_queue_name is not provided | | broker_subscriptions | True | | Subscriptions for broker | | payload_encoding | False | utf-8 | Encoding for the payload (utf-8, base64, gzip, none) | | payload_format | False | json | Format for the payload (json, yaml, text) | diff --git a/docs/components/index.md b/docs/components/index.md index 58cf8ad..5d61fbd 100644 --- a/docs/components/index.md +++ b/docs/components/index.md @@ -3,6 +3,7 @@ | Component | Description | | --- | --- | | [aggregate](aggregate.md) | Aggregate messages into one message. | +| [assembly](assembly.md) | Assembles messages till criteria is met, the output will be the assembled message | | [broker_input](broker_input.md) | Connect to a messaging broker and receive messages from it. The component will output the payload, topic, and user properties of the message. | | [broker_output](broker_output.md) | Connect to a messaging broker and send messages to it. Note that this component requires that the data is transformed into the input schema. | | [broker_request_response](broker_request_response.md) | Connect to a messaging broker, send request messages, and receive responses. This component combines the functionality of broker_input and broker_output with additional request-response handling. | @@ -17,6 +18,8 @@ | [langchain_vector_store_embedding_index](langchain_vector_store_embedding_index.md) | Use LangChain Vector Stores to index text for later semantic searches. This will take text, run it through an embedding model and then store it in a vector database. | | [langchain_vector_store_embedding_search](langchain_vector_store_embedding_search.md) | Use LangChain Vector Stores to search a vector store with a semantic search. This will take text, run it through an embedding model with a query embedding and then find the closest matches in the store. | | [message_filter](message_filter.md) | A filtering component. This will apply a user configurable expression. If the expression evaluates to True, the message will be passed on. If the expression evaluates to False, the message will be discarded. If the message is discarded, any previous components that require an acknowledgement will be acknowledged. | +| [openai_chat_model](openai_chat_model.md) | OpenAI chat model component | +| [openai_chat_model_with_history](openai_chat_model_with_history.md) | OpenAI chat model component with conversation history | | [pass_through](pass_through.md) | What goes in comes out | | [stdin_input](stdin_input.md) | STDIN input component. The component will prompt for input, which will then be placed in the message payload using the output schema below. | | [stdout_output](stdout_output.md) | STDOUT output component | diff --git a/docs/components/langchain_chat_model_with_history.md b/docs/components/langchain_chat_model_with_history.md index 100e429..e66e9a6 100644 --- a/docs/components/langchain_chat_model_with_history.md +++ b/docs/components/langchain_chat_model_with_history.md @@ -15,6 +15,7 @@ component_config: history_max_turns: history_max_message_size: history_max_tokens: + history_max_time: history_module: history_class: history_config: @@ -33,6 +34,7 @@ component_config: | history_max_turns | False | 20 | The maximum number of turns to keep in the history. If not set, the history will be limited to 20 turns. | | history_max_message_size | False | 1000 | The maximum amount of characters to keep in a single message in the history. | | history_max_tokens | False | 8000 | The maximum number of tokens to keep in the history. If not set, the history will be limited to 8000 tokens. | +| history_max_time | False | None | The maximum time (in seconds) to keep messages in the history. If not set, messages will not expire based on time. | | history_module | False | langchain_community.chat_message_histories | The module that contains the history class. Default: 'langchain_community.chat_message_histories' | | history_class | False | ChatMessageHistory | The class to use for the history. Default: 'ChatMessageHistory' | | history_config | False | | The configuration for the history class. | diff --git a/docs/components/openai_chat_model.md b/docs/components/openai_chat_model.md new file mode 100644 index 0000000..fa8a506 --- /dev/null +++ b/docs/components/openai_chat_model.md @@ -0,0 +1,62 @@ +# OpenAIChatModel + +OpenAI chat model component + +## Configuration Parameters + +```yaml +component_name: +component_module: openai_chat_model +component_config: + api_key: + model: + temperature: + base_url: + stream_to_flow: + llm_mode: + stream_batch_size: + set_response_uuid_in_user_properties: +``` + +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| api_key | True | | OpenAI API key | +| model | True | | OpenAI model to use (e.g., 'gpt-3.5-turbo') | +| temperature | False | 0.7 | Sampling temperature to use | +| base_url | False | None | Base URL for OpenAI API | +| stream_to_flow | False | | Name the flow to stream the output to - this must be configured for llm_mode='stream'. | +| llm_mode | False | none | The mode for streaming results: 'sync' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response. | +| stream_batch_size | False | 15 | The minimum number of words in a single streaming result. Default: 15. | +| set_response_uuid_in_user_properties | False | False | Whether to set the response_uuid in the user_properties of the input_message. This will allow other components to correlate streaming chunks with the full response. | + + +## Component Input Schema + +``` +{ + messages: [ + { + role: , + content: + }, + ... + ] +} +``` +| Field | Required | Description | +| --- | --- | --- | +| messages | True | | +| messages[].role | True | | +| messages[].content | True | | + + +## Component Output Schema + +``` +{ + content: +} +``` +| Field | Required | Description | +| --- | --- | --- | +| content | True | The generated response from the model | diff --git a/docs/components/openai_chat_model_with_history.md b/docs/components/openai_chat_model_with_history.md new file mode 100644 index 0000000..262bc72 --- /dev/null +++ b/docs/components/openai_chat_model_with_history.md @@ -0,0 +1,68 @@ +# OpenAIChatModelWithHistory + +OpenAI chat model component with conversation history + +## Configuration Parameters + +```yaml +component_name: +component_module: openai_chat_model_with_history +component_config: + api_key: + model: + temperature: + base_url: + stream_to_flow: + llm_mode: + stream_batch_size: + set_response_uuid_in_user_properties: + history_max_turns: + history_max_time: +``` + +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| api_key | True | | OpenAI API key | +| model | True | | OpenAI model to use (e.g., 'gpt-3.5-turbo') | +| temperature | False | 0.7 | Sampling temperature to use | +| base_url | False | None | Base URL for OpenAI API | +| stream_to_flow | False | | Name the flow to stream the output to - this must be configured for llm_mode='stream'. | +| llm_mode | False | none | The mode for streaming results: 'sync' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response. | +| stream_batch_size | False | 15 | The minimum number of words in a single streaming result. Default: 15. | +| set_response_uuid_in_user_properties | False | False | Whether to set the response_uuid in the user_properties of the input_message. This will allow other components to correlate streaming chunks with the full response. | +| history_max_turns | False | 10 | Maximum number of conversation turns to keep in history | +| history_max_time | False | 3600 | Maximum time to keep conversation history (in seconds) | + + +## Component Input Schema + +``` +{ + messages: [ + { + role: , + content: + }, + ... + ], + clear_history_but_keep_depth: +} +``` +| Field | Required | Description | +| --- | --- | --- | +| messages | True | | +| messages[].role | True | | +| messages[].content | True | | +| clear_history_but_keep_depth | False | Clear history but keep the last N messages. If 0, clear all history. If not set, do not clear history. | + + +## Component Output Schema + +``` +{ + content: +} +``` +| Field | Required | Description | +| --- | --- | --- | +| content | True | The generated response from the model | diff --git a/examples/assembly_inputs.yaml b/examples/assembly_inputs.yaml new file mode 100644 index 0000000..5a4feb4 --- /dev/null +++ b/examples/assembly_inputs.yaml @@ -0,0 +1,74 @@ +# Example for assembling inputs for a Solace AI connector +# +# It will subscribe to `demo/messages` and expect an event with the payload: +# +# The input message has the following schema: +# { +# "content": "", +# "id": "" +# } +# +# It will then send an event back to Solace with the topic: `demo/messages/assembled` +# +# +# required ENV variables: +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN + +--- +log: + stdout_log_level: INFO + log_file_level: INFO + log_file: solace_ai_connector.log + +shared_config: + - broker_config: &broker_connection + broker_type: solace + broker_url: ${SOLACE_BROKER_URL} + broker_username: ${SOLACE_BROKER_USERNAME} + broker_password: ${SOLACE_BROKER_PASSWORD} + broker_vpn: ${SOLACE_BROKER_VPN} + + +flows: + - name: Simple assemble flow + components: + # Input from a Solace broker + - component_name: solace_input + component_module: broker_input + component_config: + <<: *broker_connection + broker_subscriptions: + - topic: demo/messages + payload_encoding: utf-8 + payload_format: json + + # Assemble messages + - component_name: assemble_messages + component_module: assembly + component_config: + assemble_key: id + max_items: 3 + max_time_ms: 10000 + component_input: + source_expression: input.payload + + # Send assembled messages back to broker + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.output:payload + - type: copy + source_expression: template:{{text://input.topic}}/assembled + dest_expression: user_data.output:topic + component_input: + source_expression: user_data.output diff --git a/examples/llm/langchain_openai_with_history_chat.yaml b/examples/llm/langchain_openai_with_history_chat.yaml index 1440b33..639567f 100644 --- a/examples/llm/langchain_openai_with_history_chat.yaml +++ b/examples/llm/langchain_openai_with_history_chat.yaml @@ -37,9 +37,9 @@ shared_config: broker_password: ${SOLACE_BROKER_PASSWORD} broker_vpn: ${SOLACE_BROKER_VPN} -# Take from Slack and publish to Solace +# Take from input broker and publish back to Solace flows: - # Slack chat input processing + # broker input processing - name: Simple template to LLM components: # Input from a Solace broker diff --git a/examples/llm/openai_chat.yaml b/examples/llm/openai_chat.yaml index 65e9897..4c3f753 100644 --- a/examples/llm/openai_chat.yaml +++ b/examples/llm/openai_chat.yaml @@ -36,9 +36,9 @@ shared_config: broker_password: ${SOLACE_BROKER_PASSWORD} broker_vpn: ${SOLACE_BROKER_VPN} -# Take from Slack and publish to Solace +# Take from input broker and publish back to Solace flows: - # Slack chat input processing + # broker input processing - name: Simple template to LLM components: # Input from a Solace broker diff --git a/src/solace_ai_connector/components/component_base.py b/src/solace_ai_connector/components/component_base.py index 82203ee..8fc6270 100644 --- a/src/solace_ai_connector/components/component_base.py +++ b/src/solace_ai_connector/components/component_base.py @@ -126,6 +126,8 @@ def process_event(self, event): self.current_message = None elif event.event_type == EventType.TIMER: self.handle_timer_event(event.data) + elif event.event_type == EventType.CACHE_EXPIRY: + self.handle_cache_expiry_event(event.data) else: log.warning( "%sUnknown event type: %s", self.log_identifier, event.event_type @@ -158,6 +160,10 @@ def handle_timer_event(self, timer_data): # This method can be overridden by components that need to handle timer events pass + def handle_cache_expiry_event(self, timer_data): + # This method can be overridden by components that need to handle cache expiry events + pass + def discard_current_message(self): # If the message is to be discarded, we need to acknowledge any previous components self.current_message_has_been_discarded = True diff --git a/src/solace_ai_connector/components/general/assembly.py b/src/solace_ai_connector/components/general/assembly.py new file mode 100644 index 0000000..72f69b2 --- /dev/null +++ b/src/solace_ai_connector/components/general/assembly.py @@ -0,0 +1,119 @@ +"""Assembly component for the Solace AI Event Connector""" + +from ...common.log import log +from ..component_base import ComponentBase +from ...common.message import Message + + +info = { + "class_name": "Assembly", + "description": ( + "Assembles messages till criteria is met, " + "the output will be the assembled message" + ), + "config_parameters": [ + { + "name": "assemble_key", + "required": True, + "description": "The key from input message that would cluster the similar messages together", + }, + { + "name": "max_items", + "required": False, + "default": 10, + "description": "Maximum number of messages to assemble. Once this value is reached, the messages would be flushed to the output", + }, + { + "name": "max_time_ms", + "required": False, + "default": 10000, + "description": "The timeout in seconds to wait for the messages to assemble. If timeout is reached before the max size is reached, the messages would be flushed to the output", + }, + ], + "input_schema": { + "type": "object", + "properties": {}, + "required": [], + }, + "output_schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "payload": { + "type": "string", + }, + "topic": { + "type": "string", + }, + "user_properties": { + "type": "object", + }, + }, + }, + }, +} + +# Default timeout to flush the messages +DEFAULT_FLUSH_TIMEOUT_MS = 10000 +ASSEMBLY_EXPIRY_ID = "assembly_expiry" + + +class Assembly(ComponentBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + self.assemble_key = self.get_config("assemble_key") + self.max_items = self.get_config("max_items") + self.max_time_ms = self.get_config("max_time_ms", DEFAULT_FLUSH_TIMEOUT_MS) + + def invoke(self, message, data): + # Check if the message has the assemble key + if self.assemble_key not in data or type(data[self.assemble_key]) is not str: + log.error( + f"Message does not have the key {self.assemble_key} or it is not a string" + ) + raise ValueError( + f"Message does not have the key {self.assemble_key} or it is not a string" + ) + + event_key = data[self.assemble_key] + # Fetch the current assembly from cache + current_assembly = self.cache_service.get_data(event_key) + + # Set expiry timeout only on cache creation (not on update) + expiry = None + # Create a new assembly if not present + if not current_assembly: + expiry = self.max_time_ms / 1000 + current_assembly = { + "list": [], + "message": Message(), + } + + # Update cache with the new data + message.combine_with_message(current_assembly["message"]) + current_assembly["message"] = message + current_assembly["list"].append(data) + self.cache_service.add_data( + event_key, + current_assembly, + expiry=expiry, + metadata=ASSEMBLY_EXPIRY_ID, + component=self, + ) + + # Flush the assembly if the max size is reached + if len(current_assembly["list"]) >= self.max_items: + log.debug(f"Flushing data by size - {len(current_assembly['list'])} items") + return self.flush_assembly(event_key)["list"] + + def handle_cache_expiry_event(self, data): + if data["metadata"] == ASSEMBLY_EXPIRY_ID: + assembled_data = data["expired_data"] + log.debug(f"Flushing data by timeout - {len(assembled_data['list'])} items") + self.process_post_invoke(assembled_data["list"], assembled_data["message"]) + + def flush_assembly(self, assemble_key): + assembly = self.cache_service.get_data(assemble_key) + self.cache_service.remove_data(assemble_key) + return assembly diff --git a/src/solace_ai_connector/services/cache_service.py b/src/solace_ai_connector/services/cache_service.py index 885ec29..19f08bb 100644 --- a/src/solace_ai_connector/services/cache_service.py +++ b/src/solace_ai_connector/services/cache_service.py @@ -13,7 +13,7 @@ class CacheStorageBackend(ABC): @abstractmethod - def get(self, key: str) -> Any: + def get(self, key: str, include_meta=False) -> Any: pass @abstractmethod @@ -40,7 +40,7 @@ def __init__(self): self.store: Dict[str, Dict[str, Any]] = {} self.lock = Lock() - def get(self, key: str) -> Any: + def get(self, key: str, include_meta=False) -> Any: with self.lock: item = self.store.get(key) if item is None: @@ -48,7 +48,7 @@ def get(self, key: str) -> Any: if item["expiry"] and time.time() > item["expiry"]: del self.store[key] return None - return item["value"] + return item if include_meta else item["value"] def set( self, @@ -61,7 +61,7 @@ def set( with self.lock: self.store[key] = { "value": value, - "expiry": time.time() + expiry if expiry else None, + "expiry": expiry, "metadata": metadata, "component": component, } @@ -103,7 +103,7 @@ def __init__(self, connection_string: str): Base.metadata.create_all(self.engine) self.Session = sessionmaker(bind=self.engine) - def get(self, key: str) -> Any: + def get(self, key: str, include_meta=False) -> Any: session = self.Session() try: item = session.query(CacheItem).filter_by(key=key).first() @@ -113,6 +113,13 @@ def get(self, key: str) -> Any: session.delete(item) session.commit() return None + if include_meta: + return { + "value": pickle.loads(item.value), + "metadata": pickle.loads(item.item_metadata) if item.item_metadata else None, + "expiry": item.expiry, + "component": self._get_component_from_reference(item.component_reference), + } return pickle.loads(item.value), ( pickle.loads(item.item_metadata) if item.item_metadata else None ) @@ -207,6 +214,16 @@ def add_data( metadata: Optional[Dict] = None, component=None, ): + # Calculate the expiry time + expiry = time.time() + expiry if expiry else None + + # Check if the key already exists + cache = self.storage.get(key, include_meta=True) + if cache: + # Use the cache data to combine with the new data + expiry = expiry or cache["expiry"] + metadata = metadata or cache["metadata"] + component = component or cache["component"] self.storage.set(key, value, expiry, metadata, component) with self.lock: if expiry: @@ -242,23 +259,22 @@ def _check_expirations(self): # Use the storage backend to get all items all_items = self.storage.get_all() - for key, (value, metadata, expiry, component) in all_items.items(): if expiry and current_time > expiry: - expired_keys.append((key, metadata, component)) + expired_keys.append((key, metadata, component, value)) elif expiry and (next_expiry is None or expiry < next_expiry): next_expiry = expiry with self.lock: - for key, _, _ in expired_keys: + for key, _, _, _ in expired_keys: self.storage.delete(key) self.next_expiry = next_expiry - - for key, metadata, component in expired_keys: + + for key, metadata, component, value in expired_keys: if component: event = Event( - EventType.CACHE_EXPIRY, {"key": key, "metadata": metadata} + EventType.CACHE_EXPIRY, {"key": key, "metadata": metadata, "expired_data": value} ) component.enqueue(event) From c95b7f39bb1485a14988df474c95751d6a6039e3 Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Thu, 29 Aug 2024 13:08:32 -0400 Subject: [PATCH 04/55] Added MoA Example + UUID Invoke Function (#28) * MoA example: Broadcast to multiple agents * Added MoA event manager, added uuid invoke_function + test, updated auto-generated docs * Added assembly layer to MoA example --- docs/configuration.md | 2 + examples/llm/mixture_of_agents.yaml | 486 ++++++++++++++++++ .../common/invoke_functions.py | 4 +- .../components/inputs_outputs/broker_base.py | 2 +- tests/test_invoke.py | 20 + 5 files changed, 512 insertions(+), 2 deletions(-) create mode 100644 examples/llm/mixture_of_agents.yaml diff --git a/docs/configuration.md b/docs/configuration.md index 841ea21..5e6e5e8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -116,6 +116,8 @@ There is a module named `invoke_functions` that has a list of functions that can - `empty_tuple`: Return an empty tuple - `empty_float`: Return 0.0 - `empty_int`: Return 0 +- `if_else`: If the first value is true, return the second value, otherwise return the third value +- `uuid`: returns a universally unique identifier (UUID) Use positional parameters to pass values to the functions that expect arguments. Here is an example of using the `invoke_functions` module to do some simple operations: diff --git a/examples/llm/mixture_of_agents.yaml b/examples/llm/mixture_of_agents.yaml new file mode 100644 index 0000000..27af1f6 --- /dev/null +++ b/examples/llm/mixture_of_agents.yaml @@ -0,0 +1,486 @@ +# This will create a flow using the mixture of agents pattern (https://arxiv.org/abs/2406.04692) +# +# It will subscribe to `moa/question` and expect an event with the payload: +# The input message has the following schema: +# { +# "query": "" +# } +# +# It will then send an event back to Solace with the topic: `moa/question/response` +# +# NOTE: For horizontal scaling, partitioned queues must be used. This is not implemented in this example. +# +# Dependencies: +# pip install -U langchain-google-vertexai langchain_anthropic langchain_openai openai +# +# required ENV variables: +# - GOOGLE_APPLICATION_CREDENTIALS: the path to a service account JSON file +# - VERTEX_REGION +# - VERTEX_API_ENDPOINT - optional +# - VERTEX_MODEL_NAME +# - OPENAI_API_KEY +# - OPENAI_API_ENDPOINT - optional +# - OPENAI_MODEL_NAME +# - ANTHROPIC_API_KEY +# - ANTHROPIC_API_ENDPOINT - optional +# - ANTHROPIC_MODEL_NAME +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN +# - NUMBER_OF_MOA_LAYERS: the number of layers in the mixture of agents + +--- +log: + stdout_log_level: INFO + log_file_level: INFO + log_file: solace_ai_connector.log + +shared_config: + # Broker connection configuration + - broker_config: &broker_connection + broker_type: solace + broker_url: ${SOLACE_BROKER_URL} + broker_username: ${SOLACE_BROKER_USERNAME} + broker_password: ${SOLACE_BROKER_PASSWORD} + broker_vpn: ${SOLACE_BROKER_VPN} + + # Agent broker input configuration + - agent_broker_input: &agent_broker_input + component_name: solace_agent_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_subscriptions: + - topic: moa/broadcast + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # Agent broker output configuration + - agent_broker_output: &agent_broker_output + component_name: solace_agent_broker + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + # Copy the contents of the input event (query, id, layer_number) + - type: copy + source_expression: input.payload + dest_expression: user_data.output:payload + # Copy the output from the LLM + - type: copy + source_expression: user_data.formatted_response:content + dest_expression: user_data.output:payload.content + # Copy the agent name + - type: copy + source_expression: user_data.formatted_response:agent + dest_expression: user_data.output:payload.agent + # Copy the response topic based on input topic + - type: copy + source_expression: template:{{text://input.topic}}/next + dest_expression: user_data.output:topic + component_input: + source_expression: user_data.output + + # Agent input transformations + - agent_input_transformations: &agent_input_transformations + input_transforms: + - type: copy + source_expression: | + template:You are a helpful AI assistant. Please help with the user's request below: + + {{text://input.payload:query}} + + dest_expression: user_data.llm_input:messages.0.content + - type: copy + source_expression: static:user + dest_expression: user_data.llm_input:messages.0.role + component_input: + source_expression: user_data.llm_input + +flows: + # Event manager - Updates user message and send to all agents + - name: event manager + components: + # Broker input for user query + - component_name: user_query_input + component_module: broker_input + component_config: + <<: *broker_connection + broker_subscriptions: + - topic: moa/question + qos: 1 + - topic: moa/question/aggregate + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # Broker output for agents - Update input event with layer number and UUID + - component_name: solace_agent_broker + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + # Copy the original user query + - type: copy + source_expression: input.payload:query + dest_expression: user_data.output:payload.query + # Increase layer number by 1, if none exists, set to 1 + - type: copy + source_expression: + invoke: + # Check if layer number exists + module: invoke_functions + function: if_else + params: + positional: + - source_expression(input.payload:layer_number) + - invoke: + # Add 1 to the layer number + module: invoke_functions + function: add + params: + positional: + - invoke: + # Default to zero + module: invoke_functions + function: or_op + params: + positional: + - source_expression(input.payload:layer_number) + - 0 + - 1 + # No layer number, set to 1 + - 1 + dest_expression: user_data.output:payload.layer_number + + # Copy over the UUID, if doesn't exists create one + - type: copy + source_value: + invoke: + module: invoke_functions + function: if_else + params: + positional: + - source_expression(input.payload:id) + - source_expression(input.payload:id) + - invoke: + module: invoke_functions + function: uuid + dest_expression: user_data.output:payload.id + + # Copy the response topic based on input topic + - type: copy + source_value: moa/broadcast + dest_expression: user_data.output:topic + + component_input: + source_expression: user_data.output + + # Agent 1 - Google Vertex AI + - name: Agent 1 - Google Vertex AI + components: + # Broker input for Vertex AI + - <<: *agent_broker_input + + # Vertex AI LLM Request + - component_name: llm_request + component_module: langchain_chat_model + component_config: + langchain_module: langchain_google_vertexai + langchain_class: ChatVertexAI + langchain_component_config: + base_url: ${VERTEX_API_ENDPOINT} + location: ${VERTEX_REGION} + model: ${VERTEX_MODEL_NAME} + temperature: 0.01 + <<: *agent_input_transformations + + # Format Vertex AI response for broker output + - component_name: format_response + component_module: pass_through + input_transforms: + - type: copy + source_value: vertex_ai + dest_expression: user_data.formatted_response:agent + - type: copy + source_expression: previous + dest_expression: user_data.formatted_response:content + component_input: + source_expression: user_data.formatted_response + + # Broker output for Vertex AI + - <<: *agent_broker_output + + # Agent 2 - OpenAI + - name: Agent 2 - OpenAI + components: + # Broker input for OpenAI + - <<: *agent_broker_input + + # OpenAI LLM Request + - component_name: llm_request + component_module: openai_chat_model + component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${OPENAI_MODEL_NAME} + temperature: 0.01 + <<: *agent_input_transformations + + # Format OpenAI response for broker output + - component_name: format_response + component_module: pass_through + input_transforms: + - type: copy + source_value: openai + dest_expression: user_data.formatted_response:agent + - type: copy + source_expression: previous:content + dest_expression: user_data.formatted_response:content + component_input: + source_expression: user_data.formatted_response + + # Broker output for OpenAI + - <<: *agent_broker_output + + # Agent 3 - Anthropic + - name: Agent 3 - Anthropic + components: + # Broker input for Anthropic + - <<: *agent_broker_input + + # Anthropic LLM Request + - component_name: llm_request + component_module: langchain_chat_model + component_config: + langchain_module: langchain_anthropic + langchain_class: ChatAnthropic + langchain_component_config: + api_key: ${ANTHROPIC_API_KEY} + base_url: ${ANTHROPIC_API_ENDPOINT} + model: ${ANTHROPIC_MODEL_NAME} + temperature: 0.01 + <<: *agent_input_transformations + + # Format Anthropic response for broker output + - component_name: format_response + component_module: pass_through + input_transforms: + - type: copy + source_value: anthropic + dest_expression: user_data.formatted_response:agent + - type: copy + source_expression: previous + dest_expression: user_data.formatted_response:content + component_input: + source_expression: user_data.formatted_response + + # Broker output for Anthropic + - <<: *agent_broker_output + + # Assemble the responses and send to user/next layer + - name: Assemble agent responses + components: + # Agents responses from solace broker + - component_name: agent_responses + component_module: broker_input + component_config: + <<: *broker_connection + broker_subscriptions: + - topic: moa/broadcast/next + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # Assemble Agent responses + - component_name: assemble_responses + component_module: assembly + component_config: + assemble_key: id + max_time_ms: 30000 + max_items: 3 # Number of Agents + component_input: + source_expression: input.payload + + # Format response for the LLM request + - component_name: format_response + component_module: pass_through + input_transforms: + # Copy the ID + - type: copy + source_expression: previous:0.id + dest_expression: user_data.aggregated_data:id + # Copy the layer number + - type: copy + source_expression: previous:0.layer_number + dest_expression: user_data.aggregated_data:layer_number + # Copy the initial user query + - type: copy + source_expression: previous:0.query + dest_expression: user_data.aggregated_data:query + # Transform each response to use the template + - type: map + source_list_expression: previous + source_expression: | + template: + {{text://item:content}} + \n + dest_list_expression: user_data.temp:responses + # Transform and reduce the responses to one message + - type: reduce + source_list_expression: user_data.temp:responses + source_expression: item + initial_value: "" + accumulator_function: + invoke: + module: invoke_functions + function: add + params: + positional: + - source_expression(keyword_args:accumulated_value) + - source_expression(keyword_args:current_value) + dest_expression: user_data.aggregated_data:responses + component_input: + source_expression: user_data.aggregated_data + + # Aggregate all the outcomes from the agents + - component_name: aggregate_generations + component_module: openai_chat_model + component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${OPENAI_MODEL_NAME} + temperature: 0.01 + input_transforms: + - type: copy + source_expression: | + template:You have been provided with a set of responses from various large language models to a user query. Your task is to synthesize these responses into a single, high-quality response. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect. Your response should not simply replicate the given answers but should offer a refined, accurate, and comprehensive reply to the instruction. Ensure your response is well-structured, coherent, and adheres to the highest standards of accuracy and reliability. Do not add any extra comments about how you created the response, just synthesize these responses as instructed. Do not mention that the result is created of multiple responses. + + User Query: + + + {{text://user_data.aggregated_data:query}} + <\user-query> + + Responses: + + {{text://user_data.aggregated_data:responses}} + dest_expression: user_data.llm_input:messages.0.content + - type: copy + source_expression: static:user + dest_expression: user_data.llm_input:messages.0.role + component_input: + source_expression: user_data.llm_input + + - component_name: aggregator_output + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + # Copy the contents of the required info for next layer + - type: copy + source_expression: user_data.aggregated_data:id + dest_expression: user_data.output:payload.id + - type: copy + source_expression: user_data.aggregated_data:layer_number + dest_expression: user_data.output:payload.layer_number + # Copy to a temporary location the modified query for the next layer + - type: copy + source_expression: | + template: For the given query, the following draft response is created. Update and enhance the response to be logically accurate, sound natural, and free of errors. + Think before you reply. And only reply with the updated response. Do not add any extra comments. + + + {{text://user_data.aggregated_data:query}} + <\query> + + + {{text://previous:content}} + <\response> + dest_expression: user_data.temp:new_query + # Copy the results of the aggregation by LLM + # The LLM result is added under query for the next layer + - type: copy + source_expression: + invoke: + # If the layer number is less than the number of layers, + # modify the response for the next layer of agents + module: invoke_functions + function: if_else + params: + positional: + - invoke: + module: invoke_functions + function: less_than + params: + positional: + - source_expression(user_data.aggregated_data:layer_number, int) + - ${NUMBER_OF_MOA_LAYERS} + - source_expression(user_data.temp:new_query) + - source_expression(previous:content) + dest_expression: user_data.output:payload.query + # Copy the response topic based on layer number + - type: copy + source_expression: + invoke: + # If the layer number is less than the number of layers, + # send to the next layer, otherwise send to the user + module: invoke_functions + function: if_else + params: + positional: + - invoke: + module: invoke_functions + function: less_than + params: + positional: + - source_expression(user_data.aggregated_data:layer_number, int) + - ${NUMBER_OF_MOA_LAYERS} + - moa/question/aggregate + - moa/question/cleanup + dest_expression: user_data.output:topic + component_input: + source_expression: user_data.output + + # Cleanup the responses from the assembly and send to the user + - name: Cleanup assembled responses + components: + # Response from the assembly + - component_name: assembly_response + component_module: broker_input + component_config: + <<: *broker_connection + broker_subscriptions: + - topic: moa/question/cleanup + qos: 1 + payload_encoding: utf-8 + payload_format: json + + - component_name: aggregator_output + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + # Copy the user query and response for the final response + - type: copy + source_expression: input.payload:query + dest_expression: user_data.output:payload.response + - type: copy + source_value: moa/question/response + dest_expression: user_data.output:topic + component_input: + source_expression: user_data.output diff --git a/src/solace_ai_connector/common/invoke_functions.py b/src/solace_ai_connector/common/invoke_functions.py index db0365d..bdde26e 100644 --- a/src/solace_ai_connector/common/invoke_functions.py +++ b/src/solace_ai_connector/common/invoke_functions.py @@ -1,5 +1,7 @@ """Set of simple functions to take the place of operators in the config file""" +import uuid as uuid_module + add = lambda x, y: x + y append = lambda x, y: x + [y] subtract = lambda x, y: x - y @@ -26,7 +28,7 @@ empty_float = lambda: 0.0 empty_int = lambda: 0 if_else = lambda x, y, z: y if x else z - +uuid = lambda: str(uuid_module.uuid4()) # A few test functions def _test_positional_and_keyword_args(*args, **kwargs): diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_base.py b/src/solace_ai_connector/components/inputs_outputs/broker_base.py index 157177b..6b91a01 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_base.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_base.py @@ -4,6 +4,7 @@ import gzip import json import yaml +import uuid from abc import abstractmethod @@ -11,7 +12,6 @@ from ..component_base import ComponentBase from ...common.message import Message from ...common.messaging.messaging_builder import MessagingServiceBuilder -import uuid # TBD - at the moment, there is no connection sharing supported. It should be possible # to share a connection between multiple components and even flows. The changes diff --git a/tests/test_invoke.py b/tests/test_invoke.py index 46c6267..4834604 100644 --- a/tests/test_invoke.py +++ b/tests/test_invoke.py @@ -1076,3 +1076,23 @@ def test_filter_transform_sub_field_greater_than_2(): assert output_message.get_data("user_data.temp") == { "new_list": [{"my_val": 3}, {"my_val": 4}] } + + +def test_invoke_with_uuid_generator(): + """Verify that the uuid invoke_function returns an ID""" + response = resolve_config_values( + { + "a": { + "invoke": { + "module": "invoke_functions", + "function": "uuid" + }, + }, + } + ) + + # Check if the output is of type string + assert type(response["a"]) == str + + # Check if the output is a valid UUID + assert len(response["a"]) == 36 \ No newline at end of file From 8fa025eb43d3084e2beabba3d1563fa99d721064 Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Tue, 3 Sep 2024 08:11:12 -0400 Subject: [PATCH 05/55] Update documentation for new users + Refactored component_input & source_expression (#29) * Refactored component_input to input_selection * Updated, added, and enhanced the documentation with new users in mind * Refactored source_expression function to evaluate_expression (backward compatible) * Added tips and tricks section + info and examples on custom modules * tiny format update * tiny update --- README.md | 12 +- config.yaml | 6 +- docs/components/aggregate.md | 19 + docs/components/error_input.md | 2 +- docs/components/index.md | 2 +- docs/components/iterate.md | 13 + docs/configuration.md | 698 +++++++++++------- docs/getting_started.md | 6 +- docs/index.md | 10 +- docs/overview.md | 32 +- docs/tips_and_tricks.md | 153 ++++ docs/transforms/filter.md | 4 +- docs/transforms/map.md | 6 +- docs/transforms/reduce.md | 6 +- docs/usage.md | 85 --- examples/ack_test.yaml | 6 +- examples/anthropic_bedrock.yaml | 4 +- examples/assembly_inputs.yaml | 4 +- examples/chat_model_with_history.yaml | 2 +- examples/error_handler.yaml | 12 +- examples/llm/anthropic_chat.yaml | 4 +- examples/llm/bedrock_anthropic_chat.yaml | 2 +- .../langchain_openai_with_history_chat.yaml | 4 +- examples/llm/mixture_of_agents.yaml | 42 +- examples/llm/openai_chat.yaml | 4 +- examples/llm/vertexai_chat.yaml | 4 +- examples/milvus_store.yaml | 4 +- examples/request_reply.yaml | 6 +- examples/vector_store_search.yaml | 2 +- src/solace_ai_connector/common/utils.py | 28 +- .../components/component_base.py | 12 +- .../components/general/aggregate.py | 23 +- .../components/general/iterate.py | 10 + .../components/inputs_outputs/error_input.py | 2 +- .../services/cache_service.py | 3 +- src/solace_ai_connector/transforms/filter.py | 6 +- src/solace_ai_connector/transforms/map.py | 8 +- src/solace_ai_connector/transforms/reduce.py | 8 +- tests/test_aggregate.py | 6 +- tests/test_config_file.py | 4 +- tests/test_error_flows.py | 2 +- tests/test_filter.py | 4 +- tests/test_flows.py | 12 +- tests/test_invoke.py | 108 +-- tests/test_iterate.py | 4 +- tests/test_timer_input.py | 2 +- tests/test_transforms.py | 22 +- 47 files changed, 866 insertions(+), 552 deletions(-) create mode 100644 docs/tips_and_tricks.md delete mode 100644 docs/usage.md diff --git a/README.md b/README.md index bad7909..994a5c1 100644 --- a/README.md +++ b/README.md @@ -6,27 +6,27 @@ This project provides a standalone, Python-based application to allow Solace eve a wide range of AI models and services. The application is designed to be easily extensible to support new AI models and services. -## Getting started quickly - -Please see the [getting started guide](docs/getting_started.md) for instructions on how to get started quickly. - ## Documentation Please see the [documentation](docs/index.md) for more information. +## Getting started quickly + +Please see the [getting started guide](docs/getting_started.md) for instructions on how to get started quickly. + ## Support This is not an officially supported Solace product. For more information try these resources: + - Ask the [Solace Community](https://solace.community) - The Solace Developer Portal website at: https://solace.dev - ## Contributing Contributions are encouraged! Please read [CONTRIBUTING](CONTRIBUTING.md) for details on our code of conduct, and the process for submitting pull requests to us. - ## License + See the LICENSE file for details. diff --git a/config.yaml b/config.yaml index 0f0d7a1..fcb4667 100644 --- a/config.yaml +++ b/config.yaml @@ -41,7 +41,7 @@ flows: - type: copy source_expression: input.payload dest_expression: user_data.temp:text - component_input: + input_selection: source_expression: user_data.temp:text - component_name: solace_sw_broker @@ -49,8 +49,6 @@ flows: component_config: <<: *broker_connection payload_format: json - component_input: - source_expression: user_data.output input_transforms: - type: copy source_expression: input.payload @@ -67,5 +65,5 @@ flows: - type: copy source_expression: user_data.temp dest_expression: user_data.output:user_properties - component_input: + input_selection: source_expression: user_data.output diff --git a/docs/components/aggregate.md b/docs/components/aggregate.md index d1bcf87..825c1b3 100644 --- a/docs/components/aggregate.md +++ b/docs/components/aggregate.md @@ -1,6 +1,7 @@ # Aggregate Take multiple messages and aggregate them into one. The output of this component is a list of the exact structure of the input data. +This can be useful for batch processing or for aggregating events together before processing them. The Aggregate component will take a sequence of events and combine them into a single event before enqueuing it to the next component in the flow so that it can perform batch processing. ## Configuration Parameters @@ -37,3 +38,21 @@ component_config: ... ] ``` + + +## Example Configuration + + +```yaml + - component_name: aggretator_example + component_module: aggregate + component_config: + # The maximum number of items to aggregate before sending the data to the next component + max_items: 3 + # The maximum time to wait before sending the data to the next component + max_time_ms: 1000 + input_selection: + # Take the text field from the message and use it as the input to the aggregator + source_expression: input.payload:text +``` + diff --git a/docs/components/error_input.md b/docs/components/error_input.md index c0d8127..de06a88 100644 --- a/docs/components/error_input.md +++ b/docs/components/error_input.md @@ -1,6 +1,6 @@ # ErrorInput -Receive processing errors from the Solace AI Event Connector. Note that the component_input configuration is ignored. This component should be used to create a flow that handles errors from other flows. +Receive processing errors from the Solace AI Event Connector. Note that the input_selection configuration is ignored. This component should be used to create a flow that handles errors from other flows. ## Configuration Parameters diff --git a/docs/components/index.md b/docs/components/index.md index 5d61fbd..5832e4d 100644 --- a/docs/components/index.md +++ b/docs/components/index.md @@ -8,7 +8,7 @@ | [broker_output](broker_output.md) | Connect to a messaging broker and send messages to it. Note that this component requires that the data is transformed into the input schema. | | [broker_request_response](broker_request_response.md) | Connect to a messaging broker, send request messages, and receive responses. This component combines the functionality of broker_input and broker_output with additional request-response handling. | | [delay](delay.md) | A simple component that simply passes the input to the output, but with a configurable delay. | -| [error_input](error_input.md) | Receive processing errors from the Solace AI Event Connector. Note that the component_input configuration is ignored. This component should be used to create a flow that handles errors from other flows. | +| [error_input](error_input.md) | Receive processing errors from the Solace AI Event Connector. Note that the input_selection configuration is ignored. This component should be used to create a flow that handles errors from other flows. | | [file_output](file_output.md) | File output component | | [iterate](iterate.md) | Take a single message that is a list and output each item in that list as a separate message | | [langchain_chat_model](langchain_chat_model.md) | Provide access to all the LangChain chat models via configuration | diff --git a/docs/components/iterate.md b/docs/components/iterate.md index 157943f..53aad80 100644 --- a/docs/components/iterate.md +++ b/docs/components/iterate.md @@ -32,3 +32,16 @@ No configuration parameters } ``` + + +## Example Configuration + + +```yaml + - component_name: iterate_example + component_module: iterate + component_config: + input_selection: + # Take the list field from the message and use it as the input to the iterator + source_expression: input.payload:embeddings +``` diff --git a/docs/configuration.md b/docs/configuration.md index 5e6e5e8..8c2b511 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,16 +1,52 @@ # Configuration for the AI Event Connector +Table of Contents + +- [Configuration for the AI Event Connector](#configuration-for-the-ai-event-connector) + - [Configuration File Format and Rules](#configuration-file-format-and-rules) + - [Special values](#special-values) + - [Configuration File Structure](#configuration-file-structure) + - [Log Configuration](#log-configuration) + - [Trace Configuration](#trace-configuration) + - [Shared Configurations](#shared-configurations) + - [Flow Configuration](#flow-configuration) + - [Message Data](#message-data) + - [Expression Syntax](#expression-syntax) + - [Templates](#templates) + - [Component Configuration](#component-configuration) + - [component\_module](#component_module) + - [component\_config](#component_config) + - [input\_transforms](#input_transforms) + - [input\_selection](#input_selection) + - [queue\_depth](#queue_depth) + - [num\_instances](#num_instances) + - [Built-in components](#built-in-components) + - [Invoke Keyword](#invoke-keyword) + - [Invoke with custom function](#invoke-with-custom-function) + - [invoke\_functions](#invoke_functions) + - [evaluate\_expression()](#evaluate_expression) + - [user\_processor Component and invoke](#user_processor-component-and-invoke) + - [Usecase Examples](#usecase-examples) + The AI Event Connector is highly configurable. You can define the components of each flow, the queue depths between them, and the number of instances of each component. The configuration is done through a YAML file that is loaded when the connector starts. This allows you to easily change the configuration without having to modify the code. ## Configuration File Format and Rules -The configuration file is a YAML file that is loaded when the connector starts. +The configuration file is a YAML file that is loaded when the connector starts. Multiple YAML files can be passed to the connector at startup. The files will be merged, the latest file will overwrite the previous duplicate keys. Arrays will be concatenated. Useful to separate flows. + +For example, if you have two files: + +```bash +python3 -m solace_ai_connector.main config1.yaml config2.yaml +``` + +Since this application usings `pyyaml`, it is possible to use the `!include` directive to include the template from a file. This can be useful for very large templates or for templates that are shared across multiple components. ### Special values Within the configuration, you can have simple static values, environment variables, or dynamic values using the `invoke` keyword. -#### Environment Variables +- ***Environment Variables*** You can use environment variables in the configuration file by using the `${}` syntax. For example, if you have an environment variable `MY_VAR` you can use it in the configuration file like this: @@ -18,193 +54,31 @@ You can use environment variables in the configuration file by using the `${}` s my_key: ${MY_VAR} ``` -#### Dynamic Values (invoke keyword) +- ***Dynamic Values (invoke keyword)*** You can use dynamic values in the configuration file by using the `invoke` keyword. This allows you to do such things as import a module, instantiate a class and call a function to get the value. For example, if you want to get the operating system type you can use it in the configuration file like this: ```yaml -os_type: +os_type: invoke: module: platform function: system ``` -An `invoke` block works by specifying an 'object' to act on with one (and only one) of the following keys: -- `module`: The name of the module to import in normal Python import syntax (e.g. `os.path`) -- `object`: An object to call a function on or get an attribute from. Note that this must have an `invoke` block itself to create the object. Objects can be nested to build up complex objects. An object is the returned value from a function call or attribute get from a module or a nested object. - -It is also acceptable to specify neither `module` nor `object` if you are calling a function that is in the global namespace. - -In addition to the object specifier, you can specify one (and only one) of the following keys: -- `function`: The name of the function to call on the object -- `attribute`: The name of the attribute to get from the object - -In the case of a function, you can also specify a `params` key to pass parameters to the function. The params value has the following keys: -- `positional`: A list of positional parameters to pass to the function -- `keyword`: A dictionary of keyword parameters to pass to the function - -`invoke` blocks can be nested to build up complex objects and call functions on them. - -Here is an example of a complex `invoke` block that could be used to get AWS credentials: - -```yaml - # Get AWS credentials and give it a name to reference later - - aws_credentials: &aws_credentials - invoke: - object: - invoke: - # import boto3 - module: boto3 - # Get the session object -> boto3.Session() - function: Session - # Call the get_credentials function on the session object -> session.get_credentials() - function: get_credentials - - - aws_4_auth: - invoke: - # import requests_aws4auth - module: requests_aws4auth - # Get the AWS4Auth object -> requests_aws4auth.AWS4Auth() - function: AWS4Auth - params: - positional: - # Access key - - invoke: - object: *aws_credentials - attribute: access_key - # Secret key - - invoke: - object: *aws_credentials - attribute: secret_key - # Region (from environment variable) - - ${AWS_REGION} - # Service name (from environment variable) - - ${AWS_SERVICE} - keyword: - # Pass the session token if it exists -> session_token= - session_token: - invoke: - object: *aws_credentials - attribute: token -``` - -##### invoke_functions - -There is a module named `invoke_functions` that has a list of functions that can take the place of python operators. This is useful for when you want to use an operator in a configuration file. The following functions are available: -- `add`: param1 + param2 - can be used to add or concatenate two strings or lists -- `append`: Append the second value to the first -- `subtract`: Subtract the second number from the first -- `multiply`: Multiply two numbers together -- `divide`: Divide the first number by the second -- `modulus`: Get the modulus of the first number by the second -- `power`: Raise the first number to the power of the second -- `equal`: Check if two values are equal -- `not_equal`: Check if two values are not equal -- `greater_than`: Check if the first value is greater than the second -- `greater_than_or_equal`: Check if the first value is greater than or equal to the second -- `less_than`: Check if the first value is less than the second -- `less_than_or_equal`: Check if the first value is less than or equal to the second -- `and_op`: Check if both values are true -- `or_op`: Check if either value is true -- `not_op`: Check if the value is false -- `in_op`: Check if the first value is in the second value -- `negate`: Negate the value -- `empty_list`: Return an empty list -- `empty_dict`: Return an empty dictionary -- `empty_string`: Return an empty string -- `empty_set`: Return an empty set -- `empty_tuple`: Return an empty tuple -- `empty_float`: Return 0.0 -- `empty_int`: Return 0 -- `if_else`: If the first value is true, return the second value, otherwise return the third value -- `uuid`: returns a universally unique identifier (UUID) - -Use positional parameters to pass values to the functions that expect arguments. -Here is an example of using the `invoke_functions` module to do some simple operations: - -```yaml - # Use the invoke_functions module to do some simple operations - - simple_operations: - invoke: - module: invoke_functions - function: add - params: - positional: - - 1 - - 2 -``` - -##### source_expression() - -If the `invoke` block is used within an area of the configuration that relates to message processing -(e.g. input_transforms), an invoke function call can use the special function `source_expression([, type])` for -any of its parameters. This function will be replaced with the value of the source expression at runtime. -It is an error to use `source_expression()` outside of a message processing. The second parameter is optional -and will convert the result to the specified type. The following types are supported: -- `int` -- `float` -- `bool` -- `str` -If the value is a dict or list, the type request will be ignored - -Example: -```yaml --flows: - -my_flow: - -my_component: - input_transforms: - -type: copy - source_expression: - invoke: - module: invoke_functions - function: add - params: - positional: - - source_expression(input.payload:my_obj.val1, int) - - 2 - dest_expression: user_data.my_obj:result -``` - -In the above example, the `source_expression()` function is used to get the value of `input.payload:my_obj.val1`, -convert it to an `int` and add 2 to it. - -**Note:** In places where the yaml keys `source_expression` and `dest_expressions` are used, you can use the same type of expression to access a value. Check [Expression Syntax](#expression-syntax) for more details. - -##### user_processor component and invoke - -The `user_processor` component is a special component that allows you to define a user-defined function to process the message. This is useful for when you want to do some processing on the input message that is not possible with the built-in transforms or other components. In order to specify the user-defined function, you must define the `component_processing` property with an `invoke` block. - -Here is an example of using the `user_processor` component with an `invoke` block: - -```yaml - - my_user_processor: - component_name: my_user_processor - component_module: user_processor - component_processing: - invoke: - module: my_module - function: my_function - params: - positional: - - source_expression(input.payload:my_key) - - 2 -``` - - - - +More details [here](#invoke-keyword). ## Configuration File Structure The configuration file is a YAML file with these top-level keys: - `log`: Configuration of logging for the connector +- `trace`: Configuration of tracing for the connector - `shared_config`: Named configurations that can be used by multiple components later in the file -- `flows`: A list of flow configurations. +- `flows`: A list of flow configurations. ### Log Configuration -The `log` configuration section is used to configure the logging for the connector. It configures the logging behaviour for stdout and file logs. It has the following keys: +The `log` configuration section is used to configure the logging for the connector. It configures the logging behavior for stdout and file logs. It has the following keys: - `stdout_log_level`: - The log level for the stdout log - `log_file_level`: - The log level for the file log @@ -219,6 +93,15 @@ log: log_file: /var/log/ai_event_connector.log ``` +### Trace Configuration + +The trace option will output logs to a trace log that has all the detail of the message at each point. It gives an output when a message is pulled out of an input queue and another one before invoke is called (i.e. after transforms). + +```yaml +trace: + trace_file: /var/log/ai_event_connector_trace.log +``` + ### Shared Configurations The `shared_config` section is used to define configurations that can be used by multiple components later in the file. It is a dictionary of named configurations. Each named configuration is a dictionary of configuration values. Here is an example of a shared configuration: @@ -240,158 +123,451 @@ Later in the file, you can reference this shared configuration like this: ### Flow Configuration +A flow is an instance of a pipeline that processes events in a sequential manner. Each `flow` is completely independent of the others and can have its own set of components and configurations. + +Flows can be communicating together if programmed to do so. For example, a flow can send a message to a broker and another flow can subscribe to the same topic to receive the message. + +flows can be spread across multiple configuration files. The connector will merge the flows from all the files and run them together. + The `flows` section is a list of flow configurations. Each flow configuration is a dictionary with the following keys: + - `name`: - The unique name of the flow -- `components`: A list of component configurations +- `components`: A list of component configurations. Check [Component Configuration](#component-configuration) for more details + +## Message Data + +Between each component in a flow, a message is passed. This message is a dictionary that is used to pass data between components within the same flow. The message object has different properties, some are available throughout the whole flow, some only between two immediate components, and some have other characteristics. + +The message object has the following properties: + +- `input`: The Solace broker input message. It has the following properties: + - `payload`: The payload of the input message + - `topic`: The topic of the input message + - `topic_levels`: A list of the levels of the topic of the input message + - `user_properties`: The user properties of the input message + +This data type is available only after a topic subscription and then it will be available from that component onwards till overwritten by another input message. + +- `user_data`: The user data object. This is a storage where the user can write and read values to be used at the different places. It is an object that is passed through the flows, and can hold any valid Python data type. To write to this object, you can use the `dest_expression` in the configuration file. To read from this object, you can use the `source_expression` in the configuration file. (This object is also available in the `evaluate_expression()` function). + +- `previous`: The complete output of the previous component in the flow. This can be used to completely forward the output of the previous component as an input to the next component or be modified in the `input_transforms` section of the next component. + +- transform specific variables: Some transforms function will add specific variables to the message object that are ONLY accessible in that transform. For example, the [`map` transform](./transforms/map.md) will add `item`, `index`, and `source_list` to the message object or the [`reduce` transform](./transforms/reduce.md) will add `accumulated_value`, `current_value`, and `source_list` to the message object. You can find these details in each transform documentation. + +## Expression Syntax -#### Component Configuration +The `source_expression` and `dest_expression` values in the configuration file use a simple expression syntax to reference values in the input message and to store values in the output message. The format of the expression is: + +**`[.][:]`** + +Where: + +- `data_type`: - The type of data to reference. This can be one of the [message data type Check](#message-data) or one of the following: + - message data type: input, user_data, previous, etc mentioned in the [Message Data](#message-data) section + - `static`: A static value (e.g. `static:my_value`) + - `template`: A template ([see more below](#templates)) + + +- `qualifier`: - The qualifier to use to reference the data. This is specific to the `data_type` and is optional. If not specified, the entire data type will be used. + +- `index`: - Where to get the data in the data type. This is optional and is specific to the `data_type`. For templates, it is the template. For other data types, it is a dot separated string or an integer index. The index will be split on dots and used to traverse the data type. If it is an integer, it will be used as an index into the data type. If it is a string, it will be used as a key to get the value from the data type. + +Here are some examples of expressions: + +- `input.payload:my_key` - Get the value of `my_key` from the input payload +- `user_data.my_obj:my_key` - Get the value of `my_key` from the `my_obj` object in the user data +- `static:my_value` - Use the static value `my_value` +- `user_data:my_obj2:my_list.2.my_key` - Get the value of `my_key` from the 3rd item in the `my_list` list in the `my_obj2` object in the user data + +When using expressions for destination expressions, lists and objects will be created as needed. If the destination expression is a list index, the list will be extended to the index if it is not long enough. If the destination expression is an object key, the object will be created if it does not exist. + +### Templates + +The `template` data type is a special data type that allows you to use a template to create a value. The template is a string that can contain expressions to reference values in the input message. The format of the template is: + +**`template:text text text {{template_expression}} text text text`** + +Where: + +- `template:` is the template data type indicator. +- `{{template_expression}}` - An expression to reference values in the input message. It has the format: + + **`://`** + + Where: + + - `encoding`: - The encoding/formatting to use to print out the value. This can be one of the following (Optional, defaulted to `text`): + + - `base64`: Use base64 encoding + - `json`: Use json format + - `yaml`: Use yaml format + - `text`: Use string format + - `datauri:`: Use data uri encoding with the specified mime type + + - `source_expression`: - An expression to reference values in the input message. This has the same format as the `source_expression` in the configuration file described above. + +Here is an example of a template: + +```yaml +input_transforms: + - type: copy + source_expression: | + template:Write me a dry joke about: + {{text://input.payload}} + + Write the joke in the voice of {{text://input.user_properties:comedian}} + + dest_expression: user_data.llm_input:messages.0.content + - type: copy + source_value: user + dest_expression: user_data.llm_input:messages.0.role +``` + +In this example, the `source_expression` for the first transform is a template that uses the `text` encoding to create a string. + + +## Component Configuration Each component configuration is a dictionary with the following keys: -- `component_name`: - The unique name of the component within the flow -- `component_module`: - The module that contains the component class (python import syntax) -- `component_config`: - The configuration for the component. Its format is specific to the component -- `input_transforms`: - A list of transforms to apply to the input message before sending it to the component -- `component_input`: - A source_expression or source_value to use as the input to the component. -- `queue_depth`: - The depth of the input queue for the component -- `num_instances`: - The number of instances of the component to run -**Note: For a list of all built-in components, see the [Components](components/index.md) documentation.** +- `component_name`: - The unique name of the component within the flow. +- `component_module`: - The module that contains the component class (python import syntax) or the name of the [built-in component](#built-in-components) +- `component_config`: - The configuration for the component. Its format is specific to the component. [Optional: if the component does not require configuration] +- `input_transforms`: - A list of transforms to apply to the input message before sending it to the component. This is to ensure that the input message is in the correct format for the component. [Optional] +- `input_selection`: - A `source_expression` or `source_value` to use as the input to the component. Check [Expression Syntax](#expression-syntax) for more details. [Optional: If not specified, the complete previous component output will be used] +- `queue_depth`: - The depth of the input queue for the component. +- `num_instances`: - The number of instances of the component to run (Starts multiple threads to process messages) + + +### component_module + +The `component_module` is a string that specifies the module that contains the component class. + +Solace-ai-connector comes with a number of flexible and highly customizable [built-in components](./components/index.md) that should cover a wide range of use cases. To use a built-in component, you can specify the name of the component in the `component_module` key and configure it using the `component_config` key. For example, to use the `aggregate` component, you would specify the following: + +```yaml +- my_component: + component_module: aggregate + component_config: + max_items: 3 + max_time_ms: 1000 +``` + +The `component_module` can also be the python import syntax for the module. When using with a custom component, you can also use `component_base_path` to specify the base path of the python module. + +You're module file should also export a variable named `info` that has the name of the class to instantiate under the key `class_name`. -##### component_config +For example: -The `component_config` is a dictionary of configuration values specific to the component. The format of this dictionary is specific to the component. You must refer to the component's documentation for the specific configuration values. +```python +from solace_ai_connector.components.component_base import ComponentBase -##### input_transforms +info = { + "class_name": "CustomClass", +} + +class CustomClass(ComponentBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + + def invoke(self, _, data): + return data["text"] + " + custom class" +``` + +For example, if the component class is in a module named `my_module` in `src` directory, you can use it in the configuration file like this: + +```yaml + - component_name: custom_module_example + component_base_path: . + component_module: src.my_module +``` + +You can find an example of a custom component in the [tips and tricks](tips_and_tricks.md/#using-custom-modules-with-the-ai-connector) section. + +**Note:** If you are using a custom component, you must ensure that you're using proper relative paths or your paths are in the correct level to as where you're running the connector from. + +### component_config + +The `component_config` is a dictionary of configuration values specific to the component. The format of this dictionary is specific to the component. You must refer to the component's documentation for the specific configuration values. for example, the [`aggregate` component](./components/aggregate.md) has the following configuration: + +```yaml + component_module: aggregate + component_config: + max_items: 3 + max_time_ms: 1000 +``` + +### input_transforms The `input_transforms` is a list of transforms to apply to the input message before sending it to the component. Each transform is a dictionary with the following keys: + - `type`: - The type of transform -- `source_expression|source_value`: - The source expression or value to use as the input to the transform +- `source_expression|source_value`: - The source expression or static value to use as the input to the transform - `dest_expression`: - The destination expression for where to store the transformation output -For a list of all available transform functions check [Transforms](transforms/index.md) page. +The AI Event Connector comes with a number of built-in transforms that can be used to process messages. **For a list of all built-in transforms, see the [Transforms](transforms/index.md) documentation.** Here is an example of a component configuration with input transforms: ```yaml - - my_component: - component_module: my_module.my_component - component_config: - my_key: my_value - input_transforms: - - type: copy - # Extract the my_key value from the input payload - source_expression: input.payload:my_key - # Store the value in the newly created my_obj object in the my_keys list - # at index 2 (i.e. my_obj.my_keys[2].my_key = input.payload.my_key) - dest_expression: user_data.my_obj:my_keys.2.my_key +- my_component: + component_module: my_module.my_component + component_config: + my_key: my_value + input_transforms: + - type: copy + # Extract the my_key value from the input payload + source_expression: input.payload:my_key + # Store the value in the newly created my_obj object in the my_keys list + # at index 2 (i.e. my_obj.my_keys[2].my_key = input.payload.my_key) + dest_expression: user_data.my_obj:my_keys.2.my_key + - type: copy + # Use a static value + source_value: my_static_value + # Store the value in the newly created my_obj object in the my_keys list + # at index 3 (i.e. my_obj.my_keys[3].my_key = my_static_value) + dest_expression: user_data.my_obj:my_keys.3.my_key ``` -###### Built-in Transforms -The AI Event Connector comes with a number of built-in transforms that can be used to process messages. For a list of all built-in transforms, see the [Transforms](transforms/index.md) documentation. +### input_selection -##### component_input +The `input_selection` is a dictionary with one (and only one) of the following keys: -The `component_input` is a dictionary with one (and only one) of the following keys: - `source_expression`: - An expression to use as the input to the component (see below for expression syntax) -- `source_value`: - A value to use as the input to the component. +- `source_value`: - A static value to use as the input to the component. -Note that, as for all values in the config file, you can use the `invoke` keyword to get dynamic values +Note that, as for all values in the config file, you can use the [`invoke`](#invoke-keyword) keyword to get dynamic values Here is an example of a component configuration with a source expression: ```yaml - - my_component: - component_module: my_module.my_component - component_config: - my_key: my_value - component_input: - source_expression: input.payload:my_key +- my_component: + component_module: my_module.my_component + component_config: + my_key: my_value + input_selection: + source_expression: input.payload:my_key ``` -##### queue_depth +### queue_depth The `queue_depth` is an integer that specifies the depth of the input queue for the component. This is the number of messages that can be buffered in the queue before the component will start to block. By default, the queue depth is 100. - -##### num_instances +### num_instances The `num_instances` is an integer that specifies the number of instances of the component to run. This is the number of threads that will be started to process messages from the input queue. By default, the number of instances is 1. -#### Built-in components +### Built-in components The AI Event Connector comes with a number of built-in components that can be used to process messages. For a list of all built-in components, see the [Components](components/index.md) documentation. -### Expression Syntax +## Invoke Keyword -The `source_expression` and `dest_expression` values in the configuration file use a simple expression syntax to reference values in the input message and to store values in the output message. The format of the expression is: +The `invoke` keyword is used to get dynamic values in the configuration file. An `invoke` block works by specifying an 'object' to act on with one (and only one) of the following keys: -`[.][:]` +- `module`: The name of the module to import in normal Python import syntax (e.g. `os.path`) +- `object`: An object to call a function on or get an attribute from. Note that this must have an `invoke` block itself to create the object. Objects can be nested to build up complex objects. An object is the returned value from a function call or get attribute from a module or a nested object. -Where: +It is also acceptable to specify neither `module` nor `object` if you are calling a function that is in the global namespace. -- `data_type`: - The type of data to reference. This can be one of the following: - - `input`: The input message. It supports the qualifiers: - - `payload`: The payload of the input message - - `topic`: The topic of the input message - - `topic_levels`: A list of the levels of the topic of the input message - - `user_properties`: The user properties of the input message - - `user_data`: The user data object. The qualifier is required to specify the name of the user data object. `user_data` is an object that is passed through the flows, where the user can read and write values to it to be accessed at the different places. - - `static`: A static value (e.g. `static:my_value`) - - `template`: A template ([see more below](#templates)) - - `previous`: The output from the previous component in the flow. This could be of any type depending on the previous component +In addition to the object specifier, you can specify one (and only one) of the following keys: -- `qualifier`: - The qualifier to use to reference the data. This is specific to the `data_type` and is optional. If not specified, the entire data type will be used. +- `function`: The name of the function to call on the object +- `attribute`: The name of the attribute to get from the object -- `index`: - Where to get the data in the data type. This is optional and is specific to the `data_type`. For templates, it is the template. For other data types, it is a dot separated string or an integer index. The index will be split on dots and used to traverse the data type. If it is an integer, it will be used as an index into the data type. If it is a string, it will be used as a key to get the value from the data type. +In the case of a function, you can also specify a `params` key to pass parameters to the function. The params value has the following keys: -Here are some examples of expressions: +- `positional`: A list of positional parameters to pass to the function +- `keyword`: A dictionary of keyword parameters to pass to the function -- `input.payload:my_key` - Get the value of `my_key` from the input payload -- `user_data.my_obj:my_key` - Get the value of `my_key` from the `my_obj` object in the user data -- `static:my_value` - Use the static value `my_value` -- `user_data:my_obj2:my_list.2.my_key` - Get the value of `my_key` from the 3rd item in the `my_list` list in the `my_obj2` object in the user data +`invoke` blocks can be nested to build up complex objects and call functions on them. -When using expressions for destination expressions, lists and objects will be created as needed. If the destination expression is a list index, the list will be extended to the index if it is not long enough. If the destination expression is an object key, the object will be created if it does not exist. +Here is an example of a complex `invoke` block that could be used to get AWS credentials: -#### Templates +```yaml +# Get AWS credentials and give it a name to reference later +- aws_credentials: &aws_credentials + invoke: + object: + invoke: + # import boto3 + module: boto3 + # Get the session object -> boto3.Session() + function: Session + # Passing a parameter to the Session function + params: + keyword: + # Using a keyword parameter + profile_name: default + # Call the get_credentials function on the session object -> session.get_credentials() + function: get_credentials + +- aws_4_auth: + invoke: + # import requests_aws4auth + module: requests_aws4auth + # Get the AWS4Auth object -> requests_aws4auth.AWS4Auth() + function: AWS4Auth + params: + positional: + # Access key + - invoke: + object: *aws_credentials + attribute: access_key + # Secret key + - invoke: + object: *aws_credentials + attribute: secret_key + # Region (from environment variable) + - ${AWS_REGION} + # Service name (from environment variable) + - ${AWS_SERVICE} + keyword: + # Pass the session token if it exists -> session_token= + session_token: + invoke: + object: *aws_credentials + attribute: token +``` -The `template` data type is a special data type that allows you to use a template to create a value. The template is a string that can contain expressions to reference values in the input message. The format of the template is: +**Note:** The function parameters do not support expression syntax outside of the `evaluate_expression()` function. If you need to use an expression like template, you'd have to write it to a temporary user data value and reference it in the `source_expression` function. -`template:text text text {{template_expression}} text text text` +### Invoke with custom function -Where: +You can use invoke with your own custom functions. When using a custom functions, you can use the `path` to specify the base path of the python module. -- `template:` is the template data type indicator. -- `{{template_expression}}` - An expression to reference values in the input message. It has the format: +For example, if you have a custom function in a module named `my_module` in `src` directory and the function is named `my_function`, you can use it in the configuration file like this: - `://` +```yaml +- my_custom_function: + invoke: + path: . + module: src.my_module + function: my_function + params: + positional: + - 1 + - 2 +``` - Where: - - `encoding`: - The encoding/formatting to use to print out the value. This can be one of the following (Optional, defaulted to `text`): - - `base64`: Use base64 encoding - - `json`: Use json format - - `yaml`: Use yaml format - - `text`: Use string format - - `datauri:`: Use data uri encoding with the specified mime type +### invoke_functions - - `source_expression`: - An expression to reference values in the input message. This has the same format as the `source_expression` in the configuration file described above. +There is a module named `invoke_functions` that has a list of functions that can take the place of python operators used inside of `invoke`. This is useful for when you want to use an operator in a configuration file. -Here is an example of a template: +The following functions are available: + +- `add`: param1 + param2 - can be used to add or concatenate two strings or lists +- `append`: Append the second value to the first +- `subtract`: Subtract the second number from the first +- `multiply`: Multiply two numbers together +- `divide`: Divide the first number by the second +- `modulus`: Get the modulus of the first number by the second +- `power`: Raise the first number to the power of the second +- `equal`: Check if two values are equal +- `not_equal`: Check if two values are not equal +- `greater_than`: Check if the first value is greater than the second +- `greater_than_or_equal`: Check if the first value is greater than or equal to the second +- `less_than`: Check if the first value is less than the second +- `less_than_or_equal`: Check if the first value is less than or equal to the second +- `and_op`: Check if both values are true +- `or_op`: Check if either value is true +- `not_op`: Check if the value is false +- `in_op`: Check if the first value is in the second value +- `negate`: Negate the value +- `empty_list`: Return an empty list +- `empty_dict`: Return an empty dictionary +- `empty_string`: Return an empty string +- `empty_set`: Return an empty set +- `empty_tuple`: Return an empty tuple +- `empty_float`: Return 0.0 +- `empty_int`: Return 0 +- `if_else`: If the first value is true, return the second value, otherwise return the third value +- `uuid`: returns a universally unique identifier (UUID) + +Use positional parameters to pass values to the functions that expect arguments. + +Here is an example of using the `invoke_functions` module to do some simple operations: ```yaml - input_transforms: - - type: copy - source_expression: | - template:Write me a dry joke about: - {{text://input.payload}} - Write the joke in the voice of {{text://input.user_properties:comedian}} - dest_expression: user_data.llm_input:messages.0.content - - type: copy - source_value: user - dest_expression: user_data.llm_input:messages.0.role +# Use the invoke_functions module to do some simple operations +- simple_operations: + invoke: + module: invoke_functions + function: add + params: + positional: + - 1 + - 2 ``` -In this example, the `source_expression` for the first transform is a template that uses the `text` encoding to create a string. +### evaluate_expression() + +If the `invoke` block is used within an area of the configuration that relates to message processing +(e.g. input_transforms), an invoke function call can use the special function `evaluate_expression([, type])` for any of its parameters. This function will be replaced with the value of the source expression at runtime. + +It is an error to use `evaluate_expression()` outside of a message processing. The second parameter is optional +and will convert the result to the specified type. The following types are supported: + +- `int` +- `float` +- `bool` +- `str` + +If the value is a dict or list, the type request will be ignored + +Example: + +```yaml +- flows: + - my_flow: + - my_component: + input_transforms: + -type: copy + source_expression: + invoke: + module: invoke_functions + function: add + params: + positional: + - evaluate_expression(input.payload:my_obj.val1, int) + - 2 + dest_expression: user_data.my_obj:result +``` + +In the above example, the `evaluate_expression()` function is used to get the value of `input.payload:my_obj.val1`, +convert it to an `int` and add 2 to it. + +**Note:** In places where the yaml keys `source_expression` and `dest_expressions` are used, you can use the same type of expression to access a value. Check [Expression Syntax](#expression-syntax) for more details. + +### user_processor Component and invoke + +The `user_processor` component is a special component that allows you to define a user-defined function to process the message. This is useful for when you want to do some processing on the input message that is not possible with the built-in transforms or other components. In order to specify the user-defined function, you must define the `component_processing` property with an `invoke` block. + +Here is an example of using the `user_processor` component with an `invoke` block: + +```yaml +- my_user_processor: + component_name: my_user_processor + component_module: user_processor + component_processing: + invoke: + module: my_module + function: my_function + params: + positional: + - evaluate_expression(input.payload:my_key) + - 2 +``` + +## Usecase Examples + +You can find various usecase examples in the [examples directory](../examples/) + + + +--- + +Checkout [components.md](./components/index.md), [transforms.md](./transforms/index.md), or [tips_and_tricks](tips_and_tricks.md) next. diff --git a/docs/getting_started.md b/docs/getting_started.md index f2f7306..14b8845 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -75,7 +75,7 @@ Download the OpenAI connector example configuration file: curl https://raw.githubusercontent.com/SolaceLabs/solace-ai-connector/main/examples/llm/openai_chat.yaml > openai_chat.yaml ``` -For this one, you need to also define the following environment variables: +For this one, you need to also define the following additional environment variables: ```sh export OPENAI_API_KEY= @@ -174,4 +174,6 @@ To build a Docker image, run the following command: make build ``` -Please now visit the [Documentation Page](index.md) for more information +--- + +Checkout [configuration.md](configuration.md) or [overview.md](overview.md) next \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 85baa9e..6450165 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,11 +7,9 @@ This connector application makes it easy to connect your AI/ML models to Solace - [Overview](overview.md) - [Getting Started](getting_started.md) - [Configuration](configuration.md) -- [Usage](usage.md) -- [Components](components/index.md) -- [Transforms](transforms/index.md) + - [Components](components/index.md) + - [Transforms](transforms/index.md) +- [Tips and Tricks](tips_and_tricks.md) +- [Examples](../examples/) - [Contributing](../CONTRIBUTING.md) - [License](../LICENSE) - - - diff --git a/docs/overview.md b/docs/overview.md index b8e8efb..67540f8 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -1,5 +1,14 @@ # AI Event Connector for Solace event brokers - Overview +- [AI Event Connector for Solace event brokers - Overview](#ai-event-connector-for-solace-event-brokers---overview) + - [Architecture](#architecture) + - [Components](#components) + - [Built-in Components](#built-in-components) + - [Configuration](#configuration) + - [Extensibility](#extensibility) + - [Resilience and Fault Tolerance](#resilience-and-fault-tolerance) + - [Scalability](#scalability) + The AI Event Connector makes it easy to connect your AI/ML models to Solace event brokers. It provides a simple way to build pipelines that consume events from Solace, process them with your models, and then publish the results back to Solace. By doing this, you can greatly enhance the value of your event-driven architecture by adding AI capabilities to your event-driven applications. This connector is built using Python and the Solace Python API. It also heavily leverages LangChain, a Python library for abstracting the interface to many AI models. This allows you to easily swap out different AI models and model providers without having to change your pipeline. @@ -25,25 +34,20 @@ As shown in the flow diagram above, each flow is comprised of a sequence of comp ![Component](images/parts_of_a_component.png) The component is made up of the following parts: - - **Input Queue**: This is the queue that the component reads from. It is where the events are buffered as they flow through the system. Note that if there are multiple instances of the same component, they will all read from the same queue. - - - **Input Transforms**: This is an optional step that allows you to transform the event before it is processed. This can be useful for normalizing the data or for adding additional context to the event. - - - **Input Selection**: This selects what data should be processed by the component. The data selected should conform to the input schema of the component. It is normal to use Input Transforms and Input Selection together to ensure that the data is in the correct format for the component. If the Input Selection configuration is omitted, the component will select "previous" as the default, which will take the exact output of the previous component in the flow as the input to this component. - - **Processing**: This is where the actual processing of the event happens, such as where the AI model would be called to process the event. This is the only required part of the component. +- **Input Queue**: This is a python built queue that the component reads from. It is where the events are buffered as they flow through the system. Note that if there are multiple instances of the same component, they will all read from the same queue. - After these steps, the component will write the result to the next component's queue. The data written should conform to the output schema of the component. Some components are output components and will send the data to a Solace broker or other data sink. +- **Input Transforms**: This is an optional step that allows you to transform the event before it is processed. This can be useful for normalizing the data or for adding additional context to the event. In the yaml config file this is indicated by the **`input_transforms`** key. +- **Input Selection**: This selects what data should be processed by the component. The data selected should conform to the input schema of the component. It is normal to use Input Transforms and Input Selection together to ensure that the data is in the correct format for the component. If the Input Selection configuration is omitted, the component will select **"previous" as the default**, which will take the exact output of the previous component in the flow as the input to this component. In the yaml config file this is indicated by the **`input_selection`** key. -### Iterate and Aggregate Components +- **Processing**: This is where the actual processing of the event happens, such as where the AI model would be called to process the event. This is the only required part of the component. In the yaml config file this is controlled by the **`component_module`** and **`component_config`** keys. -In addition to the standard components, there are two special components that can be used to iterate over a list of events and to aggregate a list of events. These components can be used to process multiple events in a single component. This can be useful for batch processing or for aggregating events together before processing them. +After these steps, the component will write the result to the next component's queue. The data written should conform to the output schema of the component. Some components are output components and will send the data to a Solace broker or other data sink. -The Iterate component will take a list of events and enqueue each one individually to the next component. The Aggregate component will take a sequence of events and combine them into a single event before enqueuing it to the next component in the flow so that it can perform batch processing. - -Example usage of the Iterate and Aggregate components can be found in the [Usage](usage.md) section. +### Built-in Components +In addition to the standard components, there are a series of other built-in components that can be used to help process events. You can find a list of all built-in components in the [Components](components/index.md) section. ## Configuration @@ -63,3 +67,7 @@ The AI Event Connector is designed to be resilient and fault-tolerant. It uses q The AI Event Connector is designed to be scalable. You can increase the number of instances of a component to handle more load. This allows you to scale your pipelines to handle more events and process them faster. Additionally, you can run multiple flows or even multiple connectors that connect to the same broker queue to handle more events in parallel. Note that for Solace broker queues, they must be configured to be non-exclusive to have multiple flows receive messages. + +--- + +Checkout [Getting Started](getting_started.md) next. diff --git a/docs/tips_and_tricks.md b/docs/tips_and_tricks.md new file mode 100644 index 0000000..221ae1d --- /dev/null +++ b/docs/tips_and_tricks.md @@ -0,0 +1,153 @@ +# Some tips and tricks for using the Solace AI Connector + +- [Some tips and tricks for using the Solace AI Connector](#some-tips-and-tricks-for-using-the-solace-ai-connector) + - [Using `user_data` as temporary storage](#using-user_data-as-temporary-storage) + - [Using custom modules with the AI Connector](#using-custom-modules-with-the-ai-connector) + + +## Using `user_data` as temporary storage + +Some times you might need to chain multiple transforms together, but transforms do not support nesting. For example if you'd want to `map` through a list of strings first and then reduce them to a single string, you can write your transforms sequentially and write to a temporary place in `user_data` like `user_data.temp`: + +For example: + +```yaml + input_transforms: + # Transform each response to use the template + - type: map + source_list_expression: previous + source_expression: | + template: + {{text://item:content}} + \n + dest_list_expression: user_data.temp:responses # Temporary storage + + # Transform and reduce the responses to one message + - type: reduce + source_list_expression: user_data.temp:responses # Using the value in temporary storage + source_expression: item + initial_value: "" + accumulator_function: + invoke: + module: invoke_functions + function: add + params: + positional: + - evaluate_expression(keyword_args:accumulated_value) + - evaluate_expression(keyword_args:current_value) + dest_expression: user_data.output:responses # Value to be used in the component + + input_selection: + source_expression: user_data.output +``` + +## Using custom modules with the AI Connector + +This is a simple example that utilizes a custom component, a class based transform and a function based transform. + +First follow the following steps to create a repository to run the ai connector: + +```bash +mkdir -p module-example/src +cd module-example +python3 -m venv env +source env/bin/activate +pip install solace-ai-connector +touch config.yaml src/custom_component.py src/custom_function.py src/__init__.py +``` + +Write the following code to `src/custom_function.py`: + +```python +def custom_function(input_data): + return input_data + " + custom function value" + +class CustomFunctionClass: + def get_custom_value(self, input_data): + return input_data + " + custom function class" +``` + +Write the following code to `src/custom_component.py`: + +```python +from solace_ai_connector.components.component_base import ComponentBase + +info = { + "class_name": "CustomClass", +} + +class CustomClass(ComponentBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + + def invoke(self, _, data): + return data["text"] + " + custom class" + +``` + +Write the following config to `config.yaml`: + +```yaml +log: + stdout_log_level: INFO + log_file_level: INFO + log_file: solace_ai_connector.log + +flows: + - name: custom_module_flow + components: + # Input from a standard in + - component_name: stdin + component_module: stdin_input + + # Using Custom component + - component_name: custom_component_example + component_base_path: . + component_module: src.custom_component + input_selection: + source_expression: previous + + # Output to a standard out + - component_name: stdout + component_module: stdout_output + # Using custom transforms + input_transforms: + # Instantiating a class and calling its function + - type: copy + source_expression: + invoke: + # Creating an object of the class + object: + invoke: + path: . + module: src.custom_function + function: CustomFunctionClass + # Calling the function of the class + function: get_custom_value + params: + positional: + - source_expression(previous) + dest_expression: user_data.output:class + # Calling a function directly + - type: copy + source_expression: + invoke: + module: src.custom_function + function: custom_function + params: + positional: + - source_expression(previous) + dest_expression: user_data.output:function + component_input: + source_expression: user_data.output +``` + +Then run the AI connector with the following command: + +```bash +solace-ai-connector config.yaml +``` + +--- + +Find more examples in the [examples](../examples/) directory. diff --git a/docs/transforms/filter.md b/docs/transforms/filter.md index 310bba9..bd88f06 100644 --- a/docs/transforms/filter.md +++ b/docs/transforms/filter.md @@ -8,7 +8,7 @@ In the filter function, you have access to the following keyword arguments: * current_value: The value of the current item in the source list * source_list: The source list -These should be accessed using `source_expression(keyword_args:)`. For example, `source_expression(keyword_args:current_value)`. See the example below for more detail. +These should be accessed using `evaluate_expression(keyword_args:)`. For example, `evaluate_expression(keyword_args:current_value)`. See the example below for more detail. ## Configuration Parameters @@ -46,7 +46,7 @@ input_transforms: function: greater_than params: positional: - - source_expression(keyword_args:current_value.my_val) + - evaluate_expression(keyword_args:current_value.my_val) - 2 dest_expression: user_data.output:new_list ``` diff --git a/docs/transforms/map.md b/docs/transforms/map.md index f6f56b8..90d0d41 100644 --- a/docs/transforms/map.md +++ b/docs/transforms/map.md @@ -6,7 +6,7 @@ This is a map transform where a list is iterated over. For each item, it is poss * current_value: The value of the current item in the source list * source_list: The source list -These should be accessed using `source_expression(keyword_args:)`. For example, `source_expression(keyword_args:current_value)`. See the example below for more detail. +These should be accessed using `evaluate_expression(keyword_args:)`. For example, `evaluate_expression(keyword_args:current_value)`. See the example below for more detail. ## Configuration Parameters @@ -44,9 +44,9 @@ input_transforms: function: add params: positional: - - source_expression(keyword_args:current_value) + - evaluate_expression(keyword_args:current_value) - 2 - dest_expression: user_data.output:new_list + dest_list_expression: user_data.output:new_list ``` This transform would take a payload like this: diff --git a/docs/transforms/reduce.md b/docs/transforms/reduce.md index e23dc7f..5401308 100644 --- a/docs/transforms/reduce.md +++ b/docs/transforms/reduce.md @@ -9,7 +9,7 @@ In the accumulator function, you have access to the following keyword arguments: * current_value: The value of the current item in the source list * source_list: The source list -These should be accessed using `source_expression(keyword_args:)`. For example, `source_expression(keyword_args:current_value)`. See the example below for more detail. +These should be accessed using `evaluate_expression(keyword_args:)`. For example, `evaluate_expression(keyword_args:current_value)`. See the example below for more detail. ## Configuration Parameters @@ -48,8 +48,8 @@ input_transforms: function: add params: positional: - - source_expression(keyword_args:accumulated_value) - - source_expression(keyword_args:current_value) + - evaluate_expression(keyword_args:accumulated_value) + - evaluate_expression(keyword_args:current_value) dest_expression: user_data.output:my_obj.sum ``` This transform would take a payload like this: diff --git a/docs/usage.md b/docs/usage.md deleted file mode 100644 index 7abe92f..0000000 --- a/docs/usage.md +++ /dev/null @@ -1,85 +0,0 @@ -## Selecting Data - -Within the configuration, it is necessary to select data for processing. For example, this happens -in the `component_input` section of the configuration or for the source data for input transforms. -The selection of data uses a simple expression language that allows you to select data from the -input message. - -The details of the expression language can be found in the [Configuration](configuration.md#expression-syntax) page in the Expression Syntax section. The expression language allows for the detailed selection of data from the input message or for the -creation of new data. It even supports filling a template with data from the input message as described in detail in the next section. - -### Selecting Data by Filling Templates - -As part of the data selection expressions, it is possible to provide a full template string that will be filled with data from the input message. This is very useful for components that are interacting with Large Language Models (LLMs) or other AI models that typically take large amounts of text with some additional metadata sprinkled in. - -Here is an example configuration that uses a template to provide data for a component: - -```yaml - - component_name: template_example - component_module: some_llm_component - component_input: - # Take the text field from the message and use it as the input to the component - source_expression: | - template:You are a helpful assistant who is an expert in animal husbandry. I would like you to answer this - question by using the information following the question. Make sure to include the links to the data sources - that you used to answer the question. - - Question: {{input.payload:question}} - - Context: {{user_data.vector_store_results:results}} - -``` - -In this example, a previous component did a vector store lookup on the question to get some context data. -Those results in addition to the original question are used to fill in the template for the LLM component. - -Since this application usings `pyyaml`, it is possible to use the `!include` directive to include the template from -a file. This can be useful for very large templates or for templates that are shared across multiple components. - -## Built-in Components - -### Aggregating Messages - -The AI Event Connector has a special component called the `Aggregate` component that can be used to combine multiple events into a single event. This can be useful for batch processing or for aggregating events together before processing them. The `Aggregate` component will take a sequence of events and combine them into a single event before enqueuing it to the next component in the flow so that it can perform batch processing. - -The `Aggregate` component has the following configuration options: - - max_items: The maximum number of items to aggregate before sending the data to the next component - - max_time_ms: The maximum time to wait (in milliseconds) before sending the data to the next component - - -Example Configuration: - -```yaml - - component_name: aggretator_example - component_module: aggregate - component_config: - # The maximum number of items to aggregate before sending the data to the next component - max_items: 3 - # The maximum time to wait before sending the data to the next component - max_time_ms: 1000 - component_input: - # Take the text field from the message and use it as the input to the aggregator - source_expression: input.payload:text -``` - - -### Iterating Over Messages - -The AI Event Connector has a special component called the `Iterate` component that can be used to iterate over a list within one message to create many messages for the next component. - -There is no specific configuration for the Iterate component other than the normal component_input configuration. That source must select a list of items to iterate over. - -Example Configuration: - -```yaml - - component_name: iterate_example - component_module: iterate - component_config: - component_input: - # Take the list field from the message and use it as the input to the iterator - source_expression: input.payload:embeddings -``` - -**Note: For a list of all built-in components, see the [Components](components/index.md) documentation.** - -In addition to these, you also can create your own custom components. \ No newline at end of file diff --git a/examples/ack_test.yaml b/examples/ack_test.yaml index 278f9f0..08314aa 100644 --- a/examples/ack_test.yaml +++ b/examples/ack_test.yaml @@ -47,7 +47,7 @@ flows: - type: copy source_expression: input.payload dest_expression: user_data.temp:text - component_input: + input_selection: source_expression: user_data.temp:text - component_name: solace_sw_broker @@ -55,8 +55,6 @@ flows: component_config: <<: *broker_connection payload_format: json - component_input: - source_expression: user_data.output input_transforms: - type: copy source_expression: input.payload @@ -73,5 +71,5 @@ flows: - type: copy source_expression: user_data.temp dest_expression: user_data.output:user_properties - component_input: + input_selection: source_expression: user_data.output diff --git a/examples/anthropic_bedrock.yaml b/examples/anthropic_bedrock.yaml index 1f3c60a..7c35bb6 100644 --- a/examples/anthropic_bedrock.yaml +++ b/examples/anthropic_bedrock.yaml @@ -75,7 +75,7 @@ flows: - type: copy source_expression: static:user dest_expression: user_data.llm_input:messages.0.role - component_input: + input_selection: source_expression: user_data.llm_input - component_name: solace_sw_broker @@ -96,5 +96,5 @@ flows: - type: copy source_expression: template:response/{{text://input.topic}} dest_expression: user_data.output:topic - component_input: + input_selection: source_expression: user_data.output diff --git a/examples/assembly_inputs.yaml b/examples/assembly_inputs.yaml index 5a4feb4..b215cd6 100644 --- a/examples/assembly_inputs.yaml +++ b/examples/assembly_inputs.yaml @@ -52,7 +52,7 @@ flows: assemble_key: id max_items: 3 max_time_ms: 10000 - component_input: + input_selection: source_expression: input.payload # Send assembled messages back to broker @@ -70,5 +70,5 @@ flows: - type: copy source_expression: template:{{text://input.topic}}/assembled dest_expression: user_data.output:topic - component_input: + input_selection: source_expression: user_data.output diff --git a/examples/chat_model_with_history.yaml b/examples/chat_model_with_history.yaml index c889ea5..820d232 100644 --- a/examples/chat_model_with_history.yaml +++ b/examples/chat_model_with_history.yaml @@ -50,7 +50,7 @@ flows: - type: copy source_value: user dest_expression: user_data.temp:messages.1.role - component_input: + input_selection: source_expression: user_data.temp - component_name: stdout diff --git a/examples/error_handler.yaml b/examples/error_handler.yaml index 416d1bc..a8c700e 100644 --- a/examples/error_handler.yaml +++ b/examples/error_handler.yaml @@ -51,7 +51,7 @@ flows: - type: copy source_value: error_log.log dest_expression: user_data.log:file_path - component_input: + input_selection: source_expression: user_data.log - component_name: solace_sw_broker component_module: broker_output @@ -68,7 +68,7 @@ flows: - type: copy source_expression: input.user_properties dest_expression: user_data.output:user_properties - component_input: + input_selection: source_expression: user_data.output @@ -95,7 +95,7 @@ flows: - type: copy source_expression: input.payload dest_expression: user_data.temp:text - component_input: + input_selection: source_expression: user_data.temp:text - component_name: solace_sw_broker @@ -103,8 +103,6 @@ flows: component_config: <<: *broker_connection payload_format: json - component_input: - source_expression: user_data.output input_transforms: - type: copy source_expression: input.payload @@ -116,7 +114,7 @@ flows: function: power params: positional: - - source_expression(input.payload:value) # This will throw an error if value is not a number + - evaluate_expression(input.payload:value) # This will throw an error if value is not a number - 2 dest_expression: user_data.output:payload.valueSquared - type: copy @@ -131,5 +129,5 @@ flows: - type: copy source_expression: user_data.temp dest_expression: user_data.output:user_properties - component_input: + input_selection: source_expression: user_data.output diff --git a/examples/llm/anthropic_chat.yaml b/examples/llm/anthropic_chat.yaml index 5bc1c16..76fe7ef 100644 --- a/examples/llm/anthropic_chat.yaml +++ b/examples/llm/anthropic_chat.yaml @@ -77,7 +77,7 @@ flows: - type: copy source_expression: static:user dest_expression: user_data.llm_input:messages.0.role - component_input: + input_selection: source_expression: user_data.llm_input # Send response back to broker @@ -95,5 +95,5 @@ flows: - type: copy source_expression: template:{{text://input.topic}}/response dest_expression: user_data.output:topic - component_input: + input_selection: source_expression: user_data.output diff --git a/examples/llm/bedrock_anthropic_chat.yaml b/examples/llm/bedrock_anthropic_chat.yaml index 879bb03..421ce42 100644 --- a/examples/llm/bedrock_anthropic_chat.yaml +++ b/examples/llm/bedrock_anthropic_chat.yaml @@ -59,7 +59,7 @@ flows: - type: copy source_expression: static:user dest_expression: user_data.llm_input:messages.0.role - component_input: + input_selection: source_expression: user_data.llm_input # diff --git a/examples/llm/langchain_openai_with_history_chat.yaml b/examples/llm/langchain_openai_with_history_chat.yaml index 639567f..bd922d3 100644 --- a/examples/llm/langchain_openai_with_history_chat.yaml +++ b/examples/llm/langchain_openai_with_history_chat.yaml @@ -76,7 +76,7 @@ flows: - type: copy source_value: user dest_expression: user_data.input:messages.0.role - component_input: + input_selection: source_expression: user_data.input # Send response back to broker @@ -94,5 +94,5 @@ flows: - type: copy source_expression: template:{{text://input.topic}}/response dest_expression: user_data.output:topic - component_input: + input_selection: source_expression: user_data.output diff --git a/examples/llm/mixture_of_agents.yaml b/examples/llm/mixture_of_agents.yaml index 27af1f6..d71f46c 100644 --- a/examples/llm/mixture_of_agents.yaml +++ b/examples/llm/mixture_of_agents.yaml @@ -83,7 +83,7 @@ shared_config: - type: copy source_expression: template:{{text://input.topic}}/next dest_expression: user_data.output:topic - component_input: + input_selection: source_expression: user_data.output # Agent input transformations @@ -99,7 +99,7 @@ shared_config: - type: copy source_expression: static:user dest_expression: user_data.llm_input:messages.0.role - component_input: + input_selection: source_expression: user_data.llm_input flows: @@ -141,7 +141,7 @@ flows: function: if_else params: positional: - - source_expression(input.payload:layer_number) + - evaluate_expression(input.payload:layer_number) - invoke: # Add 1 to the layer number module: invoke_functions @@ -154,7 +154,7 @@ flows: function: or_op params: positional: - - source_expression(input.payload:layer_number) + - evaluate_expression(input.payload:layer_number) - 0 - 1 # No layer number, set to 1 @@ -169,8 +169,8 @@ flows: function: if_else params: positional: - - source_expression(input.payload:id) - - source_expression(input.payload:id) + - evaluate_expression(input.payload:id) + - evaluate_expression(input.payload:id) - invoke: module: invoke_functions function: uuid @@ -181,7 +181,7 @@ flows: source_value: moa/broadcast dest_expression: user_data.output:topic - component_input: + input_selection: source_expression: user_data.output # Agent 1 - Google Vertex AI @@ -213,7 +213,7 @@ flows: - type: copy source_expression: previous dest_expression: user_data.formatted_response:content - component_input: + input_selection: source_expression: user_data.formatted_response # Broker output for Vertex AI @@ -245,7 +245,7 @@ flows: - type: copy source_expression: previous:content dest_expression: user_data.formatted_response:content - component_input: + input_selection: source_expression: user_data.formatted_response # Broker output for OpenAI @@ -280,7 +280,7 @@ flows: - type: copy source_expression: previous dest_expression: user_data.formatted_response:content - component_input: + input_selection: source_expression: user_data.formatted_response # Broker output for Anthropic @@ -307,7 +307,7 @@ flows: assemble_key: id max_time_ms: 30000 max_items: 3 # Number of Agents - component_input: + input_selection: source_expression: input.payload # Format response for the LLM request @@ -345,10 +345,10 @@ flows: function: add params: positional: - - source_expression(keyword_args:accumulated_value) - - source_expression(keyword_args:current_value) + - evaluate_expression(keyword_args:accumulated_value) + - evaluate_expression(keyword_args:current_value) dest_expression: user_data.aggregated_data:responses - component_input: + input_selection: source_expression: user_data.aggregated_data # Aggregate all the outcomes from the agents @@ -377,7 +377,7 @@ flows: - type: copy source_expression: static:user dest_expression: user_data.llm_input:messages.0.role - component_input: + input_selection: source_expression: user_data.llm_input - component_name: aggregator_output @@ -425,10 +425,10 @@ flows: function: less_than params: positional: - - source_expression(user_data.aggregated_data:layer_number, int) + - evaluate_expression(user_data.aggregated_data:layer_number, int) - ${NUMBER_OF_MOA_LAYERS} - - source_expression(user_data.temp:new_query) - - source_expression(previous:content) + - evaluate_expression(user_data.temp:new_query) + - evaluate_expression(previous:content) dest_expression: user_data.output:payload.query # Copy the response topic based on layer number - type: copy @@ -445,12 +445,12 @@ flows: function: less_than params: positional: - - source_expression(user_data.aggregated_data:layer_number, int) + - evaluate_expression(user_data.aggregated_data:layer_number, int) - ${NUMBER_OF_MOA_LAYERS} - moa/question/aggregate - moa/question/cleanup dest_expression: user_data.output:topic - component_input: + input_selection: source_expression: user_data.output # Cleanup the responses from the assembly and send to the user @@ -482,5 +482,5 @@ flows: - type: copy source_value: moa/question/response dest_expression: user_data.output:topic - component_input: + input_selection: source_expression: user_data.output diff --git a/examples/llm/openai_chat.yaml b/examples/llm/openai_chat.yaml index 4c3f753..038903c 100644 --- a/examples/llm/openai_chat.yaml +++ b/examples/llm/openai_chat.yaml @@ -74,7 +74,7 @@ flows: - type: copy source_expression: static:user dest_expression: user_data.llm_input:messages.0.role - component_input: + input_selection: source_expression: user_data.llm_input # Send response back to broker @@ -92,5 +92,5 @@ flows: - type: copy source_expression: template:{{text://input.topic}}/response dest_expression: user_data.output:topic - component_input: + input_selection: source_expression: user_data.output diff --git a/examples/llm/vertexai_chat.yaml b/examples/llm/vertexai_chat.yaml index 16201b8..19e77ec 100644 --- a/examples/llm/vertexai_chat.yaml +++ b/examples/llm/vertexai_chat.yaml @@ -78,7 +78,7 @@ flows: - type: copy source_expression: static:user dest_expression: user_data.llm_input:messages.0.role - component_input: + input_selection: source_expression: user_data.llm_input # Send response back to broker @@ -96,5 +96,5 @@ flows: - type: copy source_expression: template:{{text://input.topic}}/response dest_expression: user_data.output:topic - component_input: + input_selection: source_expression: user_data.output diff --git a/examples/milvus_store.yaml b/examples/milvus_store.yaml index 4c6211b..c1b707c 100644 --- a/examples/milvus_store.yaml +++ b/examples/milvus_store.yaml @@ -58,7 +58,7 @@ flows: - type: copy source_expression: input.payload:text dest_expression: user_data.vector_input:texts - component_input: + input_selection: source_expression: user_data.vector_input - component_name: milvus_cohere_embed_search @@ -78,7 +78,7 @@ flows: region_name: ${AWS_BEDROCK_COHERE_EMBED_REGION} credentials_profile_name: default # Profile name in ~/.aws/credentials max_results: 5 - component_input: + input_selection: source_expression: input.payload - component_name: stdout diff --git a/examples/request_reply.yaml b/examples/request_reply.yaml index 9755203..3cdae47 100644 --- a/examples/request_reply.yaml +++ b/examples/request_reply.yaml @@ -41,7 +41,7 @@ flows: - type: copy source_value: request/topic dest_expression: user_data.request:topic - component_input: + input_selection: source_expression: user_data.request - component_name: stdout @@ -63,7 +63,7 @@ flows: - type: copy source_expression: input.payload dest_expression: user_data.reply:payload.wrapper - component_input: + input_selection: source_expression: user_data.reply:payload - component_name: broker_output @@ -80,6 +80,6 @@ flows: - type: copy source_expression: input.user_properties:__solace_ai_connector_broker_request_reply_topic__ dest_expression: user_data.output:topic - component_input: + input_selection: source_expression: user_data.output diff --git a/examples/vector_store_search.yaml b/examples/vector_store_search.yaml index ca4b3f8..3316d8c 100644 --- a/examples/vector_store_search.yaml +++ b/examples/vector_store_search.yaml @@ -108,7 +108,7 @@ flows: model_id: ${AWS_BEDROCK_COHERE_EMBED_MODEL_ID} region_name: ${AWS_BEDROCK_COHERE_EMBED_REGION} max_results: 7 - component_input: + input_selection: source_expression: input.payload - component_name: stdout diff --git a/src/solace_ai_connector/common/utils.py b/src/solace_ai_connector/common/utils.py index c04a0ca..003e2ff 100644 --- a/src/solace_ai_connector/common/utils.py +++ b/src/solace_ai_connector/common/utils.py @@ -82,7 +82,7 @@ def resolve_config_values(config, allow_source_expression=False): log.debug("Resolved config value to %s", config) return config for key, value in config.items(): - # If the key is source_expression, we sub config to use the 'source_expression()' value in + # If the key is source_expression, we sub config to use the 'evaluate_expression()' value in # invoke parameters config[key] = resolve_config_values( value, @@ -202,12 +202,13 @@ def call_function(function, params, allow_source_expression): have_lambda = False if positional: for index, value in enumerate(positional): - if isinstance(value, str) and value.startswith("source_expression("): + # source_expression check for backwards compatibility + if isinstance(value, str) and (value.startswith("evaluate_expression(") or value.startswith("source_expression(")): # if not allow_source_expression: # raise ValueError( - # "source_expression() is not allowed in this context" + # "evaluate_expression() is not allowed in this context" # ) - (expression, data_type) = extract_source_expression(value) + (expression, data_type) = extract_evaluate_expression(value) positional[index] = create_lambda_function_for_source_expression( expression, data_type=data_type ) @@ -216,12 +217,13 @@ def call_function(function, params, allow_source_expression): have_lambda = True if keyword: for key, value in keyword.items(): - if isinstance(value, str) and value.startswith("source_expression("): + # source_expression check for backwards compatibility + if isinstance(value, str) and (value.startswith("evaluate_expression(") or value.startswith("source_expression(")): if not allow_source_expression: raise ValueError( - "source_expression() is not allowed in this context" + "evaluate_expression() is not allowed in this context" ) - (expression, data_type) = extract_source_expression(value) + (expression, data_type) = extract_evaluate_expression(value) keyword[key] = create_lambda_function_for_source_expression( expression, data_type=data_type ) @@ -250,16 +252,20 @@ def install_package(package_name): subprocess.run(["pip", "install", package_name], check=True) -def extract_source_expression(se_call): - # First remove the source_expression( and the trailing ) +def extract_evaluate_expression(se_call): + # First remove the evaluate_expression( and the trailing ) # Account for possible whitespace - expression = se_call.split("source_expression(")[1].split(")")[0].strip() + if (se_call.startswith("evaluate_expression(")): + expression = se_call.split("evaluate_expression(")[1].split(")")[0].strip() + else: + # For backwards compatibility + expression = se_call.split("source_expression(")[1].split(")")[0].strip() data_type = None if "," in expression: (expression, data_type) = re.split(r"\s*,\s*", expression) if not expression: - raise ValueError("source_expression() must contain an expression") + raise ValueError("evaluate_expression() must contain an expression") return (expression, data_type) diff --git a/src/solace_ai_connector/components/component_base.py b/src/solace_ai_connector/components/component_base.py index 8fc6270..bd4c52c 100644 --- a/src/solace_ai_connector/components/component_base.py +++ b/src/solace_ai_connector/components/component_base.py @@ -173,10 +173,12 @@ def get_acknowledgement_callback(self): return None def get_input_data(self, message): - component_input = self.config.get("component_input") or { - "source_expression": "previous" - } - source_expression = get_source_expression(component_input) + input_selection = ( + self.config.get("input_selection") + or self.config.get("component_input") + or {"source_expression": "previous"} + ) + source_expression = get_source_expression(input_selection) # This should be overridden by the component if it needs to extract data from the message return message.get_data(source_expression, self) @@ -216,7 +218,7 @@ def get_config(self, key=None, default=None): if self.current_message is None: raise ValueError( f"Component {self.log_identifier} is trying to use an `invoke` config " - "that contains a 'source_expression()' in a context that does not " + "that contains a 'evaluate_expression()' in a context that does not " "have a message available. This is likely a bug in the " "component's configuration." ) diff --git a/src/solace_ai_connector/components/general/aggregate.py b/src/solace_ai_connector/components/general/aggregate.py index ef51773..3a7b33c 100644 --- a/src/solace_ai_connector/components/general/aggregate.py +++ b/src/solace_ai_connector/components/general/aggregate.py @@ -8,7 +8,11 @@ "class_name": "Aggregate", "description": "Take multiple messages and aggregate them into one. " "The output of this component is a list of the exact structure " - "of the input data.", + "of the input data.\n" + "This can be useful for batch processing or for aggregating events " + "together before processing them. The Aggregate component will take a " + "sequence of events and combine them into a single event before enqueuing " + "it to the next component in the flow so that it can perform batch processing.", "short_description": "Aggregate messages into one message.", "config_parameters": [ { @@ -38,6 +42,21 @@ "type": "object", }, }, + "example_config": """ +```yaml + - component_name: aggretator_example + component_module: aggregate + component_config: + # The maximum number of items to aggregate before sending the data to the next component + max_items: 3 + # The maximum time to wait before sending the data to the next component + max_time_ms: 1000 + input_selection: + # Take the text field from the message and use it as the input to the aggregator + source_expression: input.payload:text +``` + +""", } @@ -50,7 +69,7 @@ def __init__(self, **kwargs): self.max_items = self.get_config("max_items") def invoke(self, message, data): - # The passed in data is the date specified by component_input + # The passed in data is the date specified by input_selection # from the config file if self.current_aggregation is None: self.current_aggregation = self.start_new_aggregation() diff --git a/src/solace_ai_connector/components/general/iterate.py b/src/solace_ai_connector/components/general/iterate.py index e0af98e..43e9588 100644 --- a/src/solace_ai_connector/components/general/iterate.py +++ b/src/solace_ai_connector/components/general/iterate.py @@ -20,6 +20,16 @@ "type": "object", "properties": {}, }, + "example_config": """ +```yaml + - component_name: iterate_example + component_module: iterate + component_config: + input_selection: + # Take the list field from the message and use it as the input to the iterator + source_expression: input.payload:embeddings +``` +""", } diff --git a/src/solace_ai_connector/components/inputs_outputs/error_input.py b/src/solace_ai_connector/components/inputs_outputs/error_input.py index 88a6828..6272013 100644 --- a/src/solace_ai_connector/components/inputs_outputs/error_input.py +++ b/src/solace_ai_connector/components/inputs_outputs/error_input.py @@ -15,7 +15,7 @@ "class_name": "ErrorInput", "description": ( "Receive processing errors from the Solace AI Event Connector. Note that " - "the component_input configuration is ignored. " + "the input_selection configuration is ignored. " "This component should be used to create a flow that handles errors from other flows. " ), "config_parameters": [ diff --git a/src/solace_ai_connector/services/cache_service.py b/src/solace_ai_connector/services/cache_service.py index 19f08bb..0b0ff3f 100644 --- a/src/solace_ai_connector/services/cache_service.py +++ b/src/solace_ai_connector/services/cache_service.py @@ -5,8 +5,7 @@ from typing import Any, Optional, Dict, Tuple from threading import Lock from sqlalchemy import create_engine, Column, String, Float, LargeBinary -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm import declarative_base, sessionmaker from ..common.event import Event, EventType from ..common.log import log diff --git a/src/solace_ai_connector/transforms/filter.py b/src/solace_ai_connector/transforms/filter.py index a1e17c7..3f7fa90 100644 --- a/src/solace_ai_connector/transforms/filter.py +++ b/src/solace_ai_connector/transforms/filter.py @@ -13,8 +13,8 @@ " * index: The index of the current item in the source list\n" " * current_value: The value of the current item in the source list\n" " * source_list: The source list\n\n" - "These should be accessed using `source_expression(keyword_args:)`. " - "For example, `source_expression(keyword_args:current_value)`. " + "These should be accessed using `evaluate_expression(keyword_args:)`. " + "For example, `evaluate_expression(keyword_args:current_value)`. " "See the example below for more detail." ), "short_description": "Filter a list based on a filter function", @@ -62,7 +62,7 @@ function: greater_than params: positional: - - source_expression(keyword_args:current_value.my_val) + - evaluate_expression(keyword_args:current_value.my_val) - 2 dest_expression: user_data.output:new_list ``` diff --git a/src/solace_ai_connector/transforms/map.py b/src/solace_ai_connector/transforms/map.py index faf7359..b09865e 100644 --- a/src/solace_ai_connector/transforms/map.py +++ b/src/solace_ai_connector/transforms/map.py @@ -17,8 +17,8 @@ " * index: The index of the current item in the source list\n" " * current_value: The value of the current item in the source list\n" " * source_list: The source list\n\n" - "These should be accessed using `source_expression(keyword_args:)`. " - "For example, `source_expression(keyword_args:current_value)`. " + "These should be accessed using `evaluate_expression(keyword_args:)`. " + "For example, `evaluate_expression(keyword_args:current_value)`. " "See the example below for more detail." ), "short_description": ( @@ -69,9 +69,9 @@ function: add params: positional: - - source_expression(keyword_args:current_value) + - evaluate_expression(keyword_args:current_value) - 2 - dest_expression: user_data.output:new_list + dest_list_expression: user_data.output:new_list ``` This transform would take a payload like this: diff --git a/src/solace_ai_connector/transforms/reduce.py b/src/solace_ai_connector/transforms/reduce.py index 681e041..de08210 100644 --- a/src/solace_ai_connector/transforms/reduce.py +++ b/src/solace_ai_connector/transforms/reduce.py @@ -14,8 +14,8 @@ " * accumulated_value: The current accumulated value\n" " * current_value: The value of the current item in the source list\n" " * source_list: The source list\n\n" - "These should be accessed using `source_expression(keyword_args:)`. " - "For example, `source_expression(keyword_args:current_value)`. " + "These should be accessed using `evaluate_expression(keyword_args:)`. " + "For example, `evaluate_expression(keyword_args:current_value)`. " "See the example below for more detail." ), "short_description": "Reduce a list to a single value", @@ -64,8 +64,8 @@ function: add params: positional: - - source_expression(keyword_args:accumulated_value) - - source_expression(keyword_args:current_value) + - evaluate_expression(keyword_args:accumulated_value) + - evaluate_expression(keyword_args:current_value) dest_expression: user_data.output:my_obj.sum ``` This transform would take a payload like this: diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py index 6cccd41..a288410 100644 --- a/tests/test_aggregate.py +++ b/tests/test_aggregate.py @@ -28,7 +28,7 @@ def test_aggregate_by_time(): component_config: max_items: 10 max_time_ms: {TIMEOUT_MS} - component_input: + input_selection: source_expression: input.payload """ connector, flows = create_test_flows(config_yaml) @@ -78,7 +78,7 @@ def test_aggregate_by_items(): component_config: max_items: 3 max_time_ms: 1000 - component_input: + input_selection: source_expression: input.payload """ connector, flows = create_test_flows(config_yaml) @@ -126,7 +126,7 @@ def test_both_items_and_time(): component_config: max_items: 3 max_time_ms: {MAX_TIME_MS} - component_input: + input_selection: source_expression: input.payload """ connector, flows = create_test_flows(config_yaml) diff --git a/tests/test_config_file.py b/tests/test_config_file.py index 54bd4a3..5bd34f7 100644 --- a/tests/test_config_file.py +++ b/tests/test_config_file.py @@ -58,7 +58,7 @@ def test_no_flow_name(): - type: append source_expression: self:component_index dest_expression: user_data.path:my_path - component_input: + input_selection: source_expression: input.payload:text """ SolaceAiConnector( @@ -114,7 +114,7 @@ def test_no_component_name(): - name: test_flow components: - component_module: delay - component_input: + input_selection: source_expression: input.payload:text """ SolaceAiConnector( diff --git a/tests/test_error_flows.py b/tests/test_error_flows.py index a2ee53f..b8ff4d7 100644 --- a/tests/test_error_flows.py +++ b/tests/test_error_flows.py @@ -43,7 +43,7 @@ def test_basic_error_flow(): component_module: error_input - component_name: pass_through component_module: pass_through - component_input: + input_selection: source_expression: previous:error.text """ connector, flows = create_test_flows(config_yaml) diff --git a/tests/test_filter.py b/tests/test_filter.py index 550114c..b43b72f 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -33,7 +33,7 @@ def test_simple_filter(): function: equal params: positional: - - source_expression(input.payload:my_list.1) + - evaluate_expression(input.payload:my_list.1) - 2 """ connector, flows = create_test_flows(config_yaml) @@ -77,7 +77,7 @@ def test_missing_item_filter(): function: not_equal params: positional: - - source_expression(input.payload:my_list) + - evaluate_expression(input.payload:my_list) - null """ connector, flows = create_test_flows(config_yaml) diff --git a/tests/test_flows.py b/tests/test_flows.py index 5500f36..6196ae1 100644 --- a/tests/test_flows.py +++ b/tests/test_flows.py @@ -46,7 +46,7 @@ # - type: append # source_expression: self:component_index # dest_expression: user_data.path:my_path -# component_input: +# input_selection: # source_expression: input.payload:text # - component_name: delay2 # component_module: delay @@ -63,7 +63,7 @@ # - type: append # source_expression: self:component_index # dest_expression: user_data.path:my_path -# component_input: +# input_selection: # source_expression: user_data.temp # - component_name: delay3 # component_module: delay @@ -74,7 +74,7 @@ # - type: append # source_expression: self:component_index # dest_expression: user_data.path:my_path -# component_input: +# input_selection: # source_expression: previous # """ # ], @@ -131,13 +131,13 @@ def test_on_flow_creation_event(): components: - component_name: delay1 component_module: delay - component_input: + input_selection: source_expression: input.payload:text - name: test_flow2 components: - component_name: delay2 component_module: delay - component_input: + input_selection: source_expression: input.payload:text """ event_handler_called = False @@ -174,7 +174,7 @@ def test_multiple_flow_instances(): components: - component_name: delay1 component_module: delay - component_input: + input_selection: source_expression: input.payload:text """ # Create the connector diff --git a/tests/test_invoke.py b/tests/test_invoke.py index 4834604..58d0b77 100644 --- a/tests/test_invoke.py +++ b/tests/test_invoke.py @@ -580,8 +580,8 @@ def test_invoke_import_os_module(): ) == {"a": "posix"} -def test_invoke_with_source_expression_simple(): - """Verify that the source expression is evaluated""" +def test_invoke_with_evaluate_expression_simple(): + """Verify that the evaluate expression is evaluated""" config = resolve_config_values( { "source_expression": { @@ -590,8 +590,8 @@ def test_invoke_with_source_expression_simple(): "function": "add", "params": { "positional": [ - "source_expression(input.payload:my_obj.val1)", - "source_expression(input.payload:my_obj.val2)", + "evaluate_expression(input.payload:my_obj.val1)", + "evaluate_expression(input.payload:my_obj.val2)", ], }, }, @@ -603,8 +603,8 @@ def test_invoke_with_source_expression_simple(): assert config == {"source_expression": 3} -def test_invoke_with_source_expression_cast_to_int(): - """Verify that the source expression is evaluated""" +def test_invoke_with_evaluate_expression_cast_to_int(): + """Verify that the evaluate expression is evaluated""" config = resolve_config_values( { "source_expression": { @@ -613,7 +613,7 @@ def test_invoke_with_source_expression_cast_to_int(): "function": "add", "params": { "positional": [ - "source_expression(input.payload:my_obj.val1, int )", + "evaluate_expression(input.payload:my_obj.val1, int )", 2, ], }, @@ -626,8 +626,8 @@ def test_invoke_with_source_expression_cast_to_int(): assert config == {"source_expression": 3} -def test_invoke_with_source_expression_cast_to_float(): - """Verify that the source expression is evaluated""" +def test_invoke_with_evaluate_expression_cast_to_float(): + """Verify that the evaluate expression is evaluated""" config = resolve_config_values( { "source_expression": { @@ -636,7 +636,7 @@ def test_invoke_with_source_expression_cast_to_float(): "function": "add", "params": { "positional": [ - "source_expression(input.payload:my_obj.val1, float )", + "evaluate_expression(input.payload:my_obj.val1, float )", 2, ], }, @@ -659,7 +659,7 @@ def test_invoke_with_source_expression_cast_to_bool(): "function": "and_op", "params": { "positional": [ - "source_expression(input.payload:my_obj.val1 , bool )", + "evaluate_expression(input.payload:my_obj.val1 , bool )", True, ], }, @@ -672,8 +672,8 @@ def test_invoke_with_source_expression_cast_to_bool(): assert config == {"source_expression": True} -def test_invoke_with_source_expression_cast_to_str(): - """Verify that the source expression is evaluated""" +def test_invoke_with_evaluate_expression_cast_to_str(): + """Verify that the evaluate expression is evaluated""" config = resolve_config_values( { "source_expression": { @@ -682,7 +682,7 @@ def test_invoke_with_source_expression_cast_to_str(): "function": "add", "params": { "positional": [ - "source_expression(input.payload:my_obj.val1,str)", + "evaluate_expression(input.payload:my_obj.val1,str)", "2", ], }, @@ -695,8 +695,8 @@ def test_invoke_with_source_expression_cast_to_str(): assert config == {"source_expression": "12"} -def test_invoke_with_source_expression_keyword(): - """Verify that the source expression is evaluated""" +def test_invoke_with_evaluate_expression_keyword(): + """Verify that the evaluate expression is evaluated""" config = resolve_config_values( { "source_value": { @@ -705,8 +705,8 @@ def test_invoke_with_source_expression_keyword(): "function": "_test_keyword_args", "params": { "keyword": { - "x": "source_expression(input.payload:my_obj.val1)", - "y": "source_expression(input.payload:my_obj.val2)", + "x": "evaluate_expression(input.payload:my_obj.val1)", + "y": "evaluate_expression(input.payload:my_obj.val2)", }, }, }, @@ -718,8 +718,8 @@ def test_invoke_with_source_expression_keyword(): assert config == {"source_value": {"x": 1, "y": 2}} -def test_invoke_with_source_expression_complex(): - """Verify that the source expression is evaluated""" +def test_invoke_with_evaluate_expression_complex(): + """Verify that the evaluate expression is evaluated""" config = resolve_config_values( { "source_expression": { @@ -728,22 +728,22 @@ def test_invoke_with_source_expression_complex(): "function": "_test_positional_and_keyword_args", "params": { "positional": [ - "source_expression(input.payload:my_obj.val1)", + "evaluate_expression(input.payload:my_obj.val1)", { "invoke": { "module": "invoke_functions", "function": "add", "params": { "positional": [ - "source_expression(input.payload:my_obj.val2)", + "evaluate_expression(input.payload:my_obj.val2)", { "invoke": { "module": "invoke_functions", "function": "multiply", "params": { "positional": [ - "source_expression(input.payload:my_obj.val2)", - "source_expression(input.payload:my_obj.val2)", + "evaluate_expression(input.payload:my_obj.val2)", + "evaluate_expression(input.payload:my_obj.val2)", ], }, }, @@ -754,15 +754,15 @@ def test_invoke_with_source_expression_complex(): }, ], "keyword": { - "x": "source_expression(input.payload:my_obj.val1)", + "x": "evaluate_expression(input.payload:my_obj.val1)", "y": { "invoke": { "module": "invoke_functions", "function": "subtract", "params": { "positional": [ - "source_expression(input.payload:my_obj.val2)", - "source_expression(input.payload:my_obj.val3)", + "evaluate_expression(input.payload:my_obj.val2)", + "evaluate_expression(input.payload:my_obj.val3)", ], }, }, @@ -778,8 +778,8 @@ def test_invoke_with_source_expression_complex(): assert config == {"source_expression": ((1, 6), {"x": 1, "y": -1})} -def test_invoke_with_source_expression_missing(): - """Verify that the source expression is evaluated""" +def test_invoke_with_evaluate_expression_missing(): + """Verify that the evaluate expression is evaluated""" config = resolve_config_values( { "source_expression": { @@ -788,8 +788,8 @@ def test_invoke_with_source_expression_missing(): "function": "add", "params": { "positional": [ - "source_expression(input.payload:my_obj.val1)", - "source_expression(input.payload:my_obj.val2)", + "evaluate_expression(input.payload:my_obj.val1)", + "evaluate_expression(input.payload:my_obj.val2)", ], }, }, @@ -803,10 +803,10 @@ def test_invoke_with_source_expression_missing(): config["source_expression"] = config["source_expression"](message) -def test_invoke_with_source_expression_no_source_expression(): - """Verify that the source expression is evaluated""" +def test_invoke_with_source_expression_no_evaluate_expression(): + """Verify that the evaluated expression is evaluated""" with pytest.raises( - ValueError, match=r"source_expression\(\) must contain an expression" + ValueError, match=r"evaluate_expression\(\) must contain an expression" ): resolve_config_values( { @@ -816,7 +816,7 @@ def test_invoke_with_source_expression_no_source_expression(): "function": "add", "params": { "positional": [ - "source_expression()", + "evaluate_expression()", 2, ], }, @@ -826,8 +826,8 @@ def test_invoke_with_source_expression_no_source_expression(): ) -def test_invoke_with_source_expression_with_real_flow(): - """Verify that the source expression is evaluated properly in transforms and component_input""" +def test_invoke_with_evaluate_expression_with_real_flow(): + """Verify that the evaluate expression is evaluated properly in transforms and input_selection""" config_yaml = """ instance_name: test_instance log: @@ -846,17 +846,17 @@ def test_invoke_with_source_expression_with_real_flow(): function: add params: positional: - - source_expression(input.payload:my_obj.val1.1) + - evaluate_expression(input.payload:my_obj.val1.1) - 2 dest_expression: user_data.temp:my_val - component_input: + input_selection: source_expression: invoke: module: invoke_functions function: add params: positional: - - source_expression(input.payload:my_obj.obj2) + - evaluate_expression(input.payload:my_obj.obj2) - " test" """ message = Message(payload={"my_obj": {"val1": [1, 2, 3], "obj2": "Hello, World!"}}) @@ -868,7 +868,7 @@ def test_invoke_with_source_expression_with_real_flow(): } # The copy transform assert ( output_message.get_data("previous") == "Hello, World! test" - ) # The component_input + ) # The input_selection def atest_user_processing_component(): @@ -886,7 +886,7 @@ def atest_user_processing_component(): function: add params: positional: - - source_expression(input.payload:my_obj.val1.1) + - evaluate_expression(input.payload:my_obj.val1.1) - 2 """ @@ -919,10 +919,10 @@ def test_reduce_transform_accumulator(): function: add params: positional: - - source_expression(keyword_args:accumulated_value) - - source_expression(keyword_args:current_value) + - evaluate_expression(keyword_args:accumulated_value) + - evaluate_expression(keyword_args:current_value) dest_expression: user_data.temp:my_val - component_input: + input_selection: source_expression: user_data.temp:my_val """ message = Message(payload={"my_list": [1, 2, 3, 4, 5]}) @@ -957,10 +957,10 @@ def test_reduce_transform_make_list(): function: append params: positional: - - source_expression(keyword_args:accumulated_value) - - source_expression(keyword_args:current_value) + - evaluate_expression(keyword_args:accumulated_value) + - evaluate_expression(keyword_args:current_value) dest_expression: user_data.temp:my_val - component_input: + input_selection: source_expression: user_data.temp:my_val """ message = Message(payload={"my_list": [1, 2, 3, 4, 5]}) @@ -991,10 +991,10 @@ def test_map_transform_add_2(): function: add params: positional: - - source_expression(keyword_args:current_value) + - evaluate_expression(keyword_args:current_value) - 2 dest_list_expression: user_data.temp:new_list - component_input: + input_selection: source_expression: user_data.temp:new_list """ message = Message(payload={"my_list": [1, 2, 3, 4, 5]}) @@ -1025,10 +1025,10 @@ def test_filter_transform_greater_than_2(): function: greater_than params: positional: - - source_expression(keyword_args:current_value) + - evaluate_expression(keyword_args:current_value) - 2 dest_list_expression: user_data.temp:new_list - component_input: + input_selection: source_expression: user_data.temp:new_list """ message = Message(payload={"my_list": [1, 2, 3, 4, 5]}) @@ -1059,10 +1059,10 @@ def test_filter_transform_sub_field_greater_than_2(): function: greater_than params: positional: - - source_expression(keyword_args:current_value.my_val) + - evaluate_expression(keyword_args:current_value.my_val) - 2 dest_list_expression: user_data.temp:new_list - component_input: + input_selection: source_expression: user_data.temp:new_list """ message = Message( diff --git a/tests/test_iterate.py b/tests/test_iterate.py index 0dc947b..a33bacc 100644 --- a/tests/test_iterate.py +++ b/tests/test_iterate.py @@ -26,7 +26,7 @@ def test_small_list(): components: - component_name: iterate component_module: iterate - component_input: + input_selection: source_expression: input.payload:my_list """ connector, flows = create_test_flows(config_yaml) @@ -56,7 +56,7 @@ def test_large_list(): components: - component_name: iterate component_module: iterate - component_input: + input_selection: source_expression: input.payload:my_list """ connector, flows = create_test_flows(config_yaml) diff --git a/tests/test_timer_input.py b/tests/test_timer_input.py index c193538..343a7d8 100644 --- a/tests/test_timer_input.py +++ b/tests/test_timer_input.py @@ -40,7 +40,7 @@ def test_basic_timer(): module: time function: time dest_expression: user_data.timestamp - component_input: + input_selection: source_expression: user_data.timestamp """ diff --git a/tests/test_transforms.py b/tests/test_transforms.py index d9ed555..9b00150 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -30,7 +30,7 @@ def test_basic_copy_transform(): - type: copy source_value: "Static Greeting!" dest_expression: user_data.temp:payload.greeting - component_input: + input_selection: source_expression: user_data.temp:payload.text """ @@ -64,7 +64,7 @@ def test_basic_map_transform(): source_expression: item dest_list_expression: user_data.temp:my_list dest_expression: my_obj.full - component_input: + input_selection: source_expression: user_data.temp """ @@ -101,7 +101,7 @@ def test_map_with_index_transform(): source_expression: index dest_list_expression: user_data.temp:my_list dest_expression: my_obj.index - component_input: + input_selection: source_expression: user_data.temp """ @@ -138,7 +138,7 @@ def test_map_with_message_source_expression(): source_expression: input.payload:my_obj.two dest_list_expression: user_data.temp:my_list dest_expression: my_obj.my_obj_two - component_input: + input_selection: source_expression: user_data.temp """ @@ -176,7 +176,7 @@ def test_basic_append_transform(): - type: append source_expression: input.payload:three dest_expression: user_data.temp:my_list - component_input: + input_selection: source_expression: user_data.temp """ @@ -204,7 +204,7 @@ def test_overwrite_non_list_with_list(): - type: append source_expression: input.payload:one dest_expression: user_data.temp:my_list - component_input: + input_selection: source_expression: user_data.temp """ @@ -228,7 +228,7 @@ def test_transform_without_a_type(): input_transforms: - source_expression: input.payload:one dest_expression: user_data.temp:my_list - component_input: + input_selection: source_expression: user_data.temp """ create_connector(config_yaml) @@ -249,7 +249,7 @@ def test_transform_with_unknown_type(): - type: unknown source_expression: input.payload:one dest_expression: user_data.temp:my_list - component_input: + input_selection: source_expression: user_data.temp """ create_connector(config_yaml) @@ -270,7 +270,7 @@ def test_missing_source_expression(): input_transforms: - type: copy dest_expression: user_data.temp:my_list - component_input: + input_selection: source_expression: user_data.temp """ create_connector(config_yaml) @@ -291,7 +291,7 @@ def test_missing_dest_expression(): input_transforms: - type: copy source_expression: input.payload:one - component_input: + input_selection: source_expression: user_data.temp """ create_connector(config_yaml) @@ -314,7 +314,7 @@ def test_source_value_as_an_object(): one: 1 two: 2 dest_expression: user_data.temp:my_obj - component_input: + input_selection: source_expression: user_data.temp """ From 3f339855db057880c70430a38ac2d21bb82a112d Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Tue, 3 Sep 2024 15:08:49 -0400 Subject: [PATCH 06/55] Fixed solace disconnection issues on shutting down (#30) --- examples/llm/mixture_of_agents.yaml | 3 +++ .../components/inputs_outputs/broker_base.py | 3 +++ src/solace_ai_connector/flow/flow.py | 4 +++- src/solace_ai_connector/main.py | 2 ++ src/solace_ai_connector/solace_ai_connector.py | 8 ++++---- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/examples/llm/mixture_of_agents.yaml b/examples/llm/mixture_of_agents.yaml index d71f46c..dd72e93 100644 --- a/examples/llm/mixture_of_agents.yaml +++ b/examples/llm/mixture_of_agents.yaml @@ -193,6 +193,7 @@ flows: # Vertex AI LLM Request - component_name: llm_request component_module: langchain_chat_model + num_instances: 3 component_config: langchain_module: langchain_google_vertexai langchain_class: ChatVertexAI @@ -228,6 +229,7 @@ flows: # OpenAI LLM Request - component_name: llm_request component_module: openai_chat_model + num_instances: 3 component_config: api_key: ${OPENAI_API_KEY} base_url: ${OPENAI_API_ENDPOINT} @@ -260,6 +262,7 @@ flows: # Anthropic LLM Request - component_name: llm_request component_module: langchain_chat_model + num_instances: 3 component_config: langchain_module: langchain_anthropic langchain_class: ChatAnthropic diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_base.py b/src/solace_ai_connector/components/inputs_outputs/broker_base.py index 6b91a01..b6fd113 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_base.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_base.py @@ -59,6 +59,9 @@ def disconnect(self): self.messaging_service.disconnect() self.connected = False + def stop_component(self): + self.disconnect() + def decode_payload(self, payload): encoding = self.get_config("payload_encoding") payload_format = self.get_config("payload_format") diff --git a/src/solace_ai_connector/flow/flow.py b/src/solace_ai_connector/flow/flow.py index c13ea71..9adfb27 100644 --- a/src/solace_ai_connector/flow/flow.py +++ b/src/solace_ai_connector/flow/flow.py @@ -1,7 +1,9 @@ """Main class for the flow""" import threading +from typing import List +from ..components.component_base import ComponentBase from ..common.log import log from ..common.utils import import_module @@ -48,7 +50,7 @@ def __init__( ): self.flow_config = flow_config self.flow_index = flow_index - self.component_groups = [] + self.component_groups: List[List[ComponentBase]] = [] self.name = flow_config.get("name") self.module_info = None self.stop_signal = stop_signal diff --git a/src/solace_ai_connector/main.py b/src/solace_ai_connector/main.py index fd78588..0adb414 100644 --- a/src/solace_ai_connector/main.py +++ b/src/solace_ai_connector/main.py @@ -66,6 +66,8 @@ def main(): app.wait_for_flows() + print("Solace AI Connector exited successfully!") + if __name__ == "__main__": # Read in the configuration yaml filenames from the args diff --git a/src/solace_ai_connector/solace_ai_connector.py b/src/solace_ai_connector/solace_ai_connector.py index 83c3c2f..f41f50f 100644 --- a/src/solace_ai_connector/solace_ai_connector.py +++ b/src/solace_ai_connector/solace_ai_connector.py @@ -2,9 +2,9 @@ import threading import queue -import time from datetime import datetime +from typing import List from .common.log import log, setup_log from .common.utils import resolve_config_values from .flow.flow import Flow @@ -18,7 +18,7 @@ class SolaceAiConnector: def __init__(self, config, event_handlers=None, error_queue=None): self.config = config or {} - self.flows = [] + self.flows: List[Flow] = [] self.trace_queue = None self.trace_thread = None self.flow_input_queues = {} @@ -95,8 +95,8 @@ def wait_for_flows(self): break except KeyboardInterrupt: log.info("Received keyboard interrupt - stopping") - self.stop_signal.set() - # sys.exit(0) + self.stop() + self.cleanup() def stop(self): """Stop the Solace AI Event Connector""" From 8c9434f2d7e96f10cb204cefe77cd60509fb22e3 Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:54:04 -0400 Subject: [PATCH 07/55] Add RAG example for AI connector + delete action for vector index (#31) * Added a RAG example for AI connector * Added delete option to vectordb * Changed id to ids --- .../langchain_vector_store_embedding_index.md | 9 +- ...langchain_vector_store_embedding_search.md | 2 +- examples/llm/openai_chat.yaml | 4 +- examples/llm/openai_chroma_rag.yaml | 183 ++++++++++++++++++ examples/milvus_store.yaml | 4 +- .../langchain_vector_store_embedding_base.py | 6 +- .../langchain_vector_store_embedding_index.py | 41 +++- ...langchain_vector_store_embedding_search.py | 3 +- 8 files changed, 240 insertions(+), 12 deletions(-) create mode 100644 examples/llm/openai_chroma_rag.yaml diff --git a/docs/components/langchain_vector_store_embedding_index.md b/docs/components/langchain_vector_store_embedding_index.md index a9b3c9d..5566b8b 100644 --- a/docs/components/langchain_vector_store_embedding_index.md +++ b/docs/components/langchain_vector_store_embedding_index.md @@ -41,13 +41,20 @@ component_config: }, ... - ] + ], + ids: [ + , + ... + ], + action: } ``` | Field | Required | Description | | --- | --- | --- | | texts | True | | | metadatas | False | | +| ids | False | The ID of the text to add to the index. required for 'delete' action | +| action | False | The action to perform on the index from one of 'add', 'delete' | ## Component Output Schema diff --git a/docs/components/langchain_vector_store_embedding_search.md b/docs/components/langchain_vector_store_embedding_search.md index 7baf102..68c9678 100644 --- a/docs/components/langchain_vector_store_embedding_search.md +++ b/docs/components/langchain_vector_store_embedding_search.md @@ -28,7 +28,7 @@ component_config: | embedding_component_path | True | | The embedding library path - e.g. 'langchain_community.embeddings' | | embedding_component_name | True | | The embedding model to use - e.g. BedrockEmbeddings | | embedding_component_config | True | | Model specific configuration for the embedding model. See documentation for valid parameter names. | -| max_results | True | | The maximum number of results to return | +| max_results | True | 3 | The maximum number of results to return | | combine_context_from_same_source | False | True | Set to False if you don't want to combine all the context from the same source. Default is True | diff --git a/examples/llm/openai_chat.yaml b/examples/llm/openai_chat.yaml index 038903c..71aee14 100644 --- a/examples/llm/openai_chat.yaml +++ b/examples/llm/openai_chat.yaml @@ -16,7 +16,7 @@ # required ENV variables: # - OPENAI_API_KEY # - OPENAI_API_ENDPOINT -# - MODEL_NAME +# - OPENAI_MODEL_NAME # - SOLACE_BROKER_URL # - SOLACE_BROKER_USERNAME # - SOLACE_BROKER_PASSWORD @@ -61,7 +61,7 @@ flows: component_config: api_key: ${OPENAI_API_KEY} base_url: ${OPENAI_API_ENDPOINT} - model: ${MODEL_NAME} + model: ${OPENAI_MODEL_NAME} temperature: 0.01 input_transforms: - type: copy diff --git a/examples/llm/openai_chroma_rag.yaml b/examples/llm/openai_chroma_rag.yaml new file mode 100644 index 0000000..f78bfc0 --- /dev/null +++ b/examples/llm/openai_chroma_rag.yaml @@ -0,0 +1,183 @@ +# OpenAI RAG (Retrieval Augmented Generation) example using ChromaDB +# This will create 2 flows like these: +# +# Solace[topic:demo/rag/data] -> embed and store in ChromaDB +# Solace[topic:demo/rag/query] -> search in ChromaDB -> OpenAI -> Solace[topic:demo/rag/query/response] +# +# Load Data: +# Send data to Solace topic `demo/rag/data` with the following payload format: +# { +# "texts": [. , ...] +# } +# +# RAG Query: +# Send query to Solace topic `demo/rag/query` with the following payload format: +# { +# "query": "" +# } +# The response will be sent to Solace topic `demo/rag/query/response` +# +# Dependencies: +# pip install -U langchain_openai openai chromadb langchain-chroma +# +# Required ENV variables: +# - OPENAI_API_KEY +# - OPENAI_API_ENDPOINT +# - OPENAI_EMBEDDING_MODEL_NAME +# - OPENAI_MODEL_NAME +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN + +--- +log: + stdout_log_level: INFO + log_file_level: INFO + log_file: solace_ai_connector.log + +shared_config: + - broker_config: &broker_connection + broker_type: solace + broker_url: ${SOLACE_BROKER_URL} + broker_username: ${SOLACE_BROKER_USERNAME} + broker_password: ${SOLACE_BROKER_PASSWORD} + broker_vpn: ${SOLACE_BROKER_VPN} + +# Data ingestion and augmented inference flows +flows: + # Data ingestion to chromaDB for RAG + - name: chroma_ingest + components: + # Data Input from a Solace broker for ingestion + - component_name: solace_data_input + component_module: broker_input + component_config: + <<: *broker_connection + broker_queue_name: demo_rag_data + broker_subscriptions: + - topic: demo/rag/data + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # Embedding data & ChromaDB ingest + - component_name: chroma_embed + component_module: langchain_vector_store_embedding_index + component_config: + vector_store_component_path: langchain_chroma + vector_store_component_name: Chroma + vector_store_component_config: + persist_directory: ./chroma_data + collection_name: rag + embedding_component_path: langchain_openai + embedding_component_name: OpenAIEmbeddings + embedding_component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${OPENAI_EMBEDDING_MODEL_NAME} + input_transforms: + - type: copy + source_value: topic:demo/rag/data + dest_expression: user_data.vector_input:metadatas.source + - type: copy + source_expression: input.payload:texts + dest_expression: user_data.vector_input:texts + input_selection: + source_expression: user_data.vector_input + + # RAG Inference flow + - name: OpenAI_RAG + components: + # Inference Input from a Solace broker for completion + - component_name: solace_completion_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_queue_name: demo_rag_query + broker_subscriptions: + - topic: demo/rag/query + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # Retrieve the top-k documents from ChromaDB + - component_name: chroma_search + component_module: langchain_vector_store_embedding_search + component_config: + vector_store_component_path: langchain_chroma + vector_store_component_name: Chroma + vector_store_component_config: + persist_directory: ./chroma_data + collection_name: rag + embedding_component_path: langchain_openai + embedding_component_name: OpenAIEmbeddings + embedding_component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${OPENAI_EMBEDDING_MODEL_NAME} + max_results: 5 + input_transforms: + - type: copy + source_expression: input.payload:query + dest_expression: user_data.vector_input:text + input_selection: + source_expression: user_data.vector_input + + # Generate response using the retrieved data + - component_name: llm_request + component_module: openai_chat_model + component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${OPENAI_MODEL_NAME} + temperature: 0.01 + input_transforms: + # Extract and format the retrieved data + - type: map + source_list_expression: previous:result + source_expression: | + template:{{text://item:text}}\n\n + dest_list_expression: user_data.retrieved_data + + - type: copy + source_expression: | + template:You are a helpful AI assistant. Using the provided context, help with the user's request below. Refrain to use any knowledge outside from the provided context. If the user query can not be answered using the provided context, reject user's query. + + + {{text://user_data.retrieved_data}} + + + + {{text://input.payload:query}} + + dest_expression: user_data.llm_input:messages.0.content + - type: copy + source_expression: static:user + dest_expression: user_data.llm_input:messages.0.role + input_selection: + source_expression: user_data.llm_input + + # Send response back to broker with completion and retrieved data + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: previous:content + dest_expression: user_data.output:payload.response + - type: copy + source_expression: input.payload:query + dest_expression: user_data.output:payload.query + - type: copy + source_expression: user_data.retrieved_data + dest_expression: user_data.output:payload.retrieved_data + - type: copy + source_expression: template:{{text://input.topic}}/response + dest_expression: user_data.output:topic + input_selection: + source_expression: user_data.output diff --git a/examples/milvus_store.yaml b/examples/milvus_store.yaml index c1b707c..50c98ad 100644 --- a/examples/milvus_store.yaml +++ b/examples/milvus_store.yaml @@ -51,10 +51,10 @@ flows: invoke: module: platform function: system - dest_expression: user_data.vector_input:metadata.system + dest_expression: user_data.vector_input:metadatas.system - type: copy source_value: username - dest_expression: user_data.vector_input:metadata.user + dest_expression: user_data.vector_input:metadatas.user - type: copy source_expression: input.payload:text dest_expression: user_data.vector_input:texts diff --git a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_base.py b/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_base.py index f5543cc..5288fb1 100644 --- a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_base.py +++ b/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_base.py @@ -78,8 +78,10 @@ def init(self): self.vector_store_info["config"], vector_store_class ) except Exception: # pylint: disable=broad-except - del self.vector_store_info["config"]["embeddings"] - del self.vector_store_info["config"]["embedding_function"] + if "embeddings" in self.vector_store_info["config"]: + del self.vector_store_info["config"]["embeddings"] + if "embedding_function" in self.vector_store_info["config"]: + del self.vector_store_info["config"]["embedding_function"] self.vector_store = vector_store_class.from_texts( [], self.embedding, **self.vector_store_info["config"] ) diff --git a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_index.py b/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_index.py index 9e41b54..79064ed 100644 --- a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_index.py +++ b/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_index.py @@ -74,6 +74,18 @@ "type": "object", }, }, + "ids": { + "type": "array", + "items": { + "type": "string", + }, + "description": "The ID of the text to add to the index. required for 'delete' action", + }, + "action": { + "type": "string", + "default": "add", + "description": "The action to perform on the index from one of 'add', 'delete'", + }, }, "required": ["texts"], }, @@ -116,12 +128,35 @@ def invoke(self, message, data): # Get the metadatas if they exist metadatas = data.get("metadatas", None) - args = [texts] if metadatas is not None: if not isinstance(metadatas, list): metadatas = [metadatas] - args.append(metadatas) + # Get the ids if they exist + ids = data.get("ids", None) + if ids is not None: + if not isinstance(ids, list): + ids = [ids] + + action = data.get("action", "add") + match action: + case "add": + return self.add_data(texts, metadatas, ids) + case "delete": + return self.delete_data(ids) + case _: + raise ValueError("Invalid action: {}".format(action)) + + def add_data(self, texts, metadatas=None, ids=None): # Add the texts to the vector store - self.vector_store.add_texts(*args) + args = [texts] + if metadatas is not None: + args.append(metadatas) + self.vector_store.add_texts(*args, ids=ids) + return {"result": "OK"} + + def delete_data(self, ids): + if not ids: + raise ValueError("No IDs provided to delete") + self.vector_store.delete(ids) return {"result": "OK"} diff --git a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_search.py b/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_search.py index ef179d9..1b974eb 100644 --- a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_search.py +++ b/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_search.py @@ -59,6 +59,7 @@ "name": "max_results", "required": True, "description": "The maximum number of results to return", + "default": 3, }, { "name": "combine_context_from_same_source", @@ -92,7 +93,7 @@ def __init__(self, **kwargs): def invoke(self, message, data): text = data["text"] - k = self.get_config("max_results") + k = self.get_config("max_results", 3) combine_context_from_same_source = self.get_config( "combine_context_from_same_source" ) From 3c9b8ba006880b9dccc12456305c81b018089091 Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Wed, 4 Sep 2024 20:19:41 -0400 Subject: [PATCH 08/55] chore: Refactor make_history_start_with_user_message method (#32) Fix the method to not trim the first entry if it is a "system" role --- .../openai/openai_chat_model_with_history.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/solace_ai_connector/components/general/openai/openai_chat_model_with_history.py b/src/solace_ai_connector/components/general/openai/openai_chat_model_with_history.py index cca2e7b..5fde36f 100644 --- a/src/solace_ai_connector/components/general/openai/openai_chat_model_with_history.py +++ b/src/solace_ai_connector/components/general/openai/openai_chat_model_with_history.py @@ -132,11 +132,20 @@ def clear_history_but_keep_depth(self, session_id: str, depth: int, history): def make_history_start_with_user_message(self, session_id, history): if session_id in history: - while ( - history[session_id]["messages"] - and history[session_id]["messages"][0]["role"] != "user" - ): - history[session_id]["messages"].pop(0) + messages = history[session_id]["messages"] + if messages: + if messages[0]["role"] == "system": + # Start from the second message if the first is "system" + start_index = 1 + else: + # Start from the first message otherwise + start_index = 0 + + while ( + start_index < len(messages) + and messages[start_index]["role"] != "user" + ): + messages.pop(start_index) def handle_timer_event(self, timer_data): if timer_data["timer_id"] == "history_cleanup": From fc26447bc8b5a1faaabae2e18537589b2560e806 Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Mon, 9 Sep 2024 09:59:10 -0400 Subject: [PATCH 09/55] Keep history depth needs to be a positive integer and test refactor (#33) * chore: Refactor clear_history_but_keep_depth method to handle negative depth values * chore: small change to how this is solved * chore: one more try * refactor: move utils_for_test_files.py to solace_ai_connector module * refactor: removed the orginal utils_for_test_files.py * refactor: update import statements in test files * refactor: add sys.path.append("src") to test files * refactor: standardize import order and sys.path.append in test files * refactor: a bit more test infrastructure changes * feat: allow component_module to accept module objects directly * feat: add types module import to utils.py * test: add static import and object config test * refactor: update test_static_import_and_object_config to use create_test_flows * refactor: Improve test structure and remove duplicate test case * fix: remove duplicate import of yaml module * refactor: Modify test config to use dict instead of YAML string * refactor: convert config_yaml from string to dictionary * refactor: update static import test to use pass_through component * test: Add delay component message passing test * feat: add test for delay component message processing * feat: Added a new test function (test_one_component) to make it very easy to just run some quick tests on a single input -> expected output tests on a single component * feat: added input_transforms to the test_one_component so that input transforms can be tested with it * chore: a bit of cleanup and new tests for test_one_component * chore: rename test_one_component because it was being picked up as a test by the pytest scanner * fix: fixed a typo --- src/solace_ai_connector/common/utils.py | 22 ++--- .../openai/openai_chat_model_with_history.py | 5 ++ src/solace_ai_connector/flow/flow.py | 3 - .../test_utils}/utils_for_test_files.py | 87 +++++++++++++++++-- tests/test_acks.py | 4 +- tests/test_aggregate.py | 5 +- tests/test_config_file.py | 51 ++++++++++- tests/test_error_flows.py | 6 +- tests/test_filter.py | 6 +- tests/test_flows.py | 5 +- tests/test_invoke.py | 18 ++-- tests/test_iterate.py | 6 +- tests/test_message_get_set_data.py | 4 +- tests/test_timer_input.py | 5 +- tests/test_transforms.py | 62 ++++++++++++- 15 files changed, 243 insertions(+), 46 deletions(-) rename {tests => src/solace_ai_connector/test_utils}/utils_for_test_files.py (62%) diff --git a/src/solace_ai_connector/common/utils.py b/src/solace_ai_connector/common/utils.py index 003e2ff..4996c3b 100644 --- a/src/solace_ai_connector/common/utils.py +++ b/src/solace_ai_connector/common/utils.py @@ -6,6 +6,7 @@ import re import builtins import subprocess +import types from .log import log @@ -94,8 +95,11 @@ def resolve_config_values(config, allow_source_expression=False): return config -def import_module(name, base_path=None, component_package=None): - """Import a module by name""" +def import_module(module, base_path=None, component_package=None): + """Import a module by name or return the module object if it's already imported""" + + if isinstance(module, types.ModuleType): + return module if component_package: install_package(component_package) @@ -104,14 +108,13 @@ def import_module(name, base_path=None, component_package=None): if base_path not in sys.path: sys.path.append(base_path) try: - module = importlib.import_module(name) - return module + return importlib.import_module(module) except ModuleNotFoundError as exc: # If the module does not have a path associated with it, try # importing it from the known prefixes - annoying that this # is necessary. It seems you can't dynamically import a module # that is listed in an __init__.py file :( - if "." not in name: + if "." not in module: for prefix_prefix in ["solace_ai_connector", "."]: for prefix in [ ".components", @@ -123,22 +126,21 @@ def import_module(name, base_path=None, component_package=None): ".transforms", ".common", ]: - full_name = f"{prefix_prefix}{prefix}.{name}" + full_name = f"{prefix_prefix}{prefix}.{module}" try: if full_name.startswith("."): - module = importlib.import_module( + return importlib.import_module( full_name, package=__package__ ) else: - module = importlib.import_module(full_name) - return module + return importlib.import_module(full_name) except ModuleNotFoundError: pass except Exception as e: raise ImportError( f"Module load error for {full_name}: {e}" ) from e - raise ModuleNotFoundError(f"Module '{name}' not found") from exc + raise ModuleNotFoundError(f"Module '{module}' not found") from exc def invoke_config(config, allow_source_expression=False): diff --git a/src/solace_ai_connector/components/general/openai/openai_chat_model_with_history.py b/src/solace_ai_connector/components/general/openai/openai_chat_model_with_history.py index 5fde36f..ba7fe64 100644 --- a/src/solace_ai_connector/components/general/openai/openai_chat_model_with_history.py +++ b/src/solace_ai_connector/components/general/openai/openai_chat_model_with_history.py @@ -46,6 +46,11 @@ def __init__(self, **kwargs): def invoke(self, message, data): session_id = data.get("session_id") clear_history_but_keep_depth = data.get("clear_history_but_keep_depth") + try: + if clear_history_but_keep_depth is not None: + clear_history_but_keep_depth = max(0, int(clear_history_but_keep_depth)) + except (TypeError, ValueError): + clear_history_but_keep_depth = 0 messages = data.get("messages", []) with self.get_lock(self.history_key): diff --git a/src/solace_ai_connector/flow/flow.py b/src/solace_ai_connector/flow/flow.py index 9adfb27..782c7ce 100644 --- a/src/solace_ai_connector/flow/flow.py +++ b/src/solace_ai_connector/flow/flow.py @@ -90,10 +90,7 @@ def create_component_group(self, component, index): base_path = component.get("component_base_path", None) component_package = component.get("component_package", None) num_instances = component.get("num_instances", 1) - # component_config = component.get("component_config", {}) - # component_name = component.get("component_name", "") - # imported_module = import_from_directories(component_module) imported_module = import_module(component_module, base_path, component_package) try: diff --git a/tests/utils_for_test_files.py b/src/solace_ai_connector/test_utils/utils_for_test_files.py similarity index 62% rename from tests/utils_for_test_files.py rename to src/solace_ai_connector/test_utils/utils_for_test_files.py index 2bb9dfc..fec9bad 100644 --- a/tests/utils_for_test_files.py +++ b/src/solace_ai_connector/test_utils/utils_for_test_files.py @@ -1,8 +1,6 @@ -"""Collection of functions to be used in test files""" - +import os import queue import sys -import os import yaml sys.path.insert(0, os.path.abspath("src")) @@ -10,6 +8,7 @@ from solace_ai_connector.solace_ai_connector import SolaceAiConnector from solace_ai_connector.common.log import log from solace_ai_connector.common.event import Event, EventType +from solace_ai_connector.common.message import Message # from solace_ai_connector.common.message import Message @@ -61,12 +60,16 @@ def enqueue(self, message): self.next_component_queue.put(event) -def create_connector(config_yaml, event_handlers=None, error_queue=None): - """Create a connector from a config""" +def create_connector(config_or_yaml, event_handlers=None, error_queue=None): + """Create a connector from a config that can be an object or a yaml string""" + + config = config_or_yaml + if isinstance(config_or_yaml, str): + config = yaml.safe_load(config_or_yaml) # Create the connector connector = SolaceAiConnector( - yaml.safe_load(config_yaml), + config, event_handlers=event_handlers, error_queue=error_queue, ) @@ -76,9 +79,77 @@ def create_connector(config_yaml, event_handlers=None, error_queue=None): return connector -def create_test_flows(config_yaml, queue_timeout=None, error_queue=None, queue_size=0): +def run_component_test( + module_or_name, + validation_func, + component_config=None, + input_data=None, + input_messages=None, + input_selection=None, + input_transforms=None, +): + if not input_data and not input_messages: + raise ValueError("Either input_data or input_messages must be provided") + + if input_data and input_messages: + raise ValueError("Only one of input_data or input_messages can be provided") + + if input_data and not isinstance(input_data, list): + input_data = [input_data] + + if input_messages and not isinstance(input_messages, list): + input_messages = [input_messages] + + if not input_messages: + input_messages = [] + + if input_selection: + if isinstance(input_selection, str): + input_selection = {"source_expression": input_selection} + + connector = None + try: + connector, flows = create_test_flows( + { + "flows": [ + { + "name": "test_flow", + "components": [ + { + "component_name": "test_component", + "component_module": module_or_name, + "component_config": component_config or {}, + "input_selection": input_selection, + "input_transforms": input_transforms, + } + ], + } + ] + } + ) + + if input_data: + for data in input_data: + message = Message(payload=data) + message.set_previous(data) + input_messages.append(message) + + # Send each message through, one at a time + for message in input_messages: + send_message_to_flow(flows[0], message) + output_message = get_message_from_flow(flows[0]) + validation_func(output_message.get_previous(), output_message, message) + + finally: + if connector: + dispose_connector(connector) + + +def create_test_flows( + config_or_yaml, queue_timeout=None, error_queue=None, queue_size=0 +): # Create the connector - connector = create_connector(config_yaml, error_queue=error_queue) + connector = create_connector(config_or_yaml, error_queue=error_queue) flows = connector.get_flows() diff --git a/tests/test_acks.py b/tests/test_acks.py index c067fb5..bf0b1ea 100644 --- a/tests/test_acks.py +++ b/tests/test_acks.py @@ -1,11 +1,11 @@ """This file tests acks in a flow""" import sys -import queue sys.path.append("src") +import queue -from utils_for_test_files import ( # pylint: disable=wrong-import-position +from solace_ai_connector.test_utils.utils_for_test_files import ( # pylint: disable=wrong-import-position # create_connector, # create_and_run_component, dispose_connector, diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py index a288410..8826f6a 100644 --- a/tests/test_aggregate.py +++ b/tests/test_aggregate.py @@ -1,8 +1,11 @@ """Some tests to verify the aggregate component works as expected""" +import sys + +sys.path.append("src") import time -from utils_for_test_files import ( +from solace_ai_connector.test_utils.utils_for_test_files import ( create_test_flows, dispose_connector, send_message_to_flow, diff --git a/tests/test_config_file.py b/tests/test_config_file.py index 5bd34f7..593bc0e 100644 --- a/tests/test_config_file.py +++ b/tests/test_config_file.py @@ -1,19 +1,26 @@ """Test various things related to the configuration file""" import sys -import yaml import pytest +import yaml sys.path.append("src") -from utils_for_test_files import ( # pylint: disable=wrong-import-position +from solace_ai_connector.test_utils.utils_for_test_files import ( # pylint: disable=wrong-import-position create_connector, + create_test_flows, + dispose_connector, + send_message_to_flow, + get_message_from_flow, ) from solace_ai_connector.solace_ai_connector import ( # pylint: disable=wrong-import-position SolaceAiConnector, ) +from solace_ai_connector.common.message import Message +import solace_ai_connector.components.general.pass_through + # from solace_ai_connector.common.log import log @@ -143,6 +150,46 @@ def test_no_component_module(): assert str(e) == "component_module not provided in flow 0, component 0" +def test_static_import_and_object_config(): + """Test that we can statically import a module and pass an object for the config""" + + config = { + "log": {"log_file_level": "DEBUG", "log_file": "solace_ai_connector.log"}, + "flows": [ + { + "name": "test_flow", + "components": [ + { + "component_name": "delay1", + "component_module": solace_ai_connector.components.general.pass_through, + "component_config": {"delay": 0.1}, + "input_selection": {"source_expression": "input.payload"}, + } + ], + } + ], + } + connector = None + try: + connector, flows = create_test_flows(config) + + # Test pushing a simple message through the delay component + message = Message(payload={"text": "Hello, World!"}) + send_message_to_flow(flows[0], message) + + # Get the output message + output_message = get_message_from_flow(flows[0]) + + # Check that the output is correct + assert output_message.get_data("previous") == {"text": "Hello, World!"} + + except Exception as e: + pytest.fail(f"Test failed with exception: {e}") + finally: + if "connector" in locals(): + dispose_connector(connector) + + def test_bad_module(): """Test that the program exits if the component module is not found""" try: diff --git a/tests/test_error_flows.py b/tests/test_error_flows.py index b8ff4d7..8e7edfe 100644 --- a/tests/test_error_flows.py +++ b/tests/test_error_flows.py @@ -2,11 +2,11 @@ import sys -# import queue - sys.path.append("src") -from utils_for_test_files import ( # pylint: disable=wrong-import-position +# import queue + +from solace_ai_connector.test_utils.utils_for_test_files import ( # pylint: disable=wrong-import-position create_test_flows, # create_and_run_component, dispose_connector, diff --git a/tests/test_filter.py b/tests/test_filter.py index b43b72f..478cc94 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -1,8 +1,12 @@ """Some tests to verify the filter component works as expected""" +import sys + +sys.path.append("src") + # import pytest -from utils_for_test_files import ( +from solace_ai_connector.test_utils.utils_for_test_files import ( create_test_flows, # create_connector, dispose_connector, diff --git a/tests/test_flows.py b/tests/test_flows.py index 6196ae1..4687fda 100644 --- a/tests/test_flows.py +++ b/tests/test_flows.py @@ -1,9 +1,12 @@ """This test file tests all things to do with the flows and the components that make up the flows""" +import sys + +sys.path.append("src") import pytest import time -from utils_for_test_files import ( +from solace_ai_connector.test_utils.utils_for_test_files import ( create_test_flows, create_connector, dispose_connector, diff --git a/tests/test_invoke.py b/tests/test_invoke.py index 58d0b77..fa0de0f 100644 --- a/tests/test_invoke.py +++ b/tests/test_invoke.py @@ -5,13 +5,14 @@ sys.path.append("src") -from utils_for_test_files import ( # pylint: disable=wrong-import-position + +from solace_ai_connector.test_utils.utils_for_test_files import ( create_and_run_component, ) -from solace_ai_connector.common.utils import ( # pylint: disable=wrong-import-position +from solace_ai_connector.common.utils import ( resolve_config_values, ) -from solace_ai_connector.common.message import ( # pylint: disable=wrong-import-position +from solace_ai_connector.common.message import ( Message, ) @@ -1083,16 +1084,13 @@ def test_invoke_with_uuid_generator(): response = resolve_config_values( { "a": { - "invoke": { - "module": "invoke_functions", - "function": "uuid" - }, + "invoke": {"module": "invoke_functions", "function": "uuid"}, }, } - ) - + ) + # Check if the output is of type string assert type(response["a"]) == str # Check if the output is a valid UUID - assert len(response["a"]) == 36 \ No newline at end of file + assert len(response["a"]) == 36 diff --git a/tests/test_iterate.py b/tests/test_iterate.py index a33bacc..cffb763 100644 --- a/tests/test_iterate.py +++ b/tests/test_iterate.py @@ -1,8 +1,12 @@ """Some tests to verify the iterate component works as expected""" +import sys + +sys.path.append("src") + # import pytest -from utils_for_test_files import ( +from solace_ai_connector.test_utils.utils_for_test_files import ( create_test_flows, # create_connector, dispose_connector, diff --git a/tests/test_message_get_set_data.py b/tests/test_message_get_set_data.py index f2258d1..33622ed 100644 --- a/tests/test_message_get_set_data.py +++ b/tests/test_message_get_set_data.py @@ -1,11 +1,11 @@ """This test fixture will test the get_data and set_data methods of the Message class""" +import sys +sys.path.append("src") import json import base64 -import sys import pytest -sys.path.append("src") from solace_ai_connector.common.message import Message # Create a few different messages to test with diff --git a/tests/test_timer_input.py b/tests/test_timer_input.py index 343a7d8..b8897e2 100644 --- a/tests/test_timer_input.py +++ b/tests/test_timer_input.py @@ -1,9 +1,12 @@ """Test the timer input component""" +import sys + +sys.path.append("src") import time import pytest -from utils_for_test_files import ( +from solace_ai_connector.test_utils.utils_for_test_files import ( create_test_flows, create_connector, dispose_connector, diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 9b00150..2efe69c 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -4,14 +4,16 @@ sys.path.append("src") -from utils_for_test_files import ( # pylint: disable=wrong-import-position +from solace_ai_connector.test_utils.utils_for_test_files import ( # pylint: disable=wrong-import-position create_connector, create_and_run_component, + run_component_test, # dispose_connector, ) from solace_ai_connector.common.message import ( # pylint: disable=wrong-import-position Message, ) +import solace_ai_connector.components.general.pass_through def test_basic_copy_transform(): @@ -44,6 +46,64 @@ def test_basic_copy_transform(): assert output_message.get_data("previous") == "Hello, World!" +def test_transform_with_run_component_test(): + """This test is actually testing the test infrastructure method: run_component_test""" + + def validation_func(output_data, output_message, _input_message): + assert output_data == "Hello, World!" + assert output_message.get_data("user_data.temp") == { + "payload": {"text": "Hello, World!", "greeting": "Static Greeting!"} + } + + run_component_test( + "pass_through", + validation_func, + input_data={"text": "Hello, World!"}, + input_transforms=[ + { + "type": "copy", + "source_expression": "input.payload", + "dest_expression": "user_data.temp:payload", + }, + { + "type": "copy", + "source_value": "Static Greeting!", + "dest_expression": "user_data.temp:payload.greeting", + }, + ], + input_selection={"source_expression": "user_data.temp:payload.text"}, + ) + + +def test_transform_with_run_component_test_with_static_import(): + """This test is actually testing the test infrastructure method: run_component_test""" + + def validation_func(output_data, output_message, _input_message): + assert output_data == "Hello, World!" + assert output_message.get_data("user_data.temp") == { + "payload": {"text": "Hello, World!", "greeting": "Static Greeting!"} + } + + run_component_test( + solace_ai_connector.components.general.pass_through, + validation_func, + input_data={"text": "Hello, World!"}, + input_transforms=[ + { + "type": "copy", + "source_expression": "input.payload", + "dest_expression": "user_data.temp:payload", + }, + { + "type": "copy", + "source_value": "Static Greeting!", + "dest_expression": "user_data.temp:payload.greeting", + }, + ], + input_selection={"source_expression": "user_data.temp:payload.text"}, + ) + + def test_basic_map_transform(): """Test the basic map transform""" # Create a simple configuration From 113b9304cd4af05172c3a0a2d7c2189374c6d78a Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:53:21 -0400 Subject: [PATCH 10/55] Fix for anthropic example (#35) --- config.yaml | 1 - examples/ack_test.yaml | 1 - examples/anthropic_bedrock.yaml | 16 ++++++---------- examples/error_handler.yaml | 1 - examples/request_reply.yaml | 1 - 5 files changed, 6 insertions(+), 14 deletions(-) diff --git a/config.yaml b/config.yaml index fcb4667..360e99d 100644 --- a/config.yaml +++ b/config.yaml @@ -9,7 +9,6 @@ log: shared_config: - broker_config: &broker_connection - broker_connection_share: ${SOLACE_BROKER_URL} broker_type: solace broker_url: ${SOLACE_BROKER_URL} broker_username: ${SOLACE_BROKER_USERNAME} diff --git a/examples/ack_test.yaml b/examples/ack_test.yaml index 08314aa..41fb2eb 100644 --- a/examples/ack_test.yaml +++ b/examples/ack_test.yaml @@ -15,7 +15,6 @@ log: shared_config: - broker_config: &broker_connection - broker_connection_share: ${SOLACE_BROKER_URL} broker_type: solace broker_url: ${SOLACE_BROKER_URL} broker_username: ${SOLACE_BROKER_USERNAME} diff --git a/examples/anthropic_bedrock.yaml b/examples/anthropic_bedrock.yaml index 7c35bb6..03a0c6c 100644 --- a/examples/anthropic_bedrock.yaml +++ b/examples/anthropic_bedrock.yaml @@ -3,7 +3,10 @@ # sends a message to an Anthropic Bedrock model, and # sends the response back to the Solace broker # It will ask the model to write a dry joke about the input -# message. It takes the entire payload of the input message +# message. +# Send a message to the Solace broker topics `my/topic1` or `my/topic2` +# with a plain text payload. The model will respond with a dry joke to the +# same topic prefixed with `response/`. (e.g. `response/my/topic1`) # # Dependencies: # pip install langchain_aws langchain_community @@ -28,12 +31,12 @@ log: shared_config: - broker_config: &broker_connection - broker_connection_share: ${SOLACE_BROKER_URL} broker_type: solace broker_url: ${SOLACE_BROKER_URL} broker_username: ${SOLACE_BROKER_USERNAME} broker_password: ${SOLACE_BROKER_PASSWORD} broker_vpn: ${SOLACE_BROKER_VPN} + payload_encoding: utf-8 # List of flows flows: @@ -51,7 +54,6 @@ flows: qos: 1 - topic: my/topic2 qos: 1 - payload_encoding: utf-8 payload_format: text - component_name: llm @@ -81,13 +83,7 @@ flows: - component_name: solace_sw_broker component_module: broker_output component_config: - broker_connection_share: ${SOLACE_BROKER_URL} - broker_type: solace - broker_url: ${SOLACE_BROKER_URL} - broker_username: ${SOLACE_BROKER_USERNAME} - broker_password: ${SOLACE_BROKER_PASSWORD} - broker_vpn: ${SOLACE_BROKER_VPN} - payload_encoding: utf-8 + <<: *broker_connection payload_format: text input_transforms: - type: copy diff --git a/examples/error_handler.yaml b/examples/error_handler.yaml index a8c700e..f227794 100644 --- a/examples/error_handler.yaml +++ b/examples/error_handler.yaml @@ -25,7 +25,6 @@ log: shared_config: - broker_config: &broker_connection - broker_connection_share: ${SOLACE_BROKER_URL} broker_type: solace broker_url: ${SOLACE_BROKER_URL} broker_username: ${SOLACE_BROKER_USERNAME} diff --git a/examples/request_reply.yaml b/examples/request_reply.yaml index 3cdae47..69e5834 100644 --- a/examples/request_reply.yaml +++ b/examples/request_reply.yaml @@ -16,7 +16,6 @@ log: shared_config: - broker_config: &broker_connection - broker_connection_share: ${SOLACE_BROKER_URL} broker_type: solace broker_url: ${SOLACE_BROKER_URL} broker_username: ${SOLACE_BROKER_USERNAME} From 43821d98fa43c5f8650e6afd95ebd0758dc4b885 Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Tue, 17 Sep 2024 09:18:40 -0400 Subject: [PATCH 11/55] Updating version dependency (#37) --- pyproject.toml | 4 ++-- requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7c1f63a..2ec127b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,8 @@ classifiers = [ ] dependencies = [ "boto3~=1.34.122", - "langchain_core~=0.2.5", - "langchain~=0.2.3", + "langchain-core~=0.3.0", + "langchain~=0.3.0", "PyYAML~=6.0.1", "Requests~=2.32.3", "solace_pubsubplus>=1.8.0", diff --git a/requirements.txt b/requirements.txt index 702aa66..66ec399 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ boto3~=1.34.122 -langchain_core~=0.2.5 -langchain~=0.2.3 +langchain-core~=0.3.0 +langchain~=0.3.0 PyYAML~=6.0.1 Requests~=2.32.3 solace_pubsubplus~=1.8.0 From 1a96bae89aaeb02b362ae728f6ce31b8e400cbda Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Thu, 19 Sep 2024 10:49:48 -0400 Subject: [PATCH 12/55] Fixed url and file name in getting started (#38) --- docs/getting_started.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting_started.md b/docs/getting_started.md index 14b8845..53d847b 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -72,7 +72,7 @@ area in the Subscriber side of the "Try me!" page. Download the OpenAI connector example configuration file: ```sh -curl https://raw.githubusercontent.com/SolaceLabs/solace-ai-connector/main/examples/llm/openai_chat.yaml > openai_chat.yaml +curl https://raw.githubusercontent.com/SolaceLabs/solace-ai-connector/refs/heads/main/examples/llm/langchain_openai_with_history_chat.yaml > langchain_openai_with_history_chat.yaml ``` For this one, you need to also define the following additional environment variables: @@ -94,7 +94,7 @@ pip install langchain_openai openai Run the connector: ```sh -solace-ai-connector openai_chat.yaml +solace-ai-connector langchain_openai_with_history_chat.yaml ``` Use the "Try Me!" function on the broker's browser UI (or some other means) to publish an event like this: From 9f2891f65c55b3ae01afb0ebe3442c17f7753859 Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:40:58 -0400 Subject: [PATCH 13/55] Add guide for RAG (#39) * Added guide for RAG * update wording --- docs/guides/RAG.md | 232 +++++++++++++++++++++++++++++++++++++++++++ docs/guides/index.md | 4 + docs/index.md | 1 + 3 files changed, 237 insertions(+) create mode 100644 docs/guides/RAG.md create mode 100644 docs/guides/index.md diff --git a/docs/guides/RAG.md b/docs/guides/RAG.md new file mode 100644 index 0000000..6f3f3c7 --- /dev/null +++ b/docs/guides/RAG.md @@ -0,0 +1,232 @@ + +- [Building AI-Powered Applications with Solace AI Connector: A Deep Dive into RAG, LLMs, and Embeddings](#building-ai-powered-applications-with-solace-ai-connector-a-deep-dive-into-rag-llms-and-embeddings) + - [What is Solace AI Connector?](#what-is-solace-ai-connector) + - [Key Concepts Behind the Configuration](#key-concepts-behind-the-configuration) + - [1. Large Language Models (LLMs)](#1-large-language-models-llms) + - [2. Retrieval-Augmented Generation (RAG)](#2-retrieval-augmented-generation-rag) + - [3. Embeddings](#3-embeddings) + - [4. Solace PubSub+ Platform](#4-solace-pubsub-platform) + - [Real-Time Data Consumption with Solace AI Connector](#real-time-data-consumption-with-solace-ai-connector) + - [Real-Time Data Embedding and Storage Flow](#real-time-data-embedding-and-storage-flow) + - [YAML Configuration Breakdown](#yaml-configuration-breakdown) + - [Logging Configuration](#logging-configuration) + - [Shared Configuration for Solace Broker](#shared-configuration-for-solace-broker) + - [Data Ingestion to ChromaDB (Embedding Flow)](#data-ingestion-to-chromadb-embedding-flow) + - [1. Solace Data Input Component](#1-solace-data-input-component) + - [2. Embedding and Storage in ChromaDB](#2-embedding-and-storage-in-chromadb) + - [RAG Inference Flow (Query and Response)](#rag-inference-flow-query-and-response) + - [1. Query Ingestion from Solace Topic](#1-query-ingestion-from-solace-topic) + - [2. ChromaDB Search for Relevant Documents](#2-chromadb-search-for-relevant-documents) + - [3. Response Generation Using OpenAI](#3-response-generation-using-openai) + - [4. Sending the Response Back to Solace](#4-sending-the-response-back-to-solace) + - [Flexibility in Components](#flexibility-in-components) + - [Conclusion](#conclusion) + +# Building AI-Powered Applications with Solace AI Connector: A Deep Dive into RAG, LLMs, and Embeddings + +In the fast-evolving world of AI, businesses are increasingly looking for ways to harness advanced technologies like **Retrieval-Augmented Generation (RAG)** and **Large Language Models (LLMs)** to provide smarter, more interactive applications. **Solace AI Connector** is one such CLI tool that allows you to create AI-powered applications interconnected to Solace's PubSub+ brokers. In this guide, we will explore the configuration of Solace AI Connector, how it can be used with technologies like **RAG**, **LLMs**, and **Embeddings**, and provide a deep understanding of these concepts. + +## What is Solace AI Connector? + +Solace AI Connector is a tool that enables AI-powered applications to interface with Solace PubSub+ brokers. By integrating Solace’s event-driven architecture with AI services (such as OpenAI’s models), you can create applications that interact with real-time data, perform knowledge retrieval, and generate intelligent responses. + +In this guide, we will walk through [a sample YAML configuration](../../examples/llm/openai_chroma_rag.yaml) that sets up two essential flows: +- **Data ingestion into a vector database** using Solace topics, embedding the data into **ChromaDB**. +- **Querying the ingested data** using **RAG**, where queries are sent to OpenAI for intelligent completion and response generation. + +## Key Concepts Behind the Configuration + +Before diving into the configuration, let’s explore the key concepts that power this setup. + +### 1. Large Language Models (LLMs) +LLMs are AI models trained on vast amounts of textual data, capable of generating human-like text. They are used in various tasks like text generation, summarization, translation, and answering complex questions. Models such as OpenAI's GPT-4o or Anthropic's Claude 3.5 Sonnet are examples of LLMs. These models can generate meaningful responses based on the context they are provided, but they also have limitations, such as the risk of hallucinations (generating incorrect or fabricated facts). + +### 2. Retrieval-Augmented Generation (RAG) +RAG is a framework that enhances the performance of LLMs by combining them with external knowledge retrieval. Instead of relying solely on the LLM’s internal knowledge, RAG retrieves relevant documents from an external database (such as ChromaDB) before generating a response. This approach enhances the accuracy of responses, as the generation process is “grounded” in factual information retrieved at the time of the query. + +### 3. Embeddings +Embeddings are numerical representations of text that capture its semantic meaning. They are crucial for many NLP tasks, as they allow models to measure similarity between different pieces of text. In the context of RAG, embedding models (such as **OpenAI Embeddings**) convert input data into vector representations that can be stored in a vector database like **ChromaDB**. When a query is issued, it is also converted into a vector, and similar documents can be retrieved based on their proximity in the vector space. + +### 4. Solace PubSub+ Platform +Solace’s PubSub+ platform provides event-driven messaging and streaming services, enabling applications to publish and subscribe to topics in real-time. In the context of AI applications, Solace acts as the message broker that facilitates the flow of data between different components (e.g., input data, queries, and responses). + +## Real-Time Data Consumption with Solace AI Connector + +One of the standout features of the Solace AI Connector is its ability to seamlessly handle real-time data. As data passes through the Solace broker, it can be consumed, embedded, and stored for future retrieval and analysis. + +### Real-Time Data Embedding and Storage Flow + +Using Solace topics, the connector can subscribe to real-time data streams. This data is processed in near real-time, where each message is embedded using an embedding model like **OpenAI Embeddings**. These embeddings are then stored in vector database like **ChromaDB** making them retrievable for future queries. + +For example, imagine a system that ingests live customer support chat messages. As each message is published to the Solace broker, the system embeds the message, stores the vector in a database, and makes it available for retrieval during future interactions or analyses. This architecture is particularly useful for applications that need to build dynamic, up-to-date knowledge bases based on streaming data. + +By leveraging the real-time messaging capabilities of Solace, the system ensures that data is continuously processed and stored in a structured, retrievable way, allowing for efficient and scalable AI-driven applications. + +## YAML Configuration Breakdown + +Let’s break down the YAML configuration provided in the example, which demonstrates how to implement RAG with Solace AI Connector and ChromaDB. + +### Logging Configuration +```yaml +log: + stdout_log_level: INFO + log_file_level: INFO + log_file: solace_ai_connector.log +``` +This section sets the logging level for the connector, ensuring that logs are captured both to the console and to a file. + +### Shared Configuration for Solace Broker +```yaml +shared_config: + - broker_config: &broker_connection + broker_type: solace + broker_url: ${SOLACE_BROKER_URL} + broker_username: ${SOLACE_BROKER_USERNAME} + broker_password: ${SOLACE_BROKER_PASSWORD} + broker_vpn: ${SOLACE_BROKER_VPN} +``` +Here, we define the connection settings for the Solace broker. These environment variables (`SOLACE_BROKER_URL`, etc.) are essential for establishing communication with the Solace messaging infrastructure. Shared configs can be reused throughout the configuration to maintain consistency. In this example, we're using it for the input and output Solace broker connections. + +### Data Ingestion to ChromaDB (Embedding Flow) + +This flow ingests data into **ChromaDB**, which is a vector database that will store embeddings of the input text. You could have used any other vector database, but for this example, we are using ChromaDB. + +#### 1. Solace Data Input Component +```yaml +- component_name: solace_data_input + component_module: broker_input + component_config: + <<: *broker_connection + broker_queue_name: demo_rag_data + broker_subscriptions: + - topic: demo/rag/data + qos: 1 + payload_encoding: utf-8 + payload_format: json +``` +This component listens to the **Solace topic** `demo/rag/data` for incoming data that needs to be embedded. It subscribes to the topic and expects the payload in JSON format with UTF-8 encoding. This is could be the real-time data stream that you want to process and embed. + +#### 2. Embedding and Storage in ChromaDB +```yaml +- component_name: chroma_embed + component_module: langchain_vector_store_embedding_index + component_config: + vector_store_component_path: langchain_chroma + vector_store_component_name: Chroma + vector_store_component_config: + persist_directory: ./chroma_data + collection_name: rag + embedding_component_path: langchain_openai + embedding_component_name: OpenAIEmbeddings + embedding_component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${OPENAI_EMBEDDING_MODEL_NAME} + input_transforms: + - type: copy + source_value: topic:demo/rag/data + dest_expression: user_data.vector_input:metadatas.source + - type: copy + source_expression: input.payload:texts + dest_expression: user_data.vector_input:texts + input_selection: + source_expression: user_data.vector_input +``` +This component uses `langchain_vector_store_embedding_index` to handle embedding logic which is a built-in component for adding and deleting embeddings to a vector database. It takes the input texts, converts them into embeddings using the OpenAI embedding model, and stores the embeddings in **ChromaDB**. ChromaDB is set to persist data in the `./chroma_data` directory. + +The data from the solace input broker is first transformed into the shape that we expect in `input_transforms`. The `input_selection` section then selects the transformed data to be used as input for the embedding process. + + +### RAG Inference Flow (Query and Response) + +This flow handles the **Retrieval-Augmented Generation (RAG)** process where a query is sent, relevant documents are retrieved, and a response is generated using OpenAI's models. + +#### 1. Query Ingestion from Solace Topic +```yaml +- component_name: solace_completion_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_queue_name: demo_rag_query + broker_subscriptions: + - topic: demo/rag/query +``` +This component listens for queries on the `demo/rag/query` Solace topic. The query is received as JSON data, and the Solace broker delivers it to the next step. + +#### 2. ChromaDB Search for Relevant Documents +```yaml +- component_name: chroma_search + component_module: langchain_vector_store_embedding_search + component_config: + vector_store_component_path: langchain_chroma + vector_store_component_name: Chroma + vector_store_component_config: + persist_directory: ./chroma_data + collection_name: rag + max_results: 5 +``` +This component searches ChromaDB for documents that are most similar to the query using the built-in `langchain_vector_store_embedding_search` component. It retrieves the top 5 results based on proximity in the vector space. + +#### 3. Response Generation Using OpenAI +```yaml +- component_name: llm_request + component_module: openai_chat_model + component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${OPENAI_MODEL_NAME} + temperature: 0.01 + input_transforms: + # Extract and format the retrieved data + - type: map + source_list_expression: previous:result + source_expression: | + template:{{text://item:text}}\n\n + dest_list_expression: user_data.retrieved_data + + - type: copy + source_expression: | + template:You are a helpful AI assistant. Using the provided context, help with the user's request below. Refrain to use any knowledge outside from the provided context. If the user query can not be answered using the provided context, reject user's query. + + + {{text://user_data.retrieved_data}} + + + + {{text://input.payload:query}} + + dest_expression: user_data.llm_input:messages.0.content + - type: copy + source_expression: static:user + dest_expression: user_data.llm_input:messages.0.role + input_selection: + source_expression: user_data.llm_input +``` +Once relevant documents are retrieved, we build the prompt with retrieved context. To prevent the model from hallucination, we ask it to refuse to answer if the answer is not provided in the given context. Then the **LLM (e.g., GPT-4o)** is used to generate a response. The temperature is set to `0.01`, meaning the response will be deterministic and focused on factual accuracy. The retrieved documents provide context for the LLM, which then generates a response based solely on this context. + +#### 4. Sending the Response Back to Solace + + +```yaml +- component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json +``` +The final component sends the generated response back to the Solace broker, specifically to the topic `demo/rag/query/response`, where the response can be consumed by the requesting application. + +## Flexibility in Components + +One of the key strengths of this architecture is its flexibility. Components like the **OpenAI connector** or **ChromaDB** can easily be swapped out for other AI service providers or vector databases. For example: +- Instead of OpenAI, you can use another LLM provider like **Cohere** or **Anthropic**. +- Instead of **ChromaDB**, you could use a different vector database like **Pinecone**, **Weaviate**, or **Milvus**. + +This modularity allows developers to adapt the system to different business requirements, AI services, and database solutions, providing greater flexibility and scalability. + +## Conclusion + +The Solace AI Connector, when combined with technologies like RAG, LLMs, and embeddings, enables a powerful AI-driven ecosystem for real-time applications. By ingesting data, embedding it into vector databases, and performing retrieval-augmented generation with LLMs, developers can build applications that provide accurate, context-aware responses to user queries fast. + +This YAML configuration serves as a template for setting up such an application, you can find the complete example in the [examples directory](../../examples/llm/openai_chroma_rag.yaml). \ No newline at end of file diff --git a/docs/guides/index.md b/docs/guides/index.md new file mode 100644 index 0000000..05dfdff --- /dev/null +++ b/docs/guides/index.md @@ -0,0 +1,4 @@ +In this directory you can find complete guides on varies topics on how to use the Solace AI Connector. + + +- [Building AI-Powered Applications with Solace AI Connector: A Deep Dive into RAG, LLMs, and Embeddings](./RAG.md) \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 6450165..6f1481e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,6 +10,7 @@ This connector application makes it easy to connect your AI/ML models to Solace - [Components](components/index.md) - [Transforms](transforms/index.md) - [Tips and Tricks](tips_and_tricks.md) +- [Guides](guides/index.md) - [Examples](../examples/) - [Contributing](../CONTRIBUTING.md) - [License](../LICENSE) From be10555a013f152124143c7f8688df8143b3a06d Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Fri, 20 Sep 2024 12:44:47 -0400 Subject: [PATCH 14/55] Added link to other docs from RAG guide (#40) --- docs/guides/RAG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/guides/RAG.md b/docs/guides/RAG.md index 6f3f3c7..218969d 100644 --- a/docs/guides/RAG.md +++ b/docs/guides/RAG.md @@ -229,4 +229,8 @@ This modularity allows developers to adapt the system to different business requ The Solace AI Connector, when combined with technologies like RAG, LLMs, and embeddings, enables a powerful AI-driven ecosystem for real-time applications. By ingesting data, embedding it into vector databases, and performing retrieval-augmented generation with LLMs, developers can build applications that provide accurate, context-aware responses to user queries fast. -This YAML configuration serves as a template for setting up such an application, you can find the complete example in the [examples directory](../../examples/llm/openai_chroma_rag.yaml). \ No newline at end of file +This YAML configuration serves as a template for setting up such an application, you can find the complete example in the [examples directory](../../examples/llm/openai_chroma_rag.yaml). + +## Want to Learn More About Solace AI Connector? + +Check out the [Solace AI Connector Overview](../overview.md) to explore its features in depth, or dive right in by following the [Getting Started Guide](../getting_started.md) to begin working with Solace AI Connector today! \ No newline at end of file From 3f0c99d02a82481e4e1b9340fdf810538c017321 Mon Sep 17 00:00:00 2001 From: Alireza Parvizimosaed Date: Fri, 20 Sep 2024 16:13:25 -0400 Subject: [PATCH 15/55] Improved the installation instruction --- README.md | 2 +- docs/getting_started.md | 26 +++++++++++++++++--------- requirements.txt | 7 +++++-- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 994a5c1..fab8aec 100644 --- a/README.md +++ b/README.md @@ -29,4 +29,4 @@ Contributions are encouraged! Please read [CONTRIBUTING](CONTRIBUTING.md) for de ## License -See the LICENSE file for details. +See the [LICENSE](LICENSE) file for details. diff --git a/docs/getting_started.md b/docs/getting_started.md index 53d847b..07939bb 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -4,7 +4,7 @@ This guide will help you get started with the Solace AI Event Connector. ## Prerequisites -- Python 3.10 or later +- Python 3.10.x, 3.11.x or 3.12.x - A Solace PubSub+ event broker - A chat model to connect to (optional) @@ -16,13 +16,14 @@ To get started with creating a solace PubSub+ event broker follow the instructio ### Install the connector -Optionally create a virtual environment: +(Optional) Create a virtual environment: ```sh python3 -m venv env source env/bin/activate ``` +Set up the connector package ```sh pip install solace-ai-connector ``` @@ -53,6 +54,12 @@ export SOLACE_BROKER_PASSWORD=default export SOLACE_BROKER_VPN=default ``` +(Optional) Store the environment variables permanently in ~/.profile file and activate them by: + +```sh +source ~/.profile +``` + Run the connector: ```sh @@ -83,10 +90,9 @@ export OPENAI_API_ENDPOINT= export MODEL_NAME= ``` -Note that if you want to use the default OpenAI endpoint, just delete that line from the openai_chat.yaml file. +Note that if you want to use the default OpenAI endpoint, just delete that line from the langchain_openai_with_history_chat.yaml file. Install the langchain openai dependencies: - ```sh pip install langchain_openai openai ``` @@ -113,7 +119,7 @@ Payload: In the "Try Me!" also subscribe to `demo/joke/subject/response` to see the response -## Installation +## Running From Source Code 1. Clone the repository and enter its directory: @@ -123,7 +129,7 @@ In the "Try Me!" also subscribe to `demo/joke/subject/response` to see the respo cd solace-ai-connector ``` -2. Optionally create a virtual environment: +2. (Optional) Create a virtual environment: ```sh python -m venv .venv @@ -136,11 +142,13 @@ In the "Try Me!" also subscribe to `demo/joke/subject/response` to see the respo pip install -r requirements.txt ``` -## Configuration +### Configuration -1. Edit the example configuration file at the root of the repository: +1. (Optional) Edit the example configuration file at the root of the repository: + ```sh config.yaml + ``` 2. Set up the environment variables that you need for the config.yaml file. The default one requires the following variables: @@ -152,7 +160,7 @@ In the "Try Me!" also subscribe to `demo/joke/subject/response` to see the respo ``` -## Running the AI Event Connector +### Running the AI Event Connector 1. Start the AI Event Connector: diff --git a/requirements.txt b/requirements.txt index 66ec399..8071e4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,9 @@ boto3~=1.34.122 -langchain-core~=0.3.0 -langchain~=0.3.0 +langchain-core~=0.2.5 +langchain~=0.2.3 PyYAML~=6.0.1 Requests~=2.32.3 solace_pubsubplus~=1.8.0 +solace_ai_connector~=0.1.5 +langchain-openai~=0.1.25 +openai~=1.47.0 \ No newline at end of file From 8074a7ff09a1e6041bc5c85a7381226c0bcd0f70 Mon Sep 17 00:00:00 2001 From: Alireza Parvizimosaed Date: Mon, 23 Sep 2024 10:18:07 -0400 Subject: [PATCH 16/55] Enhanced the documentation --- docs/configuration.md | 16 +++++++++++++--- docs/getting_started.md | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8c2b511..3266161 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -127,7 +127,7 @@ A flow is an instance of a pipeline that processes events in a sequential manner Flows can be communicating together if programmed to do so. For example, a flow can send a message to a broker and another flow can subscribe to the same topic to receive the message. -flows can be spread across multiple configuration files. The connector will merge the flows from all the files and run them together. +Flows can be spread across multiple configuration files. The connector will merge the flows from all the files and run them together. The `flows` section is a list of flow configurations. Each flow configuration is a dictionary with the following keys: @@ -135,6 +135,16 @@ following keys: - `name`: - The unique name of the flow - `components`: A list of component configurations. Check [Component Configuration](#component-configuration) for more details +```yaml + flows: + - name: + components: + - component_name: + - name: + components: + - component_name: +``` + ## Message Data Between each component in a flow, a message is passed. This message is a dictionary that is used to pass data between components within the same flow. The message object has different properties, some are available throughout the whole flow, some only between two immediate components, and some have other characteristics. @@ -153,7 +163,7 @@ This data type is available only after a topic subscription and then it will be - `previous`: The complete output of the previous component in the flow. This can be used to completely forward the output of the previous component as an input to the next component or be modified in the `input_transforms` section of the next component. -- transform specific variables: Some transforms function will add specific variables to the message object that are ONLY accessible in that transform. For example, the [`map` transform](./transforms/map.md) will add `item`, `index`, and `source_list` to the message object or the [`reduce` transform](./transforms/reduce.md) will add `accumulated_value`, `current_value`, and `source_list` to the message object. You can find these details in each transform documentation. +- Transform specific variables: Some transforms function will add specific variables to the message object that are ONLY accessible in that transform. For example, the [`map` transform](./transforms/map.md) will add `item`, `index`, and `source_list` to the message object or the [`reduce` transform](./transforms/reduce.md) will add `accumulated_value`, `current_value`, and `source_list` to the message object. You can find these details in each [transform](transforms/index.md) documentation. ## Expression Syntax @@ -570,4 +580,4 @@ You can find various usecase examples in the [examples directory](../examples/) --- -Checkout [components.md](./components/index.md), [transforms.md](./transforms/index.md), or [tips_and_tricks](tips_and_tricks.md) next. +Checkout [components](./components/index.md), [transforms](./transforms/index.md), or [tips_and_tricks](tips_and_tricks.md) next. diff --git a/docs/getting_started.md b/docs/getting_started.md index 07939bb..e58b119 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -184,4 +184,4 @@ make build --- -Checkout [configuration.md](configuration.md) or [overview.md](overview.md) next \ No newline at end of file +Checkout [configuration](configuration.md) or [overview](overview.md) next \ No newline at end of file From 2e1c1c6fb61c667bcb1645ed21e11c81101b9901 Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Mon, 23 Sep 2024 11:26:15 -0400 Subject: [PATCH 17/55] chore: added a timeout setting for running component tests so that you can test situations where you don't expect any output (#34) --- .../test_utils/utils_for_test_files.py | 18 ++++++++++++++++-- tests/test_transforms.py | 8 ++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/solace_ai_connector/test_utils/utils_for_test_files.py b/src/solace_ai_connector/test_utils/utils_for_test_files.py index fec9bad..1b38e98 100644 --- a/src/solace_ai_connector/test_utils/utils_for_test_files.py +++ b/src/solace_ai_connector/test_utils/utils_for_test_files.py @@ -87,6 +87,7 @@ def run_component_test( input_messages=None, input_selection=None, input_transforms=None, + max_response_timeout=None, ): if not input_data and not input_messages: raise ValueError("Either input_data or input_messages must be provided") @@ -125,7 +126,8 @@ def run_component_test( ], } ] - } + }, + queue_timeout=max_response_timeout, ) if input_data: @@ -135,10 +137,20 @@ def run_component_test( input_messages.append(message) # Send each message through, one at a time + output_data_list = [] + output_message_list = [] for message in input_messages: send_message_to_flow(flows[0], message) output_message = get_message_from_flow(flows[0]) - validation_func(output_message.get_previous(), output_message, message) + if not output_message: + # This only happens if the max_response_timeout is reached + output_message_list.append(None) + output_data_list.append(None) + continue + output_data_list.append(output_message.get_data("previous")) + output_message_list.append(output_message) + + validation_func(output_data_list, output_message_list, message) finally: if connector: @@ -194,6 +206,8 @@ def send_message_to_flow(flow_info, message): def get_message_from_flow(flow_info): output_component = flow_info["output_component"] event = output_component.get_output() + if not event: + return event if event.event_type != EventType.MESSAGE: raise ValueError("Expected a message event") return event.data diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 2efe69c..43e0fed 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -50,8 +50,8 @@ def test_transform_with_run_component_test(): """This test is actually testing the test infrastructure method: run_component_test""" def validation_func(output_data, output_message, _input_message): - assert output_data == "Hello, World!" - assert output_message.get_data("user_data.temp") == { + assert output_data[0] == "Hello, World!" + assert output_message[0].get_data("user_data.temp") == { "payload": {"text": "Hello, World!", "greeting": "Static Greeting!"} } @@ -79,8 +79,8 @@ def test_transform_with_run_component_test_with_static_import(): """This test is actually testing the test infrastructure method: run_component_test""" def validation_func(output_data, output_message, _input_message): - assert output_data == "Hello, World!" - assert output_message.get_data("user_data.temp") == { + assert output_data == ["Hello, World!"] + assert output_message[0].get_data("user_data.temp") == { "payload": {"text": "Hello, World!", "greeting": "Static Greeting!"} } From c2d77a43b544446b50d31b0f010a59624456c147 Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Mon, 23 Sep 2024 11:28:55 -0400 Subject: [PATCH 18/55] AI-124: Add a feature to provide simple blocking broker request/response ability for components (#42) * feat: add request_response_controller.py * feat: implement RequestResponseFlowManager and RequestResponseController classes * style: format code with black and improve readability * feat: implement RequestResponseController for flow-based request-response handling * feat: implement RequestResponseController for handling request-response patterns * fix: import SolaceAiConnector for type checking * refactor: restructure Flow class and improve code organization * feat: implement multiple named RequestResponseControllers per component * refactor: initialize request-response controllers in ComponentBase * test: add request_response_controller functionality tests * feat: finished implementation and added some tests * refactor: rename RequestResponseController to RequestResponseFlowController * refactor: rename RequestResponseController to RequestResponseFlowController * refactor: some name changes * fix: update test function names for RequestResponseFlowController * refactor: more name changes * Ed/req_resp_examples_and_fixes (#41) * feat: Added a request_response_flow example and fixed a few issues along the way * feat: Reworked the broker_request_response built-in ability of components to be simpler. Instead of having to have a defined flow and then name that flow, it will automatically create a flow with a single broker_request_response component in it. Now there is a straightforward interating function call to allow components to issue a request and get streaming or non-streaming responses from that flow. * chore: fix the request_response example and remove the old one * docs: add broker request-response configuration * docs: added advanced_component_features.md * docs: add broker request-response configuration details * docs: add payload encoding and format to broker config * docs: add cache service and timer manager to advanced_component_features.md * docs: add configuration requirement for broker request-response * docs: update broker request-response section with configuration info * docs: a bit more detail about do_broker_request_response * docs: add link to advanced features page in table of contents * docs: add link to advanced features page * docs: reorder table of contents in index.md * docs: add custom components documentation * docs: Remove advanced component features from table of contents * docs: clean up a double inclusion of the same section * docs: small example change * chore: remove dead code * chore: add some extra comments to explain some test code * docs: Update description of STDIN input component Update the description of the STDIN input component to clarify that it waits for its output message to be acknowledged before prompting for the next input. This change is made in the `stdin_input.py` file. * chore: add is_broker_request_response_enabled method * chore: Some changes after review --- docs/advanced_component_features.md | 120 +++++++++ docs/configuration.md | 33 ++- docs/custom_components.md | 90 +++++++ docs/index.md | 1 + examples/llm/custom_components/__init__.py | 0 .../llm_streaming_custom_component.py | 52 ++++ .../openai_component_request_response.yaml | 147 +++++++++++ src/solace_ai_connector/common/event.py | 5 +- .../components/component_base.py | 66 ++++- .../components/general/delay.py | 3 +- .../general/for_testing/handler_callback.py | 67 +++++ .../general/openai/openai_chat_model_base.py | 110 +++++++- .../components/inputs_outputs/broker_base.py | 9 +- .../components/inputs_outputs/broker_input.py | 3 +- .../inputs_outputs/broker_request_response.py | 164 +++++++++--- .../components/inputs_outputs/stdin_input.py | 34 ++- .../inputs_outputs/stdout_output.py | 19 +- src/solace_ai_connector/flow/flow.py | 16 +- .../flow/request_response_flow_controller.py | 156 +++++++++++ .../solace_ai_connector.py | 18 +- .../test_utils/utils_for_test_files.py | 2 + tests/test_request_response_controller.py | 244 ++++++++++++++++++ 22 files changed, 1289 insertions(+), 70 deletions(-) create mode 100644 docs/advanced_component_features.md create mode 100644 docs/custom_components.md create mode 100644 examples/llm/custom_components/__init__.py create mode 100644 examples/llm/custom_components/llm_streaming_custom_component.py create mode 100644 examples/llm/openai_component_request_response.yaml create mode 100644 src/solace_ai_connector/components/general/for_testing/handler_callback.py create mode 100644 src/solace_ai_connector/flow/request_response_flow_controller.py create mode 100644 tests/test_request_response_controller.py diff --git a/docs/advanced_component_features.md b/docs/advanced_component_features.md new file mode 100644 index 0000000..7053ff6 --- /dev/null +++ b/docs/advanced_component_features.md @@ -0,0 +1,120 @@ +# Advanced Component Features + +This document describes advanced features available to custom components in the Solace AI Connector. + +## Table of Contents +- [Broker Request-Response](#broker-request-response) +- [Cache Manager](#cache-manager) +- [Timer Features](#timer-features) + +## Broker Request-Response + +Components can perform a request and get a response from the broker using the `do_broker_request_response` method. This method supports both simple request-response and streamed responses. To use this feature, the component's configuration must include a `broker_request_response` section. For details on how to configure this section, refer to the [Broker Request-Response Configuration](configuration.md#broker-request-response-configuration) in the configuration documentation. + +This feature would be used in the invoke method of a custom component. When the `do_broker_request_response` method is called, the component will send a message to the broker and then block until a response (or a series of streamed chunks) is received. This makes it very easy to call services that are available via the broker. + +### Usage + +```python +response = self.do_broker_request_response(message, stream=False) +``` + +For streamed responses: + +```python +for chunk, is_last in self.do_broker_request_response(message, stream=True, streaming_complete_expression="input.payload:streaming.last_message"): + # Process each chunk + if is_last: + break +``` + +### Parameters + +- `message`: The message to send to the broker. This must have a topic and payload. +- `stream` (optional): Boolean indicating whether to expect a streamed response. Default is False. +- `streaming_complete_expression` (optional): An expression to evaluate on each response chunk to determine if it's the last one. This is required when `stream=True`. + +### Return Value + +- For non-streamed responses: Returns the response message. +- For streamed responses: Returns a generator that yields tuples of (chunk, is_last). Each chunk is a fully formed message with the format of the response. `is_last` is a boolean indicating if the chunk is the last one. + +## Memory Cache + +The cache service provides a flexible way to store and retrieve data with optional expiration. It supports different storage backends and offers features like automatic expiry checks. + +### Features + +1. Multiple storage backends: + - In-memory storage + - SQLAlchemy-based storage (for persistent storage) + +2. Key-value storage with metadata and expiry support +3. Automatic expiry checks in a background thread +4. Thread-safe operations + +### Usage + +Components can access the cache service through `self.cache_service`. Here are some common operations: + +```python +# Set a value with expiry +self.cache_service.set("key", "value", expiry=300) # Expires in 300 seconds + +# Get a value +value = self.cache_service.get("key") + +# Delete a value +self.cache_service.delete("key") + +# Get all values (including metadata and expiry) +all_data = self.cache_service.get_all() +``` + +### Configuration + +The cache service can be configured in the main configuration file: + +```yaml +cache: + backend: "memory" # or "sqlalchemy" + connection_string: "sqlite:///cache.db" # for SQLAlchemy backend +``` + +## Timer Features + +The timer manager allows components to schedule one-time or recurring timer events. This is useful for implementing delayed actions, periodic tasks, or timeouts. + +### Features + +1. One-time and recurring timers +2. Customizable timer IDs for easy management +3. Optional payloads for timer events + +### Usage + +Components can access the timer manager through `self.timer_manager`. Here are some common operations: + +```python +# Add a one-time timer +self.add_timer(delay_ms=5000, timer_id="my_timer", payload={"key": "value"}) + +# Add a recurring timer +self.add_timer(delay_ms=5000, timer_id="recurring_timer", interval_ms=10000, payload={"type": "recurring"}) + +# Cancel a timer +self.cancel_timer(timer_id="my_timer") +``` + +### Handling Timer Events + +To handle timer events, components should implement the `handle_timer_event` method: + +```python +def handle_timer_event(self, timer_data): + timer_id = timer_data["timer_id"] + payload = timer_data["payload"] + # Process the timer event +``` + +Timer events are automatically dispatched to the appropriate component by the timer manager. diff --git a/docs/configuration.md b/docs/configuration.md index 8c2b511..9394aa8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -238,7 +238,7 @@ Each component configuration is a dictionary with the following keys: - `input_selection`: - A `source_expression` or `source_value` to use as the input to the component. Check [Expression Syntax](#expression-syntax) for more details. [Optional: If not specified, the complete previous component output will be used] - `queue_depth`: - The depth of the input queue for the component. - `num_instances`: - The number of instances of the component to run (Starts multiple threads to process messages) - +- `broker_request_response`: - Configuration for the broker request-response functionality. [Optional] ### component_module @@ -359,6 +359,37 @@ The `queue_depth` is an integer that specifies the depth of the input queue for The `num_instances` is an integer that specifies the number of instances of the component to run. This is the number of threads that will be started to process messages from the input queue. By default, the number of instances is 1. +### Broker Request-Response Configuration + +The `broker_request_response` configuration allows components to perform request-response operations with a broker. It has the following structure: + +```yaml +broker_request_response: + enabled: + broker_config: + broker_type: + broker_url: + broker_username: + broker_password: + broker_vpn: + payload_encoding: + payload_format: + request_expiry_ms: +``` + +- `enabled`: Set to `true` to enable broker request-response functionality for the component. +- `broker_config`: Configuration for the broker connection. + - `broker_type`: Type of the broker (e.g., "solace"). + - `broker_url`: URL of the broker. + - `broker_username`: Username for broker authentication. + - `broker_password`: Password for broker authentication. + - `broker_vpn`: VPN name for the broker connection. + - `payload_encoding`: Encoding for the payload (e.g., "utf-8", "base64"). + - `payload_format`: Format of the payload (e.g., "json", "text"). +- `request_expiry_ms`: Expiry time for requests in milliseconds. + +For more details on using this functionality, see the [Advanced Component Features](advanced_component_features.md#broker-request-response) documentation. + ### Built-in components The AI Event Connector comes with a number of built-in components that can be used to process messages. For a list of all built-in components, see the [Components](components/index.md) documentation. diff --git a/docs/custom_components.md b/docs/custom_components.md new file mode 100644 index 0000000..f34da91 --- /dev/null +++ b/docs/custom_components.md @@ -0,0 +1,90 @@ +# Custom Components + +## Purpose + +Custom components provide a way to extend the functionality of the Solace AI Connector beyond what's possible with the built-in components and configuration options. Sometimes, it's easier and more efficient to add custom code than to build a complex configuration file, especially for specialized or unique processing requirements. + +## Requirements of a Custom Component + +To create a custom component, you need to follow these requirements: + +1. **Inherit from ComponentBase**: Your custom component class should inherit from the `ComponentBase` class. + +2. **Info Section**: Define an `info` dictionary with the following keys: + - `class_name`: The name of your custom component class. + - `config_parameters`: A list of dictionaries describing the configuration parameters for your component. + - `input_schema`: A dictionary describing the expected input schema. + - `output_schema`: A dictionary describing the expected output schema. + +3. **Implement the `invoke` method**: This is the main method where your component's logic will be implemented. + +Here's a basic template for a custom component: + +```python +from solace_ai_connector.components.component_base import ComponentBase + +info = { + "class_name": "MyCustomComponent", + "config_parameters": [ + { + "name": "my_param", + "type": "string", + "required": True, + "description": "A custom parameter" + } + ], + "input_schema": { + "type": "object", + "properties": { + "input_data": {"type": "string"} + } + }, + "output_schema": { + "type": "object", + "properties": { + "output_data": {"type": "string"} + } + } +} + +class MyCustomComponent(ComponentBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + self.my_param = self.get_config("my_param") + + def invoke(self, message, data): + # Your custom logic here + result = f"{self.my_param}: {data['input_data']}" + return {"output_data": result} +``` + +## Overrideable Methods + +While the `invoke` method is the main one you'll implement, there are several other methods you can override to customize your component's behavior: + +1. `invoke(self, message, data)`: The main processing method for your component. +2. `get_next_event(self)`: Customize how your component receives events. +3. `send_message(self, message)`: Customize how your component sends messages to the next component. +4. `handle_timer_event(self, timer_data)`: Handle timer events if your component uses timers. +5. `handle_cache_expiry_event(self, timer_data)`: Handle cache expiry events if your component uses the cache service. +6. `process_pre_invoke(self, message)`: Customize preprocessing before `invoke` is called. +7. `process_post_invoke(self, result, message)`: Customize postprocessing after `invoke` is called. + +## Advanced Features + +Custom components can take advantage of advanced features provided by the Solace AI Connector. These include: + +- Broker request-response functionality +- Cache services +- Timer management + +For more information on these advanced features and how to use them in your custom components, please refer to the [Advanced Component Features](advanced_component_features.md) documentation. + +By creating custom components, you can extend the Solace AI Connector to meet your specific needs while still benefiting from the framework's built-in capabilities for event processing, flow management, and integration with Solace event brokers. + +## Example + +See the [Tips and Tricks page](tips_and_tricks.md) for an example of creating a custom component. + + +[] \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 6f1481e..1b89909 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,6 +9,7 @@ This connector application makes it easy to connect your AI/ML models to Solace - [Configuration](configuration.md) - [Components](components/index.md) - [Transforms](transforms/index.md) +- [Custom Components](custom_components.md) - [Tips and Tricks](tips_and_tricks.md) - [Guides](guides/index.md) - [Examples](../examples/) diff --git a/examples/llm/custom_components/__init__.py b/examples/llm/custom_components/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/llm/custom_components/llm_streaming_custom_component.py b/examples/llm/custom_components/llm_streaming_custom_component.py new file mode 100644 index 0000000..a1363d4 --- /dev/null +++ b/examples/llm/custom_components/llm_streaming_custom_component.py @@ -0,0 +1,52 @@ +# A simple pass-through component - what goes in comes out + +import sys + +sys.path.append("src") + +from solace_ai_connector.components.component_base import ComponentBase +from solace_ai_connector.common.message import Message + + +info = { + "class_name": "LlmStreamingCustomComponent", + "description": "Do a blocking LLM request/response", + "config_parameters": [ + { + "name": "llm_request_topic", + "description": "The topic to send the request to", + "type": "string", + } + ], + "input_schema": { + "type": "object", + "properties": {}, + }, + "output_schema": { + "type": "object", + "properties": {}, + }, +} + + +class LlmStreamingCustomComponent(ComponentBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + self.llm_request_topic = self.get_config("llm_request_topic") + + def invoke(self, message, data): + llm_message = Message(payload=data, topic=self.llm_request_topic) + for message, last_message in self.do_broker_request_response( + llm_message, + stream=True, + streaming_complete_expression="input.payload:last_chunk", + ): + text = message.get_data("input.payload:chunk") + if not text: + text = message.get_data("input.payload:content") or "no response" + if last_message: + return {"chunk": text} + self.output_streaming(message, {"chunk": text}) + + def output_streaming(self, message, data): + return self.process_post_invoke(data, message) diff --git a/examples/llm/openai_component_request_response.yaml b/examples/llm/openai_component_request_response.yaml new file mode 100644 index 0000000..bb102a6 --- /dev/null +++ b/examples/llm/openai_component_request_response.yaml @@ -0,0 +1,147 @@ +# This example demostrates how to use the request_response_flow_controller to +# inject another flow into an existing flow. This is commonly used when +# you want to call a service that is only accessible via the broker. +# +# Main flow: STDIN -> llm_streaming_custom_component -> STDOUT +# | ^ +# v | +# do_broker_request_response() +# | ^ +# v | +# Broker Broker +# +# +# LLM flow: Broker -> OpenAI -> Broker +# +# +# While this looks a bit complicated, it allows you to very easily use all +# the benefits of the broker to distribute service requests, such as load +# balancing, failover, and scaling to LLMs. +# +# It will subscribe to `demo/question` and expect an event with the payload: +# +# The input message has the following schema: +# { +# "text": "" +# } +# +# It will then send an event back to Solace with the topic: `demo/question/response` +# +# Dependencies: +# pip install -U langchain_openai openai +# +# required ENV variables: +# - OPENAI_API_KEY +# - OPENAI_API_ENDPOINT +# - OPENAI_MODEL_NAME +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN + +--- +log: + stdout_log_level: INFO + log_file_level: DEBUG + log_file: solace_ai_connector.log + +shared_config: + - broker_config: &broker_connection + broker_type: solace + broker_url: ${SOLACE_BROKER_URL} + broker_username: ${SOLACE_BROKER_USERNAME} + broker_password: ${SOLACE_BROKER_PASSWORD} + broker_vpn: ${SOLACE_BROKER_VPN} + +# Take from input broker and publish back to Solace +flows: + # broker input processing + - name: main_flow + components: + # Input from a Solace broker + - component_name: input + component_module: stdin_input + + # Our custom component + - component_name: llm_streaming_custom_component + component_module: llm_streaming_custom_component + # Relative path to the component + component_base_path: examples/llm/custom_components + component_config: + llm_request_topic: example/llm/best + broker_request_response: + enabled: true + broker_config: *broker_connection + request_expiry_ms: 60000 + payload_encoding: utf-8 + payload_format: json + input_transforms: + - type: copy + source_expression: | + template:You are a helpful AI assistant. Please help with the user's request below: + + {{text://input.payload:text}} + + dest_expression: user_data.llm_input:messages.0.content + - type: copy + source_expression: static:user + dest_expression: user_data.llm_input:messages.0.role + input_selection: + source_expression: user_data.llm_input + + # Send response to stdout + - component_name: send_response + component_module: stdout_output + component_config: + add_new_line_between_messages: false + input_selection: + source_expression: previous:chunk + + + + # The LLM flow that is accessible via the broker + - name: llm_flow + components: + # Input from a Solace broker + - component_name: solace_sw_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_queue_name: example_flow_streaming + broker_subscriptions: + - topic: example/llm/best + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # Do an LLM request + - component_name: llm_request + component_module: openai_chat_model + component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${MODEL_NAME} + temperature: 0.01 + llm_mode: stream + stream_to_next_component: true + stream_batch_size: 20 + input_selection: + source_expression: input.payload + + # Send response back to broker + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.output:payload + - type: copy + source_expression: input.user_properties:__solace_ai_connector_broker_request_reply_topic__ + dest_expression: user_data.output:topic + input_selection: + source_expression: user_data.output \ No newline at end of file diff --git a/src/solace_ai_connector/common/event.py b/src/solace_ai_connector/common/event.py index cc302c2..98f4217 100644 --- a/src/solace_ai_connector/common/event.py +++ b/src/solace_ai_connector/common/event.py @@ -6,7 +6,10 @@ class EventType(Enum): MESSAGE = "message" TIMER = "timer" CACHE_EXPIRY = "cache_expiry" - # Add more event types as needed + # Add more event types as need + + def __eq__(self, other): + return self.value == other.value class Event: diff --git a/src/solace_ai_connector/components/component_base.py b/src/solace_ai_connector/components/component_base.py index bd4c52c..f059c06 100644 --- a/src/solace_ai_connector/components/component_base.py +++ b/src/solace_ai_connector/components/component_base.py @@ -10,6 +10,7 @@ from ..common.message import Message from ..common.trace_message import TraceMessage from ..common.event import Event, EventType +from ..flow.request_response_flow_controller import RequestResponseFlowController DEFAULT_QUEUE_TIMEOUT_MS = 200 DEFAULT_QUEUE_MAX_DEPTH = 5 @@ -34,6 +35,9 @@ def __init__(self, module_info, **kwargs): self.cache_service = kwargs.pop("cache_service", None) self.component_config = self.config.get("component_config") or {} + self.broker_request_response_config = self.config.get( + "broker_request_response", None + ) self.name = self.config.get("component_name", "") resolve_config_values(self.component_config) @@ -51,6 +55,7 @@ def __init__(self, module_info, **kwargs): self.validate_config() self.setup_transforms() self.setup_communications() + self.setup_broker_request_response() def create_thread_and_run(self): self.thread = threading.Thread(target=self.run) @@ -66,6 +71,8 @@ def run(self): if self.trace_queue: self.trace_event(event) self.process_event(event) + except AssertionError as e: + raise e except Exception as e: log.error( "%sComponent has crashed: %s\n%s", @@ -214,7 +221,15 @@ def get_config(self, key=None, default=None): val = self.component_config.get(key, None) if val is None: val = self.config.get(key, default) - if callable(val): + + # We reserve a few callable function names for internal use + # They are used for the handler_callback component which is used + # in testing (search the tests directory for example uses) + if callable(val) and key not in [ + "invoke_handler", + "get_next_event_handler", + "send_message_handler", + ]: if self.current_message is None: raise ValueError( f"Component {self.log_identifier} is trying to use an `invoke` config " @@ -262,6 +277,32 @@ def setup_communications(self): else: self.input_queue = queue.Queue(maxsize=self.queue_max_depth) + def setup_broker_request_response(self): + if ( + not self.broker_request_response_config + or not self.broker_request_response_config.get("enabled", False) + ): + self.broker_request_response_controller = None + return + broker_config = self.broker_request_response_config.get("broker_config", {}) + request_expiry_ms = self.broker_request_response_config.get( + "request_expiry_ms", 30000 + ) + if not broker_config: + raise ValueError( + f"Broker request response config not found for component {self.name}" + ) + rrc_config = { + "broker_config": broker_config, + "request_expiry_ms": request_expiry_ms, + } + self.broker_request_response_controller = RequestResponseFlowController( + config=rrc_config, connector=self.connector + ) + + def is_broker_request_response_enabled(self): + return self.broker_request_response_controller is not None + def setup_transforms(self): self.transforms = Transforms( self.config.get("input_transforms", []), log_identifier=self.log_identifier @@ -365,3 +406,26 @@ def cleanup(self): self.input_queue.get_nowait() except queue.Empty: break + + # This should be used to do an on-the-fly broker request response + def do_broker_request_response( + self, message, stream=False, streaming_complete_expression=None + ): + if self.broker_request_response_controller: + if stream: + return ( + self.broker_request_response_controller.do_broker_request_response( + message, stream, streaming_complete_expression + ) + ) + else: + generator = ( + self.broker_request_response_controller.do_broker_request_response( + message + ) + ) + next_message, last = next(generator, None) + return next_message + raise ValueError( + f"Broker request response controller not found for component {self.name}" + ) diff --git a/src/solace_ai_connector/components/general/delay.py b/src/solace_ai_connector/components/general/delay.py index d4a05d0..8d8aaf0 100644 --- a/src/solace_ai_connector/components/general/delay.py +++ b/src/solace_ai_connector/components/general/delay.py @@ -38,5 +38,6 @@ def __init__(self, **kwargs): super().__init__(info, **kwargs) def invoke(self, message, data): - sleep(self.get_config("delay")) + delay = self.get_config("delay") + sleep(delay) return deepcopy(data) diff --git a/src/solace_ai_connector/components/general/for_testing/handler_callback.py b/src/solace_ai_connector/components/general/for_testing/handler_callback.py new file mode 100644 index 0000000..12d0ea7 --- /dev/null +++ b/src/solace_ai_connector/components/general/for_testing/handler_callback.py @@ -0,0 +1,67 @@ +"""This test component allows a tester to configure callback handlers for + get_next_event, send_message and invoke methods""" + +from ...component_base import ComponentBase + + +info = { + "class_name": "HandlerCallback", + "description": ( + "This test component allows a tester to configure callback handlers for " + "get_next_event, send_message and invoke methods" + ), + "config_parameters": [ + { + "name": "get_next_event_handler", + "required": False, + "description": "The callback handler for the get_next_event method", + "type": "function", + }, + { + "name": "send_message_handler", + "required": False, + "description": "The callback handler for the send_message method", + "type": "function", + }, + { + "name": "invoke_handler", + "required": False, + "description": "The callback handler for the invoke method", + "type": "function", + }, + ], + "input_schema": { + "type": "object", + "properties": {}, + }, + "output_schema": { + "type": "object", + "properties": {}, + }, +} + + +class HandlerCallback(ComponentBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + self.get_next_event_handler = self.get_config("get_next_event_handler") + self.send_message_handler = self.get_config("send_message_handler") + self.invoke_handler = self.get_config("invoke_handler") + + def get_next_event(self): + if self.get_next_event_handler: + return self.get_next_event_handler(self) + else: + return super().get_next_event() + + def send_message(self, message): + if self.send_message_handler: + return self.send_message_handler(self, message) + else: + return super().send_message(message) + + def invoke(self, message, data): + if self.invoke_handler: + return self.invoke_handler(self, message, data) + else: + return super().invoke(message, data) diff --git a/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py b/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py index 6a0884b..0ce6c90 100644 --- a/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py +++ b/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py @@ -1,9 +1,10 @@ """Base class for OpenAI chat models""" +import uuid + from openai import OpenAI from ...component_base import ComponentBase from ....common.message import Message -import uuid openai_info_base = { "class_name": "OpenAIChatModelBase", @@ -34,13 +35,29 @@ { "name": "stream_to_flow", "required": False, - "description": "Name the flow to stream the output to - this must be configured for llm_mode='stream'.", + "description": ( + "Name the flow to stream the output to - this must be configured for " + "llm_mode='stream'. This is mutually exclusive with stream_to_next_component." + ), "default": "", }, + { + "name": "stream_to_next_component", + "required": False, + "description": ( + "Whether to stream the output to the next component in the flow. " + "This is mutually exclusive with stream_to_flow." + ), + "default": False, + }, { "name": "llm_mode", "required": False, - "description": "The mode for streaming results: 'sync' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response.", + "description": ( + "The mode for streaming results: 'sync' or 'stream'. 'stream' " + "will just stream the results to the named flow. 'none' will " + "wait for the full response." + ), "default": "none", }, { @@ -52,7 +69,11 @@ { "name": "set_response_uuid_in_user_properties", "required": False, - "description": "Whether to set the response_uuid in the user_properties of the input_message. This will allow other components to correlate streaming chunks with the full response.", + "description": ( + "Whether to set the response_uuid in the user_properties of the " + "input_message. This will allow other components to correlate " + "streaming chunks with the full response." + ), "default": False, "type": "boolean", }, @@ -90,8 +111,6 @@ } -import uuid - class OpenAIChatModelBase(ComponentBase): def __init__(self, module_info, **kwargs): super().__init__(module_info, **kwargs) @@ -101,16 +120,22 @@ def init(self): self.model = self.get_config("model") self.temperature = self.get_config("temperature") self.stream_to_flow = self.get_config("stream_to_flow") + self.stream_to_next_component = self.get_config("stream_to_next_component") self.llm_mode = self.get_config("llm_mode") self.stream_batch_size = self.get_config("stream_batch_size") - self.set_response_uuid_in_user_properties = self.get_config("set_response_uuid_in_user_properties") + self.set_response_uuid_in_user_properties = self.get_config( + "set_response_uuid_in_user_properties" + ) + if self.stream_to_flow and self.stream_to_next_component: + raise ValueError( + "stream_to_flow and stream_to_next_component are mutually exclusive" + ) def invoke(self, message, data): messages = data.get("messages", []) client = OpenAI( - api_key=self.get_config("api_key"), - base_url=self.get_config("base_url") + api_key=self.get_config("api_key"), base_url=self.get_config("base_url") ) if self.llm_mode == "stream": @@ -134,7 +159,7 @@ def invoke_stream(self, client, message, messages): messages=messages, model=self.model, temperature=self.temperature, - stream=True + stream=True, ): if chunk.choices[0].delta.content is not None: content = chunk.choices[0].delta.content @@ -148,11 +173,31 @@ def invoke_stream(self, client, message, messages): aggregate_result, response_uuid, first_chunk, - False + False, + ) + elif self.stream_to_next_component: + self.send_to_next_component( + message, + current_batch, + aggregate_result, + response_uuid, + first_chunk, + False, ) current_batch = "" first_chunk = False + if self.stream_to_next_component: + # Just return the last chunk + return { + "content": aggregate_result, + "chunk": current_batch, + "uuid": response_uuid, + "first_chunk": first_chunk, + "last_chunk": True, + "streaming": True, + } + if self.stream_to_flow: self.send_streaming_message( message, @@ -160,12 +205,20 @@ def invoke_stream(self, client, message, messages): aggregate_result, response_uuid, first_chunk, - True + True, ) return {"content": aggregate_result, "uuid": response_uuid} - def send_streaming_message(self, input_message, chunk, aggregate_result, response_uuid, first_chunk=False, last_chunk=False): + def send_streaming_message( + self, + input_message, + chunk, + aggregate_result, + response_uuid, + first_chunk=False, + last_chunk=False, + ): message = Message( payload={ "chunk": chunk, @@ -177,3 +230,34 @@ def send_streaming_message(self, input_message, chunk, aggregate_result, respons user_properties=input_message.get_user_properties(), ) self.send_to_flow(self.stream_to_flow, message) + + def send_to_next_component( + self, + input_message, + chunk, + aggregate_result, + response_uuid, + first_chunk=False, + last_chunk=False, + ): + message = Message( + payload={ + "chunk": chunk, + "aggregate_result": aggregate_result, + "response_uuid": response_uuid, + "first_chunk": first_chunk, + "last_chunk": last_chunk, + }, + user_properties=input_message.get_user_properties(), + ) + + result = { + "content": aggregate_result, + "chunk": chunk, + "uuid": response_uuid, + "first_chunk": first_chunk, + "last_chunk": last_chunk, + "streaming": True, + } + + self.process_post_invoke(result, message) diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_base.py b/src/solace_ai_connector/components/inputs_outputs/broker_base.py index b6fd113..fac4207 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_base.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_base.py @@ -37,9 +37,12 @@ class BrokerBase(ComponentBase): def __init__(self, module_info, **kwargs): super().__init__(module_info, **kwargs) self.broker_properties = self.get_broker_properties() - self.messaging_service = ( - MessagingServiceBuilder().from_properties(self.broker_properties).build() - ) + if self.broker_properties["broker_type"] not in ["test", "test_streaming"]: + self.messaging_service = ( + MessagingServiceBuilder() + .from_properties(self.broker_properties) + .build() + ) self.current_broker_message = None self.messages_to_ack = [] self.connected = False diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_input.py b/src/solace_ai_connector/components/inputs_outputs/broker_input.py index 841c8ee..3aabd8d 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_input.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_input.py @@ -44,7 +44,8 @@ { "name": "temporary_queue", "required": False, - "description": "Whether to create a temporary queue that will be deleted after disconnection, defaulted to True if broker_queue_name is not provided", + "description": "Whether to create a temporary queue that will be deleted " + "after disconnection, defaulted to True if broker_queue_name is not provided", "default": False, }, { diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py b/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py index 3e4a2cd..b363776 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py @@ -3,6 +3,8 @@ import threading import uuid import json +import queue +from copy import deepcopy # from typing import Dict, Any @@ -63,6 +65,19 @@ "description": "Expiry time for cached requests in milliseconds", "default": 60000, }, + { + "name": "streaming", + "required": False, + "description": "The response will arrive in multiple pieces. If True, " + "the streaming_complete_expression must be set and will be used to " + "determine when the last piece has arrived.", + }, + { + "name": "streaming_complete_expression", + "required": False, + "description": "The source expression to determine when the last piece of a " + "streaming response has arrived.", + }, ], "input_schema": { "type": "object", @@ -79,6 +94,16 @@ "type": "object", "description": "User properties to send with the request message", }, + "stream": { + "type": "boolean", + "description": "Whether this will have a streaming response", + "default": False, + }, + "streaming_complete_expression": { + "type": "string", + "description": "Expression to determine when the last piece of a " + "streaming response has arrived. Required if stream is True.", + }, }, "required": ["payload", "topic"], }, @@ -115,6 +140,11 @@ def __init__(self, **kwargs): self.reply_queue_name = f"reply-queue-{uuid.uuid4()}" self.reply_topic = f"reply/{uuid.uuid4()}" self.response_thread = None + self.streaming = self.get_config("streaming") + self.streaming_complete_expression = self.get_config( + "streaming_complete_expression" + ) + self.broker_type = self.broker_properties.get("broker_type", "solace") self.broker_properties["temporary_queue"] = True self.broker_properties["queue_name"] = self.reply_queue_name self.broker_properties["subscriptions"] = [ @@ -123,7 +153,13 @@ def __init__(self, **kwargs): "qos": 1, } ] - self.connect() + self.test_mode = False + + if self.broker_type == "solace": + self.connect() + elif self.broker_type == "test" or self.broker_type == "test_streaming": + self.test_mode = True + self.setup_test_pass_through() self.start() def start(self): @@ -135,8 +171,16 @@ def setup_reply_queue(self): self.reply_queue_name, [self.reply_topic], temporary=True ) + def setup_test_pass_through(self): + self.pass_through_queue = queue.Queue() + def start_response_thread(self): - self.response_thread = threading.Thread(target=self.handle_responses) + if self.test_mode: + self.response_thread = threading.Thread( + target=self.handle_test_pass_through + ) + else: + self.response_thread = threading.Thread(target=self.handle_responses) self.response_thread.start() def handle_responses(self): @@ -148,61 +192,76 @@ def handle_responses(self): except Exception as e: log.error("Error handling response: %s", e) + def handle_test_pass_through(self): + while not self.stop_signal.is_set(): + try: + message = self.pass_through_queue.get(timeout=1) + decoded_payload = self.decode_payload(message.get_payload()) + message.set_payload(decoded_payload) + self.process_response(message) + except queue.Empty: + continue + except Exception as e: + log.error("Error handling test passthrough: %s", e) + def process_response(self, broker_message): - payload = broker_message.get_payload_as_string() - if payload is None: - payload = broker_message.get_payload_as_bytes() - payload = self.decode_payload(payload) - topic = broker_message.get_destination_name() - user_properties = broker_message.get_properties() + if self.test_mode: + payload = broker_message.get_payload() + topic = broker_message.get_topic() + user_properties = broker_message.get_user_properties() + else: + payload = broker_message.get_payload_as_string() + if payload is None: + payload = broker_message.get_payload_as_bytes() + payload = self.decode_payload(payload) + topic = broker_message.get_destination_name() + user_properties = broker_message.get_properties() metadata_json = user_properties.get( "__solace_ai_connector_broker_request_reply_metadata__" ) if not metadata_json: - log.warning("Received response without metadata: %s", payload) + log.error("Received response without metadata: %s", payload) return try: metadata_stack = json.loads(metadata_json) except json.JSONDecodeError: - log.warning( - "Received response with invalid metadata JSON: %s", metadata_json - ) + log.error("Received response with invalid metadata JSON: %s", metadata_json) return if not metadata_stack: - log.warning("Received response with empty metadata stack: %s", payload) + log.error("Received response with empty metadata stack: %s", payload) return try: current_metadata = metadata_stack.pop() except IndexError: - log.warning( + log.error( "Received response with invalid metadata stack: %s", metadata_stack ) return request_id = current_metadata.get("request_id") if not request_id: - log.warning("Received response without request_id in metadata: %s", payload) + log.error("Received response without request_id in metadata: %s", payload) return cached_request = self.cache_service.get_data(request_id) if not cached_request: - log.warning("Received response for unknown request_id: %s", request_id) + log.error("Received response for unknown request_id: %s", request_id) return + stream = cached_request.get("stream", False) + streaming_complete_expression = cached_request.get( + "streaming_complete_expression" + ) + response = { "payload": payload, "topic": topic, "user_properties": user_properties, } - result = { - "request": cached_request, - "response": response, - } - # Update the metadata in the response if metadata_stack: response["user_properties"][ @@ -221,8 +280,23 @@ def process_response(self, broker_message): "__solace_ai_connector_broker_request_reply_topic__", None ) - self.process_post_invoke(result, Message(payload=result)) - self.cache_service.remove_data(request_id) + message = Message( + payload=payload, + user_properties=user_properties, + topic=topic, + ) + self.process_post_invoke(response, message) + + # Only remove the cache entry if this isn't a streaming response or + # if it is the last piece of a streaming response + last_piece = True + if stream and streaming_complete_expression: + is_last = message.get_data(streaming_complete_expression) + if not is_last: + last_piece = False + + if last_piece: + self.cache_service.remove_data(request_id) def invoke(self, message, data): request_id = str(uuid.uuid4()) @@ -230,6 +304,12 @@ def invoke(self, message, data): if "user_properties" not in data: data["user_properties"] = {} + stream = False + if "stream" in data: + stream = data["stream"] + if "streaming_complete_expression" in data: + streaming_complete_expression = data["streaming_complete_expression"] + metadata = {"request_id": request_id, "reply_topic": self.reply_topic} if ( @@ -266,13 +346,39 @@ def invoke(self, message, data): "__solace_ai_connector_broker_request_reply_topic__" ] = self.reply_topic - encoded_payload = self.encode_payload(data["payload"]) + if self.test_mode: + if self.broker_type == "test_streaming": + # The payload should be an array. Send one message per item in the array + if not isinstance(data["payload"], list): + raise ValueError("Payload must be a list for test_streaming broker") + for item in data["payload"]: + encoded_payload = self.encode_payload(item) + self.pass_through_queue.put( + Message( + payload=encoded_payload, + user_properties=deepcopy(data["user_properties"]), + topic=data["topic"], + ) + ) + else: + encoded_payload = self.encode_payload(data["payload"]) + self.pass_through_queue.put( + Message( + payload=encoded_payload, + user_properties=data["user_properties"], + topic=data["topic"], + ) + ) + else: + encoded_payload = self.encode_payload(data["payload"]) + self.messaging_service.send_message( + destination_name=data["topic"], + payload=encoded_payload, + user_properties=data["user_properties"], + ) - self.messaging_service.send_message( - destination_name=data["topic"], - payload=encoded_payload, - user_properties=data["user_properties"], - ) + data["stream"] = stream + data["streaming_complete_expression"] = streaming_complete_expression self.cache_service.add_data( key=request_id, diff --git a/src/solace_ai_connector/components/inputs_outputs/stdin_input.py b/src/solace_ai_connector/components/inputs_outputs/stdin_input.py index a4fb83e..568333c 100644 --- a/src/solace_ai_connector/components/inputs_outputs/stdin_input.py +++ b/src/solace_ai_connector/components/inputs_outputs/stdin_input.py @@ -1,5 +1,7 @@ # An input component that reads from STDIN +import threading + from copy import deepcopy from ..component_base import ComponentBase from ...common.message import Message @@ -9,9 +11,17 @@ "class_name": "Stdin", "description": ( "STDIN input component. The component will prompt for " - "input, which will then be placed in the message payload using the output schema below." + "input, which will then be placed in the message payload using the output schema below. " + "The component will wait for its output message to be acknowledged before prompting for " + "the next input." ), - "config_parameters": [], + "config_parameters": [ + { + "name": "prompt", + "required": False, + "description": "The prompt to display when asking for input", + } + ], "output_schema": { "type": "object", "properties": { @@ -27,16 +37,28 @@ class Stdin(ComponentBase): def __init__(self, **kwargs): super().__init__(info, **kwargs) + self.need_acknowledgement = True + self.next_input_signal = threading.Event() + self.next_input_signal.set() def get_next_message(self): + # Wait for the next input signal + self.next_input_signal.wait() + + # Reset the event for the next use + self.next_input_signal.clear() + # Get the next message from STDIN - obj = {"text": input(self.config.get("prompt", "Enter text: "))} + obj = {"text": input(self.config.get("prompt", "\nEnter text: "))} # Create and return a message object return Message(payload=obj) - # def get_input_data(self, message): - # return message.payload - def invoke(self, message, data): return deepcopy(message.get_payload()) + + def acknowledge_message(self): + self.next_input_signal.set() + + def get_acknowledgement_callback(self): + return self.acknowledge_message diff --git a/src/solace_ai_connector/components/inputs_outputs/stdout_output.py b/src/solace_ai_connector/components/inputs_outputs/stdout_output.py index 0309f1f..edba4aa 100644 --- a/src/solace_ai_connector/components/inputs_outputs/stdout_output.py +++ b/src/solace_ai_connector/components/inputs_outputs/stdout_output.py @@ -6,7 +6,15 @@ info = { "class_name": "Stdout", "description": "STDOUT output component", - "config_parameters": [], + "config_parameters": [ + { + "name": "add_new_line_between_messages", + "required": False, + "description": "Add a new line between messages", + "type": "boolean", + "default": True, + } + ], "input_schema": { "type": "object", "properties": { @@ -22,8 +30,15 @@ class Stdout(ComponentBase): def __init__(self, **kwargs): super().__init__(info, **kwargs) + self.add_newline = self.get_config("add_new_line_between_messages") def invoke(self, message, data): # Print the message to STDOUT - print(yaml.dump(data)) + if isinstance(data, dict) or isinstance(data, list): + print(yaml.dump(data)) + else: + print(data, end="") + if self.add_newline: + print() + return data diff --git a/src/solace_ai_connector/flow/flow.py b/src/solace_ai_connector/flow/flow.py index 782c7ce..ea5091c 100644 --- a/src/solace_ai_connector/flow/flow.py +++ b/src/solace_ai_connector/flow/flow.py @@ -66,6 +66,9 @@ def __init__( self.cache_service = connector.cache_service if connector else None self.create_components() + def get_input_queue(self): + return self.flow_input_queue + def create_components(self): # Loop through the components and create them for index, component in enumerate(self.flow_config.get("components", [])): @@ -77,14 +80,15 @@ def create_components(self): for component in component_group: component.set_next_component(self.component_groups[index + 1][0]) + self.flow_input_queue = self.component_groups[0][0].get_input_queue() + + def run(self): # Now one more time to create threads and run them - for index, component_group in enumerate(self.component_groups): + for _index, component_group in enumerate(self.component_groups): for component in component_group: thread = component.create_thread_and_run() self.threads.append(thread) - self.flow_input_queue = self.component_groups[0][0].get_input_queue() - def create_component_group(self, component, index): component_module = component.get("component_module", "") base_path = component.get("component_base_path", None) @@ -133,6 +137,12 @@ def create_component_group(self, component, index): def get_flow_input_queue(self): return self.flow_input_queue + # This will set the next component in all the components in the + # last component group + def set_next_component(self, component): + for comp in self.component_groups[-1]: + comp.set_next_component(component) + def wait_for_threads(self): for thread in self.threads: thread.join() diff --git a/src/solace_ai_connector/flow/request_response_flow_controller.py b/src/solace_ai_connector/flow/request_response_flow_controller.py new file mode 100644 index 0000000..36dda71 --- /dev/null +++ b/src/solace_ai_connector/flow/request_response_flow_controller.py @@ -0,0 +1,156 @@ +""" +This file will handle sending a message to a named flow and then +receiving the output message from that flow. It will also support the result +message being a streamed message that comes in multiple parts. + +Each component can optionally create multiple of these using the configuration: + +```yaml +- name: example_flow + components: + - component_name: example_component + component_module: custom_component + request_response_flow_controllers: + - name: example_controller + flow_name: llm_flow + streaming: true + streaming_complete_expression: input.payload:streaming.last_message + request_expiry_ms: 300000 +``` + +""" + +import queue +import time +from typing import Dict, Any + +from ..common.message import Message +from ..common.event import Event, EventType + + +# This is a very basic component which will be stitched onto the final component in the flow +class RequestResponseControllerOuputComponent: + def __init__(self, controller): + self.controller = controller + + def enqueue(self, event): + self.controller.enqueue_response(event) + + +# This is the main class that will be used to send messages to a flow and receive the response +class RequestResponseFlowController: + def __init__(self, config: Dict[str, Any], connector): + self.config = config + self.connector = connector + self.broker_config = config.get("broker_config") + self.request_expiry_ms = config.get("request_expiry_ms", 300000) + self.request_expiry_s = self.request_expiry_ms / 1000 + self.input_queue = None + self.response_queue = None + self.enqueue_time = None + self.request_outstanding = False + + self.flow = self.create_broker_request_response_flow() + self.setup_queues(self.flow) + self.flow.run() + + def create_broker_request_response_flow(self): + self.broker_config["request_expiry_ms"] = self.request_expiry_ms + config = { + "name": "_internal_broker_request_response_flow", + "components": [ + { + "component_name": "_internal_broker_request_response", + "component_module": "broker_request_response", + "component_config": self.broker_config, + } + ], + } + return self.connector.create_flow(flow=config, index=0, flow_instance_index=0) + + def setup_queues(self, flow): + # Input queue to send the message to the flow + self.input_queue = flow.get_input_queue() + + # Response queue to receive the response from the flow + self.response_queue = queue.Queue() + rrcComponent = RequestResponseControllerOuputComponent(self) + flow.set_next_component(rrcComponent) + + def do_broker_request_response( + self, request_message, stream=False, streaming_complete_expression=None + ): + # Send the message to the broker + self.send_message(request_message, stream, streaming_complete_expression) + + # Now we will wait for the response + now = time.time() + elapsed_time = now - self.enqueue_time + remaining_timeout = self.request_expiry_s - elapsed_time + if stream: + # If we are in streaming mode, we will return individual messages + # until we receive the last message. Use the expression to determine + # if this is the last message + while True: + try: + event = self.response_queue.get(timeout=remaining_timeout) + if event.event_type == EventType.MESSAGE: + message = event.data + last_message = message.get_data(streaming_complete_expression) + yield message, last_message + if last_message: + return + except queue.Empty: + if (time.time() - self.enqueue_time) > self.request_expiry_s: + raise TimeoutError( # pylint: disable=raise-missing-from + "Timeout waiting for response" + ) + except Exception as e: + raise e + + now = time.time() + elapsed_time = now - self.enqueue_time + remaining_timeout = self.request_expiry_s - elapsed_time + + # If we are not in streaming mode, we will return a single message + # and then stop the iterator + try: + event = self.response_queue.get(timeout=remaining_timeout) + if event.event_type == EventType.MESSAGE: + message = event.data + yield message, True + return + except queue.Empty: + if (time.time() - self.enqueue_time) > self.request_expiry_s: + raise TimeoutError( # pylint: disable=raise-missing-from + "Timeout waiting for response" + ) + except Exception as e: + raise e + + def send_message( + self, message: Message, stream=False, streaming_complete_expression=None + ): + # Make a new message, but copy the data from the original message + if not self.input_queue: + raise ValueError(f"Input queue for flow {self.flow.name} not found") + + # Need to set the previous object to the required input for the + # broker_request_response component + message.set_previous( + { + "payload": message.get_payload(), + "user_properties": message.get_user_properties(), + "topic": message.get_topic(), + "stream": stream, + "streaming_complete_expression": streaming_complete_expression, + }, + ) + + event = Event(EventType.MESSAGE, message) + self.enqueue_time = time.time() + self.request_outstanding = True + self.input_queue.put(event) + + def enqueue_response(self, event): + self.response_queue.put(event) diff --git a/src/solace_ai_connector/solace_ai_connector.py b/src/solace_ai_connector/solace_ai_connector.py index f41f50f..40240cc 100644 --- a/src/solace_ai_connector/solace_ai_connector.py +++ b/src/solace_ai_connector/solace_ai_connector.py @@ -62,6 +62,8 @@ def create_flows(self): flow_input_queue = flow_instance.get_flow_input_queue() self.flow_input_queues[flow.get("name")] = flow_input_queue self.flows.append(flow_instance) + for flow in self.flows: + flow.run() def create_flow(self, flow: dict, index: int, flow_instance_index: int): """Create a single flow""" @@ -98,15 +100,6 @@ def wait_for_flows(self): self.stop() self.cleanup() - def stop(self): - """Stop the Solace AI Event Connector""" - log.info("Stopping Solace AI Event Connector") - self.stop_signal.set() - self.timer_manager.stop() # Stop the timer manager first - self.wait_for_flows() - if self.trace_thread: - self.trace_thread.join() - def cleanup(self): """Clean up resources and ensure all threads are properly joined""" log.info("Cleaning up Solace AI Event Connector") @@ -202,6 +195,13 @@ def get_flows(self): """Return the flows""" return self.flows + def get_flow(self, flow_name): + """Return a specific flow by name""" + for flow in self.flows: + if flow.name == flow_name: + return flow + return None + def setup_cache_service(self): """Setup the cache service""" cache_config = self.config.get("cache", {}) diff --git a/src/solace_ai_connector/test_utils/utils_for_test_files.py b/src/solace_ai_connector/test_utils/utils_for_test_files.py index 1b38e98..2b3421b 100644 --- a/src/solace_ai_connector/test_utils/utils_for_test_files.py +++ b/src/solace_ai_connector/test_utils/utils_for_test_files.py @@ -168,6 +168,8 @@ def create_test_flows( # For each of the flows, add the input and output components flow_info = [] for flow in flows: + if flow.flow_config.get("test_ignore", False): + continue input_component = TestInputComponent( flow.component_groups[0][0].get_input_queue() ) diff --git a/tests/test_request_response_controller.py b/tests/test_request_response_controller.py new file mode 100644 index 0000000..6ca6073 --- /dev/null +++ b/tests/test_request_response_controller.py @@ -0,0 +1,244 @@ +import sys + +sys.path.append("src") + +from solace_ai_connector.test_utils.utils_for_test_files import ( + create_test_flows, + dispose_connector, + send_message_to_flow, + get_message_from_flow, +) +from solace_ai_connector.common.message import Message + + +def test_request_response_flow_controller_basic(): + """Test basic functionality of the RequestResponseFlowController""" + + def test_invoke_handler(component, message, _data): + # Call the request_response + message = component.do_broker_request_response(message) + try: + assert message.get_data("previous") == { + "payload": {"text": "Hello, World!"}, + "topic": None, + "user_properties": {}, + } + except AssertionError as e: + return e + return "Pass" + + config = { + "flows": [ + { + "name": "test_flow", + "components": [ + { + "component_name": "requester", + "component_module": "handler_callback", + "component_config": { + "invoke_handler": test_invoke_handler, + }, + "broker_request_response": { + "enabled": True, + "broker_config": { + "broker_type": "test", + "broker_url": "test", + "broker_username": "test", + "broker_password": "test", + "broker_vpn": "test", + "payload_encoding": "utf-8", + "payload_format": "json", + }, + "request_expiry_ms": 500000, + }, + } + ], + }, + ] + } + connector, flows = create_test_flows(config) + + test_flow = flows[0] + + try: + + # Send a message to the input flow + send_message_to_flow(test_flow, Message(payload={"text": "Hello, World!"})) + + # Get the output message + output_message = get_message_from_flow(test_flow) + + result = output_message.get_data("previous") + + # if the result is an AssertionError, then raise it + if isinstance(result, AssertionError): + raise result + + assert result == "Pass" + + except Exception as e: + print(e) + assert False + + finally: + dispose_connector(connector) + + +# Test simple streaming request response +# Use the iterate component to break a single message into multiple messages +def test_request_response_flow_controller_streaming(): + """Test streaming functionality of the RequestResponseFlowController""" + + def test_invoke_handler(component, message, data): + result = [] + for message, last_message in component.do_broker_request_response( + message, stream=True, streaming_complete_expression="input.payload:last" + ): + payload = message.get_data("input.payload") + result.append(payload) + if last_message: + assert payload == {"text": "Chunk3", "last": True} + + assert result == [ + {"text": "Chunk1", "last": False}, + {"text": "Chunk2", "last": False}, + {"text": "Chunk3", "last": True}, + ] + + return "Pass" + + config = { + "flows": [ + { + "name": "test_flow", + "components": [ + { + "component_name": "requester", + "component_module": "handler_callback", + "component_config": { + "invoke_handler": test_invoke_handler, + }, + "broker_request_response": { + "enabled": True, + "broker_config": { + "broker_type": "test_streaming", + "broker_url": "test", + "broker_username": "test", + "broker_password": "test", + "broker_vpn": "test", + "payload_encoding": "utf-8", + "payload_format": "json", + }, + "request_expiry_ms": 500000, + }, + } + ], + }, + ] + } + connector, flows = create_test_flows(config) + + test_flow = flows[0] + + try: + + # Send a message to the input flow + send_message_to_flow( + test_flow, + Message( + payload=[ + {"text": "Chunk1", "last": False}, + {"text": "Chunk2", "last": False}, + {"text": "Chunk3", "last": True}, + ] + ), + ) + + # Get the output message + output_message = get_message_from_flow(test_flow) + + assert output_message.get_data("previous") == "Pass" + + except Exception as e: + print(e) + assert False + + finally: + dispose_connector(connector) + + +# Test the timeout functionality +def test_request_response_flow_controller_timeout(): + """Test timeout functionality of the RequestResponseFlowController""" + + def test_invoke_handler(component, message, data): + # # Call the request_response_flow + # data_iter = component.send_request_response_flow_message( + # "test_controller", message, {"test": "data"} + # ) + + # # This will timeout + # try: + # for message, data, _last_message in data_iter(): + # assert message.get_data("previous") == {"test": "data"} + # assert message.get_data("input.payload") == {"text": "Hello, World!"} + # except TimeoutError: + # return "timeout" + # return "done" + + # Do it the new way + try: + for message, _last_message in component.do_broker_request_response( + message, stream=True, streaming_complete_expression="input.payload:last" + ): + pass + except TimeoutError: + return "Timeout" + return "Fail" + + config = { + "flows": [ + { + "name": "test_flow", + "components": [ + { + "component_name": "requester", + "component_module": "handler_callback", + "component_config": { + "invoke_handler": test_invoke_handler, + }, + "broker_request_response": { + "enabled": True, + "broker_config": { + "broker_type": "test_streaming", + "broker_url": "test", + "broker_username": "test", + "broker_password": "test", + "broker_vpn": "test", + "payload_encoding": "utf-8", + "payload_format": "json", + }, + "request_expiry_ms": 2000, + }, + } + ], + }, + ] + } + connector, flows = create_test_flows(config) + + test_flow = flows[0] + + try: + + # Send a message with an empty list in the payload to the test_streaming broker type + # This will not send any chunks and should timeout + send_message_to_flow(test_flow, Message(payload=[])) + + # Get the output message + output_message = get_message_from_flow(test_flow) + + assert output_message.get_data("previous") == "Timeout" + + finally: + dispose_connector(connector) From 0146c9d34faa959e9daca77c1c3353b2a41991de Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Tue, 24 Sep 2024 09:24:24 -0400 Subject: [PATCH 19/55] feat: AI-129: add ability to specify a default value for a an environment variable in a .yaml config file (#43) --- src/solace_ai_connector/main.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/solace_ai_connector/main.py b/src/solace_ai_connector/main.py index 0adb414..a62cf00 100644 --- a/src/solace_ai_connector/main.py +++ b/src/solace_ai_connector/main.py @@ -1,5 +1,6 @@ import os import sys +import re import yaml from .solace_ai_connector import SolaceAiConnector @@ -12,7 +13,7 @@ def load_config(file): yaml_str = f.read() # Substitute the environment variables using os.environ - yaml_str = os.path.expandvars(yaml_str) + yaml_str = expandvars_with_defaults(yaml_str) # Load the YAML string using yaml.safe_load return yaml.safe_load(yaml_str) @@ -22,6 +23,19 @@ def load_config(file): sys.exit(1) +def expandvars_with_defaults(text): + """Expand environment variables with support for default values. + Supported syntax: ${VAR_NAME} or ${VAR_NAME, default_value}""" + pattern = re.compile(r"\$\{([^}:\s]+)(?:\s*,\s*([^}]*))?\}") + + def replacer(match): + var_name = match.group(1) + default_value = match.group(2) if match.group(2) is not None else "" + return os.environ.get(var_name, default_value) + + return pattern.sub(replacer, text) + + def merge_config(dict1, dict2): """Merge a new configuration into an existing configuration.""" merged = {} From 779f9b64067787af26a26896e2cb41e908d79e00 Mon Sep 17 00:00:00 2001 From: Alireza Parvizimosaed Date: Wed, 25 Sep 2024 12:30:47 -0400 Subject: [PATCH 20/55] Added the duckduckgo, google and bing web search. Added the web scraper --- examples/custom_components/__init__.py | 0 examples/custom_components/web_search.py | 67 +++++++++ examples/custom_components/web_search.yaml | 25 ++++ examples/websearch/duckduckgo_web_search.yaml | 24 +++ examples/websearch/google_web_search.yaml | 26 ++++ examples/websearch/web_scraping.yaml | 32 ++++ src/solace_ai_connector/common/utils.py | 1 + .../components/__init__.py | 8 + .../components/general/websearch/__init__.py | 0 .../general/websearch/web_scraper.py | 141 ++++++++++++++++++ .../general/websearch/websearch_base.py | 44 ++++++ .../general/websearch/websearch_duckduckgo.py | 91 +++++++++++ .../general/websearch/websearch_google.py | 90 +++++++++++ 13 files changed, 549 insertions(+) create mode 100644 examples/custom_components/__init__.py create mode 100644 examples/custom_components/web_search.py create mode 100644 examples/custom_components/web_search.yaml create mode 100644 examples/websearch/duckduckgo_web_search.yaml create mode 100644 examples/websearch/google_web_search.yaml create mode 100644 examples/websearch/web_scraping.yaml create mode 100644 src/solace_ai_connector/components/general/websearch/__init__.py create mode 100644 src/solace_ai_connector/components/general/websearch/web_scraper.py create mode 100644 src/solace_ai_connector/components/general/websearch/websearch_base.py create mode 100644 src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py create mode 100644 src/solace_ai_connector/components/general/websearch/websearch_google.py diff --git a/examples/custom_components/__init__.py b/examples/custom_components/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/custom_components/web_search.py b/examples/custom_components/web_search.py new file mode 100644 index 0000000..55542ff --- /dev/null +++ b/examples/custom_components/web_search.py @@ -0,0 +1,67 @@ +# A simple pass-through component - what goes in comes out +import sys +import requests + +sys.path.append("src") + +from solace_ai_connector.components.component_base import ComponentBase + + +info = { + "class_name": "WebSearchCustomComponent", + "description": "Search web using APIs.", + "config_parameters": [ + { + "name": "engine", + "description": "The search engine.", + "type": "string", + }, + { + "name": "output_format", + "description": "Output format in json or html.", + "type": "string", + "default": "json" + } + ], + "input_schema": { + "type": "object", + "properties": {}, + }, + "output_schema": { + "type": "object", + "properties": {}, + }, +} + + +class WebSearchCustomComponent(ComponentBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + self.engine = self.get_config("engine") + self.format = self.get_config("format") + + def invoke(self, message, data): + query = data["text"] + print(query) + url = None + if self.engine == "DuckDuckGo": + url = "http://api.duckduckgo.com/" + params = { + "q": query, # User query + "format": self.format, # Response format (json by default) + "pretty": 1, # Beautify the output + "no_html": 3, # Remove HTML from the response + "skip_disambig": 1 # Skip disambiguation + } + + if url != None: + response = requests.get(url, params=params) + if response.status_code == 200: + if params["format"] == 'json': + print(response) + return response.json() # Return JSON response if the format is JSON + else: + return response # Return raw response if not JSON format + else: + # Handle errors if the request fails + return f"Error: {response.status_code}" diff --git a/examples/custom_components/web_search.yaml b/examples/custom_components/web_search.yaml new file mode 100644 index 0000000..94c49fa --- /dev/null +++ b/examples/custom_components/web_search.yaml @@ -0,0 +1,25 @@ +log: + stdout_log_level: INFO + log_file_level: INFO + log_file: solace_ai_connector.log + +flows: + - name: web_search_flow + components: + # Input from a standard in + - component_name: stdin + component_module: stdin_input + + # Using Custom component + - component_name: web_search_component + component_base_path: . + component_module: web_search + component_config: + engine: DuckDuckGo + format: json + input_selection: + source_expression: previous + + # Output to a standard out + - component_name: stdout + component_module: stdout_output \ No newline at end of file diff --git a/examples/websearch/duckduckgo_web_search.yaml b/examples/websearch/duckduckgo_web_search.yaml new file mode 100644 index 0000000..b1b24f2 --- /dev/null +++ b/examples/websearch/duckduckgo_web_search.yaml @@ -0,0 +1,24 @@ +log: + stdout_log_level: INFO + log_file_level: INFO + log_file: solace_ai_connector.log + +flows: + - name: duckduckgo_web_search_flow + components: + # Input from a standard in + - component_name: stdin + component_module: stdin_input + + # Using Custom component + - component_name: web_search_component + component_module: websearch_duckduckgo + component_config: + engine: duckduckgo + detail: false + input_selection: + source_expression: previous + + # Output to a standard out + - component_name: stdout + component_module: stdout_output \ No newline at end of file diff --git a/examples/websearch/google_web_search.yaml b/examples/websearch/google_web_search.yaml new file mode 100644 index 0000000..6766a9f --- /dev/null +++ b/examples/websearch/google_web_search.yaml @@ -0,0 +1,26 @@ +log: + stdout_log_level: INFO + log_file_level: INFO + log_file: solace_ai_connector.log + +flows: + - name: google_web_search_flow + components: + # Input from a standard in + - component_name: stdin + component_module: stdin_input + + # Using Custom component + - component_name: web_search_component + component_module: websearch_google + component_config: + engine: google + api_key: + search_engine_id: + detail: false + input_selection: + source_expression: previous + + # Output to a standard out + - component_name: stdout + component_module: stdout_output \ No newline at end of file diff --git a/examples/websearch/web_scraping.yaml b/examples/websearch/web_scraping.yaml new file mode 100644 index 0000000..986436c --- /dev/null +++ b/examples/websearch/web_scraping.yaml @@ -0,0 +1,32 @@ +log: + stdout_log_level: INFO + log_file_level: INFO + log_file: solace_ai_connector.log + +flows: + - name: web_scraping_flow + components: + # Input from a standard in + - component_name: stdin + component_module: stdin_input + + # Using Custom component + - component_name: web_scraping_component + component_module: web_scraper + component_config: + max_depth: 1 + use_async: false + extractor: null + metadata_extractor: null + exclude_dirs: [] + timeout: 10 + check_response_status: true + continue_on_failure: true + prevent_outside: true + base_url: null + input_selection: + source_expression: previous + + # Output to a standard out + - component_name: stdout + component_module: stdout_output \ No newline at end of file diff --git a/src/solace_ai_connector/common/utils.py b/src/solace_ai_connector/common/utils.py index 4996c3b..7f26c98 100644 --- a/src/solace_ai_connector/common/utils.py +++ b/src/solace_ai_connector/common/utils.py @@ -122,6 +122,7 @@ def import_module(module, base_path=None, component_package=None): ".components.general.for_testing", ".components.general.langchain", ".components.general.openai", + ".components.general.websearch", ".components.inputs_outputs", ".transforms", ".common", diff --git a/src/solace_ai_connector/components/__init__.py b/src/solace_ai_connector/components/__init__.py index f373125..a42f608 100644 --- a/src/solace_ai_connector/components/__init__.py +++ b/src/solace_ai_connector/components/__init__.py @@ -33,6 +33,11 @@ langchain_vector_store_embedding_search, ) +from .general.websearch import ( + websearch_duckduckgo, + websearch_google +) + # Also import the components from the submodules from .inputs_outputs.error_input import ErrorInput from .inputs_outputs.timer_input import TimerInput @@ -62,3 +67,6 @@ from .general.langchain.langchain_vector_store_embedding_search import ( LangChainVectorStoreEmbeddingsSearch, ) +from .general.websearch.websearch_duckduckgo import WebSearchDuckDuckGo +from .general.websearch.websearch_google import WebSearchGoogle + diff --git a/src/solace_ai_connector/components/general/websearch/__init__.py b/src/solace_ai_connector/components/general/websearch/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/solace_ai_connector/components/general/websearch/web_scraper.py b/src/solace_ai_connector/components/general/websearch/web_scraper.py new file mode 100644 index 0000000..db76c9b --- /dev/null +++ b/src/solace_ai_connector/components/general/websearch/web_scraper.py @@ -0,0 +1,141 @@ +"""Scrape a website and related subpages""" +from bs4 import BeautifulSoup +from langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader +from ...component_base import ComponentBase + +info = { + "class_name": "WebScraper", + "description": "Scrape a website and related subpages.", + "config_parameters": [ + { + "name": "max_depth", + "required": False, + "description": "Maximum depth for scraping subpages.", + "default": 1 + }, + { + "name": "use_async", + "required": False, + "description": "Use asynchronous scraping.", + "default": False + }, + { + "name": "extractor", + "required": False, + "description": "Custom extractor function for processing page content.", + "default": "custom_extractor" + }, + { + "name": "metadata_extractor", + "required": False, + "description": "Function to extract metadata from pages.", + }, + { + "name": "exclude_dirs", + "required": False, + "description": "Directories to exclude from scraping.", + }, + { + "name": "timeout", + "required": False, + "description": "Maximum time to wait for a response in seconds.", + "default": 10 + }, + { + "name": "check_response_status", + "required": False, + "description": "Whether to check HTTP response status.", + "default": False + }, + { + "name": "continue_on_failure", + "required": False, + "description": "Continue scraping even if some pages fail.", + "default": True + }, + { + "name": "prevent_outside", + "required": False, + "description": "Prevent scraping outside the base URL.", + "default": True + }, + { + "name": "base_url", + "required": False, + "description": "Base URL to begin scraping from.", + } + ], + "input_schema": { + "type": "object", + "properties": {} + }, + "output_schema": { + "type": "object", + "properties": {} + } +} + +class WebScraper(ComponentBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + self.max_depth = self.get_config("max_depth") + self.use_async = self.get_config("use_async") + self.extractor = self.get_config("extractor") + self.metadata_extractor = self.get_config("metadata_extractor") + self.exclude_dirs = self.get_config("exclude_dirs") + self.timeout = self.get_config("timeout") + self.check_response_status = self.get_config("check_response_status") + self.continue_on_failure = self.get_config("continue_on_failure") + self.prevent_outside = self.get_config("prevent_outside") + self.base_url = self.get_config("base_url") + + def invoke(self, message, data): + url = data["text"] + content = self.scrape(url) + return content + + # Define a custom extractor function to extract text from HTML using BeautifulSoup + def custom_extractor(self, html_content): + soup = BeautifulSoup(html_content, "html.parser") + return soup.get_text() + + # Scrape a website + def scrape(self, url): + loader = RecursiveUrlLoader( + url=url, + extractor=self.extractor, + max_depth=self.max_depth, + use_async=self.use_async, + metadata_extractor=self.metadata_extractor, + exclude_dirs=self.exclude_dirs, + timeout=self.timeout, + check_response_status=self.check_response_status, + continue_on_failure=self.continue_on_failure, + prevent_outside=self.prevent_outside, + base_url=self.base_url + ) + + docs = loader.load() + + for doc in docs: + title = doc.metadata.get("title") + source = doc.metadata.get("source") + content = doc.page_content + + # Ensure that the title, source, and content are string type + resp = {} + if isinstance(title, str) and isinstance(source, str) and isinstance(content, str): + resp = { + "title": title, + "source": source, + "content": content + } + else: + resp = { + "title": "", + "source": "", + "content": "" + } + return resp + + diff --git a/src/solace_ai_connector/components/general/websearch/websearch_base.py b/src/solace_ai_connector/components/general/websearch/websearch_base.py new file mode 100644 index 0000000..56a6fd5 --- /dev/null +++ b/src/solace_ai_connector/components/general/websearch/websearch_base.py @@ -0,0 +1,44 @@ +"""Base class for Web Search""" +from ...component_base import ComponentBase + +info_base = { + "class_name": "WebSearchBase", + "description": "Base class for performing a query on web search engines.", + "config_parameters": [ + { + "name": "engine", + "required": True, + "description": "The type of search engine.", + "default": "DuckDuckGo" + }, + { + "name": "detail", + "required": False, + "description": "Return the detail.", + "default": False + } + ], + "input_schema": { + "type": "object", + "properties": {}, + }, + "output_schema": { + "type": "object", + "properties": {}, + } +} + +class WebSearchBase(ComponentBase): + def __init__(self, info_base, **kwargs): + super().__init__(info_base, **kwargs) + self.engine = self.get_config("engine") + self.detail = self.get_config("detail") + + def invoke(self, message, data): + pass + + # Extract required data from a message + def parse(self, message): + pass + + diff --git a/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py b/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py new file mode 100644 index 0000000..150df1c --- /dev/null +++ b/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py @@ -0,0 +1,91 @@ +# This is a DuckDuckGo search engine. +# The configuration will configure the DuckDuckGo engine. +import requests + +from .websearch_base import ( + WebSearchBase, +) + +info = { + "class_name": "WebSearchDuckDuckGo", + "description": "Perform a search query on DuckDuckGo.", + "config_parameters": [ + { + "name": "engine", + "required": True, + "description": "The type of search engine.", + "default": "duckduckgo" + }, + { + "name": "pretty", + "required": False, + "description": "Beautify the search output.", + "default": 1 + }, + { + "name": "no_html", + "required": False, + "description": "The number of output pages.", + "default": 1 + }, + { + "name": "skip_disambig", + "required": False, + "description": "Skip disambiguation.", + "default": 1 + }, + { + "name": "detail", + "required": False, + "description": "Return the detail.", + "default": False + } + ], + "input_schema": { + "type": "object", + "properties": {}, + }, + "output_schema": { + "type": "object", + "properties": {}, + }, +} + +class WebSearchDuckDuckGo(WebSearchBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + self.init() + + def init(self): + self.pretty = self.get_config("pretty", 1) + self.no_html = self.get_config("no_html", 1) + self.skip_disambig = self.get_config("skip_disambig", 1) + self.url = "http://api.duckduckgo.com/" + + def invoke(self, message, data): + query = data["text"] + if self.engine.lower() == "duckduckgo": + params = { + "q": query, # User query + "format": "json", # Response format (json by default) + "pretty": self.pretty, # Beautify the output + "no_html": self.no_html, # Remove HTML from the response + "skip_disambig": self.skip_disambig # Skip disambiguation + } + + response = requests.get(self.url, params=params) + if response.status_code == 200: + response = response.json() + response = self.parse(response) + return response + else: + return f"Error: {response.status_code}" + else: + return f"Error: The engine is not DuckDuckGo." + + # Extract required data from a message + def parse(self, message): + if self.detail: + return message + else: + return message['Abstract'] \ No newline at end of file diff --git a/src/solace_ai_connector/components/general/websearch/websearch_google.py b/src/solace_ai_connector/components/general/websearch/websearch_google.py new file mode 100644 index 0000000..3561f87 --- /dev/null +++ b/src/solace_ai_connector/components/general/websearch/websearch_google.py @@ -0,0 +1,90 @@ +# This is a Google search engine. +# The configuration will configure the Google engine. +import requests + +from .websearch_base import ( + WebSearchBase, +) + +info = { + "class_name": "WebSearchGoogle", + "description": "Perform a search query on Google.", + "config_parameters": [ + { + "name": "engine", + "required": True, + "description": "The type of search engine.", + "default": "google" + }, + { + "name": "api_key", + "required": True, + "description": "Google API Key.", + }, + { + "name": "search_engine_id", + "required": False, + "description": "The custom search engine id.", + "default": 1 + }, + { + "name": "detail", + "required": False, + "description": "Return the detail.", + "default": False + } + ], + "input_schema": { + "type": "object", + "properties": {}, + }, + "output_schema": { + "type": "object", + "properties": {}, + }, +} + +class WebSearchGoogle(WebSearchBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + self.init() + + def init(self): + self.api_key = self.get_config("api_key") + self.search_engine_id = self.get_config("search_engine_id") + self.url = "https://www.googleapis.com/customsearch/v1" + + def invoke(self, message, data): + query = data["text"] + if self.engine.lower() == "google": + params = { + "q": query, # User query + "key": self.api_key, # Google API Key + "cx": self.search_engine_id, # Google custom search engine id + } + + response = requests.get(self.url, params=params) + if response.status_code == 200: + response = response.json() + response = self.parse(response) + return response + else: + return f"Error: {response.status_code}" + else: + return f"Error: The engine is not DuckDuckGo." + + # Extract required data from a message + def parse(self, message): + if self.detail: + return message + else: + data = [] + + # Process the search results to create a summary + for item in message.get('items', []): + data.append({ + "Title": item['title'], + "Snippet": item['snippet'], + "URL": item['link'] + }) + return data \ No newline at end of file From 49cf9b67ab1c131b16b8e5652cf52bcaf889a55e Mon Sep 17 00:00:00 2001 From: Art Morozov Date: Thu, 26 Sep 2024 16:11:32 -0400 Subject: [PATCH 21/55] DATAGO-85484 Bump min python version --- .github/workflows/ci.yml | 9 ++++++--- .github/workflows/release.yaml | 2 +- pyproject.toml | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21e2b8b..439f70f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,9 @@ permissions: jobs: ci: - uses: SolaceDev/solace-public-workflows/.github/workflows/hatch_ci.yml@v1.0.0 + uses: SolaceDev/solace-public-workflows/.github/workflows/hatch_ci.yml@latest + with: + min-python-version: "3.9" secrets: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} SONAR_HOST_URL: ${{ vars.SONAR_HOST_URL }} @@ -29,8 +31,9 @@ jobs: ssh-key: ${{ secrets.COMMIT_KEY }} - name: Set up Hatch - uses: SolaceDev/solace-public-workflows/.github/actions/hatch-setup@v1.0.0 - + uses: SolaceDev/solace-public-workflows/.github/actions/hatch-setup@latest + with: + min-python-version: "3.9" - name: Set Up Docker Buildx id: builder uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c483f4b..0026e90 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,7 +18,7 @@ permissions: jobs: release: - uses: SolaceDev/solace-public-workflows/.github/workflows/hatch_release_pypi.yml@v1.0.1 + uses: SolaceDev/solace-public-workflows/.github/workflows/hatch_release_pypi.yml@latest with: ENVIRONMENT: pypi version: ${{ github.event.inputs.version }} diff --git a/pyproject.toml b/pyproject.toml index 2ec127b..5fbd262 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] description = "Solace AI Connector - make it easy to connect Solace PubSub+ Event Brokers to AI/ML frameworks" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", From a8c62dd8d7735ec0a8489db61e1917b5c7334e7a Mon Sep 17 00:00:00 2001 From: Alireza Parvizimosaed Date: Sun, 29 Sep 2024 17:55:46 -0400 Subject: [PATCH 22/55] designed a routing flow, updated the scraper to render javascript websites, and added a filtering agent --- docs/getting_started.md | 4 + examples/custom_components/web_search.py | 0 examples/custom_components/web_search.yaml | 0 examples/websearch/bing_web_search.yaml | 27 ++ examples/websearch/duckduckgo_web_search.yaml | 0 examples/websearch/google_web_search.yaml | 0 examples/websearch/web_scraping.yaml | 10 - examples/websearch/websearch_router.yaml | 384 ++++++++++++++++++ requirements.txt | 1 + .../components/__init__.py | 9 +- .../components/general/filter/__init__.py | 0 .../general/filter/filter_by_llm.py | 65 +++ .../components/general/websearch/__init__.py | 0 .../general/websearch/web_scraper.py | 130 +----- .../general/websearch/websearch_base.py | 0 .../general/websearch/websearch_bing.py | 94 +++++ .../general/websearch/websearch_duckduckgo.py | 0 .../general/websearch/websearch_google.py | 1 + 18 files changed, 604 insertions(+), 121 deletions(-) mode change 100644 => 100755 docs/getting_started.md mode change 100644 => 100755 examples/custom_components/web_search.py mode change 100644 => 100755 examples/custom_components/web_search.yaml create mode 100755 examples/websearch/bing_web_search.yaml mode change 100644 => 100755 examples/websearch/duckduckgo_web_search.yaml mode change 100644 => 100755 examples/websearch/google_web_search.yaml mode change 100644 => 100755 examples/websearch/web_scraping.yaml create mode 100755 examples/websearch/websearch_router.yaml mode change 100644 => 100755 requirements.txt mode change 100644 => 100755 src/solace_ai_connector/components/__init__.py create mode 100644 src/solace_ai_connector/components/general/filter/__init__.py create mode 100644 src/solace_ai_connector/components/general/filter/filter_by_llm.py mode change 100644 => 100755 src/solace_ai_connector/components/general/websearch/__init__.py mode change 100644 => 100755 src/solace_ai_connector/components/general/websearch/web_scraper.py mode change 100644 => 100755 src/solace_ai_connector/components/general/websearch/websearch_base.py create mode 100755 src/solace_ai_connector/components/general/websearch/websearch_bing.py mode change 100644 => 100755 src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py mode change 100644 => 100755 src/solace_ai_connector/components/general/websearch/websearch_google.py diff --git a/docs/getting_started.md b/docs/getting_started.md old mode 100644 new mode 100755 index 53d847b..e5adb4c --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -135,6 +135,10 @@ In the "Try Me!" also subscribe to `demo/joke/subject/response` to see the respo ```sh pip install -r requirements.txt ``` +4. Install the necessary browser binaries for the web scraper. + ```sh + playwright install + ``` ## Configuration diff --git a/examples/custom_components/web_search.py b/examples/custom_components/web_search.py old mode 100644 new mode 100755 diff --git a/examples/custom_components/web_search.yaml b/examples/custom_components/web_search.yaml old mode 100644 new mode 100755 diff --git a/examples/websearch/bing_web_search.yaml b/examples/websearch/bing_web_search.yaml new file mode 100755 index 0000000..2863b89 --- /dev/null +++ b/examples/websearch/bing_web_search.yaml @@ -0,0 +1,27 @@ +log: + stdout_log_level: INFO + log_file_level: INFO + log_file: solace_ai_connector.log + +flows: + - name: google_web_search_flow + components: + # Input from a standard in + - component_name: stdin + component_module: stdin_input + + # Using Custom component + - component_name: web_search_component + component_module: websearch_bing + component_config: + engine: bing + api_key: + safesearch: Moderate + count: 2 + detail: false + input_selection: + source_expression: previous + + # Output to a standard out + - component_name: stdout + component_module: stdout_output \ No newline at end of file diff --git a/examples/websearch/duckduckgo_web_search.yaml b/examples/websearch/duckduckgo_web_search.yaml old mode 100644 new mode 100755 diff --git a/examples/websearch/google_web_search.yaml b/examples/websearch/google_web_search.yaml old mode 100644 new mode 100755 diff --git a/examples/websearch/web_scraping.yaml b/examples/websearch/web_scraping.yaml old mode 100644 new mode 100755 index 986436c..cb4616e --- a/examples/websearch/web_scraping.yaml +++ b/examples/websearch/web_scraping.yaml @@ -14,16 +14,6 @@ flows: - component_name: web_scraping_component component_module: web_scraper component_config: - max_depth: 1 - use_async: false - extractor: null - metadata_extractor: null - exclude_dirs: [] - timeout: 10 - check_response_status: true - continue_on_failure: true - prevent_outside: true - base_url: null input_selection: source_expression: previous diff --git a/examples/websearch/websearch_router.yaml b/examples/websearch/websearch_router.yaml new file mode 100755 index 0000000..4bad28f --- /dev/null +++ b/examples/websearch/websearch_router.yaml @@ -0,0 +1,384 @@ +# This will create a flow like this: +# Solace -> OpenAI -> Solace +# +# It will subscribe to `demo/joke/subject` and expect an event with the payload: +# +# { +# "joke": { +# "subject": "" +# } +# } +# +# It will then send an event back to Solace with the topic: `demo/joke/subject/response` +# +# Dependencies: +# pip install -U langchain_openai openai +# +# required ENV variables: +# - OPENAI_API_KEY +# - OPENAI_API_ENDPOINT - optional +# - MODEL_NAME +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN + +--- +log: + stdout_log_level: INFO + log_file_level: DEBUG + log_file: solace_ai_connector.log + +shared_config: + - broker_config: &broker_connection + broker_type: solace + broker_url: ${SOLACE_BROKER_URL} + broker_username: ${SOLACE_BROKER_USERNAME} + broker_password: ${SOLACE_BROKER_PASSWORD} + broker_vpn: ${SOLACE_BROKER_VPN} + +# Take from input broker and publish back to Solace +flows: + # Flow 1: Route a request + - name: Router + components: + # Input from a Solace broker + - component_name: solace_sw_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_queue_name: web_search_broker + broker_subscriptions: + - topic: search/query + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # Go to the LLM and keep history + - component_name: chat_request_llm + component_module: langchain_chat_model_with_history + component_config: + langchain_module: langchain_openai + langchain_class: ChatOpenAI + langchain_component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${MODEL_NAME} + temperature: 0.01 + history_module: langchain_core.chat_history + history_class: InMemoryChatMessageHistory + history_max_turns: 20 + history_max_length: 1 + input_transforms: + - type: copy + source_expression: | + template: You are a helpful AI assistant tasked with selecting the most suitable agent to respond to the user's query. Based on the nature of the question, determine whether the query requires searching external web sources or if it can be answered using an internal knowledge model. Return ONLY the agent name (web_search or LLM) that is most appropriate for the given query. + + { + name: LLM + description: Use for retrieving general knowledge, answering questions that do not require up-to-date data, performing analysis, creative tasks, or providing explanations based on established information. This agent is suitable for queries about historical events, scientific concepts, literary analysis, mathematical calculations, coding help, or general advice that doesn't depend on current events. + } + { + name: web_search + description: Use for looking up real-time or current information, recent events, latest news, up-to-date statistics, current prices, weather forecasts, ongoing developments, or any query that requires the most recent data. This agent is also suitable for fact-checking or verifying claims about recent occurrences. + } + + + Guidelines for selection: + 1. If the query explicitly mentions needing the latest information or current data, choose web_search. + 2. For questions about historical events, established scientific facts, or timeless concepts, choose LLM. + 3. If the query is about a potentially evolving situation but doesn't specifically request current information, lean towards web_search to ensure accuracy. + 4. For tasks involving analysis, creative writing, or explanation of concepts, choose LLM. + 5. If unsure, err on the side of web_search to provide the most up-to-date information. + + Examples: + 1. Query: "What is the current stock price of Apple?" + Agent: web_search + + 2. Query: "Explain the process of photosynthesis in plants." + Agent: LLM + + 3. Query: "Who won the last presidential election in France?" + Agent: web_search + + 4. Query: "Can you help me write a short story about a time traveler?" + Agent: LLM + + 5. Query: "What were the major causes of World War I?" + Agent: LLM + + 6. Query: "What are the latest developments in the Israel-Palestine conflict?" + Agent: web_search + + 7. Query: "How do I calculate the area of a circle?" + Agent: LLM + + 8. Query: "What's the weather forecast for New York City tomorrow?" + Agent: web_search + + 9. Query: "Can you explain the theory of relativity?" + Agent: LLM + + 10. Query: "What are the current COVID-19 restrictions in California?" + Agent: web_search + + Respond ONLY with the name of the chosen agent: either "LLM" or "web_search". Do not include any explanation or additional text in your response. + + + {{text://input.payload:text}} + + dest_expression: user_data.input:messages.0.content + - type: copy + source_value: user + dest_expression: user_data.input:messages.0.role + input_selection: + source_expression: user_data.input + + # Route a request + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: input.payload + dest_expression: user_data.output:payload + - type: copy + source_value: + invoke: + module: invoke_functions + function: if_else + params: + positional: + - invoke: + module: invoke_functions + function: equal + params: + positional: + - evaluate_expression(user_data.output:payload, text) + - "web_search" + - search/query/llm + - search/query/llm + dest_expression: user_data.output:topic + input_selection: + source_expression: user_data.output + + # Flow 2: Web search flow + - name: web_search + components: + # Agent broker input configuration + - agent_broker_input: &agent_broker_input + component_name: solace_agent_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_subscriptions: + - topic: search/query/web_search + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # Using Custom component + - component_name: web_search_component + component_module: websearch_google + component_config: + engine: google + api_key: AIzaSyA7kKgIBK_Clw-HFb_YZYSxwQgKEX1rza8 + search_engine_id: d244da49663d44d5e + detail: false + input_selection: + source_expression: input.payload + + # Clean results by LLM + - component_name: cleaner_llm + component_module: langchain_chat_model_with_history + component_config: + langchain_module: langchain_openai + langchain_class: ChatOpenAI + langchain_component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${MODEL_NAME} + temperature: 0.01 + history_module: langchain_core.chat_history + history_class: InMemoryChatMessageHistory + history_max_turns: 20 + history_max_length: 1 + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.input:payload + - type: copy + source_expression: | + template: You are an AI assistant specialized in cleaning and formatting JSON data. Your task is to clean the text content within a JSON object while preserving its structure. Given a JSON object of any structure, perform the following cleaning operations on all string values throughout the object: + Convert all Unicode escape sequences (e.g., \u03c0) to their corresponding characters. + Remove any HTML entities (e.g., , &) and replace them with their corresponding characters. + Remove any trailing ellipsis (...) at the end of text fields. + Trim leading and trailing whitespace from all string values. + Replace multiple consecutive spaces with a single space. + Ensure proper capitalization for title-like fields (use your judgment based on the field name or content). + For URL-like fields, ensure they are in a valid URL format and remove any unnecessary query parameters or fragments. + Remove any control characters or invisible formatting characters. + Maintain the overall JSON structure, including all keys and nested objects or arrays. Apply these cleaning operations recursively to all levels of the JSON object. + Return the cleaned JSON object as plain text, without any code block formatting (such as ```json). Do not add any additional text before or after the JSON object. + Here is the JSON data to clean: + {{text://user_data.input:payload}} + dest_expression: user_data.input:messages.0.content + - type: copy + source_value: user + dest_expression: user_data.input:messages.0.role + input_selection: + source_expression: user_data.input + + # # Filter json + # - component_name: filter_json_component + # component_module: filter_by_llm + # input_transforms: + # - type: copy + # source_expression: previous + # dest_expression: user_data.input:payload + # input_selection: + # source_expression: input.payload + + # Send response back to broker with completion and retrieved data + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.output:payload.response + - type: copy + source_expression: input.payload:query + dest_expression: user_data.output:payload.query + - type: copy + source_expression: template:{{text://input.topic}}/response + dest_expression: user_data.output:topic + input_selection: + source_expression: user_data.output + +# Flow 3: LLM + - name: LLM + components: + # Agent broker input configuration + - agent_broker_input: &llm_agent_broker_input + component_name: solace_agent_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_subscriptions: + - topic: search/query/llm + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # Request from LLM + - component_name: llm + component_module: langchain_chat_model_with_history + component_config: + langchain_module: langchain_openai + langchain_class: ChatOpenAI + langchain_component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${MODEL_NAME} + temperature: 0.01 + history_module: langchain_core.chat_history + history_class: InMemoryChatMessageHistory + history_max_turns: 20 + history_max_length: 1 + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.input:payload + - type: copy + source_expression: | + template:You are a helpful AI assistant. Please help with the user's request below: + + {{text://user_data.input:payload}} + + dest_expression: user_data.input:messages.0.content + - type: copy + source_value: user + dest_expression: user_data.input:messages.0.role + input_selection: + source_expression: user_data.input + + # Clean results by LLM + - component_name: cleaner_llm + component_module: langchain_chat_model_with_history + component_config: + langchain_module: langchain_openai + langchain_class: ChatOpenAI + langchain_component_config: + api_key: ${OPENAI_API_KEY} + base_url: ${OPENAI_API_ENDPOINT} + model: ${MODEL_NAME} + temperature: 0.01 + history_module: langchain_core.chat_history + history_class: InMemoryChatMessageHistory + history_max_turns: 20 + history_max_length: 1 + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.input:payload + - type: copy + source_expression: | + template: You are an AI assistant specialized in cleaning and formatting JSON data. Your task is to clean the text content within a JSON object while preserving its structure. Given a JSON object of any structure, perform the following cleaning operations on all string values throughout the object: + Convert all Unicode escape sequences (e.g., \u03c0) to their corresponding characters. + Remove any HTML entities (e.g., , &) and replace them with their corresponding characters. + Remove any trailing ellipsis (...) at the end of text fields. + Trim leading and trailing whitespace from all string values. + Replace multiple consecutive spaces with a single space. + Ensure proper capitalization for title-like fields (use your judgment based on the field name or content). + For URL-like fields, ensure they are in a valid URL format and remove any unnecessary query parameters or fragments. + Remove any control characters or invisible formatting characters. + Maintain the overall JSON structure, including all keys and nested objects or arrays. Apply these cleaning operations recursively to all levels of the JSON object. + Return the cleaned JSON object as plain text, without any code block formatting (such as ```json). Do not add any additional text before or after the JSON object. + Here is the JSON data to clean: + {{text://user_data.input:payload}} + dest_expression: user_data.input:messages.0.content + - type: copy + source_value: user + dest_expression: user_data.input:messages.0.role + input_selection: + source_expression: user_data.input + + # # Filter json + # - component_name: filter_json_component + # component_module: filter_by_llm + # input_transforms: + # - type: copy + # source_expression: previous + # dest_expression: user_data.input:payload + # input_selection: + # source_expression: input.payload + + # Send response back to broker with completion and retrieved data + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.output:payload.response + - type: copy + source_expression: input.payload:query + dest_expression: user_data.output:payload.query + - type: copy + source_expression: template:{{text://input.topic}}/response + dest_expression: user_data.output:topic + input_selection: + source_expression: user_data.output \ No newline at end of file diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 index 66ec399..7bbfb9d --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ langchain~=0.3.0 PyYAML~=6.0.1 Requests~=2.32.3 solace_pubsubplus~=1.8.0 +playwright~=1.47.0 \ No newline at end of file diff --git a/src/solace_ai_connector/components/__init__.py b/src/solace_ai_connector/components/__init__.py old mode 100644 new mode 100755 index a42f608..3bb8b7a --- a/src/solace_ai_connector/components/__init__.py +++ b/src/solace_ai_connector/components/__init__.py @@ -35,7 +35,12 @@ from .general.websearch import ( websearch_duckduckgo, - websearch_google + websearch_google, + websearch_bing +) + +from .general.filter import ( + filter_by_llm ) # Also import the components from the submodules @@ -69,4 +74,6 @@ ) from .general.websearch.websearch_duckduckgo import WebSearchDuckDuckGo from .general.websearch.websearch_google import WebSearchGoogle +from .general.websearch.websearch_bing import WebSearchBing +from .general.filter.filter_by_llm import CleanJsonObject diff --git a/src/solace_ai_connector/components/general/filter/__init__.py b/src/solace_ai_connector/components/general/filter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/solace_ai_connector/components/general/filter/filter_by_llm.py b/src/solace_ai_connector/components/general/filter/filter_by_llm.py new file mode 100644 index 0000000..16429a5 --- /dev/null +++ b/src/solace_ai_connector/components/general/filter/filter_by_llm.py @@ -0,0 +1,65 @@ +"""Remove any unnecessary texts before and after a json object.""" + +import json + +from ...component_base import ComponentBase + +info = { + "class_name": "CleanJsonObject", + "description": "Scrape javascript based websites.", + "config_parameters": [ + ], + "input_schema": { + "type": "object", + "properties": {} + }, + "output_schema": { + "type": "object", + "properties": {} + } +} + +class CleanJsonObject(ComponentBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + + def invoke(self, message, data): + text = data["text"] + json_obj = self.extract_json(text) + return json_obj + + # Clean a json object + def extract_json(self, text): + start_index = text.find('{') + end_index = start_index + brace_count = 0 + + for i, char in enumerate(text[start_index:], start=start_index): + if char == '{': + brace_count += 1 + elif char == '}': + brace_count -= 1 + if brace_count == 0: + end_index = i + break + + if brace_count == 0: + json_str = text[start_index:end_index + 1] + # If a JSON object is found, convert to JSON + if json_str: + try: + json_object = json.loads(json_str) + return json.dumps(json_object, indent=4) + except json.JSONDecodeError as e: + raise ValueError(f"Error converting text to json: {str(e)}") from e + else: + return None + else: + return None + + + + + + + diff --git a/src/solace_ai_connector/components/general/websearch/__init__.py b/src/solace_ai_connector/components/general/websearch/__init__.py old mode 100644 new mode 100755 diff --git a/src/solace_ai_connector/components/general/websearch/web_scraper.py b/src/solace_ai_connector/components/general/websearch/web_scraper.py old mode 100644 new mode 100755 index db76c9b..eee6858 --- a/src/solace_ai_connector/components/general/websearch/web_scraper.py +++ b/src/solace_ai_connector/components/general/websearch/web_scraper.py @@ -1,69 +1,11 @@ -"""Scrape a website and related subpages""" -from bs4 import BeautifulSoup -from langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader +"""Scrape a website""" +from playwright.sync_api import sync_playwright from ...component_base import ComponentBase info = { "class_name": "WebScraper", - "description": "Scrape a website and related subpages.", + "description": "Scrape javascript based websites.", "config_parameters": [ - { - "name": "max_depth", - "required": False, - "description": "Maximum depth for scraping subpages.", - "default": 1 - }, - { - "name": "use_async", - "required": False, - "description": "Use asynchronous scraping.", - "default": False - }, - { - "name": "extractor", - "required": False, - "description": "Custom extractor function for processing page content.", - "default": "custom_extractor" - }, - { - "name": "metadata_extractor", - "required": False, - "description": "Function to extract metadata from pages.", - }, - { - "name": "exclude_dirs", - "required": False, - "description": "Directories to exclude from scraping.", - }, - { - "name": "timeout", - "required": False, - "description": "Maximum time to wait for a response in seconds.", - "default": 10 - }, - { - "name": "check_response_status", - "required": False, - "description": "Whether to check HTTP response status.", - "default": False - }, - { - "name": "continue_on_failure", - "required": False, - "description": "Continue scraping even if some pages fail.", - "default": True - }, - { - "name": "prevent_outside", - "required": False, - "description": "Prevent scraping outside the base URL.", - "default": True - }, - { - "name": "base_url", - "required": False, - "description": "Base URL to begin scraping from.", - } ], "input_schema": { "type": "object", @@ -78,64 +20,32 @@ class WebScraper(ComponentBase): def __init__(self, **kwargs): super().__init__(info, **kwargs) - self.max_depth = self.get_config("max_depth") - self.use_async = self.get_config("use_async") - self.extractor = self.get_config("extractor") - self.metadata_extractor = self.get_config("metadata_extractor") - self.exclude_dirs = self.get_config("exclude_dirs") - self.timeout = self.get_config("timeout") - self.check_response_status = self.get_config("check_response_status") - self.continue_on_failure = self.get_config("continue_on_failure") - self.prevent_outside = self.get_config("prevent_outside") - self.base_url = self.get_config("base_url") def invoke(self, message, data): url = data["text"] content = self.scrape(url) return content - - # Define a custom extractor function to extract text from HTML using BeautifulSoup - def custom_extractor(self, html_content): - soup = BeautifulSoup(html_content, "html.parser") - return soup.get_text() # Scrape a website def scrape(self, url): - loader = RecursiveUrlLoader( - url=url, - extractor=self.extractor, - max_depth=self.max_depth, - use_async=self.use_async, - metadata_extractor=self.metadata_extractor, - exclude_dirs=self.exclude_dirs, - timeout=self.timeout, - check_response_status=self.check_response_status, - continue_on_failure=self.continue_on_failure, - prevent_outside=self.prevent_outside, - base_url=self.base_url - ) + with sync_playwright() as p: + # Launch a browser instance (Chromium, Firefox, or WebKit) + browser = p.chromium.launch(headless=True) # Set headless=False to see the browser in action + page = browser.new_page() + page.goto(url) + + # Wait for the page to fully load + page.wait_for_load_state("networkidle") + + # Scrape the text content of the page + title = page.title() + content = page.evaluate("document.body.innerText") + resp = { + "title": title, + "content": content + } + browser.close() - docs = loader.load() - - for doc in docs: - title = doc.metadata.get("title") - source = doc.metadata.get("source") - content = doc.page_content - - # Ensure that the title, source, and content are string type - resp = {} - if isinstance(title, str) and isinstance(source, str) and isinstance(content, str): - resp = { - "title": title, - "source": source, - "content": content - } - else: - resp = { - "title": "", - "source": "", - "content": "" - } return resp diff --git a/src/solace_ai_connector/components/general/websearch/websearch_base.py b/src/solace_ai_connector/components/general/websearch/websearch_base.py old mode 100644 new mode 100755 diff --git a/src/solace_ai_connector/components/general/websearch/websearch_bing.py b/src/solace_ai_connector/components/general/websearch/websearch_bing.py new file mode 100755 index 0000000..2cda058 --- /dev/null +++ b/src/solace_ai_connector/components/general/websearch/websearch_bing.py @@ -0,0 +1,94 @@ +# This is a Bing search engine. +# The configuration will configure the Bing engine. +import requests + +from .websearch_base import ( + WebSearchBase, +) + +info = { + "class_name": "WebSearchBing", + "description": "Perform a search query on Bing.", + "config_parameters": [ + { + "name": "engine", + "required": True, + "description": "The type of search engine.", + "default": "bing" + }, + { + "name": "api_key", + "required": True, + "description": "Bing API Key.", + }, + { + "name": "count", + "required": False, + "description": "Number of search results to return.", + "default": 10 + }, + { + "name": "safesearch", + "required": False, + "description": "Safe search setting: Off, Moderate, or Strict.", + "default": "Moderate" + } + ], + "input_schema": { + "type": "object", + "properties": {}, + }, + "output_schema": { + "type": "object", + "properties": {}, + }, +} + +class WebSearchBing(WebSearchBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + self.init() + + def init(self): + self.api_key = self.get_config("api_key") + self.count = self.get_config("count") + self.safesearch = self.get_config("safesearch") + self.url = "https://api.bing.microsoft.com/v7.0/search" + + def invoke(self, message, data): + query = data["text"] + if self.engine.lower() == "bing": + params = { + "q": query, # User query + "count": self.count, # Number of results to return + "safesearch": self.safesearch # Safe search filter + } + headers = { + "Ocp-Apim-Subscription-Key": self.api_key # Bing API Key + } + + response = requests.get(self.url, headers=headers, params=params) + if response.status_code == 200: + response = response.json() + response = self.parse(response) + return response + else: + return f"Error: {response.status_code}" + else: + return f"Error: The engine is not Bing." + + # Extract required data from a message + def parse(self, message): + if self.detail: + return message + else: + data = [] + + # Process the search results to create a summary + for web_page in message.get("webPages", {}).get("value", []): + data.append({ + "Title": web_page['name'], + "Snippet": web_page['snippet'], + "URL": web_page['url'] + }) + return data diff --git a/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py b/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py old mode 100644 new mode 100755 diff --git a/src/solace_ai_connector/components/general/websearch/websearch_google.py b/src/solace_ai_connector/components/general/websearch/websearch_google.py old mode 100644 new mode 100755 index 3561f87..ecacdf9 --- a/src/solace_ai_connector/components/general/websearch/websearch_google.py +++ b/src/solace_ai_connector/components/general/websearch/websearch_google.py @@ -1,6 +1,7 @@ # This is a Google search engine. # The configuration will configure the Google engine. import requests +import json from .websearch_base import ( WebSearchBase, From 714417434fc057dda6711ced248a5b52974abc24 Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Mon, 30 Sep 2024 11:29:12 -0400 Subject: [PATCH 23/55] AI-95: Add retries to openai llm requests, cleaned up some request/response terminology and rebuild all the docs (#45) * feat: AI-95: Support a more configurable reply topic for broker_request_response * refactor: AI-95: Consitently use 'response' rather than a mix of response/reply * docs: Update component docs * chore: Small changes after AI code review --- docs/components/broker_request_response.md | 20 +++- docs/components/index.md | 2 +- docs/components/openai_chat_model.md | 4 +- .../openai_chat_model_with_history.md | 4 +- docs/components/stdin_input.md | 7 +- docs/components/stdout_output.md | 5 +- .../openai_component_request_response.yaml | 2 +- examples/request_reply.yaml | 2 +- src/solace_ai_connector/common/utils.py | 28 +++++- .../general/openai/openai_chat_model_base.py | 97 ++++++++++++------- .../inputs_outputs/broker_request_response.py | 64 +++++++++--- 11 files changed, 176 insertions(+), 59 deletions(-) diff --git a/docs/components/broker_request_response.md b/docs/components/broker_request_response.md index 3d1af10..333b050 100644 --- a/docs/components/broker_request_response.md +++ b/docs/components/broker_request_response.md @@ -15,7 +15,12 @@ component_config: broker_vpn: payload_encoding: payload_format: - request_expiry_ms: + response_topic_prefix: + response_topic_suffix: + reply_queue_prefix: + request_expiry_ms: + streaming: + streaming_complete_expression: ``` | Parameter | Required | Default | Description | @@ -27,7 +32,12 @@ component_config: | broker_vpn | True | | Client VPN for broker | | payload_encoding | False | utf-8 | Encoding for the payload (utf-8, base64, gzip, none) | | payload_format | False | json | Format for the payload (json, yaml, text) | +| response_topic_prefix | False | reply | Prefix for reply topics | +| response_topic_suffix | False | | Suffix for reply topics | +| reply_queue_prefix | False | reply-queue | Prefix for reply queues | | request_expiry_ms | False | 60000 | Expiry time for cached requests in milliseconds | +| streaming | False | | The response will arrive in multiple pieces. If True, the streaming_complete_expression must be set and will be used to determine when the last piece has arrived. | +| streaming_complete_expression | False | | The source expression to determine when the last piece of a streaming response has arrived. | ## Component Input Schema @@ -38,7 +48,10 @@ component_config: topic: , user_properties: { - } + }, + response_topic_suffix: , + stream: , + streaming_complete_expression: } ``` | Field | Required | Description | @@ -46,6 +59,9 @@ component_config: | payload | True | Payload of the request message to be sent to the broker | | topic | True | Topic to send the request message to | | user_properties | False | User properties to send with the request message | +| response_topic_suffix | False | Suffix for the reply topic | +| stream | False | Whether this will have a streaming response | +| streaming_complete_expression | False | Expression to determine when the last piece of a streaming response has arrived. Required if stream is True. | ## Component Output Schema diff --git a/docs/components/index.md b/docs/components/index.md index 5832e4d..e46fd1f 100644 --- a/docs/components/index.md +++ b/docs/components/index.md @@ -21,7 +21,7 @@ | [openai_chat_model](openai_chat_model.md) | OpenAI chat model component | | [openai_chat_model_with_history](openai_chat_model_with_history.md) | OpenAI chat model component with conversation history | | [pass_through](pass_through.md) | What goes in comes out | -| [stdin_input](stdin_input.md) | STDIN input component. The component will prompt for input, which will then be placed in the message payload using the output schema below. | +| [stdin_input](stdin_input.md) | STDIN input component. The component will prompt for input, which will then be placed in the message payload using the output schema below. The component will wait for its output message to be acknowledged before prompting for the next input. | | [stdout_output](stdout_output.md) | STDOUT output component | | [timer_input](timer_input.md) | An input that will generate a message at a specified interval. | | [user_processor](user_processor.md) | A component that allows the processing stage to be defined in the configuration file. | diff --git a/docs/components/openai_chat_model.md b/docs/components/openai_chat_model.md index fa8a506..b9cc612 100644 --- a/docs/components/openai_chat_model.md +++ b/docs/components/openai_chat_model.md @@ -13,6 +13,7 @@ component_config: temperature: base_url: stream_to_flow: + stream_to_next_component: llm_mode: stream_batch_size: set_response_uuid_in_user_properties: @@ -24,7 +25,8 @@ component_config: | model | True | | OpenAI model to use (e.g., 'gpt-3.5-turbo') | | temperature | False | 0.7 | Sampling temperature to use | | base_url | False | None | Base URL for OpenAI API | -| stream_to_flow | False | | Name the flow to stream the output to - this must be configured for llm_mode='stream'. | +| stream_to_flow | False | | Name the flow to stream the output to - this must be configured for llm_mode='stream'. This is mutually exclusive with stream_to_next_component. | +| stream_to_next_component | False | False | Whether to stream the output to the next component in the flow. This is mutually exclusive with stream_to_flow. | | llm_mode | False | none | The mode for streaming results: 'sync' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response. | | stream_batch_size | False | 15 | The minimum number of words in a single streaming result. Default: 15. | | set_response_uuid_in_user_properties | False | False | Whether to set the response_uuid in the user_properties of the input_message. This will allow other components to correlate streaming chunks with the full response. | diff --git a/docs/components/openai_chat_model_with_history.md b/docs/components/openai_chat_model_with_history.md index 262bc72..c72f818 100644 --- a/docs/components/openai_chat_model_with_history.md +++ b/docs/components/openai_chat_model_with_history.md @@ -13,6 +13,7 @@ component_config: temperature: base_url: stream_to_flow: + stream_to_next_component: llm_mode: stream_batch_size: set_response_uuid_in_user_properties: @@ -26,7 +27,8 @@ component_config: | model | True | | OpenAI model to use (e.g., 'gpt-3.5-turbo') | | temperature | False | 0.7 | Sampling temperature to use | | base_url | False | None | Base URL for OpenAI API | -| stream_to_flow | False | | Name the flow to stream the output to - this must be configured for llm_mode='stream'. | +| stream_to_flow | False | | Name the flow to stream the output to - this must be configured for llm_mode='stream'. This is mutually exclusive with stream_to_next_component. | +| stream_to_next_component | False | False | Whether to stream the output to the next component in the flow. This is mutually exclusive with stream_to_flow. | | llm_mode | False | none | The mode for streaming results: 'sync' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response. | | stream_batch_size | False | 15 | The minimum number of words in a single streaming result. Default: 15. | | set_response_uuid_in_user_properties | False | False | Whether to set the response_uuid in the user_properties of the input_message. This will allow other components to correlate streaming chunks with the full response. | diff --git a/docs/components/stdin_input.md b/docs/components/stdin_input.md index 05f1186..c670eef 100644 --- a/docs/components/stdin_input.md +++ b/docs/components/stdin_input.md @@ -1,6 +1,6 @@ # Stdin -STDIN input component. The component will prompt for input, which will then be placed in the message payload using the output schema below. +STDIN input component. The component will prompt for input, which will then be placed in the message payload using the output schema below. The component will wait for its output message to be acknowledged before prompting for the next input. ## Configuration Parameters @@ -8,9 +8,12 @@ STDIN input component. The component will prompt for input, which will then be p component_name: component_module: stdin_input component_config: + prompt: ``` -No configuration parameters +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| prompt | False | | The prompt to display when asking for input | diff --git a/docs/components/stdout_output.md b/docs/components/stdout_output.md index deba25b..66a4700 100644 --- a/docs/components/stdout_output.md +++ b/docs/components/stdout_output.md @@ -8,9 +8,12 @@ STDOUT output component component_name: component_module: stdout_output component_config: + add_new_line_between_messages: ``` -No configuration parameters +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| add_new_line_between_messages | False | True | Add a new line between messages | ## Component Input Schema diff --git a/examples/llm/openai_component_request_response.yaml b/examples/llm/openai_component_request_response.yaml index bb102a6..f00ec8e 100644 --- a/examples/llm/openai_component_request_response.yaml +++ b/examples/llm/openai_component_request_response.yaml @@ -141,7 +141,7 @@ flows: source_expression: previous dest_expression: user_data.output:payload - type: copy - source_expression: input.user_properties:__solace_ai_connector_broker_request_reply_topic__ + source_expression: input.user_properties:__solace_ai_connector_broker_request_response_topic__ dest_expression: user_data.output:topic input_selection: source_expression: user_data.output \ No newline at end of file diff --git a/examples/request_reply.yaml b/examples/request_reply.yaml index 69e5834..a6fb75c 100644 --- a/examples/request_reply.yaml +++ b/examples/request_reply.yaml @@ -77,7 +77,7 @@ flows: source_expression: input.user_properties dest_expression: user_data.output:user_properties - type: copy - source_expression: input.user_properties:__solace_ai_connector_broker_request_reply_topic__ + source_expression: input.user_properties:__solace_ai_connector_broker_request_response_topic__ dest_expression: user_data.output:topic input_selection: source_expression: user_data.output diff --git a/src/solace_ai_connector/common/utils.py b/src/solace_ai_connector/common/utils.py index 4996c3b..4004e87 100644 --- a/src/solace_ai_connector/common/utils.py +++ b/src/solace_ai_connector/common/utils.py @@ -205,7 +205,10 @@ def call_function(function, params, allow_source_expression): if positional: for index, value in enumerate(positional): # source_expression check for backwards compatibility - if isinstance(value, str) and (value.startswith("evaluate_expression(") or value.startswith("source_expression(")): + if isinstance(value, str) and ( + value.startswith("evaluate_expression(") + or value.startswith("source_expression(") + ): # if not allow_source_expression: # raise ValueError( # "evaluate_expression() is not allowed in this context" @@ -220,7 +223,10 @@ def call_function(function, params, allow_source_expression): if keyword: for key, value in keyword.items(): # source_expression check for backwards compatibility - if isinstance(value, str) and (value.startswith("evaluate_expression(") or value.startswith("source_expression(")): + if isinstance(value, str) and ( + value.startswith("evaluate_expression(") + or value.startswith("source_expression(") + ): if not allow_source_expression: raise ValueError( "evaluate_expression() is not allowed in this context" @@ -257,7 +263,7 @@ def install_package(package_name): def extract_evaluate_expression(se_call): # First remove the evaluate_expression( and the trailing ) # Account for possible whitespace - if (se_call.startswith("evaluate_expression(")): + if se_call.startswith("evaluate_expression("): expression = se_call.split("evaluate_expression(")[1].split(")")[0].strip() else: # For backwards compatibility @@ -313,3 +319,19 @@ def get_obj_text(block_format, text): if f"```{block_format}" in text: return text.split(f"```{block_format}")[1].split("```")[0] return text + + +def ensure_slash_on_end(string): + if not string: + return "" + if not string.endswith("/"): + return string + "/" + return string + + +def ensure_slash_on_start(string): + if not string: + return "" + if not string.startswith("/"): + return "/" + string + return string diff --git a/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py b/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py index 0ce6c90..68f78ea 100644 --- a/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py +++ b/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py @@ -1,10 +1,12 @@ """Base class for OpenAI chat models""" import uuid +import time from openai import OpenAI from ...component_base import ComponentBase from ....common.message import Message +from ....common.log import log openai_info_base = { "class_name": "OpenAIChatModelBase", @@ -141,10 +143,22 @@ def invoke(self, message, data): if self.llm_mode == "stream": return self.invoke_stream(client, message, messages) else: - response = client.chat.completions.create( - messages=messages, model=self.model, temperature=self.temperature - ) - return {"content": response.choices[0].message.content} + max_retries = 3 + while max_retries > 0: + try: + response = client.chat.completions.create( + messages=messages, + model=self.model, + temperature=self.temperature, + ) + return {"content": response.choices[0].message.content} + except Exception as e: + log.error("Error invoking OpenAI: %s", e) + max_retries -= 1 + if max_retries <= 0: + raise e + else: + time.sleep(1) def invoke_stream(self, client, message, messages): response_uuid = str(uuid.uuid4()) @@ -155,37 +169,50 @@ def invoke_stream(self, client, message, messages): current_batch = "" first_chunk = True - for chunk in client.chat.completions.create( - messages=messages, - model=self.model, - temperature=self.temperature, - stream=True, - ): - if chunk.choices[0].delta.content is not None: - content = chunk.choices[0].delta.content - aggregate_result += content - current_batch += content - if len(current_batch.split()) >= self.stream_batch_size: - if self.stream_to_flow: - self.send_streaming_message( - message, - current_batch, - aggregate_result, - response_uuid, - first_chunk, - False, - ) - elif self.stream_to_next_component: - self.send_to_next_component( - message, - current_batch, - aggregate_result, - response_uuid, - first_chunk, - False, - ) - current_batch = "" - first_chunk = False + max_retries = 3 + while max_retries > 0: + try: + for chunk in client.chat.completions.create( + messages=messages, + model=self.model, + temperature=self.temperature, + stream=True, + ): + # If we get any response, then don't retry + max_retries = 0 + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + aggregate_result += content + current_batch += content + if len(current_batch.split()) >= self.stream_batch_size: + if self.stream_to_flow: + self.send_streaming_message( + message, + current_batch, + aggregate_result, + response_uuid, + first_chunk, + False, + ) + elif self.stream_to_next_component: + self.send_to_next_component( + message, + current_batch, + aggregate_result, + response_uuid, + first_chunk, + False, + ) + current_batch = "" + first_chunk = False + except Exception as e: + log.error("Error invoking OpenAI: %s", e) + max_retries -= 1 + if max_retries <= 0: + raise e + else: + # Small delay before retrying + time.sleep(1) if self.stream_to_next_component: # Just return the last chunk diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py b/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py index b363776..cb217b9 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py @@ -11,6 +11,7 @@ from ...common.log import log from .broker_base import BrokerBase from ...common.message import Message +from ...common.utils import ensure_slash_on_end, ensure_slash_on_start # from ...common.event import Event, EventType @@ -59,11 +60,30 @@ "description": "Format for the payload (json, yaml, text)", "default": "json", }, + { + "name": "response_topic_prefix", + "required": False, + "description": "Prefix for reply topics", + "default": "reply", + }, + { + "name": "response_topic_suffix", + "required": False, + "description": "Suffix for reply topics", + "default": "", + }, + { + "name": "reply_queue_prefix", + "required": False, + "description": "Prefix for reply queues", + "default": "reply-queue", + }, { "name": "request_expiry_ms", "required": False, "description": "Expiry time for cached requests in milliseconds", "default": 60000, + "type": "integer", }, { "name": "streaming", @@ -94,6 +114,10 @@ "type": "object", "description": "User properties to send with the request message", }, + "response_topic_suffix": { + "type": "string", + "description": "Suffix for the reply topic", + }, "stream": { "type": "boolean", "description": "Whether this will have a streaming response", @@ -137,8 +161,18 @@ def __init__(self, **kwargs): super().__init__(info, **kwargs) self.need_acknowledgement = False self.request_expiry_ms = self.get_config("request_expiry_ms") - self.reply_queue_name = f"reply-queue-{uuid.uuid4()}" - self.reply_topic = f"reply/{uuid.uuid4()}" + self.response_topic_prefix = ensure_slash_on_end( + self.get_config("response_topic_prefix") + ) + self.response_topic_suffix = ensure_slash_on_start( + self.get_config("response_topic_suffix") + ) + self.reply_queue_prefix = ensure_slash_on_end( + self.get_config("reply_queue_prefix") + ) + self.requestor_id = str(uuid.uuid4()) + self.reply_queue_name = f"{self.reply_queue_prefix}{self.requestor_id}" + self.response_topic = f"{self.response_topic_prefix}{self.requestor_id}{self.response_topic_suffix}" self.response_thread = None self.streaming = self.get_config("streaming") self.streaming_complete_expression = self.get_config( @@ -149,9 +183,13 @@ def __init__(self, **kwargs): self.broker_properties["queue_name"] = self.reply_queue_name self.broker_properties["subscriptions"] = [ { - "topic": self.reply_topic, + "topic": self.response_topic, "qos": 1, - } + }, + { + "topic": self.response_topic + "/>", + "qos": 1, + }, ] self.test_mode = False @@ -168,7 +206,7 @@ def start(self): def setup_reply_queue(self): self.messaging_service.bind_to_queue( - self.reply_queue_name, [self.reply_topic], temporary=True + self.reply_queue_name, [self.response_topic], temporary=True ) def setup_test_pass_through(self): @@ -269,15 +307,15 @@ def process_response(self, broker_message): ] = json.dumps(metadata_stack) # Put the last reply topic back in the user properties response["user_properties"][ - "__solace_ai_connector_broker_request_reply_topic__" - ] = metadata_stack[-1]["reply_topic"] + "__solace_ai_connector_broker_request_response_topic__" + ] = metadata_stack[-1]["response_topic"] else: # Remove the metadata and reply topic from the user properties response["user_properties"].pop( "__solace_ai_connector_broker_request_reply_metadata__", None ) response["user_properties"].pop( - "__solace_ai_connector_broker_request_reply_topic__", None + "__solace_ai_connector_broker_request_response_topic__", None ) message = Message( @@ -310,7 +348,11 @@ def invoke(self, message, data): if "streaming_complete_expression" in data: streaming_complete_expression = data["streaming_complete_expression"] - metadata = {"request_id": request_id, "reply_topic": self.reply_topic} + topic = self.response_topic + if "response_topic_suffix" in data: + topic = f"{topic}/{data['response_topic_suffix']}" + + metadata = {"request_id": request_id, "response_topic": topic} if ( "__solace_ai_connector_broker_request_reply_metadata__" @@ -343,8 +385,8 @@ def invoke(self, message, data): "__solace_ai_connector_broker_request_reply_metadata__" ] = json.dumps(metadata) data["user_properties"][ - "__solace_ai_connector_broker_request_reply_topic__" - ] = self.reply_topic + "__solace_ai_connector_broker_request_response_topic__" + ] = self.response_topic if self.test_mode: if self.broker_type == "test_streaming": From 5aef3f7dd6763dc862a397d7c817ab388e6f9af0 Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Mon, 30 Sep 2024 13:47:22 -0400 Subject: [PATCH 24/55] Add a bit more visibility on startup to mirror the shutdown (#46) --- src/solace_ai_connector/solace_ai_connector.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/solace_ai_connector/solace_ai_connector.py b/src/solace_ai_connector/solace_ai_connector.py index 40240cc..91c349e 100644 --- a/src/solace_ai_connector/solace_ai_connector.py +++ b/src/solace_ai_connector/solace_ai_connector.py @@ -35,7 +35,7 @@ def __init__(self, config, event_handlers=None, error_queue=None): def run(self): """Run the Solace AI Event Connector""" - log.debug("Starting Solace AI Event Connector") + log.info("Starting Solace AI Event Connector") try: self.create_flows() @@ -44,6 +44,7 @@ def run(self): if on_flow_creation: on_flow_creation(self.flows) + log.info("Solace AI Event Connector started successfully") except Exception as e: log.error("Error during Solace AI Event Connector startup: %s", str(e)) self.stop() @@ -53,7 +54,7 @@ def run(self): def create_flows(self): """Loop through the flows and create them""" for index, flow in enumerate(self.config.get("flows", [])): - log.debug("Creating flow %s", flow.get("name")) + log.info("Creating flow %s", flow.get("name")) num_instances = flow.get("num_instances", 1) if num_instances < 1: num_instances = 1 From 4033193516e217cbe992617fe9b1703fa5ed680a Mon Sep 17 00:00:00 2001 From: Alireza Parvizimosaed Date: Tue, 1 Oct 2024 01:28:13 -0400 Subject: [PATCH 25/55] added some tags to convert OpenAI and Langchain output to JSON format --- .../langchain_openai_with_history_chat.yaml | 1 + examples/llm/openai_chat.yaml | 1 + examples/websearch/websearch_router.yaml | 62 ++++++------------- src/solace_ai_connector/common/utils.py | 1 + .../general/openai/openai_chat_model_base.py | 3 +- .../general/websearch/websearch_duckduckgo.py | 6 +- 6 files changed, 29 insertions(+), 45 deletions(-) mode change 100644 => 100755 examples/llm/langchain_openai_with_history_chat.yaml mode change 100644 => 100755 examples/llm/openai_chat.yaml mode change 100644 => 100755 src/solace_ai_connector/common/utils.py mode change 100644 => 100755 src/solace_ai_connector/components/general/openai/openai_chat_model_base.py diff --git a/examples/llm/langchain_openai_with_history_chat.yaml b/examples/llm/langchain_openai_with_history_chat.yaml old mode 100644 new mode 100755 index bd922d3..fa27307 --- a/examples/llm/langchain_openai_with_history_chat.yaml +++ b/examples/llm/langchain_openai_with_history_chat.yaml @@ -65,6 +65,7 @@ flows: base_url: ${OPENAI_API_ENDPOINT} model: ${MODEL_NAME} temperature: 0.01 + llm_response_format: 'text' history_module: langchain_core.chat_history history_class: InMemoryChatMessageHistory history_max_turns: 20 diff --git a/examples/llm/openai_chat.yaml b/examples/llm/openai_chat.yaml old mode 100644 new mode 100755 index 71aee14..2009320 --- a/examples/llm/openai_chat.yaml +++ b/examples/llm/openai_chat.yaml @@ -63,6 +63,7 @@ flows: base_url: ${OPENAI_API_ENDPOINT} model: ${OPENAI_MODEL_NAME} temperature: 0.01 + response_format: text # json_object or json_schema input_transforms: - type: copy source_expression: | diff --git a/examples/websearch/websearch_router.yaml b/examples/websearch/websearch_router.yaml index 4bad28f..3e14ce7 100755 --- a/examples/websearch/websearch_router.yaml +++ b/examples/websearch/websearch_router.yaml @@ -1,15 +1,15 @@ -# This will create a flow like this: -# Solace -> OpenAI -> Solace +# This is a router that search either a LLM or web search engine +# Flow 1: Interprets a query and route the next flow (#2 or #3). It gets requests from Solace broker on search/query topic and forward the request to either search/query/llm or search/query/web_search topic. +# Flow 2: Listens to the search/query/web_search topic and uses the Google API to answer the query. It cleans the result by a LLM agent, and finally returns the response to search/query/web_search/response topic. +# Flow 3: Listens to the search/query/llm topic and uses the LLM agent to answer the query. It cleans the result by a LLM agent, and finally returns the response to search/query/llm/response topic. # -# It will subscribe to `demo/joke/subject` and expect an event with the payload: +# It will subscribe to `search/query` and expect an event with the payload: # # { -# "joke": { -# "subject": "" -# } +# "text": "" # } # -# It will then send an event back to Solace with the topic: `demo/joke/subject/response` +# It will then send an event back to Solace with the topics: `search/query/web_search/response` or `search/query/llm/response` # # Dependencies: # pip install -U langchain_openai openai @@ -23,7 +23,6 @@ # - SOLACE_BROKER_PASSWORD # - SOLACE_BROKER_VPN ---- log: stdout_log_level: INFO log_file_level: DEBUG @@ -47,7 +46,7 @@ flows: component_module: broker_input component_config: <<: *broker_connection - broker_queue_name: web_search_broker + broker_queue_name: web_search_broker1 broker_subscriptions: - topic: search/query qos: 1 @@ -67,7 +66,7 @@ flows: temperature: 0.01 history_module: langchain_core.chat_history history_class: InMemoryChatMessageHistory - history_max_turns: 20 + history_max_turns: 0 history_max_length: 1 input_transforms: - type: copy @@ -80,7 +79,7 @@ flows: } { name: web_search - description: Use for looking up real-time or current information, recent events, latest news, up-to-date statistics, current prices, weather forecasts, ongoing developments, or any query that requires the most recent data. This agent is also suitable for fact-checking or verifying claims about recent occurrences. + description: Use for searching real-time, current or up-to-date information, recent events, latest news, up-to-date statistics, current prices, weather forecasts, ongoing developments, or any query that requires the most recent data. This agent is also suitable for fact-checking or verifying claims about recent occurrences. } @@ -89,7 +88,7 @@ flows: 2. For questions about historical events, established scientific facts, or timeless concepts, choose LLM. 3. If the query is about a potentially evolving situation but doesn't specifically request current information, lean towards web_search to ensure accuracy. 4. For tasks involving analysis, creative writing, or explanation of concepts, choose LLM. - 5. If unsure, err on the side of web_search to provide the most up-to-date information. + 5. If unsure, select web_search to provide the most up-to-date information. Examples: 1. Query: "What is the current stock price of Apple?" @@ -158,10 +157,10 @@ flows: function: equal params: positional: - - evaluate_expression(user_data.output:payload, text) - - "web_search" - - search/query/llm + - evaluate_expression(previous, text) + - "LLM" - search/query/llm + - search/query/web_search dest_expression: user_data.output:topic input_selection: source_expression: user_data.output @@ -205,7 +204,7 @@ flows: temperature: 0.01 history_module: langchain_core.chat_history history_class: InMemoryChatMessageHistory - history_max_turns: 20 + history_max_turns: 0 history_max_length: 1 input_transforms: - type: copy @@ -233,16 +232,6 @@ flows: input_selection: source_expression: user_data.input - # # Filter json - # - component_name: filter_json_component - # component_module: filter_by_llm - # input_transforms: - # - type: copy - # source_expression: previous - # dest_expression: user_data.input:payload - # input_selection: - # source_expression: input.payload - # Send response back to broker with completion and retrieved data - component_name: send_response component_module: broker_output @@ -264,7 +253,7 @@ flows: input_selection: source_expression: user_data.output -# Flow 3: LLM + # Flow 3: LLM - name: LLM components: # Agent broker input configuration @@ -292,17 +281,14 @@ flows: temperature: 0.01 history_module: langchain_core.chat_history history_class: InMemoryChatMessageHistory - history_max_turns: 20 + history_max_turns: 0 history_max_length: 1 input_transforms: - - type: copy - source_expression: previous - dest_expression: user_data.input:payload - type: copy source_expression: | template:You are a helpful AI assistant. Please help with the user's request below: - {{text://user_data.input:payload}} + {{text://input.payload:text}} dest_expression: user_data.input:messages.0.content - type: copy @@ -324,7 +310,7 @@ flows: temperature: 0.01 history_module: langchain_core.chat_history history_class: InMemoryChatMessageHistory - history_max_turns: 20 + history_max_turns: 0 history_max_length: 1 input_transforms: - type: copy @@ -352,16 +338,6 @@ flows: input_selection: source_expression: user_data.input - # # Filter json - # - component_name: filter_json_component - # component_module: filter_by_llm - # input_transforms: - # - type: copy - # source_expression: previous - # dest_expression: user_data.input:payload - # input_selection: - # source_expression: input.payload - # Send response back to broker with completion and retrieved data - component_name: send_response component_module: broker_output diff --git a/src/solace_ai_connector/common/utils.py b/src/solace_ai_connector/common/utils.py old mode 100644 new mode 100755 index 7f26c98..2959720 --- a/src/solace_ai_connector/common/utils.py +++ b/src/solace_ai_connector/common/utils.py @@ -124,6 +124,7 @@ def import_module(module, base_path=None, component_package=None): ".components.general.openai", ".components.general.websearch", ".components.inputs_outputs", + ".components.general.filter", ".transforms", ".common", ]: diff --git a/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py b/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py old mode 100644 new mode 100755 index 0ce6c90..3d2541b --- a/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py +++ b/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py @@ -123,6 +123,7 @@ def init(self): self.stream_to_next_component = self.get_config("stream_to_next_component") self.llm_mode = self.get_config("llm_mode") self.stream_batch_size = self.get_config("stream_batch_size") + self.response_format = self.get_config("response_format", "text") self.set_response_uuid_in_user_properties = self.get_config( "set_response_uuid_in_user_properties" ) @@ -142,7 +143,7 @@ def invoke(self, message, data): return self.invoke_stream(client, message, messages) else: response = client.chat.completions.create( - messages=messages, model=self.model, temperature=self.temperature + messages=messages, model=self.model, temperature=self.temperature, response_format={"type": self.response_format} ) return {"content": response.choices[0].message.content} diff --git a/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py b/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py index 150df1c..088cc91 100755 --- a/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py +++ b/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py @@ -88,4 +88,8 @@ def parse(self, message): if self.detail: return message else: - return message['Abstract'] \ No newline at end of file + return { + "Title": message['AbstractSource'], + "Snippet": message['Abstract'], + "URL": message['AbstractURL'] + } \ No newline at end of file From baa15cdb140a7f669048ce7c694ce852b7789ea1 Mon Sep 17 00:00:00 2001 From: Alireza Parvizimosaed Date: Thu, 3 Oct 2024 14:45:50 -0400 Subject: [PATCH 26/55] added unicode_escape decoder to the list of broker formats --- .../components/inputs_outputs/broker_base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_base.py b/src/solace_ai_connector/components/inputs_outputs/broker_base.py index fac4207..a641427 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_base.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_base.py @@ -76,6 +76,9 @@ def decode_payload(self, payload): isinstance(payload, bytes) or isinstance(payload, bytearray) ): payload = payload.decode("utf-8") + elif encoding == "unicode_escape": + payload = payload.decode('unicode_escape') + if payload_format == "json": payload = json.loads(payload) elif payload_format == "yaml": From ff349d76d24b4826348f7d3aa52ce9d9d6ca69f3 Mon Sep 17 00:00:00 2001 From: Alireza Parvizimosaed Date: Thu, 3 Oct 2024 14:48:01 -0400 Subject: [PATCH 27/55] returned the requirements.txt back --- requirements.txt | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8071e4f..3d40e69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,6 @@ boto3~=1.34.122 -langchain-core~=0.2.5 -langchain~=0.2.3 +langchain-core~=0.3.0 +langchain~=0.3.0 PyYAML~=6.0.1 Requests~=2.32.3 -solace_pubsubplus~=1.8.0 -solace_ai_connector~=0.1.5 -langchain-openai~=0.1.25 -openai~=1.47.0 \ No newline at end of file +solace_pubsubplus~=1.8.0 \ No newline at end of file From be8d9e360ec60c6202024359d6747c014dec41f3 Mon Sep 17 00:00:00 2001 From: Alireza Parvizimosaed Date: Thu, 3 Oct 2024 14:52:45 -0400 Subject: [PATCH 28/55] returned the requirements.txt back --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d40e69..66ec399 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ langchain-core~=0.3.0 langchain~=0.3.0 PyYAML~=6.0.1 Requests~=2.32.3 -solace_pubsubplus~=1.8.0 \ No newline at end of file +solace_pubsubplus~=1.8.0 From d508c06d634477c1081e192c04f3c170b34eecfd Mon Sep 17 00:00:00 2001 From: Alireza Parvizimosaed Date: Thu, 3 Oct 2024 15:42:20 -0400 Subject: [PATCH 29/55] clarified some workflows execution --- examples/websearch/bing_web_search.yaml | 6 ++++++ examples/websearch/duckduckgo_web_search.yaml | 4 ++++ examples/websearch/google_web_search.yaml | 6 ++++++ examples/websearch/web_scraping.yaml | 10 ++++++++++ 4 files changed, 26 insertions(+) diff --git a/examples/websearch/bing_web_search.yaml b/examples/websearch/bing_web_search.yaml index 2863b89..e828c2d 100755 --- a/examples/websearch/bing_web_search.yaml +++ b/examples/websearch/bing_web_search.yaml @@ -1,3 +1,9 @@ +# This is a Bing search engine workflow +# The input payload is: +# +# +# Do not forget to set the bing api_key in the below configuration. + log: stdout_log_level: INFO log_file_level: INFO diff --git a/examples/websearch/duckduckgo_web_search.yaml b/examples/websearch/duckduckgo_web_search.yaml index b1b24f2..85e4e2d 100755 --- a/examples/websearch/duckduckgo_web_search.yaml +++ b/examples/websearch/duckduckgo_web_search.yaml @@ -1,3 +1,7 @@ +# This is a Duck Duck Go search engine workflow +# The input payload is: +# + log: stdout_log_level: INFO log_file_level: INFO diff --git a/examples/websearch/google_web_search.yaml b/examples/websearch/google_web_search.yaml index 6766a9f..34d9d7b 100755 --- a/examples/websearch/google_web_search.yaml +++ b/examples/websearch/google_web_search.yaml @@ -1,3 +1,9 @@ +# This is a Google search engine workflow +# The input payload is: +# +# +# Do not forget to set the api_key and search_engine_id in the below configuration. + log: stdout_log_level: INFO log_file_level: INFO diff --git a/examples/websearch/web_scraping.yaml b/examples/websearch/web_scraping.yaml index cb4616e..6336302 100755 --- a/examples/websearch/web_scraping.yaml +++ b/examples/websearch/web_scraping.yaml @@ -1,3 +1,13 @@ +# This example gets a url and scrapes the website. +# +# The input payload is: +# +# +# Ensure that the browser binaries are installed. If not, install them by: +# ```sh +# playwright install +# ``` + log: stdout_log_level: INFO log_file_level: INFO From 44db905cd6ca928b512cb4b164c4f7924ae33ee6 Mon Sep 17 00:00:00 2001 From: alimosaed Date: Thu, 10 Oct 2024 10:48:11 -0400 Subject: [PATCH 30/55] addressed code review comments --- docs/getting_started.md | 4 -- .../langchain_openai_with_history_chat.yaml | 2 +- examples/llm/openai_chat.yaml | 2 +- examples/parser.yaml | 28 ++++++++ examples/websearch/bing_web_search.yaml | 6 +- examples/websearch/duckduckgo_web_search.yaml | 1 - examples/websearch/google_web_search.yaml | 9 +-- examples/websearch/web_scraping.yaml | 6 +- examples/websearch/websearch_router.yaml | 7 +- requirements.txt | 3 +- .../components/__init__.py | 7 +- .../components/general/filter/__init__.py | 0 .../general/filter/filter_by_llm.py | 65 ------------------- .../langchain/langchain_chat_model_base.py | 6 +- .../components/general/parser.py | 43 ++++++++++++ .../general/websearch/web_scraper.py | 26 +++++++- .../general/websearch/websearch_base.py | 1 - .../general/websearch/websearch_bing.py | 37 ++++------- .../general/websearch/websearch_duckduckgo.py | 35 ++++------ .../general/websearch/websearch_google.py | 33 ++++------ 20 files changed, 158 insertions(+), 163 deletions(-) create mode 100755 examples/parser.yaml delete mode 100644 src/solace_ai_connector/components/general/filter/__init__.py delete mode 100644 src/solace_ai_connector/components/general/filter/filter_by_llm.py create mode 100644 src/solace_ai_connector/components/general/parser.py diff --git a/docs/getting_started.md b/docs/getting_started.md index e5adb4c..53d847b 100755 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -135,10 +135,6 @@ In the "Try Me!" also subscribe to `demo/joke/subject/response` to see the respo ```sh pip install -r requirements.txt ``` -4. Install the necessary browser binaries for the web scraper. - ```sh - playwright install - ``` ## Configuration diff --git a/examples/llm/langchain_openai_with_history_chat.yaml b/examples/llm/langchain_openai_with_history_chat.yaml index fa27307..bef45af 100755 --- a/examples/llm/langchain_openai_with_history_chat.yaml +++ b/examples/llm/langchain_openai_with_history_chat.yaml @@ -65,7 +65,7 @@ flows: base_url: ${OPENAI_API_ENDPOINT} model: ${MODEL_NAME} temperature: 0.01 - llm_response_format: 'text' + llm_response_format: text # options: text, json, or yaml history_module: langchain_core.chat_history history_class: InMemoryChatMessageHistory history_max_turns: 20 diff --git a/examples/llm/openai_chat.yaml b/examples/llm/openai_chat.yaml index 2009320..54db782 100755 --- a/examples/llm/openai_chat.yaml +++ b/examples/llm/openai_chat.yaml @@ -63,7 +63,7 @@ flows: base_url: ${OPENAI_API_ENDPOINT} model: ${OPENAI_MODEL_NAME} temperature: 0.01 - response_format: text # json_object or json_schema + response_format: text # options: text, json_object, or json_schema input_transforms: - type: copy source_expression: | diff --git a/examples/parser.yaml b/examples/parser.yaml new file mode 100755 index 0000000..ead9324 --- /dev/null +++ b/examples/parser.yaml @@ -0,0 +1,28 @@ +# This is a simple parser workflow +# The input payload is: +# +# + +log: + stdout_log_level: INFO + log_file_level: INFO + log_file: solace_ai_connector.log + +flows: + - name: parser + components: + # Input from a standard in + - component_name: stdin + component_module: stdin_input + + # Using Custom component + - component_name: parser_component + component_module: parser + component_config: + input_format: yaml + input_selection: + source_expression: previous + + # Output to a standard out + - component_name: stdout + component_module: stdout_output \ No newline at end of file diff --git a/examples/websearch/bing_web_search.yaml b/examples/websearch/bing_web_search.yaml index e828c2d..378fb48 100755 --- a/examples/websearch/bing_web_search.yaml +++ b/examples/websearch/bing_web_search.yaml @@ -2,7 +2,8 @@ # The input payload is: # # -# Do not forget to set the bing api_key in the below configuration. +# Required ENV variables: +# - BING_API_KEY log: stdout_log_level: INFO @@ -20,8 +21,7 @@ flows: - component_name: web_search_component component_module: websearch_bing component_config: - engine: bing - api_key: + api_key: ${BING_API_KEY} safesearch: Moderate count: 2 detail: false diff --git a/examples/websearch/duckduckgo_web_search.yaml b/examples/websearch/duckduckgo_web_search.yaml index 85e4e2d..76eb588 100755 --- a/examples/websearch/duckduckgo_web_search.yaml +++ b/examples/websearch/duckduckgo_web_search.yaml @@ -18,7 +18,6 @@ flows: - component_name: web_search_component component_module: websearch_duckduckgo component_config: - engine: duckduckgo detail: false input_selection: source_expression: previous diff --git a/examples/websearch/google_web_search.yaml b/examples/websearch/google_web_search.yaml index 34d9d7b..f42e8df 100755 --- a/examples/websearch/google_web_search.yaml +++ b/examples/websearch/google_web_search.yaml @@ -2,7 +2,9 @@ # The input payload is: # # -# Do not forget to set the api_key and search_engine_id in the below configuration. +# Required ENV variables: +# - Google_API_KEY +# - GOOGLE_ENGINE_ID log: stdout_log_level: INFO @@ -20,9 +22,8 @@ flows: - component_name: web_search_component component_module: websearch_google component_config: - engine: google - api_key: - search_engine_id: + api_key: ${GOOGLE_API_KEY} + search_engine_id: ${GOOGLE_ENGINE_ID} detail: false input_selection: source_expression: previous diff --git a/examples/websearch/web_scraping.yaml b/examples/websearch/web_scraping.yaml index 6336302..df06494 100755 --- a/examples/websearch/web_scraping.yaml +++ b/examples/websearch/web_scraping.yaml @@ -3,7 +3,11 @@ # The input payload is: # # -# Ensure that the browser binaries are installed. If not, install them by: +# Ensure that the playwright package and the browser binaries are installed. If not, install them by: +# ```sh +# pip install playwright +# ``` +# and then run: # ```sh # playwright install # ``` diff --git a/examples/websearch/websearch_router.yaml b/examples/websearch/websearch_router.yaml index 3e14ce7..aea0d16 100755 --- a/examples/websearch/websearch_router.yaml +++ b/examples/websearch/websearch_router.yaml @@ -22,6 +22,8 @@ # - SOLACE_BROKER_USERNAME # - SOLACE_BROKER_PASSWORD # - SOLACE_BROKER_VPN +# - GOOGLE_API_KEY +# - GOOGLE_ENGINE_ID log: stdout_log_level: INFO @@ -184,9 +186,8 @@ flows: - component_name: web_search_component component_module: websearch_google component_config: - engine: google - api_key: AIzaSyA7kKgIBK_Clw-HFb_YZYSxwQgKEX1rza8 - search_engine_id: d244da49663d44d5e + api_key: ${GOOGLE_API_KEY} + search_engine_id: ${GOOGLE_ENGINE_ID} detail: false input_selection: source_expression: input.payload diff --git a/requirements.txt b/requirements.txt index 7bbfb9d..3d40e69 100755 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,4 @@ langchain-core~=0.3.0 langchain~=0.3.0 PyYAML~=6.0.1 Requests~=2.32.3 -solace_pubsubplus~=1.8.0 -playwright~=1.47.0 \ No newline at end of file +solace_pubsubplus~=1.8.0 \ No newline at end of file diff --git a/src/solace_ai_connector/components/__init__.py b/src/solace_ai_connector/components/__init__.py index 3bb8b7a..e9e54b9 100755 --- a/src/solace_ai_connector/components/__init__.py +++ b/src/solace_ai_connector/components/__init__.py @@ -16,6 +16,7 @@ delay, iterate, message_filter, + parser, ) from .general.for_testing import ( @@ -39,10 +40,6 @@ websearch_bing ) -from .general.filter import ( - filter_by_llm -) - # Also import the components from the submodules from .inputs_outputs.error_input import ErrorInput from .inputs_outputs.timer_input import TimerInput @@ -59,6 +56,7 @@ from .general.delay import Delay from .general.iterate import Iterate from .general.message_filter import MessageFilter +from .general.parser import Parser from .general.langchain.langchain_base import LangChainBase from .general.langchain.langchain_embeddings import LangChainEmbeddings from .general.langchain.langchain_vector_store_delete import LangChainVectorStoreDelete @@ -75,5 +73,4 @@ from .general.websearch.websearch_duckduckgo import WebSearchDuckDuckGo from .general.websearch.websearch_google import WebSearchGoogle from .general.websearch.websearch_bing import WebSearchBing -from .general.filter.filter_by_llm import CleanJsonObject diff --git a/src/solace_ai_connector/components/general/filter/__init__.py b/src/solace_ai_connector/components/general/filter/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/solace_ai_connector/components/general/filter/filter_by_llm.py b/src/solace_ai_connector/components/general/filter/filter_by_llm.py deleted file mode 100644 index 16429a5..0000000 --- a/src/solace_ai_connector/components/general/filter/filter_by_llm.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Remove any unnecessary texts before and after a json object.""" - -import json - -from ...component_base import ComponentBase - -info = { - "class_name": "CleanJsonObject", - "description": "Scrape javascript based websites.", - "config_parameters": [ - ], - "input_schema": { - "type": "object", - "properties": {} - }, - "output_schema": { - "type": "object", - "properties": {} - } -} - -class CleanJsonObject(ComponentBase): - def __init__(self, **kwargs): - super().__init__(info, **kwargs) - - def invoke(self, message, data): - text = data["text"] - json_obj = self.extract_json(text) - return json_obj - - # Clean a json object - def extract_json(self, text): - start_index = text.find('{') - end_index = start_index - brace_count = 0 - - for i, char in enumerate(text[start_index:], start=start_index): - if char == '{': - brace_count += 1 - elif char == '}': - brace_count -= 1 - if brace_count == 0: - end_index = i - break - - if brace_count == 0: - json_str = text[start_index:end_index + 1] - # If a JSON object is found, convert to JSON - if json_str: - try: - json_object = json.loads(json_str) - return json.dumps(json_object, indent=4) - except json.JSONDecodeError as e: - raise ValueError(f"Error converting text to json: {str(e)}") from e - else: - return None - else: - return None - - - - - - - diff --git a/src/solace_ai_connector/components/general/langchain/langchain_chat_model_base.py b/src/solace_ai_connector/components/general/langchain/langchain_chat_model_base.py index 8965324..bd3b464 100644 --- a/src/solace_ai_connector/components/general/langchain/langchain_chat_model_base.py +++ b/src/solace_ai_connector/components/general/langchain/langchain_chat_model_base.py @@ -1,9 +1,9 @@ # This is the base class of a wrapper around all the LangChain chat models # The configuration will control dynamic loading of the chat models -import json import yaml from abc import abstractmethod +from langchain_core.output_parsers import JsonOutputParser from ....common.utils import get_obj_text from langchain.schema.messages import ( @@ -116,9 +116,9 @@ def invoke(self, message, data): res_format = self.get_config("llm_response_format", "text") if res_format == "json": - obj_text = get_obj_text("json", llm_res.content) try: - json_res = json.loads(obj_text) + parser = JsonOutputParser() + json_res = parser.invoke(llm_res.content) return json_res except Exception as e: raise ValueError(f"Error parsing LLM JSON response: {str(e)}") from e diff --git a/src/solace_ai_connector/components/general/parser.py b/src/solace_ai_connector/components/general/parser.py new file mode 100644 index 0000000..e6dd705 --- /dev/null +++ b/src/solace_ai_connector/components/general/parser.py @@ -0,0 +1,43 @@ +"""Parse a JSON or YAML file.""" +import yaml +from langchain_core.output_parsers import JsonOutputParser + +from ..component_base import ComponentBase +from ...common.utils import get_obj_text + + +info = { + "class_name": "Parser", + "description": "Parse a JSON string and extract data fields.", + "config_parameters": [ + { + "name": "input_format", + "required": True, + "description": "The input format of the data. Options: 'json', 'yaml'.", + }, + ], +} + +class Parser(ComponentBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + + def invoke(self, message, data): + text = data["text"] + res_format = self.get_config("input_format", "json") + if res_format == "json": + try: + parser = JsonOutputParser() + json_res = parser.invoke(text) + return json_res + except Exception as e: + raise ValueError(f"Error parsing the input JSON: {str(e)}") from e + elif res_format == "yaml": + obj_text = get_obj_text("yaml", text) + try: + yaml_res = yaml.safe_load(obj_text) + return yaml_res + except Exception as e: + raise ValueError(f"Error parsing the input YAML: {str(e)}") from e + else: + return text diff --git a/src/solace_ai_connector/components/general/websearch/web_scraper.py b/src/solace_ai_connector/components/general/websearch/web_scraper.py index eee6858..1ce922b 100755 --- a/src/solace_ai_connector/components/general/websearch/web_scraper.py +++ b/src/solace_ai_connector/components/general/websearch/web_scraper.py @@ -1,6 +1,8 @@ """Scrape a website""" -from playwright.sync_api import sync_playwright +import subprocess +import sys from ...component_base import ComponentBase +from ....common.log import log info = { "class_name": "WebScraper", @@ -28,9 +30,27 @@ def invoke(self, message, data): # Scrape a website def scrape(self, url): + try: + from playwright.sync_api import sync_playwright + except ImportError: + log.error( + "Please install playwright by running 'pip install playwright' and 'playwright install'." + ) + raise ValueError( + "Please install playwright by running 'pip install playwright' and 'playwright install'." + ) + with sync_playwright() as p: - # Launch a browser instance (Chromium, Firefox, or WebKit) - browser = p.chromium.launch(headless=True) # Set headless=False to see the browser in action + try: + # Launch a Chromium browser instance + browser = p.chromium.launch(headless=True) # Set headless=False to see the browser in action + except ImportError: + log.error( + f"Failed to launch the Chromium instance. Please install the browser binaries by running 'playwright install'" + ) + raise ValueError( + f"Failed to launch the Chromium instance. Please install the browser binaries by running 'playwright install'" + ) page = browser.new_page() page.goto(url) diff --git a/src/solace_ai_connector/components/general/websearch/websearch_base.py b/src/solace_ai_connector/components/general/websearch/websearch_base.py index 56a6fd5..773ff94 100755 --- a/src/solace_ai_connector/components/general/websearch/websearch_base.py +++ b/src/solace_ai_connector/components/general/websearch/websearch_base.py @@ -31,7 +31,6 @@ class WebSearchBase(ComponentBase): def __init__(self, info_base, **kwargs): super().__init__(info_base, **kwargs) - self.engine = self.get_config("engine") self.detail = self.get_config("detail") def invoke(self, message, data): diff --git a/src/solace_ai_connector/components/general/websearch/websearch_bing.py b/src/solace_ai_connector/components/general/websearch/websearch_bing.py index 2cda058..2f6c1b5 100755 --- a/src/solace_ai_connector/components/general/websearch/websearch_bing.py +++ b/src/solace_ai_connector/components/general/websearch/websearch_bing.py @@ -10,12 +10,6 @@ "class_name": "WebSearchBing", "description": "Perform a search query on Bing.", "config_parameters": [ - { - "name": "engine", - "required": True, - "description": "The type of search engine.", - "default": "bing" - }, { "name": "api_key", "required": True, @@ -57,25 +51,22 @@ def init(self): def invoke(self, message, data): query = data["text"] - if self.engine.lower() == "bing": - params = { - "q": query, # User query - "count": self.count, # Number of results to return - "safesearch": self.safesearch # Safe search filter - } - headers = { - "Ocp-Apim-Subscription-Key": self.api_key # Bing API Key - } + params = { + "q": query, # User query + "count": self.count, # Number of results to return + "safesearch": self.safesearch # Safe search filter + } + headers = { + "Ocp-Apim-Subscription-Key": self.api_key # Bing API Key + } - response = requests.get(self.url, headers=headers, params=params) - if response.status_code == 200: - response = response.json() - response = self.parse(response) - return response - else: - return f"Error: {response.status_code}" + response = requests.get(self.url, headers=headers, params=params) + if response.status_code == 200: + response = response.json() + response = self.parse(response) + return response else: - return f"Error: The engine is not Bing." + return f"Error: {response.status_code}" # Extract required data from a message def parse(self, message): diff --git a/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py b/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py index 088cc91..71a76b4 100755 --- a/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py +++ b/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py @@ -10,12 +10,6 @@ "class_name": "WebSearchDuckDuckGo", "description": "Perform a search query on DuckDuckGo.", "config_parameters": [ - { - "name": "engine", - "required": True, - "description": "The type of search engine.", - "default": "duckduckgo" - }, { "name": "pretty", "required": False, @@ -64,24 +58,21 @@ def init(self): def invoke(self, message, data): query = data["text"] - if self.engine.lower() == "duckduckgo": - params = { - "q": query, # User query - "format": "json", # Response format (json by default) - "pretty": self.pretty, # Beautify the output - "no_html": self.no_html, # Remove HTML from the response - "skip_disambig": self.skip_disambig # Skip disambiguation - } + params = { + "q": query, # User query + "format": "json", # Response format (json by default) + "pretty": self.pretty, # Beautify the output + "no_html": self.no_html, # Remove HTML from the response + "skip_disambig": self.skip_disambig # Skip disambiguation + } - response = requests.get(self.url, params=params) - if response.status_code == 200: - response = response.json() - response = self.parse(response) - return response - else: - return f"Error: {response.status_code}" + response = requests.get(self.url, params=params) + if response.status_code == 200: + response = response.json() + response = self.parse(response) + return response else: - return f"Error: The engine is not DuckDuckGo." + return f"Error: {response.status_code}" # Extract required data from a message def parse(self, message): diff --git a/src/solace_ai_connector/components/general/websearch/websearch_google.py b/src/solace_ai_connector/components/general/websearch/websearch_google.py index ecacdf9..5371d80 100755 --- a/src/solace_ai_connector/components/general/websearch/websearch_google.py +++ b/src/solace_ai_connector/components/general/websearch/websearch_google.py @@ -11,12 +11,6 @@ "class_name": "WebSearchGoogle", "description": "Perform a search query on Google.", "config_parameters": [ - { - "name": "engine", - "required": True, - "description": "The type of search engine.", - "default": "google" - }, { "name": "api_key", "required": True, @@ -57,23 +51,20 @@ def init(self): def invoke(self, message, data): query = data["text"] - if self.engine.lower() == "google": - params = { - "q": query, # User query - "key": self.api_key, # Google API Key - "cx": self.search_engine_id, # Google custom search engine id - } + params = { + "q": query, # User query + "key": self.api_key, # Google API Key + "cx": self.search_engine_id, # Google custom search engine id + } - response = requests.get(self.url, params=params) - if response.status_code == 200: - response = response.json() - response = self.parse(response) - return response - else: - return f"Error: {response.status_code}" + response = requests.get(self.url, params=params) + if response.status_code == 200: + response = response.json() + response = self.parse(response) + return response else: - return f"Error: The engine is not DuckDuckGo." - + return f"Error: {response.status_code}" + # Extract required data from a message def parse(self, message): if self.detail: From e8f6a343c1be966b833ed985f2ee93e52a59d360 Mon Sep 17 00:00:00 2001 From: alimosaed Date: Thu, 10 Oct 2024 11:20:22 -0400 Subject: [PATCH 31/55] updated the document and reverted the python version --- docs/getting_started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started.md b/docs/getting_started.md index e58b119..3887eb5 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -4,7 +4,7 @@ This guide will help you get started with the Solace AI Event Connector. ## Prerequisites -- Python 3.10.x, 3.11.x or 3.12.x +- Python 3.10 or later - A Solace PubSub+ event broker - A chat model to connect to (optional) From 920b0e48980d7d78f164038a5631fa0bcfb24108 Mon Sep 17 00:00:00 2001 From: alimosaed Date: Fri, 11 Oct 2024 09:43:35 -0400 Subject: [PATCH 32/55] addressed the code review comments --- docs/components/index.md | 5 +++ docs/components/parser.md | 17 ++++++++ docs/components/web_scraper.md | 31 ++++++++++++++ docs/components/websearch_bing.md | 38 ++++++++++++++++++ docs/components/websearch_duckduckgo.md | 40 +++++++++++++++++++ docs/components/websearch_google.md | 38 ++++++++++++++++++ .../general/websearch/web_scraper.py | 12 +++--- .../general/websearch/websearch_google.py | 1 - 8 files changed, 175 insertions(+), 7 deletions(-) create mode 100644 docs/components/parser.md create mode 100644 docs/components/web_scraper.md create mode 100644 docs/components/websearch_bing.md create mode 100644 docs/components/websearch_duckduckgo.md create mode 100644 docs/components/websearch_google.md diff --git a/docs/components/index.md b/docs/components/index.md index e46fd1f..c55fcc9 100644 --- a/docs/components/index.md +++ b/docs/components/index.md @@ -20,8 +20,13 @@ | [message_filter](message_filter.md) | A filtering component. This will apply a user configurable expression. If the expression evaluates to True, the message will be passed on. If the expression evaluates to False, the message will be discarded. If the message is discarded, any previous components that require an acknowledgement will be acknowledged. | | [openai_chat_model](openai_chat_model.md) | OpenAI chat model component | | [openai_chat_model_with_history](openai_chat_model_with_history.md) | OpenAI chat model component with conversation history | +| [parser](parser.md) | Parse a JSON string and extract data fields. | | [pass_through](pass_through.md) | What goes in comes out | | [stdin_input](stdin_input.md) | STDIN input component. The component will prompt for input, which will then be placed in the message payload using the output schema below. The component will wait for its output message to be acknowledged before prompting for the next input. | | [stdout_output](stdout_output.md) | STDOUT output component | | [timer_input](timer_input.md) | An input that will generate a message at a specified interval. | | [user_processor](user_processor.md) | A component that allows the processing stage to be defined in the configuration file. | +| [web_scraper](web_scraper.md) | Scrape javascript based websites. | +| [websearch_bing](websearch_bing.md) | Perform a search query on Bing. | +| [websearch_duckduckgo](websearch_duckduckgo.md) | Perform a search query on DuckDuckGo. | +| [websearch_google](websearch_google.md) | Perform a search query on Google. | diff --git a/docs/components/parser.md b/docs/components/parser.md new file mode 100644 index 0000000..7bb7c5d --- /dev/null +++ b/docs/components/parser.md @@ -0,0 +1,17 @@ +# Parser + +Parse a JSON string and extract data fields. + +## Configuration Parameters + +```yaml +component_name: +component_module: parser +component_config: + input_format: +``` + +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| input_format | True | | The input format of the data. Options: 'json', 'yaml'. | + diff --git a/docs/components/web_scraper.md b/docs/components/web_scraper.md new file mode 100644 index 0000000..ce52f93 --- /dev/null +++ b/docs/components/web_scraper.md @@ -0,0 +1,31 @@ +# WebScraper + +Scrape javascript based websites. + +## Configuration Parameters + +```yaml +component_name: +component_module: web_scraper +component_config: +``` + +No configuration parameters + + +## Component Input Schema + +``` +{ + +} +``` + + +## Component Output Schema + +``` +{ + +} +``` diff --git a/docs/components/websearch_bing.md b/docs/components/websearch_bing.md new file mode 100644 index 0000000..54d2960 --- /dev/null +++ b/docs/components/websearch_bing.md @@ -0,0 +1,38 @@ +# WebSearchBing + +Perform a search query on Bing. + +## Configuration Parameters + +```yaml +component_name: +component_module: websearch_bing +component_config: + api_key: + count: + safesearch: +``` + +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| api_key | True | | Bing API Key. | +| count | False | 10 | Number of search results to return. | +| safesearch | False | Moderate | Safe search setting: Off, Moderate, or Strict. | + + +## Component Input Schema + +``` +{ + +} +``` + + +## Component Output Schema + +``` +{ + +} +``` diff --git a/docs/components/websearch_duckduckgo.md b/docs/components/websearch_duckduckgo.md new file mode 100644 index 0000000..86f6e0a --- /dev/null +++ b/docs/components/websearch_duckduckgo.md @@ -0,0 +1,40 @@ +# WebSearchDuckDuckGo + +Perform a search query on DuckDuckGo. + +## Configuration Parameters + +```yaml +component_name: +component_module: websearch_duckduckgo +component_config: + pretty: + no_html: + skip_disambig: + detail: +``` + +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| pretty | False | 1 | Beautify the search output. | +| no_html | False | 1 | The number of output pages. | +| skip_disambig | False | 1 | Skip disambiguation. | +| detail | False | False | Return the detail. | + + +## Component Input Schema + +``` +{ + +} +``` + + +## Component Output Schema + +``` +{ + +} +``` diff --git a/docs/components/websearch_google.md b/docs/components/websearch_google.md new file mode 100644 index 0000000..4219a57 --- /dev/null +++ b/docs/components/websearch_google.md @@ -0,0 +1,38 @@ +# WebSearchGoogle + +Perform a search query on Google. + +## Configuration Parameters + +```yaml +component_name: +component_module: websearch_google +component_config: + api_key: + search_engine_id: + detail: +``` + +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| api_key | True | | Google API Key. | +| search_engine_id | False | 1 | The custom search engine id. | +| detail | False | False | Return the detail. | + + +## Component Input Schema + +``` +{ + +} +``` + + +## Component Output Schema + +``` +{ + +} +``` diff --git a/src/solace_ai_connector/components/general/websearch/web_scraper.py b/src/solace_ai_connector/components/general/websearch/web_scraper.py index 1ce922b..457f93b 100755 --- a/src/solace_ai_connector/components/general/websearch/web_scraper.py +++ b/src/solace_ai_connector/components/general/websearch/web_scraper.py @@ -1,6 +1,4 @@ """Scrape a website""" -import subprocess -import sys from ...component_base import ComponentBase from ....common.log import log @@ -30,26 +28,28 @@ def invoke(self, message, data): # Scrape a website def scrape(self, url): + err_msg = "Please install playwright by running 'pip install playwright' and 'playwright install'." try: from playwright.sync_api import sync_playwright except ImportError: log.error( - "Please install playwright by running 'pip install playwright' and 'playwright install'." + err_msg ) raise ValueError( - "Please install playwright by running 'pip install playwright' and 'playwright install'." + err_msg ) with sync_playwright() as p: + err_msg = "Failed to launch the Chromium instance. Please install the browser binaries by running 'playwright install'" try: # Launch a Chromium browser instance browser = p.chromium.launch(headless=True) # Set headless=False to see the browser in action except ImportError: log.error( - f"Failed to launch the Chromium instance. Please install the browser binaries by running 'playwright install'" + err_msg ) raise ValueError( - f"Failed to launch the Chromium instance. Please install the browser binaries by running 'playwright install'" + err_msg ) page = browser.new_page() page.goto(url) diff --git a/src/solace_ai_connector/components/general/websearch/websearch_google.py b/src/solace_ai_connector/components/general/websearch/websearch_google.py index 5371d80..66d6e14 100755 --- a/src/solace_ai_connector/components/general/websearch/websearch_google.py +++ b/src/solace_ai_connector/components/general/websearch/websearch_google.py @@ -1,7 +1,6 @@ # This is a Google search engine. # The configuration will configure the Google engine. import requests -import json from .websearch_base import ( WebSearchBase, From ec6944cfe3c038885e725f320ac68f5c459e044d Mon Sep 17 00:00:00 2001 From: alimosaed Date: Fri, 11 Oct 2024 09:52:17 -0400 Subject: [PATCH 33/55] fixed minior issue --- .../components/general/websearch/web_scraper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/solace_ai_connector/components/general/websearch/web_scraper.py b/src/solace_ai_connector/components/general/websearch/web_scraper.py index 457f93b..f237150 100755 --- a/src/solace_ai_connector/components/general/websearch/web_scraper.py +++ b/src/solace_ai_connector/components/general/websearch/web_scraper.py @@ -28,10 +28,10 @@ def invoke(self, message, data): # Scrape a website def scrape(self, url): - err_msg = "Please install playwright by running 'pip install playwright' and 'playwright install'." try: from playwright.sync_api import sync_playwright except ImportError: + err_msg = "Please install playwright by running 'pip install playwright' and 'playwright install'." log.error( err_msg ) @@ -40,11 +40,11 @@ def scrape(self, url): ) with sync_playwright() as p: - err_msg = "Failed to launch the Chromium instance. Please install the browser binaries by running 'playwright install'" try: # Launch a Chromium browser instance browser = p.chromium.launch(headless=True) # Set headless=False to see the browser in action except ImportError: + err_msg = "Failed to launch the Chromium instance. Please install the browser binaries by running 'playwright install'" log.error( err_msg ) From 6d8f4b7600f8c9a8f2de810c683881d827de6b7b Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Tue, 15 Oct 2024 09:33:40 -0400 Subject: [PATCH 34/55] AI-173: Support !include in yaml config files (#52) * feat: implement #include directive for config files * feat: implement indentation for included file content * A few more tweaks --- src/solace_ai_connector/main.py | 34 ++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/solace_ai_connector/main.py b/src/solace_ai_connector/main.py index a62cf00..7bf43cf 100644 --- a/src/solace_ai_connector/main.py +++ b/src/solace_ai_connector/main.py @@ -2,15 +2,18 @@ import sys import re import yaml +from pathlib import Path from .solace_ai_connector import SolaceAiConnector def load_config(file): """Load configuration from a YAML file.""" try: - # Load the YAML file as a string - with open(file, "r", encoding="utf8") as f: - yaml_str = f.read() + # Get the directory of the current file + file_dir = os.path.dirname(os.path.abspath(file)) + + # Load the YAML file as a string, processing includes + yaml_str = process_includes(file, file_dir) # Substitute the environment variables using os.environ yaml_str = expandvars_with_defaults(yaml_str) @@ -23,6 +26,31 @@ def load_config(file): sys.exit(1) +def process_includes(file_path, base_dir): + """Process #include directives in the given file.""" + with open(file_path, "r", encoding="utf8") as f: + content = f.read() + + def include_repl(match): + indent = match.group(1) # Capture the leading spaces + indent = indent.replace("\n", "") # Remove newlines + include_path = match.group(2).strip("'\"") + full_path = os.path.join(base_dir, include_path) + if not os.path.exists(full_path): + raise FileNotFoundError(f"Included file not found: {full_path}") + included_content = process_includes(full_path, os.path.dirname(full_path)) + # Indent each line of the included content + indented_content = "\n".join( + indent + line for line in included_content.splitlines() + ) + return indented_content + + include_pattern = re.compile( + r'^(\s*)!include\s+(["\']?[^"\s\']+)["\']?', re.MULTILINE + ) + return include_pattern.sub(include_repl, content) + + def expandvars_with_defaults(text): """Expand environment variables with support for default values. Supported syntax: ${VAR_NAME} or ${VAR_NAME, default_value}""" From 2995ef0475c3e1c202065414823ee42ddadaacda Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Tue, 15 Oct 2024 09:47:40 -0400 Subject: [PATCH 35/55] AI-170: Added websocket_input and websocket_output components (#50) * feat: AI-170: websocket input and output components (start) * Refactor HttpServerInputBase to work as a websocket input component * refactor: implement WebSocket input component using Flask-SocketIO * style: Format code and improve consistency in websocket_input.py * refactor: update WebSocket input to use send_message and improve invoke method * refactor: wrap WebSocket messages in events before enqueueing * feat: implement WebsocketOutput component for sending messages * refactor: implement WebSocket input/output with shared connection management * refactor: simplify WebSocket output component and improve error handling * style: replace f-strings with format directives in logging statements * feat: implement WebSocket connection tracking and improve message handling * refactor: replace f-strings with format directives in logging statements * chore: Remove unused comment about f-strings in logging * fix: replace self.socketio.sid with request.sid for correct session ID handling * More changes * feat: add HTML serving capability to WebsocketInput * refactor: Improve WebsocketInput class structure and remove unused import * refactor: simplify WebsocketInput component * fix: resolve HTML file path issues in WebsocketInput * refactor: implement event processing for WebSocket input * Add example for websocket * refactor: centralize event processing and error handling in ComponentBase * fix: update websocket.yaml to correctly pass payload and socket_id * style: enhance UI design and responsiveness of WebSocket example app * style: implement light/dark mode theme based on browser preference * fix: improve dark mode implementation and add smooth transitions * style: adjust input and button spacing in WebSocket example app * feat: add info icon with popup explanation to WebSocket example app * refactor: simplify WebSocket example app interface * More adjustments * style: reduce padding above top title and adjust container spacing * Remove unnecessary whitespace * Fix a few warnings * feat: add payload encoding and format to WebSocket components * refactor: reorganize imports and improve code formatting * refactor: simplify encode_payload function * refactor: use centralized payload encoding/decoding functions * Some additional cleanup * change the default payload encoding to 'none' so that strings will be sent * Fix warning * refactor: implement WebSocket base class and enhance output functionality * feat: import request object in websocket_input.py * refactor: move common config to base class and use deep copy * refactor: merge base_info into websocket component info structures * More refactoring to allow websocket_output to share some code * refactor: make WebsocketBase abstract and enforce listen_port requirement * refactor: remove listen_port validation in WebsocketBase * feat: implement threaded WebSocket server for output component * default the encoding to none * feat: enable WebSocket server debugging * Tweak the dynamic module loader to give better output if a dynamically loaded module fails due to it not being able to be loaded by it failing to load another module. Also turn off websocket debug logs * AI-170: Add new broker type: dev_broker (#51) * feat: implement DevBroker for development purposes * refactor: convert Solace message to dictionary in receive_message * refactor: align DevBroker with solace_messaging subscription handling * More changes * More tweaks * Last few issues * refactor: standardize use of 'queue_name' in dev_broker_messaging * Remove queue_id --- examples/websocket/websocket.yaml | 42 +++ examples/websocket/websocket_example_app.html | 266 ++++++++++++++++++ .../common/messaging/dev_broker_messaging.py | 104 +++++++ .../common/messaging/messaging.py | 20 +- .../common/messaging/messaging_builder.py | 9 +- .../common/messaging/solace_messaging.py | 20 +- src/solace_ai_connector/common/utils.py | 54 +++- .../components/component_base.py | 30 +- .../components/inputs_outputs/broker_base.py | 51 +--- .../components/inputs_outputs/broker_input.py | 14 +- .../inputs_outputs/broker_request_response.py | 19 +- .../inputs_outputs/websocket_base.py | 143 ++++++++++ .../inputs_outputs/websocket_input.py | 84 ++++++ .../inputs_outputs/websocket_output.py | 73 +++++ 14 files changed, 839 insertions(+), 90 deletions(-) create mode 100644 examples/websocket/websocket.yaml create mode 100644 examples/websocket/websocket_example_app.html create mode 100644 src/solace_ai_connector/common/messaging/dev_broker_messaging.py create mode 100644 src/solace_ai_connector/components/inputs_outputs/websocket_base.py create mode 100644 src/solace_ai_connector/components/inputs_outputs/websocket_input.py create mode 100644 src/solace_ai_connector/components/inputs_outputs/websocket_output.py diff --git a/examples/websocket/websocket.yaml b/examples/websocket/websocket.yaml new file mode 100644 index 0000000..4efabfa --- /dev/null +++ b/examples/websocket/websocket.yaml @@ -0,0 +1,42 @@ +--- + # Example configuration for a WebSocket flow + # This flow creates a WebSocket server that echoes messages back to clients + # It also serves an example HTML file for easy testing + + log: + stdout_log_level: INFO + log_file_level: DEBUG + log_file: solace_ai_connector.log + + flows: + - name: websocket_echo + components: + # WebSocket Input + - component_name: websocket_input + component_module: websocket_input + component_config: + listen_port: 5000 + serve_html: true + html_path: "examples/websocket/websocket_example_app.html" + + # Pass Through + - component_name: pass_through + component_module: pass_through + component_config: {} + input_transforms: + - type: copy + source_expression: input.payload + dest_expression: user_data.input:payload + - type: copy + source_expression: input.user_properties:socket_id + dest_expression: user_data.input:socket_id + input_selection: + source_expression: user_data.input + + # WebSocket Output + - component_name: websocket_output + component_module: websocket_output + component_config: + payload_encoding: none + input_selection: + source_expression: previous diff --git a/examples/websocket/websocket_example_app.html b/examples/websocket/websocket_example_app.html new file mode 100644 index 0000000..02d54db --- /dev/null +++ b/examples/websocket/websocket_example_app.html @@ -0,0 +1,266 @@ + + + + + + WebSocket Example App + + + + +
+

WebSocket Example App

+

This is a simple app to show how JSON can be sent into a solace-ai-connector flow and how to receive output from it. Just hit connect to connect to the flow, type in some JSON and hit send. Your JSON should be echoed back to you.

+ +
+ + + Disconnected +
+ +
+ + +
+
+ +
+

Received Messages

+
+ +
+
+ + + + + diff --git a/src/solace_ai_connector/common/messaging/dev_broker_messaging.py b/src/solace_ai_connector/common/messaging/dev_broker_messaging.py new file mode 100644 index 0000000..0d10cd6 --- /dev/null +++ b/src/solace_ai_connector/common/messaging/dev_broker_messaging.py @@ -0,0 +1,104 @@ +"""This is a simple broker for testing purposes. It allows sending and receiving +messages to/from queues. It supports subscriptions based on topics.""" + +from typing import Dict, List, Any +import queue +import re +from copy import deepcopy +from .messaging import Messaging + + +class DevBroker(Messaging): + def __init__(self, broker_properties: dict, flow_lock_manager, flow_kv_store): + super().__init__(broker_properties) + self.flow_lock_manager = flow_lock_manager + self.flow_kv_store = flow_kv_store + self.connected = False + self.subscriptions_lock = self.flow_lock_manager.get_lock("subscriptions") + with self.subscriptions_lock: + self.subscriptions = self.flow_kv_store.get("dev_broker:subscriptions") + if self.subscriptions is None: + self.subscriptions: Dict[str, List[str]] = {} + self.flow_kv_store.set("dev_broker:subscriptions", self.subscriptions) + self.queues = self.flow_kv_store.get("dev_broker:queues") + if self.queues is None: + self.queues: Dict[str, queue.Queue] = {} + self.flow_kv_store.set("dev_broker:queues", self.queues) + + def connect(self): + self.connected = True + queue_name = self.broker_properties.get("queue_name") + subscriptions = self.broker_properties.get("subscriptions", []) + if queue_name: + self.queues[queue_name] = queue.Queue() + for subscription in subscriptions: + self.subscribe(subscription["topic"], queue_name) + + def disconnect(self): + self.connected = False + + def receive_message(self, timeout_ms, queue_name: str): + if not self.connected: + raise RuntimeError("DevBroker is not connected") + + try: + return self.queues[queue_name].get(timeout=timeout_ms / 1000) + except queue.Empty: + return None + + def send_message( + self, + destination_name: str, + payload: Any, + user_properties: Dict = None, + user_context: Dict = None, + ): + if not self.connected: + raise RuntimeError("DevBroker is not connected") + + message = { + "payload": payload, + "topic": destination_name, + "user_properties": user_properties or {}, + } + + matching_queue_names = self._get_matching_queue_names(destination_name) + + for queue_name in matching_queue_names: + # Clone the message for each queue to ensure isolation + self.queues[queue_name].put(deepcopy(message)) + + if user_context and "callback" in user_context: + user_context["callback"](user_context) + + def subscribe(self, subscription: str, queue_name: str): + if not self.connected: + raise RuntimeError("DevBroker is not connected") + + subscription = self._subscription_to_regex(subscription) + + with self.subscriptions_lock: + if queue_name not in self.queues: + self.queues[queue_name] = queue.Queue() + if subscription not in self.subscriptions: + self.subscriptions[subscription] = [] + self.subscriptions[subscription].append(queue_name) + + def ack_message(self, message): + pass + + def _get_matching_queue_names(self, topic: str) -> List[str]: + matching_queue_names = [] + with self.subscriptions_lock: + for subscription, queue_names in self.subscriptions.items(): + if self._topic_matches(subscription, topic): + matching_queue_names.extend(queue_names) + return list(set(matching_queue_names)) # Remove duplicates + + @staticmethod + def _topic_matches(subscription: str, topic: str) -> bool: + return re.match(f"^{subscription}$", topic) is not None + + @staticmethod + def _subscription_to_regex(subscription: str) -> str: + return subscription.replace("*", "[^/]+").replace(">", ".*") diff --git a/src/solace_ai_connector/common/messaging/messaging.py b/src/solace_ai_connector/common/messaging/messaging.py index 0844863..5847eff 100644 --- a/src/solace_ai_connector/common/messaging/messaging.py +++ b/src/solace_ai_connector/common/messaging/messaging.py @@ -1,4 +1,4 @@ -# messaging.py - Base class for EDA messaging services +from typing import Any, Dict class Messaging: @@ -11,14 +11,14 @@ def connect(self): def disconnect(self): raise NotImplementedError - def receive_message(self, timeout_ms): + def receive_message(self, timeout_ms, queue_id: str): raise NotImplementedError - # def is_connected(self): - # raise NotImplementedError - - # def send_message(self, destination_name: str, message: str): - # raise NotImplementedError - - # def subscribe(self, subscription: str, message_handler): #: MessageHandler): - # raise NotImplementedError + def send_message( + self, + destination_name: str, + payload: Any, + user_properties: Dict = None, + user_context: Dict = None, + ): + raise NotImplementedError diff --git a/src/solace_ai_connector/common/messaging/messaging_builder.py b/src/solace_ai_connector/common/messaging/messaging_builder.py index 423d246..826cdd4 100644 --- a/src/solace_ai_connector/common/messaging/messaging_builder.py +++ b/src/solace_ai_connector/common/messaging/messaging_builder.py @@ -1,12 +1,15 @@ """Class to build a Messaging Service object""" from .solace_messaging import SolaceMessaging +from .dev_broker_messaging import DevBroker # Make a Messaging Service builder - this is a factory for Messaging Service objects class MessagingServiceBuilder: - def __init__(self): + def __init__(self, flow_lock_manager, flow_kv_store): self.broker_properties = {} + self.flow_lock_manager = flow_lock_manager + self.flow_kv_store = flow_kv_store def from_properties(self, broker_properties: dict): self.broker_properties = broker_properties @@ -15,6 +18,10 @@ def from_properties(self, broker_properties: dict): def build(self): if self.broker_properties["broker_type"] == "solace": return SolaceMessaging(self.broker_properties) + elif self.broker_properties["broker_type"] == "dev_broker": + return DevBroker( + self.broker_properties, self.flow_lock_manager, self.flow_kv_store + ) raise ValueError( f"Unsupported broker type: {self.broker_properties['broker_type']}" diff --git a/src/solace_ai_connector/common/messaging/solace_messaging.py b/src/solace_ai_connector/common/messaging/solace_messaging.py index ed33091..4b03e7a 100644 --- a/src/solace_ai_connector/common/messaging/solace_messaging.py +++ b/src/solace_ai_connector/common/messaging/solace_messaging.py @@ -246,8 +246,19 @@ def send_message( user_context=user_context, ) - def receive_message(self, timeout_ms): - return self.persistent_receivers[0].receive_message(timeout_ms) + def receive_message(self, timeout_ms, queue_id): + broker_message = self.persistent_receivers[0].receive_message(timeout_ms) + if broker_message is None: + return None + + # Convert Solace message to dictionary format + return { + "payload": broker_message.get_payload_as_string() + or broker_message.get_payload_as_bytes(), + "topic": broker_message.get_destination_name(), + "user_properties": broker_message.get_properties(), + "_original_message": broker_message, # Keep original message for acknowledgement + } def subscribe( self, subscription: str, persistent_receiver: PersistentMessageReceiver @@ -256,4 +267,7 @@ def subscribe( persistent_receiver.add_subscription(sub) def ack_message(self, broker_message): - self.persistent_receiver.ack(broker_message) + if "_original_message" in broker_message: + self.persistent_receiver.ack(broker_message["_original_message"]) + else: + log.warning("Cannot acknowledge message: original Solace message not found") diff --git a/src/solace_ai_connector/common/utils.py b/src/solace_ai_connector/common/utils.py index 4004e87..e53fb61 100644 --- a/src/solace_ai_connector/common/utils.py +++ b/src/solace_ai_connector/common/utils.py @@ -7,6 +7,11 @@ import builtins import subprocess import types +import base64 +import gzip +import json +import yaml + from .log import log @@ -134,8 +139,13 @@ def import_module(module, base_path=None, component_package=None): ) else: return importlib.import_module(full_name) - except ModuleNotFoundError: - pass + except ModuleNotFoundError as e: + name = str(e.name) + if ( + name != "solace_ai_connector" + and name.split(".")[-1] != full_name.split(".")[-1] + ): + raise e except Exception as e: raise ImportError( f"Module load error for {full_name}: {e}" @@ -335,3 +345,43 @@ def ensure_slash_on_start(string): if not string.startswith("/"): return "/" + string return string + + +def encode_payload(payload, encoding, payload_format): + # First, format the payload + if payload_format == "json": + formatted_payload = json.dumps(payload) + elif payload_format == "yaml": + formatted_payload = yaml.dump(payload) + elif isinstance(payload, bytes) or isinstance(payload, bytearray): + formatted_payload = payload + else: + formatted_payload = str(payload) + + # Then, encode the formatted payload + if encoding == "utf-8": + return formatted_payload.encode("utf-8") + elif encoding == "base64": + return base64.b64encode(formatted_payload.encode("utf-8")) + elif encoding == "gzip": + return gzip.compress(formatted_payload.encode("utf-8")) + else: + return formatted_payload + + +def decode_payload(payload, encoding, payload_format): + if encoding == "base64": + payload = base64.b64decode(payload) + elif encoding == "gzip": + payload = gzip.decompress(payload) + elif encoding == "utf-8" and ( + isinstance(payload, bytes) or isinstance(payload, bytearray) + ): + payload = payload.decode("utf-8") + + if payload_format == "json": + payload = json.loads(payload) + elif payload_format == "yaml": + payload = yaml.safe_load(payload) + + return payload diff --git a/src/solace_ai_connector/components/component_base.py b/src/solace_ai_connector/components/component_base.py index f059c06..f7c8c41 100644 --- a/src/solace_ai_connector/components/component_base.py +++ b/src/solace_ai_connector/components/component_base.py @@ -12,7 +12,7 @@ from ..common.event import Event, EventType from ..flow.request_response_flow_controller import RequestResponseFlowController -DEFAULT_QUEUE_TIMEOUT_MS = 200 +DEFAULT_QUEUE_TIMEOUT_MS = 1000 DEFAULT_QUEUE_MAX_DEPTH = 5 @@ -68,23 +68,29 @@ def run(self): try: event = self.get_next_event() if event is not None: - if self.trace_queue: - self.trace_event(event) - self.process_event(event) + self.process_event_with_tracing(event) except AssertionError as e: raise e except Exception as e: - log.error( - "%sComponent has crashed: %s\n%s", - self.log_identifier, - e, - traceback.format_exc(), - ) - if self.error_queue: - self.handle_error(e, event) + self.handle_component_error(e, event) self.stop_component() + def process_event_with_tracing(self, event): + if self.trace_queue: + self.trace_event(event) + self.process_event(event) + + def handle_component_error(self, e, event): + log.error( + "%sComponent has crashed: %s\n%s", + self.log_identifier, + e, + traceback.format_exc(), + ) + if self.error_queue: + self.handle_error(e, event) + def get_next_event(self): # Check if there is a get_next_message defined by a # component that inherits from this class - this is diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_base.py b/src/solace_ai_connector/components/inputs_outputs/broker_base.py index fac4207..c312740 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_base.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_base.py @@ -1,17 +1,13 @@ """Base class for broker input/output components for the Solace AI Event Connector""" -import base64 -import gzip -import json -import yaml import uuid from abc import abstractmethod -# from solace_ai_connector.common.log import log from ..component_base import ComponentBase from ...common.message import Message from ...common.messaging.messaging_builder import MessagingServiceBuilder +from ...common.utils import encode_payload, decode_payload # TBD - at the moment, there is no connection sharing supported. It should be possible # to share a connection between multiple components and even flows. The changes @@ -39,7 +35,7 @@ def __init__(self, module_info, **kwargs): self.broker_properties = self.get_broker_properties() if self.broker_properties["broker_type"] not in ["test", "test_streaming"]: self.messaging_service = ( - MessagingServiceBuilder() + MessagingServiceBuilder(self.flow_lock_manager, self.flow_kv_store) .from_properties(self.broker_properties) .build() ) @@ -68,51 +64,12 @@ def stop_component(self): def decode_payload(self, payload): encoding = self.get_config("payload_encoding") payload_format = self.get_config("payload_format") - if encoding == "base64": - payload = base64.b64decode(payload) - elif encoding == "gzip": - payload = gzip.decompress(payload) - elif encoding == "utf-8" and ( - isinstance(payload, bytes) or isinstance(payload, bytearray) - ): - payload = payload.decode("utf-8") - if payload_format == "json": - payload = json.loads(payload) - elif payload_format == "yaml": - payload = yaml.safe_load(payload) - return payload + return decode_payload(payload, encoding, payload_format) def encode_payload(self, payload): encoding = self.get_config("payload_encoding") payload_format = self.get_config("payload_format") - if encoding == "utf-8": - if payload_format == "json": - return json.dumps(payload).encode("utf-8") - elif payload_format == "yaml": - return yaml.dump(payload).encode("utf-8") - else: - return str(payload).encode("utf-8") - elif encoding == "base64": - if payload_format == "json": - return base64.b64encode(json.dumps(payload).encode("utf-8")) - elif payload_format == "yaml": - return base64.b64encode(yaml.dump(payload).encode("utf-8")) - else: - return base64.b64encode(str(payload).encode("utf-8")) - elif encoding == "gzip": - if payload_format == "json": - return gzip.compress(json.dumps(payload).encode("utf-8")) - elif payload_format == "yaml": - return gzip.compress(yaml.dump(payload).encode("utf-8")) - else: - return gzip.compress(str(payload).encode("utf-8")) - else: - if payload_format == "json": - return json.dumps(payload) - elif payload_format == "yaml": - return yaml.dump(payload) - else: - return str(payload) + return encode_payload(payload, encoding, payload_format) def get_egress_topic(self, message: Message): pass diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_input.py b/src/solace_ai_connector/components/inputs_outputs/broker_input.py index 3aabd8d..2d277cb 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_input.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_input.py @@ -110,16 +110,18 @@ def invoke(self, message, data): def get_next_message(self, timeout_ms=None): if timeout_ms is None: timeout_ms = DEFAULT_TIMEOUT_MS - broker_message = self.messaging_service.receive_message(timeout_ms) + broker_message = self.messaging_service.receive_message( + timeout_ms, self.broker_properties["queue_name"] + ) if not broker_message: return None self.current_broker_message = broker_message - payload = broker_message.get_payload_as_string() - topic = broker_message.get_destination_name() - if payload is None: - payload = broker_message.get_payload_as_bytes() + + payload = broker_message.get("payload") payload = self.decode_payload(payload) - user_properties = broker_message.get_properties() + + topic = broker_message.get("topic") + user_properties = broker_message.get("user_properties", {}) log.debug( "Received message from broker: topic=%s, user_properties=%s, payload length=%d", topic, diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py b/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py index cb217b9..4c33ddb 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py @@ -193,11 +193,12 @@ def __init__(self, **kwargs): ] self.test_mode = False - if self.broker_type == "solace": - self.connect() - elif self.broker_type == "test" or self.broker_type == "test_streaming": + if self.broker_type == "test" or self.broker_type == "test_streaming": self.test_mode = True self.setup_test_pass_through() + else: + self.connect() + self.start() def start(self): @@ -224,7 +225,9 @@ def start_response_thread(self): def handle_responses(self): while not self.stop_signal.is_set(): try: - broker_message = self.messaging_service.receive_message(1000) + broker_message = self.messaging_service.receive_message( + 1000, self.reply_queue_name + ) if broker_message: self.process_response(broker_message) except Exception as e: @@ -248,12 +251,10 @@ def process_response(self, broker_message): topic = broker_message.get_topic() user_properties = broker_message.get_user_properties() else: - payload = broker_message.get_payload_as_string() - if payload is None: - payload = broker_message.get_payload_as_bytes() + payload = broker_message.get("payload") payload = self.decode_payload(payload) - topic = broker_message.get_destination_name() - user_properties = broker_message.get_properties() + topic = broker_message.get("topic") + user_properties = broker_message.get("user_properties", {}) metadata_json = user_properties.get( "__solace_ai_connector_broker_request_reply_metadata__" diff --git a/src/solace_ai_connector/components/inputs_outputs/websocket_base.py b/src/solace_ai_connector/components/inputs_outputs/websocket_base.py new file mode 100644 index 0000000..ffe1ad9 --- /dev/null +++ b/src/solace_ai_connector/components/inputs_outputs/websocket_base.py @@ -0,0 +1,143 @@ +"""Base class for WebSocket components.""" + +from abc import ABC, abstractmethod +from flask import Flask, send_file, request +from flask_socketio import SocketIO +import logging +from ...common.log import log +from ..component_base import ComponentBase +import copy +from flask.logging import default_handler + +base_info = { + "config_parameters": [ + { + "name": "listen_port", + "type": "int", + "required": False, + "description": "Port to listen on (optional)", + }, + { + "name": "serve_html", + "type": "bool", + "required": False, + "description": "Serve the example HTML file", + "default": False, + }, + { + "name": "html_path", + "type": "string", + "required": False, + "description": "Path to the HTML file to serve", + "default": "examples/websocket/websocket_example_app.html", + }, + { + "name": "payload_encoding", + "required": False, + "description": "Encoding for the payload (utf-8, base64, gzip, none)", + "default": "none", + }, + { + "name": "payload_format", + "required": False, + "description": "Format for the payload (json, yaml, text)", + "default": "json", + }, + ], +} + + +class WebsocketBase(ComponentBase, ABC): + def __init__(self, info, **kwargs): + super().__init__(info, **kwargs) + self.listen_port = self.get_config("listen_port") + self.serve_html = self.get_config("serve_html", False) + self.html_path = self.get_config("html_path", "") + self.sockets = {} + self.app = None + self.socketio = None + + if self.listen_port: + self.setup_websocket_server() + + def setup_websocket_server(self): + self.app = Flask(__name__) + + # Enable Flask debugging + self.app.debug = False + + # Set up Flask logging + # self.app.logger.setLevel(logging.DEBUG) + # self.app.logger.addHandler(default_handler) + + # Enable SocketIO logging + # logging.getLogger("socketio").setLevel(logging.DEBUG) + # logging.getLogger("engineio").setLevel(logging.DEBUG) + + self.socketio = SocketIO( + self.app, cors_allowed_origins="*", logger=False, engineio_logger=False + ) + self.setup_websocket() + + if self.serve_html: + self.setup_html_route() + + def setup_html_route(self): + @self.app.route("/") + def serve_html(): + return send_file(self.html_path) + + def setup_websocket(self): + @self.socketio.on("connect") + def handle_connect(): + socket_id = request.sid + self.sockets[socket_id] = self.socketio + self.kv_store_set("websocket_connections", self.sockets) + log.info("New WebSocket connection established. Socket ID: %s", socket_id) + return socket_id + + @self.socketio.on("disconnect") + def handle_disconnect(): + socket_id = request.sid + if socket_id in self.sockets: + del self.sockets[socket_id] + self.kv_store_set("websocket_connections", self.sockets) + log.info("WebSocket connection closed. Socket ID: %s", socket_id) + + def run_server(self): + if self.socketio: + self.socketio.run( + self.app, port=self.listen_port, debug=False, use_reloader=False + ) + + def stop_server(self): + if self.socketio: + self.socketio.stop() + if self.app: + func = request.environ.get("werkzeug.server.shutdown") + if func is None: + raise RuntimeError("Not running with the Werkzeug Server") + func() + + def get_sockets(self): + if not self.sockets: + self.sockets = self.kv_store_get("websocket_connections") or {} + return self.sockets + + def send_to_socket(self, socket_id, payload): + sockets = self.get_sockets() + if socket_id == "*": + for socket in sockets.values(): + socket.emit("message", payload) + log.debug("Message sent to all WebSocket connections") + elif socket_id in sockets: + sockets[socket_id].emit("message", payload) + log.debug("Message sent to WebSocket connection %s", socket_id) + else: + log.error("No active connection found for socket_id: %s", socket_id) + return False + return True + + @abstractmethod + def invoke(self, message, data): + pass diff --git a/src/solace_ai_connector/components/inputs_outputs/websocket_input.py b/src/solace_ai_connector/components/inputs_outputs/websocket_input.py new file mode 100644 index 0000000..6b80617 --- /dev/null +++ b/src/solace_ai_connector/components/inputs_outputs/websocket_input.py @@ -0,0 +1,84 @@ +"""This component receives messages from a websocket connection and sends them to the next component in the flow.""" + +import json +import os +import copy + +from flask import request +from ...common.log import log +from ...common.message import Message +from ...common.event import Event, EventType +from ...common.utils import decode_payload +from .websocket_base import WebsocketBase, base_info + + +# Merge base_info into info +info = copy.deepcopy(base_info) +info.update( + { + "class_name": "WebsocketInput", + "description": "Listen for incoming messages on a websocket connection.", + "output_schema": { + "type": "object", + "properties": { + "payload": { + "type": "object", + "description": "The decoded JSON payload received from the WebSocket", + }, + }, + "required": ["payload"], + }, + } +) + + +class WebsocketInput(WebsocketBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + self.payload_encoding = self.get_config("payload_encoding") + self.payload_format = self.get_config("payload_format") + + if not self.listen_port: + raise ValueError("listen_port is required for WebsocketInput") + + if not os.path.isabs(self.html_path): + self.html_path = os.path.join(os.getcwd(), self.html_path) + + self.setup_message_handler() + + def setup_message_handler(self): + @self.socketio.on("message") + def handle_message(data): + try: + decoded_payload = decode_payload( + data, self.payload_encoding, self.payload_format + ) + socket_id = request.sid + message = Message( + payload=decoded_payload, user_properties={"socket_id": socket_id} + ) + event = Event(EventType.MESSAGE, message) + self.process_event_with_tracing(event) + except json.JSONDecodeError: + log.error("Received invalid payload: %s", data) + except AssertionError as e: + raise e + except Exception as e: + self.handle_component_error(e, event) + + def run(self): + self.run_server() + + def stop_component(self): + self.stop_server() + + def invoke(self, message, data): + try: + return { + "payload": message.get_payload(), + "topic": message.get_topic(), + "user_properties": message.get_user_properties(), + } + except Exception as e: + log.error("Error processing WebSocket message: %s", str(e)) + return None diff --git a/src/solace_ai_connector/components/inputs_outputs/websocket_output.py b/src/solace_ai_connector/components/inputs_outputs/websocket_output.py new file mode 100644 index 0000000..d732064 --- /dev/null +++ b/src/solace_ai_connector/components/inputs_outputs/websocket_output.py @@ -0,0 +1,73 @@ +"""This component sends messages to a websocket connection.""" + +import copy +import threading +from ...common.log import log +from ...common.utils import encode_payload +from .websocket_base import WebsocketBase, base_info + +info = copy.deepcopy(base_info) +info.update( + { + "class_name": "WebsocketOutput", + "description": "Send messages to a websocket connection.", + "input_schema": { + "type": "object", + "properties": { + "payload": { + "type": "object", + "description": "The payload to be sent via WebSocket", + }, + "socket_id": { + "type": "string", + "description": "Identifier for the WebSocket connection", + }, + }, + "required": ["payload", "user_properties"], + }, + } +) + + +class WebsocketOutput(WebsocketBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + self.payload_encoding = self.get_config("payload_encoding") + self.payload_format = self.get_config("payload_format") + self.server_thread = None + + def run(self): + if self.listen_port: + self.server_thread = threading.Thread(target=self.run_server) + self.server_thread.start() + super().run() + + def stop_component(self): + self.stop_server() + if self.server_thread: + self.server_thread.join() + + def invoke(self, message, data): + try: + payload = data.get("payload") + socket_id = data.get("socket_id") + + if not socket_id: + log.error("No socket_id provided") + self.discard_current_message() + return None + + encoded_payload = encode_payload( + payload, self.payload_encoding, self.payload_format + ) + + if not self.send_to_socket(socket_id, encoded_payload): + self.discard_current_message() + return None + + except Exception as e: + log.error("Error sending message via WebSocket: %s", str(e)) + self.discard_current_message() + return None + + return data From ab2083f33e6616a42d9310ca142d6037af43c83b Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Tue, 15 Oct 2024 09:58:58 -0400 Subject: [PATCH 36/55] feat: AI-166: fix inconsistency with payload/data responses (#49) * Refactor OpenAIChatModelBase to include additional response information * feat: AI-166: Make streaming data results more consistent * feat: AI-167: fix inconsistency with payload/data responses --- .../langchain_chat_model_with_history.py | 4 +-- .../general/openai/openai_chat_model_base.py | 36 +++++++++++++++---- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/solace_ai_connector/components/general/langchain/langchain_chat_model_with_history.py b/src/solace_ai_connector/components/general/langchain/langchain_chat_model_with_history.py index b559ee4..9885c68 100644 --- a/src/solace_ai_connector/components/general/langchain/langchain_chat_model_with_history.py +++ b/src/solace_ai_connector/components/general/langchain/langchain_chat_model_with_history.py @@ -213,7 +213,7 @@ def invoke_model( True, ) - result = namedtuple("Result", ["content", "uuid"])( + result = namedtuple("Result", ["content", "response_uuid"])( aggregate_result, response_uuid ) @@ -233,7 +233,7 @@ def send_streaming_message( message = Message( payload={ "chunk": chunk, - "aggregate_result": aggregate_result, + "content": aggregate_result, "response_uuid": response_uuid, "first_chunk": first_chunk, "last_chunk": last_chunk, diff --git a/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py b/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py index 68f78ea..6577df9 100644 --- a/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py +++ b/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py @@ -106,7 +106,27 @@ "content": { "type": "string", "description": "The generated response from the model", - } + }, + "chunk": { + "type": "string", + "description": "The current chunk of the response", + }, + "response_uuid": { + "type": "string", + "description": "The UUID of the response", + }, + "first_chunk": { + "type": "boolean", + "description": "Whether this is the first chunk of the response", + }, + "last_chunk": { + "type": "boolean", + "description": "Whether this is the last chunk of the response", + }, + "streaming": { + "type": "boolean", + "description": "Whether this is a streaming response", + }, }, "required": ["content"], }, @@ -219,7 +239,7 @@ def invoke_stream(self, client, message, messages): return { "content": aggregate_result, "chunk": current_batch, - "uuid": response_uuid, + "response_uuid": response_uuid, "first_chunk": first_chunk, "last_chunk": True, "streaming": True, @@ -235,7 +255,7 @@ def invoke_stream(self, client, message, messages): True, ) - return {"content": aggregate_result, "uuid": response_uuid} + return {"content": aggregate_result, "response_uuid": response_uuid} def send_streaming_message( self, @@ -249,10 +269,11 @@ def send_streaming_message( message = Message( payload={ "chunk": chunk, - "aggregate_result": aggregate_result, + "content": aggregate_result, "response_uuid": response_uuid, "first_chunk": first_chunk, "last_chunk": last_chunk, + "streaming": True, }, user_properties=input_message.get_user_properties(), ) @@ -270,18 +291,19 @@ def send_to_next_component( message = Message( payload={ "chunk": chunk, - "aggregate_result": aggregate_result, + "content": aggregate_result, "response_uuid": response_uuid, "first_chunk": first_chunk, "last_chunk": last_chunk, + "streaming": True, }, user_properties=input_message.get_user_properties(), ) result = { - "content": aggregate_result, "chunk": chunk, - "uuid": response_uuid, + "content": aggregate_result, + "response_uuid": response_uuid, "first_chunk": first_chunk, "last_chunk": last_chunk, "streaming": True, From e13d6edb920a1e6de372a2921281fc5a2761a406 Mon Sep 17 00:00:00 2001 From: Ali Parvizi <91437594+alimosaed@users.noreply.github.com> Date: Thu, 31 Oct 2024 10:29:25 -0400 Subject: [PATCH 37/55] fix:resolved the unit test error (#54) --- src/solace_ai_connector/common/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/solace_ai_connector/common/utils.py b/src/solace_ai_connector/common/utils.py index 9050bdc..cc8299a 100755 --- a/src/solace_ai_connector/common/utils.py +++ b/src/solace_ai_connector/common/utils.py @@ -129,7 +129,6 @@ def import_module(module, base_path=None, component_package=None): ".components.general.openai", ".components.general.websearch", ".components.inputs_outputs", - ".components.general.filter", ".transforms", ".common", ]: From 948554f2ca6d0ad103da3a899695857ddef7bd6b Mon Sep 17 00:00:00 2001 From: Ali Parvizi <91437594+alimosaed@users.noreply.github.com> Date: Mon, 4 Nov 2024 09:31:10 -0500 Subject: [PATCH 38/55] Add LiteLLM as an agent for model connections (#53) * added litellm component * support chat history * trimmed comments * dynamically get the model parameters * added llm load balancer * added the AI PR reviewer workflow * fixed minor issues * controlled session id * refactored chat history and reused codes * fixed minor logging issue * reverted minor changes * handle all LiteLLM inferences and embedding requests by the load balancer * updated documents * fix: remove useless import command * refactor: restructure the LLM components * refactor: divided litellm chat and embedding * fix: update the embedding input format * fix: update naming * fix: update naming --- .github/workflows/workflows/pr-agent.yaml | 9 + .pr_agent.toml | 134 +++++++++++ docs/components/index.md | 4 + docs/components/litellm_chat_model.md | 84 +++++++ .../litellm_chat_model_with_history.md | 84 +++++++ docs/components/openai_chat_model.md | 12 +- .../openai_chat_model_with_history.md | 12 +- docs/components/websocket_input.md | 39 ++++ docs/components/websocket_output.md | 40 ++++ examples/llm/litellm_chat.yaml | 117 ++++++++++ examples/llm/litellm_chat_with_history.yaml | 121 ++++++++++ examples/llm/litellm_embedding.yaml | 106 +++++++++ src/solace_ai_connector/common/utils.py | 5 +- .../components/__init__.py | 25 +- .../llm/common/chat_history_handler.py | 86 +++++++ .../general/{ => llm}/langchain/__init__.py | 0 .../{ => llm}/langchain/langchain_base.py | 4 +- .../langchain/langchain_chat_model.py | 0 .../langchain/langchain_chat_model_base.py | 2 +- .../langchain_chat_model_with_history.py | 2 +- .../langchain/langchain_embeddings.py | 0 .../langchain_vector_store_delete.py | 2 +- .../langchain_vector_store_embedding_base.py | 0 .../langchain_vector_store_embedding_index.py | 0 ...langchain_vector_store_embedding_search.py | 2 +- .../{openai => llm/litellm}/__init__.py | 0 .../general/llm/litellm/litellm_base.py | 128 ++++++++++ .../general/llm/litellm/litellm_chat_model.py | 11 + .../llm/litellm/litellm_chat_model_base.py | 219 ++++++++++++++++++ .../litellm_chat_model_with_history.py | 105 +++++++++ .../general/llm/litellm/litellm_embeddings.py | 51 ++++ .../components/general/llm/openai/__init__.py | 0 .../{ => llm}/openai/openai_chat_model.py | 0 .../openai/openai_chat_model_base.py | 6 +- .../openai/openai_chat_model_with_history.py | 76 +----- 35 files changed, 1393 insertions(+), 93 deletions(-) create mode 100644 .github/workflows/workflows/pr-agent.yaml create mode 100644 .pr_agent.toml create mode 100644 docs/components/litellm_chat_model.md create mode 100644 docs/components/litellm_chat_model_with_history.md create mode 100644 docs/components/websocket_input.md create mode 100644 docs/components/websocket_output.md create mode 100644 examples/llm/litellm_chat.yaml create mode 100644 examples/llm/litellm_chat_with_history.yaml create mode 100644 examples/llm/litellm_embedding.yaml create mode 100644 src/solace_ai_connector/components/general/llm/common/chat_history_handler.py rename src/solace_ai_connector/components/general/{ => llm}/langchain/__init__.py (100%) rename src/solace_ai_connector/components/general/{ => llm}/langchain/langchain_base.py (94%) rename src/solace_ai_connector/components/general/{ => llm}/langchain/langchain_chat_model.py (100%) rename src/solace_ai_connector/components/general/{ => llm}/langchain/langchain_chat_model_base.py (99%) rename src/solace_ai_connector/components/general/{ => llm}/langchain/langchain_chat_model_with_history.py (99%) rename src/solace_ai_connector/components/general/{ => llm}/langchain/langchain_embeddings.py (100%) rename src/solace_ai_connector/components/general/{ => llm}/langchain/langchain_vector_store_delete.py (99%) rename src/solace_ai_connector/components/general/{ => llm}/langchain/langchain_vector_store_embedding_base.py (100%) rename src/solace_ai_connector/components/general/{ => llm}/langchain/langchain_vector_store_embedding_index.py (100%) rename src/solace_ai_connector/components/general/{ => llm}/langchain/langchain_vector_store_embedding_search.py (99%) rename src/solace_ai_connector/components/general/{openai => llm/litellm}/__init__.py (100%) create mode 100644 src/solace_ai_connector/components/general/llm/litellm/litellm_base.py create mode 100644 src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model.py create mode 100644 src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py create mode 100644 src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_with_history.py create mode 100644 src/solace_ai_connector/components/general/llm/litellm/litellm_embeddings.py create mode 100644 src/solace_ai_connector/components/general/llm/openai/__init__.py rename src/solace_ai_connector/components/general/{ => llm}/openai/openai_chat_model.py (100%) rename src/solace_ai_connector/components/general/{ => llm}/openai/openai_chat_model_base.py (98%) rename src/solace_ai_connector/components/general/{ => llm}/openai/openai_chat_model_with_history.py (54%) diff --git a/.github/workflows/workflows/pr-agent.yaml b/.github/workflows/workflows/pr-agent.yaml new file mode 100644 index 0000000..79c6db9 --- /dev/null +++ b/.github/workflows/workflows/pr-agent.yaml @@ -0,0 +1,9 @@ +name: AI PR Agent + +on: + pull_request: + types: [opened, reopened, ready_for_review] + +jobs: + pr_agent_job: + uses: SolaceDev/ai-build-actions/.github/workflows/ai_pr.yaml@use_sonnet_3_5 \ No newline at end of file diff --git a/.pr_agent.toml b/.pr_agent.toml new file mode 100644 index 0000000..1b057f6 --- /dev/null +++ b/.pr_agent.toml @@ -0,0 +1,134 @@ +[config] +model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model_turbo="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +fallback_models="[bedrock/anthropic.claude-3-sonnet-20240229-v1:0]" +git_provider="github" +publish_output=true +publish_output_progress=false +verbosity_level=2 # 0,1,2 +use_extra_bad_extensions=false +use_wiki_settings_file=true +use_repo_settings_file=true +use_global_settings_file=true +ai_timeout=120 # 2minutes +max_description_tokens = 800 +max_commits_tokens = 500 +max_model_tokens = 64000 # Limits the maximum number of tokens that can be used by any model, regardless of the model's default capabilities. +patch_extra_lines = 200 +secret_provider="google_cloud_storage" +cli_mode=false +ai_disclaimer_title="" # Pro feature, title for a collapsible disclaimer to AI outputs +ai_disclaimer="" # Pro feature, full text for the AI disclaimer + +[pr_reviewer] # /review # +# enable/disable features +require_score_review=false +require_tests_review=true +require_estimate_effort_to_review=true +require_can_be_split_review=false +# soc2 +require_soc2_ticket=false +soc2_ticket_prompt="Does the PR description include a link to ticket in a project management system (e.g., Jira, Asana, Trello, etc.) ?" +# general options +num_code_suggestions=4 +inline_code_comments = true +ask_and_reflect=false +#automatic_review=true +persistent_comment=true +extra_instructions = "" +final_update_message = true +# review labels +enable_review_labels_security=true +enable_review_labels_effort=true +# specific configurations for incremental review (/review -i) +require_all_thresholds_for_incremental_review=false +minimal_commits_for_incremental_review=0 +minimal_minutes_for_incremental_review=0 +enable_help_text=true # Determines whether to include help text in the PR review. Enabled by default. +# auto approval +enable_auto_approval=false +maximal_review_effort=5 + +[pr_description] # /describe # +publish_labels=true +add_original_user_description=true +keep_original_user_title=true +generate_ai_title=false +use_bullet_points=true +extra_instructions = "" +enable_pr_type=true +final_update_message = true +enable_help_text=false +enable_help_comment=false +# describe as comment +publish_description_as_comment=false +publish_description_as_comment_persistent=true +## changes walkthrough section +enable_semantic_files_types=true +collapsible_file_list='adaptive' # true, false, 'adaptive' +inline_file_summary=false # false, true, 'table' +# markers +use_description_markers=false +include_generated_by_header=true + +[pr_code_suggestions] # /improve # +max_context_tokens=8000 +num_code_suggestions=4 +commitable_code_suggestions = false +extra_instructions = "" +rank_suggestions = false +enable_help_text=true +persistent_comment=false +# params for '/improve --extended' mode +auto_extended_mode=true +num_code_suggestions_per_chunk=5 +max_number_of_calls = 3 +parallel_calls = true +rank_extended_suggestions = false +final_clip_factor = 0.8 + +[pr_add_docs] # /add_docs # +extra_instructions = "" +docs_style = "Sphinx Style" # "Google Style with Args, Returns, Attributes...etc", "Numpy Style", "Sphinx Style", "PEP257", "reStructuredText" + +[pr_update_changelog] # /update_changelog # +push_changelog_changes=false +extra_instructions = "" + +[pr_analyze] # /analyze # + +[pr_test] # /test # +extra_instructions = "" +testing_framework = "" # specify the testing framework you want to use +num_tests=3 # number of tests to generate. max 5. +avoid_mocks=true # if true, the generated tests will prefer to use real objects instead of mocks +file = "" # in case there are several components with the same name, you can specify the relevant file +class_name = "" # in case there are several methods with the same name in the same file, you can specify the relevant class name +enable_help_text=true + +[pr_improve_component] # /improve_component # +num_code_suggestions=4 +extra_instructions = "" +file = "" # in case there are several components with the same name, you can specify the relevant file +class_name = "" + +[checks] # /checks (pro feature) # +enable_auto_checks_feedback=true +excluded_checks_list=["lint"] # list of checks to exclude, for example: ["check1", "check2"] +persistent_comment=true +enable_help_text=true + +[pr_help] # /help # + +[pr_config] # /config # + +[github] +# The type of deployment to create. Valid values are 'app' or 'user'. +deployment_type = "user" +ratelimit_retries = 5 +base_url = "https://api.github.com" +publish_inline_comments_fallback_with_verification = true +try_fix_invalid_inline_comments = true + +[litellm] +drop_params = true \ No newline at end of file diff --git a/docs/components/index.md b/docs/components/index.md index c55fcc9..f599aa1 100644 --- a/docs/components/index.md +++ b/docs/components/index.md @@ -17,6 +17,8 @@ | [langchain_vector_store_delete](langchain_vector_store_delete.md) | This component allows for entries in a LangChain Vector Store to be deleted. This is needed for the continued maintenance of the vector store. Due to the nature of langchain vector stores, you need to specify an embedding component even though it is not used in this component. | | [langchain_vector_store_embedding_index](langchain_vector_store_embedding_index.md) | Use LangChain Vector Stores to index text for later semantic searches. This will take text, run it through an embedding model and then store it in a vector database. | | [langchain_vector_store_embedding_search](langchain_vector_store_embedding_search.md) | Use LangChain Vector Stores to search a vector store with a semantic search. This will take text, run it through an embedding model with a query embedding and then find the closest matches in the store. | +| [litellm_chat_model](litellm_chat_model.md) | LiteLLM chat model component | +| [litellm_chat_model_with_history](litellm_chat_model_with_history.md) | LiteLLM model handler component with conversation history | | [message_filter](message_filter.md) | A filtering component. This will apply a user configurable expression. If the expression evaluates to True, the message will be passed on. If the expression evaluates to False, the message will be discarded. If the message is discarded, any previous components that require an acknowledgement will be acknowledged. | | [openai_chat_model](openai_chat_model.md) | OpenAI chat model component | | [openai_chat_model_with_history](openai_chat_model_with_history.md) | OpenAI chat model component with conversation history | @@ -30,3 +32,5 @@ | [websearch_bing](websearch_bing.md) | Perform a search query on Bing. | | [websearch_duckduckgo](websearch_duckduckgo.md) | Perform a search query on DuckDuckGo. | | [websearch_google](websearch_google.md) | Perform a search query on Google. | +| [websocket_input](websocket_input.md) | Listen for incoming messages on a websocket connection. | +| [websocket_output](websocket_output.md) | Send messages to a websocket connection. | diff --git a/docs/components/litellm_chat_model.md b/docs/components/litellm_chat_model.md new file mode 100644 index 0000000..5dd4cf6 --- /dev/null +++ b/docs/components/litellm_chat_model.md @@ -0,0 +1,84 @@ +# LiteLLMChatModel + +LiteLLM chat model component + +## Configuration Parameters + +```yaml +component_name: +component_module: litellm_chat_model +component_config: + action: + load_balancer: + embedding_params: + temperature: + stream_to_flow: + stream_to_next_component: + llm_mode: + stream_batch_size: + set_response_uuid_in_user_properties: + history_max_turns: + history_max_time: + history_max_turns: + history_max_time: +``` + +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| action | True | inference | The action to perform (e.g., 'inference', 'embedding') | +| load_balancer | False | | Add a list of models to load balancer. | +| embedding_params | False | | LiteLLM model parameters. The model, api_key and base_url are mandatory.find more models at https://docs.litellm.ai/docs/providersfind more parameters at https://docs.litellm.ai/docs/completion/input | +| temperature | False | 0.7 | Sampling temperature to use | +| stream_to_flow | False | | Name the flow to stream the output to - this must be configured for llm_mode='stream'. This is mutually exclusive with stream_to_next_component. | +| stream_to_next_component | False | False | Whether to stream the output to the next component in the flow. This is mutually exclusive with stream_to_flow. | +| llm_mode | False | none | The mode for streaming results: 'sync' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response. | +| stream_batch_size | False | 15 | The minimum number of words in a single streaming result. Default: 15. | +| set_response_uuid_in_user_properties | False | False | Whether to set the response_uuid in the user_properties of the input_message. This will allow other components to correlate streaming chunks with the full response. | +| history_max_turns | False | 10 | Maximum number of conversation turns to keep in history | +| history_max_time | False | 3600 | Maximum time to keep conversation history (in seconds) | +| history_max_turns | False | 10 | Maximum number of conversation turns to keep in history | +| history_max_time | False | 3600 | Maximum time to keep conversation history (in seconds) | + + +## Component Input Schema + +``` +{ + messages: [ + { + role: , + content: + }, + ... + ], + clear_history_but_keep_depth: +} +``` +| Field | Required | Description | +| --- | --- | --- | +| messages | True | | +| messages[].role | True | | +| messages[].content | True | | +| clear_history_but_keep_depth | False | Clear history but keep the last N messages. If 0, clear all history. If not set, do not clear history. | + + +## Component Output Schema + +``` +{ + content: , + chunk: , + response_uuid: , + first_chunk: , + last_chunk: , + streaming: +} +``` +| Field | Required | Description | +| --- | --- | --- | +| content | True | The generated response from the model | +| chunk | False | The current chunk of the response | +| response_uuid | False | The UUID of the response | +| first_chunk | False | Whether this is the first chunk of the response | +| last_chunk | False | Whether this is the last chunk of the response | +| streaming | False | Whether this is a streaming response | diff --git a/docs/components/litellm_chat_model_with_history.md b/docs/components/litellm_chat_model_with_history.md new file mode 100644 index 0000000..8c5ea58 --- /dev/null +++ b/docs/components/litellm_chat_model_with_history.md @@ -0,0 +1,84 @@ +# LiteLLMChatModelWithHistory + +LiteLLM model handler component with conversation history + +## Configuration Parameters + +```yaml +component_name: +component_module: litellm_chat_model_with_history +component_config: + action: + load_balancer: + embedding_params: + temperature: + stream_to_flow: + stream_to_next_component: + llm_mode: + stream_batch_size: + set_response_uuid_in_user_properties: + history_max_turns: + history_max_time: + history_max_turns: + history_max_time: +``` + +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| action | True | inference | The action to perform (e.g., 'inference', 'embedding') | +| load_balancer | False | | Add a list of models to load balancer. | +| embedding_params | False | | LiteLLM model parameters. The model, api_key and base_url are mandatory.find more models at https://docs.litellm.ai/docs/providersfind more parameters at https://docs.litellm.ai/docs/completion/input | +| temperature | False | 0.7 | Sampling temperature to use | +| stream_to_flow | False | | Name the flow to stream the output to - this must be configured for llm_mode='stream'. This is mutually exclusive with stream_to_next_component. | +| stream_to_next_component | False | False | Whether to stream the output to the next component in the flow. This is mutually exclusive with stream_to_flow. | +| llm_mode | False | none | The mode for streaming results: 'sync' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response. | +| stream_batch_size | False | 15 | The minimum number of words in a single streaming result. Default: 15. | +| set_response_uuid_in_user_properties | False | False | Whether to set the response_uuid in the user_properties of the input_message. This will allow other components to correlate streaming chunks with the full response. | +| history_max_turns | False | 10 | Maximum number of conversation turns to keep in history | +| history_max_time | False | 3600 | Maximum time to keep conversation history (in seconds) | +| history_max_turns | False | 10 | Maximum number of conversation turns to keep in history | +| history_max_time | False | 3600 | Maximum time to keep conversation history (in seconds) | + + +## Component Input Schema + +``` +{ + messages: [ + { + role: , + content: + }, + ... + ], + clear_history_but_keep_depth: +} +``` +| Field | Required | Description | +| --- | --- | --- | +| messages | True | | +| messages[].role | True | | +| messages[].content | True | | +| clear_history_but_keep_depth | False | Clear history but keep the last N messages. If 0, clear all history. If not set, do not clear history. | + + +## Component Output Schema + +``` +{ + content: , + chunk: , + response_uuid: , + first_chunk: , + last_chunk: , + streaming: +} +``` +| Field | Required | Description | +| --- | --- | --- | +| content | True | The generated response from the model | +| chunk | False | The current chunk of the response | +| response_uuid | False | The UUID of the response | +| first_chunk | False | Whether this is the first chunk of the response | +| last_chunk | False | Whether this is the last chunk of the response | +| streaming | False | Whether this is a streaming response | diff --git a/docs/components/openai_chat_model.md b/docs/components/openai_chat_model.md index b9cc612..e41c669 100644 --- a/docs/components/openai_chat_model.md +++ b/docs/components/openai_chat_model.md @@ -56,9 +56,19 @@ component_config: ``` { - content: + content: , + chunk: , + response_uuid: , + first_chunk: , + last_chunk: , + streaming: } ``` | Field | Required | Description | | --- | --- | --- | | content | True | The generated response from the model | +| chunk | False | The current chunk of the response | +| response_uuid | False | The UUID of the response | +| first_chunk | False | Whether this is the first chunk of the response | +| last_chunk | False | Whether this is the last chunk of the response | +| streaming | False | Whether this is a streaming response | diff --git a/docs/components/openai_chat_model_with_history.md b/docs/components/openai_chat_model_with_history.md index c72f818..9c7c4dc 100644 --- a/docs/components/openai_chat_model_with_history.md +++ b/docs/components/openai_chat_model_with_history.md @@ -62,9 +62,19 @@ component_config: ``` { - content: + content: , + chunk: , + response_uuid: , + first_chunk: , + last_chunk: , + streaming: } ``` | Field | Required | Description | | --- | --- | --- | | content | True | The generated response from the model | +| chunk | False | The current chunk of the response | +| response_uuid | False | The UUID of the response | +| first_chunk | False | Whether this is the first chunk of the response | +| last_chunk | False | Whether this is the last chunk of the response | +| streaming | False | Whether this is a streaming response | diff --git a/docs/components/websocket_input.md b/docs/components/websocket_input.md new file mode 100644 index 0000000..44bb2a1 --- /dev/null +++ b/docs/components/websocket_input.md @@ -0,0 +1,39 @@ +# WebsocketInput + +Listen for incoming messages on a websocket connection. + +## Configuration Parameters + +```yaml +component_name: +component_module: websocket_input +component_config: + listen_port: + serve_html: + html_path: + payload_encoding: + payload_format: +``` + +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| listen_port | False | | Port to listen on (optional) | +| serve_html | False | False | Serve the example HTML file | +| html_path | False | examples/websocket/websocket_example_app.html | Path to the HTML file to serve | +| payload_encoding | False | none | Encoding for the payload (utf-8, base64, gzip, none) | +| payload_format | False | json | Format for the payload (json, yaml, text) | + + + +## Component Output Schema + +``` +{ + payload: { + + } +} +``` +| Field | Required | Description | +| --- | --- | --- | +| payload | True | The decoded JSON payload received from the WebSocket | diff --git a/docs/components/websocket_output.md b/docs/components/websocket_output.md new file mode 100644 index 0000000..d631dd9 --- /dev/null +++ b/docs/components/websocket_output.md @@ -0,0 +1,40 @@ +# WebsocketOutput + +Send messages to a websocket connection. + +## Configuration Parameters + +```yaml +component_name: +component_module: websocket_output +component_config: + listen_port: + serve_html: + html_path: + payload_encoding: + payload_format: +``` + +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| listen_port | False | | Port to listen on (optional) | +| serve_html | False | False | Serve the example HTML file | +| html_path | False | examples/websocket/websocket_example_app.html | Path to the HTML file to serve | +| payload_encoding | False | none | Encoding for the payload (utf-8, base64, gzip, none) | +| payload_format | False | json | Format for the payload (json, yaml, text) | + + +## Component Input Schema + +``` +{ + payload: { + + }, + socket_id: +} +``` +| Field | Required | Description | +| --- | --- | --- | +| payload | True | The payload to be sent via WebSocket | +| socket_id | False | Identifier for the WebSocket connection | diff --git a/examples/llm/litellm_chat.yaml b/examples/llm/litellm_chat.yaml new file mode 100644 index 0000000..83ba283 --- /dev/null +++ b/examples/llm/litellm_chat.yaml @@ -0,0 +1,117 @@ +# This process will create a flow where the LiteLLM agent distributes requests across multiple LLM models. +# Solace -> LiteLLM -> Solace +# +# It will subscribe to `demo/question` and expect an event with the payload: +# +# The input message has the following schema: +# { +# "text": "" +# } +# +# Output is published to the topic `demo/question/response` +# +# It will then send an event back to Solace with the topic: `demo/question/response` +# +# Dependencies: +# pip install litellm +# +# required ENV variables: +# - OPENAI_API_KEY +# - OPENAI_API_ENDPOINT +# - OPENAI_MODEL_NAME +# - ANTHROPIC_MODEL_NAME +# - ANTHROPIC_API_KEY +# - ANTHROPIC_API_ENDPOINT +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN +# +# Supported models: OpenAI, Anthropic, Azure, Huggingface, Ollama, Google VertexAI +# More models are available in https://docs.litellm.ai/docs/providers +# Note: For most models, the model provider’s name should be used as a prefix for the model name (e.g. azure/chatgpt-v-2) + +--- +log: + stdout_log_level: INFO + log_file_level: DEBUG + log_file: solace_ai_connector.log + +shared_config: + - broker_config: &broker_connection + broker_type: solace + broker_url: ${SOLACE_BROKER_URL} + broker_username: ${SOLACE_BROKER_USERNAME} + broker_password: ${SOLACE_BROKER_PASSWORD} + broker_vpn: ${SOLACE_BROKER_VPN} + +# Take from input broker and publish back to Solace +flows: + # broker input processing + - name: Simple template to LLM + components: + # Input from a Solace broker + - component_name: solace_sw_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_queue_name: demo_question + broker_subscriptions: + - topic: demo/question + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # + # Do an LLM request + # + - component_name: llm_request + component_module: litellm_chat_model + component_config: + llm_mode: none # options: none or stream + load_balancer: + - model_name: "gpt-4o" # model alias + litellm_params: + model: ${OPENAI_MODEL_NAME} + api_key: ${OPENAI_API_KEY} + api_base: ${OPENAI_API_ENDPOINT} + temperature: 0.01 + # add any other parameters here + - model_name: "claude-3-5-sonnet" # model alias + litellm_params: + model: ${ANTHROPIC_MODEL_NAME} + api_key: ${ANTHROPIC_API_KEY} + api_base: ${ANTHROPIC_API_ENDPOINT} + # add any other parameters here + # add more models here + input_transforms: + - type: copy + source_expression: | + template:You are a helpful AI assistant. Please help with the user's request below: + + {{text://input.payload:text}} + + dest_expression: user_data.llm_input:messages.0.content + - type: copy + source_expression: static:user + dest_expression: user_data.llm_input:messages.0.role + input_selection: + source_expression: user_data.llm_input + + # Send response back to broker + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.output:payload + - type: copy + source_expression: template:{{text://input.topic}}/response + dest_expression: user_data.output:topic + input_selection: + source_expression: user_data.output diff --git a/examples/llm/litellm_chat_with_history.yaml b/examples/llm/litellm_chat_with_history.yaml new file mode 100644 index 0000000..3fcb96a --- /dev/null +++ b/examples/llm/litellm_chat_with_history.yaml @@ -0,0 +1,121 @@ +# This process will establish the following flow where the LiteLLM agent retains the history of questions and answers. +# Solace -> LiteLLM -> Solace +# +# It will subscribe to `demo/question` and expect an event with the payload: +# +# The input message has the following schema: +# { +# "text": "" +# "session_id": "" +# } +# +# Output is published to the topic `demo/question/response` +# +# It will then send an event back to Solace with the topic: `demo/question/response` +# +# Dependencies: +# pip install litellm +# +# Required ENV variables: +# - OPENAI_API_KEY +# - OPENAI_API_ENDPOINT +# - OPENAI_MODEL_NAME +# - ANTHROPIC_MODEL_NAME +# - ANTHROPIC_API_KEY +# - ANTHROPIC_API_ENDPOINT +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN +# +# Supported models: OpenAI, Anthropic, Azure, Huggingface, Ollama, Google VertexAI +# More models are available in https://docs.litellm.ai/docs/providers +# Note: For most models, the model provider’s name should be used as a prefix for the model name (e.g. azure/chatgpt-v-2) + + +--- +log: + stdout_log_level: INFO + log_file_level: DEBUG + log_file: solace_ai_connector.log + +shared_config: + - broker_config: &broker_connection + broker_type: solace + broker_url: ${SOLACE_BROKER_URL} + broker_username: ${SOLACE_BROKER_USERNAME} + broker_password: ${SOLACE_BROKER_PASSWORD} + broker_vpn: ${SOLACE_BROKER_VPN} + +# Take from input broker and publish back to Solace +flows: + # broker input processing + - name: Simple template to LLM + components: + # Input from a Solace broker + - component_name: solace_sw_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_queue_name: demo_question67 + broker_subscriptions: + - topic: demo/question + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # + # Do an LLM request + # + - component_name: llm_request + component_module: litellm_chat_model_with_history + component_config: + load_balancer: + - model_name: "gpt-4o" # model alias + litellm_params: + model: ${OPENAI_MODEL_NAME} + api_key: ${OPENAI_API_KEY} + api_base: ${OPENAI_API_ENDPOINT} + temperature: 0.01 + # add any other parameters here + - model_name: "claude-3-5-sonnet" # model alias + litellm_params: + model: ${ANTHROPIC_MODEL_NAME} + api_key: ${ANTHROPIC_API_KEY} + api_base: ${ANTHROPIC_API_ENDPOINT} + # add any other parameters here + # add more models here + input_transforms: + - type: copy + source_expression: | + template:You are a helpful AI assistant. Please help with the user's request below: + + {{text://input.payload:text}} + + dest_expression: user_data.llm_input:messages.0.content + - type: copy + source_expression: static:user + dest_expression: user_data.llm_input:messages.0.role + - type: copy + source_expression: input.payload:session_id + dest_expression: user_data.llm_input:session_id + input_selection: + source_expression: user_data.llm_input + + # Send response back to broker + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.output:payload + - type: copy + source_expression: template:{{text://input.topic}}/response + dest_expression: user_data.output:topic + input_selection: + source_expression: user_data.output diff --git a/examples/llm/litellm_embedding.yaml b/examples/llm/litellm_embedding.yaml new file mode 100644 index 0000000..3784859 --- /dev/null +++ b/examples/llm/litellm_embedding.yaml @@ -0,0 +1,106 @@ +# This below flow embeds the input text and sends it back to the Solace broker. +# Solace -> LiteLLM -> Solace +# +# It will subscribe to `demo/question` and expect an event with the payload: +# +# The input message has the following schema: +# { +# "items": ["item1", "item2", ...] +# } +# +# Output is published to the topic `demo/question/response` +# +# It will then send an event back to Solace with the topic: `demo/question/response` +# +# Dependencies: +# pip install litellm +# +# required ENV variables: +# - OPENAI_EMBEDDING_MODEL_NAME +# - OPENAI_API_KEY +# - OPENAI_API_ENDPOINT +# - AZURE_EMBEDDING_MODEL_NAME +# - AZURE_API_KEY +# - AZURE_API_ENDPOINT +# - SOLACE_BROKER_URL +# - SOLACE_BROKER_USERNAME +# - SOLACE_BROKER_PASSWORD +# - SOLACE_BROKER_VPN +# +# Supported models: OpenAI, Azure and Huggingface +# More models are available in https://docs.litellm.ai/docs/providers + +--- +log: + stdout_log_level: INFO + log_file_level: DEBUG + log_file: solace_ai_connector.log + +shared_config: + - broker_config: &broker_connection + broker_type: solace + broker_url: ${SOLACE_BROKER_URL} + broker_username: ${SOLACE_BROKER_USERNAME} + broker_password: ${SOLACE_BROKER_PASSWORD} + broker_vpn: ${SOLACE_BROKER_VPN} + +# Take from input broker and publish back to Solace +flows: + # broker input processing + - name: Simple template to LLM + components: + # Input from a Solace broker + - component_name: solace_sw_broker + component_module: broker_input + component_config: + <<: *broker_connection + broker_queue_name: demo_question434 + broker_subscriptions: + - topic: demo/question + qos: 1 + payload_encoding: utf-8 + payload_format: json + + # + # Do an LLM request + # + - component_name: llm_request + component_module: litellm_embeddings + component_config: + load_balancer: + - model_name: "text-embedding-ada-002" # model alias + litellm_params: + model: ${OPENAI_EMBEDDING_MODEL_NAME} + api_key: ${OPENAI_API_KEY} + api_base: ${OPENAI_API_ENDPOINT} + # add any other parameters here + - model_name: "text-embedding-3-large" # model alias + itellm_params: + model: ${AZURE_EMBEDDING_MODEL_NAME} + api_key: ${AZURE_API_KEY} + api_base: ${AZURE_API_ENDPOINT} + # add any other parameters here + input_transforms: + - type: copy + source_expression: input.payload + dest_expression: user_data.llm_input:items + input_selection: + source_expression: user_data.llm_input:items + + # Send response back to broker + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: previous + dest_expression: user_data.output:payload + - type: copy + source_expression: template:{{text://input.topic}}/response + dest_expression: user_data.output:topic + input_selection: + source_expression: user_data.output diff --git a/src/solace_ai_connector/common/utils.py b/src/solace_ai_connector/common/utils.py index cc8299a..aaf9ee4 100755 --- a/src/solace_ai_connector/common/utils.py +++ b/src/solace_ai_connector/common/utils.py @@ -125,8 +125,9 @@ def import_module(module, base_path=None, component_package=None): ".components", ".components.general", ".components.general.for_testing", - ".components.general.langchain", - ".components.general.openai", + ".components.general.llm.langchain", + ".components.general.llm.openai", + ".components.general.llm.litellm", ".components.general.websearch", ".components.inputs_outputs", ".transforms", diff --git a/src/solace_ai_connector/components/__init__.py b/src/solace_ai_connector/components/__init__.py index e9e54b9..d20da98 100755 --- a/src/solace_ai_connector/components/__init__.py +++ b/src/solace_ai_connector/components/__init__.py @@ -25,7 +25,7 @@ give_ack_output, ) -from .general.langchain import ( +from .general.llm.langchain import ( langchain_embeddings, langchain_vector_store_delete, langchain_chat_model, @@ -34,6 +34,12 @@ langchain_vector_store_embedding_search, ) +from .general.llm.litellm import ( + litellm_chat_model, + litellm_embeddings, + litellm_chat_model_with_history, +) + from .general.websearch import ( websearch_duckduckgo, websearch_google, @@ -57,20 +63,19 @@ from .general.iterate import Iterate from .general.message_filter import MessageFilter from .general.parser import Parser -from .general.langchain.langchain_base import LangChainBase -from .general.langchain.langchain_embeddings import LangChainEmbeddings -from .general.langchain.langchain_vector_store_delete import LangChainVectorStoreDelete -from .general.langchain.langchain_chat_model import LangChainChatModel -from .general.langchain.langchain_chat_model_with_history import ( +from .general.llm.langchain.langchain_base import LangChainBase +from .general.llm.langchain.langchain_embeddings import LangChainEmbeddings +from .general.llm.langchain.langchain_vector_store_delete import LangChainVectorStoreDelete +from .general.llm.langchain.langchain_chat_model import LangChainChatModel +from .general.llm.langchain.langchain_chat_model_with_history import ( LangChainChatModelWithHistory, ) -from .general.langchain.langchain_vector_store_embedding_index import ( +from .general.llm.langchain.langchain_vector_store_embedding_index import ( LangChainVectorStoreEmbeddingsIndex, ) -from .general.langchain.langchain_vector_store_embedding_search import ( +from .general.llm.langchain.langchain_vector_store_embedding_search import ( LangChainVectorStoreEmbeddingsSearch, ) from .general.websearch.websearch_duckduckgo import WebSearchDuckDuckGo from .general.websearch.websearch_google import WebSearchGoogle -from .general.websearch.websearch_bing import WebSearchBing - +from .general.websearch.websearch_bing import WebSearchBing \ No newline at end of file diff --git a/src/solace_ai_connector/components/general/llm/common/chat_history_handler.py b/src/solace_ai_connector/components/general/llm/common/chat_history_handler.py new file mode 100644 index 0000000..c811f6e --- /dev/null +++ b/src/solace_ai_connector/components/general/llm/common/chat_history_handler.py @@ -0,0 +1,86 @@ +"""Generic chat history handler.""" + +import time +from ....component_base import ComponentBase +from .....common.log import log + +class ChatHistoryHandler(ComponentBase): + def __init__(self, info, **kwargs): + super().__init__(info, **kwargs) + self.history_max_turns = self.get_config("history_max_turns", 10) + self.history_max_time = self.get_config("history_max_time", 3600) + self.history_key = f"{self.flow_name}_{self.name}_history" + + # Set up hourly timer for history cleanup + self.add_timer(3600000, "history_cleanup", interval_ms=3600000) + + def prune_history(self, session_id, history): + current_time = time.time() + if current_time - history[session_id]["last_accessed"] > self.history_max_time: + history[session_id]["messages"] = [] + elif len(history[session_id]["messages"]) > self.history_max_turns * 2: + history[session_id]["messages"] = history[session_id]["messages"][ + -self.history_max_turns * 2 : + ] + log.debug(f"Pruned history for session {session_id}") + self.make_history_start_with_user_message(session_id, history) + + def clear_history_but_keep_depth(self, session_id: str, depth: int, history): + if session_id in history: + messages = history[session_id]["messages"] + # If the depth is 0, then clear all history + if depth == 0: + history[session_id]["messages"] = [] + history[session_id]["last_accessed"] = time.time() + return + + # Check if the history is already shorter than the depth + if len(messages) <= depth: + # Do nothing, since the history is already shorter than the depth + return + + # If the message at depth is not a user message, then + # increment the depth until a user message is found + while depth < len(messages) and messages[-depth]["role"] != "user": + depth += 1 + history[session_id]["messages"] = messages[-depth:] + history[session_id]["last_accessed"] = time.time() + + # In the unlikely case that the history starts with a non-user message, + # remove it + self.make_history_start_with_user_message(session_id, history) + log.info(f"Cleared history for session {session_id}") + + def make_history_start_with_user_message(self, session_id, history): + if session_id in history: + messages = history[session_id]["messages"] + if messages: + if messages[0]["role"] == "system": + # Start from the second message if the first is "system" + start_index = 1 + else: + # Start from the first message otherwise + start_index = 0 + + while ( + start_index < len(messages) + and messages[start_index]["role"] != "user" + ): + messages.pop(start_index) + + def handle_timer_event(self, timer_data): + if timer_data["timer_id"] == "history_cleanup": + self.history_age_out() + + def history_age_out(self): + with self.get_lock(self.history_key): + history = self.kv_store_get(self.history_key) or {} + current_time = time.time() + for session_id in list(history.keys()): + if ( + current_time - history[session_id]["last_accessed"] + > self.history_max_time + ): + del history[session_id] + log.info(f"Removed history for session {session_id}") + self.kv_store_set(self.history_key, history) diff --git a/src/solace_ai_connector/components/general/langchain/__init__.py b/src/solace_ai_connector/components/general/llm/langchain/__init__.py similarity index 100% rename from src/solace_ai_connector/components/general/langchain/__init__.py rename to src/solace_ai_connector/components/general/llm/langchain/__init__.py diff --git a/src/solace_ai_connector/components/general/langchain/langchain_base.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_base.py similarity index 94% rename from src/solace_ai_connector/components/general/langchain/langchain_base.py rename to src/solace_ai_connector/components/general/llm/langchain/langchain_base.py index 4ffda1b..6bf94f4 100644 --- a/src/solace_ai_connector/components/general/langchain/langchain_base.py +++ b/src/solace_ai_connector/components/general/llm/langchain/langchain_base.py @@ -2,8 +2,8 @@ import importlib -from ...component_base import ComponentBase -from ....common.utils import resolve_config_values +from ....component_base import ComponentBase +from .....common.utils import resolve_config_values class LangChainBase(ComponentBase): diff --git a/src/solace_ai_connector/components/general/langchain/langchain_chat_model.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model.py similarity index 100% rename from src/solace_ai_connector/components/general/langchain/langchain_chat_model.py rename to src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model.py diff --git a/src/solace_ai_connector/components/general/langchain/langchain_chat_model_base.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_base.py similarity index 99% rename from src/solace_ai_connector/components/general/langchain/langchain_chat_model_base.py rename to src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_base.py index bd3b464..58c7ae5 100644 --- a/src/solace_ai_connector/components/general/langchain/langchain_chat_model_base.py +++ b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_base.py @@ -5,7 +5,7 @@ from abc import abstractmethod from langchain_core.output_parsers import JsonOutputParser -from ....common.utils import get_obj_text +from .....common.utils import get_obj_text from langchain.schema.messages import ( HumanMessage, SystemMessage, diff --git a/src/solace_ai_connector/components/general/langchain/langchain_chat_model_with_history.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_with_history.py similarity index 99% rename from src/solace_ai_connector/components/general/langchain/langchain_chat_model_with_history.py rename to src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_with_history.py index 9885c68..4569c30 100644 --- a/src/solace_ai_connector/components/general/langchain/langchain_chat_model_with_history.py +++ b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_with_history.py @@ -16,7 +16,7 @@ SystemMessage, ) -from ....common.message import Message +from .....common.message import Message from .langchain_chat_model_base import ( LangChainChatModelBase, info_base, diff --git a/src/solace_ai_connector/components/general/langchain/langchain_embeddings.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_embeddings.py similarity index 100% rename from src/solace_ai_connector/components/general/langchain/langchain_embeddings.py rename to src/solace_ai_connector/components/general/llm/langchain/langchain_embeddings.py diff --git a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_delete.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_vector_store_delete.py similarity index 99% rename from src/solace_ai_connector/components/general/langchain/langchain_vector_store_delete.py rename to src/solace_ai_connector/components/general/llm/langchain/langchain_vector_store_delete.py index 667d882..7d84712 100644 --- a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_delete.py +++ b/src/solace_ai_connector/components/general/llm/langchain/langchain_vector_store_delete.py @@ -3,7 +3,7 @@ # as well, so the configuration for this component will also include the # embedding model configuration -from ....common.log import log +from .....common.log import log from .langchain_vector_store_embedding_base import ( LangChainVectorStoreEmbeddingsBase, ) diff --git a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_base.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_vector_store_embedding_base.py similarity index 100% rename from src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_base.py rename to src/solace_ai_connector/components/general/llm/langchain/langchain_vector_store_embedding_base.py diff --git a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_index.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_vector_store_embedding_index.py similarity index 100% rename from src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_index.py rename to src/solace_ai_connector/components/general/llm/langchain/langchain_vector_store_embedding_index.py diff --git a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_search.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_vector_store_embedding_search.py similarity index 99% rename from src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_search.py rename to src/solace_ai_connector/components/general/llm/langchain/langchain_vector_store_embedding_search.py index 1b974eb..5295abd 100644 --- a/src/solace_ai_connector/components/general/langchain/langchain_vector_store_embedding_search.py +++ b/src/solace_ai_connector/components/general/llm/langchain/langchain_vector_store_embedding_search.py @@ -4,7 +4,7 @@ embedding model configuration """ -from ....common.log import log +from .....common.log import log from .langchain_vector_store_embedding_base import ( LangChainVectorStoreEmbeddingsBase, ) diff --git a/src/solace_ai_connector/components/general/openai/__init__.py b/src/solace_ai_connector/components/general/llm/litellm/__init__.py similarity index 100% rename from src/solace_ai_connector/components/general/openai/__init__.py rename to src/solace_ai_connector/components/general/llm/litellm/__init__.py diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py new file mode 100644 index 0000000..bcd5ddf --- /dev/null +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py @@ -0,0 +1,128 @@ +"""Base class for LiteLLM chat models""" + +import uuid +import time +import asyncio +import litellm + +from ....component_base import ComponentBase +from .....common.message import Message +from .....common.log import log + +litellm_info_base = { + "class_name": "LiteLLMChatModelBase", + "description": "Base class for LiteLLM chat models", + "config_parameters": [ + { + "name": "load_balancer", + "required": False, + "description": ( + "Add a list of models to load balancer." + ), + "default": "", + }, + { + "name": "embedding_params", + "required": False, + "description": ( + "LiteLLM model parameters. The model, api_key and base_url are mandatory." + "find more models at https://docs.litellm.ai/docs/providers" + "find more parameters at https://docs.litellm.ai/docs/completion/input" + ), + "default": "", + }, + { + "name": "temperature", + "required": False, + "description": "Sampling temperature to use", + "default": 0.7, + }, + { + "name": "stream_to_flow", + "required": False, + "description": ( + "Name the flow to stream the output to - this must be configured for " + "llm_mode='stream'. This is mutually exclusive with stream_to_next_component." + ), + "default": "", + }, + { + "name": "stream_to_next_component", + "required": False, + "description": ( + "Whether to stream the output to the next component in the flow. " + "This is mutually exclusive with stream_to_flow." + ), + "default": False, + }, + { + "name": "llm_mode", + "required": False, + "description": ( + "The mode for streaming results: 'sync' or 'stream'. 'stream' " + "will just stream the results to the named flow. 'none' will " + "wait for the full response." + ), + "default": "none", + }, + { + "name": "stream_batch_size", + "required": False, + "description": "The minimum number of words in a single streaming result. Default: 15.", + "default": 15, + }, + { + "name": "set_response_uuid_in_user_properties", + "required": False, + "description": ( + "Whether to set the response_uuid in the user_properties of the " + "input_message. This will allow other components to correlate " + "streaming chunks with the full response." + ), + "default": False, + "type": "boolean", + }, + ], +} + + +class LiteLLMBase(ComponentBase): + def __init__(self, module_info, **kwargs): + super().__init__(module_info, **kwargs) + self.init() + self.init_load_balancer() + + def init(self): + litellm.suppress_debug_info = True + self.load_balancer = self.get_config("load_balancer") + self.stream_to_flow = self.get_config("stream_to_flow") + self.stream_to_next_component = self.get_config("stream_to_next_component") + self.llm_mode = self.get_config("llm_mode") + self.stream_batch_size = self.get_config("stream_batch_size") + self.set_response_uuid_in_user_properties = self.get_config( + "set_response_uuid_in_user_properties" + ) + if self.stream_to_flow and self.stream_to_next_component: + raise ValueError( + "stream_to_flow and stream_to_next_component are mutually exclusive" + ) + self.router = None + + def init_load_balancer(self): + """initialize a load balancer""" + try: + self.router = litellm.Router(model_list=self.load_balancer) + log.debug("Load balancer initialized with models: %s", self.load_balancer) + except Exception as e: + raise ValueError(f"Error initializing load balancer: {e}") + + def load_balance(self, messages, stream): + """load balance the messages""" + response = self.router.completion(model=self.load_balancer[0]["model_name"], + messages=messages, stream=stream) + log.debug("Load balancer response: %s", response) + return response + + def invoke(self, message, data): + """invoke the model""" + pass \ No newline at end of file diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model.py new file mode 100644 index 0000000..5ae9c12 --- /dev/null +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model.py @@ -0,0 +1,11 @@ +"""LiteLLM chat model component""" + +from .litellm_chat_model_base import LiteLLMChatModelBase, litellm_chat_info_base + +info = litellm_chat_info_base.copy() +info["class_name"] = "LiteLLMChatModel" +info["description"] = "LiteLLM chat component" + +class LiteLLMChatModel(LiteLLMChatModelBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) \ No newline at end of file diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py new file mode 100644 index 0000000..9aea43c --- /dev/null +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py @@ -0,0 +1,219 @@ +"""LiteLLM chat model component""" + +from .litellm_base import LiteLLMBase, litellm_info_base +from .....common.log import log + +litellm_chat_info_base = litellm_info_base.copy() +litellm_chat_info_base.update( + { + "class_name": "LiteLLMChatModelBase", + "description": "LiteLLM chat model base component", + "input_schema": { + "type": "object", + "properties": { + "messages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "role": { + "type": "string", + "enum": ["system", "user", "assistant"], + }, + "content": {"type": "string"}, + }, + "required": ["role", "content"], + }, + }, + }, + "required": ["messages"], + }, + "output_schema": { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "The generated response from the model", + }, + "chunk": { + "type": "string", + "description": "The current chunk of the response", + }, + "response_uuid": { + "type": "string", + "description": "The UUID of the response", + }, + "first_chunk": { + "type": "boolean", + "description": "Whether this is the first chunk of the response", + }, + "last_chunk": { + "type": "boolean", + "description": "Whether this is the last chunk of the response", + }, + "streaming": { + "type": "boolean", + "description": "Whether this is a streaming response", + }, + }, + "required": ["content"], + }, + }, +) + +class LiteLLMChatModelBase(LiteLLMBase): + def __init__(self, info, **kwargs): + super().__init__(info, **kwargs) + + def invoke(self, message, data): + """invoke the model""" + messages = data.get("messages", []) + + if self.llm_mode == "stream": + return self.invoke_stream(message, messages) + else: + return self.invoke_non_stream(messages) + + def invoke_non_stream(self, messages): + """invoke the model without streaming""" + max_retries = 3 + while max_retries > 0: + try: + response = self.load_balance(messages, stream=False) + return {"content": response.choices[0].message.content} + except Exception as e: + log.error("Error invoking LiteLLM: %s", e) + max_retries -= 1 + if max_retries <= 0: + raise e + else: + time.sleep(1) + + def invoke_stream(self, message, messages): + """invoke the model with streaming""" + response_uuid = str(uuid.uuid4()) + if self.set_response_uuid_in_user_properties: + message.set_data("input.user_properties:response_uuid", response_uuid) + + aggregate_result = "" + current_batch = "" + first_chunk = True + + max_retries = 3 + while max_retries > 0: + try: + response = self.load_balance(messages, stream=True) + + for chunk in response: + # If we get any response, then don't retry + max_retries = 0 + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + aggregate_result += content + current_batch += content + if len(current_batch.split()) >= self.stream_batch_size: + if self.stream_to_flow: + self.send_streaming_message( + message, + current_batch, + aggregate_result, + response_uuid, + first_chunk, + False, + ) + elif self.stream_to_next_component: + self.send_to_next_component( + message, + current_batch, + aggregate_result, + response_uuid, + first_chunk, + False, + ) + current_batch = "" + first_chunk = False + except Exception as e: + log.error("Error invoking LiteLLM: %s", e) + max_retries -= 1 + if max_retries <= 0: + raise e + else: + # Small delay before retrying + time.sleep(1) + + if self.stream_to_next_component: + # Just return the last chunk + return { + "content": aggregate_result, + "chunk": current_batch, + "response_uuid": response_uuid, + "first_chunk": first_chunk, + "last_chunk": True, + "streaming": True, + } + + if self.stream_to_flow: + self.send_streaming_message( + message, + current_batch, + aggregate_result, + response_uuid, + first_chunk, + True, + ) + + return {"content": aggregate_result, "response_uuid": response_uuid} + + def send_streaming_message( + self, + input_message, + chunk, + aggregate_result, + response_uuid, + first_chunk=False, + last_chunk=False, + ): + message = Message( + payload={ + "chunk": chunk, + "content": aggregate_result, + "response_uuid": response_uuid, + "first_chunk": first_chunk, + "last_chunk": last_chunk, + "streaming": True, + }, + user_properties=input_message.get_user_properties(), + ) + self.send_to_flow(self.stream_to_flow, message) + + def send_to_next_component( + self, + input_message, + chunk, + aggregate_result, + response_uuid, + first_chunk=False, + last_chunk=False, + ): + message = Message( + payload={ + "chunk": chunk, + "content": aggregate_result, + "response_uuid": response_uuid, + "first_chunk": first_chunk, + "last_chunk": last_chunk, + "streaming": True, + }, + user_properties=input_message.get_user_properties(), + ) + + result = { + "chunk": chunk, + "content": aggregate_result, + "response_uuid": response_uuid, + "first_chunk": first_chunk, + "last_chunk": last_chunk, + "streaming": True, + } + + self.process_post_invoke(result, message) diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_with_history.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_with_history.py new file mode 100644 index 0000000..c98e353 --- /dev/null +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_with_history.py @@ -0,0 +1,105 @@ +"""LiteLLM chat model component with conversation history""" + +import time + +from .litellm_chat_model_base import LiteLLMChatModelBase, litellm_chat_info_base +from ..common.chat_history_handler import ChatHistoryHandler +from .....common.log import log + +info = litellm_chat_info_base.copy() +info["class_name"] = "LiteLLMChatModelWithHistory" +info["description"] = "LiteLLM model handler component with conversation history" +info["config_parameters"].extend( + [ + { + "name": "history_max_turns", + "required": False, + "description": "Maximum number of conversation turns to keep in history", + "default": 10, + }, + { + "name": "history_max_time", + "required": False, + "description": "Maximum time to keep conversation history (in seconds)", + "default": 3600, + }, + ] +) + +info["input_schema"]["properties"]["clear_history_but_keep_depth"] = { + "type": "integer", + "minimum": 0, + "description": "Clear history but keep the last N messages. If 0, clear all history. If not set, do not clear history.", +} + +class LiteLLMChatModelWithHistory(LiteLLMChatModelBase, ChatHistoryHandler): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + self.history_max_turns = self.get_config("history_max_turns", 10) + self.history_max_time = self.get_config("history_max_time", 3600) + self.history_key = f"{self.flow_name}_{self.name}_history" + + # Set up hourly timer for history cleanup + self.add_timer(3600000, "history_cleanup", interval_ms=3600000) + + def invoke(self, message, data): + session_id = data.get("session_id") + if not session_id: + raise ValueError("session_id is not provided") + + clear_history_but_keep_depth = data.get("clear_history_but_keep_depth") + try: + if clear_history_but_keep_depth is not None: + clear_history_but_keep_depth = max(0, int(clear_history_but_keep_depth)) + except (TypeError, ValueError): + log.error("Invalid clear_history_but_keep_depth value. Defaulting to 0.") + clear_history_but_keep_depth = 0 + messages = data.get("messages", []) + + with self.get_lock(self.history_key): + history = self.kv_store_get(self.history_key) or {} + if session_id not in history: + history[session_id] = {"messages": [], "last_accessed": time.time()} + + if clear_history_but_keep_depth is not None: + self.clear_history_but_keep_depth( + session_id, clear_history_but_keep_depth, history + ) + + session_history = history[session_id]["messages"] + log.debug(f"Session history: {session_history}") + + # If the passed in messages have a system message and the history's + # first message is a system message, then replace the history's first + # message with the passed in messages' system message + if ( + len(messages) + and messages[0]["role"] == "system" + and len(session_history) + and session_history[0]["role"] == "system" + ): + session_history[0] = messages[0] + session_history.extend(messages[1:]) + else: + session_history.extend(messages) + + history[session_id]["last_accessed"] = time.time() + + self.prune_history(session_id, history) + + response = super().invoke( + message, {"messages": history[session_id]["messages"]} + ) + + # Add the assistant's response to the history + history[session_id]["messages"].append( + { + "role": "assistant", + "content": response["content"], + } + ) + + self.kv_store_set(self.history_key, history) + log.debug(f"Updated history: {history}") + + return response \ No newline at end of file diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_embeddings.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_embeddings.py new file mode 100644 index 0000000..7c52170 --- /dev/null +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_embeddings.py @@ -0,0 +1,51 @@ +"""LiteLLM embedding component""" + +from .litellm_base import LiteLLMBase, litellm_info_base +from .....common.log import log + +info = litellm_info_base.copy() +info.update( + { + "class_name": "LiteLLMEmbeddings", + "description": "Embed text using a LiteLLM model", + "input_schema": { + "type": "object", + "properties": { + "items": { + "type": "array", + "description": "A single element or a list of elements to embed", + }, + }, + "required": ["items"], + }, + "output_schema": { + "type": "object", + "properties": { + "embeddings": { + "type": "array", + "description": ( + "A list of floating point numbers representing the embeddings. " + "Its length is the size of vector that the embedding model produces" + ), + "items": {"type": "float"}, + } + }, + "required": ["embeddings"], + }, + } +) + +class LiteLLMEmbeddings(LiteLLMBase): + def __init__(self, **kwargs): + super().__init__(info, **kwargs) + + def invoke(self, message, data): + """invoke the embedding model""" + items = data.get("items", []) + + response = self.router.embedding(model=self.load_balancer[0]["model_name"], + input=items) + + # Extract the embedding data from the response + embedding_data = response['data'][0]['embedding'] + return {"embeddings": embedding_data} \ No newline at end of file diff --git a/src/solace_ai_connector/components/general/llm/openai/__init__.py b/src/solace_ai_connector/components/general/llm/openai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/solace_ai_connector/components/general/openai/openai_chat_model.py b/src/solace_ai_connector/components/general/llm/openai/openai_chat_model.py similarity index 100% rename from src/solace_ai_connector/components/general/openai/openai_chat_model.py rename to src/solace_ai_connector/components/general/llm/openai/openai_chat_model.py diff --git a/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py b/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_base.py similarity index 98% rename from src/solace_ai_connector/components/general/openai/openai_chat_model_base.py rename to src/solace_ai_connector/components/general/llm/openai/openai_chat_model_base.py index d0a53d5..beabd07 100755 --- a/src/solace_ai_connector/components/general/openai/openai_chat_model_base.py +++ b/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_base.py @@ -4,9 +4,9 @@ import time from openai import OpenAI -from ...component_base import ComponentBase -from ....common.message import Message -from ....common.log import log +from ....component_base import ComponentBase +from .....common.message import Message +from .....common.log import log openai_info_base = { "class_name": "OpenAIChatModelBase", diff --git a/src/solace_ai_connector/components/general/openai/openai_chat_model_with_history.py b/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_with_history.py similarity index 54% rename from src/solace_ai_connector/components/general/openai/openai_chat_model_with_history.py rename to src/solace_ai_connector/components/general/llm/openai/openai_chat_model_with_history.py index ba7fe64..e9f0da8 100644 --- a/src/solace_ai_connector/components/general/openai/openai_chat_model_with_history.py +++ b/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_with_history.py @@ -4,6 +4,7 @@ import time from .openai_chat_model_base import OpenAIChatModelBase, openai_info_base +from ..common.chat_history_handler import ChatHistoryHandler info = openai_info_base.copy() info["class_name"] = "OpenAIChatModelWithHistory" @@ -33,7 +34,7 @@ } -class OpenAIChatModelWithHistory(OpenAIChatModelBase): +class OpenAIChatModelWithHistory(OpenAIChatModelBase, ChatHistoryHandler): def __init__(self, **kwargs): super().__init__(info, **kwargs) self.history_max_turns = self.get_config("history_max_turns", 10) @@ -45,6 +46,9 @@ def __init__(self, **kwargs): def invoke(self, message, data): session_id = data.get("session_id") + if not session_id: + raise ValueError("session_id is not provided") + clear_history_but_keep_depth = data.get("clear_history_but_keep_depth") try: if clear_history_but_keep_depth is not None: @@ -98,72 +102,4 @@ def invoke(self, message, data): self.kv_store_set(self.history_key, history) - return response - - def prune_history(self, session_id, history): - current_time = time.time() - if current_time - history[session_id]["last_accessed"] > self.history_max_time: - history[session_id]["messages"] = [] - elif len(history[session_id]["messages"]) > self.history_max_turns * 2: - history[session_id]["messages"] = history[session_id]["messages"][ - -self.history_max_turns * 2 : - ] - self.make_history_start_with_user_message(session_id, history) - - def clear_history_but_keep_depth(self, session_id: str, depth: int, history): - if session_id in history: - messages = history[session_id]["messages"] - # If the depth is 0, then clear all history - if depth == 0: - history[session_id]["messages"] = [] - history[session_id]["last_accessed"] = time.time() - return - - # Check if the history is already shorter than the depth - if len(messages) <= depth: - # Do nothing, since the history is already shorter than the depth - return - - # If the message at depth is not a user message, then - # increment the depth until a user message is found - while depth < len(messages) and messages[-depth]["role"] != "user": - depth += 1 - history[session_id]["messages"] = messages[-depth:] - history[session_id]["last_accessed"] = time.time() - - # In the unlikely case that the history starts with a non-user message, - # remove it - self.make_history_start_with_user_message(session_id, history) - - def make_history_start_with_user_message(self, session_id, history): - if session_id in history: - messages = history[session_id]["messages"] - if messages: - if messages[0]["role"] == "system": - # Start from the second message if the first is "system" - start_index = 1 - else: - # Start from the first message otherwise - start_index = 0 - - while ( - start_index < len(messages) - and messages[start_index]["role"] != "user" - ): - messages.pop(start_index) - - def handle_timer_event(self, timer_data): - if timer_data["timer_id"] == "history_cleanup": - self.history_age_out() - - def history_age_out(self): - with self.get_lock(self.history_key): - history = self.kv_store_get(self.history_key) or {} - current_time = time.time() - for session_id in list(history.keys()): - if ( - current_time - history[session_id]["last_accessed"] - > self.history_max_time - ): - del history[session_id] - self.kv_store_set(self.history_key, history) + return response \ No newline at end of file From 3b4c99a6a81600389e7c7e145406cdb4f099b4c8 Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:48:43 -0500 Subject: [PATCH 39/55] FEATURE: Enhance embedding functionality with batch and image support. (#55) * added litellm component * support chat history * trimmed comments * dynamically get the model parameters * added llm load balancer * added the AI PR reviewer workflow * fixed minor issues * controlled session id * refactored chat history and reused codes * fixed minor logging issue * reverted minor changes * handle all LiteLLM inferences and embedding requests by the load balancer * updated documents * fix: remove useless import command * refactor: restructure the LLM components * Added support for batch embedding + image embedding * Added init py * typo * fixed embedding --------- Co-authored-by: alimosaed --- docs/components/langchain_embeddings.md | 9 +++-- examples/llm/litellm_embedding.yaml | 4 +- .../components/general/llm/__init__.py | 0 .../components/general/llm/common/__init__.py | 0 .../llm/langchain/langchain_embeddings.py | 37 +++++++++++++------ .../general/llm/litellm/litellm_base.py | 2 +- .../general/llm/litellm/litellm_chat_model.py | 2 +- .../general/llm/litellm/litellm_embeddings.py | 7 +++- 8 files changed, 41 insertions(+), 20 deletions(-) create mode 100644 src/solace_ai_connector/components/general/llm/__init__.py create mode 100644 src/solace_ai_connector/components/general/llm/common/__init__.py diff --git a/docs/components/langchain_embeddings.md b/docs/components/langchain_embeddings.md index 669bb52..f9fd71f 100644 --- a/docs/components/langchain_embeddings.md +++ b/docs/components/langchain_embeddings.md @@ -24,14 +24,17 @@ component_config: ``` { - text: , + items: [ +, + ... + ], type: } ``` | Field | Required | Description | | --- | --- | --- | -| text | True | The text to embed | -| type | False | The type of embedding to use: 'document' or 'query' - default is 'document' | +| items | True | A single element or a list of elements to embed | +| type | False | The type of embedding to use: 'document', 'query', or 'image' - default is 'document' | ## Component Output Schema diff --git a/examples/llm/litellm_embedding.yaml b/examples/llm/litellm_embedding.yaml index 3784859..9cb8e3b 100644 --- a/examples/llm/litellm_embedding.yaml +++ b/examples/llm/litellm_embedding.yaml @@ -62,9 +62,9 @@ flows: payload_format: json # - # Do an LLM request + # Do an Embedding request # - - component_name: llm_request + - component_name: embedding_request component_module: litellm_embeddings component_config: load_balancer: diff --git a/src/solace_ai_connector/components/general/llm/__init__.py b/src/solace_ai_connector/components/general/llm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/solace_ai_connector/components/general/llm/common/__init__.py b/src/solace_ai_connector/components/general/llm/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/solace_ai_connector/components/general/llm/langchain/langchain_embeddings.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_embeddings.py index 7951622..770afdf 100644 --- a/src/solace_ai_connector/components/general/llm/langchain/langchain_embeddings.py +++ b/src/solace_ai_connector/components/general/llm/langchain/langchain_embeddings.py @@ -33,16 +33,16 @@ "input_schema": { "type": "object", "properties": { - "text": { - "type": "string", - "description": "The text to embed", + "items": { + "type": "array", + "description": "A single element or a list of elements to embed", }, "type": { - "type": "string", # This is document or query - "description": "The type of embedding to use: 'document' or 'query' - default is 'document'", + "type": "string", # This is document, query, or image + "description": "The type of embedding to use: 'document', 'query', or 'image' - default is 'document'", }, }, - "required": ["text"], + "required": ["items"], }, "output_schema": { "type": "object", @@ -66,13 +66,28 @@ def __init__(self, **kwargs): super().__init__(info, **kwargs) def invoke(self, message, data): - text = data["text"] + items = data["items"] embedding_type = data.get("type", "document") - embeddings = None + items = [items] if type(items) != list else items + if embedding_type == "document": - embeddings = self.component.embed_documents([text]) + return self.embed_documents(items) elif embedding_type == "query": - embeddings = [self.component.embed_query(text)] + return self.embed_queries(items) + elif embedding_type == "image": + return self.embed_images(items) + + def embed_documents(self, documents): + embeddings = self.component.embed_documents(documents) + return {"embeddings": embeddings} + + def embed_queries(self, queries): + embeddings = [] + for query in queries: + embeddings.append(self.component.embed_query(query)) + return {"embeddings": embeddings} - return {"embedding": embeddings[0]} + def embed_images(self, images): + embeddings = self.component.embed_images(images) + return {"embeddings": embeddings} diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py index bcd5ddf..da0ccf5 100644 --- a/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py @@ -125,4 +125,4 @@ def load_balance(self, messages, stream): def invoke(self, message, data): """invoke the model""" - pass \ No newline at end of file + pass diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model.py index 5ae9c12..4bc3ecc 100644 --- a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model.py +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model.py @@ -8,4 +8,4 @@ class LiteLLMChatModel(LiteLLMChatModelBase): def __init__(self, **kwargs): - super().__init__(info, **kwargs) \ No newline at end of file + super().__init__(info, **kwargs) diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_embeddings.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_embeddings.py index 7c52170..3f5e0cd 100644 --- a/src/solace_ai_connector/components/general/llm/litellm/litellm_embeddings.py +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_embeddings.py @@ -47,5 +47,8 @@ def invoke(self, message, data): input=items) # Extract the embedding data from the response - embedding_data = response['data'][0]['embedding'] - return {"embeddings": embedding_data} \ No newline at end of file + embeddings = [] + for embedding in response.get("data", []): + embeddings.append(embedding['embedding']) + + return {"embeddings": embeddings} \ No newline at end of file From 0143aca686bd4936af5adca0866881611b7f1178 Mon Sep 17 00:00:00 2001 From: Ali Parvizi <91437594+alimosaed@users.noreply.github.com> Date: Mon, 4 Nov 2024 11:26:52 -0500 Subject: [PATCH 40/55] Alireza/none/hotfix package dependencies (#56) * updated the requirements * fix: update the requirements --- requirements.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d40e69..c7e9047 100755 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,8 @@ langchain-core~=0.3.0 langchain~=0.3.0 PyYAML~=6.0.1 Requests~=2.32.3 -solace_pubsubplus~=1.8.0 \ No newline at end of file +solace_pubsubplus~=1.8.0 +litellm~=1.51.3 +Flask~=3.0.3 +Flask-SocketIO~=5.4.1 +build~=1.2.2.post1 \ No newline at end of file From 6008a20eab76a6c4c7acf15d2e9556e219f85c2f Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Wed, 6 Nov 2024 15:24:35 -0500 Subject: [PATCH 41/55] AI-131: Improve exit process to support force quit and fix ctrl-c issue (#57) * Fixed exiting on ctrl-c issue * enhanced the exit process to support force quit for single component --- docs/components/index.md | 3 +- docs/components/litellm_chat_model.md | 4 +- .../litellm_chat_model_with_history.md | 2 - docs/components/litellm_embeddings.md | 68 +++++++++++++++++++ .../components/component_base.py | 5 +- .../inputs_outputs/websocket_base.py | 23 ++++--- src/solace_ai_connector/main.py | 20 ++++-- .../solace_ai_connector.py | 4 +- 8 files changed, 107 insertions(+), 22 deletions(-) create mode 100644 docs/components/litellm_embeddings.md diff --git a/docs/components/index.md b/docs/components/index.md index f599aa1..8390909 100644 --- a/docs/components/index.md +++ b/docs/components/index.md @@ -17,8 +17,9 @@ | [langchain_vector_store_delete](langchain_vector_store_delete.md) | This component allows for entries in a LangChain Vector Store to be deleted. This is needed for the continued maintenance of the vector store. Due to the nature of langchain vector stores, you need to specify an embedding component even though it is not used in this component. | | [langchain_vector_store_embedding_index](langchain_vector_store_embedding_index.md) | Use LangChain Vector Stores to index text for later semantic searches. This will take text, run it through an embedding model and then store it in a vector database. | | [langchain_vector_store_embedding_search](langchain_vector_store_embedding_search.md) | Use LangChain Vector Stores to search a vector store with a semantic search. This will take text, run it through an embedding model with a query embedding and then find the closest matches in the store. | -| [litellm_chat_model](litellm_chat_model.md) | LiteLLM chat model component | +| [litellm_chat_model](litellm_chat_model.md) | LiteLLM chat component | | [litellm_chat_model_with_history](litellm_chat_model_with_history.md) | LiteLLM model handler component with conversation history | +| [litellm_embeddings](litellm_embeddings.md) | Embed text using a LiteLLM model | | [message_filter](message_filter.md) | A filtering component. This will apply a user configurable expression. If the expression evaluates to True, the message will be passed on. If the expression evaluates to False, the message will be discarded. If the message is discarded, any previous components that require an acknowledgement will be acknowledged. | | [openai_chat_model](openai_chat_model.md) | OpenAI chat model component | | [openai_chat_model_with_history](openai_chat_model_with_history.md) | OpenAI chat model component with conversation history | diff --git a/docs/components/litellm_chat_model.md b/docs/components/litellm_chat_model.md index 5dd4cf6..556ee08 100644 --- a/docs/components/litellm_chat_model.md +++ b/docs/components/litellm_chat_model.md @@ -1,6 +1,6 @@ # LiteLLMChatModel -LiteLLM chat model component +LiteLLM chat component ## Configuration Parameters @@ -8,7 +8,6 @@ LiteLLM chat model component component_name: component_module: litellm_chat_model component_config: - action: load_balancer: embedding_params: temperature: @@ -25,7 +24,6 @@ component_config: | Parameter | Required | Default | Description | | --- | --- | --- | --- | -| action | True | inference | The action to perform (e.g., 'inference', 'embedding') | | load_balancer | False | | Add a list of models to load balancer. | | embedding_params | False | | LiteLLM model parameters. The model, api_key and base_url are mandatory.find more models at https://docs.litellm.ai/docs/providersfind more parameters at https://docs.litellm.ai/docs/completion/input | | temperature | False | 0.7 | Sampling temperature to use | diff --git a/docs/components/litellm_chat_model_with_history.md b/docs/components/litellm_chat_model_with_history.md index 8c5ea58..29aa640 100644 --- a/docs/components/litellm_chat_model_with_history.md +++ b/docs/components/litellm_chat_model_with_history.md @@ -8,7 +8,6 @@ LiteLLM model handler component with conversation history component_name: component_module: litellm_chat_model_with_history component_config: - action: load_balancer: embedding_params: temperature: @@ -25,7 +24,6 @@ component_config: | Parameter | Required | Default | Description | | --- | --- | --- | --- | -| action | True | inference | The action to perform (e.g., 'inference', 'embedding') | | load_balancer | False | | Add a list of models to load balancer. | | embedding_params | False | | LiteLLM model parameters. The model, api_key and base_url are mandatory.find more models at https://docs.litellm.ai/docs/providersfind more parameters at https://docs.litellm.ai/docs/completion/input | | temperature | False | 0.7 | Sampling temperature to use | diff --git a/docs/components/litellm_embeddings.md b/docs/components/litellm_embeddings.md new file mode 100644 index 0000000..6542c29 --- /dev/null +++ b/docs/components/litellm_embeddings.md @@ -0,0 +1,68 @@ +# LiteLLMEmbeddings + +Embed text using a LiteLLM model + +## Configuration Parameters + +```yaml +component_name: +component_module: litellm_embeddings +component_config: + load_balancer: + embedding_params: + temperature: + stream_to_flow: + stream_to_next_component: + llm_mode: + stream_batch_size: + set_response_uuid_in_user_properties: + history_max_turns: + history_max_time: + history_max_turns: + history_max_time: +``` + +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| load_balancer | False | | Add a list of models to load balancer. | +| embedding_params | False | | LiteLLM model parameters. The model, api_key and base_url are mandatory.find more models at https://docs.litellm.ai/docs/providersfind more parameters at https://docs.litellm.ai/docs/completion/input | +| temperature | False | 0.7 | Sampling temperature to use | +| stream_to_flow | False | | Name the flow to stream the output to - this must be configured for llm_mode='stream'. This is mutually exclusive with stream_to_next_component. | +| stream_to_next_component | False | False | Whether to stream the output to the next component in the flow. This is mutually exclusive with stream_to_flow. | +| llm_mode | False | none | The mode for streaming results: 'sync' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response. | +| stream_batch_size | False | 15 | The minimum number of words in a single streaming result. Default: 15. | +| set_response_uuid_in_user_properties | False | False | Whether to set the response_uuid in the user_properties of the input_message. This will allow other components to correlate streaming chunks with the full response. | +| history_max_turns | False | 10 | Maximum number of conversation turns to keep in history | +| history_max_time | False | 3600 | Maximum time to keep conversation history (in seconds) | +| history_max_turns | False | 10 | Maximum number of conversation turns to keep in history | +| history_max_time | False | 3600 | Maximum time to keep conversation history (in seconds) | + + +## Component Input Schema + +``` +{ + items: [ +, + ... + ] +} +``` +| Field | Required | Description | +| --- | --- | --- | +| items | True | A single element or a list of elements to embed | + + +## Component Output Schema + +``` +{ + embeddings: [ + , + ... + ] +} +``` +| Field | Required | Description | +| --- | --- | --- | +| embeddings | True | A list of floating point numbers representing the embeddings. Its length is the size of vector that the embedding model produces | diff --git a/src/solace_ai_connector/components/component_base.py b/src/solace_ai_connector/components/component_base.py index f7c8c41..4284bfb 100644 --- a/src/solace_ai_connector/components/component_base.py +++ b/src/solace_ai_connector/components/component_base.py @@ -405,7 +405,10 @@ def stop_component(self): def cleanup(self): """Clean up resources used by the component""" log.debug("%sCleaning up component", self.log_identifier) - self.stop_component() + try: + self.stop_component() + except KeyboardInterrupt: + pass if hasattr(self, "input_queue"): while not self.input_queue.empty(): try: diff --git a/src/solace_ai_connector/components/inputs_outputs/websocket_base.py b/src/solace_ai_connector/components/inputs_outputs/websocket_base.py index ffe1ad9..de01c4c 100644 --- a/src/solace_ai_connector/components/inputs_outputs/websocket_base.py +++ b/src/solace_ai_connector/components/inputs_outputs/websocket_base.py @@ -1,12 +1,12 @@ """Base class for WebSocket components.""" +import os +import signal from abc import ABC, abstractmethod from flask import Flask, send_file, request from flask_socketio import SocketIO -import logging from ...common.log import log from ..component_base import ComponentBase -import copy from flask.logging import default_handler base_info = { @@ -111,14 +111,21 @@ def run_server(self): ) def stop_server(self): - if self.socketio: - self.socketio.stop() - if self.app: - func = request.environ.get("werkzeug.server.shutdown") + try: + func = request.environ.get('werkzeug.server.shutdown') if func is None: - raise RuntimeError("Not running with the Werkzeug Server") + raise RuntimeError('Not running with the Werkzeug Server') func() - + except RuntimeError: + # Ignore the error if the server is already shutdown + pass + try: + self.socketio.stop() + except Exception as e: + pass + # force exiting component + os.kill(os.getpid(), signal.SIGINT) + def get_sockets(self): if not self.sockets: self.sockets = self.kv_store_get("websocket_connections") or {} diff --git a/src/solace_ai_connector/main.py b/src/solace_ai_connector/main.py index 7bf43cf..fe2ccf6 100644 --- a/src/solace_ai_connector/main.py +++ b/src/solace_ai_connector/main.py @@ -2,7 +2,8 @@ import sys import re import yaml -from pathlib import Path +import atexit + from .solace_ai_connector import SolaceAiConnector @@ -103,13 +104,22 @@ def main(): # Create the application app = SolaceAiConnector(full_config) + def shutdown(): + """Shutdown the application.""" + print("Stopping Solace AI Connector") + app.stop() + app.cleanup() + print("Solace AI Connector exited successfully!") + os._exit(0) + atexit.register(shutdown) + # Start the application app.run() - app.wait_for_flows() - - print("Solace AI Connector exited successfully!") - + try: + app.wait_for_flows() + except KeyboardInterrupt: + shutdown() if __name__ == "__main__": # Read in the configuration yaml filenames from the args diff --git a/src/solace_ai_connector/solace_ai_connector.py b/src/solace_ai_connector/solace_ai_connector.py index 91c349e..2bce481 100644 --- a/src/solace_ai_connector/solace_ai_connector.py +++ b/src/solace_ai_connector/solace_ai_connector.py @@ -91,7 +91,8 @@ def send_message_to_flow(self, flow_name, message): def wait_for_flows(self): """Wait for the flows to finish""" - while True: + while not self.stop_signal.is_set(): + print("-- Press Ctrl+C to stop --") try: for flow in self.flows: flow.wait_for_threads() @@ -216,6 +217,5 @@ def stop(self): self.stop_signal.set() self.timer_manager.stop() # Stop the timer manager first self.cache_service.stop() # Stop the cache service - self.wait_for_flows() if self.trace_thread: self.trace_thread.join() From 015df58ecf209c16c940b2e51d3a29a2ebf4710d Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Tue, 12 Nov 2024 08:53:04 -0500 Subject: [PATCH 42/55] AI-187: Small change to allow event-mesh-gateway components to inherit from broker_input and output (#59) * refactor: Improve BrokerInput initialization with optional module_info * Allow broker_output to be inherited with flexible info --- .../components/inputs_outputs/broker_input.py | 5 +++-- .../components/inputs_outputs/broker_output.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_input.py b/src/solace_ai_connector/components/inputs_outputs/broker_input.py index 2d277cb..38be39f 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_input.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_input.py @@ -88,8 +88,9 @@ class BrokerInput(BrokerBase): - def __init__(self, **kwargs): - super().__init__(info, **kwargs) + def __init__(self, module_info=None, **kwargs): + module_info = module_info or info + super().__init__(module_info, **kwargs) self.need_acknowledgement = True self.temporary_queue = self.get_config("temporary_queue", False) # If broker_queue_name is not provided, use temporary queue diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_output.py b/src/solace_ai_connector/components/inputs_outputs/broker_output.py index a0de440..25809b8 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_output.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_output.py @@ -93,8 +93,9 @@ class BrokerOutput(BrokerBase): - def __init__(self, **kwargs): - super().__init__(info, **kwargs) + def __init__(self, module_info=None, **kwargs): + module_info = module_info or info + super().__init__(module_info, **kwargs) self.needs_acknowledgement = False self.propagate_acknowledgements = self.get_config("propagate_acknowledgements") self.copy_user_properties = self.get_config("copy_user_properties") From 9c224178af148d33d254b247879a1980ba5977f0 Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:50:21 -0500 Subject: [PATCH 43/55] updated the parser component to support more conversions (#61) --- examples/custom_components/web_search.py | 2 +- examples/parser.yaml | 11 +++- .../components/general/parser.py | 66 ++++++++++++++----- .../solace_ai_connector.py | 5 +- 4 files changed, 61 insertions(+), 23 deletions(-) diff --git a/examples/custom_components/web_search.py b/examples/custom_components/web_search.py index 55542ff..136d97a 100755 --- a/examples/custom_components/web_search.py +++ b/examples/custom_components/web_search.py @@ -1,4 +1,4 @@ -# A simple pass-through component - what goes in comes out +# A component to search the web using APIs. import sys import requests diff --git a/examples/parser.yaml b/examples/parser.yaml index ead9324..d4c93b7 100755 --- a/examples/parser.yaml +++ b/examples/parser.yaml @@ -16,10 +16,19 @@ flows: component_module: stdin_input # Using Custom component - - component_name: parser_component + - component_name: json_to_yaml + component_module: parser + component_config: + input_format: json + output_format: yaml + input_selection: + source_expression: previous:text + + - component_name: yaml_to_dict component_module: parser component_config: input_format: yaml + output_format: dict input_selection: source_expression: previous diff --git a/src/solace_ai_connector/components/general/parser.py b/src/solace_ai_connector/components/general/parser.py index e6dd705..466e4e9 100644 --- a/src/solace_ai_connector/components/general/parser.py +++ b/src/solace_ai_connector/components/general/parser.py @@ -1,5 +1,7 @@ """Parse a JSON or YAML file.""" + import yaml +import json from langchain_core.output_parsers import JsonOutputParser from ..component_base import ComponentBase @@ -8,36 +10,64 @@ info = { "class_name": "Parser", - "description": "Parse a JSON string and extract data fields.", + "description": "Parse input from the given type to output type.", "config_parameters": [ { "name": "input_format", "required": True, - "description": "The input format of the data. Options: 'json', 'yaml'.", + "description": "The input format of the data. Options: 'dict', 'json' 'yaml'. 'yaml' and 'json' must be string formatted.", + }, + { + "name": "output_format", + "required": True, + "description": "The input format of the data. Options: 'dict', 'json' 'yaml'. 'yaml' and 'json' will be string formatted.", }, ], } + class Parser(ComponentBase): def __init__(self, **kwargs): super().__init__(info, **kwargs) - def invoke(self, message, data): - text = data["text"] - res_format = self.get_config("input_format", "json") - if res_format == "json": - try: - parser = JsonOutputParser() - json_res = parser.invoke(text) - return json_res - except Exception as e: - raise ValueError(f"Error parsing the input JSON: {str(e)}") from e - elif res_format == "yaml": + def str_to_dict(self, text, format): + if format == "json": + return JsonOutputParser().invoke(text) + elif format == "yaml": obj_text = get_obj_text("yaml", text) - try: - yaml_res = yaml.safe_load(obj_text) - return yaml_res - except Exception as e: - raise ValueError(f"Error parsing the input YAML: {str(e)}") from e + return yaml.safe_load(obj_text) else: return text + + def dict_to_format(self, data, format): + if format == "json": + return json.dumps(data, indent=4) + elif format == "yaml": + return yaml.dump(data) + else: + return data + + def invoke(self, message, data): + input_format = self.get_config("input_format") + output_format = self.get_config("output_format") + if not input_format or not output_format: + raise ValueError("Input and output format must be provided.") + + dict_data = data # By default assuming it's already a dictionary + try: + if input_format == "json" or input_format == "yaml": + dict_data = self.str_to_dict(data, input_format) + else: + raise ValueError(f"Invalid input format: {input_format}") + except Exception as e: + raise ValueError(f"Error converting input: {str(e)}") from e + + try: + if output_format == "json" or output_format == "yaml": + return self.dict_to_format(dict_data, output_format) + elif output_format == "dict": + return dict_data + else: + raise ValueError(f"Invalid output format: {output_format}") + except Exception as e: + raise ValueError(f"Error converting output: {str(e)}") from e diff --git a/src/solace_ai_connector/solace_ai_connector.py b/src/solace_ai_connector/solace_ai_connector.py index 2bce481..dd04442 100644 --- a/src/solace_ai_connector/solace_ai_connector.py +++ b/src/solace_ai_connector/solace_ai_connector.py @@ -92,7 +92,6 @@ def send_message_to_flow(self, flow_name, message): def wait_for_flows(self): """Wait for the flows to finish""" while not self.stop_signal.is_set(): - print("-- Press Ctrl+C to stop --") try: for flow in self.flows: flow.wait_for_threads() @@ -120,9 +119,9 @@ def setup_logging(self): """Setup logging""" log_config = self.config.get("log", {}) stdout_log_level = log_config.get("stdout_log_level", "INFO") - file_log_level = log_config.get("file_log_level", "DEBUG") + log_file_level = log_config.get("log_file_level", "DEBUG") log_file = log_config.get("log_file", "solace_ai_connector.log") - setup_log(log_file, stdout_log_level, file_log_level) + setup_log(log_file, stdout_log_level, log_file_level) def setup_trace(self): """Setup trace""" From 14d6a5771d3e7932f37b93c5823d8692e46e81a7 Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Wed, 13 Nov 2024 15:26:30 -0500 Subject: [PATCH 44/55] AI-287: A few changes to improve error reporting (#60) * refactor: Improve BrokerInput initialization with optional module_info * Allow broker_output to be inherited with flexible info * Add flag to disable error queue for a flow * A few changes during testing --- src/solace_ai_connector/components/component_base.py | 8 ++++++-- src/solace_ai_connector/flow/flow.py | 9 ++++++++- tests/test_acks.py | 6 ++---- tests/test_error_flows.py | 8 ++++---- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/solace_ai_connector/components/component_base.py b/src/solace_ai_connector/components/component_base.py index 4284bfb..0d04dae 100644 --- a/src/solace_ai_connector/components/component_base.py +++ b/src/solace_ai_connector/components/component_base.py @@ -17,6 +17,7 @@ class ComponentBase: + def __init__(self, module_info, **kwargs): self.module_info = module_info self.config = kwargs.pop("config", {}) @@ -33,6 +34,7 @@ def __init__(self, module_info, **kwargs): self.connector = kwargs.pop("connector", None) self.timer_manager = kwargs.pop("timer_manager", None) self.cache_service = kwargs.pop("cache_service", None) + self.put_errors_in_error_queue = kwargs.pop("put_errors_in_error_queue", True) self.component_config = self.config.get("component_config") or {} self.broker_request_response_config = self.config.get( @@ -88,8 +90,7 @@ def handle_component_error(self, e, event): e, traceback.format_exc(), ) - if self.error_queue: - self.handle_error(e, event) + self.handle_error(e, event) def get_next_event(self): # Check if there is a get_next_message defined by a @@ -353,10 +354,13 @@ def trace_data(self, data): ) def handle_error(self, exception, event): + if self.error_queue is None or not self.put_errors_in_error_queue: + return error_message = { "error": { "text": str(exception), "exception": type(exception).__name__, + "traceback": traceback.format_exc(), }, "location": { "instance": self.instance_name, diff --git a/src/solace_ai_connector/flow/flow.py b/src/solace_ai_connector/flow/flow.py index ea5091c..555c9f1 100644 --- a/src/solace_ai_connector/flow/flow.py +++ b/src/solace_ai_connector/flow/flow.py @@ -9,6 +9,7 @@ class FlowLockManager: + def __init__(self): self._lock = threading.Lock() self.locks = {} @@ -22,6 +23,7 @@ def get_lock(self, lock_name): class FlowKVStore: + def __init__(self): self.store = {} @@ -54,7 +56,6 @@ def __init__( self.name = flow_config.get("name") self.module_info = None self.stop_signal = stop_signal - self.error_queue = error_queue self.instance_name = instance_name self.trace_queue = trace_queue self.flow_instance_index = flow_instance_index @@ -64,6 +65,11 @@ def __init__( self.flow_lock_manager = Flow._lock_manager self.flow_kv_store = Flow._kv_store self.cache_service = connector.cache_service if connector else None + self.error_queue = error_queue + self.put_errors_in_error_queue = flow_config.get( + "put_errors_in_error_queue", True + ) + self.create_components() def get_input_queue(self): @@ -125,6 +131,7 @@ def create_component_group(self, component, index): connector=self.connector, timer_manager=self.connector.timer_manager, cache_service=self.cache_service, + put_errors_in_error_queue=self.put_errors_in_error_queue, ) sibling_component = component_instance diff --git a/tests/test_acks.py b/tests/test_acks.py index bf0b1ea..2486e39 100644 --- a/tests/test_acks.py +++ b/tests/test_acks.py @@ -54,10 +54,8 @@ def test_basic_ack(): "component": "give_ack_output", "component_index": 0, } - assert payload["error"] == { - "text": "This is an ack message", - "exception": "Exception", - } + assert payload["error"]["text"] == "This is an ack message" + assert payload["error"]["exception"] == "Exception" assert payload["message"]["payload"] == {"text": "Hello, World!"} finally: dispose_connector(connector) diff --git a/tests/test_error_flows.py b/tests/test_error_flows.py index 8e7edfe..fca11b8 100644 --- a/tests/test_error_flows.py +++ b/tests/test_error_flows.py @@ -59,9 +59,9 @@ def test_basic_error_flow(): try: assert output_message.get_data("previous") == "This is an error message" - assert output_message.get_data("input.payload")["error"] == { - "exception": "ValueError", - "text": "This is an error message", - } + + error = output_message.get_data("input.payload")["error"] + assert error["exception"] == "ValueError" + assert error["text"] == "This is an error message" finally: dispose_connector(connector) From 1c4b5d8454407a28b4c7c89cede05a4683252082 Mon Sep 17 00:00:00 2001 From: Ali Parvizi <91437594+alimosaed@users.noreply.github.com> Date: Wed, 13 Nov 2024 15:45:07 -0500 Subject: [PATCH 45/55] Alireza/none/hotfix package dependencies (#58) * updated the requirements * fix: update the requirements * fix: import required libraries --- .../general/llm/litellm/litellm_base.py | 15 ++++++--------- .../llm/litellm/litellm_chat_model_base.py | 17 +++++++++++------ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py index da0ccf5..f9d4282 100644 --- a/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py @@ -1,8 +1,5 @@ """Base class for LiteLLM chat models""" -import uuid -import time -import asyncio import litellm from ....component_base import ComponentBase @@ -16,9 +13,7 @@ { "name": "load_balancer", "required": False, - "description": ( - "Add a list of models to load balancer." - ), + "description": ("Add a list of models to load balancer."), "default": "", }, { @@ -87,6 +82,7 @@ class LiteLLMBase(ComponentBase): + def __init__(self, module_info, **kwargs): super().__init__(module_info, **kwargs) self.init() @@ -115,11 +111,12 @@ def init_load_balancer(self): log.debug("Load balancer initialized with models: %s", self.load_balancer) except Exception as e: raise ValueError(f"Error initializing load balancer: {e}") - + def load_balance(self, messages, stream): """load balance the messages""" - response = self.router.completion(model=self.load_balancer[0]["model_name"], - messages=messages, stream=stream) + response = self.router.completion( + model=self.load_balancer[0]["model_name"], messages=messages, stream=stream + ) log.debug("Load balancer response: %s", response) return response diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py index 9aea43c..a1ecff5 100644 --- a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py @@ -1,6 +1,9 @@ """LiteLLM chat model component""" +import time +import uuid from .litellm_base import LiteLLMBase, litellm_info_base +from .....common.message import Message from .....common.log import log litellm_chat_info_base = litellm_info_base.copy() @@ -61,7 +64,9 @@ }, ) + class LiteLLMChatModelBase(LiteLLMBase): + def __init__(self, info, **kwargs): super().__init__(info, **kwargs) @@ -82,12 +87,12 @@ def invoke_non_stream(self, messages): response = self.load_balance(messages, stream=False) return {"content": response.choices[0].message.content} except Exception as e: - log.error("Error invoking LiteLLM: %s", e) - max_retries -= 1 - if max_retries <= 0: - raise e - else: - time.sleep(1) + log.error("Error invoking LiteLLM: %s", e) + max_retries -= 1 + if max_retries <= 0: + raise e + else: + time.sleep(1) def invoke_stream(self, message, messages): """invoke the model with streaming""" From b0f0870ec871518a93c330991f5796aab412cfd2 Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Thu, 14 Nov 2024 12:12:50 -0500 Subject: [PATCH 46/55] AI-289: API refactor for web components + clean up (#62) * API refactor for web components + clean up * update duckduckgo to return array as well * raising error to preserve interface * update * updated scraper with timeout * minor fix * Fix scraper using Ali's solution --- docs/components/index.md | 2 +- docs/components/parser.md | 6 +- docs/components/web_scraper.md | 17 +++- docs/components/websearch_bing.md | 22 +++-- docs/components/websearch_duckduckgo.md | 22 +++-- docs/components/websearch_google.md | 22 +++-- examples/websearch/bing_web_search.yaml | 8 +- examples/websearch/duckduckgo_web_search.yaml | 8 +- examples/websearch/google_web_search.yaml | 10 +- examples/websearch/web_scraping.yaml | 12 ++- examples/websearch/websearch_router.yaml | 4 +- .../components/general/parser.py | 3 +- .../general/websearch/web_scraper.py | 98 ++++++++++++------- .../general/websearch/websearch_base.py | 18 ++-- .../general/websearch/websearch_bing.py | 59 ++++++----- .../general/websearch/websearch_duckduckgo.py | 82 +++++++++++----- .../general/websearch/websearch_google.py | 63 +++++++----- .../inputs_outputs/broker_request_response.py | 3 + 18 files changed, 298 insertions(+), 161 deletions(-) diff --git a/docs/components/index.md b/docs/components/index.md index 8390909..1b3b851 100644 --- a/docs/components/index.md +++ b/docs/components/index.md @@ -23,7 +23,7 @@ | [message_filter](message_filter.md) | A filtering component. This will apply a user configurable expression. If the expression evaluates to True, the message will be passed on. If the expression evaluates to False, the message will be discarded. If the message is discarded, any previous components that require an acknowledgement will be acknowledged. | | [openai_chat_model](openai_chat_model.md) | OpenAI chat model component | | [openai_chat_model_with_history](openai_chat_model_with_history.md) | OpenAI chat model component with conversation history | -| [parser](parser.md) | Parse a JSON string and extract data fields. | +| [parser](parser.md) | Parse input from the given type to output type. | | [pass_through](pass_through.md) | What goes in comes out | | [stdin_input](stdin_input.md) | STDIN input component. The component will prompt for input, which will then be placed in the message payload using the output schema below. The component will wait for its output message to be acknowledged before prompting for the next input. | | [stdout_output](stdout_output.md) | STDOUT output component | diff --git a/docs/components/parser.md b/docs/components/parser.md index 7bb7c5d..305d545 100644 --- a/docs/components/parser.md +++ b/docs/components/parser.md @@ -1,6 +1,6 @@ # Parser -Parse a JSON string and extract data fields. +Parse input from the given type to output type. ## Configuration Parameters @@ -9,9 +9,11 @@ component_name: component_module: parser component_config: input_format: + output_format: ``` | Parameter | Required | Default | Description | | --- | --- | --- | --- | -| input_format | True | | The input format of the data. Options: 'json', 'yaml'. | +| input_format | True | | The input format of the data. Options: 'dict', 'json' 'yaml'. 'yaml' and 'json' must be string formatted. | +| output_format | True | | The input format of the data. Options: 'dict', 'json' 'yaml'. 'yaml' and 'json' will be string formatted. | diff --git a/docs/components/web_scraper.md b/docs/components/web_scraper.md index ce52f93..0c75a61 100644 --- a/docs/components/web_scraper.md +++ b/docs/components/web_scraper.md @@ -8,24 +8,35 @@ Scrape javascript based websites. component_name: component_module: web_scraper component_config: + timeout: ``` -No configuration parameters +| Parameter | Required | Default | Description | +| --- | --- | --- | --- | +| timeout | False | 30000 | The timeout for the browser in milliseconds. | ## Component Input Schema ``` { - + url: } ``` +| Field | Required | Description | +| --- | --- | --- | +| url | False | The URL of the website to scrape. | ## Component Output Schema ``` { - + title: , + content: } ``` +| Field | Required | Description | +| --- | --- | --- | +| title | False | The title of the website. | +| content | False | The content of the website. | diff --git a/docs/components/websearch_bing.md b/docs/components/websearch_bing.md index 54d2960..4ce7a52 100644 --- a/docs/components/websearch_bing.md +++ b/docs/components/websearch_bing.md @@ -16,23 +16,31 @@ component_config: | Parameter | Required | Default | Description | | --- | --- | --- | --- | | api_key | True | | Bing API Key. | -| count | False | 10 | Number of search results to return. | +| count | False | 10 | Max number of search results to return. | | safesearch | False | Moderate | Safe search setting: Off, Moderate, or Strict. | ## Component Input Schema ``` -{ - -} + ``` ## Component Output Schema ``` -{ - -} +[ + { + title: , + snippet: , + url: + }, + ... +] ``` +| Field | Required | Description | +| --- | --- | --- | +| [].title | False | | +| [].snippet | False | | +| [].url | False | | diff --git a/docs/components/websearch_duckduckgo.md b/docs/components/websearch_duckduckgo.md index 86f6e0a..e4db95a 100644 --- a/docs/components/websearch_duckduckgo.md +++ b/docs/components/websearch_duckduckgo.md @@ -10,6 +10,7 @@ component_module: websearch_duckduckgo component_config: pretty: no_html: + count: skip_disambig: detail: ``` @@ -18,6 +19,7 @@ component_config: | --- | --- | --- | --- | | pretty | False | 1 | Beautify the search output. | | no_html | False | 1 | The number of output pages. | +| count | False | 10 | Max Number of search results to return. | | skip_disambig | False | 1 | Skip disambiguation. | | detail | False | False | Return the detail. | @@ -25,16 +27,24 @@ component_config: ## Component Input Schema ``` -{ - -} + ``` ## Component Output Schema ``` -{ - -} +[ + { + title: , + snippet: , + url: + }, + ... +] ``` +| Field | Required | Description | +| --- | --- | --- | +| [].title | False | | +| [].snippet | False | | +| [].url | False | | diff --git a/docs/components/websearch_google.md b/docs/components/websearch_google.md index 4219a57..2dbf9e7 100644 --- a/docs/components/websearch_google.md +++ b/docs/components/websearch_google.md @@ -10,6 +10,7 @@ component_module: websearch_google component_config: api_key: search_engine_id: + count: detail: ``` @@ -17,22 +18,31 @@ component_config: | --- | --- | --- | --- | | api_key | True | | Google API Key. | | search_engine_id | False | 1 | The custom search engine id. | +| count | False | 10 | Max Number of search results to return. | | detail | False | False | Return the detail. | ## Component Input Schema ``` -{ - -} + ``` ## Component Output Schema ``` -{ - -} +[ + { + title: , + snippet: , + url: + }, + ... +] ``` +| Field | Required | Description | +| --- | --- | --- | +| [].title | False | | +| [].snippet | False | | +| [].url | False | | diff --git a/examples/websearch/bing_web_search.yaml b/examples/websearch/bing_web_search.yaml index 378fb48..94b0644 100755 --- a/examples/websearch/bing_web_search.yaml +++ b/examples/websearch/bing_web_search.yaml @@ -17,7 +17,7 @@ flows: - component_name: stdin component_module: stdin_input - # Using Custom component + # Using web component - component_name: web_search_component component_module: websearch_bing component_config: @@ -26,8 +26,8 @@ flows: count: 2 detail: false input_selection: - source_expression: previous + source_expression: previous:text - # Output to a standard out + # Output to a standard out - component_name: stdout - component_module: stdout_output \ No newline at end of file + component_module: stdout_output diff --git a/examples/websearch/duckduckgo_web_search.yaml b/examples/websearch/duckduckgo_web_search.yaml index 76eb588..beef9ba 100755 --- a/examples/websearch/duckduckgo_web_search.yaml +++ b/examples/websearch/duckduckgo_web_search.yaml @@ -14,14 +14,14 @@ flows: - component_name: stdin component_module: stdin_input - # Using Custom component + # Using web component - component_name: web_search_component component_module: websearch_duckduckgo component_config: detail: false input_selection: - source_expression: previous + source_expression: previous:text - # Output to a standard out + # Output to a standard out - component_name: stdout - component_module: stdout_output \ No newline at end of file + component_module: stdout_output diff --git a/examples/websearch/google_web_search.yaml b/examples/websearch/google_web_search.yaml index f42e8df..7331cc8 100755 --- a/examples/websearch/google_web_search.yaml +++ b/examples/websearch/google_web_search.yaml @@ -2,6 +2,8 @@ # The input payload is: # # +# Get API key from: https://developers.google.com/custom-search/v1/introduction +# Create a search engine from: https://programmablesearchengine.google.com/controlpanel/create # Required ENV variables: # - Google_API_KEY # - GOOGLE_ENGINE_ID @@ -18,7 +20,7 @@ flows: - component_name: stdin component_module: stdin_input - # Using Custom component + # Using web component - component_name: web_search_component component_module: websearch_google component_config: @@ -26,8 +28,8 @@ flows: search_engine_id: ${GOOGLE_ENGINE_ID} detail: false input_selection: - source_expression: previous + source_expression: previous:text - # Output to a standard out + # Output to a standard out - component_name: stdout - component_module: stdout_output \ No newline at end of file + component_module: stdout_output diff --git a/examples/websearch/web_scraping.yaml b/examples/websearch/web_scraping.yaml index df06494..0a96f91 100755 --- a/examples/websearch/web_scraping.yaml +++ b/examples/websearch/web_scraping.yaml @@ -24,13 +24,17 @@ flows: - component_name: stdin component_module: stdin_input - # Using Custom component + # Using Custom component - component_name: web_scraping_component component_module: web_scraper component_config: + input_transforms: + - type: copy + source_expression: previous:text + dest_expression: user_data.scraper:url input_selection: - source_expression: previous + source_expression: user_data.scraper - # Output to a standard out + # Output to a standard out - component_name: stdout - component_module: stdout_output \ No newline at end of file + component_module: stdout_output diff --git a/examples/websearch/websearch_router.yaml b/examples/websearch/websearch_router.yaml index aea0d16..b7941ab 100755 --- a/examples/websearch/websearch_router.yaml +++ b/examples/websearch/websearch_router.yaml @@ -190,7 +190,7 @@ flows: search_engine_id: ${GOOGLE_ENGINE_ID} detail: false input_selection: - source_expression: input.payload + source_expression: input.payload:text # Clean results by LLM - component_name: cleaner_llm @@ -358,4 +358,4 @@ flows: source_expression: template:{{text://input.topic}}/response dest_expression: user_data.output:topic input_selection: - source_expression: user_data.output \ No newline at end of file + source_expression: user_data.output diff --git a/src/solace_ai_connector/components/general/parser.py b/src/solace_ai_connector/components/general/parser.py index 466e4e9..b26d38c 100644 --- a/src/solace_ai_connector/components/general/parser.py +++ b/src/solace_ai_connector/components/general/parser.py @@ -27,6 +27,7 @@ class Parser(ComponentBase): + def __init__(self, **kwargs): super().__init__(info, **kwargs) @@ -57,7 +58,7 @@ def invoke(self, message, data): try: if input_format == "json" or input_format == "yaml": dict_data = self.str_to_dict(data, input_format) - else: + elif input_format != "dict": raise ValueError(f"Invalid input format: {input_format}") except Exception as e: raise ValueError(f"Error converting input: {str(e)}") from e diff --git a/src/solace_ai_connector/components/general/websearch/web_scraper.py b/src/solace_ai_connector/components/general/websearch/web_scraper.py index f237150..ff3e550 100755 --- a/src/solace_ai_connector/components/general/websearch/web_scraper.py +++ b/src/solace_ai_connector/components/general/websearch/web_scraper.py @@ -1,4 +1,5 @@ """Scrape a website""" + from ...component_base import ComponentBase from ....common.log import log @@ -6,23 +7,42 @@ "class_name": "WebScraper", "description": "Scrape javascript based websites.", "config_parameters": [ + { + "name": "timeout", + "required": False, + "description": "The timeout for the browser in milliseconds.", + "default": 30000, + } ], "input_schema": { "type": "object", - "properties": {} + "properties": { + "url": { + "type": "string", + "description": "The URL of the website to scrape.", + } + }, }, "output_schema": { "type": "object", - "properties": {} - } + "properties": { + "title": {"type": "string", "description": "The title of the website."}, + "content": {"type": "string", "description": "The content of the website."}, + }, + }, } + class WebScraper(ComponentBase): + def __init__(self, **kwargs): super().__init__(info, **kwargs) + self.timeout = self.get_config("timeout", 30000) def invoke(self, message, data): - url = data["text"] + url = data["url"] + if type(url) != str or not url: + raise ValueError("No URL provided") content = self.scrape(url) return content @@ -30,42 +50,50 @@ def invoke(self, message, data): def scrape(self, url): try: from playwright.sync_api import sync_playwright - except ImportError: - err_msg = "Please install playwright by running 'pip install playwright' and 'playwright install'." - log.error( - err_msg - ) - raise ValueError( - err_msg - ) - + except ImportError as e: + err_msg = "Please install playwright by running 'pip install playwright' and 'playwright install'." + log.error(err_msg) + raise ValueError(err_msg) from e + with sync_playwright() as p: try: # Launch a Chromium browser instance - browser = p.chromium.launch(headless=True) # Set headless=False to see the browser in action - except ImportError: + browser = p.chromium.launch( + headless=True, + timeout=self.timeout, + ) # Set headless=False to see the browser in action + except ImportError as e: err_msg = "Failed to launch the Chromium instance. Please install the browser binaries by running 'playwright install'" - log.error( - err_msg - ) - raise ValueError( - err_msg - ) - page = browser.new_page() - page.goto(url) - - # Wait for the page to fully load - page.wait_for_load_state("networkidle") + log.error(err_msg) + raise ValueError(err_msg) from e - # Scrape the text content of the page - title = page.title() - content = page.evaluate("document.body.innerText") - resp = { - "title": title, - "content": content - } - browser.close() + resp = {} + try: + context = browser.new_context( + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + ) + log.debug(f"Scraping the website: {url}") + page = context.new_page() + page.goto(url) - return resp + # Wait for the page to fully load + page.wait_for_load_state("load", timeout=self.timeout) + # Scroll to the bottom of the page to load more content + page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + # Scrape the text content of the page + title = page.title() + content = page.evaluate("document.body.innerText") + resp = {"title": title, "content": content} + log.debug(f"Scraped the website: {url}. \n Content is {content}") + browser.close() + return resp + except Exception as e: + log.error(f"Failed to scrape the website: {e}") + browser.close() + return { + "title": "", + "content": "", + "error": "Failed to scrape the website.", + } diff --git a/src/solace_ai_connector/components/general/websearch/websearch_base.py b/src/solace_ai_connector/components/general/websearch/websearch_base.py index 773ff94..d118002 100755 --- a/src/solace_ai_connector/components/general/websearch/websearch_base.py +++ b/src/solace_ai_connector/components/general/websearch/websearch_base.py @@ -1,4 +1,5 @@ """Base class for Web Search""" + from ...component_base import ComponentBase info_base = { @@ -9,35 +10,34 @@ "name": "engine", "required": True, "description": "The type of search engine.", - "default": "DuckDuckGo" + "default": "DuckDuckGo", }, { "name": "detail", "required": False, "description": "Return the detail.", - "default": False - } + "default": False, + }, ], "input_schema": { - "type": "object", - "properties": {}, + "type": "string", }, "output_schema": { "type": "object", "properties": {}, - } + }, } + class WebSearchBase(ComponentBase): + def __init__(self, info_base, **kwargs): super().__init__(info_base, **kwargs) self.detail = self.get_config("detail") def invoke(self, message, data): pass - + # Extract required data from a message def parse(self, message): pass - - diff --git a/src/solace_ai_connector/components/general/websearch/websearch_bing.py b/src/solace_ai_connector/components/general/websearch/websearch_bing.py index 2f6c1b5..fbbef92 100755 --- a/src/solace_ai_connector/components/general/websearch/websearch_bing.py +++ b/src/solace_ai_connector/components/general/websearch/websearch_bing.py @@ -18,47 +18,54 @@ { "name": "count", "required": False, - "description": "Number of search results to return.", - "default": 10 + "description": "Max number of search results to return.", + "default": 10, }, { "name": "safesearch", "required": False, "description": "Safe search setting: Off, Moderate, or Strict.", - "default": "Moderate" - } + "default": "Moderate", + }, ], "input_schema": { - "type": "object", - "properties": {}, + "type": "string", }, "output_schema": { - "type": "object", - "properties": {}, + "type": "array", + "items": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "snippet": {"type": "string"}, + "url": {"type": "string"}, + }, + }, }, } + class WebSearchBing(WebSearchBase): + def __init__(self, **kwargs): super().__init__(info, **kwargs) self.init() - + def init(self): self.api_key = self.get_config("api_key") - self.count = self.get_config("count") + self.count = self.get_config("count", 10) self.safesearch = self.get_config("safesearch") self.url = "https://api.bing.microsoft.com/v7.0/search" def invoke(self, message, data): - query = data["text"] + if type(data) != str or not data: + raise ValueError("Invalid search query") params = { - "q": query, # User query - "count": self.count, # Number of results to return - "safesearch": self.safesearch # Safe search filter - } - headers = { - "Ocp-Apim-Subscription-Key": self.api_key # Bing API Key + "q": data, # User query + "count": self.count, # Number of results to return + "safesearch": self.safesearch, # Safe search filter } + headers = {"Ocp-Apim-Subscription-Key": self.api_key} # Bing API Key response = requests.get(self.url, headers=headers, params=params) if response.status_code == 200: @@ -66,20 +73,22 @@ def invoke(self, message, data): response = self.parse(response) return response else: - return f"Error: {response.status_code}" - + raise ValueError(f"Error: {response.status_code}") + # Extract required data from a message def parse(self, message): if self.detail: return message else: data = [] - + # Process the search results to create a summary for web_page in message.get("webPages", {}).get("value", []): - data.append({ - "Title": web_page['name'], - "Snippet": web_page['snippet'], - "URL": web_page['url'] - }) + data.append( + { + "title": web_page["name"], + "snippet": web_page["snippet"], + "url": web_page["url"], + } + ) return data diff --git a/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py b/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py index 71a76b4..5519a78 100755 --- a/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py +++ b/src/solace_ai_connector/components/general/websearch/websearch_duckduckgo.py @@ -14,56 +14,70 @@ "name": "pretty", "required": False, "description": "Beautify the search output.", - "default": 1 + "default": 1, }, { "name": "no_html", "required": False, "description": "The number of output pages.", - "default": 1 + "default": 1, + }, + { + "name": "count", + "required": False, + "description": "Max Number of search results to return.", + "default": 10, }, { "name": "skip_disambig", "required": False, "description": "Skip disambiguation.", - "default": 1 + "default": 1, }, { "name": "detail", "required": False, "description": "Return the detail.", - "default": False - } + "default": False, + }, ], - "input_schema": { - "type": "object", - "properties": {}, - }, + "input_schema": {"type": "string"}, "output_schema": { - "type": "object", - "properties": {}, + "type": "array", + "items": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "snippet": {"type": "string"}, + "url": {"type": "string"}, + }, + }, }, } + class WebSearchDuckDuckGo(WebSearchBase): + def __init__(self, **kwargs): super().__init__(info, **kwargs) self.init() - + def init(self): self.pretty = self.get_config("pretty", 1) self.no_html = self.get_config("no_html", 1) + self.count = self.get_config("count", 10) self.skip_disambig = self.get_config("skip_disambig", 1) self.url = "http://api.duckduckgo.com/" def invoke(self, message, data): - query = data["text"] + if type(data) != str or not data: + raise ValueError("Invalid search query") params = { - "q": query, # User query - "format": "json", # Response format (json by default) - "pretty": self.pretty, # Beautify the output - "no_html": self.no_html, # Remove HTML from the response - "skip_disambig": self.skip_disambig # Skip disambiguation + "q": data, # User query + "format": "json", # Response format (json by default) + "pretty": self.pretty, # Beautify the output + "no_html": self.no_html, # Remove HTML from the response + "skip_disambig": self.skip_disambig, # Skip disambiguation } response = requests.get(self.url, params=params) @@ -72,15 +86,33 @@ def invoke(self, message, data): response = self.parse(response) return response else: - return f"Error: {response.status_code}" - + raise ValueError(f"Error: {response.status_code}") + # Extract required data from a message def parse(self, message): if self.detail: return message else: - return { - "Title": message['AbstractSource'], - "Snippet": message['Abstract'], - "URL": message['AbstractURL'] - } \ No newline at end of file + data = [] + if ( + message.get("AbstractSource") + and message.get("Abstract") + and message.get("AbstractURL") + ): + data.append( + { + "title": message["AbstractSource"], + "snippet": message["Abstract"], + "url": message["AbstractURL"], + } + ) + for message in message["RelatedTopics"]: + if "FirstURL" in message and "Text" in message and "Result" in message: + data.append( + { + "url": message["FirstURL"], + "title": message["Text"], + "snippet": message["Result"], + } + ) + return data[: self.count] diff --git a/src/solace_ai_connector/components/general/websearch/websearch_google.py b/src/solace_ai_connector/components/general/websearch/websearch_google.py index 66d6e14..5f7b997 100755 --- a/src/solace_ai_connector/components/general/websearch/websearch_google.py +++ b/src/solace_ai_connector/components/general/websearch/websearch_google.py @@ -19,41 +19,55 @@ "name": "search_engine_id", "required": False, "description": "The custom search engine id.", - "default": 1 + "default": 1, + }, + { + "name": "count", + "required": False, + "description": "Max Number of search results to return.", + "default": 10, }, { "name": "detail", "required": False, "description": "Return the detail.", - "default": False - } + "default": False, + }, ], - "input_schema": { - "type": "object", - "properties": {}, - }, + "input_schema": {"type": "string"}, "output_schema": { - "type": "object", - "properties": {}, + "type": "array", + "items": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "snippet": {"type": "string"}, + "url": {"type": "string"}, + }, + }, }, } + class WebSearchGoogle(WebSearchBase): + def __init__(self, **kwargs): super().__init__(info, **kwargs) self.init() - + def init(self): self.api_key = self.get_config("api_key") self.search_engine_id = self.get_config("search_engine_id") + self.count = self.get_config("count", 10) self.url = "https://www.googleapis.com/customsearch/v1" def invoke(self, message, data): - query = data["text"] + if type(data) != str or not data: + raise ValueError("Invalid search query") params = { - "q": query, # User query - "key": self.api_key, # Google API Key - "cx": self.search_engine_id, # Google custom search engine id + "q": data, # User query + "key": self.api_key, # Google API Key + "cx": self.search_engine_id, # Google custom search engine id } response = requests.get(self.url, params=params) @@ -62,7 +76,8 @@ def invoke(self, message, data): response = self.parse(response) return response else: - return f"Error: {response.status_code}" + error = response.json().get("error", {}).get("message", "Unknown error") + raise ValueError(f"Error: {response.status_code}: {error}") # Extract required data from a message def parse(self, message): @@ -70,12 +85,14 @@ def parse(self, message): return message else: data = [] - + # Process the search results to create a summary - for item in message.get('items', []): - data.append({ - "Title": item['title'], - "Snippet": item['snippet'], - "URL": item['link'] - }) - return data \ No newline at end of file + for item in message.get("items", []): + data.append( + { + "title": item["title"], + "snippet": item["snippet"], + "url": item["link"], + } + ) + return data[: self.count] diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py b/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py index 4c33ddb..88068fd 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py @@ -157,6 +157,7 @@ class BrokerRequestResponse(BrokerBase): + def __init__(self, **kwargs): super().__init__(info, **kwargs) self.need_acknowledgement = False @@ -256,6 +257,7 @@ def process_response(self, broker_message): topic = broker_message.get("topic") user_properties = broker_message.get("user_properties", {}) + streaming_complete_expression = None metadata_json = user_properties.get( "__solace_ai_connector_broker_request_reply_metadata__" ) @@ -346,6 +348,7 @@ def invoke(self, message, data): stream = False if "stream" in data: stream = data["stream"] + streaming_complete_expression = None if "streaming_complete_expression" in data: streaming_complete_expression = data["streaming_complete_expression"] From 7b0272a41715e033547f29a628072fe0b7bfc84c Mon Sep 17 00:00:00 2001 From: Ali Parvizi <91437594+alimosaed@users.noreply.github.com> Date: Fri, 15 Nov 2024 08:18:53 -0500 Subject: [PATCH 47/55] Alireza/ai 167/ai micro integration new (#63) * feat: add json log format * fix: add comments * fix: resolve comments * fix: change json to jsonl format --- src/solace_ai_connector/common/log.py | 51 ++++++++++++++++--- .../solace_ai_connector.py | 3 +- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/solace_ai_connector/common/log.py b/src/solace_ai_connector/common/log.py index 17c837b..ac15151 100644 --- a/src/solace_ai_connector/common/log.py +++ b/src/solace_ai_connector/common/log.py @@ -1,15 +1,51 @@ -# log.py - Logging utilities - import sys import logging import logging.handlers +import json log = logging.getLogger("solace_ai_connector") -# Function to setup the configuration for the logger -def setup_log(logFilePath, stdOutLogLevel, fileLogLevel): +class JsonFormatter(logging.Formatter): + """ + Custom formatter to output logs in JSON format. + """ + + def format(self, record): + log_record = { + "time": self.formatTime(record, self.datefmt), + "level": record.levelname, + "message": record.getMessage(), + } + return json.dumps(log_record) + + +class JsonlFormatter(logging.Formatter): + """ + Custom formatter to output logs in JSON Lines (JSONL) format. + """ + + def format(self, record): + log_record = { + "time": self.formatTime(record, self.datefmt), + "level": record.levelname, + "message": record.getMessage(), + } + return json.dumps(log_record) + + +def setup_log(logFilePath, stdOutLogLevel, fileLogLevel, logFormat): + """ + Set up the configuration for the logger. + + Parameters: + logFilePath (str): Path to the log file. + stdOutLogLevel (int): Logging level for standard output. + fileLogLevel (int): Logging level for the log file. + logFormat (str): Format of the log output ('jsonl' or 'pipe-delimited'). + + """ # Set the global logger level to the lowest of the two levels log.setLevel(min(stdOutLogLevel, fileLogLevel)) @@ -20,12 +56,15 @@ def setup_log(logFilePath, stdOutLogLevel, fileLogLevel): # Create an empty file at logFilePath (this will overwrite any existing content) with open(logFilePath, "w") as file: - file.write("") + file.write("") # file_handler = logging.handlers.TimedRotatingFileHandler( # filename=logFilePath, when='midnight', backupCount=30, mode='w') file_handler = logging.FileHandler(filename=logFilePath, mode="a") - file_formatter = logging.Formatter("%(asctime)s | %(levelname)s: %(message)s") + if logFormat == "jsonl": + file_formatter = JsonlFormatter() + else: + file_formatter = logging.Formatter("%(asctime)s | %(levelname)s: %(message)s") file_handler.setFormatter(file_formatter) file_handler.setLevel(fileLogLevel) diff --git a/src/solace_ai_connector/solace_ai_connector.py b/src/solace_ai_connector/solace_ai_connector.py index dd04442..ec2d5f0 100644 --- a/src/solace_ai_connector/solace_ai_connector.py +++ b/src/solace_ai_connector/solace_ai_connector.py @@ -121,7 +121,8 @@ def setup_logging(self): stdout_log_level = log_config.get("stdout_log_level", "INFO") log_file_level = log_config.get("log_file_level", "DEBUG") log_file = log_config.get("log_file", "solace_ai_connector.log") - setup_log(log_file, stdout_log_level, log_file_level) + log_format = log_config.get("log_format", "pipe-delimited") + setup_log(log_file, stdout_log_level, log_file_level, log_format) def setup_trace(self): """Setup trace""" From 41b67631725cefd97116af51aba40a358d655ab8 Mon Sep 17 00:00:00 2001 From: Art Morozov Date: Wed, 20 Nov 2024 15:00:14 -0500 Subject: [PATCH 48/55] Bump min-python to 3.10 --- .github/workflows/ci.yml | 8 ++++---- pyproject.toml | 13 ++++++++++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 439f70f..052f4c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,9 +14,9 @@ permissions: jobs: ci: - uses: SolaceDev/solace-public-workflows/.github/workflows/hatch_ci.yml@latest + uses: SolaceDev/solace-public-workflows/.github/workflows/hatch_ci.yml@test with: - min-python-version: "3.9" + min-python-version: "3.10" secrets: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} SONAR_HOST_URL: ${{ vars.SONAR_HOST_URL }} @@ -31,9 +31,9 @@ jobs: ssh-key: ${{ secrets.COMMIT_KEY }} - name: Set up Hatch - uses: SolaceDev/solace-public-workflows/.github/actions/hatch-setup@latest + uses: SolaceDev/solace-public-workflows/.github/actions/hatch-setup@test with: - min-python-version: "3.9" + min-python-version: "3.10" - name: Set Up Docker Buildx id: builder uses: docker/setup-buildx-action@v3 diff --git a/pyproject.toml b/pyproject.toml index 5fbd262..e24e428 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] description = "Solace AI Connector - make it easy to connect Solace PubSub+ Event Brokers to AI/ML frameworks" readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", @@ -24,6 +24,10 @@ dependencies = [ "PyYAML~=6.0.1", "Requests~=2.32.3", "solace_pubsubplus>=1.8.0", + "litellm~=1.51.3", + "Flask~=3.0.3", + "Flask-SocketIO~=5.4.1", + "build~=1.2.2.post1", ] [project.urls] @@ -36,6 +40,13 @@ documentation = "https://github.com/SolaceLabs/solace-ai-connector/blob/main/doc solace-ai-connector = "solace_ai_connector.main:main" solace-ai-connector-gen-docs = "solace_ai_connector.tools.gen_component_docs:main" +[tool.hatch.envs.hatch-test] +installer = "pip" + +# # Specify minimum and maximum Python versions to test +[[tool.hatch.envs.hatch-test.matrix]] +python = ["3.10", "3.12"] + [tool.hatch.build.targets.wheel] packages = ["src/solace_ai_connector"] From 531866b9a021df14d9d67860c16fc27414c3961f Mon Sep 17 00:00:00 2001 From: Art Morozov Date: Wed, 20 Nov 2024 15:07:21 -0500 Subject: [PATCH 49/55] Use updated workflows --- .github/workflows/ci.yml | 4 ++-- .github/workflows/release.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 052f4c2..6acdaf6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ permissions: jobs: ci: - uses: SolaceDev/solace-public-workflows/.github/workflows/hatch_ci.yml@test + uses: SolaceDev/solace-public-workflows/.github/workflows/hatch_ci.yml@main with: min-python-version: "3.10" secrets: @@ -31,7 +31,7 @@ jobs: ssh-key: ${{ secrets.COMMIT_KEY }} - name: Set up Hatch - uses: SolaceDev/solace-public-workflows/.github/actions/hatch-setup@test + uses: SolaceDev/solace-public-workflows/.github/actions/hatch-setup@main with: min-python-version: "3.10" - name: Set Up Docker Buildx diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0026e90..de21a17 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,7 +18,7 @@ permissions: jobs: release: - uses: SolaceDev/solace-public-workflows/.github/workflows/hatch_release_pypi.yml@latest + uses: SolaceDev/solace-public-workflows/.github/workflows/hatch_release_pypi.yml@main with: ENVIRONMENT: pypi version: ${{ github.event.inputs.version }} From f4677f9d54f4d7824ed460cad10f1f494fbfd472 Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Mon, 25 Nov 2024 13:24:54 -0500 Subject: [PATCH 50/55] FEATURE: Enable stream overwrite for LLM Chat at the event level (#66) * Allowing stream overwrite at event level for LLM Chat * Added overwrite flag --- examples/llm/anthropic_chat.yaml | 31 +++++++- .../llm/langchain/langchain_chat_model.py | 48 +++++++++++- .../langchain/langchain_chat_model_base.py | 76 ++++++++++++++++++- .../langchain_chat_model_with_history.py | 51 ++----------- .../general/llm/litellm/litellm_base.py | 38 ---------- .../llm/litellm/litellm_chat_model_base.py | 63 ++++++++++++++- .../litellm_chat_model_with_history.py | 9 ++- .../llm/openai/openai_chat_model_base.py | 26 ++++++- .../openai/openai_chat_model_with_history.py | 10 ++- 9 files changed, 252 insertions(+), 100 deletions(-) diff --git a/examples/llm/anthropic_chat.yaml b/examples/llm/anthropic_chat.yaml index 76fe7ef..60cf396 100644 --- a/examples/llm/anthropic_chat.yaml +++ b/examples/llm/anthropic_chat.yaml @@ -5,7 +5,8 @@ # # The input message has the following schema: # { -# "text": "" +# "query": "", +# "stream": false # } # # It will then send an event back to Solace with the topic: `demo/question/response` @@ -66,17 +67,23 @@ flows: base_url: ${ANTHROPIC_API_ENDPOINT} model: ${MODEL_NAME} temperature: 0.01 + llm_mode: stream + allow_overwrite_llm_mode: true + stream_to_flow: stream_output input_transforms: - type: copy source_expression: | template:You are a helpful AI assistant. Please help with the user's request below: - {{text://input.payload:text}} + {{text://input.payload:query}} dest_expression: user_data.llm_input:messages.0.content - type: copy source_expression: static:user dest_expression: user_data.llm_input:messages.0.role + - type: copy + source_expression: input.payload:stream + dest_expression: user_data.llm_input:stream input_selection: source_expression: user_data.llm_input @@ -97,3 +104,23 @@ flows: dest_expression: user_data.output:topic input_selection: source_expression: user_data.output + + - name: stream_output + components: + # Send response back to broker + - component_name: send_response + component_module: broker_output + component_config: + <<: *broker_connection + payload_encoding: utf-8 + payload_format: json + copy_user_properties: true + input_transforms: + - type: copy + source_expression: input.payload + dest_expression: user_data.output:payload + - type: copy + source_value: demo/question/stream + dest_expression: user_data.output:topic + input_selection: + source_expression: user_data.output diff --git a/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model.py index 85cd194..510323e 100644 --- a/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model.py +++ b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model.py @@ -1,6 +1,8 @@ # This is a wrapper around all the LangChain chat models # The configuration will control dynamic loading of the chat models +from uuid import uuid4 from copy import deepcopy +from collections import namedtuple from .langchain_chat_model_base import ( LangChainChatModelBase, info_base, @@ -17,6 +19,48 @@ def __init__(self, **kwargs): super().__init__(info, **kwargs) def invoke_model( - self, input_message, messages, session_id=None, clear_history=False + self, + input_message, + messages, + session_id=None, + clear_history=False, + stream=False, ): - return self.component.invoke(messages) + if not stream: + return self.component.invoke(messages) + + aggregate_result = "" + current_batch = "" + response_uuid = str(uuid4()) + first_chunk = True + + for chunk in self.component.stream(messages): + aggregate_result += chunk.content + current_batch += chunk.content + if len(current_batch) >= self.stream_batch_size: + if self.stream_to_flow: + self.send_streaming_message( + input_message, + current_batch, + aggregate_result, + response_uuid, + first_chunk, + ) + current_batch = "" + first_chunk = False + + if self.stream_to_flow: + self.send_streaming_message( + input_message, + current_batch, + aggregate_result, + response_uuid, + first_chunk, + True, + ) + + result = namedtuple("Result", ["content", "response_uuid"])( + aggregate_result, response_uuid + ) + + return result diff --git a/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_base.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_base.py index 58c7ae5..089ec08 100644 --- a/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_base.py +++ b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_base.py @@ -5,6 +5,7 @@ from abc import abstractmethod from langchain_core.output_parsers import JsonOutputParser +from .....common.message import Message from .....common.utils import get_obj_text from langchain.schema.messages import ( HumanMessage, @@ -39,6 +40,28 @@ "description": "Model specific configuration for the chat model. " "See documentation for valid parameter names.", }, + { + "name": "llm_mode", + "required": False, + "description": "The mode for streaming results: 'none' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response.", + }, + { + "name": "allow_overwrite_llm_mode", + "required": False, + "description": "Whether to allow the llm_mode to be overwritten by the `stream` from the input message.", + }, + { + "name": "stream_to_flow", + "required": False, + "description": "Name the flow to stream the output to - this must be configured for llm_mode='stream'.", + "default": "", + }, + { + "name": "stream_batch_size", + "required": False, + "description": "The minimum number of words in a single streaming result. Default: 15.", + "default": 15, + }, { "name": "llm_response_format", "required": False, @@ -88,10 +111,18 @@ class LangChainChatModelBase(LangChainBase): + + def __init__(self, info, **kwargs): + super().__init__(info, **kwargs) + self.llm_mode = self.get_config("llm_mode", "none") + self.allow_overwrite_llm_mode = self.get_config("allow_overwrite_llm_mode") + self.stream_to_flow = self.get_config("stream_to_flow", "") + self.stream_batch_size = self.get_config("stream_batch_size", 15) + def invoke(self, message, data): messages = [] - for item in data["messages"]: + for item in data.get("messages"): if item["role"] == "system": messages.append(SystemMessage(content=item["content"])) elif item["role"] == "user" or item["role"] == "human": @@ -109,9 +140,22 @@ def invoke(self, message, data): session_id = data.get("session_id", None) clear_history = data.get("clear_history", False) + stream = data.get("stream") + + should_stream = self.llm_mode == "stream" + if ( + self.allow_overwrite_llm_mode + and stream is not None + and isinstance(stream, bool) + ): + should_stream = stream llm_res = self.invoke_model( - message, messages, session_id=session_id, clear_history=clear_history + message, + messages, + session_id=session_id, + clear_history=clear_history, + stream=should_stream, ) res_format = self.get_config("llm_response_format", "text") @@ -134,6 +178,32 @@ def invoke(self, message, data): @abstractmethod def invoke_model( - self, input_message, messages, session_id=None, clear_history=False + self, + input_message, + messages, + session_id=None, + clear_history=False, + stream=False, ): pass + + def send_streaming_message( + self, + input_message, + chunk, + aggregate_result, + response_uuid, + first_chunk=False, + last_chunk=False, + ): + message = Message( + payload={ + "chunk": chunk, + "content": aggregate_result, + "response_uuid": response_uuid, + "first_chunk": first_chunk, + "last_chunk": last_chunk, + }, + user_properties=input_message.get_user_properties(), + ) + self.send_to_flow(self.stream_to_flow, message) diff --git a/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_with_history.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_with_history.py index 4569c30..7e708dd 100644 --- a/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_with_history.py +++ b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_with_history.py @@ -16,7 +16,6 @@ SystemMessage, ) -from .....common.message import Message from .langchain_chat_model_base import ( LangChainChatModelBase, info_base, @@ -78,23 +77,6 @@ "description": "The configuration for the history class.", "type": "object", }, - { - "name": "stream_to_flow", - "required": False, - "description": "Name the flow to stream the output to - this must be configured for llm_mode='stream'.", - "default": "", - }, - { - "name": "llm_mode", - "required": False, - "description": "The mode for streaming results: 'sync' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response.", - }, - { - "name": "stream_batch_size", - "required": False, - "description": "The minimum number of words in a single streaming result. Default: 15.", - "default": 15, - }, { "name": "set_response_uuid_in_user_properties", "required": False, @@ -128,15 +110,17 @@ def __init__(self, **kwargs): ) self.history_max_tokens = self.get_config("history_max_tokens", 8000) self.history_max_time = self.get_config("history_max_time", None) - self.stream_to_flow = self.get_config("stream_to_flow", "") - self.llm_mode = self.get_config("llm_mode", "none") - self.stream_batch_size = self.get_config("stream_batch_size", 15) self.set_response_uuid_in_user_properties = self.get_config( "set_response_uuid_in_user_properties", False ) def invoke_model( - self, input_message, messages, session_id=None, clear_history=False + self, + input_message, + messages, + session_id=None, + clear_history=False, + stream=False, ): if clear_history: @@ -171,7 +155,7 @@ def invoke_model( history_messages_key="chat_history", ) - if self.llm_mode == "none": + if not stream: return runnable.invoke( {"input": human_message}, config={ @@ -221,27 +205,6 @@ def invoke_model( return result - def send_streaming_message( - self, - input_message, - chunk, - aggregate_result, - response_uuid, - first_chunk=False, - last_chunk=False, - ): - message = Message( - payload={ - "chunk": chunk, - "content": aggregate_result, - "response_uuid": response_uuid, - "first_chunk": first_chunk, - "last_chunk": last_chunk, - }, - user_properties=input_message.get_user_properties(), - ) - self.send_to_flow(self.stream_to_flow, message) - def create_history(self): history_class = self.load_component( diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py index f9d4282..4f8d5aa 100644 --- a/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py @@ -32,40 +32,6 @@ "description": "Sampling temperature to use", "default": 0.7, }, - { - "name": "stream_to_flow", - "required": False, - "description": ( - "Name the flow to stream the output to - this must be configured for " - "llm_mode='stream'. This is mutually exclusive with stream_to_next_component." - ), - "default": "", - }, - { - "name": "stream_to_next_component", - "required": False, - "description": ( - "Whether to stream the output to the next component in the flow. " - "This is mutually exclusive with stream_to_flow." - ), - "default": False, - }, - { - "name": "llm_mode", - "required": False, - "description": ( - "The mode for streaming results: 'sync' or 'stream'. 'stream' " - "will just stream the results to the named flow. 'none' will " - "wait for the full response." - ), - "default": "none", - }, - { - "name": "stream_batch_size", - "required": False, - "description": "The minimum number of words in a single streaming result. Default: 15.", - "default": 15, - }, { "name": "set_response_uuid_in_user_properties", "required": False, @@ -91,10 +57,6 @@ def __init__(self, module_info, **kwargs): def init(self): litellm.suppress_debug_info = True self.load_balancer = self.get_config("load_balancer") - self.stream_to_flow = self.get_config("stream_to_flow") - self.stream_to_next_component = self.get_config("stream_to_next_component") - self.llm_mode = self.get_config("llm_mode") - self.stream_batch_size = self.get_config("stream_batch_size") self.set_response_uuid_in_user_properties = self.get_config( "set_response_uuid_in_user_properties" ) diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py index a1ecff5..6336a4b 100644 --- a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py @@ -28,6 +28,10 @@ "required": ["role", "content"], }, }, + "stream": { + "type": "boolean", + "description": "Whether to stream the response - overwrites llm_mode", + }, }, "required": ["messages"], }, @@ -63,18 +67,75 @@ }, }, ) +litellm_chat_info_base["config_parameters"].extend( + [ + { + "name": "stream_to_flow", + "required": False, + "description": ( + "Name the flow to stream the output to - this must be configured for " + "llm_mode='stream'. This is mutually exclusive with stream_to_next_component." + ), + "default": "", + }, + { + "name": "stream_to_next_component", + "required": False, + "description": ( + "Whether to stream the output to the next component in the flow. " + "This is mutually exclusive with stream_to_flow." + ), + "default": False, + }, + { + "name": "llm_mode", + "required": False, + "description": ( + "The mode for streaming results: 'none' or 'stream'. 'stream' " + "will just stream the results to the named flow. 'none' will " + "wait for the full response." + ), + "default": "none", + }, + { + "name": "allow_overwrite_llm_mode", + "required": False, + "description": "Whether to allow the llm_mode to be overwritten by the `stream` from the input message.", + }, + { + "name": "stream_batch_size", + "required": False, + "description": "The minimum number of words in a single streaming result. Default: 15.", + "default": 15, + }, + ] +) class LiteLLMChatModelBase(LiteLLMBase): def __init__(self, info, **kwargs): super().__init__(info, **kwargs) + self.stream_to_flow = self.get_config("stream_to_flow") + self.stream_to_next_component = self.get_config("stream_to_next_component") + self.llm_mode = self.get_config("llm_mode") + self.allow_overwrite_llm_mode = self.get_config("allow_overwrite_llm_mode") + self.stream_batch_size = self.get_config("stream_batch_size") def invoke(self, message, data): """invoke the model""" messages = data.get("messages", []) + stream = data.get("stream") + + should_stream = self.llm_mode == "stream" + if ( + self.allow_overwrite_llm_mode + and stream is not None + and isinstance(stream, bool) + ): + should_stream = stream - if self.llm_mode == "stream": + if should_stream: return self.invoke_stream(message, messages) else: return self.invoke_non_stream(messages) diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_with_history.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_with_history.py index c98e353..4ee0b02 100644 --- a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_with_history.py +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_with_history.py @@ -32,7 +32,9 @@ "description": "Clear history but keep the last N messages. If 0, clear all history. If not set, do not clear history.", } + class LiteLLMChatModelWithHistory(LiteLLMChatModelBase, ChatHistoryHandler): + def __init__(self, **kwargs): super().__init__(info, **kwargs) self.history_max_turns = self.get_config("history_max_turns", 10) @@ -45,7 +47,7 @@ def __init__(self, **kwargs): def invoke(self, message, data): session_id = data.get("session_id") if not session_id: - raise ValueError("session_id is not provided") + raise ValueError("session_id is not provided") clear_history_but_keep_depth = data.get("clear_history_but_keep_depth") try: @@ -55,6 +57,7 @@ def invoke(self, message, data): log.error("Invalid clear_history_but_keep_depth value. Defaulting to 0.") clear_history_but_keep_depth = 0 messages = data.get("messages", []) + stream = data.get("stream") with self.get_lock(self.history_key): history = self.kv_store_get(self.history_key) or {} @@ -88,7 +91,7 @@ def invoke(self, message, data): self.prune_history(session_id, history) response = super().invoke( - message, {"messages": history[session_id]["messages"]} + message, {"messages": history[session_id]["messages"], "stream": stream} ) # Add the assistant's response to the history @@ -102,4 +105,4 @@ def invoke(self, message, data): self.kv_store_set(self.history_key, history) log.debug(f"Updated history: {history}") - return response \ No newline at end of file + return response diff --git a/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_base.py b/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_base.py index beabd07..3ac9bcd 100755 --- a/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_base.py +++ b/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_base.py @@ -56,12 +56,17 @@ "name": "llm_mode", "required": False, "description": ( - "The mode for streaming results: 'sync' or 'stream'. 'stream' " + "The mode for streaming results: 'none' or 'stream'. 'stream' " "will just stream the results to the named flow. 'none' will " "wait for the full response." ), "default": "none", }, + { + "name": "allow_overwrite_llm_mode", + "required": False, + "description": "Whether to allow the llm_mode to be overwritten by the `stream` from the input message.", + }, { "name": "stream_batch_size", "required": False, @@ -97,6 +102,10 @@ "required": ["role", "content"], }, }, + "stream": { + "type": "boolean", + "description": "Whether to stream the response - overwrites llm_mode", + }, }, "required": ["messages"], }, @@ -134,6 +143,7 @@ class OpenAIChatModelBase(ComponentBase): + def __init__(self, module_info, **kwargs): super().__init__(module_info, **kwargs) self.init() @@ -144,6 +154,7 @@ def init(self): self.stream_to_flow = self.get_config("stream_to_flow") self.stream_to_next_component = self.get_config("stream_to_next_component") self.llm_mode = self.get_config("llm_mode") + self.allow_overwrite_llm_mode = self.get_config("allow_overwrite_llm_mode") self.stream_batch_size = self.get_config("stream_batch_size") self.response_format = self.get_config("response_format", "text") self.set_response_uuid_in_user_properties = self.get_config( @@ -156,12 +167,21 @@ def init(self): def invoke(self, message, data): messages = data.get("messages", []) + stream = data.get("stream") client = OpenAI( api_key=self.get_config("api_key"), base_url=self.get_config("base_url") ) - if self.llm_mode == "stream": + should_stream = self.llm_mode == "stream" + if ( + self.allow_overwrite_llm_mode + and stream is not None + and isinstance(stream, bool) + ): + should_stream = stream + + if should_stream: return self.invoke_stream(client, message, messages) else: max_retries = 3 @@ -171,7 +191,7 @@ def invoke(self, message, data): messages=messages, model=self.model, temperature=self.temperature, - response_format={"type": self.response_format} + response_format={"type": self.response_format}, ) return {"content": response.choices[0].message.content} except Exception as e: diff --git a/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_with_history.py b/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_with_history.py index e9f0da8..fb164c7 100644 --- a/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_with_history.py +++ b/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_with_history.py @@ -35,6 +35,7 @@ class OpenAIChatModelWithHistory(OpenAIChatModelBase, ChatHistoryHandler): + def __init__(self, **kwargs): super().__init__(info, **kwargs) self.history_max_turns = self.get_config("history_max_turns", 10) @@ -47,8 +48,8 @@ def __init__(self, **kwargs): def invoke(self, message, data): session_id = data.get("session_id") if not session_id: - raise ValueError("session_id is not provided") - + raise ValueError("session_id is not provided") + clear_history_but_keep_depth = data.get("clear_history_but_keep_depth") try: if clear_history_but_keep_depth is not None: @@ -56,6 +57,7 @@ def invoke(self, message, data): except (TypeError, ValueError): clear_history_but_keep_depth = 0 messages = data.get("messages", []) + stream = data.get("stream") with self.get_lock(self.history_key): history = self.kv_store_get(self.history_key) or {} @@ -89,7 +91,7 @@ def invoke(self, message, data): self.prune_history(session_id, history) response = super().invoke( - message, {"messages": history[session_id]["messages"]} + message, {"messages": history[session_id]["messages"], "stream": stream} ) # Add the assistant's response to the history @@ -102,4 +104,4 @@ def invoke(self, message, data): self.kv_store_set(self.history_key, history) - return response \ No newline at end of file + return response From 6e1e3fce6f9543e883b38b21205b555b09f9f956 Mon Sep 17 00:00:00 2001 From: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Date: Tue, 26 Nov 2024 14:47:12 -0500 Subject: [PATCH 51/55] AI-95: Enhance request/response handling for streaming LLM access (#69) * Changes for request/response for streaming LLM access * Updated with main * update --------- Co-authored-by: Edward Funnekotter --- docs/components/broker_request_response.md | 4 ++-- .../langchain_chat_model_with_history.md | 6 ----- docs/components/litellm_chat_model.md | 8 +++++++ docs/components/litellm_embeddings.md | 8 +++++++ examples/llm/anthropic_chat.yaml | 1 - .../components/component_base.py | 10 ++++++++ .../langchain/langchain_chat_model_base.py | 18 ++------------- .../general/llm/litellm/litellm_base.py | 5 ---- .../llm/litellm/litellm_chat_model_base.py | 23 ++++++------------- .../llm/openai/openai_chat_model_base.py | 23 ++++--------------- .../inputs_outputs/broker_request_response.py | 8 +++---- .../flow/request_response_flow_controller.py | 7 ++++-- 12 files changed, 50 insertions(+), 71 deletions(-) mode change 100755 => 100644 src/solace_ai_connector/components/general/llm/openai/openai_chat_model_base.py diff --git a/docs/components/broker_request_response.md b/docs/components/broker_request_response.md index 333b050..30ee616 100644 --- a/docs/components/broker_request_response.md +++ b/docs/components/broker_request_response.md @@ -17,7 +17,7 @@ component_config: payload_format: response_topic_prefix: response_topic_suffix: - reply_queue_prefix: + response_queue_prefix: request_expiry_ms: streaming: streaming_complete_expression: @@ -34,7 +34,7 @@ component_config: | payload_format | False | json | Format for the payload (json, yaml, text) | | response_topic_prefix | False | reply | Prefix for reply topics | | response_topic_suffix | False | | Suffix for reply topics | -| reply_queue_prefix | False | reply-queue | Prefix for reply queues | +| response_queue_prefix | False | reply-queue | Prefix for reply queues | | request_expiry_ms | False | 60000 | Expiry time for cached requests in milliseconds | | streaming | False | | The response will arrive in multiple pieces. If True, the streaming_complete_expression must be set and will be used to determine when the last piece has arrived. | | streaming_complete_expression | False | | The source expression to determine when the last piece of a streaming response has arrived. | diff --git a/docs/components/langchain_chat_model_with_history.md b/docs/components/langchain_chat_model_with_history.md index e66e9a6..8686061 100644 --- a/docs/components/langchain_chat_model_with_history.md +++ b/docs/components/langchain_chat_model_with_history.md @@ -19,9 +19,6 @@ component_config: history_module: history_class: history_config: - stream_to_flow: - llm_mode: - stream_batch_size: set_response_uuid_in_user_properties: ``` @@ -38,9 +35,6 @@ component_config: | history_module | False | langchain_community.chat_message_histories | The module that contains the history class. Default: 'langchain_community.chat_message_histories' | | history_class | False | ChatMessageHistory | The class to use for the history. Default: 'ChatMessageHistory' | | history_config | False | | The configuration for the history class. | -| stream_to_flow | False | | Name the flow to stream the output to - this must be configured for llm_mode='stream'. | -| llm_mode | False | | The mode for streaming results: 'sync' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response. | -| stream_batch_size | False | 15 | The minimum number of words in a single streaming result. Default: 15. | | set_response_uuid_in_user_properties | False | False | Whether to set the response_uuid in the user_properties of the input_message. This will allow other components to correlate streaming chunks with the full response. | diff --git a/docs/components/litellm_chat_model.md b/docs/components/litellm_chat_model.md index 556ee08..acd1985 100644 --- a/docs/components/litellm_chat_model.md +++ b/docs/components/litellm_chat_model.md @@ -20,6 +20,10 @@ component_config: history_max_time: history_max_turns: history_max_time: + stream_to_flow: + stream_to_next_component: + llm_mode: + stream_batch_size: ``` | Parameter | Required | Default | Description | @@ -36,6 +40,10 @@ component_config: | history_max_time | False | 3600 | Maximum time to keep conversation history (in seconds) | | history_max_turns | False | 10 | Maximum number of conversation turns to keep in history | | history_max_time | False | 3600 | Maximum time to keep conversation history (in seconds) | +| stream_to_flow | False | | Name the flow to stream the output to - this must be configured for llm_mode='stream'. This is mutually exclusive with stream_to_next_component. | +| stream_to_next_component | False | False | Whether to stream the output to the next component in the flow. This is mutually exclusive with stream_to_flow. | +| llm_mode | False | none | The mode for streaming results: 'none' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response. | +| stream_batch_size | False | 15 | The minimum number of words in a single streaming result. Default: 15. | ## Component Input Schema diff --git a/docs/components/litellm_embeddings.md b/docs/components/litellm_embeddings.md index 6542c29..8393008 100644 --- a/docs/components/litellm_embeddings.md +++ b/docs/components/litellm_embeddings.md @@ -20,6 +20,10 @@ component_config: history_max_time: history_max_turns: history_max_time: + stream_to_flow: + stream_to_next_component: + llm_mode: + stream_batch_size: ``` | Parameter | Required | Default | Description | @@ -36,6 +40,10 @@ component_config: | history_max_time | False | 3600 | Maximum time to keep conversation history (in seconds) | | history_max_turns | False | 10 | Maximum number of conversation turns to keep in history | | history_max_time | False | 3600 | Maximum time to keep conversation history (in seconds) | +| stream_to_flow | False | | Name the flow to stream the output to - this must be configured for llm_mode='stream'. This is mutually exclusive with stream_to_next_component. | +| stream_to_next_component | False | False | Whether to stream the output to the next component in the flow. This is mutually exclusive with stream_to_flow. | +| llm_mode | False | none | The mode for streaming results: 'none' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response. | +| stream_batch_size | False | 15 | The minimum number of words in a single streaming result. Default: 15. | ## Component Input Schema diff --git a/examples/llm/anthropic_chat.yaml b/examples/llm/anthropic_chat.yaml index 60cf396..cc6e8fd 100644 --- a/examples/llm/anthropic_chat.yaml +++ b/examples/llm/anthropic_chat.yaml @@ -68,7 +68,6 @@ flows: model: ${MODEL_NAME} temperature: 0.01 llm_mode: stream - allow_overwrite_llm_mode: true stream_to_flow: stream_output input_transforms: - type: copy diff --git a/src/solace_ai_connector/components/component_base.py b/src/solace_ai_connector/components/component_base.py index 0d04dae..63dd8fb 100644 --- a/src/solace_ai_connector/components/component_base.py +++ b/src/solace_ai_connector/components/component_base.py @@ -303,6 +303,16 @@ def setup_broker_request_response(self): "broker_config": broker_config, "request_expiry_ms": request_expiry_ms, } + + if "response_topic_prefix" in self.broker_request_response_config: + rrc_config["response_topic_prefix"] = self.broker_request_response_config[ + "response_topic_prefix" + ] + if "response_queue_prefix" in self.broker_request_response_config: + rrc_config["response_queue_prefix"] = self.broker_request_response_config[ + "response_queue_prefix" + ] + self.broker_request_response_controller = RequestResponseFlowController( config=rrc_config, connector=self.connector ) diff --git a/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_base.py b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_base.py index 089ec08..86cc5c2 100644 --- a/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_base.py +++ b/src/solace_ai_connector/components/general/llm/langchain/langchain_chat_model_base.py @@ -45,11 +45,6 @@ "required": False, "description": "The mode for streaming results: 'none' or 'stream'. 'stream' will just stream the results to the named flow. 'none' will wait for the full response.", }, - { - "name": "allow_overwrite_llm_mode", - "required": False, - "description": "Whether to allow the llm_mode to be overwritten by the `stream` from the input message.", - }, { "name": "stream_to_flow", "required": False, @@ -115,7 +110,6 @@ class LangChainChatModelBase(LangChainBase): def __init__(self, info, **kwargs): super().__init__(info, **kwargs) self.llm_mode = self.get_config("llm_mode", "none") - self.allow_overwrite_llm_mode = self.get_config("allow_overwrite_llm_mode") self.stream_to_flow = self.get_config("stream_to_flow", "") self.stream_batch_size = self.get_config("stream_batch_size", 15) @@ -140,22 +134,14 @@ def invoke(self, message, data): session_id = data.get("session_id", None) clear_history = data.get("clear_history", False) - stream = data.get("stream") - - should_stream = self.llm_mode == "stream" - if ( - self.allow_overwrite_llm_mode - and stream is not None - and isinstance(stream, bool) - ): - should_stream = stream + stream = data.get("stream", self.llm_mode == "stream") llm_res = self.invoke_model( message, messages, session_id=session_id, clear_history=clear_history, - stream=should_stream, + stream=stream, ) res_format = self.get_config("llm_response_format", "text") diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py index 4f8d5aa..22bb2e0 100644 --- a/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_base.py @@ -3,7 +3,6 @@ import litellm from ....component_base import ComponentBase -from .....common.message import Message from .....common.log import log litellm_info_base = { @@ -60,10 +59,6 @@ def init(self): self.set_response_uuid_in_user_properties = self.get_config( "set_response_uuid_in_user_properties" ) - if self.stream_to_flow and self.stream_to_next_component: - raise ValueError( - "stream_to_flow and stream_to_next_component are mutually exclusive" - ) self.router = None def init_load_balancer(self): diff --git a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py index 6336a4b..24358bd 100644 --- a/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py +++ b/src/solace_ai_connector/components/general/llm/litellm/litellm_chat_model_base.py @@ -97,11 +97,6 @@ ), "default": "none", }, - { - "name": "allow_overwrite_llm_mode", - "required": False, - "description": "Whether to allow the llm_mode to be overwritten by the `stream` from the input message.", - }, { "name": "stream_batch_size", "required": False, @@ -119,23 +114,19 @@ def __init__(self, info, **kwargs): self.stream_to_flow = self.get_config("stream_to_flow") self.stream_to_next_component = self.get_config("stream_to_next_component") self.llm_mode = self.get_config("llm_mode") - self.allow_overwrite_llm_mode = self.get_config("allow_overwrite_llm_mode") self.stream_batch_size = self.get_config("stream_batch_size") + if self.stream_to_flow and self.stream_to_next_component: + raise ValueError( + "stream_to_flow and stream_to_next_component are mutually exclusive" + ) + def invoke(self, message, data): """invoke the model""" messages = data.get("messages", []) - stream = data.get("stream") - - should_stream = self.llm_mode == "stream" - if ( - self.allow_overwrite_llm_mode - and stream is not None - and isinstance(stream, bool) - ): - should_stream = stream + stream = data.get("stream", self.llm_mode == "stream") - if should_stream: + if stream: return self.invoke_stream(message, messages) else: return self.invoke_non_stream(messages) diff --git a/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_base.py b/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_base.py old mode 100755 new mode 100644 index 3ac9bcd..012d1df --- a/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_base.py +++ b/src/solace_ai_connector/components/general/llm/openai/openai_chat_model_base.py @@ -62,11 +62,6 @@ ), "default": "none", }, - { - "name": "allow_overwrite_llm_mode", - "required": False, - "description": "Whether to allow the llm_mode to be overwritten by the `stream` from the input message.", - }, { "name": "stream_batch_size", "required": False, @@ -104,7 +99,8 @@ }, "stream": { "type": "boolean", - "description": "Whether to stream the response - overwrites llm_mode", + "description": "Whether to stream the response. It is is not provided, it will default to the value of llm_mode.", + "required": False, }, }, "required": ["messages"], @@ -154,9 +150,7 @@ def init(self): self.stream_to_flow = self.get_config("stream_to_flow") self.stream_to_next_component = self.get_config("stream_to_next_component") self.llm_mode = self.get_config("llm_mode") - self.allow_overwrite_llm_mode = self.get_config("allow_overwrite_llm_mode") self.stream_batch_size = self.get_config("stream_batch_size") - self.response_format = self.get_config("response_format", "text") self.set_response_uuid_in_user_properties = self.get_config( "set_response_uuid_in_user_properties" ) @@ -167,21 +161,13 @@ def init(self): def invoke(self, message, data): messages = data.get("messages", []) - stream = data.get("stream") + stream = data.get("stream", self.llm_mode == "stream") client = OpenAI( api_key=self.get_config("api_key"), base_url=self.get_config("base_url") ) - should_stream = self.llm_mode == "stream" - if ( - self.allow_overwrite_llm_mode - and stream is not None - and isinstance(stream, bool) - ): - should_stream = stream - - if should_stream: + if stream: return self.invoke_stream(client, message, messages) else: max_retries = 3 @@ -191,7 +177,6 @@ def invoke(self, message, data): messages=messages, model=self.model, temperature=self.temperature, - response_format={"type": self.response_format}, ) return {"content": response.choices[0].message.content} except Exception as e: diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py b/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py index 88068fd..bdaea62 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py @@ -73,7 +73,7 @@ "default": "", }, { - "name": "reply_queue_prefix", + "name": "response_queue_prefix", "required": False, "description": "Prefix for reply queues", "default": "reply-queue", @@ -168,11 +168,11 @@ def __init__(self, **kwargs): self.response_topic_suffix = ensure_slash_on_start( self.get_config("response_topic_suffix") ) - self.reply_queue_prefix = ensure_slash_on_end( - self.get_config("reply_queue_prefix") + self.response_queue_prefix = ensure_slash_on_end( + self.get_config("response_queue_prefix") ) self.requestor_id = str(uuid.uuid4()) - self.reply_queue_name = f"{self.reply_queue_prefix}{self.requestor_id}" + self.reply_queue_name = f"{self.response_queue_prefix}{self.requestor_id}" self.response_topic = f"{self.response_topic_prefix}{self.requestor_id}{self.response_topic_suffix}" self.response_thread = None self.streaming = self.get_config("streaming") diff --git a/src/solace_ai_connector/flow/request_response_flow_controller.py b/src/solace_ai_connector/flow/request_response_flow_controller.py index 36dda71..37a4fe9 100644 --- a/src/solace_ai_connector/flow/request_response_flow_controller.py +++ b/src/solace_ai_connector/flow/request_response_flow_controller.py @@ -30,6 +30,7 @@ # This is a very basic component which will be stitched onto the final component in the flow class RequestResponseControllerOuputComponent: + def __init__(self, controller): self.controller = controller @@ -39,6 +40,7 @@ def enqueue(self, event): # This is the main class that will be used to send messages to a flow and receive the response class RequestResponseFlowController: + def __init__(self, config: Dict[str, Any], connector): self.config = config self.connector = connector @@ -55,14 +57,15 @@ def __init__(self, config: Dict[str, Any], connector): self.flow.run() def create_broker_request_response_flow(self): - self.broker_config["request_expiry_ms"] = self.request_expiry_ms + full_config = self.broker_config.copy() + full_config.update(self.config) config = { "name": "_internal_broker_request_response_flow", "components": [ { "component_name": "_internal_broker_request_response", "component_module": "broker_request_response", - "component_config": self.broker_config, + "component_config": full_config, } ], } From a21b0526ca85ad76105f0bcb7ebf4cf69f22d492 Mon Sep 17 00:00:00 2001 From: Art Morozov Date: Mon, 9 Sep 2024 12:32:26 -0400 Subject: [PATCH 52/55] Fix env variable name --- .github/workflows/sync.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 1510126..216f10d 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -10,7 +10,6 @@ jobs: contents: write if: github.repository == 'SolaceLabs/solace-ai-connector' steps: - - run: gh repo sync SolaceDev/solace-ai-connector --source SolaceLabs/solace-ai-connector --branch $BRANCH_NAME + - run: gh repo sync SolaceDev/solace-ai-connector --source SolaceLabs/solace-ai-connector --branch ${{github.ref_name}} env: - GITHUB_TOKEN: ${{ secrets.GH_PAT }} - BRANCH_NAME: ${{ github.ref_name }} + GH_TOKEN: ${{ secrets.GH_PAT }} From 94eacf6aad630630be1773c32fa746de0c5aa748 Mon Sep 17 00:00:00 2001 From: Art Morozov Date: Mon, 9 Sep 2024 12:41:49 -0400 Subject: [PATCH 53/55] Test on branch --- .github/workflows/sync.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 216f10d..f17b22a 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -8,8 +8,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: write + env: + GH_TOKEN: ${{ secrets.GH_PAT }} if: github.repository == 'SolaceLabs/solace-ai-connector' steps: - run: gh repo sync SolaceDev/solace-ai-connector --source SolaceLabs/solace-ai-connector --branch ${{github.ref_name}} - env: - GH_TOKEN: ${{ secrets.GH_PAT }} From 37a6bede4080a15ad78850a1979046ee62e9f92f Mon Sep 17 00:00:00 2001 From: Edward Funnekotter Date: Thu, 26 Sep 2024 16:50:15 -0400 Subject: [PATCH 54/55] New features: inline broker request-response, temporary queues, improved docs and examples and better testing (#39) * Examples Update + Code Refactor (#25) * Removed StorageManager * Added examples for OpenAI, Bedrock, Anthropic, and VertexAI * Updating old examples (1/2) * Updating old examples (2/2) * Added support for temporary queue + UUID queue name (#26) * Add assembly component and auto-generated documents (#27) * Added the assembly component * Auto-generated documents * Added type check * Update the cache service expiry logic + Update the assembly component to use cache expiry for timeout * Moved assembly to the correct place * Added MoA Example + UUID Invoke Function (#28) * MoA example: Broadcast to multiple agents * Added MoA event manager, added uuid invoke_function + test, updated auto-generated docs * Added assembly layer to MoA example * Update documentation for new users + Refactored component_input & source_expression (#29) * Refactored component_input to input_selection * Updated, added, and enhanced the documentation with new users in mind * Refactored source_expression function to evaluate_expression (backward compatible) * Added tips and tricks section + info and examples on custom modules * tiny format update * tiny update * Fixed solace disconnection issues on shutting down (#30) * Add RAG example for AI connector + delete action for vector index (#31) * Added a RAG example for AI connector * Added delete option to vectordb * Changed id to ids * chore: Refactor make_history_start_with_user_message method (#32) Fix the method to not trim the first entry if it is a "system" role * Keep history depth needs to be a positive integer and test refactor (#33) * chore: Refactor clear_history_but_keep_depth method to handle negative depth values * chore: small change to how this is solved * chore: one more try * refactor: move utils_for_test_files.py to solace_ai_connector module * refactor: removed the orginal utils_for_test_files.py * refactor: update import statements in test files * refactor: add sys.path.append("src") to test files * refactor: standardize import order and sys.path.append in test files * refactor: a bit more test infrastructure changes * feat: allow component_module to accept module objects directly * feat: add types module import to utils.py * test: add static import and object config test * refactor: update test_static_import_and_object_config to use create_test_flows * refactor: Improve test structure and remove duplicate test case * fix: remove duplicate import of yaml module * refactor: Modify test config to use dict instead of YAML string * refactor: convert config_yaml from string to dictionary * refactor: update static import test to use pass_through component * test: Add delay component message passing test * feat: add test for delay component message processing * feat: Added a new test function (test_one_component) to make it very easy to just run some quick tests on a single input -> expected output tests on a single component * feat: added input_transforms to the test_one_component so that input transforms can be tested with it * chore: a bit of cleanup and new tests for test_one_component * chore: rename test_one_component because it was being picked up as a test by the pytest scanner * fix: fixed a typo * Fix for anthropic example (#35) * Updating version dependency (#37) * Fixed url and file name in getting started (#38) * Add guide for RAG (#39) * Added guide for RAG * update wording * Added link to other docs from RAG guide (#40) * chore: added a timeout setting for running component tests so that you can test situations where you don't expect any output (#34) * AI-124: Add a feature to provide simple blocking broker request/response ability for components (#42) * feat: add request_response_controller.py * feat: implement RequestResponseFlowManager and RequestResponseController classes * style: format code with black and improve readability * feat: implement RequestResponseController for flow-based request-response handling * feat: implement RequestResponseController for handling request-response patterns * fix: import SolaceAiConnector for type checking * refactor: restructure Flow class and improve code organization * feat: implement multiple named RequestResponseControllers per component * refactor: initialize request-response controllers in ComponentBase * test: add request_response_controller functionality tests * feat: finished implementation and added some tests * refactor: rename RequestResponseController to RequestResponseFlowController * refactor: rename RequestResponseController to RequestResponseFlowController * refactor: some name changes * fix: update test function names for RequestResponseFlowController * refactor: more name changes * Ed/req_resp_examples_and_fixes (#41) * feat: Added a request_response_flow example and fixed a few issues along the way * feat: Reworked the broker_request_response built-in ability of components to be simpler. Instead of having to have a defined flow and then name that flow, it will automatically create a flow with a single broker_request_response component in it. Now there is a straightforward interating function call to allow components to issue a request and get streaming or non-streaming responses from that flow. * chore: fix the request_response example and remove the old one * docs: add broker request-response configuration * docs: added advanced_component_features.md * docs: add broker request-response configuration details * docs: add payload encoding and format to broker config * docs: add cache service and timer manager to advanced_component_features.md * docs: add configuration requirement for broker request-response * docs: update broker request-response section with configuration info * docs: a bit more detail about do_broker_request_response * docs: add link to advanced features page in table of contents * docs: add link to advanced features page * docs: reorder table of contents in index.md * docs: add custom components documentation * docs: Remove advanced component features from table of contents * docs: clean up a double inclusion of the same section * docs: small example change * chore: remove dead code * chore: add some extra comments to explain some test code * docs: Update description of STDIN input component Update the description of the STDIN input component to clarify that it waits for its output message to be acknowledged before prompting for the next input. This change is made in the `stdin_input.py` file. * chore: add is_broker_request_response_enabled method * chore: Some changes after review * feat: AI-129: add ability to specify a default value for a an environment variable in a .yaml config file (#43) * DATAGO-85484 Bump min python version --------- Co-authored-by: Cyrus Mobini <68962752+cyrus2281@users.noreply.github.com> Co-authored-by: Art Morozov Co-authored-by: Art Morozov --- .../inputs_outputs/broker_request_response.py | 13 +++++++++++++ .../flow/request_response_flow_controller.py | 2 -- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py b/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py index bdaea62..71b33df 100644 --- a/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py +++ b/src/solace_ai_connector/components/inputs_outputs/broker_request_response.py @@ -98,6 +98,19 @@ "description": "The source expression to determine when the last piece of a " "streaming response has arrived.", }, + { + "name": "streaming", + "required": False, + "description": "The response will arrive in multiple pieces. If True, " + "the streaming_complete_expression must be set and will be used to " + "determine when the last piece has arrived.", + }, + { + "name": "streaming_complete_expression", + "required": False, + "description": "The source expression to determine when the last piece of a " + "streaming response has arrived.", + }, ], "input_schema": { "type": "object", diff --git a/src/solace_ai_connector/flow/request_response_flow_controller.py b/src/solace_ai_connector/flow/request_response_flow_controller.py index 37a4fe9..ce87f1c 100644 --- a/src/solace_ai_connector/flow/request_response_flow_controller.py +++ b/src/solace_ai_connector/flow/request_response_flow_controller.py @@ -30,7 +30,6 @@ # This is a very basic component which will be stitched onto the final component in the flow class RequestResponseControllerOuputComponent: - def __init__(self, controller): self.controller = controller @@ -40,7 +39,6 @@ def enqueue(self, event): # This is the main class that will be used to send messages to a flow and receive the response class RequestResponseFlowController: - def __init__(self, config: Dict[str, Any], connector): self.config = config self.connector = connector From ce8047b69328892b50490fcbfa92a7513eba0c38 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 26 Sep 2024 20:54:01 +0000 Subject: [PATCH 55/55] [ci skip] Bump version to 0.3.0 --- src/solace_ai_connector/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/solace_ai_connector/__init__.py b/src/solace_ai_connector/__init__.py index f189002..900e817 100644 --- a/src/solace_ai_connector/__init__.py +++ b/src/solace_ai_connector/__init__.py @@ -1,3 +1,3 @@ # Internal components that are dynamically loaded by the AI Connector # Listing them here allows for them to use relative imports -__version__ = "0.2.0" +__version__ = "0.3.0"