implementing debug mode

llm-server/copilot_exceptions/api_call_failed_exception.py

@@ -0,0 +1,6 @@
+class APICallFailedException(Exception):
+    """Custom exception for API call failures."""
+
+    def __init__(self, message="API call failed"):
+        self.message = message
+        super().__init__(self.message)

llm-server/custom_types/response_dict.py

@@ -1,6 +1,27 @@
-from typing import TypedDict, Optional
+from typing import NamedTuple, TypedDict, Optional, Dict, Any
+import requests
+from dataclasses import dataclass, field
 
 
 class ResponseDict(TypedDict):
     response: Optional[str]
     error: Optional[str]
+
+
+@dataclass
+class ApiRequestResult:
+    api_requests: Dict[str, str] = field(default_factory=dict)
+
+
+class LLMResponse(NamedTuple):
+    message: Optional[str]
+    error: Optional[str]
+    api_request_response: ApiRequestResult
+
+    @classmethod
+    def create_default(cls):
+        return cls(
+            message=None,
+            error=None,
+            api_request_response=ApiRequestResult(),
+        )

llm-server/models/migrations/versions/62ef7ae67c7d_add_debug_json_column_to_chat_history.py

@@ -0,0 +1,31 @@
+"""Add debug_json column to chat history
+
+Revision ID: 62ef7ae67c7d
+Revises: 228d50d1fc45
+Create Date: 2024-01-20 19:04:21.677356
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision: str = "62ef7ae67c7d"
+down_revision: Union[str, None] = "228d50d1fc45"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    if (
+        not op.get_bind()
+        .execute(sa.text("SHOW COLUMNS FROM chat_history LIKE 'debug_json'"))
+        .fetchone()
+    ):
+        op.add_column("chat_history", sa.Column("debug_json", sa.Text(), nullable=True))
+
+
+def downgrade():
+    op.drop_column("chatbots", "global_variables")

llm-server/models/repository/chat_history_repo.py

@@ -255,6 +255,7 @@ def create_chat_histories(
                 session_id=record["session_id"],
                 from_user=record["from_user"],
                 message=record["message"],
+                debug_json=record.get("debug_json"),
             )
 
             session.add(chat_history)

llm-server/routes/chat/chat_controller.py

@@ -3,7 +3,7 @@
 from typing import cast, Dict
 
 from flask import jsonify, Blueprint, request, Response, abort, Request
-from custom_types.response_dict import ResponseDict
+from custom_types.response_dict import LLMResponse, ResponseDict
 
 from models.repository.chat_history_repo import (
     get_all_chat_history_by_session_id,
@@ -23,6 +23,7 @@
 from utils.sqlalchemy_objs_to_json_array import sqlalchemy_objs_to_json_array
 from .. import root_service
 from flask_socketio import emit
+import asyncio
 
 logger = CustomLogger(module_name=__name__)
 
@@ -127,10 +128,6 @@ async def send_chat_stream(
 
 @chat_workflow.route("/send", methods=["POST"])
 async def send_chat():
-    response_data: ResponseDict = {
-        "error": "",
-        "response": "Something went wrong, please try again!",
-    }
     json_data = request.get_json()
 
     input_data = ChatInput(**json_data)
@@ -170,7 +167,8 @@ async def handle_chat_send_common(
             strategy = ToolStrategy()
 
         headers_from_json.update(bot.global_variables or {})
-        response_data = await strategy.handle_request(
+
+        result = await strategy.handle_request(
             message,
             session_id,
             str(base_prompt),
@@ -180,7 +178,8 @@ async def handle_chat_send_common(
             is_streaming,
         )
 
-        if response_data["response"]:
+        # if the llm replied correctly
+        if result.message is not None:
             chat_records = [
                 {
                     "session_id": session_id,
@@ -190,27 +189,25 @@ async def handle_chat_send_common(
                 {
                     "session_id": session_id,
                     "from_user": False,
-                    "message": response_data["response"]
-                    or response_data["error"]
-                    or "",
+                    "message": result.message,
+                    "debug_json": str(result.api_request_response.__dict__),
                 },
             ]
 
             upsert_analytics_record(
                 chatbot_id=str(bot.id), successful_operations=1, total_operations=1
             )
             create_chat_histories(str(bot.id), chat_records)
-        elif response_data["error"]:
+
+        elif result.error:
             upsert_analytics_record(
                 chatbot_id=str(bot.id),
                 successful_operations=0,
                 total_operations=1,
-                logs=response_data["error"],
+                logs=result.error,
             )
-
-        emit(session_id, "|im_end|") if is_streaming else None
-        return jsonify(
-            {"type": "text", "response": {"text": response_data["response"]}}
+        emit(session_id, "|im_end|") if is_streaming else jsonify(
+            {"type": "text", "response": {"text": result.message}}
         )
     except Exception as e:
         logger.error(

llm-server/routes/chat/implementation/chain_strategy.py

@@ -1,10 +1,10 @@
 from flask_socketio import emit
 from models.repository.chat_history_repo import get_chat_message_as_llm_conversation
 from routes.chat.implementation.handler_interface import ChatRequestHandler
-from typing import Callable, Dict, Optional
+from typing import Awaitable, Callable, Dict, Optional
 import asyncio
 
-from custom_types.response_dict import ResponseDict
+from custom_types.response_dict import LLMResponse
 from routes.flow.utils.api_retrievers import (
     get_relevant_actions,
     get_relevant_flows,
@@ -32,12 +32,7 @@ async def handle_request(
         headers: Dict[str, str],
         app: Optional[str],
         is_streaming: bool,
-    ) -> ResponseDict:
-        # Dict
-        response: ResponseDict = {
-            "error": "",
-            "response": "Something went wrong, please try again!",
-        }
+    ) -> LLMResponse:
         check_required_fields(base_prompt, text)
 
         tasks = [

llm-server/routes/chat/implementation/functions_strategy.py

@@ -1,12 +1,12 @@
 from routes.chat.implementation.handler_interface import ChatRequestHandler
 from typing import Dict, Optional
 
-from custom_types.response_dict import ResponseDict
+from custom_types.response_dict import LLMResponse
 from asyncio import Future
 
 
 class FunctionStrategy(ChatRequestHandler):
-    def handle_request(
+    async def handle_request(
         self,
         text: str,
         session_id: str,
@@ -15,6 +15,6 @@ def handle_request(
         headers: Dict[str, str],
         app: Optional[str],
         is_streaming: bool,
-    ) -> Future[ResponseDict]:
+    ) -> LLMResponse:
         # Extract relevant information from inputs
         raise NotImplementedError("Subclasses must override handle_request.")

llm-server/routes/chat/implementation/handler_interface.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
-from typing import Dict, Optional
-from custom_types.response_dict import ResponseDict
+from typing import Awaitable, Dict, Optional
+from custom_types.response_dict import LLMResponse
 from asyncio import Future
 
 
@@ -15,5 +15,5 @@ async def handle_request(
         headers: Dict[str, str],
         app: Optional[str],
         is_streaming: bool,
-    ) -> Future[ResponseDict]:
+    ) -> LLMResponse:
         pass

llm-server/routes/chat/implementation/tools_strategy.py

@@ -1,12 +1,12 @@
 from asyncio import Future
 from routes.chat.implementation.handler_interface import ChatRequestHandler
-from typing import Dict, Optional
+from typing import Awaitable, Dict, Optional
 
-from custom_types.response_dict import ResponseDict
+from custom_types.response_dict import LLMResponse
 
 
 class ToolStrategy(ChatRequestHandler):
-    def handle_request(
+    async def handle_request(
         self,
         text: str,
         session_id: str,
@@ -15,6 +15,6 @@ def handle_request(
         headers: Dict[str, str],
         app: Optional[str],
         is_streaming: bool,
-    ) -> Future[ResponseDict]:
+    ) -> LLMResponse:
         # Extract relevant information from inputs
         raise NotImplementedError("Subclasses must override handle_request.")

llm-server/routes/flow/utils/run_openapi_ops.py

@@ -1,10 +1,10 @@
 import json
-from typing import Optional
+from typing import Optional, Tuple
 from flask_socketio import emit
-from openai import InvalidRequestError
 
 from werkzeug.datastructures import Headers
 from requests.exceptions import MissingSchema
+from copilot_exceptions.api_call_failed_exception import APICallFailedException
 from entities.flow_entity import FlowDTO
 from extractors.convert_json_to_text import (
     convert_json_error_to_text,
@@ -29,7 +29,7 @@ async def run_actions(
     bot_id: str,
     session_id: str,
     is_streaming: bool,
-) -> str:
+) -> Tuple[str, dict]:
     api_request_data = {}
     prev_api_response = ""
     apis_calls_history = {}
@@ -69,21 +69,24 @@ async def run_actions(
                         operation_id=operation_id,
                         app=app,
                     )
-                    apis_calls_history[operation_id] = api_response.text
+                    apis_calls_history[operation_id] = api_response.api_requests[
+                        "response"
+                    ]
                 else:
                     logger.info(
                         "API Response",
                         incident="log_api_response",
-                        api_response=api_response.text,
+                        api_response=api_response.api_requests,
                         json_config_used=partial_json,
                         next_action="summarize_with_partial_json",
                     )
-                    api_json = json.loads(api_response.text)
+                    api_json = json.loads(api_response.api_requests["response"])
                     apis_calls_history[operation_id] = json.dumps(
                         transform_response(
                             full_json=api_json, partial_json=partial_json
                         )
                     )
+
             except Exception as e:
                 logger.error(
                     "Error occurred during workflow check in store",
@@ -98,23 +101,23 @@ async def run_actions(
                 formatted_error = convert_json_error_to_text(
                     str(e), is_streaming, session_id
                 )
-                return str(formatted_error)
+                return str(formatted_error), api_request_data
+
+    try:
+        readable_response = convert_json_to_text(
+            text,
+            apis_calls_history,
+            api_request_data,
+            bot_id=bot_id,
+            session_id=session_id,
+            is_streaming=is_streaming,
+        )
 
-        try:
-            return convert_json_to_text(
-                text,
-                apis_calls_history,
-                api_request_data,
-                bot_id=bot_id,
-                session_id=session_id,
-                is_streaming=is_streaming,
-            )
-        except InvalidRequestError as e:
-            error_message = (
-                f"Api response too large for the endpoint: {api_payload.endpoint}"
-                if api_payload is not None
-                else ""
-            )
-            logger.error("OpenAI exception", bot_id=bot_id, error=str(e))
-            emit(session_id, error_message) if is_streaming else None
-            return error_message
+        return readable_response, api_request_data
+    except Exception as e:
+        error_message = (
+            f"{str(e)}: {api_payload.endpoint}" if api_payload is not None else ""
+        )
+        logger.error("OpenAI exception", bot_id=bot_id, error=str(e))
+        emit(session_id, error_message) if is_streaming else None
+        return error_message, api_request_data

llm-server/routes/flow/utils/run_workflow.py

@@ -1,10 +1,11 @@
 import json
 import logging
 from typing import Optional
+import requests
 
 from werkzeug.datastructures import Headers
 
-from custom_types.response_dict import ResponseDict
+from custom_types.response_dict import ApiRequestResult, LLMResponse
 from custom_types.run_workflow_input import ChatContext
 from entities.flow_entity import FlowDTO
 from routes.flow.utils import run_actions
@@ -20,14 +21,14 @@ async def run_flow(
     bot_id: str,
     session_id: str,
     is_streaming: bool,
-) -> ResponseDict:
+) -> LLMResponse:
     headers = chat_context.headers or Headers()
 
     result = ""
     error = None
-
+    api_request_data = {}
     try:
-        result = await run_actions(
+        response, api_request_data = await run_actions(
             flow=flow,
             text=chat_context.text,
             headers=headers,
@@ -52,4 +53,8 @@ async def run_flow(
     output = {"response": result if not error else "", "error": error}
 
     logging.info("Workflow output %s", json.dumps(output, separators=(",", ":")))
-    return output
+    return LLMResponse(
+        api_request_response=ApiRequestResult(api_request_data),
+        error=output["error"],
+        message=output["response"],
+    )