Merge pull request #574 from openchatai/feat/analytics_2

Adding columns to track if api call / knowledgebase was called

llm-server/custom_types/response_dict.py

@@ -13,15 +13,14 @@ class ApiRequestResult:
     api_requests: Dict[str, str] = field(default_factory=dict)
 
 
-class LLMResponse(NamedTuple):
-    message: Optional[str]
-    error: Optional[str]
-    api_request_response: ApiRequestResult
+@dataclass
+class LLMResponse:
+    message: Optional[str] = None
+    error: Optional[str] = None
+    api_request_response: ApiRequestResult = ApiRequestResult()
+    api_called: bool = False
+    knowledgebase_called: bool = False
 
     @classmethod
     def create_default(cls):
-        return cls(
-            message=None,
-            error=None,
-            api_request_response=ApiRequestResult(),
-        )
+        return cls()

llm-server/models/migrations/versions/b9828e70d171_add_api_call_made_and_knowledgebase_.py

@@ -0,0 +1,35 @@
+"""Add api_call_made and knowledgebase_called columns
+
+Revision ID: b9828e70d171
+Revises: 62ef7ae67c7d
+Create Date: 2024-01-22 20:57:25.839823
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import Column, Boolean
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision: str = "b9828e70d171"
+down_revision: Union[str, None] = "62ef7ae67c7d"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+# Add two new boolean columns
+def upgrade():
+    op.add_column("chat_history", Column("api_called", Boolean, default=False))
+    op.add_column(
+        "chat_history", Column("knowledgebase_called", Boolean, default=False)
+    )
+
+
+# Remove the two boolean columns
+def downgrade():
+    op.drop_column("chat_history", "api_called")
+    op.drop_column("chat_history", "knowledgebase_called")

llm-server/models/repository/chat_history_repo.py

@@ -2,11 +2,13 @@
 from typing import Optional, List, Dict, Union, Tuple
 from shared.models.opencopilot_db import ChatHistory, engine
 from sqlalchemy.orm import sessionmaker
-from sqlalchemy import distinct
+from sqlalchemy import Integer, distinct
 from sqlalchemy.orm import class_mapper
 from langchain.schema import BaseMessage, AIMessage, HumanMessage
 from sqlalchemy import func
 
+from shared.models.opencopilot_db.chatbot import Chatbot
+
 Session = sessionmaker(bind=engine)
 
 
@@ -266,10 +268,48 @@ def create_chat_histories(
                 from_user=record["from_user"],
                 message=record["message"],
                 debug_json=record.get("debug_json"),
+                api_called=record.get("api_called"),
+                knowledgebase_called=record.get("knowledgebase_called"),
             )
 
             session.add(chat_history)
             chat_histories.append(chat_history)
         session.commit()
 
     return chat_histories
+
+
+async def get_analytics(email: str):
+    with Session() as session:
+        chat_histories = []
+
+        # Step 1: Get chatbot_ids associated with the given email
+        chatbot_ids = session.query(Chatbot.id).filter(Chatbot.email == email).all()
+
+        if chatbot_ids:
+            chatbot_ids = [chatbot_id[0] for chatbot_id in chatbot_ids]
+
+            # Step 2: Get analytics data in a single query
+            analytics_data = (
+                session.query(
+                    func.sum(func.cast(ChatHistory.api_called, Integer)).label("api_called_count"),
+                    func.sum(func.cast(ChatHistory.knowledgebase_called, Integer)).label("knowledgebase_called_count"),
+                    func.count().label("total"),
+                    func.sum(func.cast(
+                        ~ChatHistory.api_called & ~ChatHistory.knowledgebase_called, Integer)).label("other_count")
+                )
+                .filter(ChatHistory.chatbot_id.in_(chatbot_ids))
+                .one()
+            )
+
+            # Append the results to chat_histories
+            chat_histories.append(
+                {
+                    "api_called_count": analytics_data.api_called_count or 0,
+                    "knowledgebase_called_count": analytics_data.knowledgebase_called_count or 0,
+                    "total": analytics_data.total or 0,
+                    "other_count": analytics_data.other_count or 0,
+                }
+            )
+
+    return chat_histories

llm-server/routes/chat/chat_controller.py

@@ -8,6 +8,7 @@
     get_all_chat_history_by_session_id_with_total,
     get_unique_sessions_with_first_message_by_bot_id,
     create_chat_histories,
+    get_analytics,
 )
 from models.repository.copilot_repo import find_one_or_fail_by_token
 from routes.analytics.analytics_service import upsert_analytics_record
@@ -19,9 +20,7 @@
 from utils.get_logger import CustomLogger
 from utils.llm_consts import X_App_Name, chat_strategy, ChatStrategy
 from utils.sqlalchemy_objs_to_json_array import sqlalchemy_objs_to_json_array
-from .. import root_service
 from flask_socketio import emit
-import asyncio
 
 logger = CustomLogger(module_name=__name__)
 
@@ -211,6 +210,8 @@ async def handle_chat_send_common(
                     "session_id": session_id,
                     "from_user": False,
                     "message": result.message or result.error,
+                    "api_called": result.api_called,
+                    "knowledgebase_called": result.knowledgebase_called,
                     "debug_json": result.api_request_response.api_requests,
                 },
             ]
@@ -252,3 +253,18 @@ async def handle_chat_send_common(
             ),
             500,
         )
+
+
+# curl -X POST http://localhost:5000/analytics -H "x_consumer_username: your_username"
+@chat_workflow.route("/analytics", methods=["GET"])
+async def get_analytics_by_email():
+    x_consumer_username = request.headers.get("x_consumer_username") or "guest"
+    if not x_consumer_username:
+        return Response(
+            response='{"error": "x_consumer_username is required"}',
+            status=400,
+            content_type="application/json",
+        )
+
+    result = await get_analytics(x_consumer_username)
+    return jsonify(result)

llm-server/routes/chat/implementation/chain_strategy.py

@@ -87,6 +87,8 @@ async def handle_request(
                 is_streaming=is_streaming,
                 session_id=session_id,
             )
+
+            response.api_called = True
             return response
         else:
             # it means that the user query is "informative" and can be answered using text only
@@ -102,4 +104,6 @@ async def handle_request(
                 is_streaming=is_streaming,
                 session_id=session_id,
             )
+
+            response.knowledgebase_called = True
             return response

llm-server/routes/flow/utils/run_workflow.py

@@ -57,4 +57,5 @@ async def run_flow(
         api_request_response=ApiRequestResult(api_request_data),
         error=output["error"],
         message=output["response"],
+        api_called=True,
     )

llm-server/routes/root_service.py

@@ -175,5 +175,8 @@ def run_informative_item(
 
     # return {"response": content, "error": None}
     return LLMResponse(
-        message=content, error=None, api_request_response=ApiRequestResult()
+        message=content,
+        error=None,
+        api_request_response=ApiRequestResult(),
+        knowledgebase_called=True,
     )

llm-server/shared/models/opencopilot_db/chat_history.py

@@ -4,9 +4,9 @@
 
 from .database_setup import engine
 from .get_declarative_base import Base
-import json
 from dataclasses import dataclass
 
+
 @dataclass
 class ChatHistory(Base):
     __tablename__ = "chat_history"
@@ -21,6 +21,8 @@ class ChatHistory(Base):
         DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow
     )
     debug_json = Column(JSON, default={}, nullable=True)
+    api_called = Column(Boolean, default=False)
+    knowledgebase_called = Column(Boolean, default=False)
 
 
 Base.metadata.create_all(engine)