✨ #149: Streaming Chat Workflow (#150)

- Initing a new type of workflow, a streaming (async) routing workflow using the Streaming Chat API as an example - Updated the Bruno collection - Updated the LanguageModel API to include `ChatStream()` and `SupportChatStream()` methods - Get the streaming router working - Implemented SSE event parsing to be able to work with OpenAI streaming chat API - Integrated OpenAI chat streaming into the Glide's streaming chat API - Covered the happy workflow by tests
EinStack · Mar 2, 2024 · 4edb644 · 4edb644
1 parent e9e7246
commit 4edb644
Show file tree

Hide file tree

Showing 46 changed files with 1,279 additions and 250 deletions.
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -69,9 +69,6 @@ jobs:
       - name: login into Github Container Registry
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
 
-      - name: login into Github Container Registry
-        run: echo "${{ secrets.DOCKER_HUB_TOKEN }}" | docker login -u einstack --password-stdin
-
       - name: login into Docker
         run: echo "${{ secrets.DOCKER_HUB_TOKEN }}" | docker login -u einstack --password-stdin
 

diff --git a/docs/api/Language API/💬 Chat Stream.bru b/docs/api/Language API/💬 Chat Stream.bru
@@ -0,0 +1,11 @@
+meta {
+  name: 💬 Chat Stream
+  type: http
+  seq: 2
+}
+
+get {
+  url: {{base_url}}/language/default/chatStream
+  body: none
+  auth: none
+}
diff --git a/docs/api/[Lang] Chat.bru → docs/api/Language API/💬 Chat.bru b/docs/api/[Lang] Chat.bru → docs/api/Language API/💬 Chat.bru
@@ -1,11 +1,11 @@
 meta {
-  name: [Lang] Chat
+  name: 💬 Chat
   type: http
-  seq: 2
+  seq: 1
 }
 
 post {
-  url: {{base_url}}/v1/language/myrouter/chat/
+  url: {{base_url}}/language/default/chat/
   body: json
   auth: none
 }

diff --git a/docs/api/[Lang] Router List.bru → docs/api/Language API/🔧 Router List.bru b/docs/api/[Lang] Router List.bru → docs/api/Language API/🔧 Router List.bru
@@ -1,11 +1,11 @@
 meta {
-  name: [Lang] Router List
+  name: 🔧 Router List
   type: http
   seq: 3
 }
 
 get {
-  url: {{base_url}}/v1/language/
+  url: {{base_url}}/language/
   body: json
   auth: none
 }

diff --git a/docs/api/environments/Development.bru b/docs/api/environments/Development.bru
@@ -1,3 +1,3 @@
 vars {
-  base_url: http://127.0.0.1:9099
+  base_url: http://127.0.0.1:9099/v1
 }
diff --git a/docs/api/Health.bru → docs/api/🔧 Health.bru b/docs/api/Health.bru → docs/api/🔧 Health.bru
@@ -1,11 +1,11 @@
 meta {
-  name: Health
+  name: 🔧 Health
   type: http
   seq: 1
 }
 
 get {
-  url: {{base_url}}/health
+  url: {{base_url}}/health/
   body: none
   auth: none
 }
diff --git a/docs/docs.go b/docs/docs.go
@@ -72,7 +72,7 @@ const docTemplate = `{
         },
         "/v1/language/{router}/chat": {
             "post": {
-                "description": "Talk to different LLMs Chat API via unified endpoint",
+                "description": "Talk to different LLM Chat APIs via unified endpoint",
                 "consumes": [
                     "application/json"
                 ],
@@ -123,6 +123,70 @@ const docTemplate = `{
                     }
                 }
             }
+        },
+        "/v1/language/{router}/chatStream": {
+            "get": {
+                "description": "Talk to different LLM Stream Chat APIs via a unified websocket endpoint",
+                "consumes": [
+                    "application/json"
+                ],
+                "tags": [
+                    "Language"
+                ],
+                "summary": "Language Chat",
+                "operationId": "glide-language-chat-stream",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "Router ID",
+                        "name": "router",
+                        "in": "path",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Websocket Connection Type",
+                        "name": "Connection",
+                        "in": "header",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Upgrade header",
+                        "name": "Upgrade",
+                        "in": "header",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Websocket Security Token",
+                        "name": "Sec-WebSocket-Key",
+                        "in": "header",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Websocket Security Token",
+                        "name": "Sec-WebSocket-Version",
+                        "in": "header",
+                        "required": true
+                    }
+                ],
+                "responses": {
+                    "101": {
+                        "description": "Switching Protocols"
+                    },
+                    "404": {
+                        "description": "Not Found",
+                        "schema": {
+                            "$ref": "#/definitions/http.ErrorSchema"
+                        }
+                    },
+                    "426": {
+                        "description": "Upgrade Required"
+                    }
+                }
+            }
         }
     },
     "definitions": {
@@ -742,6 +806,10 @@ const docTemplate = `{
         },
         "schemas.ChatMessage": {
             "type": "object",
+            "required": [
+                "content",
+                "role"
+            ],
             "properties": {
                 "content": {
                     "description": "The content of the message.",
@@ -759,6 +827,9 @@ const docTemplate = `{
         },
         "schemas.ChatRequest": {
             "type": "object",
+            "required": [
+                "message"
+            ],
             "properties": {
                 "message": {
                     "$ref": "#/definitions/schemas.ChatMessage"
@@ -790,7 +861,7 @@ const docTemplate = `{
                     "type": "string"
                 },
                 "modelResponse": {
-                    "$ref": "#/definitions/schemas.ProviderResponse"
+                    "$ref": "#/definitions/schemas.ModelResponse"
                 },
                 "model_id": {
                     "type": "string"
@@ -803,18 +874,7 @@ const docTemplate = `{
                 }
             }
         },
-        "schemas.OverrideChatRequest": {
-            "type": "object",
-            "properties": {
-                "message": {
-                    "$ref": "#/definitions/schemas.ChatMessage"
-                },
-                "model_id": {
-                    "type": "string"
-                }
-            }
-        },
-        "schemas.ProviderResponse": {
+        "schemas.ModelResponse": {
             "type": "object",
             "properties": {
                 "message": {
@@ -831,6 +891,21 @@ const docTemplate = `{
                 }
             }
         },
+        "schemas.OverrideChatRequest": {
+            "type": "object",
+            "required": [
+                "message",
+                "model_id"
+            ],
+            "properties": {
+                "message": {
+                    "$ref": "#/definitions/schemas.ChatMessage"
+                },
+                "model_id": {
+                    "type": "string"
+                }
+            }
+        },
         "schemas.TokenUsage": {
             "type": "object",
             "properties": {

diff --git a/docs/swagger.json b/docs/swagger.json
@@ -69,7 +69,7 @@
         },
         "/v1/language/{router}/chat": {
             "post": {
-                "description": "Talk to different LLMs Chat API via unified endpoint",
+                "description": "Talk to different LLM Chat APIs via unified endpoint",
                 "consumes": [
                     "application/json"
                 ],
@@ -120,6 +120,70 @@
                     }
                 }
             }
+        },
+        "/v1/language/{router}/chatStream": {
+            "get": {
+                "description": "Talk to different LLM Stream Chat APIs via a unified websocket endpoint",
+                "consumes": [
+                    "application/json"
+                ],
+                "tags": [
+                    "Language"
+                ],
+                "summary": "Language Chat",
+                "operationId": "glide-language-chat-stream",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "Router ID",
+                        "name": "router",
+                        "in": "path",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Websocket Connection Type",
+                        "name": "Connection",
+                        "in": "header",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Upgrade header",
+                        "name": "Upgrade",
+                        "in": "header",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Websocket Security Token",
+                        "name": "Sec-WebSocket-Key",
+                        "in": "header",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Websocket Security Token",
+                        "name": "Sec-WebSocket-Version",
+                        "in": "header",
+                        "required": true
+                    }
+                ],
+                "responses": {
+                    "101": {
+                        "description": "Switching Protocols"
+                    },
+                    "404": {
+                        "description": "Not Found",
+                        "schema": {
+                            "$ref": "#/definitions/http.ErrorSchema"
+                        }
+                    },
+                    "426": {
+                        "description": "Upgrade Required"
+                    }
+                }
+            }
         }
     },
     "definitions": {
@@ -739,6 +803,10 @@
         },
         "schemas.ChatMessage": {
             "type": "object",
+            "required": [
+                "content",
+                "role"
+            ],
             "properties": {
                 "content": {
                     "description": "The content of the message.",
@@ -756,6 +824,9 @@
         },
         "schemas.ChatRequest": {
             "type": "object",
+            "required": [
+                "message"
+            ],
             "properties": {
                 "message": {
                     "$ref": "#/definitions/schemas.ChatMessage"
@@ -787,7 +858,7 @@
                     "type": "string"
                 },
                 "modelResponse": {
-                    "$ref": "#/definitions/schemas.ProviderResponse"
+                    "$ref": "#/definitions/schemas.ModelResponse"
                 },
                 "model_id": {
                     "type": "string"
@@ -800,18 +871,7 @@
                 }
             }
         },
-        "schemas.OverrideChatRequest": {
-            "type": "object",
-            "properties": {
-                "message": {
-                    "$ref": "#/definitions/schemas.ChatMessage"
-                },
-                "model_id": {
-                    "type": "string"
-                }
-            }
-        },
-        "schemas.ProviderResponse": {
+        "schemas.ModelResponse": {
             "type": "object",
             "properties": {
                 "message": {
@@ -828,6 +888,21 @@
                 }
             }
         },
+        "schemas.OverrideChatRequest": {
+            "type": "object",
+            "required": [
+                "message",
+                "model_id"
+            ],
+            "properties": {
+                "message": {
+                    "$ref": "#/definitions/schemas.ChatMessage"
+                },
+                "model_id": {
+                    "type": "string"
+                }
+            }
+        },
         "schemas.TokenUsage": {
             "type": "object",
             "properties": {