From a6d471c75939b2f4708a4e1cb1aa3b7b993ee54b Mon Sep 17 00:00:00 2001 From: Jae-Won Chung Date: Thu, 29 Feb 2024 01:04:07 -0500 Subject: [PATCH] Fix: `AttributeError` in OpenAI-compatible server (#3018) --- vllm/entrypoints/openai/protocol.py | 2 +- vllm/entrypoints/openai/serving_chat.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index e85e7e2b1ede9..97cfd797587c4 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -57,7 +57,7 @@ class UsageInfo(BaseModel): class ChatCompletionRequest(BaseModel): model: str - messages: Union[str, List[Dict[str, str]]] + messages: List[Dict[str, str]] temperature: Optional[float] = 0.7 top_p: Optional[float] = 1.0 n: Optional[int] = 1 diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 5635ac6c9e106..e5ae39e110a40 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -80,7 +80,7 @@ def get_chat_request_role(self, request: ChatCompletionRequest) -> str: if request.add_generation_prompt: return self.response_role else: - return request.messages[-1].role + return request.messages[-1]["role"] async def chat_completion_stream_generator( self, request: ChatCompletionRequest,