[Bugfix] Fixing max token error message for openai compatible server (#…

…4016)
vllm-project · Apr 23, 2024 · d3c8180 · d3c8180
1 parent 62b8aeb
commit d3c8180
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py
@@ -206,6 +206,12 @@ def _validate_prompt_and_tokenize(
         token_num = len(input_ids)
 
         if request.max_tokens is None:
+            if token_num >= self.max_model_len:
+                raise ValueError(
+                    f"This model's maximum context length is "
+                    f"{self.max_model_len} tokens. However, you requested "
+                    f"{token_num} tokens in the messages, "
+                    f"Please reduce the length of the messages.", )
             request.max_tokens = self.max_model_len - token_num
 
         if token_num + request.max_tokens > self.max_model_len: