[Bugfix] Include encoder prompts len to non-stream api usage response (…

…vllm-project#8861)
axonn-ai · Sep 30, 2024 · 5d23aff · 5d23aff
1 parent 3467dc3
commit 5d23aff
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py
@@ -726,6 +726,8 @@ async def chat_completion_full_generator(
 
         assert final_res.prompt_token_ids is not None
         num_prompt_tokens = len(final_res.prompt_token_ids)
+        if final_res.encoder_prompt_token_ids is not None:
+            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
         num_generated_tokens = sum(
             len(output.token_ids) for output in final_res.outputs)
         usage = UsageInfo(