[Frontend] Separate OpenAI Batch Runner usage from API Server (vllm-p…

…roject#4851)
neuralmagic · May 19, 2024 · de61ba7 · de61ba7
1 parent 3426d29
commit de61ba7
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/vllm/entrypoints/openai/run_batch.py b/vllm/entrypoints/openai/run_batch.py
@@ -101,7 +101,7 @@ async def main(args):
 
     engine_args = AsyncEngineArgs.from_cli_args(args)
     engine = AsyncLLMEngine.from_engine_args(
-        engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
+        engine_args, usage_context=UsageContext.OPENAI_BATCH_RUNNER)
 
     # When using single vLLM without engine_use_ray
     model_config = await engine.get_model_config()

diff --git a/vllm/usage/usage_lib.py b/vllm/usage/usage_lib.py
@@ -90,6 +90,7 @@ class UsageContext(str, Enum):
     LLM_CLASS = "LLM_CLASS"
     API_SERVER = "API_SERVER"
     OPENAI_API_SERVER = "OPENAI_API_SERVER"
+    OPENAI_BATCH_RUNNER = "OPENAI_BATCH_RUNNER"
     ENGINE_CONTEXT = "ENGINE_CONTEXT"