Fix vLLM CPU api_server params (intel-analytics#11384)

RyuKosei · Jul 19, 2024 · 390efd2 · 390efd2
1 parent 7adb00c
commit 390efd2
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/python/llm/src/ipex_llm/vllm/cpu/entrypoints/openai/api_server.py b/python/llm/src/ipex_llm/vllm/cpu/entrypoints/openai/api_server.py
@@ -175,7 +175,9 @@ async def authentication(request: Request, call_next):
         served_model_names = [args.model]
     engine_args = AsyncEngineArgs.from_cli_args(args)
     engine = IPEXLLMAsyncLLMEngine.from_engine_args(
-        engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
+        engine_args, usage_context=UsageContext.OPENAI_API_SERVER,
+        load_in_low_bit=args.load_in_low_bit,
+    )
     openai_serving_chat = OpenAIServingChat(engine, served_model_names,
                                             args.response_role,
                                             args.lora_modules,