fix

intel-analytics · Jun 21, 2024 · 145a8a1 · 145a8a1
1 parent 9a3a21e
commit 145a8a1
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/python/llm/src/ipex_llm/vllm/cpu/entrypoints/openai/api_server.py b/python/llm/src/ipex_llm/vllm/cpu/entrypoints/openai/api_server.py
@@ -175,7 +175,9 @@ async def authentication(request: Request, call_next):
         served_model_names = [args.model]
     engine_args = AsyncEngineArgs.from_cli_args(args)
     engine = IPEXLLMAsyncLLMEngine.from_engine_args(
-        engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
+        engine_args, usage_context=UsageContext.OPENAI_API_SERVER,
+        load_in_low_bit=args.load_in_low_bit,
+    )
     openai_serving_chat = OpenAIServingChat(engine, served_model_names,
                                             args.response_role,
                                             args.lora_modules,