From 24377de1e009b1ffe9f8c7f9f473266f53733403 Mon Sep 17 00:00:00 2001 From: xiangyuT Date: Thu, 11 Jul 2024 14:10:20 +0800 Subject: [PATCH] refine --- .../llm/serving/xpu/docker/start-pp_serving-service.sh | 5 ++++- .../llm/example/GPU/Pipeline-Parallel-FastAPI/run.sh | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docker/llm/serving/xpu/docker/start-pp_serving-service.sh b/docker/llm/serving/xpu/docker/start-pp_serving-service.sh index 9e3333aa766..c1c52291b67 100644 --- a/docker/llm/serving/xpu/docker/start-pp_serving-service.sh +++ b/docker/llm/serving/xpu/docker/start-pp_serving-service.sh @@ -14,11 +14,14 @@ if [[ $KERNEL_VERSION != *"6.5"* ]]; then fi export TORCH_LLM_ALLREDUCE=0 +export IPEX_LLM_LAST_LM_HEAD=1 export IPEX_LLM_QUANTIZE_KV_CACHE=1 export num_gpus=2 export model_path="/llm/models/Llama-2-7b-chat-hf" export low_bit="fp8" # max requests = max_num_reqs * rank_num export max_num_seqs="4" +export max_prefilled_seqs="0" + cd /llm/pp_serving -CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $num_gpus pipeline_serving.py --repo-id-or-model-path $model_path --low-bit $low_bit --max-num-seqs $max_num_seqs +CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $num_gpus pipeline_serving.py --repo-id-or-model-path $model_path --low-bit $low_bit --max-num-seqs $max_num_seqs --max-prefilled-seqs $max_prefilled_seqs diff --git a/python/llm/example/GPU/Pipeline-Parallel-FastAPI/run.sh b/python/llm/example/GPU/Pipeline-Parallel-FastAPI/run.sh index 91bf6161bff..5f7719bc0ae 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-FastAPI/run.sh +++ b/python/llm/example/GPU/Pipeline-Parallel-FastAPI/run.sh @@ -28,9 +28,15 @@ if [[ $KERNEL_VERSION != *"6.5"* ]]; then export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 fi export TORCH_LLM_ALLREDUCE=0 +export IPEX_LLM_QUANTIZE_KV_CACHE=1 +export IPEX_LLM_LAST_LM_HEAD=1 export MODEL_PATH=YOUR_MODEL_PATH export NUM_GPUS=2 -export IPEX_LLM_QUANTIZE_KV_CACHE=1 +export ZE_AFFINITY_MASK=0,1 +export LOW_BIT="fp8" +# max requests = max_num_reqs * rank_num +export MAX_NUM_SEQS="4" +export MAX_PREFILLED_SEQS=0 -CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS pipeline_serving.py --repo-id-or-model-path $MODEL_PATH --low-bit fp8 --max-num-seqs 4 --max-prefilled-seqs 0 +CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS pipeline_serving.py --repo-id-or-model-path $MODEL_PATH --low-bit $LOW_BIT --max-num-seqs $MAX_NUM_SEQS --max-prefilled-seqs $MAX_PREFILLED_SEQS