refine

docker/llm/serving/xpu/docker/start-pp_serving-service.sh

@@ -14,11 +14,14 @@ if [[ $KERNEL_VERSION != *"6.5"* ]]; then
 fi
 export TORCH_LLM_ALLREDUCE=0
 
+export IPEX_LLM_LAST_LM_HEAD=1
 export IPEX_LLM_QUANTIZE_KV_CACHE=1
 export num_gpus=2
 export model_path="/llm/models/Llama-2-7b-chat-hf"
 export low_bit="fp8"
 # max requests = max_num_reqs * rank_num
 export max_num_seqs="4"
+export max_prefilled_seqs="0"
+
 cd /llm/pp_serving
-CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $num_gpus pipeline_serving.py --repo-id-or-model-path $model_path --low-bit $low_bit --max-num-seqs $max_num_seqs
+CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $num_gpus pipeline_serving.py --repo-id-or-model-path $model_path --low-bit $low_bit --max-num-seqs $max_num_seqs --max-prefilled-seqs $max_prefilled_seqs

python/llm/example/GPU/Pipeline-Parallel-FastAPI/run.sh

@@ -28,9 +28,15 @@ if [[ $KERNEL_VERSION != *"6.5"* ]]; then
     export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
 fi
 export TORCH_LLM_ALLREDUCE=0
+export IPEX_LLM_QUANTIZE_KV_CACHE=1
+export IPEX_LLM_LAST_LM_HEAD=1
 
 export MODEL_PATH=YOUR_MODEL_PATH
 export NUM_GPUS=2
-export IPEX_LLM_QUANTIZE_KV_CACHE=1
+export ZE_AFFINITY_MASK=0,1
+export LOW_BIT="fp8"
+# max requests = max_num_reqs * rank_num
+export MAX_NUM_SEQS="4"
+export MAX_PREFILLED_SEQS=0
 
-CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS pipeline_serving.py --repo-id-or-model-path $MODEL_PATH --low-bit fp8 --max-num-seqs 4 --max-prefilled-seqs 0
+CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS pipeline_serving.py --repo-id-or-model-path $MODEL_PATH --low-bit $LOW_BIT --max-num-seqs $MAX_NUM_SEQS --max-prefilled-seqs $MAX_PREFILLED_SEQS