diff --git a/python/llm/src/ipex_llm/transformers/npu_models/qwen2_mp.py b/python/llm/src/ipex_llm/transformers/npu_models/qwen2_mp.py index 8459ddf5efe..9ad999475ef 100644 --- a/python/llm/src/ipex_llm/transformers/npu_models/qwen2_mp.py +++ b/python/llm/src/ipex_llm/transformers/npu_models/qwen2_mp.py @@ -229,7 +229,10 @@ def __init__( new_value_states = self.convert_to_fp16(curr_key_values[i][1]) print(f"{mode} start compiling") - self.compile() + if group_size != 0 and (mode == "prefill" or num_layers == 2): + self.compile(npu_dpu_groups=6) + else: + self.compile() print(f"{mode} end compiling") def build_decoder(