diff --git a/python/llm/src/ipex_llm/transformers/npu_models/qwen2_mp.py b/python/llm/src/ipex_llm/transformers/npu_models/qwen2_mp.py
index 8459ddf5efe..9ad999475ef 100644
--- a/python/llm/src/ipex_llm/transformers/npu_models/qwen2_mp.py
+++ b/python/llm/src/ipex_llm/transformers/npu_models/qwen2_mp.py
@@ -229,7 +229,10 @@ def __init__(
             new_value_states = self.convert_to_fp16(curr_key_values[i][1])
 
         print(f"{mode} start compiling")
-        self.compile()
+        if group_size != 0 and (mode == "prefill" or num_layers == 2):
+            self.compile(npu_dpu_groups=6)
+        else:
+            self.compile()
         print(f"{mode} end compiling")
 
     def build_decoder(