exl2 phi does not use packed QKV/gate-up projections

huggingface · Jun 26, 2024 · 7947c34 · 7947c34
1 parent be2d380
commit 7947c34
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
@@ -60,7 +60,7 @@ def load_attention(config, prefix, weights, layer_id):
     sizes = None
     prefixes = None
 
-    if config.model_type == "phi3":
+    if config.model_type == "phi3" and config.quantize != "exl2":
         prefix = f"{prefix}.qkv_proj"
         base_layer = TensorParallelColumnLinear.load_qkv(
             config,
@@ -246,7 +246,7 @@ def __init__(self, prefix, config, weights, index):
 
         # Fuse gate and up proj
         bias = getattr(config, "mlp_bias", False)
-        if config.model_type == "phi3":
+        if config.model_type == "phi3" and config.quantize != "exl2":
             gate_up_proj = TensorParallelColumnLinear.load_gate_up(
                 config,
                 prefix=f"{prefix}.gate_up_proj",