Support vpm and resampler module of minicpm-v on NPU

python/llm/example/NPU/HF-Transformers-AutoModels/Multimodal/minicpm-llama3-v2.5.py

@@ -56,7 +56,7 @@
 
     model = AutoModelForCausalLM.from_pretrained(
         model_path,
-        torch_dtype=torch.float32,
+        torch_dtype=torch.float16,
         trust_remote_code=True,
         attn_implementation="eager",
         load_in_low_bit="sym_int4",
@@ -66,7 +66,6 @@
         intra_pp=args.intra_pp,
         inter_pp=args.inter_pp,
         transpose_value_cache=not args.disable_transpose_value_cache,
-        modules_to_not_convert=['vpm', 'resampler']
     )
     tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
 

python/llm/example/NPU/HF-Transformers-AutoModels/Multimodal/minicpm_v_2_6.py

@@ -47,7 +47,7 @@
     image_path = args.image_url_or_path
 
     model = AutoModel.from_pretrained(model_path, 
-                                      torch_dtype=torch.float32,
+                                      torch_dtype=torch.float16,
                                       trust_remote_code=True,
                                       attn_implementation="eager",
                                       load_in_low_bit="sym_int4",
@@ -57,8 +57,7 @@
                                       intra_pp=args.intra_pp,
                                       inter_pp=args.inter_pp,
                                       transpose_value_cache=not args.disable_transpose_value_cache,
-                                      modules_to_not_convert=['vpm', 'resampler']
-                                     )
+                                      )
     tokenizer = AutoTokenizer.from_pretrained(model_path,
                                               trust_remote_code=True)
     model.eval()

python/llm/src/ipex_llm/transformers/npu_models/convert_mp.py

@@ -46,12 +46,7 @@ def optimize_llm_pre(model: torch.nn.Module, qtype, mixed_precision,
             from ipex_llm.transformers.models.baichuan import pre_compute_inv_freq
             model.apply(pre_compute_inv_freq)
 
-    # MiniCPM-V 2.6 must put lm_head on CPU now
-    cpu_lm_head = (
-        (model.config.model_type == "minicpmv" and model.config.hidden_size == 3584 and
-         model.config.vocab_size == 151666)
-        or os.environ.get("IPEX_LLM_CPU_LM_HEAD", "0") != "0"
-    )
+    cpu_lm_head = os.environ.get("IPEX_LLM_CPU_LM_HEAD", "0") != "0"
 
     # workaround for MiniCPM-2B
     if model.config.model_type == "minicpm" and model.config.num_hidden_layers == 40:
@@ -76,6 +71,48 @@ def optimize_llm_pre(model: torch.nn.Module, qtype, mixed_precision,
 
     if model.config.model_type == "minicpmv" and hasattr(model, "llm"):
         # MiniCPM-V
+        # convert conv2d and layernorm
+        from ipex_llm.transformers.npu_models.minicpmv_mp import MinicpmVPatchEmbedding, \
+            replace_with_Layernorm
+        origin_conv = model.vpm.embeddings.patch_embedding
+        new_conv = MinicpmVPatchEmbedding(
+            weight=origin_conv.weight.to(torch.float16),
+            bias=origin_conv.bias.to(torch.float16),
+            strides=model.config.vision_config.patch_size,
+        )
+        model.vpm.embeddings.patch_embedding = new_conv
+        del new_conv
+        replace_with_Layernorm(model, qtype=None, device='NPU',
+                               modules_to_not_convert=[], group_size=0)
+
+        # replace forward function
+        from ipex_llm.transformers.npu_models.minicpmv_mp import pad_mlp_fc2, pad_mlp_forward, \
+            encoder_attn_forward, multi_head_attn_forward, resampler_forward
+        model.apply(pad_mlp_fc2)    # pad mlp.fc2 to avoid compile error
+        modeling_module_name = model.__class__.__module__
+        module = importlib.import_module(modeling_module_name)
+        setattr(module.Resampler, "forward", resampler_forward)
+        module = importlib.import_module(modeling_module_name.replace("modeling_minicpmv",
+                                                                      "resampler"))
+        setattr(module.MultiheadAttention, "multi_head_attention_forward", multi_head_attn_forward)
+        if model.config.hidden_size == 3584 and model.config.vocab_size == 151666:
+            # MiniCPM-V 2.6
+            module = importlib.import_module(modeling_module_name.replace("modeling_minicpmv",
+                                                                          "modeling_navit_siglip"))
+            setattr(module.SiglipAttention, "forward", encoder_attn_forward)
+            setattr(module.SiglipMLP, "forward", pad_mlp_forward)
+
+            # workaround for lm_head on NPU
+            from ipex_llm.transformers.npu_models.minicpmv_mp import pad_lm_head, lm_head_forward
+            model.apply(pad_lm_head)    # pad lm_head to avoid compile error
+            setattr(model.llm.lm_head, "forward", lm_head_forward)
+        elif model.config.hidden_size == 4096 and model.config.vocab_size == 128256:
+            # MiniCPM-V 2.5
+            from transformers.models.idefics2.modeling_idefics2 import Idefics2VisionMLP, \
+                Idefics2VisionAttention
+            convert_forward(model, Idefics2VisionAttention, encoder_attn_forward)
+            convert_forward(model, Idefics2VisionMLP, pad_mlp_forward)
+
         if model.config.hidden_size == 2304 and model.config.vocab_size == 122753:
             # MiniCPM-V 2
             model.llm.config.model_type = "minicpm"