glm-4v-9b support (#11327)

* chatglm4v support * fix style check * update glm4v
intel-analytics · Jun 17, 2024 · 183e0c6 · 183e0c6
1 parent bca5cbd
commit 183e0c6
Show file tree

Hide file tree

Showing 2 changed files with 379 additions and 15 deletions.
diff --git a/python/llm/src/ipex_llm/transformers/convert.py b/python/llm/src/ipex_llm/transformers/convert.py
@@ -1055,23 +1055,47 @@ def _optimize_post(model, lightweight_bmm=False):
                             module.SelfAttention,
                             chatglm_attention_forward
                             )
-        elif (model.config.num_layers == 40 and hasattr(model.config, 'rope_ratio')
-                and model.config.rope_ratio == 500):
-            # glm-4-9b-chat
+        elif model.config.num_layers == 40 and hasattr(model.config, 'rope_ratio'):
             modeling_module_name = model.__class__.__module__
             module = importlib.import_module(modeling_module_name)
-            from ipex_llm.transformers.models.chatglm4 import chatglm4_attention_forward
-            from ipex_llm.transformers.models.chatglm4 import chatglm4_model_forward
-            from ipex_llm.transformers.models.chatglm2 import chatglm_rms_norm_forward
-            convert_forward(model,
-                            module.SelfAttention,
-                            chatglm4_attention_forward)
-            convert_forward(model,
-                            module.ChatGLMModel,
-                            chatglm4_model_forward)
-            convert_forward(model,
-                            module.RMSNorm,
-                            chatglm_rms_norm_forward)
+            if hasattr(model.transformer, "vision"):
+                # glm-4v-9b
+                modeling_module_name = model.transformer.vision.__class__.__module__
+                vision_module = importlib.import_module(modeling_module_name)
+                from ipex_llm.transformers.models.chatglm4v import chatglm4v_attention_forward
+                from ipex_llm.transformers.models.chatglm4v import chatglm4v_model_forward
+                from ipex_llm.transformers.models.chatglm4v import visual_attention_forward
+                from ipex_llm.transformers.models.chatglm4v import patch_embedding_forward
+                from ipex_llm.transformers.models.chatglm2 import chatglm_rms_norm_forward
+                convert_forward(model,
+                                module.SelfAttention,
+                                chatglm4v_attention_forward)
+                convert_forward(model,
+                                module.ChatGLMModel,
+                                chatglm4v_model_forward)
+                convert_forward(model,
+                                module.RMSNorm,
+                                chatglm_rms_norm_forward)
+                convert_forward(model,
+                                vision_module.Attention,
+                                visual_attention_forward)
+                convert_forward(model,
+                                vision_module.PatchEmbedding,
+                                patch_embedding_forward)
+            else:
+                # glm-4-9b-chat
+                from ipex_llm.transformers.models.chatglm4 import chatglm4_attention_forward
+                from ipex_llm.transformers.models.chatglm4 import chatglm4_model_forward
+                from ipex_llm.transformers.models.chatglm2 import chatglm_rms_norm_forward
+                convert_forward(model,
+                                module.SelfAttention,
+                                chatglm4_attention_forward)
+                convert_forward(model,
+                                module.ChatGLMModel,
+                                chatglm4_model_forward)
+                convert_forward(model,
+                                module.RMSNorm,
+                                chatglm_rms_norm_forward)
 
     elif "mpt" in model.config.model_type:
         if model.config.architectures is not None: