Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor llama convert to fix minicpm-v 2.5 optimization #11783

Merged
merged 2 commits into from
Aug 14, 2024
Merged

refactor llama convert to fix minicpm-v 2.5 optimization #11783

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
317d3be
refactor llama convert
MeouSker77 Aug 13, 2024
12f1dd0
fix
MeouSker77 Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 68 additions & 101 deletions python/llm/src/ipex_llm/transformers/convert.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -754,6 +754,10 @@ def _optimize_pre(model, qtype=None):
model.llm.config.model_type = "qwen2"
_optimize_pre(model.llm, qtype=qtype)
model.llm.config.model_type = "minicpmv"
elif model.config.hidden_size == 4096 and model.config.vocab_size == 128256:
model.llm.config.model_type = "llama"
_optimize_pre(model.llm, qtype=qtype)
model.llm.config.model_type = "minicpmv"

return model

Expand Down Expand Up @@ -933,16 +937,6 @@ def _optimize_ipex(model, qtype=ggml_tensor_qtype["bf16"]):


def _optimize_post(model, lightweight_bmm=False):
from packaging import version
from ipex_llm.transformers.models.llama import llama_attention_forward_4_31
from ipex_llm.transformers.models.llama import llama_attention_selective_batching_forward_4_31
from ipex_llm.transformers.models.llama import llama_model_selective_batching_forward_4_31
from ipex_llm.transformers.models.llama import llama_rms_norm_forward
from ipex_llm.transformers.models.llama import llama_mlp_forward
from ipex_llm.transformers.models.llama import llama_decoder_forward
from ipex_llm.transformers.models.llama import llama_model_forward
from transformers.modeling_utils import PreTrainedModel

try:
from sentence_transformers.SentenceTransformer import SentenceTransformer
if isinstance(model, SentenceTransformer):
Expand All @@ -961,110 +955,80 @@ def _optimize_post(model, lightweight_bmm=False):
except ModuleNotFoundError:
pass

from transformers.modeling_utils import PreTrainedModel
# All huggingface format models are inherited from `PreTrainedModel`
if not isinstance(model, PreTrainedModel):
logger.info("Only HuggingFace Transformers models are currently "
"supported for further optimizations")
return model

vllm_selective_batching = os.getenv("VLLM_ENABLE_SELECTIVE_BATCHING")
enable_vllm_se_batching = vllm_selective_batching is not None
enable_vllm_se_batching = enable_vllm_se_batching and vllm_selective_batching.lower() == "true"

from packaging import version
trans_version = transformers.__version__
if version.parse(trans_version) >= version.parse("4.31.0"):
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaRMSNorm,
llama_rms_norm_forward,)
convert_forward(model,
transformers.models.llama.modeling_llama.LlamaMLP,
llama_mlp_forward)
convert_forward(model,
transformers.models.llama.modeling_llama.LlamaDecoderLayer,
llama_decoder_forward)

# convert all nn.LayerNorm
from ipex_llm.transformers.models.bloom import bloom_layer_norm_forward
convert_forward(model,
nn.LayerNorm,
bloom_layer_norm_forward)
from ipex_llm.transformers.models.llama import llama_rms_norm_forward
from ipex_llm.transformers.models.llama import llama_mlp_forward

if model.config.model_type == "llama":
from transformers.models.llama.modeling_llama import LlamaRMSNorm
from transformers.models.llama.modeling_llama import LlamaMLP
from transformers.models.llama.modeling_llama import LlamaAttention
from transformers.models.llama.modeling_llama import LlamaDecoderLayer
from transformers.models.llama.modeling_llama import LlamaModel
if version.parse(trans_version) >= version.parse("4.36.0"):
# transformers version >= 4.36.0
from transformers.models.llama.modeling_llama import LlamaSdpaAttention

from ipex_llm.transformers.models.llama import llama_rms_norm_forward
from ipex_llm.transformers.models.llama import llama_mlp_forward
from ipex_llm.transformers.models.llama import llama_decoder_forward

convert_forward(model, LlamaRMSNorm, llama_rms_norm_forward)
convert_forward(model, LlamaMLP, llama_mlp_forward)
convert_forward(model, LlamaDecoderLayer, llama_decoder_forward)

if version.parse(trans_version) >= version.parse("4.41.0"):
from ipex_llm.transformers.models.llama import llama_model_forward_4_41
from ipex_llm.transformers.models.llama import llama_attention_forward_4_41
convert_forward(model, LlamaModel, llama_model_forward_4_41)
convert_forward(model, LlamaAttention, llama_attention_forward_4_41)
convert_forward(model, LlamaSdpaAttention, llama_attention_forward_4_41)
elif version.parse(trans_version) >= version.parse("4.38.0"):
from ipex_llm.transformers.models.llama import llama_model_forward_4_38
from ipex_llm.transformers.models.llama import llama_attention_forward_4_38
if version.parse(trans_version) >= version.parse("4.38.0"):
if version.parse(trans_version) >= version.parse("4.41.0"):
from ipex_llm.transformers.models.llama import llama_model_forward_4_41
from ipex_llm.transformers.models.llama import llama_attention_forward_4_41
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaModel,
llama_model_forward_4_41)
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaAttention,
llama_attention_forward_4_41)
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaSdpaAttention,
llama_attention_forward_4_41)
else:
from ipex_llm.transformers.models.llama import llama_model_forward_4_38
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaModel,
llama_model_forward_4_38)
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaAttention,
llama_attention_forward_4_38)
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaSdpaAttention,
llama_attention_forward_4_38)
else:
from ipex_llm.transformers.models.llama import llama_model_forward_4_36
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaModel,
llama_model_forward_4_36)
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaAttention,
llama_attention_forward_4_38)
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaSdpaAttention,
llama_attention_forward_4_38)
convert_forward(model, LlamaModel, llama_model_forward_4_38)
convert_forward(model, LlamaAttention, llama_attention_forward_4_38)
convert_forward(model, LlamaSdpaAttention, llama_attention_forward_4_38)
elif version.parse(trans_version) >= version.parse("4.36.0"):
from ipex_llm.transformers.models.llama import llama_model_forward_4_36
from ipex_llm.transformers.models.llama import llama_attention_forward_4_38
convert_forward(model, LlamaModel, llama_model_forward_4_36)
convert_forward(model, LlamaAttention, llama_attention_forward_4_38)
convert_forward(model, LlamaSdpaAttention, llama_attention_forward_4_38)
else:
# transformers version between 4.31.0 - 4.35.2
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaAttention,
llama_attention_forward_4_31, )
if enable_vllm_se_batching:
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaModel,
vllm_se_batching = os.getenv("VLLM_ENABLE_SELECTIVE_BATCHING", "").lower() == "true"
if vllm_se_batching:
from ipex_llm.transformers.models.llama import (
llama_model_selective_batching_forward_4_31,
)
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaAttention,
llama_attention_selective_batching_forward_4_31,
)
convert_forward(model, LlamaModel,
llama_model_selective_batching_forward_4_31)
convert_forward(model, LlamaAttention,
llama_attention_selective_batching_forward_4_31)
else:
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaModel,
llama_model_forward)
else:
# todo implement 4.28.0 ~ 4.30.2
pass

# convert all nn.LayerNorm
from ipex_llm.transformers.models.bloom import bloom_layer_norm_forward
convert_forward(model,
nn.LayerNorm,
bloom_layer_norm_forward)

if model.config.architectures is not None \
and model.config.architectures[0] in ["ChatGLMModel", "ChatGLMForConditionalGeneration"]:
from ipex_llm.transformers.models.llama import llama_model_forward
from ipex_llm.transformers.models.llama import llama_attention_forward_4_31
convert_forward(model, LlamaModel, llama_model_forward)
convert_forward(model, LlamaAttention, llama_attention_forward_4_31)

elif (
model.config.architectures is not None
and model.config.architectures[0] in ["ChatGLMModel", "ChatGLMForConditionalGeneration"]
):
if hasattr(model.config, 'padded_vocab_size') and \
model.config.padded_vocab_size in [65024, 64896]:
# chatglm2-6b, chatglm2-6b-32k, chatglm3-6b, chatglm3-6b-32k, chatglm3-6b-128k
Expand Down Expand Up @@ -1370,6 +1334,7 @@ def _optimize_post(model, lightweight_bmm=False):
from ipex_llm.transformers.models.qwen2_moe import qwen2moe_model_forward
from ipex_llm.transformers.models.qwen2_moe import qwen2_moe_causal_lm_forward
from ipex_llm.transformers.models.qwen2 import qwen2_attention_forward
from ipex_llm.transformers.models.qwen2 import qwen2_mlp_forward
convert_forward(model,
module.Qwen2MoeModel,
qwen2moe_model_forward)
Expand All @@ -1384,7 +1349,7 @@ def _optimize_post(model, lightweight_bmm=False):
qwen2moe_moeblock_forward)
convert_forward(model,
module.Qwen2MoeMLP,
llama_mlp_forward)
qwen2_mlp_forward)
convert_forward(model,
module.Qwen2MoeAttention,
qwen2_attention_forward)
Expand Down Expand Up @@ -1768,7 +1733,9 @@ def safe_bmm_fwd(*args, **kwargs):
model.llm.config.model_type = "minicpmv"
elif model.config.hidden_size == 4096 and model.config.vocab_size == 128256:
# MiniCPM-V 2.5
pass
model.llm.config.model_type = "llama"
_optimize_post(model.llm, lightweight_bmm=lightweight_bmm)
model.llm.config.model_type = "minicpmv"

vpm_modeling_module_name = model.vpm.__class__.__module__
vpm_module = importlib.import_module(vpm_modeling_module_name)
Expand Down
Loading