diff --git a/python/llm/src/ipex_llm/utils/ipex_importer.py b/python/llm/src/ipex_llm/utils/ipex_importer.py
index 7758258d0f4..fdcd270f364 100644
--- a/python/llm/src/ipex_llm/utils/ipex_importer.py
+++ b/python/llm/src/ipex_llm/utils/ipex_importer.py
@@ -146,6 +146,12 @@ def directly_import_ipex(self):
 
         Raises ImportError and invalidInputError if failed
         """
+
+        # insert a fake module to avoid importing real `intel_extension_for_pytorch.llm`
+        # which will replace some `transformers`'s functions and bring some bugs in ipex 2.3
+        from ipex_llm.utils.modules import insert_fake_module
+        insert_fake_module("intel_extension_for_pytorch.llm", "fake module")
+
         # import ipex
         import intel_extension_for_pytorch as ipex
         if ipex is not None:
diff --git a/python/llm/src/ipex_llm/utils/modules.py b/python/llm/src/ipex_llm/utils/modules.py
new file mode 100644
index 00000000000..de379dfa8b9
--- /dev/null
+++ b/python/llm/src/ipex_llm/utils/modules.py
@@ -0,0 +1,25 @@
+#
+# Copyright 2016 The BigDL Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+from types import ModuleType
+
+
+def insert_fake_module(name, doc=None):
+    m = ModuleType(name, doc)
+    m.__file__ = __file__
+    sys.modules[name] = m
+    return m