From bc274a28a9a72c5fe98b070e2083fe689c8fd5e9 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Fri, 26 Apr 2024 00:23:39 +0800 Subject: [PATCH] Do not use deprecated `SourceFileLoader.load_module()` in dynamic module loading (#30370) --- src/transformers/dynamic_module_utils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/transformers/dynamic_module_utils.py b/src/transformers/dynamic_module_utils.py index a89105029868a1..f9abade2809166 100644 --- a/src/transformers/dynamic_module_utils.py +++ b/src/transformers/dynamic_module_utils.py @@ -15,6 +15,7 @@ """Utilities to dynamically load objects from the Hub.""" import filecmp import importlib +import importlib.util import os import re import shutil @@ -196,9 +197,15 @@ def get_class_in_module(class_name: str, module_path: Union[str, os.PathLike]) - Returns: `typing.Type`: The class looked for. """ - name = os.path.normpath(module_path).replace(".py", "").replace(os.path.sep, ".") - module_path = str(Path(HF_MODULES_CACHE) / module_path) - module = importlib.machinery.SourceFileLoader(name, module_path).load_module() + name = os.path.normpath(module_path).rstrip(".py").replace(os.path.sep, ".") + module_spec = importlib.util.spec_from_file_location(name, location=Path(HF_MODULES_CACHE) / module_path) + module = sys.modules.get(name) + if module is None: + module = importlib.util.module_from_spec(module_spec) + # insert it into sys.modules before any loading begins + sys.modules[name] = module + # reload in both cases + module_spec.loader.exec_module(module) return getattr(module, class_name)