From 2f71515cb0624f9686b488935f482d423a60b274 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 27 Jun 2024 17:07:04 -0700 Subject: [PATCH 1/4] Make dependabot target the dev branch --- .github/dependabot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 91abb11fdf..93aaf445f0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,5 +7,6 @@ version: 2 updates: - package-ecosystem: "pip" # See documentation for possible values directory: "/" # Location of package manifests + target-branch: "dev" schedule: interval: "weekly" From d9476d6a7332c4cd857070e334fbf6da1666a741 Mon Sep 17 00:00:00 2001 From: Invectorgator Date: Fri, 12 Jul 2024 13:06:45 -0400 Subject: [PATCH 2/4] Fix for MacOS users - 'already loaded' error on model load --- modules/llama_cpp_python_hijack.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/modules/llama_cpp_python_hijack.py b/modules/llama_cpp_python_hijack.py index 6ec1562629..2812dbffae 100644 --- a/modules/llama_cpp_python_hijack.py +++ b/modules/llama_cpp_python_hijack.py @@ -43,21 +43,23 @@ def module_to_purpose(module_name): except: pass - if return_lib is None: - if imported_module and imported_module != 'llama_cpp_cuda': - raise Exception(f"The {module_to_purpose(imported_module)} version of llama-cpp-python is already loaded. Switching to the default version currently requires a server restart.") + if return_lib is None and not shared.args.cpu and imported_module != 'llama_cpp_cuda': + if imported_module and imported_module != 'llama_cpp': + raise Exception( + f"The {module_to_purpose(imported_module)} version of llama-cpp-python is already loaded. Switching to the CPU version currently requires a server restart.") try: - return_lib = importlib.import_module('llama_cpp_cuda') - imported_module = 'llama_cpp_cuda' + return_lib = importlib.import_module('llama_cpp') + imported_module = 'llama_cpp' except: pass - if return_lib is None and not shared.args.cpu: - if imported_module and imported_module != 'llama_cpp': - raise Exception(f"The {module_to_purpose(imported_module)} version of llama-cpp-python is already loaded. Switching to the CPU version currently requires a server restart.") + if return_lib is None: + if imported_module and imported_module != 'llama_cpp_cuda': + raise Exception( + f"The {module_to_purpose(imported_module)} version of llama-cpp-python is already loaded. Switching to the default version currently requires a server restart.") try: - return_lib = importlib.import_module('llama_cpp') - imported_module = 'llama_cpp' + return_lib = importlib.import_module('llama_cpp_cuda') + imported_module = 'llama_cpp_cuda' except: pass From 526bda8acb4c973ca8aa179e8e064a5e44a8b943 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 12 Jul 2024 20:00:29 -0700 Subject: [PATCH 3/4] Alternative solution --- modules/llama_cpp_python_hijack.py | 87 ++++++++++++------------------ 1 file changed, 33 insertions(+), 54 deletions(-) diff --git a/modules/llama_cpp_python_hijack.py b/modules/llama_cpp_python_hijack.py index 2812dbffae..2c2be45066 100644 --- a/modules/llama_cpp_python_hijack.py +++ b/modules/llama_cpp_python_hijack.py @@ -1,4 +1,5 @@ import importlib +import platform from typing import Sequence from tqdm import tqdm @@ -13,60 +14,38 @@ def llama_cpp_lib(): global imported_module - def module_to_purpose(module_name): - if module_name == 'llama_cpp': - return 'CPU' - elif module_name == 'llama_cpp_cuda_tensorcores': - return 'tensorcores' - elif module_name == 'llama_cpp_cuda': - return 'default' - - return 'unknown' - - return_lib = None - - if shared.args.cpu: - if imported_module and imported_module != 'llama_cpp': - raise Exception(f"The {module_to_purpose(imported_module)} version of llama-cpp-python is already loaded. Switching to the CPU version currently requires a server restart.") - try: - return_lib = importlib.import_module('llama_cpp') - imported_module = 'llama_cpp' - except: - pass - - if shared.args.tensorcores and return_lib is None: - if imported_module and imported_module != 'llama_cpp_cuda_tensorcores': - raise Exception(f"The {module_to_purpose(imported_module)} version of llama-cpp-python is already loaded. Switching to the tensorcores version currently requires a server restart.") - try: - return_lib = importlib.import_module('llama_cpp_cuda_tensorcores') - imported_module = 'llama_cpp_cuda_tensorcores' - except: - pass - - if return_lib is None and not shared.args.cpu and imported_module != 'llama_cpp_cuda': - if imported_module and imported_module != 'llama_cpp': - raise Exception( - f"The {module_to_purpose(imported_module)} version of llama-cpp-python is already loaded. Switching to the CPU version currently requires a server restart.") - try: - return_lib = importlib.import_module('llama_cpp') - imported_module = 'llama_cpp' - except: - pass - - if return_lib is None: - if imported_module and imported_module != 'llama_cpp_cuda': - raise Exception( - f"The {module_to_purpose(imported_module)} version of llama-cpp-python is already loaded. Switching to the default version currently requires a server restart.") - try: - return_lib = importlib.import_module('llama_cpp_cuda') - imported_module = 'llama_cpp_cuda' - except: - pass - - if return_lib is not None: - monkey_patch_llama_cpp_python(return_lib) - - return return_lib + # Determine the platform + is_macos = platform.system() == 'Darwin' + + # Define the library names based on the platform + if is_macos: + lib_names = [ + (None, 'llama_cpp') + ] + else: + lib_names = [ + ('cpu', 'llama_cpp'), + ('tensorcores', 'llama_cpp_cuda_tensorcores'), + (None, 'llama_cpp_cuda'), + (None, 'llama_cpp') + ] + + for arg, lib_name in lib_names: + should_import = (arg is None or getattr(shared.args, arg)) + + if should_import: + if imported_module and imported_module != lib_name: + # Conflict detected, raise an exception + raise Exception(f"Cannot import `{lib_name}` because `{imported_module}` is already imported. Switching to a different version of llama-cpp-python currently requires a server restart.") + try: + return_lib = importlib.import_module(lib_name) + imported_module = lib_name + monkey_patch_llama_cpp_python(return_lib) + return return_lib + except ImportError: + continue + + return None def eval_with_progress(self, tokens: Sequence[int]): From f19d201e440917e95f58cb58402d9071e7996309 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 12 Jul 2024 20:01:50 -0700 Subject: [PATCH 4/4] Format --- modules/llama_cpp_python_hijack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/llama_cpp_python_hijack.py b/modules/llama_cpp_python_hijack.py index 2c2be45066..64280dc9a0 100644 --- a/modules/llama_cpp_python_hijack.py +++ b/modules/llama_cpp_python_hijack.py @@ -37,6 +37,7 @@ def llama_cpp_lib(): if imported_module and imported_module != lib_name: # Conflict detected, raise an exception raise Exception(f"Cannot import `{lib_name}` because `{imported_module}` is already imported. Switching to a different version of llama-cpp-python currently requires a server restart.") + try: return_lib = importlib.import_module(lib_name) imported_module = lib_name