Fix dependency problems with basicsr

core/install_requirements.py

@@ -20,20 +20,29 @@
     "HyperTile": "hyper-tile",
     "stable-fast": "sfast",
     "sentry-sdk[fastapi]": "sentry_sdk",
+    "pyyaml": "yaml",
+    "tb-nightly": "tensorboard",
+    "basicsr-volta": "basicsr",
 }
+importlib_metadata_overwrite = {"basicsr": "basicsr-volta"}
 logger = logging.getLogger(__name__)
 
 
 class NoModuleSpecFound(Exception):
     "Exception raised when no module spec is found"
 
 
+class ImportErrorNoDeps(Exception):
+    "Import can't find module, or can't find name in module - shall not install dependencies"
+
+
 def install_requirements(path_to_requirements: str = "requirements.txt"):
     "Check if requirements are installed, if not, install them"
 
     with open(path_to_requirements, encoding="utf-8", mode="r") as f:
         requirements = {}
-        for line in [r.strip() for r in f.read().splitlines()]:
+        do_deps_requirements = {}
+        for line in [x for x in [r.strip() for r in f.read().splitlines()] if x != ""]:
             split = line.split(";")
             i: str = split[0].strip()
 
@@ -42,10 +51,22 @@ def install_requirements(path_to_requirements: str = "requirements.txt"):
             else:
                 check = ""
 
+            # OS specific checks
             if check == 'platform_system == "Linux"':
                 logger.debug("Install check for Linux only")
                 if platform.system() != "Linux":
                     continue
+            elif check == 'platform_system == "Windows"':
+                logger.debug("Install check for Windows only")
+                if platform.system() != "Windows":
+                    continue
+            elif check == 'platform_system == "Darwin"':
+                logger.debug("Install check for MacOS only")
+                if platform.system() != "Darwin":
+                    continue
+
+            # No dependencies
+            target = requirements if check != "no_deps" else do_deps_requirements
 
             if "git+http" in i:
                 tmp = i.split("@")[0].split("/")[-1].replace(".git", "").strip()
@@ -58,13 +79,19 @@ def install_requirements(path_to_requirements: str = "requirements.txt"):
                 i = tmp
 
             if "==" in i:
-                requirements[i.split("==")[0]] = i.replace(i.split("==")[0], "").strip()
+                target[i.split("==")[0]] = (
+                    i.replace(i.split("==")[0], "").replace("#", "").strip()
+                )
             elif ">=" in i:
-                requirements[i.split(">=")[0]] = i.replace(i.split(">=")[0], "").strip()
+                target[i.split(">=")[0]] = (
+                    i.replace(i.split(">=")[0], "").replace("#", "").strip()
+                )
             elif "<=" in i:
-                requirements[i.split("<=")[0]] = i.replace(i.split("<=")[0], "").strip()
+                target[i.split("<=")[0]] = (
+                    i.replace(i.split("<=")[0], "").replace("#", "").strip()
+                )
             else:
-                requirements[i] = None
+                target[i] = None
 
         try:
             for requirement in requirements:
@@ -86,6 +113,25 @@ def install_requirements(path_to_requirements: str = "requirements.txt"):
                     logger.debug(f"Requirement {requirement_name} is not installed")
                     raise ImportError
 
+            for requirement in do_deps_requirements:
+                # Skip extra commands for pip and comments
+                if requirement.startswith("#") or requirement.startswith("--"):
+                    continue
+
+                if requirement in renamed_requirements:
+                    logger.debug(
+                        f"Requirement {requirement} is renamed to {renamed_requirements[requirement]}"
+                    )
+                    requirement_name = renamed_requirements[requirement]
+                else:
+                    requirement_name = requirement
+
+                logger.debug(f"Checking requirement: {requirement}")
+                fixed_name = requirement_name.replace("-", "_").lower()
+                if not is_installed(fixed_name, do_deps_requirements[requirement]):
+                    logger.debug(f"Requirement {requirement_name} is not installed")
+                    raise ImportErrorNoDeps
+
         except ImportError as e:
             logger.debug(
                 f"Installing requirements: {path_to_requirements}, because: {e} ({e.__class__.__name__})"
@@ -105,6 +151,26 @@ def install_requirements(path_to_requirements: str = "requirements.txt"):
                 logger.error(f"Failed to install requirements: {path_to_requirements}")
                 sys.exit(1)
 
+        except ImportErrorNoDeps as e:
+            logger.debug(
+                f"Installing requirements: {path_to_requirements}, because: {e} ({e.__class__.__name__})"
+            )
+            try:
+                subprocess.check_call(
+                    [
+                        sys.executable,
+                        "-m",
+                        "pip",
+                        "install",
+                        "--no-deps",
+                        "-r",
+                        path_to_requirements,
+                    ]
+                )
+            except subprocess.CalledProcessError:
+                logger.error(f"Failed to install requirements: {path_to_requirements}")
+                sys.exit(1)
+
 
 @dataclass
 class PytorchDistribution:
@@ -321,9 +387,13 @@ def is_installed(package: str, version: Optional[str] = None):
                 version_number = version.split("=")[-1]
                 version_type = version[:2]
                 required_version = packaging_version.parse(version_number)
-                current_version = packaging_version.parse(
-                    importlib.metadata.version(package)
+
+                importlib_found_version = importlib.metadata.version(
+                    package
+                    if package not in importlib_metadata_overwrite
+                    else importlib_metadata_overwrite[package]
                 )
+                current_version = packaging_version.parse(importlib_found_version)
                 logger.debug(
                     f"Required version: {required_version} - Current version: {current_version} - version type: {version_type}"
                 )

docker/ait-no-mount.docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - EXTRA_ARGS=${EXTRA_ARGS:-}
 
       # Extra api keys
-      - FASTAPI_ANALYTICS_KEY=${FASTAPI_ANALYTICS_KEY:-}
+      - SENTRY_DSN=${SENTRY_DSN:-}
       - DISCORD_BOT_TOKEN=${DISCORD_BOT_TOKEN:-}
 
       # R2

docker/ait.docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - EXTRA_ARGS=${EXTRA_ARGS:-}
 
       # Extra api keys
-      - FASTAPI_ANALYTICS_KEY=${FASTAPI_ANALYTICS_KEY:-}
+      - SENTRY_DSN=${SENTRY_DSN:-}
       - DISCORD_BOT_TOKEN=${DISCORD_BOT_TOKEN:-}
 
       # R2

docker/ait/dockerfile

@@ -31,6 +31,9 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install -r requirem
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install -r requirements/pytorch.txt
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install -r requirements/interrogation.txt
 
+# Overwrite the default BasicSR package with our fixed version
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install --force-reinstall --no-deps basicsr-volta==1.4.4
+
 COPY . /app
 
 # Install frontend dependencies and build the frontend

docker/cuda-no-mount.docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - EXTRA_ARGS=${EXTRA_ARGS:-}
 
       # Extra api keys
-      - FASTAPI_ANALYTICS_KEY=${FASTAPI_ANALYTICS_KEY:-}
+      - SENTRY_DSN=${SENTRY_DSN:-}
       - DISCORD_BOT_TOKEN=${DISCORD_BOT_TOKEN:-}
 
       # R2

docker/cuda.docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - EXTRA_ARGS=${EXTRA_ARGS:-}
 
       # Extra api keys
-      - FASTAPI_ANALYTICS_KEY=${FASTAPI_ANALYTICS_KEY:-}
+      - SENTRY_DSN=${SENTRY_DSN:-}
       - DISCORD_BOT_TOKEN=${DISCORD_BOT_TOKEN:-}
 
       # R2

docker/cuda/dockerfile

@@ -31,6 +31,9 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install -r requirem
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install -r requirements/pytorch.txt
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install -r requirements/interrogation.txt
 
+# Overwrite the default BasicSR package with our fixed version
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install --force-reinstall --no-deps basicsr-volta==1.4.4
+
 COPY . /app
 
 # Install frontend dependencies and build the frontend

requirements/pytorch.txt

@@ -14,7 +14,6 @@ pytorch-lightning==2.1.1
 gpustat==1.1.1
 opencv-contrib-python-headless==4.7.0.72
 controlnet-aux==0.0.7
-realesrgan==0.3.0
 timm==0.9.10
 boto3==1.28.83
 tomesd==0.1.3
@@ -25,3 +24,16 @@ piexif==1.1.3
 git+https://github.com/tfernd/HyperTile@2ef64b2800d007d305755c33550537410310d7df # 0.1.5
 k-diffusion==0.1.1
 asdff==0.2.1
+
+# RealESRGAN workaround
+realesrgan==0.3.0 #; no_deps
+gfpgan==1.3.8 #; no_deps
+facexlib==0.3.0
+lmdb==1.4.1
+pyyaml
+scipy
+numpy
+tb-nightly
+tqdm
+yapf
+basicsr-volta==1.4.4