yarn_v1: Automated Yarn version detector

In order to simplify user interface some logic is added to automatically distinguish between supported Yarn versions. Addresses: #624 Signed-off-by: Alexey Ovchinnikov <[email protected]>
containerbuildsystem · Dec 16, 2024 · ceb2583 · ceb2583
1 parent 4739b4b
commit ceb2583
Show file tree

Hide file tree

Showing 9 changed files with 250 additions and 42 deletions.
diff --git a/cachi2/core/config.py b/cachi2/core/config.py
@@ -25,6 +25,8 @@ class Config(BaseModel, extra="forbid"):
     requests_timeout: int = 300
     concurrency_limit: int = 5
 
+    allow_yarnberry_processing: bool = True
+
     @model_validator(mode="before")
     @classmethod
     def _print_deprecation_warning(cls, data: Any) -> Any:

diff --git a/cachi2/core/package_managers/metayarn.py b/cachi2/core/package_managers/metayarn.py
@@ -0,0 +1,28 @@
+from cachi2.core.config import get_config
+from cachi2.core.models.input import Request
+from cachi2.core.models.output import RequestOutput
+from cachi2.core.package_managers.yarn.main import fetch_yarn_source as fetch_yarnberry_source
+from cachi2.core.package_managers.yarn_classic.main import MissingLockfile, NotV1Lockfile
+from cachi2.core.package_managers.yarn_classic.main import (
+    fetch_yarn_source as fetch_yarn_classic_source,
+)
+from cachi2.core.utils import merge_outputs
+
+
+def fetch_yarn_source(request: Request) -> RequestOutput:
+    """Fetch yarn source."""
+    # Packages could be a mixture of yarn v1 and v2 (at least this is how it
+    # looks now). To preserve this behavior each request is split into individual
+    # packages which are assessed one by one.
+    fetched_packages = []
+    for package in request.packages:
+        new_request = request.model_copy(update={"packages": [package]})
+        try:
+            fetched_packages.append(fetch_yarn_classic_source(new_request))
+        except (MissingLockfile, NotV1Lockfile) as e:
+            # It is assumed that if a package is not v1 then it is probably v2.
+            if get_config().allow_yarnberry_processing:
+                fetched_packages.append(fetch_yarnberry_source(new_request))
+            else:
+                raise e
+    return merge_outputs(fetched_packages)
diff --git a/cachi2/core/package_managers/yarn_classic/main.py b/cachi2/core/package_managers/yarn_classic/main.py
@@ -2,7 +2,7 @@
 import re
 from collections import Counter
 from pathlib import Path
-from typing import Iterable
+from typing import Any, Iterable
 from urllib.parse import urlparse
 
 from cachi2.core.errors import PackageManagerError, PackageRejected
@@ -27,6 +27,26 @@
 
 
 MIRROR_DIR = "deps/yarn-classic"
+_yarn_classic_pattern = "yarn lockfile v1"  # See [yarn_classic_trait].
+
+
+class MissingLockfile(PackageRejected):
+    """Indicate that a lock file is missing."""
+
+    def __init__(self) -> None:
+        """Initialize a Missing Lockfile error."""
+        reason = "Yarn lockfile 'yarn.lock' missing, refusing to continue"
+        solution = "Make sure your repository has a Yarn lockfile (i.e. yarn.lock) checked in"
+        super().__init__(reason, solution=solution)
+
+
+class NotV1Lockfile(PackageRejected):
+    """Indicate that a lockfile is of wrong tyrpoe."""
+
+    def __init__(self, package_path: Any) -> None:
+        """Initialize a Missing Lockfile error."""
+        reason = f"{package_path} not a Yarn v1"
+        super().__init__(reason, solution=None)
 
 
 def fetch_yarn_source(request: Request) -> RequestOutput:
@@ -36,7 +56,7 @@ def fetch_yarn_source(request: Request) -> RequestOutput:
     def _ensure_mirror_dir_exists(output_dir: RootedPath) -> None:
         output_dir.join_within_root(MIRROR_DIR).path.mkdir(parents=True, exist_ok=True)
 
-    for package in request.yarn_classic_packages:
+    for package in request.yarn_packages:
         package_path = request.source_dir.join_within_root(package.path)
         _ensure_mirror_dir_exists(request.output_dir)
         _resolve_yarn_project(Project.from_source_dir(package_path), request.output_dir)
@@ -121,9 +141,27 @@ def _reject_if_pnp_install(project: Project) -> None:
         )
 
 
+def _get_path_to_yarn_lock(project: Project) -> Path:
+    return project.source_dir.join_within_root("yarn.lock").path
+
+
+def _reject_if_wrong_lockfile_version(project: Project) -> None:
+    yarnlock_path = _get_path_to_yarn_lock(project)
+    text = yarnlock_path.read_text()
+    if _yarn_classic_pattern not in text:
+        raise NotV1Lockfile(project.source_dir)
+
+
+def _reject_if_lockfile_is_missing(project: Project) -> None:
+    yarnlock_path = _get_path_to_yarn_lock(project)
+    if not yarnlock_path.exists():
+        raise MissingLockfile()
+
+
 def _verify_repository(project: Project) -> None:
+    _reject_if_lockfile_is_missing(project)
+    _reject_if_wrong_lockfile_version(project)
     _reject_if_pnp_install(project)
-    # _check_lockfile(project)
 
 
 def _verify_corepack_yarn_version(source_dir: RootedPath, env: dict[str, str]) -> None:
@@ -194,3 +232,7 @@ def _get_git_tarball_mirror_name(url: str) -> str:
         package_filename = package_filename[1:]
 
     return package_filename
+
+
+# References
+# [yarn_classic_trait]:  https://github.com/yarnpkg/berry/blob/13d5b3041794c33171808fdce635461ff4ab5c4e/packages/yarnpkg-core/sources/Project.ts#L434
diff --git a/cachi2/core/resolver.py b/cachi2/core/resolver.py
@@ -1,14 +1,13 @@
-from collections.abc import Iterable
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any, Callable
 
 from cachi2.core.errors import UnsupportedFeature
 from cachi2.core.models.input import PackageManagerType, Request
 from cachi2.core.models.output import RequestOutput
-from cachi2.core.package_managers import bundler, generic, gomod, npm, pip, rpm, yarn, yarn_classic
+from cachi2.core.package_managers import bundler, generic, gomod, metayarn, npm, pip, rpm
 from cachi2.core.rooted_path import RootedPath
-from cachi2.core.utils import copy_directory
+from cachi2.core.utils import copy_directory, merge_outputs
 
 Handler = Callable[[Request], RequestOutput]
 
@@ -17,15 +16,14 @@
     "gomod": gomod.fetch_gomod_source,
     "npm": npm.fetch_npm_source,
     "pip": pip.fetch_pip_source,
-    "yarn": yarn.fetch_yarn_source,
+    "yarn": metayarn.fetch_yarn_source,
     "generic": generic.fetch_generic_source,
 }
 
 # This is where we put package managers currently under development in order to
 # invoke them via CLI
 _dev_package_managers: dict[PackageManagerType, Handler] = {
     "rpm": rpm.fetch_rpm_source,
-    "yarn-classic": yarn_classic.fetch_yarn_source,
 }
 
 # This is *only* used to provide a list for `cachi2 --version`
@@ -68,26 +66,7 @@ def _resolve_packages(request: Request) -> RequestOutput:
             solution="But the good news is that we're already working on it!",
         )
     pkg_managers = [_supported_package_managers[type_] for type_ in sorted(requested_types)]
-    return _merge_outputs(pkg_manager(request) for pkg_manager in pkg_managers)
-
-
-def _merge_outputs(outputs: Iterable[RequestOutput]) -> RequestOutput:
-    """Merge RequestOutput instances."""
-    components = []
-    env_vars = []
-    project_files = []
-
-    for output in outputs:
-        components.extend(output.components)
-        env_vars.extend(output.build_config.environment_variables)
-        project_files.extend(output.build_config.project_files)
-
-    return RequestOutput.from_obj_list(
-        components=components,
-        environment_variables=env_vars,
-        project_files=project_files,
-        options=output.build_config.options if output.build_config.options else None,
-    )
+    return merge_outputs(pkg_manager(request) for pkg_manager in pkg_managers)
 
 
 def inject_files_post(from_output_dir: Path, for_output_dir: Path, **kwargs: Any) -> None:

diff --git a/cachi2/core/utils.py b/cachi2/core/utils.py
@@ -8,10 +8,11 @@
 import sys
 from functools import cache
 from pathlib import Path
-from typing import Callable, Iterator, Optional, Sequence
+from typing import Callable, Iterable, Iterator, Optional, Sequence
 
 from cachi2.core.config import get_config
 from cachi2.core.errors import Cachi2Error
+from cachi2.core.models.output import RequestOutput
 
 log = logging.getLogger(__name__)
 
@@ -195,3 +196,22 @@ def get_cache_dir() -> Path:
     except KeyError:
         cache_dir = Path.home().joinpath(".cache")
     return cache_dir.joinpath("cachi2")
+
+
+def merge_outputs(outputs: Iterable[RequestOutput]) -> RequestOutput:
+    """Merge RequestOutput instances."""
+    components = []
+    env_vars = []
+    project_files = []
+
+    for output in outputs:
+        components.extend(output.components)
+        env_vars.extend(output.build_config.environment_variables)
+        project_files.extend(output.build_config.project_files)
+
+    return RequestOutput.from_obj_list(
+        components=components,
+        environment_variables=env_vars,
+        project_files=project_files,
+        options=output.build_config.options if output.build_config.options else None,
+    )
diff --git a/tests/integration/test_yarn_classic.py b/tests/integration/test_yarn_classic.py
@@ -15,7 +15,7 @@
             utils.TestParameters(
                 repo="https://github.com/cachito-testing/cachi2-yarn.git",
                 ref="corepack_packagemanager_ignored",
-                packages=({"path": ".", "type": "yarn-classic"},),
+                packages=({"path": ".", "type": "yarn"},),
                 flags=["--dev-package-managers"],
                 check_output=False,
                 check_deps_checksums=False,
@@ -29,7 +29,7 @@
             utils.TestParameters(
                 repo="https://github.com/cachito-testing/cachi2-yarn.git",
                 ref="yarnpath_ignored",
-                packages=({"path": ".", "type": "yarn-classic"},),
+                packages=({"path": ".", "type": "yarn"},),
                 flags=["--dev-package-managers"],
                 check_output=False,
                 check_deps_checksums=False,
@@ -43,7 +43,7 @@
             utils.TestParameters(
                 repo="https://github.com/cachito-testing/cachi2-yarn.git",
                 ref="invalid_checksum",
-                packages=({"path": ".", "type": "yarn-classic"},),
+                packages=({"path": ".", "type": "yarn"},),
                 flags=["--dev-package-managers"],
                 check_output=False,
                 check_deps_checksums=False,
@@ -57,7 +57,7 @@
             utils.TestParameters(
                 repo="https://github.com/cachito-testing/cachi2-yarn.git",
                 ref="invalid_frozen_lockfile_add_dependency",
-                packages=({"path": ".", "type": "yarn-classic"},),
+                packages=({"path": ".", "type": "yarn"},),
                 flags=["--dev-package-managers"],
                 check_output=False,
                 check_deps_checksums=False,
@@ -71,7 +71,7 @@
             utils.TestParameters(
                 repo="https://github.com/cachito-testing/cachi2-yarn.git",
                 ref="lifecycle_scripts",
-                packages=({"path": ".", "type": "yarn-classic"},),
+                packages=({"path": ".", "type": "yarn"},),
                 flags=["--dev-package-managers"],
                 check_output=False,
                 check_deps_checksums=False,
@@ -85,7 +85,7 @@
             utils.TestParameters(
                 repo="https://github.com/cachito-testing/cachi2-yarn.git",
                 ref="offline-mirror-collision",
-                packages=({"path": ".", "type": "yarn-classic"},),
+                packages=({"path": ".", "type": "yarn"},),
                 flags=["--dev-package-managers"],
                 check_output=False,
                 check_deps_checksums=False,
@@ -128,8 +128,7 @@ def test_yarn_classic_packages(
             utils.TestParameters(
                 repo="https://github.com/cachito-testing/cachi2-yarn.git",
                 ref="valid_yarn_all_dependency_types",
-                packages=({"path": ".", "type": "yarn-classic"},),
-                flags=["--dev-package-managers"],
+                packages=({"path": ".", "type": "yarn"},),
                 check_vendor_checksums=False,
                 expected_exit_code=0,
                 expected_output="All dependencies fetched successfully",
@@ -143,10 +142,9 @@ def test_yarn_classic_packages(
                 repo="https://github.com/cachito-testing/cachi2-yarn.git",
                 ref="valid_multiple_packages",
                 packages=(
-                    {"path": "first-pkg", "type": "yarn-classic"},
-                    {"path": "second-pkg", "type": "yarn-classic"},
+                    {"path": "first-pkg", "type": "yarn"},
+                    {"path": "second-pkg", "type": "yarn"},
                 ),
-                flags=["--dev-package-managers"],
                 check_vendor_checksums=False,
                 expected_exit_code=0,
                 expected_output="All dependencies fetched successfully",