generic fetcher: Load lockfile

This commit adds functionality to load custom lockfile for the generic
fetcher.

Signed-off-by: Jan Koscielniak <[email protected]>

cachi2/core/package_managers/generic/__init__.py

@@ -0,0 +1,3 @@
+from cachi2.core.package_managers.generic.main import fetch_generic_source
+
+__all__ = ["fetch_generic_source"]

cachi2/core/package_managers/generic/main.py

@@ -0,0 +1,81 @@
+import logging
+
+import yaml
+from pydantic import ValidationError
+
+from cachi2.core.errors import PackageManagerError, PackageRejected
+from cachi2.core.models.input import Request
+from cachi2.core.models.output import RequestOutput
+from cachi2.core.models.sbom import Component
+from cachi2.core.package_managers.generic.models import GenericLockfileV1
+from cachi2.core.rooted_path import RootedPath
+
+log = logging.getLogger(__name__)
+DEFAULT_LOCKFILE_NAME = "generic_lockfile.yaml"
+DEFAULT_DEPS_DIR = "deps/generic"
+
+
+def fetch_generic_source(request: Request) -> RequestOutput:
+    """
+    Resolve and fetch generic dependencies for a given request.
+
+    :param request: the request to process
+    """
+    components = []
+    for package in request.generic_packages:
+        path = request.source_dir.join_within_root(package.path)
+        components.extend(_resolve_generic_lockfile(path, request.output_dir))
+    return RequestOutput.from_obj_list(components=components)
+
+
+def _resolve_generic_lockfile(source_dir: RootedPath, output_dir: RootedPath) -> list[Component]:
+    """
+    Resolve the generic lockfile and pre-fetch the dependencies.
+
+    :param source_dir: the source directory to resolve the lockfile from
+    :param output_dir: the output directory to store the dependencies
+    """
+    lockfile_path = source_dir.join_within_root(DEFAULT_LOCKFILE_NAME)
+    if not lockfile_path.path.exists():
+        raise PackageRejected(
+            f"Cachi2 generic lockfile '{DEFAULT_LOCKFILE_NAME}' missing, refusing to continue.",
+            solution=(
+                f"Make sure your repository has cachi2 generic lockfile '{DEFAULT_LOCKFILE_NAME}' checked in "
+                "to the repository."
+            ),
+        )
+
+    log.info(f"Reading generic lockfile: {lockfile_path}")
+    lockfile = _load_lockfile(lockfile_path)
+    for artifact in lockfile.artifacts:
+        log.debug(f"Resolving artifact: {artifact.download_url}")
+    return []
+
+
+def _load_lockfile(lockfile_path: RootedPath) -> GenericLockfileV1:
+    """
+    Load the cachi2 generic lockfile from the given path.
+
+    :param lockfile_path: the path to the lockfile
+    """
+    with open(lockfile_path, "r") as f:
+        try:
+            lockfile_data = yaml.safe_load(f)
+        except yaml.YAMLError as e:
+            raise PackageRejected(
+                f"Cachi2 lockfile '{lockfile_path}' yaml format is not correct: {e}",
+                solution="Check correct 'yaml' syntax in the lockfile.",
+            )
+
+        try:
+            lockfile = GenericLockfileV1.model_validate(lockfile_data)
+        except ValidationError as e:
+            loc = e.errors()[0]["loc"]
+            msg = e.errors()[0]["msg"]
+            raise PackageManagerError(
+                f"Cachi2 lockfile '{lockfile_path}' format is not valid: '{loc}: {msg}'",
+                solution=(
+                    "Check the correct format and whether any keys are missing in the lockfile."
+                ),
+            )
+    return lockfile

tests/unit/package_managers/test_generic.py

@@ -1,26 +1,62 @@
+from typing import Type
 from unittest import mock
 
 import pytest
 
-from cachi2.core.errors import PackageRejected
+from cachi2.core.errors import Cachi2Error, PackageManagerError, PackageRejected
 from cachi2.core.models.input import GenericPackageInput
 from cachi2.core.models.sbom import Component
-from cachi2.core.package_managers.generic import (
+from cachi2.core.package_managers.generic.main import (
     DEFAULT_LOCKFILE_NAME,
+    _load_lockfile,
     _resolve_generic_lockfile,
     fetch_generic_source,
 )
+from cachi2.core.package_managers.generic.models import GenericLockfileV1
 from cachi2.core.rooted_path import RootedPath
 
+LOCKFILE_WRONG_VERSION = """
+metadata:
+    version: '0.42'
+artifacts:
+    - download_url: https://example.com/artifact
+      checksums:
+        md5: 3a18656e1cea70504b905836dee14db0
+"""
+
+LOCKFILE_CHECKSUM_MISSING = """
+metadata:
+    version: '1.0'
+artifacts:
+    - download_url: https://example.com/artifact
+"""
+
+LOCKFILE_CHECKSUM_EMPTY = """
+metadata:
+    version: '1.0'
+artifacts:
+    - download_url: https://example.com/artifact
+      checksums: {}
+"""
+
+LOCKFILE_VALID = """
+metadata:
+    version: '1.0'
+artifacts:
+    - download_url: https://example.com/artifact
+      checksums:
+        md5: 3a18656e1cea70504b905836dee14db0
+"""
+
 
 @pytest.mark.parametrize(
     ["model_input", "components"],
     [
         pytest.param(GenericPackageInput.model_construct(type="generic"), [], id="single_input"),
     ],
 )
-@mock.patch("cachi2.core.package_managers.rpm.main.RequestOutput.from_obj_list")
-@mock.patch("cachi2.core.package_managers.generic._resolve_generic_lockfile")
+@mock.patch("cachi2.core.package_managers.generic.main.RequestOutput.from_obj_list")
+@mock.patch("cachi2.core.package_managers.generic.main._resolve_generic_lockfile")
 def test_fetch_generic_source(
     mock_resolve_generic_lockfile: mock.Mock,
     mock_from_obj_list: mock.Mock,
@@ -39,10 +75,79 @@ def test_fetch_generic_source(
     mock_from_obj_list.assert_called_with(components=components)
 
 
-def test_resolve_generic_no_lockfile(rooted_tmp_path: RootedPath) -> None:
+@mock.patch("cachi2.core.package_managers.generic.main._load_lockfile")
+def test_resolve_generic_no_lockfile(mock_load: mock.Mock, rooted_tmp_path: RootedPath) -> None:
     with pytest.raises(PackageRejected) as exc_info:
         _resolve_generic_lockfile(rooted_tmp_path, rooted_tmp_path)
     assert (
         f"Cachi2 generic lockfile '{DEFAULT_LOCKFILE_NAME}' missing, refusing to continue"
         in str(exc_info.value)
     )
+    mock_load.assert_not_called()
+
+
+@pytest.mark.parametrize(
+    ["lockfile", "expected_exception", "expected_err"],
+    [
+        pytest.param("{", PackageRejected, "yaml format is not correct", id="invalid_yaml"),
+        pytest.param(
+            LOCKFILE_WRONG_VERSION, PackageManagerError, "Input should be '1.0'", id="wrong_version"
+        ),
+        pytest.param(
+            LOCKFILE_CHECKSUM_MISSING, PackageManagerError, "Field required", id="checksum_missing"
+        ),
+        pytest.param(
+            LOCKFILE_CHECKSUM_EMPTY,
+            PackageManagerError,
+            "At least one checksum must be provided",
+            id="checksum_empty",
+        ),
+    ],
+)
+def test_resolve_generic_lockfile_invalid(
+    lockfile: str,
+    expected_exception: Type[Cachi2Error],
+    expected_err: str,
+    rooted_tmp_path: RootedPath,
+) -> None:
+    # setup lockfile
+    with open(rooted_tmp_path.join_within_root(DEFAULT_LOCKFILE_NAME), "w") as f:
+        f.write(lockfile)
+
+    with pytest.raises(expected_exception) as exc_info:
+        _resolve_generic_lockfile(rooted_tmp_path, rooted_tmp_path)
+
+    assert expected_err in str(exc_info.value)
+
+
+@pytest.mark.parametrize(
+    ["lockfile", "expected_lockfile"],
+    [
+        pytest.param(
+            LOCKFILE_VALID,
+            GenericLockfileV1.model_validate(
+                {
+                    "metadata": {"version": "1.0"},
+                    "artifacts": [
+                        {
+                            "download_url": "https://example.com/artifact",
+                            "checksums": {"md5": "3a18656e1cea70504b905836dee14db0"},
+                        }
+                    ],
+                }
+            ),
+        ),
+    ],
+)
+def test_resolve_generic_lockfile_valid(
+    lockfile: str,
+    expected_lockfile: GenericLockfileV1,
+    rooted_tmp_path: RootedPath,
+) -> None:
+    # setup lockfile
+    with open(rooted_tmp_path.join_within_root(DEFAULT_LOCKFILE_NAME), "w") as f:
+        f.write(lockfile)
+
+    assert (
+        _load_lockfile(rooted_tmp_path.join_within_root(DEFAULT_LOCKFILE_NAME)) == expected_lockfile
+    )