diff --git a/piptools/_compat/__init__.py b/piptools/_compat/__init__.py index 78d29b387..27d7372f8 100644 --- a/piptools/_compat/__init__.py +++ b/piptools/_compat/__init__.py @@ -1,5 +1,10 @@ from __future__ import annotations -from .pip_compat import PIP_VERSION, create_wheel_cache, parse_requirements +from .pip_compat import ( + PIP_VERSION, + Distribution, + create_wheel_cache, + parse_requirements, +) -__all__ = ["PIP_VERSION", "parse_requirements", "create_wheel_cache"] +__all__ = ["PIP_VERSION", "Distribution", "parse_requirements", "create_wheel_cache"] diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py index cb6df94a0..700576455 100644 --- a/piptools/_compat/pip_compat.py +++ b/piptools/_compat/pip_compat.py @@ -1,11 +1,15 @@ from __future__ import annotations import optparse -from typing import Callable, Iterable, Iterator, cast +from dataclasses import dataclass +from typing import TYPE_CHECKING, Iterable, Iterator import pip from pip._internal.cache import WheelCache from pip._internal.index.package_finder import PackageFinder +from pip._internal.metadata import BaseDistribution +from pip._internal.metadata.pkg_resources import Distribution as _PkgResourcesDist +from pip._internal.models.direct_url import DirectUrl from pip._internal.network.session import PipSession from pip._internal.req import InstallRequirement from pip._internal.req import parse_requirements as _parse_requirements @@ -15,11 +19,48 @@ PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split("."))) -__all__ = [ - "dist_requires", - "uses_pkg_resources", - "Distribution", -] +# The Distribution interface has changed between pkg_resources and +# importlib.metadata, so this compat layer allows for a consistent access +# pattern. In pip 22.1, importlib.metadata became the default on Python 3.11 +# (and later), but is overridable. `select_backend` returns what's being used. +if TYPE_CHECKING: + from pip._internal.metadata.importlib import Distribution as _ImportLibDist + + +@dataclass(frozen=True) +class Distribution: + key: str + version: str + requires: Iterable[Requirement] + direct_url: DirectUrl | None + + @classmethod + def from_pip_distribution(cls, dist: BaseDistribution) -> Distribution: + # TODO: Use only the BaseDistribution protocol properties and methods + # instead of specializing by type. + if isinstance(dist, _PkgResourcesDist): + return cls._from_pkg_resources(dist) + else: + return cls._from_importlib(dist) + + @classmethod + def _from_pkg_resources(cls, dist: _PkgResourcesDist) -> Distribution: + return cls( + dist._dist.key, dist._dist.version, dist._dist.requires(), dist.direct_url + ) + + @classmethod + def _from_importlib(cls, dist: _ImportLibDist) -> Distribution: + """Mimics pkg_resources.Distribution.requires for the case of no + extras. This doesn't fulfill that API's `extras` parameter but + satisfies the needs of pip-tools.""" + reqs = (Requirement.parse(req) for req in (dist._dist.requires or ())) + requires = [ + req + for req in reqs + if not req.marker or req.marker.evaluate({"extra": None}) + ] + return cls(dist._dist.name, dist._dist.version, requires, dist.direct_url) def parse_requirements( @@ -36,46 +77,6 @@ def parse_requirements( yield install_req_from_parsed_requirement(parsed_req, isolated=isolated) -# The Distribution interface has changed between pkg_resources and -# importlib.metadata, so this compat layer allows for a consistent access -# pattern. In pip 22.1, importlib.metadata became the default on Python 3.11 -# (and later), but is overridable. `select_backend` returns what's being used. - - -def _uses_pkg_resources() -> bool: - from pip._internal.metadata import select_backend - from pip._internal.metadata.pkg_resources import Distribution as _Dist - - return select_backend().Distribution is _Dist - - -uses_pkg_resources = _uses_pkg_resources() - -if uses_pkg_resources: - from operator import methodcaller - - from pip._vendor.pkg_resources import Distribution - - dist_requires = cast( - Callable[[Distribution], Iterable[Requirement]], methodcaller("requires") - ) -else: - from pip._internal.metadata import select_backend - - Distribution = select_backend().Distribution - - def dist_requires(dist: Distribution) -> Iterable[Requirement]: - """Mimics pkg_resources.Distribution.requires for the case of no - extras. This doesn't fulfill that API's `extras` parameter but - satisfies the needs of pip-tools.""" - reqs = (Requirement.parse(req) for req in (dist.requires or ())) - return [ - req - for req in reqs - if not req.marker or req.marker.evaluate({"extra": None}) - ] - - def create_wheel_cache(cache_dir: str, format_control: str | None = None) -> WheelCache: kwargs: dict[str, str | None] = {"cache_dir": cache_dir} if PIP_VERSION[:2] <= (23, 0): diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 63e4eb816..f20a635ae 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -15,8 +15,7 @@ from pip._internal.metadata import get_environment from .. import sync -from .._compat import parse_requirements -from .._compat.pip_compat import Distribution +from .._compat import Distribution, parse_requirements from ..exceptions import PipToolsError from ..locations import CONFIG_FILE_NAME from ..logging import log @@ -303,4 +302,4 @@ def _get_installed_distributions( user_only=user_only, skip=[], ) - return [cast(Distribution, dist)._dist for dist in dists] + return [Distribution.from_pip_distribution(dist) for dist in dists] diff --git a/piptools/sync.py b/piptools/sync.py index 184d78a1b..2dd57a976 100644 --- a/piptools/sync.py +++ b/piptools/sync.py @@ -9,10 +9,15 @@ import click from pip._internal.commands.freeze import DEV_PKGS +from pip._internal.models.direct_url import ArchiveInfo from pip._internal.req import InstallRequirement from pip._internal.utils.compat import stdlib_pkgs +from pip._internal.utils.direct_url_helpers import ( + direct_url_as_pep440_direct_reference, + direct_url_from_link, +) -from ._compat.pip_compat import Distribution, dist_requires +from ._compat import Distribution from .exceptions import IncompatibleRequirements from .logging import log from .utils import ( @@ -55,13 +60,13 @@ def dependency_tree( while queue: v = queue.popleft() - key = key_from_req(v) + key = v.key if key in dependencies: continue dependencies.add(key) - for dep_specifier in dist_requires(v): + for dep_specifier in v.requires: dep_name = key_from_req(dep_specifier) if dep_name in installed_keys: dep = installed_keys[dep_name] @@ -81,7 +86,7 @@ def get_dists_to_ignore(installed: Iterable[Distribution]) -> list[str]: locally, click should also be installed/uninstalled depending on the given requirements. """ - installed_keys = {key_from_req(r): r for r in installed} + installed_keys = {r.key: r for r in installed} return list( flat_map(lambda req: dependency_tree(installed_keys, req), PACKAGES_TO_IGNORE) ) @@ -120,22 +125,36 @@ def diff_key_from_ireq(ireq: InstallRequirement) -> str: """ Calculate a key for comparing a compiled requirement with installed modules. For URL requirements, only provide a useful key if the url includes - #egg=name==version, which will set ireq.req.name and ireq.specifier. + a hash, e.g. #sha1=..., in any of the supported hash algorithms. Otherwise return ireq.link so the key will not match and the package will reinstall. Reinstall is necessary to ensure that packages will reinstall - if the URL is changed but the version is not. + if the contents at the URL have changed but the version has not. """ if is_url_requirement(ireq): - if ( - ireq.req - and (getattr(ireq.req, "key", None) or getattr(ireq.req, "name", None)) - and ireq.specifier - ): - return key_from_ireq(ireq) + if getattr(ireq.req, "name", None) and ireq.link.has_hash: + return str( + direct_url_as_pep440_direct_reference( + direct_url_from_link(ireq.link), ireq.req.name + ) + ) + # TODO: Also support VCS and editable installs. return str(ireq.link) return key_from_ireq(ireq) +def diff_key_from_req(req: Distribution) -> str: + """Get a unique key for the requirement.""" + key = req.key + if ( + req.direct_url + and isinstance(req.direct_url.info, ArchiveInfo) + and req.direct_url.info.hash + ): + key = direct_url_as_pep440_direct_reference(req.direct_url, key) + # TODO: Also support VCS and editable installs. + return key + + def diff( compiled_requirements: Iterable[InstallRequirement], installed_dists: Iterable[Distribution], @@ -152,7 +171,7 @@ def diff( pkgs_to_ignore = get_dists_to_ignore(installed_dists) for dist in installed_dists: - key = key_from_req(dist) + key = diff_key_from_req(dist) if key not in requirements_lut or not requirements_lut[key].match_markers(): to_uninstall.add(key) elif requirements_lut[key].specifier.contains(dist.version): diff --git a/piptools/utils.py b/piptools/utils.py index 6b7d2d2c8..d80fdf70c 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -23,10 +23,11 @@ from pip._internal.utils.misc import redact_auth_from_url from pip._internal.vcs import is_url from pip._vendor.packaging.markers import Marker +from pip._vendor.packaging.requirements import Requirement from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.packaging.version import Version -from pip._vendor.pkg_resources import Distribution, Requirement, get_distribution +from pip._vendor.pkg_resources import get_distribution from piptools._compat import PIP_VERSION from piptools.locations import CONFIG_FILE_NAME @@ -64,15 +65,9 @@ def key_from_ireq(ireq: InstallRequirement) -> str: return key_from_req(ireq.req) -def key_from_req(req: InstallRequirement | Distribution | Requirement) -> str: +def key_from_req(req: InstallRequirement | Requirement) -> str: """Get an all-lowercase version of the requirement's name.""" - if hasattr(req, "key"): - # from pkg_resources, such as installed dists for pip-sync - key = req.key - else: - # from packaging, such as install requirements from requirements.txt - key = req.name - return str(canonicalize_name(key)) + return str(canonicalize_name(req.name)) def comment(text: str) -> str: diff --git a/tests/conftest.py b/tests/conftest.py index 513b72430..024efa480 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,15 +18,17 @@ from pip._internal.commands.install import InstallCommand from pip._internal.index.package_finder import PackageFinder from pip._internal.models.candidate import InstallationCandidate +from pip._internal.models.link import Link from pip._internal.network.session import PipSession from pip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) +from pip._internal.utils.direct_url_helpers import direct_url_from_link from pip._vendor.packaging.version import Version from pip._vendor.pkg_resources import Requirement -from piptools._compat.pip_compat import PIP_VERSION, uses_pkg_resources +from piptools._compat import PIP_VERSION, Distribution from piptools.cache import DependencyCache from piptools.exceptions import NoCandidateFound from piptools.locations import CONFIG_FILE_NAME @@ -38,7 +40,6 @@ as_tuple, is_url_requirement, key_from_ireq, - key_from_req, make_install_requirement, ) @@ -125,34 +126,6 @@ def command(self) -> InstallCommand: """Not used""" -class FakeInstalledDistribution: - def __init__(self, line, deps=None): - if deps is None: - deps = [] - self.dep_strs = deps - self.deps = [Requirement.parse(d) for d in deps] - - self.req = Requirement.parse(line) - - self.key = key_from_req(self.req) - self.specifier = self.req.specifier - - self.version = line.split("==")[1] - - # The Distribution interface has changed between pkg_resources and - # importlib.metadata. - if uses_pkg_resources: - - def requires(self): - return self.deps - - else: - - @property - def requires(self): - return self.dep_strs - - def pytest_collection_modifyitems(config, items): for item in items: # Mark network tests as flaky @@ -162,7 +135,22 @@ def pytest_collection_modifyitems(config, items): @pytest.fixture def fake_dist(): - return FakeInstalledDistribution + def _fake_dist(line, deps=None): + if deps is None: + deps = [] + req = Requirement.parse(line) + key = req.key + if "==" in line: + version = line.split("==")[1] + else: + version = "0+unknown" + requires = [Requirement.parse(d) for d in deps] + direct_url = None + if req.url: + direct_url = direct_url_from_link(Link(req.url)) + return Distribution(key, version, requires, direct_url) + + return _fake_dist @pytest.fixture diff --git a/tests/test_sync.py b/tests/test_sync.py index 2eabaaa78..ebd26a21e 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -263,27 +263,41 @@ def test_diff_with_editable(fake_dist, from_editable): assert package.link.url == path_to_url(path_to_package) -def test_diff_with_matching_url_versions(fake_dist, from_line): +def test_diff_with_matching_url_hash(fake_dist, from_line): # if URL version is explicitly provided, use it to avoid reinstalling - installed = [fake_dist("example==1.0")] - reqs = [from_line("file:///example.zip#egg=example==1.0")] + line = "example@file:///example.zip#sha1=abc" + installed = [fake_dist(line)] + reqs = [from_line(line)] to_install, to_uninstall = diff(reqs, installed) assert to_install == set() assert to_uninstall == set() -def test_diff_with_no_url_versions(fake_dist, from_line): - # if URL version is not provided, assume the contents have +def test_diff_with_no_url_hash(fake_dist, from_line): + # if URL hash is not provided, assume the contents have # changed and reinstall - installed = [fake_dist("example==1.0")] - reqs = [from_line("file:///example.zip#egg=example")] + line = "example@file:///example.zip" + installed = [fake_dist(line)] + reqs = [from_line(line)] to_install, to_uninstall = diff(reqs, installed) assert to_install == set(reqs) assert to_uninstall == {"example"} +def test_diff_with_unequal_url_hash(fake_dist, from_line): + # if URL hashes mismatch, assume the contents have + # changed and reinstall + line = "example@file:///example.zip#" + installed = [fake_dist(line + "sha1=abc")] + reqs = [from_line(line + "sha1=def")] + + to_install, to_uninstall = diff(reqs, installed) + assert to_install == set(reqs) + assert to_uninstall == {"example @ file:///example.zip#sha1=abc"} + + def test_sync_install_temporary_requirement_file( from_line, from_editable, mocked_tmp_req_file ):