From 6a6263e05a4f2dabde45cc91d83ea7fa7d12d494 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 6 Nov 2023 14:06:48 -0800 Subject: [PATCH] Implement support for `--exclude `. When excluding a requirement from a PEX, any resolved distribution matching that requirement, as well as any of its transitive dependencies not also needed by non-excluded requirements, are elided from the PEX. At runtime these missing dependencies will not trigger boot resolve errors, but they will cause errors if the modules they would have provided are attempted to be imported. If the intention is to load the modules from the runtime environment, then `--pex-inherit-path` / `PEX_INHERIT_PATH` or `PEX_EXTRA_SYS_PATH` knobs must be used to allow the PEX to see distributions installed in the runtime environment. Clearly, you must know what you're doing to use this option and not encounter runtime errors due to import errors. Be ware! A forthcoming `--provided` option, with similar effects on the PEX contents, will both automatically inherit any needed missing distributions from the runtime environment and require all missing distributions are found; failing fast if they are not. Work towards #2097. --- pex/bin/pex.py | 50 +- pex/dependency_manager.py | 108 ++++ pex/environment.py | 27 +- pex/exclude_configuration.py | 38 ++ pex/pex_info.py | 16 + testing/data/__init__.py | 27 + .../data/locks}/__init__.py | 2 +- testing/data/locks/requests.lock.json | 519 ++++++++++++++++++ .../data/platforms}/__init__.py | 0 .../macosx_10_13_x86_64-cp-36-m.tags.txt | 0 tests/integration/test_excludes.py | 77 +++ tests/test_dependency_manager.py | 155 ++++++ tests/test_platform.py | 3 +- 13 files changed, 1002 insertions(+), 20 deletions(-) create mode 100644 pex/dependency_manager.py create mode 100644 pex/exclude_configuration.py create mode 100644 testing/data/__init__.py rename {tests/data/platforms => testing/data/locks}/__init__.py (51%) create mode 100644 testing/data/locks/requests.lock.json rename {tests/data => testing/data/platforms}/__init__.py (100%) rename {tests => testing}/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt (100%) create mode 100644 tests/integration/test_excludes.py create mode 100644 tests/test_dependency_manager.py diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 263edee88..e8c0a0ee0 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -24,6 +24,7 @@ register_global_arguments, ) from pex.common import die, is_pyc_dir, is_pyc_file, safe_mkdtemp +from pex.dependency_manager import DependencyManager from pex.enum import Enum from pex.inherit_path import InheritPath from pex.interpreter_constraints import InterpreterConstraints @@ -290,6 +291,22 @@ def configure_clp_pex_options(parser): ), ) + group.add_argument( + "--exclude", + dest="excluded", + default=[], + type=str, + action="append", + help=( + "Adds a requirement to exclude from the built PEX. Any distribution included in the " + "PEX's resolve that matches the requirement is excluded from the built PEX along with " + "all of its transitive dependencies that are not also required by other non-excluded " + "distributions. At runtime, the PEX will boot without checking the excluded " + "dependencies are available (say, via `--inherit-path`). This option can be used " + "multiple times." + ), + ) + group.add_argument( "--compile", "--no-compile", @@ -808,11 +825,15 @@ def build_pex( pex_info.strip_pex_env = options.strip_pex_env pex_info.interpreter_constraints = interpreter_constraints - for requirements_pex in options.requirements_pexes: - pex_builder.add_from_requirements_pex(requirements_pex) + dependency_manager = DependencyManager() + with TRACER.timed( + "Adding distributions from pexes: {}".format(" ".join(options.requirements_pexes)) + ): + for requirements_pex in options.requirements_pexes: + dependency_manager.add_from_pex(requirements_pex) with TRACER.timed( - "Resolving distributions ({})".format( + "Resolving distributions for requirements: {}".format( " ".join( itertools.chain.from_iterable( ( @@ -824,22 +845,21 @@ def build_pex( ) ): try: - result = resolve( - targets=targets, - requirement_configuration=requirement_configuration, - resolver_configuration=resolver_configuration, - compile_pyc=options.compile, - ignore_errors=options.ignore_errors, - ) - for installed_dist in result.installed_distributions: - pex_builder.add_distribution( - installed_dist.distribution, fingerprint=installed_dist.fingerprint + dependency_manager.add_from_installed( + resolve( + targets=targets, + requirement_configuration=requirement_configuration, + resolver_configuration=resolver_configuration, + compile_pyc=options.compile, + ignore_errors=options.ignore_errors, ) - for direct_req in installed_dist.direct_requirements: - pex_builder.add_requirement(direct_req) + ) except Unsatisfiable as e: die(str(e)) + with TRACER.timed("Configuring PEX dependencies"): + dependency_manager.configure(pex_builder, excluded=options.excluded) + if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: diff --git a/pex/dependency_manager.py b/pex/dependency_manager.py new file mode 100644 index 000000000..8a2557173 --- /dev/null +++ b/pex/dependency_manager.py @@ -0,0 +1,108 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from collections import defaultdict + +from pex import pex_warnings +from pex.dist_metadata import Requirement +from pex.environment import PEXEnvironment +from pex.exclude_configuration import ExcludeConfiguration +from pex.fingerprinted_distribution import FingerprintedDistribution +from pex.orderedset import OrderedSet +from pex.pep_503 import ProjectName +from pex.pex_builder import PEXBuilder +from pex.pex_info import PexInfo +from pex.resolve.resolvers import Installed +from pex.tracer import TRACER +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import DefaultDict, Iterable, Iterator + + import attr # vendor:skip +else: + from pex.third_party import attr + + +@attr.s +class DependencyManager(object): + _requirements = attr.ib(factory=OrderedSet) # type: OrderedSet[Requirement] + _distributions = attr.ib(factory=OrderedSet) # type: OrderedSet[FingerprintedDistribution] + + def add_from_pex(self, pex): + # type: (str) -> None + + pex_info = PexInfo.from_pex(pex) + self._requirements.update(Requirement.parse(req) for req in pex_info.requirements) + + pex_environment = PEXEnvironment.mount(pex, pex_info=pex_info) + self._distributions.update(pex_environment.iter_distributions()) + + def add_from_installed(self, installed): + # type: (Installed) -> None + + for installed_dist in installed.installed_distributions: + self._requirements.update(installed_dist.direct_requirements) + self._distributions.add(installed_dist.fingerprinted_distribution) + + def configure( + self, + pex_builder, # type: PEXBuilder + excluded=(), # type: Iterable[str] + ): + # type: (...) -> None + + exclude_configuration = ExcludeConfiguration.create(excluded) + exclude_configuration.configure(pex_builder.info) + + dists_by_project_name = defaultdict( + OrderedSet + ) # type: DefaultDict[ProjectName, OrderedSet[FingerprintedDistribution]] + for dist in self._distributions: + dists_by_project_name[dist.distribution.metadata.project_name].add(dist) + + def iter_non_excluded_distributions(requirements): + # type: (Iterable[Requirement]) -> Iterator[FingerprintedDistribution] + for req in requirements: + candidate_dists = dists_by_project_name[req.project_name] + for candidate_dist in tuple(candidate_dists): + if candidate_dist.distribution not in req: + continue + candidate_dists.discard(candidate_dist) + + excluded_by = exclude_configuration.excluded_by(candidate_dist.distribution) + if excluded_by: + excludes = " and ".join(map(str, excluded_by)) + TRACER.log( + "Skipping adding {candidate}: excluded by {excludes}".format( + candidate=candidate_dist.distribution, excludes=excludes + ) + ) + for root_req in self._requirements: + if candidate_dist.distribution in root_req: + pex_warnings.warn( + "The distribution {dist} was required by the input requirement " + "{root_req} but excluded by configured excludes: " + "{excludes}".format( + dist=candidate_dist.distribution, + root_req=root_req, + excludes=excludes, + ) + ) + continue + + yield candidate_dist + for dep in iter_non_excluded_distributions( + candidate_dist.distribution.requires() + ): + yield dep + + for fingerprinted_dist in iter_non_excluded_distributions(self._requirements): + pex_builder.add_distribution( + dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint + ) + + for requirement in self._requirements: + pex_builder.add_requirement(requirement) diff --git a/pex/environment.py b/pex/environment.py index cb61b09fa..6cc26ce0f 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -12,6 +12,7 @@ from pex import dist_metadata, pex_warnings, targets from pex.common import pluralize from pex.dist_metadata import Distribution, Requirement +from pex.exclude_configuration import ExcludeConfiguration from pex.fingerprinted_distribution import FingerprintedDistribution from pex.inherit_path import InheritPath from pex.interpreter import PythonInterpreter @@ -348,12 +349,23 @@ def _resolve_requirement( resolved_dists_by_key, # type: MutableMapping[_RequirementKey, FingerprintedDistribution] required, # type: bool required_by=None, # type: Optional[Distribution] + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): # type: (...) -> Iterator[_DistributionNotFound] requirement_key = _RequirementKey.create(requirement) if requirement_key in resolved_dists_by_key: return + excluded_by = exclude_configuration.excluded_by(requirement) + if excluded_by: + TRACER.log( + "Skipping resolving {requirement}: excluded by {excludes}".format( + requirement=requirement, + excludes=" and ".join(map(str, excluded_by)), + ) + ) + return + available_distributions = [ ranked_dist for ranked_dist in self._available_ranked_dists_by_project_name[ @@ -409,6 +421,7 @@ def _resolve_requirement( resolved_dists_by_key, required, required_by=resolved_distribution.distribution, + exclude_configuration=exclude_configuration, ): yield not_found @@ -502,14 +515,21 @@ def resolve(self): # type: () -> Iterable[Distribution] if self._resolved_dists is None: all_reqs = [Requirement.parse(req) for req in self._pex_info.requirements] + exclude_configuration = ExcludeConfiguration.create(excluded=self._pex_info.excluded) self._resolved_dists = tuple( fingerprinted_distribution.distribution - for fingerprinted_distribution in self.resolve_dists(all_reqs) + for fingerprinted_distribution in self.resolve_dists( + all_reqs, exclude_configuration=exclude_configuration + ) ) return self._resolved_dists - def resolve_dists(self, reqs): - # type: (Iterable[Requirement]) -> Iterable[FingerprintedDistribution] + def resolve_dists( + self, + reqs, # type: Iterable[Requirement] + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration + ): + # type: (...) -> Iterable[FingerprintedDistribution] self._update_candidate_distributions(self.iter_distributions()) @@ -538,6 +558,7 @@ def record_unresolved(dist_not_found): requirement=qualified_req_or_not_found.requirement, required=qualified_req_or_not_found.required, resolved_dists_by_key=resolved_dists_by_key, + exclude_configuration=exclude_configuration, ): record_unresolved(not_found) diff --git a/pex/exclude_configuration.py b/pex/exclude_configuration.py new file mode 100644 index 000000000..e5ccd20d7 --- /dev/null +++ b/pex/exclude_configuration.py @@ -0,0 +1,38 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from typing import Iterable, Tuple, Union + +from pex.dist_metadata import Distribution, Requirement +from pex.pex_info import PexInfo +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Iterable + + import attr # vendor:skip +else: + from pex.third_party import attr + + +@attr.s(frozen=True) +class ExcludeConfiguration(object): + @classmethod + def create(cls, excluded): + # type: (Iterable[str]) -> ExcludeConfiguration + return cls(excluded=tuple(Requirement.parse(req) for req in excluded)) + + _excluded = attr.ib(factory=tuple) # type: Tuple[Requirement, ...] + + def configure(self, pex_info): + # type: (PexInfo) -> None + for excluded in self._excluded: + pex_info.add_excluded(excluded) + + def excluded_by(self, item): + # type: (Union[Distribution, Requirement]) -> Iterable[Requirement] + if isinstance(item, Distribution): + return tuple(req for req in self._excluded if item in req) + return tuple(req for req in self._excluded if item.project_name == req.project_name) diff --git a/pex/pex_info.py b/pex/pex_info.py index e99700d4f..6fc9b4320 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -21,6 +21,8 @@ if TYPE_CHECKING: from typing import Any, Dict, Iterable, Mapping, Optional, Text, Tuple, Union + from pex.dist_metadata import Requirement + # N.B.: These are expensive imports and PexInfo is used during PEX bootstrapping which we want # to be as fast as possible. from pex.interpreter import PythonInterpreter @@ -144,6 +146,8 @@ def __init__(self, info=None): raise ValueError("Expected requirements to be a list, got %s" % type(requirements)) self._requirements = OrderedSet(self._parse_requirement_tuple(req) for req in requirements) + self._excluded = OrderedSet(self._pex_info.get("excluded", ())) # type: OrderedSet[str] + def _get_safe(self, key): if key not in self._pex_info: return None @@ -445,6 +449,15 @@ def add_requirement(self, requirement): def requirements(self): return self._requirements + def add_excluded(self, requirement): + # type: (Requirement) -> None + self._excluded.add(str(requirement)) + + @property + def excluded(self): + # type: () -> Iterable[str] + return self._excluded + def add_distribution(self, location, sha): self._distributions[location] = sha @@ -527,12 +540,14 @@ def update(self, other): other.interpreter_constraints ) self._requirements.update(other.requirements) + self._excluded.update(other.excluded) def as_json_dict(self): # type: () -> Dict[str, Any] data = self._pex_info.copy() data["inherit_path"] = self.inherit_path.value data["requirements"] = list(self._requirements) + data["excluded"] = list(self._excluded) data["interpreter_constraints"] = [str(ic) for ic in self.interpreter_constraints] data["distributions"] = self._distributions.copy() return data @@ -541,6 +556,7 @@ def dump(self): # type: (...) -> str data = self.as_json_dict() data["requirements"].sort() + data["excluded"].sort() data["interpreter_constraints"].sort() return json.dumps(data, sort_keys=True) diff --git a/testing/data/__init__.py b/testing/data/__init__.py new file mode 100644 index 000000000..740b3a1ff --- /dev/null +++ b/testing/data/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import pkgutil + + +def load(rel_path): + # type: (str) -> bytes + data = pkgutil.get_data(__name__, rel_path) + if data is None: + raise ValueError( + "No resource found at {rel_path} from package {name}.".format( + rel_path=rel_path, name=__name__ + ) + ) + return data + + +def path(*rel_path): + # type: (*str) -> str + path = os.path.join(os.path.dirname(__file__), *rel_path) + if not os.path.isfile(path): + raise ValueError("No resource found at {path}.".format(path=path)) + return path diff --git a/tests/data/platforms/__init__.py b/testing/data/locks/__init__.py similarity index 51% rename from tests/data/platforms/__init__.py rename to testing/data/locks/__init__.py index 87308fe2b..6227712e5 100644 --- a/tests/data/platforms/__init__.py +++ b/testing/data/locks/__init__.py @@ -1,2 +1,2 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). diff --git a/testing/data/locks/requests.lock.json b/testing/data/locks/requests.lock.json new file mode 100644 index 000000000..d091745b4 --- /dev/null +++ b/testing/data/locks/requests.lock.json @@ -0,0 +1,519 @@ +{ + "allow_builds": true, + "allow_prereleases": false, + "allow_wheels": true, + "build_isolation": true, + "constraints": [], + "locked_resolves": [ + { + "locked_requirements": [ + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9", + "url": "https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", + "url": "https://files.pythonhosted.org/packages/98/98/c2ff18671db109c9f10ed27f5ef610ae05b73bd876664139cf95bd1429aa/certifi-2023.7.22.tar.gz" + } + ], + "project_name": "certifi", + "requires_dists": [], + "requires_python": ">=3.6", + "version": "2023.7.22" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", + "url": "https://files.pythonhosted.org/packages/28/76/e6222113b83e3622caa4bb41032d0b1bf785250607392e1b778aca0b8a7d/charset_normalizer-3.3.2-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", + "url": "https://files.pythonhosted.org/packages/05/31/e1f51c76db7be1d4aef220d29fbfa5dbb4a99165d9833dcbf166753b6dc0/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", + "url": "https://files.pythonhosted.org/packages/05/8c/eb854996d5fef5e4f33ad56927ad053d04dc820e4a3d39023f35cad72617/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", + "url": "https://files.pythonhosted.org/packages/07/07/7e554f2bbce3295e191f7e653ff15d55309a9ca40d0362fcdab36f01063c/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", + "url": "https://files.pythonhosted.org/packages/13/82/83c188028b6f38d39538442dd127dc794c602ae6d45d66c469f4063a4c30/charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", + "url": "https://files.pythonhosted.org/packages/13/f8/eefae0629fa9260f83b826ee3363e311bb03cfdd518dad1bd10d57cb2d84/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", + "url": "https://files.pythonhosted.org/packages/16/ea/a9e284aa38cccea06b7056d4cbc7adf37670b1f8a668a312864abf1ff7c6/charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", + "url": "https://files.pythonhosted.org/packages/19/28/573147271fd041d351b438a5665be8223f1dd92f273713cb882ddafe214c/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", + "url": "https://files.pythonhosted.org/packages/1e/49/7ab74d4ac537ece3bc3334ee08645e231f39f7d6df6347b29a74b0537103/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", + "url": "https://files.pythonhosted.org/packages/1f/8d/33c860a7032da5b93382cbe2873261f81467e7b37f4ed91e25fed62fd49b/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", + "url": "https://files.pythonhosted.org/packages/24/9d/2e3ef673dfd5be0154b20363c5cdcc5606f35666544381bee15af3778239/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", + "url": "https://files.pythonhosted.org/packages/2a/9d/a6d15bd1e3e2914af5955c8eb15f4071997e7078419328fee93dfd497eb7/charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", + "url": "https://files.pythonhosted.org/packages/2b/61/095a0aa1a84d1481998b534177c8566fdc50bb1233ea9a0478cd3cc075bd/charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", + "url": "https://files.pythonhosted.org/packages/2d/dc/9dacba68c9ac0ae781d40e1a0c0058e26302ea0660e574ddf6797a0347f7/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", + "url": "https://files.pythonhosted.org/packages/2e/37/9223632af0872c86d8b851787f0edd3fe66be4a5378f51242b25212f8374/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", + "url": "https://files.pythonhosted.org/packages/2e/7d/2259318c202f3d17f3fe6438149b3b9e706d1070fe3fcbb28049730bb25c/charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", + "url": "https://files.pythonhosted.org/packages/33/95/ef68482e4a6adf781fae8d183fb48d6f2be8facb414f49c90ba6a5149cd1/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", + "url": "https://files.pythonhosted.org/packages/33/c3/3b96a435c5109dd5b6adc8a59ba1d678b302a97938f032e3770cc84cd354/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", + "url": "https://files.pythonhosted.org/packages/34/2a/f392457d45e24a0c9bfc012887ed4f3c54bf5d4d05a5deb970ffec4b7fc0/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", + "url": "https://files.pythonhosted.org/packages/3a/52/9f9d17c3b54dc238de384c4cb5a2ef0e27985b42a0e5cc8e8a31d918d48d/charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", + "url": "https://files.pythonhosted.org/packages/3d/09/d82fe4a34c5f0585f9ea1df090e2a71eb9bb1e469723053e1ee9f57c16f3/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", + "url": "https://files.pythonhosted.org/packages/3d/85/5b7416b349609d20611a64718bed383b9251b5a601044550f0c8983b8900/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", + "url": "https://files.pythonhosted.org/packages/3e/33/21a875a61057165e92227466e54ee076b73af1e21fe1b31f1e292251aa1e/charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", + "url": "https://files.pythonhosted.org/packages/3f/ba/3f5e7be00b215fa10e13d64b1f6237eb6ebea66676a41b2bcdd09fe74323/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", + "url": "https://files.pythonhosted.org/packages/40/26/f35951c45070edc957ba40a5b1db3cf60a9dbb1b350c2d5bef03e01e61de/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", + "url": "https://files.pythonhosted.org/packages/43/05/3bf613e719efe68fb3a77f9c536a389f35b95d75424b96b426a47a45ef1d/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", + "url": "https://files.pythonhosted.org/packages/44/80/b339237b4ce635b4af1c73742459eee5f97201bd92b2371c53e11958392e/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", + "url": "https://files.pythonhosted.org/packages/45/59/3d27019d3b447a88fe7e7d004a1e04be220227760264cc41b405e863891b/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", + "url": "https://files.pythonhosted.org/packages/46/6a/d5c26c41c49b546860cc1acabdddf48b0b3fb2685f4f5617ac59261b44ae/charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", + "url": "https://files.pythonhosted.org/packages/4f/d1/d547cc26acdb0cc458b152f79b2679d7422f29d41581e6fa907861e88af1/charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", + "url": "https://files.pythonhosted.org/packages/51/fd/0ee5b1c2860bb3c60236d05b6e4ac240cf702b67471138571dad91bcfed8/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", + "url": "https://files.pythonhosted.org/packages/53/cd/aa4b8a4d82eeceb872f83237b2d27e43e637cac9ffaef19a1321c3bafb67/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561", + "url": "https://files.pythonhosted.org/packages/54/7f/cad0b328759630814fcf9d804bfabaf47776816ad4ef2e9938b7e1123d04/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", + "url": "https://files.pythonhosted.org/packages/58/78/a0bc646900994df12e07b4ae5c713f2b3e5998f58b9d3720cce2aa45652f/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", + "url": "https://files.pythonhosted.org/packages/58/a2/0c63d5d7ffac3104b86631b7f2690058c97bf72d3145c0a9cd4fb90c58c2/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", + "url": "https://files.pythonhosted.org/packages/5b/ae/ce2c12fcac59cb3860b2e2d76dc405253a4475436b1861d95fe75bdea520/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", + "url": "https://files.pythonhosted.org/packages/63/09/c1bc53dab74b1816a00d8d030de5bf98f724c52c1635e07681d312f20be8/charset-normalizer-3.3.2.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", + "url": "https://files.pythonhosted.org/packages/66/fe/c7d3da40a66a6bf2920cce0f436fa1f62ee28aaf92f412f0bf3b84c8ad6c/charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", + "url": "https://files.pythonhosted.org/packages/68/77/02839016f6fbbf808e8b38601df6e0e66c17bbab76dff4613f7511413597/charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", + "url": "https://files.pythonhosted.org/packages/72/1a/641d5c9f59e6af4c7b53da463d07600a695b9824e20849cb6eea8a627761/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", + "url": "https://files.pythonhosted.org/packages/74/f1/0d9fe69ac441467b737ba7f48c68241487df2f4522dd7246d9426e7c690e/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", + "url": "https://files.pythonhosted.org/packages/79/66/8946baa705c588521afe10b2d7967300e49380ded089a62d38537264aece/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", + "url": "https://files.pythonhosted.org/packages/7b/ef/5eb105530b4da8ae37d506ccfa25057961b7b63d581def6f99165ea89c7e/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", + "url": "https://files.pythonhosted.org/packages/81/b2/160893421adfa3c45554fb418e321ed342bb10c0a4549e855b2b2a3699cb/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", + "url": "https://files.pythonhosted.org/packages/8d/b7/9e95102e9a8cce6654b85770794b582dda2921ec1fd924c10fbcf215ad31/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", + "url": "https://files.pythonhosted.org/packages/91/33/749df346e93d7a30cdcb90cbfdd41a06026317bfbfb62cd68307c1a3c543/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", + "url": "https://files.pythonhosted.org/packages/91/95/e2cfa7ce962e6c4b59a44a6e19e541c3a0317e543f0e0923f844e8d7d21d/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", + "url": "https://files.pythonhosted.org/packages/98/69/5d8751b4b670d623aa7a47bef061d69c279e9f922f6705147983aa76c3ce/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", + "url": "https://files.pythonhosted.org/packages/99/b0/9c365f6d79a9f0f3c379ddb40a256a67aa69c59609608fe7feb6235896e1/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", + "url": "https://files.pythonhosted.org/packages/9e/ef/cd47a63d3200b232792e361cd67530173a09eb011813478b1c0fb8aa7226/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", + "url": "https://files.pythonhosted.org/packages/a0/b1/4e72ef73d68ebdd4748f2df97130e8428c4625785f2b6ece31f555590c2d/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", + "url": "https://files.pythonhosted.org/packages/a2/51/e5023f937d7f307c948ed3e5c29c4b7a3e42ed2ee0b8cdf8f3a706089bf0/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", + "url": "https://files.pythonhosted.org/packages/a8/31/47d018ef89f95b8aded95c589a77c072c55e94b50a41aa99c0a2008a45a4/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", + "url": "https://files.pythonhosted.org/packages/a8/6f/4ff299b97da2ed6358154b6eb3a2db67da2ae204e53d205aacb18a7e4f34/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", + "url": "https://files.pythonhosted.org/packages/b2/62/5a5dcb9a71390a9511a253bde19c9c89e0b20118e41080185ea69fb2c209/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", + "url": "https://files.pythonhosted.org/packages/b3/c1/ebca8e87c714a6a561cfee063f0655f742e54b8ae6e78151f60ba8708b3a/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", + "url": "https://files.pythonhosted.org/packages/b8/60/e2f67915a51be59d4539ed189eb0a2b0d292bf79270410746becb32bc2c3/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", + "url": "https://files.pythonhosted.org/packages/bd/28/7ea29e73eea52c7e15b4b9108d0743fc9e4cc2cdb00d275af1df3d46d360/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", + "url": "https://files.pythonhosted.org/packages/be/4d/9e370f8281cec2fcc9452c4d1ac513324c32957c5f70c73dd2fa8442a21a/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", + "url": "https://files.pythonhosted.org/packages/c2/65/52aaf47b3dd616c11a19b1052ce7fa6321250a7a0b975f48d8c366733b9f/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", + "url": "https://files.pythonhosted.org/packages/c9/7a/6d8767fac16f2c80c7fa9f14e0f53d4638271635c306921844dc0b5fd8a6/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", + "url": "https://files.pythonhosted.org/packages/cc/94/f7cf5e5134175de79ad2059edf2adce18e0685ebdb9227ff0139975d0e93/charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", + "url": "https://files.pythonhosted.org/packages/cf/7c/f3b682fa053cc21373c9a839e6beba7705857075686a05c72e0f8c4980ca/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", + "url": "https://files.pythonhosted.org/packages/d1/2f/0d1efd07c74c52b6886c32a3b906fb8afd2fecf448650e73ecb90a5a27f1/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", + "url": "https://files.pythonhosted.org/packages/d1/b2/fcedc8255ec42afee97f9e6f0145c734bbe104aac28300214593eb326f1d/charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", + "url": "https://files.pythonhosted.org/packages/d8/b5/eb705c313100defa57da79277d9207dc8d8e45931035862fa64b625bfead/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", + "url": "https://files.pythonhosted.org/packages/da/f1/3702ba2a7470666a62fd81c58a4c40be00670e5006a67f4d626e57f013ae/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", + "url": "https://files.pythonhosted.org/packages/dd/51/68b61b90b24ca35495956b718f35a9756ef7d3dd4b3c1508056fa98d1a1b/charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", + "url": "https://files.pythonhosted.org/packages/df/3e/a06b18788ca2eb6695c9b22325b6fde7dde0f1d1838b1792a0076f58fe9d/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", + "url": "https://files.pythonhosted.org/packages/e1/9c/60729bf15dc82e3aaf5f71e81686e42e50715a1399770bcde1a9e43d09db/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", + "url": "https://files.pythonhosted.org/packages/e4/a6/7ee57823d46331ddc37dd00749c95b0edec2c79b15fc0d6e6efb532e89ac/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", + "url": "https://files.pythonhosted.org/packages/eb/5c/97d97248af4920bc68687d9c3b3c0f47c910e21a8ff80af4565a576bd2f0/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", + "url": "https://files.pythonhosted.org/packages/ee/fb/14d30eb4956408ee3ae09ad34299131fb383c47df355ddb428a7331cfa1e/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", + "url": "https://files.pythonhosted.org/packages/ef/d4/a1d72a8f6aa754fdebe91b848912025d30ab7dced61e9ed8aabbf791ed65/charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", + "url": "https://files.pythonhosted.org/packages/f2/0e/e06bc07ef4673e4d24dc461333c254586bb759fdd075031539bab6514d07/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", + "url": "https://files.pythonhosted.org/packages/f6/93/bb6cbeec3bf9da9b2eba458c15966658d1daa8b982c642f81c93ad9b40e1/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", + "url": "https://files.pythonhosted.org/packages/f6/d3/bfc699ab2c4f9245867060744e8136d359412ff1e5ad93be38a46d160f9d/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", + "url": "https://files.pythonhosted.org/packages/f7/9d/bcf4a449a438ed6f19790eee543a86a740c77508fbc5ddab210ab3ba3a9a/charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl" + } + ], + "project_name": "charset-normalizer", + "requires_dists": [], + "requires_python": ">=3.7.0", + "version": "3.3.2" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2", + "url": "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", + "url": "https://files.pythonhosted.org/packages/8b/e1/43beb3d38dba6cb420cefa297822eac205a277ab43e5ba5d5c46faf96438/idna-3.4.tar.gz" + } + ], + "project_name": "idna", + "requires_dists": [], + "requires_python": ">=3.5", + "version": "3.4" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", + "url": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1", + "url": "https://files.pythonhosted.org/packages/9d/be/10918a2eac4ae9f02f6cfe6414b7a155ccd8f7f9d4380d62fd5b955065c3/requests-2.31.0.tar.gz" + } + ], + "project_name": "requests", + "requires_dists": [ + "PySocks!=1.5.7,>=1.5.6; extra == \"socks\"", + "certifi>=2017.4.17", + "chardet<6,>=3.0.2; extra == \"use-chardet-on-py3\"", + "charset-normalizer<4,>=2", + "idna<4,>=2.5", + "urllib3<3,>=1.21.1" + ], + "requires_python": ">=3.7", + "version": "2.31.0" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e", + "url": "https://files.pythonhosted.org/packages/d2/b2/b157855192a68541a91ba7b2bbcb91f1b4faa51f8bae38d8005c034be524/urllib3-2.0.7-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", + "url": "https://files.pythonhosted.org/packages/af/47/b215df9f71b4fdba1025fc05a77db2ad243fa0926755a52c5e71659f4e3c/urllib3-2.0.7.tar.gz" + } + ], + "project_name": "urllib3", + "requires_dists": [ + "brotli>=1.0.9; platform_python_implementation == \"CPython\" and extra == \"brotli\"", + "brotlicffi>=0.8.0; platform_python_implementation != \"CPython\" and extra == \"brotli\"", + "certifi; extra == \"secure\"", + "cryptography>=1.9; extra == \"secure\"", + "idna>=2.0.0; extra == \"secure\"", + "pyopenssl>=17.1.0; extra == \"secure\"", + "pysocks!=1.5.7,<2.0,>=1.5.6; extra == \"socks\"", + "urllib3-secure-extra; extra == \"secure\"", + "zstandard>=0.18.0; extra == \"zstd\"" + ], + "requires_python": ">=3.7", + "version": "2.0.7" + } + ], + "platform_tag": null + } + ], + "path_mappings": {}, + "pex_version": "2.1.150", + "pip_version": "23.2", + "prefer_older_binary": false, + "requirements": [ + "requests" + ], + "requires_python": [ + "<3.13,>=3.7" + ], + "resolver_version": "pip-2020-resolver", + "style": "universal", + "target_systems": [ + "linux", + "mac" + ], + "transitive": true, + "use_pep517": null +} diff --git a/tests/data/__init__.py b/testing/data/platforms/__init__.py similarity index 100% rename from tests/data/__init__.py rename to testing/data/platforms/__init__.py diff --git a/tests/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt b/testing/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt similarity index 100% rename from tests/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt rename to testing/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt diff --git a/tests/integration/test_excludes.py b/tests/integration/test_excludes.py new file mode 100644 index 000000000..41d520014 --- /dev/null +++ b/tests/integration/test_excludes.py @@ -0,0 +1,77 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import os.path +import subprocess +from os.path import commonprefix + +import pytest + +from pex.executor import Executor +from pex.pep_503 import ProjectName +from pex.pex import PEX +from pex.typing import TYPE_CHECKING +from pex.venv.virtualenv import Virtualenv +from testing import PY_VER, data, make_env, run_pex_command + +if TYPE_CHECKING: + from typing import Any + + +@pytest.mark.skipif(PY_VER < (3, 7) or PY_VER >= (3, 13), reason="The lock used is for >=3.7,<3.13") +def test_exclude(tmpdir): + # type: (Any) -> None + + requests_lock = data.path("locks", "requests.lock.json") + pex_root = os.path.join(str(tmpdir), "pex_root") + pex = os.path.join(str(tmpdir), "pex") + run_pex_command( + args=[ + "--lock", + requests_lock, + "--exclude", + "certifi", + "-o", + pex, + "--pex-root", + pex_root, + "--runtime-pex-root", + pex_root, + ] + ).assert_success() + + assert ProjectName("certifi") not in frozenset( + dist.metadata.project_name for dist in PEX(pex).resolve() + ) + + # The exclude option is buyer beware. A PEX using this option will not work if the excluded + # distributions carry modules that are, in fact, needed at run time. + requests_cmd = [pex, "-c", "import requests, sys; print(sys.modules['certifi'].__file__)"] + expected_import_error_msg = "ModuleNotFoundError: No module named 'certifi'" + + process = subprocess.Popen(args=requests_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, stderr = process.communicate() + assert process.returncode != 0 + + assert expected_import_error_msg in stderr.decode("utf-8"), stderr.decode("utf-8") + + venv_dir = os.path.join(str(tmpdir), "venv") + venv = Virtualenv.create(venv_dir) + pip = venv.install_pip() + + # N.B.: The constraining lock requirement is the one expressed by requests: certifi>=2017.4.17 + # The actual locked version is 2023.7.22; so we stress this crease and use a different, but + # allowed, version. + subprocess.check_call(args=[pip, "install", "certifi==2017.4.17"]) + + # Although the venv has certifi available, a PEX is hermetic by default; so it shouldn't be + # used. + with pytest.raises(Executor.NonZeroExit) as exc: + venv.interpreter.execute(args=requests_cmd) + assert expected_import_error_msg in exc.value.stderr + + # Allowing the `sys.path` to be inherited should allow the certifi hole to be filled in. + _, stdout, _ = venv.interpreter.execute( + args=requests_cmd, env=make_env(PEX_INHERIT_PATH="fallback") + ) + assert venv.site_packages_dir == commonprefix([venv.site_packages_dir, stdout.strip()]) diff --git a/tests/test_dependency_manager.py b/tests/test_dependency_manager.py new file mode 100644 index 000000000..98648f9e4 --- /dev/null +++ b/tests/test_dependency_manager.py @@ -0,0 +1,155 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import hashlib +import os.path +import warnings + +import pytest + +from pex.dependency_manager import DependencyManager +from pex.dist_metadata import DistMetadata, Distribution, Requirement +from pex.fingerprinted_distribution import FingerprintedDistribution +from pex.orderedset import OrderedSet +from pex.pep_440 import Version +from pex.pep_503 import ProjectName +from pex.pex_builder import PEXBuilder +from pex.pex_info import PexInfo +from pex.pex_warnings import PEXWarning +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Tuple + + import attr # vendor:skip +else: + from pex.third_party import attr + + +@attr.s(frozen=True) +class DistFactory(object): + install_base_dir = attr.ib() # type: str + + def create( + self, + name, # type: str + *requires # type: str + ): + # type: (...) -> FingerprintedDistribution + fingerprint = hashlib.sha256(name.encode("utf-8")).hexdigest() + location = os.path.join(self.install_base_dir, fingerprint, name) + os.makedirs(location) + return FingerprintedDistribution( + distribution=Distribution( + location=location, + metadata=DistMetadata( + project_name=ProjectName(name), + version=Version("0.1.0"), + requires_dists=tuple(Requirement.parse(req) for req in requires), + ), + ), + fingerprint=fingerprint, + ) + + +@pytest.fixture +def dist_factory(tmpdir): + # type: (Any) -> DistFactory + return DistFactory(os.path.join(str(tmpdir), "installed_wheels")) + + +@attr.s(frozen=True) +class DistGraph(object): + root_reqs = attr.ib() # type: Tuple[Requirement, ...] + dists = attr.ib() # type: Tuple[FingerprintedDistribution, ...] + + def dist(self, name): + # type: (str) -> FingerprintedDistribution + project_name = ProjectName(name) + dists = [ + dist for dist in self.dists if project_name == dist.distribution.metadata.project_name + ] + assert len(dists) == 1, "Expected {name} to match one dist, found {found}".format( + name=name, found=" ".join(map(str, dists)) if dists else "none" + ) + return dists[0] + + +@pytest.fixture +def dist_graph(dist_factory): + # type: (DistFactory) -> DistGraph + + # distA distB <--------\ + # \ / \ | + # v v v | + # distC distD (cycle) + # / \ / | + # V v v | + # distE distF ---------/ + + return DistGraph( + root_reqs=(Requirement.parse("a"), Requirement.parse("b")), + dists=( + dist_factory.create("A", "c"), + dist_factory.create("B", "c", "d"), + dist_factory.create("C", "e", "f"), + dist_factory.create("D", "f"), + dist_factory.create("E"), + dist_factory.create("F", "b"), + ), + ) + + +def test_exclude_root_reqs(dist_graph): + # type: (DistGraph) -> None + + dependency_manager = DependencyManager( + requirements=OrderedSet(dist_graph.root_reqs), distributions=OrderedSet(dist_graph.dists) + ) + + pex_info = PexInfo.default() + pex_builder = PEXBuilder(pex_info=pex_info) + + with warnings.catch_warnings(record=True) as events: + dependency_manager.configure(pex_builder, excluded=["a", "b"]) + assert 2 == len(events) + + warning = events[0] + assert PEXWarning == warning.category + assert ( + "The distribution A 0.1.0 was required by the input requirement a but excluded by " + "configured excludes: a" + ) == str(warning.message) + + warning = events[1] + assert PEXWarning == warning.category + assert ( + "The distribution B 0.1.0 was required by the input requirement b but excluded by " + "configured excludes: b" + ) == str(warning.message) + + pex_builder.freeze() + + assert ["a", "b"] == list(pex_info.requirements) + assert ["a", "b"] == list(pex_info.excluded) + assert {} == pex_info.distributions + + +def test_exclude_complex(dist_graph): + # type: (DistGraph) -> None + + dependency_manager = DependencyManager( + requirements=OrderedSet(dist_graph.root_reqs), distributions=OrderedSet(dist_graph.dists) + ) + + pex_info = PexInfo.default() + pex_builder = PEXBuilder(pex_info=pex_info) + dependency_manager.configure(pex_builder, excluded=["c"]) + pex_builder.freeze() + + assert ["a", "b"] == list(pex_info.requirements) + assert ["c"] == list(pex_info.excluded) + expected_dists = [dist_graph.dist(name) for name in ("A", "B", "D", "F")] + assert { + os.path.basename(dist.location): dist.fingerprint for dist in expected_dists + } == pex_info.distributions diff --git a/tests/test_platform.py b/tests/test_platform.py index 49943758d..178f93249 100644 --- a/tests/test_platform.py +++ b/tests/test_platform.py @@ -11,6 +11,7 @@ from pex.pep_425 import CompatibilityTags from pex.platforms import Platform from pex.third_party.packaging import tags +from testing import data EXPECTED_BASE = [("py27", "none", "any"), ("py2", "none", "any")] @@ -130,7 +131,7 @@ def test_platform_supported_tags(): # A golden file test. This could break if we upgrade Pip and it upgrades packaging which, from # time to time, corrects omissions in tag sets. - golden_tags = pkgutil.get_data(__name__, "data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt") + golden_tags = data.load("platforms/macosx_10_13_x86_64-cp-36-m.tags.txt") assert golden_tags is not None assert ( CompatibilityTags(