From 17ab2c3fc5b31af41e78ffe2d9602341e3f279f9 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 12 Jan 2021 15:09:44 -0800 Subject: [PATCH] Tighten requirements parsing. In order to fix #899, we need to map local project requirements to built wheels after the fact in order to learn the project name and version and thus the full requirement needed for PexInfo metadata. In order to do this with confidence, break up the ReqInfo type into three seperate types to cover the three main parse cases. --- pex/requirements.py | 326 ++++++++++++++++++++----------------- pex/resolver.py | 18 +- tests/test_integration.py | 37 ++--- tests/test_requirements.py | 169 +++++++++++-------- 4 files changed, 311 insertions(+), 239 deletions(-) diff --git a/pex/requirements.py b/pex/requirements.py index e44bdde92..0ea6e521c 100644 --- a/pex/requirements.py +++ b/pex/requirements.py @@ -31,14 +31,13 @@ from typing import ( BinaryIO, Dict, - FrozenSet, + Iterable, Iterator, Match, Optional, Text, Tuple, Union, - Iterable, ) @@ -236,91 +235,120 @@ def create_parse_error(msg): raise create_parse_error(str(e)) -class ReqInfo( - namedtuple( - "ReqInfo", - [ - "line", - "project_name", - "url", - "extras", - "specifier", - "marker", - "editable", - "is_local_project", - ], +class PyPIRequirement(namedtuple("PyPIRequirement", ["line", "requirement", "editable"])): + """A requirement realized through a package index or find links repository.""" + + @classmethod + def create( + cls, + line, # type: LogicalLine + requirement, # type: Requirement + editable=False, # type: bool + ): + # type: (...) -> PyPIRequirement + return cls(line, requirement, editable=editable) + + @property + def requirement(self): + # type: () -> Requirement + return cast(Requirement, super(PyPIRequirement, self).requirement) + + +class URLRequirement(namedtuple("URLRequirement", ["line", "url", "requirement", "editable"])): + """A requirement realized through an distribution archive at a fixed URL.""" + + @classmethod + def create( + cls, + line, # type: LogicalLine + url, # type: str + requirement, # type: Requirement + editable=False, # type: bool + ): + # type: (...) -> URLRequirement + return cls(line, url, requirement, editable=editable) + + @property + def requirement(self): + # type: () -> Requirement + return cast(Requirement, super(URLRequirement, self).requirement) + + +def parse_requirement_from_project_name_and_specifier( + project_name, # type: str + extras=None, # type: Optional[Iterable[str]] + specifier=None, # type: Optional[SpecifierSet] + marker=None, # type: Optional[Marker] +): + # type: (...) -> Requirement + requirement_string = "{project_name}{extras}{specifier}".format( + project_name=project_name, + extras="[{extras}]".format(extras=", ".join(extras)) if extras else "", + specifier=specifier or SpecifierSet(), ) + if marker: + requirement_string += ";" + str(marker) + return Requirement.parse(requirement_string) + + +def parse_requirement_from_dist( + dist, # type: str + extras=None, # type: Optional[Iterable[str]] + marker=None, # type: Optional[Marker] ): + # type: (...) -> Requirement + project_name_and_version = dist_metadata.project_name_and_version(dist) + if project_name_and_version is None: + raise ValueError( + "Failed to find a project name and version from the given wheel path: " + "{wheel}".format(wheel=dist) + ) + return parse_requirement_from_project_name_and_specifier( + project_name_and_version.project_name, + extras=extras, + specifier=SpecifierSet("=={}".format(project_name_and_version.version)), + marker=marker, + ) + + +class LocalProjectRequirement( + namedtuple("LocalProjectRequirement", ["line", "path", "extras", "marker", "editable"]) +): + """A requirement realized by building a distribution from local sources.""" + @classmethod def create( cls, line, # type: LogicalLine - project_name=None, # type: Optional[str] - url=None, # type: Optional[str] + path, # type: str extras=None, # type: Optional[Iterable[str]] - specifier=None, # type: Optional[SpecifierSet] marker=None, # type: Optional[Marker] editable=False, # type: bool - is_local_project=False, # type: bool ): - # type: (...) -> ReqInfo + # type: (...) -> LocalProjectRequirement return cls( line=line, - project_name=project_name, - url=url, - extras=frozenset(extras or ()), - specifier=specifier, + path=path, + extras=tuple(extras or ()), marker=marker, editable=editable, - is_local_project=is_local_project, ) - @property - def line(self): - # type: () -> LogicalLine - return cast(LogicalLine, super(ReqInfo, self).line) + def as_requirement(self, dist): + # type: (str) -> Requirement + """Create a requirement given a distribution that was built from this local project.""" + return parse_requirement_from_dist(dist, self.extras, self.marker) - @property - def project_name(self): - # type: () -> Optional[str] - return cast("Optional[str]", super(ReqInfo, self).project_name) - @property - def url(self): - # type: () -> Optional[str] - return cast("Optional[str]", super(ReqInfo, self).url) - - @property - def extras(self): - # type: () -> FrozenSet[str] - return cast("FrozenSet[str]", super(ReqInfo, self).extras) - - @property - def specifier(self): - # type: () -> Optional[SpecifierSet] - return cast("Optional[SpecifierSet]", super(ReqInfo, self).specifier) - - @property - def marker(self): - # type: () -> Optional[Marker] - return cast("Optional[Marker]", super(ReqInfo, self).marker) - - @property - def editable(self): - # type: () -> bool - return cast(bool, super(ReqInfo, self).editable) - - @property - def is_local_project(self): - # type: () -> bool - return cast(bool, super(ReqInfo, self).is_local_project) +if TYPE_CHECKING: + ParsedRequirement = Union[PyPIRequirement, URLRequirement, LocalProjectRequirement] -class Constraint(namedtuple("Constraint", ["req_info"])): +class Constraint(namedtuple("Constraint", ["line", "requirement"])): @property - def req_info(self): - # type: () -> ReqInfo - return cast(ReqInfo, super(Constraint, self).req_info) + def requirement(self): + # type: () -> Requirement + return cast(Requirement, super(Constraint, self).requirement) class ParseError(Exception): @@ -349,15 +377,14 @@ def _strip_requirement_options(line): return editable, re.sub(r"\s--(global-option|install-option|hash).*$", "", processed_text) -def _is_recognized_pip_url_scheme(scheme): +def _is_recognized_non_local_pip_url_scheme(scheme): # type: (str) -> bool return bool( re.match( r""" ( # Archives - file - | ftp + ftp | https? # VCSs: https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support @@ -423,22 +450,14 @@ def from_project_name_and_version(cls, project_name_and_version): ) -def _try_parse_project_name_and_specifier_from_path( - path, # type: str - try_read_metadata=False, # type:bool -): - # type: (...) -> Optional[ProjectNameAndSpecifier] +def _try_parse_project_name_and_specifier_from_path(path): + # type: (str) -> Optional[ProjectNameAndSpecifier] try: - project_name_and_version = ( - dist_metadata.project_name_and_version(path, fallback_to_filename=True) - if try_read_metadata - else ProjectNameAndVersion.from_filename(path) + return ProjectNameAndSpecifier.from_project_name_and_version( + ProjectNameAndVersion.from_filename(path) ) - if project_name_and_version is not None: - return ProjectNameAndSpecifier.from_project_name_and_version(project_name_and_version) except MetadataError: - pass - return None + return None def _try_parse_pip_local_formats( @@ -484,11 +503,7 @@ def _try_parse_pip_local_formats( if not os.path.exists(abs_stripped_path): return None - if not os.path.isdir(abs_stripped_path): - # Maybe a local archive path. - return ProjectNameExtrasAndMarker.create(abs_stripped_path) - - # Maybe a local project path. + # Maybe a local archive or project path. requirement_parts = match.group("requirement_parts") if not requirement_parts: return ProjectNameExtrasAndMarker.create(abs_stripped_path) @@ -504,101 +519,108 @@ def _try_parse_pip_local_formats( def _split_direct_references(processed_text): - # type: (str) -> Tuple[str, Optional[str]] - parts = processed_text.split("@", 1) - if len(parts) == 1: - return processed_text, None - return parts[0].strip(), parts[1].strip() + # type: (str) -> Union[Tuple[str, str], Tuple[None, None]] + match = re.match( + r""" + ^ + (?P[a-zA-Z0-9]+(?:[-_.]+[a-zA-Z0-9]+)*) + \s* + @ + \s* + (?P.+)? + $ + """, + processed_text, + re.VERBOSE, + ) + if not match: + return None, None + project_name, url = match.groups() + return project_name, url def _parse_requirement_line( line, # type: LogicalLine basepath=None, # type: Optional[str] ): - # type: (...) -> ReqInfo + # type: (...) -> ParsedRequirement basepath = basepath or os.getcwd() editable, processed_text = _strip_requirement_options(line) + project_name, direct_reference_url = _split_direct_references(processed_text) + parsed_url = urlparse.urlparse(direct_reference_url or processed_text) - # Handle urls (Pip proprietary). - parsed_url = urlparse.urlparse(processed_text) - if _is_recognized_pip_url_scheme(parsed_url.scheme): + # Handle non local URLs (Pip proprietary). + if _is_recognized_non_local_pip_url_scheme(parsed_url.scheme): project_name_extras_and_marker = _try_parse_fragment_project_name_and_marker( parsed_url.fragment ) project_name, extras, marker = ( - project_name_extras_and_marker if project_name_extras_and_marker else (None, None, None) + project_name_extras_and_marker + if project_name_extras_and_marker + else (project_name, None, None) ) specifier = None # type: Optional[SpecifierSet] if not project_name: - is_local_file = parsed_url.scheme == "file" project_name_and_specifier = _try_parse_project_name_and_specifier_from_path( - parsed_url.path, try_read_metadata=is_local_file + parsed_url.path ) if project_name_and_specifier is not None: project_name = project_name_and_specifier.project_name specifier = project_name_and_specifier.specifier + if project_name is None: + raise ParseError( + line, + ( + "Could not determine a project name for URL requirement {}, consider using " + "#egg=." + ), + ) url = parsed_url._replace(fragment="").geturl() - return ReqInfo.create( - line, - project_name=project_name, - url=url, + requirement = parse_requirement_from_project_name_and_specifier( + project_name, extras=extras, specifier=specifier, marker=marker, - editable=editable, ) + return URLRequirement.create(line, url, requirement, editable=editable) - # Handle local archives and project directories (Pip proprietary). - project_name_extras_and_marker = _try_parse_pip_local_formats(processed_text, basepath=basepath) + # Handle local archives and project directories via path or file URL (Pip proprietary). + local_requirement = parsed_url._replace(scheme="").geturl() + project_name_extras_and_marker = _try_parse_pip_local_formats( + local_requirement, basepath=basepath + ) maybe_abs_path, extras, marker = ( - project_name_extras_and_marker if project_name_extras_and_marker else (None, None, None) + project_name_extras_and_marker + if project_name_extras_and_marker + else (project_name, None, None) ) if maybe_abs_path is not None and any( os.path.isfile(os.path.join(maybe_abs_path, *p)) for p in ((), ("setup.py",), ("pyproject.toml",)) ): archive_or_project_path = os.path.realpath(maybe_abs_path) - is_local_project = os.path.isdir(archive_or_project_path) - project_name_and_specifier = ( - None - if is_local_project - else _try_parse_project_name_and_specifier_from_path( - archive_or_project_path, try_read_metadata=True + if os.path.isdir(archive_or_project_path): + return LocalProjectRequirement.create( + line, + archive_or_project_path, + extras=extras, + marker=marker, + editable=editable, ) + requirement = parse_requirement_from_dist( + archive_or_project_path, extras=extras, marker=marker ) - project_name, specifier = ( - project_name_and_specifier if project_name_and_specifier else (None, None) - ) - return ReqInfo.create( - line, - project_name=project_name, - url=archive_or_project_path, - extras=extras, - specifier=specifier, - marker=marker, - editable=editable, - is_local_project=is_local_project, - ) + return URLRequirement.create(line, archive_or_project_path, requirement, editable=editable) # Handle PEP-440. See: https://www.python.org/dev/peps/pep-0440. # # The `pkg_resources.Requirement.parse` method does all of this for us (via # `packaging.requirements.Requirement`) except for the handling of PEP-440 direct url - # references; so we strip those urls out first. - requirement, direct_reference_url = _split_direct_references(processed_text) + # references; which we handled above and won't encounter here. try: - req = Requirement.parse(requirement) - return ReqInfo.create( - line, - project_name=req.name, - url=direct_reference_url or req.url, - extras=req.extras, - specifier=req.specifier, - marker=req.marker, - editable=editable, - ) + return PyPIRequirement.create(line, Requirement.parse(processed_text), editable=editable) except RequirementParseError as e: raise ParseError( line, "Problem parsing {!r} as a requirement: {}".format(processed_text, e) @@ -640,7 +662,7 @@ def parse_requirements( source, # type: Source fetcher=None, # type: Optional[URLFetcher] ): - # type: (...) -> Iterator[Union[ReqInfo, Constraint]] + # type: (...) -> Iterator[Union[ParsedRequirement, Constraint]] # For the format specification, see: # https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format @@ -688,8 +710,8 @@ def parse_requirements( is_constraints=constraint_file, fetcher=fetcher, ) as other_source: - for req_info in parse_requirements(other_source, fetcher=fetcher): - yield req_info + for requirement in parse_requirements(other_source, fetcher=fetcher): + yield requirement continue # Skip empty lines, comment lines and all other Pip options. @@ -697,10 +719,22 @@ def parse_requirements( continue # Only requirement lines remain. - req_info = _parse_requirement_line( + requirement = _parse_requirement_line( logical_line, basepath=os.path.dirname(source.origin) if source.is_file else None ) - yield Constraint(req_info) if source.is_constraints else req_info + if source.is_constraints: + if not isinstance(requirement, PyPIRequirement) or requirement.requirement.extras: + raise ParseError( + logical_line, + "Constraint files do not support VCS, URL or local project requirements" + "and they do not support requirements with extras. Search for 'We are also " + "changing our support for Constraints Files' here: " + "https://pip.pypa.io/en/stable/user_guide/" + "#changes-to-the-pip-dependency-resolver-in-20-3-2020.", + ) + yield Constraint(line, requirement.requirement) + else: + yield requirement finally: start_line = 0 del line_buffer[:] @@ -712,7 +746,7 @@ def parse_requirement_file( is_constraints=False, # type: bool fetcher=None, # type: Optional[URLFetcher] ): - # type: (...) -> Iterator[Union[ReqInfo, Constraint]] + # type: (...) -> Iterator[Union[ParsedRequirement, Constraint]] def open_source(): url = urlparse.urlparse(location) if url.scheme and url.netloc: @@ -732,12 +766,12 @@ def open_source(): def parse_requirement_strings(requirements): - # type: (Iterable[str]) -> Iterator[ReqInfo] + # type: (Iterable[str]) -> Iterator[ParsedRequirement] for requirement in requirements: yield _parse_requirement_line( LogicalLine( raw_text=requirement, - processed_text=requirement, + processed_text=requirement.strip(), source="", start_line=1, end_line=1, diff --git a/pex/resolver.py b/pex/resolver.py index 38409ef2e..9557321f6 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -21,7 +21,12 @@ from pex.pex_info import PexInfo from pex.pip import PackageIndexConfiguration, get_pip from pex.platforms import Platform -from pex.requirements import ReqInfo, URLFetcher, parse_requirement_file, parse_requirement_strings +from pex.requirements import ( + LocalProjectRequirement, + URLFetcher, + parse_requirement_file, + parse_requirement_strings, +) from pex.third_party.packaging.markers import Marker from pex.third_party.packaging.version import InvalidVersion, Version from pex.third_party.pkg_resources import Distribution, Environment, Requirement @@ -218,9 +223,9 @@ class DownloadRequest( def iter_local_projects(self): if self.requirements: for req in parse_requirement_strings(self.requirements): - if req.is_local_project: + if isinstance(req, LocalProjectRequirement): for target in self.targets: - yield BuildRequest.create(target=target, source_path=req.url) + yield BuildRequest.create(target=target, source_path=req.path) if self.requirement_files: fetcher = URLFetcher( @@ -228,13 +233,10 @@ def iter_local_projects(self): ) for requirement_file in self.requirement_files: for req_or_constraint in parse_requirement_file(requirement_file, fetcher=fetcher): - if ( - isinstance(req_or_constraint, ReqInfo) - and req_or_constraint.is_local_project - ): + if isinstance(req_or_constraint, LocalProjectRequirement): for target in self.targets: yield BuildRequest.create( - target=target, source_path=req_or_constraint.url + target=target, source_path=req_or_constraint.path ) def download_distributions(self, dest=None, max_parallel_jobs=None): diff --git a/tests/test_integration.py b/tests/test_integration.py index 935c38aa2..824658db9 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -38,7 +38,7 @@ from pex.orderedset import OrderedSet from pex.pex_info import PexInfo from pex.pip import get_pip -from pex.requirements import LogicalLine, ReqInfo, URLFetcher, parse_requirement_file +from pex.requirements import LogicalLine, PyPIRequirement, URLFetcher, parse_requirement_file from pex.testing import ( IS_PYPY, NOT_CPYTHON27, @@ -60,6 +60,7 @@ temporary_content, ) from pex.third_party import pkg_resources +from pex.third_party.pkg_resources import Requirement from pex.typing import TYPE_CHECKING, cast from pex.util import DistributionHelper, named_temporary_file from pex.variables import ENV, unzip_dir, venv_dir @@ -2529,17 +2530,19 @@ def _run_proxy( def test_requirements_network_configuration(run_proxy, tmp_workdir): # type: (Callable[[Optional[str]], ContextManager[Tuple[int, str]]], str) -> None - def line( + def req( contents, # type: str line_no, # type: int ): - # type: (...) -> LogicalLine - return LogicalLine( - "{}\n".format(contents), - contents, - source=EXAMPLE_PYTHON_REQUIREMENTS_URL, - start_line=line_no, - end_line=line_no, + return PyPIRequirement.create( + LogicalLine( + "{}\n".format(contents), + contents, + source=EXAMPLE_PYTHON_REQUIREMENTS_URL, + start_line=line_no, + end_line=line_no, + ), + Requirement.parse(contents), ) proxy_auth = "jake:jones" @@ -2554,18 +2557,10 @@ def line( ), ) assert [ - ReqInfo.create( - line=line("ansicolors>=1.0.2", 4), project_name="ansicolors", specifier=">=1.0.2" - ), - ReqInfo.create( - line=line("setuptools>=42.0.0", 5), project_name="setuptools", specifier=">=42.0.0" - ), - ReqInfo.create( - line=line("translate>=3.2.1", 6), project_name="translate", specifier=">=3.2.1" - ), - ReqInfo.create( - line=line("protobuf>=3.11.3", 7), project_name="protobuf", specifier=">=3.11.3" - ), + req("ansicolors>=1.0.2", 4), + req("setuptools>=42.0.0", 5), + req("translate>=3.2.1", 6), + req("protobuf>=3.11.3", 7), ] == list(reqs) diff --git a/tests/test_requirements.py b/tests/test_requirements.py index b99ef2084..99a131be8 100644 --- a/tests/test_requirements.py +++ b/tests/test_requirements.py @@ -11,22 +11,29 @@ from pex.common import safe_open, temporary_dir, touch from pex.requirements import ( Constraint, + LocalProjectRequirement, LogicalLine, ParseError, - ReqInfo, + PyPIRequirement, Source, URLFetcher, + URLRequirement, parse_requirement_file, + parse_requirement_from_project_name_and_specifier, parse_requirements, ) from pex.testing import environment_as from pex.third_party.packaging.markers import Marker -from pex.third_party.packaging.specifiers import SpecifierSet +from pex.third_party.pkg_resources import Requirement from pex.typing import TYPE_CHECKING if TYPE_CHECKING: from typing import Any, Iterable, Iterator, List, Optional, Union + from pex.requirements import ParsedRequirement + + ParsedRequirementOrConstraint = Union[ParsedRequirement, Constraint] + @pytest.fixture def chroot(): @@ -78,13 +85,13 @@ def test_parse_requirements_failure_bad_requirement(chroot): req_iter = parse_requirements(Source.from_text("-r other-requirements.txt")) - req_info = next(req_iter) - assert isinstance(req_info, ReqInfo) - assert "GoodRequirement" == req_info.project_name + parsed_requirement = next(req_iter) + assert isinstance(parsed_requirement, PyPIRequirement) + assert "GoodRequirement" == parsed_requirement.requirement.project_name - req_info = next(req_iter) - assert isinstance(req_info, ReqInfo) - assert "AnotherRequirement" == req_info.project_name + parsed_requirement = next(req_iter) + assert isinstance(parsed_requirement, PyPIRequirement) + assert "AnotherRequirement" == parsed_requirement.requirement.project_name with pytest.raises(ParseError) as exc_info: next(req_iter) @@ -113,46 +120,70 @@ def __eq__(self, other): return type(other) == MarkerWithEq and str(self) == str(other) +DUMMY_LINE = LogicalLine("", "", "", 1, 1) + + def req( - project_name=None, # type: Optional[str] - url=None, # type: Optional[str] + project_name, # type: str + extras=None, # type: Optional[Iterable[str]] + specifier=None, # type: Optional[str] + marker=None, # type: Optional[str] + editable=False, # type: bool +): + # type: (...) -> PyPIRequirement + return PyPIRequirement.create( + line=DUMMY_LINE, + requirement=parse_requirement_from_project_name_and_specifier( + project_name, extras=extras, specifier=specifier, marker=marker + ), + editable=editable, + ) + + +def url_req( + url, # type: str + project_name, # type: str extras=None, # type: Optional[Iterable[str]] specifier=None, # type: Optional[str] marker=None, # type: Optional[str] editable=False, # type: bool - is_local_project=False, # type: bool ): - # type: (...) -> ReqInfo - return ReqInfo( - line=None, - project_name=project_name, + # type: (...) -> URLRequirement + return URLRequirement.create( + line=DUMMY_LINE, url=url, - extras=frozenset(extras or ()), - specifier=SpecifierSet(specifiers=specifier or ""), - marker=MarkerWithEq.wrap(marker), + requirement=parse_requirement_from_project_name_and_specifier( + project_name, extras=extras, specifier=specifier, marker=marker + ), editable=editable, - is_local_project=is_local_project, ) -def normalize_results(req_infos): - # type: (Iterable[Union[Constraint, ReqInfo]]) -> List[Union[Constraint, ReqInfo]] - def normalize_req_info(req_info): - return ( - req_info._replace(line=None) - ._replace( - specifier=SpecifierSet( - specifiers=str(req_info.specifier) if req_info.specifier else "" - ) - ) - ._replace(marker=MarkerWithEq.wrap(req_info.marker)) - ) +def local_req( + path, # type: str + extras=None, # type: Optional[Iterable[str]] + marker=None, # type: Optional[str] + editable=False, # type: bool +): + # type: (...) -> LocalProjectRequirement + return LocalProjectRequirement.create( + line=DUMMY_LINE, + path=path, + extras=extras, + marker=MarkerWithEq.wrap(marker), + editable=editable, + ) + +def normalize_results(parsed_requirements): + # type: (Iterable[ParsedRequirementOrConstraint]) -> List[ParsedRequirementOrConstraint] return [ - normalize_req_info(req_info) - if isinstance(req_info, ReqInfo) - else Constraint(normalize_req_info(req_info.req_info)) - for req_info in req_infos + parsed_requirement._replace( + line=DUMMY_LINE, marker=MarkerWithEq.wrap(parsed_requirement.marker) + ) + if isinstance(parsed_requirement, LocalProjectRequirement) + else parsed_requirement._replace(line=DUMMY_LINE) + for parsed_requirement in parsed_requirements ] @@ -175,7 +206,8 @@ def test_parse_requirements_stress(chroot): SomeProject ==5.4 ; python_version < '2.7' SomeProject; sys_platform == 'win32' - SomeProject @ file:///somewhere/over/here + SomeProject @ https://example.com/somewhere/over/here + SomeProject @ file:somewhere/over/here FooProject >= 1.2 --global-option="--no-user-cfg" \\ --install-option="--prefix='/usr/local'" \\ @@ -191,6 +223,7 @@ def test_parse_requirements_stress(chroot): """ ) ) + touch("somewhere/over/here/pyproject.toml") with safe_open(os.path.join(chroot, "extra", "stress.txt"), "w") as fp: fp.write( @@ -218,6 +251,7 @@ def test_parse_requirements_stress(chroot): Django@ git+https://github.com/django/django.git Django@git+https://github.com/django/django.git@stable/2.1.x Django@ git+https://github.com/django/django.git@fd209f62f1d83233cc634443cfac5ee4328d98b8 + Django @ file:projects/django-2.3.zip; python_version >= "3.10" """ ) ) @@ -225,6 +259,7 @@ def test_parse_requirements_stress(chroot): touch("extra/a/local/project/pyproject.toml") touch("extra/another/local/project/setup.py") touch("extra/tmp/tmpW8tdb_/setup.py") + touch("extra/projects/django-2.3.zip") with safe_open(os.path.join(chroot, "subdir", "more-requirements.txt"), "w") as fp: fp.write( @@ -293,61 +328,65 @@ def test_parse_requirements_stress(chroot): req(project_name="SomeProject", specifier="~=1.4.2"), req(project_name="SomeProject", specifier="==5.4", marker="python_version < '2.7'"), req(project_name="SomeProject", marker="sys_platform == 'win32'"), - req(project_name="SomeProject", url="file:///somewhere/over/here"), + url_req(project_name="SomeProject", url="https://example.com/somewhere/over/here"), + local_req(path=os.path.realpath("somewhere/over/here")), req(project_name="FooProject", specifier=">=1.2"), - req( + url_req( project_name="MyProject", url="git+https://git.example.com/MyProject.git@da39a3ee5e6b4b0d3255bfef95601890afd80709", ), - req(project_name="MyProject", url="git+ssh://git.example.com/MyProject"), - req(project_name="MyProject", url="git+file:/home/user/projects/MyProject"), - Constraint(req(project_name="AnotherProject")), - req( - url=os.path.realpath("extra/a/local/project"), + url_req(project_name="MyProject", url="git+ssh://git.example.com/MyProject"), + url_req(project_name="MyProject", url="git+file:/home/user/projects/MyProject"), + Constraint(DUMMY_LINE, Requirement.parse("AnotherProject")), + local_req( + path=os.path.realpath("extra/a/local/project"), extras=["foo"], marker="python_full_version == '2.7.8'", - is_local_project=True, ), - req( - url=os.path.realpath("extra/another/local/project"), + local_req( + path=os.path.realpath("extra/another/local/project"), marker="python_version == '2.7.*'", - is_local_project=True, ), - req(url=os.path.realpath("extra/another/local/project"), is_local_project=True), - req(url=os.path.realpath("extra"), is_local_project=True), - req(url=os.path.realpath("extra/tmp/tmpW8tdb_"), is_local_project=True), - req(url=os.path.realpath("extra/tmp/tmpW8tdb_"), extras=["foo"], is_local_project=True), - req( - url=os.path.realpath("extra/tmp/tmpW8tdb_"), + local_req(path=os.path.realpath("extra/another/local/project")), + local_req(path=os.path.realpath("extra")), + local_req(path=os.path.realpath("extra/tmp/tmpW8tdb_")), + local_req(path=os.path.realpath("extra/tmp/tmpW8tdb_"), extras=["foo"]), + local_req( + path=os.path.realpath("extra/tmp/tmpW8tdb_"), extras=["foo"], marker="python_version == '3.9'", - is_local_project=True, ), - req( + url_req( project_name="AnotherProject", url="hg+http://hg.example.com/MyProject@da39a3ee5e6b", extras=["more", "extra"], marker="python_version == '3.9.*'", ), - req(project_name="Project", url="ftp://a/Project-1.0.tar.gz", specifier="==1.0"), - req(project_name="Project", url="http://a/Project-1.0.zip", specifier="==1.0"), - req( + url_req(project_name="Project", url="ftp://a/Project-1.0.tar.gz", specifier="==1.0"), + url_req(project_name="Project", url="http://a/Project-1.0.zip", specifier="==1.0"), + url_req( project_name="numpy", url="https://a/numpy-1.9.2-cp34-none-win32.whl", specifier="==1.9.2", ), - req(project_name="Django", url="git+https://github.com/django/django.git"), - req(project_name="Django", url="git+https://github.com/django/django.git@stable/2.1.x"), - req( + url_req(project_name="Django", url="git+https://github.com/django/django.git"), + url_req(project_name="Django", url="git+https://github.com/django/django.git@stable/2.1.x"), + url_req( project_name="Django", url="git+https://github.com/django/django.git@fd209f62f1d83233cc634443cfac5ee4328d98b8", ), - req( + url_req( + project_name="Django", + url=os.path.realpath("extra/projects/django-2.3.zip"), + specifier="==2.3", + marker="python_version>='3.10'", + ), + url_req( project_name="numpy", url=os.path.realpath("./downloads/numpy-1.9.2-cp34-none-win32.whl"), specifier="==1.9.2", ), - req( + url_req( project_name="wxPython_Phoenix", url="http://wxpython.org/Phoenix/snapshot-builds/wxPython_Phoenix-3.0.3.dev1820+49a8884-cp34-none-win_amd64.whl", specifier="==3.0.3.dev1820+49a8884", @@ -388,7 +427,9 @@ def test_parse_constraints_from_url(): fetcher=URLFetcher(), ) results = normalize_results(req_iter) - assert [Constraint(req) for req in EXPECTED_EXAMPLE_PYTHON_REQ_INFOS] == results + assert [ + Constraint(req.line, req.requirement) for req in EXPECTED_EXAMPLE_PYTHON_REQ_INFOS + ] == results def test_parse_requirement_file_from_url():