From bba2f38d91b0eba9c752b3b4517badba9b14bdc6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 25 May 2018 20:20:55 -0400 Subject: [PATCH] Patch piptools to use current environment python - Fixes #2088, #2234, #1901 - Fully leverage piptools' compile functionality by using constraints in the same `RequirementSet` during resolution - Use `PIP_PYTHON_PATH` for compatibility check to filter out `requires_python` markers - Fix vcs resolution - Update JSON API endpoints - Enhance resolution for editable dependencies - Minor fix for adding packages to pipfiles Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 36 ++++++--- pipenv/project.py | 6 +- pipenv/utils.py | 56 ++++++-------- .../vendoring/patches/patched/piptools.patch | 75 +++++++++++++------ 4 files changed, 106 insertions(+), 67 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 6355f959e0..ea709a1bcf 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -4,6 +4,7 @@ import hashlib import os +import sys from contextlib import contextmanager from shutil import rmtree @@ -20,14 +21,17 @@ SafeFileCache, ) -from notpip._vendor.packaging.requirements import InvalidRequirement +from notpip._vendor.packaging.requirements import InvalidRequirement, Requirement +from notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version +from notpip._vendor.packaging.specifiers import SpecifierSet from notpip._vendor.pyparsing import ParseException from ..cache import CACHE_DIR from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound -from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, + make_install_requirement, format_requirement, dedup) + from .base import BaseRepository @@ -159,7 +163,15 @@ def find_best_match(self, ireq, prereleases=None): if ireq.editable: return ireq # return itself as the best match - all_candidates = self.find_all_candidates(ireq.name) + _all_candidates = self.find_all_candidates(ireq.name) + all_candidates = [] + py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3]))) + for c in _all_candidates: + if c.requires_python: + python_specifier = SpecifierSet(c.requires_python) + if not python_specifier.contains(py_version): + continue + all_candidates.append(c) candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) try: matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), @@ -194,11 +206,12 @@ def gen(ireq): r = self.session.get(url) # TODO: Latest isn't always latest. - latest = list(r.json()['releases'].keys())[-1] - if str(ireq.req.specifier) == '=={0}'.format(latest): - latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest) - latest_requires = self.session.get(latest_url) - for requires in latest_requires.json().get('info', {}).get('requires_dist', {}): + releases = list(r.json()['releases'].keys()) + match = [r for r in releases if '=={0}'.format(r) == str(ireq.req.specifier)] + if match: + release_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, match[0]) + release_requires = self.session.get(release_url) + for requires in release_requires.json().get('info', {}).get('requires_dist', {}): i = InstallRequirement.from_line(requires) if 'extra' not in repr(i.markers): @@ -245,7 +258,10 @@ def get_legacy_dependencies(self, ireq): setup_requires = self.finder.get_extras_links( dist.get_metadata_lines('requires.txt') ) - except TypeError: + ireq.version = dist.version + ireq.project_name = dist.project_name + ireq.req = dist.as_requirement() + except (TypeError, ValueError): pass if ireq not in self._dependencies_cache: diff --git a/pipenv/project.py b/pipenv/project.py index 986b7683c0..bf86b10c97 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -20,7 +20,7 @@ from pathlib2 import Path from .cmdparse import Script -from .vendor.requirementslib import Requirement +from .vendor.requirementslib.requirements import Requirement from .utils import ( atomic_open_for_write, mkdir_p, @@ -728,8 +728,8 @@ def add_package_to_pipfile(self, package_name, dev=False): # Read and append Pipfile. p = self.parsed_pipfile # Don't re-capitalize file URLs or VCSs. - package = Requirement.from_line(package_name) - converted = first(package.as_pipfile().values()) + package = Requirement.from_line(package_name.strip()) + _, converted = package.pipfile_entry key = 'dev-packages' if dev else 'packages' # Set empty group if it doesn't exist yet. if key not in p: diff --git a/pipenv/utils.py b/pipenv/utils.py index 01cf6001ca..e0a8070b7d 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -221,6 +221,7 @@ def actually_resolve_reps( ): from .patched.notpip._internal import basecommand from .patched.notpip._internal.req import parse_requirements + from .patched.notpip._internal.req.req_install import InstallRequirement from .patched.notpip._vendor import requests as pip_requests from .patched.notpip._internal.exceptions import DistributionNotFound from .patched.notpip._vendor.requests.exceptions import HTTPError @@ -236,48 +237,37 @@ class PipCommand(basecommand.Command): name = 'PipCommand' constraints = [] - tmpfile_constraints = [] cleanup_req_dir = False if not req_dir: req_dir = TemporaryDirectory(suffix='-requirements', prefix='pipenv-') cleanup_req_dir = True for dep in deps: if dep: - if dep.startswith('-e '): - constraint = req.InstallRequirement.from_editable( - dep[len('-e '):] - ) - else: - tmpfile_constraints.append(dep) - req = Requirement.from_line(dep) - # extra_constraints = [] + url = None if ' -i ' in dep: - index_lookup[req.name] = project.get_source( - url=dep.split(' -i ')[1] - ).get( - 'name' - ) - if dep.markers: - markers_lookup[dep.name] = str( - dep.markers_as_pip - ).replace( - '"', "'" - ) - constraints.append(req) + dep, url = dep.split(' -i ') + req = Requirement.from_line(dep) + _line = req.as_line() + constraints.append(_line) + # extra_constraints = [] + if url: + index_lookup[req.name] = project.get_source(url=url).get('name') + if req.markers: + markers_lookup[req.name] = req.markers_as_pip constraints_file = None - with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f: - f.write('\n'.join(tmpfile_constraints)) - constraints_file = f.name pip_command = get_pip_command() pip_args = [] if sources: pip_args = prepare_pip_source_args(sources, pip_args) + with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f: + f.write(u'\n'.join([_constraint for _constraint in constraints])) + constraints_file = f.name if verbose: print('Using pip: {0}'.format(' '.join(pip_args))) pip_options, _ = pip_command.parse_args(pip_args) session = pip_command._build_session(pip_options) pypi = PyPIRepository( - pip_options=pip_options, use_json=False, session=session + pip_options=pip_options, use_json=True, session=session ) if verbose: logging.log.verbose = True @@ -1138,7 +1128,7 @@ def install_or_update_vcs(vcs_obj, src_dir, name, rev=None): def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=False, pre=False, allow_global=False, dev=False): - from ._compat import vcs + from .patched.notpip._internal.vcs import VcsSupport section = 'vcs_dev_packages' if dev else 'vcs_packages' lines = [] lockfiles = [] @@ -1146,7 +1136,7 @@ def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=Fals packages = getattr(project, section) except AttributeError: return [], [] - vcs_registry = vcs() + vcs_registry = VcsSupport vcs_uri_map = { extract_uri_from_vcs_dep(v): {'name': k, 'ref': v.get('ref')} for k, v in packages.items() @@ -1162,13 +1152,15 @@ def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=Fals pipfile_rev = vcs_uri_map[_vcs_match]['ref'] src_dir = os.environ.get('PIP_SRC', os.path.join(project.virtualenv_location, 'src')) mkdir_p(src_dir) + pipfile_req = Requirement.from_pipfile(pipfile_name, [], packages[pipfile_name]) names = {pipfile_name} - _pip_uri = line.lstrip('-e ') - backend_name = str(_pip_uri.split('+', 1)[0]) - backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)] - __vcs = backend(url=_pip_uri) - + backend = vcs_registry()._registry.get(pipfile_req.vcs) + # TODO: Why doesn't pip freeze list 'git+git://' formatted urls? + if line.startswith('-e ') and not '{0}+'.format(pipfile_req.vcs) in line: + line = line.replace('-e ', '-e {0}+'.format(pipfile_req.vcs)) installed = Requirement.from_line(line) + __vcs = backend(url=installed.req.uri) + names.add(installed.normalized_name) locked_rev = None for _name in names: diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 6beaad8ea8..a3220d3931 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,10 +19,18 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..8320e14 100644 +index 1c4b943..858d697 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -15,10 +15,16 @@ from .._compat import ( +@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, + + import hashlib + import os ++import sys + from contextlib import contextmanager + from shutil import rmtree + +@@ -15,13 +16,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -32,15 +40,23 @@ index 1c4b943..8320e14 100644 + SafeFileCache, ) -+from pip._vendor.packaging.requirements import InvalidRequirement ++from pip._vendor.packaging.requirements import InvalidRequirement, Requirement ++from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version ++from pip._vendor.packaging.specifiers import SpecifierSet +from pip._vendor.pyparsing import ParseException + from ..cache import CACHE_DIR +from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound - from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) -@@ -37,6 +43,40 @@ except ImportError: +-from ..utils import (fs_str, is_pinned_requirement, lookup_table, +- make_install_requirement) ++from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, ++ make_install_requirement, format_requirement, dedup) ++ + from .base import BaseRepository + + +@@ -37,6 +47,40 @@ except ImportError: from pip.wheel import WheelCache @@ -81,7 +97,7 @@ index 1c4b943..8320e14 100644 class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url -@@ -46,10 +86,11 @@ class PyPIRepository(BaseRepository): +@@ -46,10 +90,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -95,7 +111,7 @@ index 1c4b943..8320e14 100644 index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: -@@ -74,11 +115,15 @@ class PyPIRepository(BaseRepository): +@@ -74,11 +119,15 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -113,9 +129,20 @@ index 1c4b943..8320e14 100644 def freshen_build_caches(self): """ -@@ -116,8 +161,11 @@ class PyPIRepository(BaseRepository): - - all_candidates = self.find_all_candidates(ireq.name) +@@ -114,10 +163,21 @@ class PyPIRepository(BaseRepository): + if ireq.editable: + return ireq # return itself as the best match + +- all_candidates = self.find_all_candidates(ireq.name) ++ _all_candidates = self.find_all_candidates(ireq.name) ++ all_candidates = [] ++ py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3]))) ++ for c in _all_candidates: ++ if c.requires_python: ++ python_specifier = SpecifierSet(c.requires_python) ++ if not python_specifier.contains(py_version): ++ continue ++ all_candidates.append(c) candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) - matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), + try: @@ -126,7 +153,7 @@ index 1c4b943..8320e14 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +174,60 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +186,61 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -153,11 +180,12 @@ index 1c4b943..8320e14 100644 + r = self.session.get(url) + + # TODO: Latest isn't always latest. -+ latest = list(r.json()['releases'].keys())[-1] -+ if str(ireq.req.specifier) == '=={0}'.format(latest): -+ latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest) -+ latest_requires = self.session.get(latest_url) -+ for requires in latest_requires.json().get('info', {}).get('requires_dist', {}): ++ releases = list(r.json()['releases'].keys()) ++ match = [r for r in releases if '=={0}'.format(r) == str(ireq.req.specifier)] ++ if match: ++ release_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, match[0]) ++ release_requires = self.session.get(release_url) ++ for requires in release_requires.json().get('info', {}).get('requires_dist', {}): + i = InstallRequirement.from_line(requires) + + if 'extra' not in repr(i.markers): @@ -190,7 +218,7 @@ index 1c4b943..8320e14 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -139,6 +236,18 @@ class PyPIRepository(BaseRepository): +@@ -139,6 +249,21 @@ class PyPIRepository(BaseRepository): if not (ireq.editable or is_pinned_requirement(ireq)): raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq)) @@ -203,13 +231,16 @@ index 1c4b943..8320e14 100644 + setup_requires = self.finder.get_extras_links( + dist.get_metadata_lines('requires.txt') + ) -+ except TypeError: ++ ireq.version = dist.version ++ ireq.project_name = dist.project_name ++ ireq.req = dist.as_requirement() ++ except (TypeError, ValueError): + pass + if ireq not in self._dependencies_cache: if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # No download_dir for locally available editable requirements. -@@ -164,11 +273,14 @@ class PyPIRepository(BaseRepository): +@@ -164,11 +289,14 @@ class PyPIRepository(BaseRepository): download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, @@ -226,7 +257,7 @@ index 1c4b943..8320e14 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -190,14 +302,44 @@ class PyPIRepository(BaseRepository): +@@ -190,14 +318,44 @@ class PyPIRepository(BaseRepository): upgrade_strategy="to-satisfy-only", force_reinstall=False, ignore_dependencies=False, @@ -273,7 +304,7 @@ index 1c4b943..8320e14 100644 reqset.cleanup_files() return set(self._dependencies_cache[ireq]) -@@ -224,17 +366,10 @@ class PyPIRepository(BaseRepository): +@@ -224,17 +382,10 @@ class PyPIRepository(BaseRepository): matching_candidates = candidates_by_version[matching_versions[0]] return {