From 1dfaceb7d95fd4ef4f169a5a2db6ecef99a48d32 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 29 Jun 2018 19:20:11 -0400 Subject: [PATCH] Update piptools to avoid reusing `InstallRequirement` - Prevents re-preparation of deleted `PKG-INFO` files - Fixes #2435 Signed-off-by: Dan Ryan --- news/2480.bugfix | 1 + pipenv/patched/piptools/resolver.py | 11 ++- pipenv/patched/piptools/utils.py | 41 +++++++++ pipenv/resolver.py | 2 +- .../vendoring/patches/patched/piptools.patch | 86 +++++++++++++++---- 5 files changed, 117 insertions(+), 24 deletions(-) create mode 100644 news/2480.bugfix diff --git a/news/2480.bugfix b/news/2480.bugfix new file mode 100644 index 0000000000..9e60e9b2ad --- /dev/null +++ b/news/2480.bugfix @@ -0,0 +1 @@ +Resolved a long-standing issue with re-using previously generated ``InstallRequirement`` objects for resolution which could cause ``PKG-INFO`` file information to be deleted, raising a ``TypeError``. diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 2ba85f4bf4..61c49f2941 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -15,7 +15,7 @@ from .cache import DependencyCache from .exceptions import UnsupportedConstraint from .logging import log -from .utils import (format_requirement, format_specifier, full_groupby, dedup, +from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) green = partial(click.style, fg='green') @@ -274,18 +274,18 @@ def _iter_dependencies(self, ireq): Editable requirements will never be looked up, as they may have changed at any time. """ + _iter_ireq = simplify_markers(ireq) if ireq.editable: - for dependency in self.repository.get_dependencies(ireq): + for dependency in self.repository.get_dependencies(_iter_ireq): yield dependency return elif ireq.markers: - for dependency in self.repository.get_dependencies(ireq): + for dependency in self.repository.get_dependencies(_iter_ireq): dependency.prepared = False yield dependency - return elif ireq.extras: valid_markers = default_environment().keys() - for dependency in self.repository.get_dependencies(ireq): + for dependency in self.repository.get_dependencies(_iter_ireq): dependency.prepared = False if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers): dependency.markers = None @@ -296,7 +296,6 @@ def _iter_dependencies(self, ireq): ireq.extras = ireq.extra yield dependency - return elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index 0ee22a56ce..4a16e30dfe 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -2,6 +2,7 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import six import os import sys from itertools import chain, groupby @@ -13,12 +14,52 @@ from first import first from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version +from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable from .click import style UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} +def simplify_markers(ireq): + """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`" + + Clean and deduplicate markers. + + :param ireq: An InstallRequirement to clean + :type ireq: :class:`~pip._internal.req.req_install.InstallRequirement` + :return: An InstallRequirement with cleaned Markers + :rtype: :class:`~pip._internal.req.req_install.InstallRequirement` + """ + + if not getattr(ireq, 'markers', None): + return ireq + markers = ireq.markers + marker_list = [] + if isinstance(markers, six.string_types): + if ';' in markers: + markers = [Marker(m_str.strip()) for m_str in markers.split(';')] + else: + markers = Marker(markers) + for m in markers._markers: + _single_marker = [] + if isinstance(m[0], six.string_types): + continue + if not isinstance(m[0], (list, tuple)): + marker_list.append(''.join([_piece.serialize() for _piece in m])) + continue + for _marker_part in m: + if isinstance(_marker_part, six.string_types): + _single_marker.append(_marker_part) + continue + _single_marker.append(''.join([_piece.serialize() for _piece in _marker_part])) + _single_marker = [_m.strip() for _m in _single_marker] + marker_list.append(tuple(_single_marker,)) + marker_str = ' and '.join(list(dedup(tuple(marker_list,)))) if marker_list else '' + new_markers = Marker(marker_str) + return make_install_requirement(ireq.name, first(ireq.specifier).version, ireq.extras, new_markers, constraint=ireq.constraint) + + def clean_requires_python(candidates): """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" all_candidates = [] diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 234697b855..9f06caa3b4 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -34,7 +34,6 @@ def main(): new_sys_argv.append(v) sys.argv = new_sys_argv - from .utils import create_mirror_source, resolve_deps, replace_pypi_sources os.environ['PIP_PYTHON_VERSION'] = '.'.join([str(s) for s in sys.version_info[:3]]) os.environ['PIP_PYTHON_PATH'] = sys.executable if is_verbose: @@ -49,6 +48,7 @@ def main(): for i, package in enumerate(packages): if package.startswith('--'): del packages[i] + from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources pypi_mirror_source = create_mirror_source(os.environ['PIPENV_PYPI_MIRROR']) if 'PIPENV_PYPI_MIRROR' in os.environ else None def resolve(packages, pre, project, sources, verbose, clear, system): diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index b1ba98a576..cad127f584 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..c922be1 100644 +index 1c4b943..c922be1 10064 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, @@ -406,7 +406,7 @@ index 1c4b943..c922be1 100644 def allow_all_wheels(self): """ diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py -index 05ec8fd..c5eb728 100644 +index 05ec8fd..465414c 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -8,13 +8,14 @@ from itertools import chain, count @@ -421,7 +421,7 @@ index 05ec8fd..c5eb728 100644 from .exceptions import UnsupportedConstraint from .logging import log -from .utils import (format_requirement, format_specifier, full_groupby, -+from .utils import (format_requirement, format_specifier, full_groupby, dedup, ++from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) green = partial(click.style, fg='green') @@ -473,20 +473,23 @@ index 05ec8fd..c5eb728 100644 # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq -@@ -271,6 +276,25 @@ class Resolver(object): +@@ -269,10 +274,28 @@ class Resolver(object): + Editable requirements will never be looked up, as they may have + changed at any time. """ ++ _iter_ireq = simplify_markers(ireq) if ireq.editable: - for dependency in self.repository.get_dependencies(ireq): -+ yield dependency -+ return +- for dependency in self.repository.get_dependencies(ireq): ++ for dependency in self.repository.get_dependencies(_iter_ireq): + yield dependency + return + elif ireq.markers: -+ for dependency in self.repository.get_dependencies(ireq): ++ for dependency in self.repository.get_dependencies(_iter_ireq): + dependency.prepared = False + yield dependency -+ return + elif ireq.extras: + valid_markers = default_environment().keys() -+ for dependency in self.repository.get_dependencies(ireq): ++ for dependency in self.repository.get_dependencies(_iter_ireq): + dependency.prepared = False + if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers): + dependency.markers = None @@ -496,10 +499,11 @@ index 05ec8fd..c5eb728 100644 + else: + ireq.extras = ireq.extra + - yield dependency - return ++ yield dependency elif not is_pinned_requirement(ireq): -@@ -283,14 +307,25 @@ class Resolver(object): + raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) + +@@ -283,14 +306,25 @@ class Resolver(object): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') dependencies = self.repository.get_dependencies(ireq) @@ -541,22 +545,70 @@ index 08dabe1..480ad1e 100644 else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py -index fde5816..fb71882 100644 +index fde5816..8732673 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py -@@ -11,13 +11,35 @@ from contextlib import contextmanager +@@ -2,6 +2,7 @@ + from __future__ import (absolute_import, division, print_function, + unicode_literals) + ++import six + import os + import sys + from itertools import chain, groupby +@@ -11,13 +12,75 @@ from contextlib import contextmanager from ._compat import InstallRequirement from first import first - +from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier +from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version ++from pip._vendor.packaging.markers import Marker, Op, Value, Variable from .click import style UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} ++def simplify_markers(ireq): ++ """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`" ++ ++ Clean and deduplicate markers. ++ ++ :param ireq: An InstallRequirement to clean ++ :type ireq: :class:`~pip._internal.req.req_install.InstallRequirement` ++ :return: An InstallRequirement with cleaned Markers ++ :rtype: :class:`~pip._internal.req.req_install.InstallRequirement` ++ """ ++ ++ if not getattr(ireq, 'markers', None): ++ return ireq ++ markers = ireq.markers ++ marker_list = [] ++ if isinstance(markers, six.string_types): ++ if ';' in markers: ++ markers = [Marker(m_str.strip()) for m_str in markers.split(';')] ++ else: ++ markers = Marker(markers) ++ for m in markers._markers: ++ _single_marker = [] ++ if isinstance(m[0], six.string_types): ++ continue ++ if not isinstance(m[0], (list, tuple)): ++ marker_list.append(''.join([_piece.serialize() for _piece in m])) ++ continue ++ for _marker_part in m: ++ if isinstance(_marker_part, six.string_types): ++ _single_marker.append(_marker_part) ++ continue ++ _single_marker.append(''.join([_piece.serialize() for _piece in _marker_part])) ++ _single_marker = [_m.strip() for _m in _single_marker] ++ marker_list.append(tuple(_single_marker,)) ++ marker_str = ' and '.join(list(dedup(tuple(marker_list,)))) if marker_list else '' ++ new_markers = Marker(marker_str) ++ return make_install_requirement(ireq.name, first(ireq.specifier).version, ireq.extras, new_markers, constraint=ireq.constraint) ++ ++ +def clean_requires_python(candidates): + """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" + all_candidates = [] @@ -581,7 +633,7 @@ index fde5816..fb71882 100644 def key_from_ireq(ireq): """Get a standardized key for an InstallRequirement.""" if ireq.req is None and ireq.link is not None: -@@ -43,16 +65,51 @@ def comment(text): +@@ -43,16 +106,51 @@ def comment(text): return style(text, fg='green') @@ -637,7 +689,7 @@ index fde5816..fb71882 100644 def format_requirement(ireq, marker=None): -@@ -63,10 +120,10 @@ def format_requirement(ireq, marker=None): +@@ -63,10 +161,10 @@ def format_requirement(ireq, marker=None): if ireq.editable: line = '-e {}'.format(ireq.link) else: