From 8ab6008e9ad2afde81b59bb5a78364f2dd299948 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 29 Jun 2018 03:16:26 -0400 Subject: [PATCH 01/11] Fix resolver imports Signed-off-by: Dan Ryan --- pipenv/resolver.py | 2 +- pipenv/utils.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 337ef22bf7..991bd22ba1 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -34,7 +34,7 @@ def main(): new_sys_argv.append(v) sys.argv = new_sys_argv - from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources + from .utils import create_mirror_source, resolve_deps, replace_pypi_sources if is_verbose: logging.getLogger('notpip').setLevel(logging.INFO) diff --git a/pipenv/utils.py b/pipenv/utils.py index 2068a227b4..e369c644c4 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -916,9 +916,11 @@ def is_valid_url(url): def is_pypi_url(url): return bool(re.match(r'^http[s]?:\/\/pypi(?:\.python)?\.org\/simple[\/]?$', url)) + def replace_pypi_sources(sources, pypi_replacement_source): return [pypi_replacement_source] + [source for source in sources if not is_pypi_url(source['url'])] + def create_mirror_source(url): return {'url': url, 'verify_ssl': url.startswith('https://'), 'name': urlparse(url).hostname} From 1dfaceb7d95fd4ef4f169a5a2db6ecef99a48d32 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 29 Jun 2018 19:20:11 -0400 Subject: [PATCH 02/11] Update piptools to avoid reusing `InstallRequirement` - Prevents re-preparation of deleted `PKG-INFO` files - Fixes #2435 Signed-off-by: Dan Ryan --- news/2480.bugfix | 1 + pipenv/patched/piptools/resolver.py | 11 ++- pipenv/patched/piptools/utils.py | 41 +++++++++ pipenv/resolver.py | 2 +- .../vendoring/patches/patched/piptools.patch | 86 +++++++++++++++---- 5 files changed, 117 insertions(+), 24 deletions(-) create mode 100644 news/2480.bugfix diff --git a/news/2480.bugfix b/news/2480.bugfix new file mode 100644 index 0000000000..9e60e9b2ad --- /dev/null +++ b/news/2480.bugfix @@ -0,0 +1 @@ +Resolved a long-standing issue with re-using previously generated ``InstallRequirement`` objects for resolution which could cause ``PKG-INFO`` file information to be deleted, raising a ``TypeError``. diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 2ba85f4bf4..61c49f2941 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -15,7 +15,7 @@ from .cache import DependencyCache from .exceptions import UnsupportedConstraint from .logging import log -from .utils import (format_requirement, format_specifier, full_groupby, dedup, +from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) green = partial(click.style, fg='green') @@ -274,18 +274,18 @@ def _iter_dependencies(self, ireq): Editable requirements will never be looked up, as they may have changed at any time. """ + _iter_ireq = simplify_markers(ireq) if ireq.editable: - for dependency in self.repository.get_dependencies(ireq): + for dependency in self.repository.get_dependencies(_iter_ireq): yield dependency return elif ireq.markers: - for dependency in self.repository.get_dependencies(ireq): + for dependency in self.repository.get_dependencies(_iter_ireq): dependency.prepared = False yield dependency - return elif ireq.extras: valid_markers = default_environment().keys() - for dependency in self.repository.get_dependencies(ireq): + for dependency in self.repository.get_dependencies(_iter_ireq): dependency.prepared = False if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers): dependency.markers = None @@ -296,7 +296,6 @@ def _iter_dependencies(self, ireq): ireq.extras = ireq.extra yield dependency - return elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index 0ee22a56ce..4a16e30dfe 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -2,6 +2,7 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import six import os import sys from itertools import chain, groupby @@ -13,12 +14,52 @@ from first import first from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version +from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable from .click import style UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} +def simplify_markers(ireq): + """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`" + + Clean and deduplicate markers. + + :param ireq: An InstallRequirement to clean + :type ireq: :class:`~pip._internal.req.req_install.InstallRequirement` + :return: An InstallRequirement with cleaned Markers + :rtype: :class:`~pip._internal.req.req_install.InstallRequirement` + """ + + if not getattr(ireq, 'markers', None): + return ireq + markers = ireq.markers + marker_list = [] + if isinstance(markers, six.string_types): + if ';' in markers: + markers = [Marker(m_str.strip()) for m_str in markers.split(';')] + else: + markers = Marker(markers) + for m in markers._markers: + _single_marker = [] + if isinstance(m[0], six.string_types): + continue + if not isinstance(m[0], (list, tuple)): + marker_list.append(''.join([_piece.serialize() for _piece in m])) + continue + for _marker_part in m: + if isinstance(_marker_part, six.string_types): + _single_marker.append(_marker_part) + continue + _single_marker.append(''.join([_piece.serialize() for _piece in _marker_part])) + _single_marker = [_m.strip() for _m in _single_marker] + marker_list.append(tuple(_single_marker,)) + marker_str = ' and '.join(list(dedup(tuple(marker_list,)))) if marker_list else '' + new_markers = Marker(marker_str) + return make_install_requirement(ireq.name, first(ireq.specifier).version, ireq.extras, new_markers, constraint=ireq.constraint) + + def clean_requires_python(candidates): """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" all_candidates = [] diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 234697b855..9f06caa3b4 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -34,7 +34,6 @@ def main(): new_sys_argv.append(v) sys.argv = new_sys_argv - from .utils import create_mirror_source, resolve_deps, replace_pypi_sources os.environ['PIP_PYTHON_VERSION'] = '.'.join([str(s) for s in sys.version_info[:3]]) os.environ['PIP_PYTHON_PATH'] = sys.executable if is_verbose: @@ -49,6 +48,7 @@ def main(): for i, package in enumerate(packages): if package.startswith('--'): del packages[i] + from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources pypi_mirror_source = create_mirror_source(os.environ['PIPENV_PYPI_MIRROR']) if 'PIPENV_PYPI_MIRROR' in os.environ else None def resolve(packages, pre, project, sources, verbose, clear, system): diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index b1ba98a576..cad127f584 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..c922be1 100644 +index 1c4b943..c922be1 10064 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, @@ -406,7 +406,7 @@ index 1c4b943..c922be1 100644 def allow_all_wheels(self): """ diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py -index 05ec8fd..c5eb728 100644 +index 05ec8fd..465414c 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -8,13 +8,14 @@ from itertools import chain, count @@ -421,7 +421,7 @@ index 05ec8fd..c5eb728 100644 from .exceptions import UnsupportedConstraint from .logging import log -from .utils import (format_requirement, format_specifier, full_groupby, -+from .utils import (format_requirement, format_specifier, full_groupby, dedup, ++from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) green = partial(click.style, fg='green') @@ -473,20 +473,23 @@ index 05ec8fd..c5eb728 100644 # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq -@@ -271,6 +276,25 @@ class Resolver(object): +@@ -269,10 +274,28 @@ class Resolver(object): + Editable requirements will never be looked up, as they may have + changed at any time. """ ++ _iter_ireq = simplify_markers(ireq) if ireq.editable: - for dependency in self.repository.get_dependencies(ireq): -+ yield dependency -+ return +- for dependency in self.repository.get_dependencies(ireq): ++ for dependency in self.repository.get_dependencies(_iter_ireq): + yield dependency + return + elif ireq.markers: -+ for dependency in self.repository.get_dependencies(ireq): ++ for dependency in self.repository.get_dependencies(_iter_ireq): + dependency.prepared = False + yield dependency -+ return + elif ireq.extras: + valid_markers = default_environment().keys() -+ for dependency in self.repository.get_dependencies(ireq): ++ for dependency in self.repository.get_dependencies(_iter_ireq): + dependency.prepared = False + if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers): + dependency.markers = None @@ -496,10 +499,11 @@ index 05ec8fd..c5eb728 100644 + else: + ireq.extras = ireq.extra + - yield dependency - return ++ yield dependency elif not is_pinned_requirement(ireq): -@@ -283,14 +307,25 @@ class Resolver(object): + raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) + +@@ -283,14 +306,25 @@ class Resolver(object): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') dependencies = self.repository.get_dependencies(ireq) @@ -541,22 +545,70 @@ index 08dabe1..480ad1e 100644 else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py -index fde5816..fb71882 100644 +index fde5816..8732673 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py -@@ -11,13 +11,35 @@ from contextlib import contextmanager +@@ -2,6 +2,7 @@ + from __future__ import (absolute_import, division, print_function, + unicode_literals) + ++import six + import os + import sys + from itertools import chain, groupby +@@ -11,13 +12,75 @@ from contextlib import contextmanager from ._compat import InstallRequirement from first import first - +from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier +from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version ++from pip._vendor.packaging.markers import Marker, Op, Value, Variable from .click import style UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} ++def simplify_markers(ireq): ++ """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`" ++ ++ Clean and deduplicate markers. ++ ++ :param ireq: An InstallRequirement to clean ++ :type ireq: :class:`~pip._internal.req.req_install.InstallRequirement` ++ :return: An InstallRequirement with cleaned Markers ++ :rtype: :class:`~pip._internal.req.req_install.InstallRequirement` ++ """ ++ ++ if not getattr(ireq, 'markers', None): ++ return ireq ++ markers = ireq.markers ++ marker_list = [] ++ if isinstance(markers, six.string_types): ++ if ';' in markers: ++ markers = [Marker(m_str.strip()) for m_str in markers.split(';')] ++ else: ++ markers = Marker(markers) ++ for m in markers._markers: ++ _single_marker = [] ++ if isinstance(m[0], six.string_types): ++ continue ++ if not isinstance(m[0], (list, tuple)): ++ marker_list.append(''.join([_piece.serialize() for _piece in m])) ++ continue ++ for _marker_part in m: ++ if isinstance(_marker_part, six.string_types): ++ _single_marker.append(_marker_part) ++ continue ++ _single_marker.append(''.join([_piece.serialize() for _piece in _marker_part])) ++ _single_marker = [_m.strip() for _m in _single_marker] ++ marker_list.append(tuple(_single_marker,)) ++ marker_str = ' and '.join(list(dedup(tuple(marker_list,)))) if marker_list else '' ++ new_markers = Marker(marker_str) ++ return make_install_requirement(ireq.name, first(ireq.specifier).version, ireq.extras, new_markers, constraint=ireq.constraint) ++ ++ +def clean_requires_python(candidates): + """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" + all_candidates = [] @@ -581,7 +633,7 @@ index fde5816..fb71882 100644 def key_from_ireq(ireq): """Get a standardized key for an InstallRequirement.""" if ireq.req is None and ireq.link is not None: -@@ -43,16 +65,51 @@ def comment(text): +@@ -43,16 +106,51 @@ def comment(text): return style(text, fg='green') @@ -637,7 +689,7 @@ index fde5816..fb71882 100644 def format_requirement(ireq, marker=None): -@@ -63,10 +120,10 @@ def format_requirement(ireq, marker=None): +@@ -63,10 +161,10 @@ def format_requirement(ireq, marker=None): if ireq.editable: line = '-e {}'.format(ireq.link) else: From 7761c3d321c14d28dd1f20e3c4b83968b0809de1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 29 Jun 2018 20:23:54 -0400 Subject: [PATCH 03/11] Fix patches Signed-off-by: Dan Ryan --- pipenv/patched/piptools/resolver.py | 9 ++-- pipenv/patched/piptools/utils.py | 8 +++- .../vendoring/patches/patched/piptools.patch | 42 ++++++++++--------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 61c49f2941..00b08705fb 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -281,12 +281,13 @@ def _iter_dependencies(self, ireq): return elif ireq.markers: for dependency in self.repository.get_dependencies(_iter_ireq): - dependency.prepared = False + # dependency.prepared = False yield dependency + return elif ireq.extras: valid_markers = default_environment().keys() for dependency in self.repository.get_dependencies(_iter_ireq): - dependency.prepared = False + # dependency.prepared = False if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers): dependency.markers = None if hasattr(ireq, 'extra'): @@ -296,6 +297,7 @@ def _iter_dependencies(self, ireq): ireq.extras = ireq.extra yield dependency + return elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) @@ -306,8 +308,7 @@ def _iter_dependencies(self, ireq): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') dependencies = self.repository.get_dependencies(ireq) - import sys - self.dependency_cache[ireq] = sorted(format_requirement(ireq) for ireq in dependencies) + self.dependency_cache[ireq] = sorted(format_requirement(_ireq) for _ireq in dependencies) # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4'] dependency_strings = self.dependency_cache[ireq] diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index 4a16e30dfe..2f389eecaf 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -57,7 +57,11 @@ def simplify_markers(ireq): marker_list.append(tuple(_single_marker,)) marker_str = ' and '.join(list(dedup(tuple(marker_list,)))) if marker_list else '' new_markers = Marker(marker_str) - return make_install_requirement(ireq.name, first(ireq.specifier).version, ireq.extras, new_markers, constraint=ireq.constraint) + ireq.markers = new_markers + new_ireq = InstallRequirement.from_line(format_requirement(ireq)) + if ireq.constraint: + new_ireq.constraint = ireq.constraint + return new_ireq def clean_requires_python(candidates): @@ -163,7 +167,7 @@ def format_requirement(ireq, marker=None): else: line = _requirement_to_str_lowercase_name(ireq.req) - if marker: + if marker and ';' not in line: line = '{}; {}'.format(line, marker) return line diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index cad127f584..289071aa43 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..c922be1 10064 +index 1c4b943..c922be1 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, @@ -406,7 +406,7 @@ index 1c4b943..c922be1 10064 def allow_all_wheels(self): """ diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py -index 05ec8fd..465414c 100644 +index 05ec8fd..3100f9d 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -8,13 +8,14 @@ from itertools import chain, count @@ -473,7 +473,7 @@ index 05ec8fd..465414c 100644 # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq -@@ -269,10 +274,28 @@ class Resolver(object): +@@ -269,8 +274,28 @@ class Resolver(object): Editable requirements will never be looked up, as they may have changed at any time. """ @@ -481,16 +481,17 @@ index 05ec8fd..465414c 100644 if ireq.editable: - for dependency in self.repository.get_dependencies(ireq): + for dependency in self.repository.get_dependencies(_iter_ireq): - yield dependency - return ++ yield dependency ++ return + elif ireq.markers: + for dependency in self.repository.get_dependencies(_iter_ireq): -+ dependency.prepared = False ++ # dependency.prepared = False + yield dependency ++ return + elif ireq.extras: + valid_markers = default_environment().keys() + for dependency in self.repository.get_dependencies(_iter_ireq): -+ dependency.prepared = False ++ # dependency.prepared = False + if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers): + dependency.markers = None + if hasattr(ireq, 'extra'): @@ -499,17 +500,15 @@ index 05ec8fd..465414c 100644 + else: + ireq.extras = ireq.extra + -+ yield dependency + yield dependency + return elif not is_pinned_requirement(ireq): - raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) - -@@ -283,14 +306,25 @@ class Resolver(object): +@@ -283,14 +308,24 @@ class Resolver(object): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') dependencies = self.repository.get_dependencies(ireq) - self.dependency_cache[ireq] = sorted(str(ireq.req) for ireq in dependencies) -+ import sys -+ self.dependency_cache[ireq] = sorted(format_requirement(ireq) for ireq in dependencies) ++ self.dependency_cache[ireq] = sorted(format_requirement(_ireq) for _ireq in dependencies) # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4'] dependency_strings = self.dependency_cache[ireq] @@ -545,7 +544,7 @@ index 08dabe1..480ad1e 100644 else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py -index fde5816..8732673 100644 +index fde5816..23a05f2 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -2,6 +2,7 @@ @@ -556,7 +555,7 @@ index fde5816..8732673 100644 import os import sys from itertools import chain, groupby -@@ -11,13 +12,75 @@ from contextlib import contextmanager +@@ -11,13 +12,79 @@ from contextlib import contextmanager from ._compat import InstallRequirement from first import first @@ -606,7 +605,11 @@ index fde5816..8732673 100644 + marker_list.append(tuple(_single_marker,)) + marker_str = ' and '.join(list(dedup(tuple(marker_list,)))) if marker_list else '' + new_markers = Marker(marker_str) -+ return make_install_requirement(ireq.name, first(ireq.specifier).version, ireq.extras, new_markers, constraint=ireq.constraint) ++ ireq.markers = new_markers ++ new_ireq = InstallRequirement.from_line(format_requirement(ireq)) ++ if ireq.constraint: ++ new_ireq.constraint = ireq.constraint ++ return new_ireq + + +def clean_requires_python(candidates): @@ -633,7 +636,7 @@ index fde5816..8732673 100644 def key_from_ireq(ireq): """Get a standardized key for an InstallRequirement.""" if ireq.req is None and ireq.link is not None: -@@ -43,16 +106,51 @@ def comment(text): +@@ -43,16 +110,51 @@ def comment(text): return style(text, fg='green') @@ -689,15 +692,16 @@ index fde5816..8732673 100644 def format_requirement(ireq, marker=None): -@@ -63,10 +161,10 @@ def format_requirement(ireq, marker=None): +@@ -63,10 +165,10 @@ def format_requirement(ireq, marker=None): if ireq.editable: line = '-e {}'.format(ireq.link) else: - line = str(ireq.req).lower() + line = _requirement_to_str_lowercase_name(ireq.req) - if marker: +- if marker: - line = '{} ; {}'.format(line, marker) ++ if marker and ';' not in line: + line = '{}; {}'.format(line, marker) return line From b9559951b663b934938947ffe7c5af97bd8c7280 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 29 Jun 2018 21:18:34 -0400 Subject: [PATCH 04/11] Fix test failures Signed-off-by: Dan Ryan --- tests/integration/test_install_markers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index ca6b70fe99..48b178f7b4 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -52,7 +52,8 @@ def test_platform_python_implementation_marker(PipenvInstance, pypi): # depends-on-marked-package has an install_requires of 'pytz; platform_python_implementation=="CPython"' # Verify that that marker shows up in our lockfile unaltered. - assert p.lockfile['default']['pytz']['markers'] == "platform_python_implementation == 'CPython'" + assert 'pytz' in p.lockfile['default'] + assert p.lockfile['default']['pytz'].get('markers') == "platform_python_implementation == 'CPython'" @pytest.mark.run From 6e8586032a06b6c12a7476bce3539fa486fb16e9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 29 Jun 2018 21:31:19 -0400 Subject: [PATCH 05/11] Move click-didyoumean.license Signed-off-by: Dan Ryan --- .../vendor/{click-didyoumean.LICENSE => click_didyoumean/LICENSE} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pipenv/vendor/{click-didyoumean.LICENSE => click_didyoumean/LICENSE} (100%) diff --git a/pipenv/vendor/click-didyoumean.LICENSE b/pipenv/vendor/click_didyoumean/LICENSE similarity index 100% rename from pipenv/vendor/click-didyoumean.LICENSE rename to pipenv/vendor/click_didyoumean/LICENSE From ad3aca8b240aec95c527509f4ed30b270b2911ad Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 29 Jun 2018 22:00:06 -0400 Subject: [PATCH 06/11] Include all markers Signed-off-by: Dan Ryan --- pipenv/patched/piptools/resolver.py | 6 +++--- tasks/vendoring/patches/patched/piptools.patch | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 00b08705fb..ab7940a531 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -279,12 +279,12 @@ def _iter_dependencies(self, ireq): for dependency in self.repository.get_dependencies(_iter_ireq): yield dependency return - elif ireq.markers: + if ireq.markers: for dependency in self.repository.get_dependencies(_iter_ireq): # dependency.prepared = False yield dependency - return - elif ireq.extras: + + if ireq.extras: valid_markers = default_environment().keys() for dependency in self.repository.get_dependencies(_iter_ireq): # dependency.prepared = False diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 289071aa43..fb68c6496b 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -406,7 +406,7 @@ index 1c4b943..c922be1 100644 def allow_all_wheels(self): """ diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py -index 05ec8fd..3100f9d 100644 +index 05ec8fd..5655f4c 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -8,13 +8,14 @@ from itertools import chain, count @@ -483,12 +483,12 @@ index 05ec8fd..3100f9d 100644 + for dependency in self.repository.get_dependencies(_iter_ireq): + yield dependency + return -+ elif ireq.markers: ++ if ireq.markers: + for dependency in self.repository.get_dependencies(_iter_ireq): + # dependency.prepared = False + yield dependency -+ return -+ elif ireq.extras: ++ ++ if ireq.extras: + valid_markers = default_environment().keys() + for dependency in self.repository.get_dependencies(_iter_ireq): + # dependency.prepared = False From cbbcc8017093c0ba042ddc0e965c57cba65010a6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 29 Jun 2018 22:40:34 -0400 Subject: [PATCH 07/11] I think this is good? Signed-off-by: Dan Ryan --- pipenv/patched/piptools/resolver.py | 38 ++++++------- .../vendoring/patches/patched/piptools.patch | 53 +++++++++---------- 2 files changed, 44 insertions(+), 47 deletions(-) diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index ab7940a531..d047be4c4e 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -274,30 +274,30 @@ def _iter_dependencies(self, ireq): Editable requirements will never be looked up, as they may have changed at any time. """ - _iter_ireq = simplify_markers(ireq) if ireq.editable: - for dependency in self.repository.get_dependencies(_iter_ireq): + for dependency in self.repository.get_dependencies(ireq): yield dependency return - if ireq.markers: - for dependency in self.repository.get_dependencies(_iter_ireq): - # dependency.prepared = False - yield dependency + ireq = simplify_markers(ireq) + # if ireq.markers: + # for dependency in self.repository.get_dependencies(_iter_ireq): + # # dependency.prepared = False + # yield dependency if ireq.extras: - valid_markers = default_environment().keys() - for dependency in self.repository.get_dependencies(_iter_ireq): - # dependency.prepared = False - if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers): - dependency.markers = None - if hasattr(ireq, 'extra'): - if ireq.extras: - ireq.extras.extend(ireq.extra) - else: - ireq.extras = ireq.extra - - yield dependency - return + if hasattr(ireq, 'extra'): + if ireq.extras: + ireq.extras.extend(ireq.extra) + else: + ireq.extras = ireq.extra + # valid_markers = default_environment().keys() + # for dependency in self.repository.get_dependencies(_iter_ireq): + # # dependency.prepared = False + # if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers): + # dependency.markers = None + + # yield dependency + # return elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index fb68c6496b..75e1c2fe90 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -406,7 +406,7 @@ index 1c4b943..c922be1 100644 def allow_all_wheels(self): """ diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py -index 05ec8fd..5655f4c 100644 +index 05ec8fd..5ed36ae 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -8,13 +8,14 @@ from itertools import chain, count @@ -473,36 +473,33 @@ index 05ec8fd..5655f4c 100644 # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq -@@ -269,8 +274,28 @@ class Resolver(object): - Editable requirements will never be looked up, as they may have - changed at any time. - """ -+ _iter_ireq = simplify_markers(ireq) - if ireq.editable: -- for dependency in self.repository.get_dependencies(ireq): -+ for dependency in self.repository.get_dependencies(_iter_ireq): -+ yield dependency -+ return -+ if ireq.markers: -+ for dependency in self.repository.get_dependencies(_iter_ireq): -+ # dependency.prepared = False -+ yield dependency -+ -+ if ireq.extras: -+ valid_markers = default_environment().keys() -+ for dependency in self.repository.get_dependencies(_iter_ireq): -+ # dependency.prepared = False -+ if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers): -+ dependency.markers = None -+ if hasattr(ireq, 'extra'): -+ if ireq.extras: -+ ireq.extras.extend(ireq.extra) -+ else: -+ ireq.extras = ireq.extra -+ +@@ -273,6 +278,26 @@ class Resolver(object): + for dependency in self.repository.get_dependencies(ireq): yield dependency return ++ ireq = simplify_markers(ireq) ++ # if ireq.markers: ++ # for dependency in self.repository.get_dependencies(_iter_ireq): ++ # # dependency.prepared = False ++ # yield dependency ++ ++ if ireq.extras: ++ if hasattr(ireq, 'extra'): ++ if ireq.extras: ++ ireq.extras.extend(ireq.extra) ++ else: ++ ireq.extras = ireq.extra ++ # valid_markers = default_environment().keys() ++ # for dependency in self.repository.get_dependencies(_iter_ireq): ++ # # dependency.prepared = False ++ # if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers): ++ # dependency.markers = None ++ ++ # yield dependency ++ # return elif not is_pinned_requirement(ireq): + raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) + @@ -283,14 +308,24 @@ class Resolver(object): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') From d5b91f13e705b2ae48f091ad9f3cb6dc4e63f7b3 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 30 Jun 2018 13:46:56 +0800 Subject: [PATCH 08/11] Format --- tests/integration/test_install_markers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index 48b178f7b4..42401c159b 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -37,7 +37,8 @@ def test_package_environment_markers(PipenvInstance, pypi): @pytest.mark.markers @flaky def test_platform_python_implementation_marker(PipenvInstance, pypi): - """Markers should be converted during locking to help users who input this incorrectly + """Markers should be converted during locking to help users who input this + incorrectly. """ with PipenvInstance(pypi=pypi) as p: with open(p.pipfile_path, 'w') as f: @@ -50,10 +51,12 @@ def test_platform_python_implementation_marker(PipenvInstance, pypi): c = p.pipenv('install') assert c.return_code == 0 - # depends-on-marked-package has an install_requires of 'pytz; platform_python_implementation=="CPython"' + # depends-on-marked-package has an install_requires of + # 'pytz; platform_python_implementation=="CPython"' # Verify that that marker shows up in our lockfile unaltered. assert 'pytz' in p.lockfile['default'] - assert p.lockfile['default']['pytz'].get('markers') == "platform_python_implementation == 'CPython'" + assert p.lockfile['default']['pytz'].get('markers') == \ + "platform_python_implementation == 'CPython'" @pytest.mark.run From a797691421715e3ea4bf019d6b3a354e49ab4f14 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 30 Jun 2018 02:48:48 -0400 Subject: [PATCH 09/11] Remove old patches to piptools Signed-off-by: Dan Ryan --- news/2480.vendor | 1 + pipenv/patched/piptools/resolver.py | 24 +-------------- .../vendoring/patches/patched/piptools.patch | 30 +++---------------- 3 files changed, 6 insertions(+), 49 deletions(-) create mode 100644 news/2480.vendor diff --git a/news/2480.vendor b/news/2480.vendor new file mode 100644 index 0000000000..f96f2b3081 --- /dev/null +++ b/news/2480.vendor @@ -0,0 +1 @@ +Unraveled a lot of old, unnecessary patches to ``pip-tools`` which were causing non-deterministic resolution errors. diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index d047be4c4e..5bdb589972 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -278,26 +278,14 @@ def _iter_dependencies(self, ireq): for dependency in self.repository.get_dependencies(ireq): yield dependency return - ireq = simplify_markers(ireq) - # if ireq.markers: - # for dependency in self.repository.get_dependencies(_iter_ireq): - # # dependency.prepared = False - # yield dependency + # fix our malformed extras if ireq.extras: if hasattr(ireq, 'extra'): if ireq.extras: ireq.extras.extend(ireq.extra) else: ireq.extras = ireq.extra - # valid_markers = default_environment().keys() - # for dependency in self.repository.get_dependencies(_iter_ireq): - # # dependency.prepared = False - # if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers): - # dependency.markers = None - - # yield dependency - # return elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) @@ -314,17 +302,7 @@ def _iter_dependencies(self, ireq): dependency_strings = self.dependency_cache[ireq] log.debug(' {:25} requires {}'.format(format_requirement(ireq), ', '.join(sorted(dependency_strings, key=lambda s: s.lower())) or '-')) - from pipenv.patched.notpip._vendor.packaging.markers import InvalidMarker for dependency_string in dependency_strings: - try: - _dependency_string = dependency_string - if ';' in dependency_string: - # split off markers and remove any duplicates by comparing against deps - _dependencies = [dep.strip() for dep in dependency_string.split(';')] - _dependency_string = '; '.join([dep for dep in dedup(_dependencies)]) - - yield InstallRequirement.from_line(_dependency_string, constraint=ireq.constraint) - except InvalidMarker: yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint) def reverse_dependencies(self, ireqs): diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 75e1c2fe90..750a68a70f 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -406,7 +406,7 @@ index 1c4b943..c922be1 100644 def allow_all_wheels(self): """ diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py -index 05ec8fd..5ed36ae 100644 +index 05ec8fd..8952478 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -8,13 +8,14 @@ from itertools import chain, count @@ -473,34 +473,22 @@ index 05ec8fd..5ed36ae 100644 # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq -@@ -273,6 +278,26 @@ class Resolver(object): +@@ -273,6 +278,14 @@ class Resolver(object): for dependency in self.repository.get_dependencies(ireq): yield dependency return -+ ireq = simplify_markers(ireq) -+ # if ireq.markers: -+ # for dependency in self.repository.get_dependencies(_iter_ireq): -+ # # dependency.prepared = False -+ # yield dependency + ++ # fix our malformed extras + if ireq.extras: + if hasattr(ireq, 'extra'): + if ireq.extras: + ireq.extras.extend(ireq.extra) + else: + ireq.extras = ireq.extra -+ # valid_markers = default_environment().keys() -+ # for dependency in self.repository.get_dependencies(_iter_ireq): -+ # # dependency.prepared = False -+ # if dependency.markers and not any(dependency.markers._markers[0][0].value.startswith(k) for k in valid_markers): -+ # dependency.markers = None -+ -+ # yield dependency -+ # return elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) -@@ -283,14 +308,24 @@ class Resolver(object): +@@ -283,14 +296,14 @@ class Resolver(object): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') dependencies = self.repository.get_dependencies(ireq) @@ -511,18 +499,8 @@ index 05ec8fd..5ed36ae 100644 dependency_strings = self.dependency_cache[ireq] log.debug(' {:25} requires {}'.format(format_requirement(ireq), ', '.join(sorted(dependency_strings, key=lambda s: s.lower())) or '-')) -+ from pip._vendor.packaging.markers import InvalidMarker for dependency_string in dependency_strings: - yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint) -+ try: -+ _dependency_string = dependency_string -+ if ';' in dependency_string: -+ # split off markers and remove any duplicates by comparing against deps -+ _dependencies = [dep.strip() for dep in dependency_string.split(';')] -+ _dependency_string = '; '.join([dep for dep in dedup(_dependencies)]) -+ -+ yield InstallRequirement.from_line(_dependency_string, constraint=ireq.constraint) -+ except InvalidMarker: + yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint) def reverse_dependencies(self, ireqs): From 5990f4224e2616cada9fd49802a7e902edb96211 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 30 Jun 2018 15:49:32 -0400 Subject: [PATCH 10/11] Fix markers being dropped Signed-off-by: Dan Ryan --- pipenv/patched/piptools/resolver.py | 7 ++++++- tasks/vendoring/patches/patched/piptools.patch | 15 ++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 5bdb589972..889abce61b 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -160,7 +160,12 @@ def _group_constraints(self, constraints): if ireq.req.specifier._specs and not combined_ireq.req.specifier._specs: combined_ireq.req.specifier._specs = ireq.req.specifier._specs combined_ireq.constraint &= ireq.constraint - combined_ireq.markers = ireq.markers + if not combined_ireq.markers: + combined_ireq.markers = ireq.markers + else: + _markers = combined_ireq.markers._markers + if not isinstance(_markers[0], (tuple, list)): + combined_ireq.markers._markers = [markers, 'and', ireq.markers._markers] # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 750a68a70f..32cfbdf7fd 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -406,7 +406,7 @@ index 1c4b943..c922be1 100644 def allow_all_wheels(self): """ diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py -index 05ec8fd..8952478 100644 +index 05ec8fd..2f94f6b 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -8,13 +8,14 @@ from itertools import chain, count @@ -451,7 +451,7 @@ index 05ec8fd..8952478 100644 msg = ('pip-compile does not support URLs as packages, unless they are editable. ' 'Perhaps add -e option?') raise UnsupportedConstraint(msg, constraint) -@@ -147,15 +149,18 @@ class Resolver(object): +@@ -147,15 +149,23 @@ class Resolver(object): if editable_ireq: yield editable_ireq # ignore all the other specs: the editable one is the one that counts continue @@ -469,11 +469,16 @@ index 05ec8fd..8952478 100644 + if ireq.req.specifier._specs and not combined_ireq.req.specifier._specs: + combined_ireq.req.specifier._specs = ireq.req.specifier._specs combined_ireq.constraint &= ireq.constraint -+ combined_ireq.markers = ireq.markers ++ if not combined_ireq.markers: ++ combined_ireq.markers = ireq.markers ++ else: ++ _markers = combined_ireq.markers._markers ++ if not isinstance(_markers[0], (tuple, list)): ++ combined_ireq.markers._markers = [markers, 'and', ireq.markers._markers] # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq -@@ -273,6 +278,14 @@ class Resolver(object): +@@ -273,6 +283,14 @@ class Resolver(object): for dependency in self.repository.get_dependencies(ireq): yield dependency return @@ -488,7 +493,7 @@ index 05ec8fd..8952478 100644 elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) -@@ -283,14 +296,14 @@ class Resolver(object): +@@ -283,14 +301,14 @@ class Resolver(object): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') dependencies = self.repository.get_dependencies(ireq) From 7dd8579d8e58fb757f25af5b10f4dd97f72ae17c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 30 Jun 2018 16:13:32 -0400 Subject: [PATCH 11/11] Fix typo Signed-off-by: Dan Ryan --- pipenv/patched/piptools/resolver.py | 2 +- tasks/vendoring/patches/patched/piptools.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 889abce61b..807cf518b0 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -165,7 +165,7 @@ def _group_constraints(self, constraints): else: _markers = combined_ireq.markers._markers if not isinstance(_markers[0], (tuple, list)): - combined_ireq.markers._markers = [markers, 'and', ireq.markers._markers] + combined_ireq.markers._markers = [_markers, 'and', ireq.markers._markers] # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 32cfbdf7fd..f5fb822f4a 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -474,7 +474,7 @@ index 05ec8fd..2f94f6b 100644 + else: + _markers = combined_ireq.markers._markers + if not isinstance(_markers[0], (tuple, list)): -+ combined_ireq.markers._markers = [markers, 'and', ireq.markers._markers] ++ combined_ireq.markers._markers = [_markers, 'and', ireq.markers._markers] # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq