From fde4a826b97e9a253ce56c538ea0e9dafa0d9289 Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Sun, 28 Jan 2018 22:06:21 -0800 Subject: [PATCH 1/6] Preliminary manylinux2 support. --- src/pip/_internal/pep425tags.py | 29 +++++++- tests/functional/test_download.py | 113 +++++++++++++++++------------- tests/unit/test_pep425tags.py | 83 +++++++++++++++++++--- tests/unit/test_utils.py | 2 +- 4 files changed, 165 insertions(+), 62 deletions(-) diff --git a/src/pip/_internal/pep425tags.py b/src/pip/_internal/pep425tags.py index 0b5c7832d4f..6695f0ba5f9 100644 --- a/src/pip/_internal/pep425tags.py +++ b/src/pip/_internal/pep425tags.py @@ -157,6 +157,23 @@ def is_manylinux1_compatible(): return pip._internal.utils.glibc.have_compatible_glibc(2, 5) +def is_manylinux2_compatible(): + # Only Linux, and only x86-64 / i686 + if get_platform() not in {"linux_x86_64", "linux_i686"}: + return False + + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux2_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass + + # Check glibc version. CentOS 6 uses glibc 2.12. + return pip._internal.utils.glibc.have_compatible_glibc(2, 12) + + def get_darwin_arches(major, minor, machine): """Return a list of supported arches (including group arches) for the given major, minor and machine architecture of an macOS machine. @@ -276,8 +293,16 @@ def get_supported(versions=None, noarch=False, platform=None, else: # arch pattern didn't match (?!) arches = [arch] - elif platform is None and is_manylinux1_compatible(): - arches = [arch.replace('linux', 'manylinux1'), arch] + elif arch.startswith('manylinux2'): + # manylinux1 wheels run on manylinux2 systems. + arches = [arch, arch.replace('manylinux2', 'manylinux1')] + elif platform is None: + arches = [] + if is_manylinux2_compatible(): + arches.append(arch.replace('linux', 'manylinux2')) + if is_manylinux1_compatible(): + arches.append(arch.replace('linux', 'manylinux1')) + arches.append(arch) else: arches = [arch] diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 4cbfb5665a3..997b87d3e44 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -322,54 +322,71 @@ def test_download_specify_platform(script, data): ) -def test_download_platform_manylinux(script, data): - """ - Test using "pip download --platform" to download a .whl archive - supported for a specific platform. - """ - fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') - # Confirm that universal wheels are returned even for specific - # platforms. - result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', 'linux_x86_64', - 'fake', - ) - assert ( - Path('scratch') / 'fake-1.0-py2.py3-none-any.whl' - in result.files_created - ) - - data.reset() - fake_wheel(data, 'fake-1.0-py2.py3-none-manylinux1_x86_64.whl') - result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', 'manylinux1_x86_64', - 'fake', - ) - assert ( - Path('scratch') / - 'fake-1.0-py2.py3-none-manylinux1_x86_64.whl' - in result.files_created - ) - - # When specifying the platform, manylinux1 needs to be the - # explicit platform--it won't ever be added to the compatible - # tags. - data.reset() - fake_wheel(data, 'fake-1.0-py2.py3-none-linux_x86_64.whl') - result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', 'linux_x86_64', - 'fake', - expect_error=True, - ) +class TestDownloadPlatformManylinuxes(object): + """ + "pip download --platform" downloads a .whl archive supported for + manylinux platforms. + """ + + @pytest.mark.parametrize("platform", [ + "linux_x86_64", + "manylinux1_x86_64", + "manylinux2_x86_64", + ]) + def test_download_universal(self, platform, script, data): + """ + Universal wheels are returned even for specific platforms. + """ + fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', platform, + 'fake', + ) + assert ( + Path('scratch') / 'fake-1.0-py2.py3-none-any.whl' + in result.files_created + ) + + @pytest.mark.parametrize("wheel_abi,platform", [ + ("manylinux1_x86_64", "manylinux1_x86_64"), + ("manylinux1_x86_64", "manylinux2_x86_64"), + ("manylinux2_x86_64", "manylinux2_x86_64"), + ]) + def test_download_compatible_manylinuxes( + self, wheel_abi, platform, script, data, + ): + """ + Earlier manylinuxes are compatible with later manylinuxes. + """ + wheel = 'fake-1.0-py2.py3-none-{}.whl'.format(wheel_abi) + fake_wheel(data, wheel) + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', platform, + 'fake', + ) + assert Path('scratch') / wheel in result.files_created + + def test_explicit_platform_only(self, data, script): + """ + When specifying the platform, manylinux1 needs to be the + explicit platform--it won't ever be added to the compatible + tags. + """ + fake_wheel(data, 'fake-1.0-py2.py3-none-linux_x86_64.whl') + script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', 'linux_x86_64', + 'fake', + expect_error=True, + ) def test_download_specify_python_version(script, data): diff --git a/tests/unit/test_pep425tags.py b/tests/unit/test_pep425tags.py index d55353adbcd..548648561c6 100644 --- a/tests/unit/test_pep425tags.py +++ b/tests/unit/test_pep425tags.py @@ -1,5 +1,7 @@ import sys +import pytest + from mock import patch from pip._internal import pep425tags @@ -114,44 +116,57 @@ def test_manual_abi_dm_flags(self): self.abi_tag_unicode('dm', {'Py_DEBUG': True, 'WITH_PYMALLOC': True}) -class TestManylinux1Tags(object): - +@pytest.mark.parametrize('is_manylinux_compatible', [ + pep425tags.is_manylinux1_compatible, + pep425tags.is_manylinux2_compatible, +]) +class TestManylinuxTags(object): + """ + Tests common to all manylinux tags (e.g. manylinux1, manylinux2, + ...) + """ @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') @patch('pip._internal.utils.glibc.have_compatible_glibc', lambda major, minor: True) - def test_manylinux1_compatible_on_linux_x86_64(self): + def test_manylinux_compatible_on_linux_x86_64(self, + is_manylinux_compatible): """ - Test that manylinux1 is enabled on linux_x86_64 + Test that manylinuxes are enabled on linux_x86_64 """ - assert pep425tags.is_manylinux1_compatible() + assert is_manylinux_compatible() @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_i686') @patch('pip._internal.utils.glibc.have_compatible_glibc', lambda major, minor: True) - def test_manylinux1_compatible_on_linux_i686(self): + def test_manylinux1_compatible_on_linux_i686(self, + is_manylinux_compatible): """ Test that manylinux1 is enabled on linux_i686 """ - assert pep425tags.is_manylinux1_compatible() + assert is_manylinux_compatible() @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') @patch('pip._internal.utils.glibc.have_compatible_glibc', lambda major, minor: False) - def test_manylinux1_2(self): + def test_manylinux1_2(self, is_manylinux_compatible): """ Test that manylinux1 is disabled with incompatible glibc """ - assert not pep425tags.is_manylinux1_compatible() + assert not is_manylinux_compatible() @patch('pip._internal.pep425tags.get_platform', lambda: 'arm6vl') @patch('pip._internal.utils.glibc.have_compatible_glibc', lambda major, minor: True) - def test_manylinux1_3(self): + def test_manylinux1_3(self, is_manylinux_compatible): """ Test that manylinux1 is disabled on arm6vl """ - assert not pep425tags.is_manylinux1_compatible() + assert not is_manylinux_compatible() + +class TestManylinux1Tags(object): + + @patch('pip._internal.pep425tags.is_manylinux2_compatible', lambda: False) @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') @patch('pip._internal.utils.glibc.have_compatible_glibc', lambda major, minor: True) @@ -172,3 +187,49 @@ def test_manylinux1_tag_is_first(self): assert arches == ['manylinux1_x86_64', 'linux_x86_64', 'any'] else: assert arches == ['manylinux1_x86_64', 'linux_x86_64'] + + +class TestManylinux2Tags(object): + + @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') + @patch('pip._internal.utils.glibc.have_compatible_glibc', + lambda major, minor: True) + @patch('sys.platform', 'linux2') + def test_manylinux2_tag_is_first(self): + """ + Test that the more specific tag manylinux2 comes first. + """ + groups = {} + for pyimpl, abi, arch in pep425tags.get_supported(): + groups.setdefault((pyimpl, abi), []).append(arch) + + for arches in groups.values(): + if arches == ['any']: + continue + # Expect the most specific arch first: + if len(arches) == 4: + assert arches == ['manylinux2_x86_64', + 'manylinux1_x86_64', + 'linux_x86_64', + 'any'] + else: + assert arches == ['manylinux2_x86_64', + 'manylinux1_x86_64', + 'linux_x86_64'] + + @pytest.mark.parametrize("manylinux2,manylinux1", [ + ("manylinux2_x86_64", "manylinux1_x86_64"), + ("manylinux2_i686", "manylinux1_i686"), + ]) + def test_manylinux2_implies_manylinux1(self, manylinux2, manylinux1): + """ + Specifying manylinux2 implies manylinux1. + """ + groups = {} + for pyimpl, abi, arch in pep425tags.get_supported(platform=manylinux2): + groups.setdefault((pyimpl, abi), []).append(arch) + + for arches in groups.values(): + if arches == ['any']: + continue + assert arches[:2] == [manylinux2, manylinux1] diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index d6a27dc180b..9e3c6a54a59 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -539,7 +539,7 @@ def test_create_and_cleanup_work(self): class TestGlibc(object): - def test_manylinux1_check_glibc_version(self): + def test_manylinux_check_glibc_version(self): """ Test that the check_glibc_version function is robust against weird glibc version strings. From 94f3318dedeedd3cefb32dcdd47ce7e7a4dc69f2 Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Wed, 31 Jan 2018 15:40:36 -0800 Subject: [PATCH 2/6] News file. --- news/5008.feature | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 news/5008.feature diff --git a/news/5008.feature b/news/5008.feature new file mode 100644 index 00000000000..529d55a7b10 --- /dev/null +++ b/news/5008.feature @@ -0,0 +1,3 @@ +Implement manylinux2 platform tag support. manylinux2 is the successor +to manylinux1. It allows carefully compiled binary wheels to be installed +on compatible Linux platforms. From 8d2068906b17699dbaa7d6f8a77b092a04e2014e Mon Sep 17 00:00:00 2001 From: William T Olson Date: Tue, 15 May 2018 10:16:35 -0400 Subject: [PATCH 3/6] Rename manylinux2 tag as manylinux2010 This reflects the change in the tag name in PEP 571. --- news/5008.feature | 2 +- src/pip/_internal/pep425tags.py | 14 +++++++------- tests/functional/test_download.py | 6 +++--- tests/unit/test_pep425tags.py | 32 ++++++++++++++++--------------- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/news/5008.feature b/news/5008.feature index 529d55a7b10..b42457c85f7 100644 --- a/news/5008.feature +++ b/news/5008.feature @@ -1,3 +1,3 @@ -Implement manylinux2 platform tag support. manylinux2 is the successor +Implement manylinux2010 platform tag support. manylinux2010 is the successor to manylinux1. It allows carefully compiled binary wheels to be installed on compatible Linux platforms. diff --git a/src/pip/_internal/pep425tags.py b/src/pip/_internal/pep425tags.py index 6695f0ba5f9..a4a35c62a91 100644 --- a/src/pip/_internal/pep425tags.py +++ b/src/pip/_internal/pep425tags.py @@ -157,7 +157,7 @@ def is_manylinux1_compatible(): return pip._internal.utils.glibc.have_compatible_glibc(2, 5) -def is_manylinux2_compatible(): +def is_manylinux2010_compatible(): # Only Linux, and only x86-64 / i686 if get_platform() not in {"linux_x86_64", "linux_i686"}: return False @@ -165,7 +165,7 @@ def is_manylinux2_compatible(): # Check for presence of _manylinux module try: import _manylinux - return bool(_manylinux.manylinux2_compatible) + return bool(_manylinux.manylinux2010_compatible) except (ImportError, AttributeError): # Fall through to heuristic check below pass @@ -293,13 +293,13 @@ def get_supported(versions=None, noarch=False, platform=None, else: # arch pattern didn't match (?!) arches = [arch] - elif arch.startswith('manylinux2'): - # manylinux1 wheels run on manylinux2 systems. - arches = [arch, arch.replace('manylinux2', 'manylinux1')] + elif arch.startswith('manylinux2010'): + # manylinux1 wheels run on manylinux2010 systems. + arches = [arch, arch.replace('manylinux2010', 'manylinux1')] elif platform is None: arches = [] - if is_manylinux2_compatible(): - arches.append(arch.replace('linux', 'manylinux2')) + if is_manylinux2010_compatible(): + arches.append(arch.replace('linux', 'manylinux2010')) if is_manylinux1_compatible(): arches.append(arch.replace('linux', 'manylinux1')) arches.append(arch) diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 997b87d3e44..1469b5277ad 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -331,7 +331,7 @@ class TestDownloadPlatformManylinuxes(object): @pytest.mark.parametrize("platform", [ "linux_x86_64", "manylinux1_x86_64", - "manylinux2_x86_64", + "manylinux2010_x86_64", ]) def test_download_universal(self, platform, script, data): """ @@ -352,8 +352,8 @@ def test_download_universal(self, platform, script, data): @pytest.mark.parametrize("wheel_abi,platform", [ ("manylinux1_x86_64", "manylinux1_x86_64"), - ("manylinux1_x86_64", "manylinux2_x86_64"), - ("manylinux2_x86_64", "manylinux2_x86_64"), + ("manylinux1_x86_64", "manylinux2010_x86_64"), + ("manylinux2010_x86_64", "manylinux2010_x86_64"), ]) def test_download_compatible_manylinuxes( self, wheel_abi, platform, script, data, diff --git a/tests/unit/test_pep425tags.py b/tests/unit/test_pep425tags.py index 548648561c6..9b2366c27b6 100644 --- a/tests/unit/test_pep425tags.py +++ b/tests/unit/test_pep425tags.py @@ -118,11 +118,11 @@ def test_manual_abi_dm_flags(self): @pytest.mark.parametrize('is_manylinux_compatible', [ pep425tags.is_manylinux1_compatible, - pep425tags.is_manylinux2_compatible, + pep425tags.is_manylinux2010_compatible, ]) class TestManylinuxTags(object): """ - Tests common to all manylinux tags (e.g. manylinux1, manylinux2, + Tests common to all manylinux tags (e.g. manylinux1, manylinux2010, ...) """ @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') @@ -166,7 +166,8 @@ def test_manylinux1_3(self, is_manylinux_compatible): class TestManylinux1Tags(object): - @patch('pip._internal.pep425tags.is_manylinux2_compatible', lambda: False) + @patch('pip._internal.pep425tags.is_manylinux2010_compatible', + lambda: False) @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') @patch('pip._internal.utils.glibc.have_compatible_glibc', lambda major, minor: True) @@ -189,15 +190,15 @@ def test_manylinux1_tag_is_first(self): assert arches == ['manylinux1_x86_64', 'linux_x86_64'] -class TestManylinux2Tags(object): +class TestManylinux2010Tags(object): @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') @patch('pip._internal.utils.glibc.have_compatible_glibc', lambda major, minor: True) @patch('sys.platform', 'linux2') - def test_manylinux2_tag_is_first(self): + def test_manylinux2010_tag_is_first(self): """ - Test that the more specific tag manylinux2 comes first. + Test that the more specific tag manylinux2010 comes first. """ groups = {} for pyimpl, abi, arch in pep425tags.get_supported(): @@ -208,28 +209,29 @@ def test_manylinux2_tag_is_first(self): continue # Expect the most specific arch first: if len(arches) == 4: - assert arches == ['manylinux2_x86_64', + assert arches == ['manylinux2010_x86_64', 'manylinux1_x86_64', 'linux_x86_64', 'any'] else: - assert arches == ['manylinux2_x86_64', + assert arches == ['manylinux2010_x86_64', 'manylinux1_x86_64', 'linux_x86_64'] - @pytest.mark.parametrize("manylinux2,manylinux1", [ - ("manylinux2_x86_64", "manylinux1_x86_64"), - ("manylinux2_i686", "manylinux1_i686"), + @pytest.mark.parametrize("manylinux2010,manylinux1", [ + ("manylinux2010_x86_64", "manylinux1_x86_64"), + ("manylinux2010_i686", "manylinux1_i686"), ]) - def test_manylinux2_implies_manylinux1(self, manylinux2, manylinux1): + def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): """ - Specifying manylinux2 implies manylinux1. + Specifying manylinux2010 implies manylinux1. """ groups = {} - for pyimpl, abi, arch in pep425tags.get_supported(platform=manylinux2): + supported = pep425tags.get_supported(platform=manylinux2010) + for pyimpl, abi, arch in supported: groups.setdefault((pyimpl, abi), []).append(arch) for arches in groups.values(): if arches == ['any']: continue - assert arches[:2] == [manylinux2, manylinux1] + assert arches[:2] == [manylinux2010, manylinux1] From 84eef80e314c3e71d9cce77da1abd4122c2587c7 Mon Sep 17 00:00:00 2001 From: William T Olson Date: Tue, 15 May 2018 10:32:37 -0400 Subject: [PATCH 4/6] Explicitly check the platform arch --- src/pip/_internal/pep425tags.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/pep425tags.py b/src/pip/_internal/pep425tags.py index a4a35c62a91..458ed5b79a0 100644 --- a/src/pip/_internal/pep425tags.py +++ b/src/pip/_internal/pep425tags.py @@ -280,7 +280,8 @@ def get_supported(versions=None, noarch=False, platform=None, if not noarch: arch = platform or get_platform() - if arch.startswith('macosx'): + arch_prefix, arch_sep, arch_suffix = arch.partition('_') + if arch_prefix == 'macosx': # support macosx-10.6-intel on macosx-10.9-x86_64 match = _osx_arch_pat.match(arch) if match: @@ -293,15 +294,15 @@ def get_supported(versions=None, noarch=False, platform=None, else: # arch pattern didn't match (?!) arches = [arch] - elif arch.startswith('manylinux2010'): + elif arch_prefix == 'manylinux2010': # manylinux1 wheels run on manylinux2010 systems. - arches = [arch, arch.replace('manylinux2010', 'manylinux1')] + arches = [arch, 'manylinux1' + arch_sep + arch_suffix] elif platform is None: arches = [] if is_manylinux2010_compatible(): - arches.append(arch.replace('linux', 'manylinux2010')) + arches.append('manylinux2010' + arch_sep + arch_suffix) if is_manylinux1_compatible(): - arches.append(arch.replace('linux', 'manylinux1')) + arches.append('manylinux1' + arch_sep + arch_suffix) arches.append(arch) else: arches = [arch] From eb867a65bea7bff7fad572bf4ad2b3749bae23e8 Mon Sep 17 00:00:00 2001 From: William T Olson Date: Tue, 15 May 2018 13:19:53 -0400 Subject: [PATCH 5/6] Revert explicit arch detection for macosx --- src/pip/_internal/pep425tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/pep425tags.py b/src/pip/_internal/pep425tags.py index 458ed5b79a0..5736f313f5f 100644 --- a/src/pip/_internal/pep425tags.py +++ b/src/pip/_internal/pep425tags.py @@ -281,7 +281,7 @@ def get_supported(versions=None, noarch=False, platform=None, if not noarch: arch = platform or get_platform() arch_prefix, arch_sep, arch_suffix = arch.partition('_') - if arch_prefix == 'macosx': + if arch.startswith('macosx'): # support macosx-10.6-intel on macosx-10.9-x86_64 match = _osx_arch_pat.match(arch) if match: From 93038958d1e7742fbfbce24efd82eb29e8d1eef6 Mon Sep 17 00:00:00 2001 From: William T Olson Date: Tue, 15 May 2018 13:31:02 -0400 Subject: [PATCH 6/6] Expand details for manylinux1 compatibility with manylinux2010 --- src/pip/_internal/pep425tags.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/pep425tags.py b/src/pip/_internal/pep425tags.py index 5736f313f5f..16497632a86 100644 --- a/src/pip/_internal/pep425tags.py +++ b/src/pip/_internal/pep425tags.py @@ -295,7 +295,10 @@ def get_supported(versions=None, noarch=False, platform=None, # arch pattern didn't match (?!) arches = [arch] elif arch_prefix == 'manylinux2010': - # manylinux1 wheels run on manylinux2010 systems. + # manylinux1 wheels run on most manylinux2010 systems with the + # exception of wheels depending on ncurses. PEP 571 states + # manylinux1 wheels should be considered manylinux2010 wheels: + # https://www.python.org/dev/peps/pep-0571/#backwards-compatibility-with-manylinux1-wheels arches = [arch, 'manylinux1' + arch_sep + arch_suffix] elif platform is None: arches = []