diff --git a/MANIFEST.in b/MANIFEST.in index b25e0780..19afb627 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,9 +8,12 @@ include tox.ini recursive-include docs * recursive-include tests *.py +recursive-include tests hello-world-* exclude .travis.yml exclude dev-requirements.txt +exclude tests/build-hello-world.sh +exclude tests/hello-world.c prune docs/_build prune tasks diff --git a/packaging/tags.py b/packaging/tags.py index db29deb5..20512aaf 100644 --- a/packaging/tags.py +++ b/packaging/tags.py @@ -17,6 +17,7 @@ import os import platform import re +import struct import sys import sysconfig import warnings @@ -27,6 +28,7 @@ from typing import ( Dict, FrozenSet, + IO, Iterable, Iterator, List, @@ -514,16 +516,143 @@ def _have_compatible_glibc(required_major, minimum_minor): return _check_glibc_version(version_str, required_major, minimum_minor) +# Python does not provide platform information at sufficient granularity to +# identify the architecture of the running executable in some cases, so we +# determine it dynamically by reading the information from the running +# process. This only applies on Linux, which uses the ELF format. +class _ELFFileHeader(object): + # https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header + class _InvalidELFFileHeader(ValueError): + """ + An invalid ELF file header was found. + """ + + ELF_MAGIC_NUMBER = 0x7F454C46 + ELFCLASS32 = 1 + ELFCLASS64 = 2 + ELFDATA2LSB = 1 + ELFDATA2MSB = 2 + EM_386 = 3 + EM_S390 = 22 + EM_ARM = 40 + EM_X86_64 = 62 + EF_ARM_ABIMASK = 0xFF000000 + EF_ARM_ABI_VER5 = 0x05000000 + EF_ARM_ABI_FLOAT_HARD = 0x00000400 + + def __init__(self, file): + # type: (IO[bytes]) -> None + def unpack(fmt): + # type: (str) -> int + try: + result, = struct.unpack( + fmt, file.read(struct.calcsize(fmt)) + ) # type: (int, ) + except struct.error: + raise _ELFFileHeader._InvalidELFFileHeader() + return result + + self.e_ident_magic = unpack(">I") + if self.e_ident_magic != self.ELF_MAGIC_NUMBER: + raise _ELFFileHeader._InvalidELFFileHeader() + self.e_ident_class = unpack("B") + if self.e_ident_class not in {self.ELFCLASS32, self.ELFCLASS64}: + raise _ELFFileHeader._InvalidELFFileHeader() + self.e_ident_data = unpack("B") + if self.e_ident_data not in {self.ELFDATA2LSB, self.ELFDATA2MSB}: + raise _ELFFileHeader._InvalidELFFileHeader() + self.e_ident_version = unpack("B") + self.e_ident_osabi = unpack("B") + self.e_ident_abiversion = unpack("B") + self.e_ident_pad = file.read(7) + format_h = "H" + format_i = "I" + format_q = "Q" + format_p = format_i if self.e_ident_class == self.ELFCLASS32 else format_q + self.e_type = unpack(format_h) + self.e_machine = unpack(format_h) + self.e_version = unpack(format_i) + self.e_entry = unpack(format_p) + self.e_phoff = unpack(format_p) + self.e_shoff = unpack(format_p) + self.e_flags = unpack(format_i) + self.e_ehsize = unpack(format_h) + self.e_phentsize = unpack(format_h) + self.e_phnum = unpack(format_h) + self.e_shentsize = unpack(format_h) + self.e_shnum = unpack(format_h) + self.e_shstrndx = unpack(format_h) + + +def _get_elf_header(): + # type: () -> Optional[_ELFFileHeader] + try: + with open(sys.executable, "rb") as f: + elf_header = _ELFFileHeader(f) + except (IOError, OSError, TypeError, _ELFFileHeader._InvalidELFFileHeader): + return None + return elf_header + + +def _is_linux_armhf(): + # type: () -> bool + # hard-float ABI can be detected from the ELF header of the running + # process + # https://static.docs.arm.com/ihi0044/g/aaelf32.pdf + elf_header = _get_elf_header() + if elf_header is None: + return False + result = elf_header.e_ident_class == elf_header.ELFCLASS32 + result &= elf_header.e_ident_data == elf_header.ELFDATA2LSB + result &= elf_header.e_machine == elf_header.EM_ARM + result &= ( + elf_header.e_flags & elf_header.EF_ARM_ABIMASK + ) == elf_header.EF_ARM_ABI_VER5 + result &= ( + elf_header.e_flags & elf_header.EF_ARM_ABI_FLOAT_HARD + ) == elf_header.EF_ARM_ABI_FLOAT_HARD + return result + + +def _is_linux_i686(): + # type: () -> bool + elf_header = _get_elf_header() + if elf_header is None: + return False + result = elf_header.e_ident_class == elf_header.ELFCLASS32 + result &= elf_header.e_ident_data == elf_header.ELFDATA2LSB + result &= elf_header.e_machine == elf_header.EM_386 + return result + + +def _have_compatible_manylinux_abi(arch): + # type: (str) -> bool + if arch == "armv7l": + return _is_linux_armhf() + if arch == "i686": + return _is_linux_i686() + return True + + def _linux_platforms(is_32bit=_32_BIT_INTERPRETER): # type: (bool) -> Iterator[str] linux = _normalize_string(distutils.util.get_platform()) if linux == "linux_x86_64" and is_32bit: linux = "linux_i686" - manylinux_support = ( - ("manylinux2014", (2, 17)), # CentOS 7 w/ glibc 2.17 (PEP 599) - ("manylinux2010", (2, 12)), # CentOS 6 w/ glibc 2.12 (PEP 571) - ("manylinux1", (2, 5)), # CentOS 5 w/ glibc 2.5 (PEP 513) - ) + manylinux_support = [] + _, arch = linux.split("_", 1) + if _have_compatible_manylinux_abi(arch): + if arch in {"x86_64", "i686", "aarch64", "armv7l", "ppc64", "ppc64le", "s390x"}: + manylinux_support.append( + ("manylinux2014", (2, 17)) + ) # CentOS 7 w/ glibc 2.17 (PEP 599) + if arch in {"x86_64", "i686"}: + manylinux_support.append( + ("manylinux2010", (2, 12)) + ) # CentOS 6 w/ glibc 2.12 (PEP 571) + manylinux_support.append( + ("manylinux1", (2, 5)) + ) # CentOS 5 w/ glibc 2.5 (PEP 513) manylinux_support_iter = iter(manylinux_support) for name, glibc_version in manylinux_support_iter: if _is_manylinux_compatible(name, glibc_version): diff --git a/tests/build-hello-world.sh b/tests/build-hello-world.sh new file mode 100755 index 00000000..9c3e1e18 --- /dev/null +++ b/tests/build-hello-world.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -x +set -e + +if [ $# -eq 0 ]; then + docker run --rm -v $(pwd):/home/hello-world arm32v5/debian /home/hello-world/build-hello-world.sh incontainer 52 + docker run --rm -v $(pwd):/home/hello-world arm32v7/debian /home/hello-world/build-hello-world.sh incontainer 52 + docker run --rm -v $(pwd):/home/hello-world i386/debian /home/hello-world/build-hello-world.sh incontainer 52 + docker run --rm -v $(pwd):/home/hello-world s390x/debian /home/hello-world/build-hello-world.sh incontainer 64 + docker run --rm -v $(pwd):/home/hello-world debian /home/hello-world/build-hello-world.sh incontainer 64 + docker run --rm -v $(pwd):/home/hello-world debian /home/hello-world/build-hello-world.sh x32 52 + cp -f hello-world-x86_64-i386 hello-world-invalid-magic + printf "\x00" | dd of=hello-world-invalid-magic bs=1 seek=0x00 count=1 conv=notrunc + cp -f hello-world-x86_64-i386 hello-world-invalid-class + printf "\x00" | dd of=hello-world-invalid-class bs=1 seek=0x04 count=1 conv=notrunc + cp -f hello-world-x86_64-i386 hello-world-invalid-data + printf "\x00" | dd of=hello-world-invalid-data bs=1 seek=0x05 count=1 conv=notrunc + head -c 40 hello-world-x86_64-i386 > hello-world-too-short + exit 0 +fi + +export DEBIAN_FRONTEND=noninteractive +cd /home/hello-world/ +apt-get update +apt-get install -y --no-install-recommends gcc libc6-dev +if [ "$1" == "incontainer" ]; then + ARCH=$(dpkg --print-architecture) + CFLAGS="" +else + ARCH=$1 + dpkg --add-architecture ${ARCH} + apt-get install -y --no-install-recommends gcc-multilib libc6-dev-${ARCH} + CFLAGS="-mx32" +fi +NAME=hello-world-$(uname -m)-${ARCH} +gcc -Os -s ${CFLAGS} -o ${NAME}-full hello-world.c +head -c $2 ${NAME}-full > ${NAME} +rm -f ${NAME}-full diff --git a/tests/hello-world-armv7l-armel b/tests/hello-world-armv7l-armel new file mode 100755 index 00000000..1dfd23fa Binary files /dev/null and b/tests/hello-world-armv7l-armel differ diff --git a/tests/hello-world-armv7l-armhf b/tests/hello-world-armv7l-armhf new file mode 100755 index 00000000..965ab300 Binary files /dev/null and b/tests/hello-world-armv7l-armhf differ diff --git a/tests/hello-world-invalid-class b/tests/hello-world-invalid-class new file mode 100755 index 00000000..5e9899fc Binary files /dev/null and b/tests/hello-world-invalid-class differ diff --git a/tests/hello-world-invalid-data b/tests/hello-world-invalid-data new file mode 100755 index 00000000..2659b8ee Binary files /dev/null and b/tests/hello-world-invalid-data differ diff --git a/tests/hello-world-invalid-magic b/tests/hello-world-invalid-magic new file mode 100755 index 00000000..46066ad2 Binary files /dev/null and b/tests/hello-world-invalid-magic differ diff --git a/tests/hello-world-s390x-s390x b/tests/hello-world-s390x-s390x new file mode 100644 index 00000000..c4e95788 Binary files /dev/null and b/tests/hello-world-s390x-s390x differ diff --git a/tests/hello-world-too-short b/tests/hello-world-too-short new file mode 100644 index 00000000..4e5c0396 Binary files /dev/null and b/tests/hello-world-too-short differ diff --git a/tests/hello-world-x86_64-amd64 b/tests/hello-world-x86_64-amd64 new file mode 100644 index 00000000..c7f5b0b5 Binary files /dev/null and b/tests/hello-world-x86_64-amd64 differ diff --git a/tests/hello-world-x86_64-i386 b/tests/hello-world-x86_64-i386 new file mode 100755 index 00000000..ff1d540a Binary files /dev/null and b/tests/hello-world-x86_64-i386 differ diff --git a/tests/hello-world-x86_64-x32 b/tests/hello-world-x86_64-x32 new file mode 100755 index 00000000..daf85d34 Binary files /dev/null and b/tests/hello-world-x86_64-x32 differ diff --git a/tests/hello-world.c b/tests/hello-world.c new file mode 100644 index 00000000..5e591c3e --- /dev/null +++ b/tests/hello-world.c @@ -0,0 +1,7 @@ +#include + +int main(int argc, char* argv[]) +{ + printf("Hello world"); + return 0; +} diff --git a/tests/test_tags.py b/tests/test_tags.py index e17b4d07..1eacf686 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -469,6 +469,150 @@ def test_linux_platforms_manylinux2014(self, monkeypatch): ] assert platforms == expected + def test_linux_platforms_manylinux2014_armhf_abi(self, monkeypatch): + monkeypatch.setattr( + tags, "_is_manylinux_compatible", lambda name, _: name == "manylinux2014" + ) + monkeypatch.setattr(distutils.util, "get_platform", lambda: "linux_armv7l") + monkeypatch.setattr( + sys, + "executable", + os.path.join(os.path.dirname(__file__), "hello-world-armv7l-armhf"), + ) + platforms = list(tags._linux_platforms(is_32bit=True)) + expected = ["manylinux2014_armv7l", "linux_armv7l"] + assert platforms == expected + + def test_linux_platforms_manylinux2014_i386_abi(self, monkeypatch): + monkeypatch.setattr( + tags, "_is_manylinux_compatible", lambda name, _: name == "manylinux2014" + ) + monkeypatch.setattr(distutils.util, "get_platform", lambda: "linux_x86_64") + monkeypatch.setattr( + sys, + "executable", + os.path.join(os.path.dirname(__file__), "hello-world-x86_64-i386"), + ) + platforms = list(tags._linux_platforms(is_32bit=True)) + expected = [ + "manylinux2014_i686", + "manylinux2010_i686", + "manylinux1_i686", + "linux_i686", + ] + assert platforms == expected + + def test_linux_platforms_manylinux2014_armv6l(self, monkeypatch): + monkeypatch.setattr( + tags, "_is_manylinux_compatible", lambda name, _: name == "manylinux2014" + ) + monkeypatch.setattr(distutils.util, "get_platform", lambda: "linux_armv6l") + platforms = list(tags._linux_platforms(is_32bit=True)) + expected = ["linux_armv6l"] + assert platforms == expected + + @pytest.mark.parametrize( + "machine, abi, alt_machine", + [("x86_64", "x32", "i686"), ("armv7l", "armel", "armv7l")], + ) + def test_linux_platforms_not_manylinux_abi( + self, monkeypatch, machine, abi, alt_machine + ): + monkeypatch.setattr(tags, "_is_manylinux_compatible", lambda name, _: True) + monkeypatch.setattr( + distutils.util, "get_platform", lambda: "linux_{}".format(machine) + ) + monkeypatch.setattr( + sys, + "executable", + os.path.join( + os.path.dirname(__file__), "hello-world-{}-{}".format(machine, abi) + ), + ) + platforms = list(tags._linux_platforms(is_32bit=True)) + expected = ["linux_{}".format(alt_machine)] + assert platforms == expected + + @pytest.mark.parametrize( + "machine, abi, elf_class, elf_data, elf_machine", + [ + ( + "x86_64", + "x32", + tags._ELFFileHeader.ELFCLASS32, + tags._ELFFileHeader.ELFDATA2LSB, + tags._ELFFileHeader.EM_X86_64, + ), + ( + "x86_64", + "i386", + tags._ELFFileHeader.ELFCLASS32, + tags._ELFFileHeader.ELFDATA2LSB, + tags._ELFFileHeader.EM_386, + ), + ( + "x86_64", + "amd64", + tags._ELFFileHeader.ELFCLASS64, + tags._ELFFileHeader.ELFDATA2LSB, + tags._ELFFileHeader.EM_X86_64, + ), + ( + "armv7l", + "armel", + tags._ELFFileHeader.ELFCLASS32, + tags._ELFFileHeader.ELFDATA2LSB, + tags._ELFFileHeader.EM_ARM, + ), + ( + "armv7l", + "armhf", + tags._ELFFileHeader.ELFCLASS32, + tags._ELFFileHeader.ELFDATA2LSB, + tags._ELFFileHeader.EM_ARM, + ), + ( + "s390x", + "s390x", + tags._ELFFileHeader.ELFCLASS64, + tags._ELFFileHeader.ELFDATA2MSB, + tags._ELFFileHeader.EM_S390, + ), + ], + ) + def test_get_elf_header( + self, monkeypatch, machine, abi, elf_class, elf_data, elf_machine + ): + path = os.path.join( + os.path.dirname(__file__), "hello-world-{}-{}".format(machine, abi) + ) + monkeypatch.setattr(sys, "executable", path) + elf_header = tags._get_elf_header() + assert elf_header.e_ident_class == elf_class + assert elf_header.e_ident_data == elf_data + assert elf_header.e_machine == elf_machine + + @pytest.mark.parametrize( + "content", [None, "invalid-magic", "invalid-class", "invalid-data", "too-short"] + ) + def test_get_elf_header_bad_excutable(self, monkeypatch, content): + if content: + path = os.path.join( + os.path.dirname(__file__), "hello-world-{}".format(content) + ) + else: + path = None + monkeypatch.setattr(sys, "executable", path) + assert tags._get_elf_header() is None + + def test_is_linux_armhf_not_elf(self, monkeypatch): + monkeypatch.setattr(tags, "_get_elf_header", lambda: None) + assert not tags._is_linux_armhf() + + def test_is_linux_i686_not_elf(self, monkeypatch): + monkeypatch.setattr(tags, "_get_elf_header", lambda: None) + assert not tags._is_linux_i686() + @pytest.mark.parametrize( "platform_name,dispatch_func",