From 3f308f15c0982be6f6b030426ce55616184bfa55 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 19 Dec 2018 15:14:15 -0500 Subject: [PATCH 01/16] needed non-py extensions can also be full_external_refs --- auditwheel/wheel_abi.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/auditwheel/wheel_abi.py b/auditwheel/wheel_abi.py index 160b9573..ba8fbee2 100644 --- a/auditwheel/wheel_abi.py +++ b/auditwheel/wheel_abi.py @@ -118,10 +118,16 @@ def get_wheel_elfdata(wheel_fn: str): # we should walk its elftree. if basename(fn) not in needed_libs: full_elftree[fn] = nonpy_elftree[fn] - full_external_refs[fn] = lddtree_external_references( - nonpy_elftree[fn], ctx.path) - log.debug(json.dumps(full_elftree, indent=4)) + # Even if a non-pyextension ELF file is not needed, we + # should include it as an external references, because + # they might also require external libraries. + full_external_refs[fn] = lddtree_external_references(nonpy_elftree[fn], + ctx.path) + + log.debug('full_elftree:\n%s', json.dumps(full_elftree, indent=4)) + log.debug('full_external_refs (will be repaired):\n%s', + json.dumps(full_external_refs, indent=4)) return (full_elftree, full_external_refs, versioned_symbols, uses_ucs2_symbols, uses_PyFPE_jbuf) From f32b1bf197f10b2c5354d5573f9161e1c433adac Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 21 Aug 2019 13:37:56 -0400 Subject: [PATCH 02/16] Test cases: Unwanted RPATH overwrite and Failing to patch internal non-py extensions (#134) * Adding minimal test wheel based on PR #134 * Separate tests for each change in PR #134 https://github.com/pypa/auditwheel/pull/134 * non-py extension does not show in auditwheel show (ignoring assert) * Confirming tests on macOS * pytest in travis is weirdly triggering a manual test * Sorry! Forgot to install zlib-devel --- tests/pr134/hello_module/.gitignore | 168 ++++++++++++++++++ tests/pr134/hello_module/README.md | 4 + .../pr134/hello_module/extensions/.gitignore | 1 + .../hello_module/extensions/testzlib.cpp | 137 ++++++++++++++ .../pr134/hello_module/extensions/testzlib.h | 11 ++ .../pr134/hello_module/extensions/testzlib.sh | 8 + tests/pr134/hello_module/hello.cpp | 96 ++++++++++ tests/pr134/hello_module/hello/__init__.py | 1 + tests/pr134/hello_module/setup.py | 84 +++++++++ tests/pr134/hello_module/tests/manual_test.py | 3 + tests/test_hello.py | 97 ++++++++++ 11 files changed, 610 insertions(+) create mode 100644 tests/pr134/hello_module/.gitignore create mode 100755 tests/pr134/hello_module/README.md create mode 100644 tests/pr134/hello_module/extensions/.gitignore create mode 100644 tests/pr134/hello_module/extensions/testzlib.cpp create mode 100644 tests/pr134/hello_module/extensions/testzlib.h create mode 100644 tests/pr134/hello_module/extensions/testzlib.sh create mode 100755 tests/pr134/hello_module/hello.cpp create mode 100644 tests/pr134/hello_module/hello/__init__.py create mode 100755 tests/pr134/hello_module/setup.py create mode 100644 tests/pr134/hello_module/tests/manual_test.py create mode 100644 tests/test_hello.py diff --git a/tests/pr134/hello_module/.gitignore b/tests/pr134/hello_module/.gitignore new file mode 100644 index 00000000..414521ef --- /dev/null +++ b/tests/pr134/hello_module/.gitignore @@ -0,0 +1,168 @@ + +# Created by https://www.gitignore.io/api/python,linux,macos +# Edit at https://www.gitignore.io/?templates=python,linux,macos + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +### Python Patch ### +.venv/ + +# End of https://www.gitignore.io/api/python,linux,macos diff --git a/tests/pr134/hello_module/README.md b/tests/pr134/hello_module/README.md new file mode 100755 index 00000000..c9c1c965 --- /dev/null +++ b/tests/pr134/hello_module/README.md @@ -0,0 +1,4 @@ +# Python 3 extension example + +This example was inspired from https://gist.github.com/physacco/2e1b52415f3a964ad2a542a99bebed8f + diff --git a/tests/pr134/hello_module/extensions/.gitignore b/tests/pr134/hello_module/extensions/.gitignore new file mode 100644 index 00000000..451dfd06 --- /dev/null +++ b/tests/pr134/hello_module/extensions/.gitignore @@ -0,0 +1 @@ +testzlib diff --git a/tests/pr134/hello_module/extensions/testzlib.cpp b/tests/pr134/hello_module/extensions/testzlib.cpp new file mode 100644 index 00000000..88d91df1 --- /dev/null +++ b/tests/pr134/hello_module/extensions/testzlib.cpp @@ -0,0 +1,137 @@ +// Copyright 2007 Timo Bingmann +// Distributed under the Boost Software License, Version 1.0. +// (See http://www.boost.org/LICENSE_1_0.txt) +// Taken from https://panthema.net/2007/0328-ZLibString.html + +#include +#include +#include +#include +#include + +#include +#include "testzlib.h" + +/** Compress a STL string using zlib with given compression level and return + * the binary data. */ +std::string compress_string(const std::string& str, + int compressionlevel) +{ + z_stream zs; // z_stream is zlib's control structure + memset(&zs, 0, sizeof(zs)); + + if (deflateInit(&zs, compressionlevel) != Z_OK) + throw(std::runtime_error("deflateInit failed while compressing.")); + + zs.next_in = (Bytef*)str.data(); + zs.avail_in = str.size(); // set the z_stream's input + + int ret; + char outbuffer[32768]; + std::string outstring; + + // retrieve the compressed bytes blockwise + do { + zs.next_out = reinterpret_cast(outbuffer); + zs.avail_out = sizeof(outbuffer); + + ret = deflate(&zs, Z_FINISH); + + if (outstring.size() < zs.total_out) { + // append the block to the output string + outstring.append(outbuffer, + zs.total_out - outstring.size()); + } + } while (ret == Z_OK); + + deflateEnd(&zs); + + if (ret != Z_STREAM_END) { // an error occurred that was not EOF + std::ostringstream oss; + oss << "Exception during zlib compression: (" << ret << ") " << zs.msg; + throw(std::runtime_error(oss.str())); + } + + return outstring; +} + +/** Decompress an STL string using zlib and return the original data. */ +std::string decompress_string(const std::string& str) +{ + z_stream zs; // z_stream is zlib's control structure + memset(&zs, 0, sizeof(zs)); + + if (inflateInit(&zs) != Z_OK) + throw(std::runtime_error("inflateInit failed while decompressing.")); + + zs.next_in = (Bytef*)str.data(); + zs.avail_in = str.size(); + + int ret; + char outbuffer[32768]; + std::string outstring; + + // get the decompressed bytes blockwise using repeated calls to inflate + do { + zs.next_out = reinterpret_cast(outbuffer); + zs.avail_out = sizeof(outbuffer); + + ret = inflate(&zs, 0); + + if (outstring.size() < zs.total_out) { + outstring.append(outbuffer, + zs.total_out - outstring.size()); + } + + } while (ret == Z_OK); + + inflateEnd(&zs); + + if (ret != Z_STREAM_END) { // an error occurred that was not EOF + std::ostringstream oss; + oss << "Exception during zlib decompression: (" << ret << ") " + << zs.msg; + throw(std::runtime_error(oss.str())); + } + + return outstring; +} + +/** Small dumb tool (de)compressing cin to cout. It holds all input in memory, + * so don't use it for huge files. */ +int main(int argc, char* argv[]) +{ + std::string allinput; + + while (std::cin.good()) // read all input from cin + { + char inbuffer[32768]; + std::cin.read(inbuffer, sizeof(inbuffer)); + allinput.append(inbuffer, std::cin.gcount()); + } + + if (argc >= 2 && strcmp(argv[1], "-d") == 0) + { + std::string cstr = decompress_string( allinput ); + + std::cerr << "Inflated data: " + << allinput.size() << " -> " << cstr.size() + << " (" << std::setprecision(1) << std::fixed + << ( ((float)cstr.size() / (float)allinput.size() - 1.0) * 100.0 ) + << "% increase).\n"; + + std::cout << cstr; + } + else + { + std::string cstr = compress_string( allinput ); + + std::cerr << "Deflated data: " + << allinput.size() << " -> " << cstr.size() + << " (" << std::setprecision(1) << std::fixed + << ( (1.0 - (float)cstr.size() / (float)allinput.size()) * 100.0) + << "% saved).\n"; + + std::cout << cstr; + } +} diff --git a/tests/pr134/hello_module/extensions/testzlib.h b/tests/pr134/hello_module/extensions/testzlib.h new file mode 100644 index 00000000..2bfb62b9 --- /dev/null +++ b/tests/pr134/hello_module/extensions/testzlib.h @@ -0,0 +1,11 @@ +#include +#include + +#ifndef ZLIB_EXAMPLE // include guard +#define ZLIB_EXAMPLE + +std::string compress_string(const std::string& str, + int compressionlevel = Z_BEST_COMPRESSION); +std::string decompress_string(const std::string& str); + +#endif /* ZLIB_EXAMPLE */ diff --git a/tests/pr134/hello_module/extensions/testzlib.sh b/tests/pr134/hello_module/extensions/testzlib.sh new file mode 100644 index 00000000..dae26bde --- /dev/null +++ b/tests/pr134/hello_module/extensions/testzlib.sh @@ -0,0 +1,8 @@ +# compile and run +g++ testzlib.cpp -lz -o testzlib +if [ $? == 0 ]; then + echo Hello Hello Hello Hello Hello Hello! | ./testzlib | ./testzlib -d +fi +# Deflated data: 37 -> 19 (48.6% saved). +# Inflated data: 19 -> 37 (94.7% increase). +# Hello Hello Hello Hello Hello Hello! diff --git a/tests/pr134/hello_module/hello.cpp b/tests/pr134/hello_module/hello.cpp new file mode 100755 index 00000000..c080f7c8 --- /dev/null +++ b/tests/pr134/hello_module/hello.cpp @@ -0,0 +1,96 @@ +#include +#include "extensions/testzlib.h" + +// Module method definitions +static PyObject* hello_world(PyObject *self, PyObject *args) { + printf("Hello, World!"); + Py_RETURN_NONE; +} + +// static PyObject* zlib_example(PyObject *self, PyObject *args) { +// main(); +// Py_RETURN_NONE; +// } + +static PyObject* z_compress(PyObject *self, PyObject *args) { + const char* str_compress; + if (!PyArg_ParseTuple(args, "s", &str_compress)) { + return NULL; + } + + std::string str_compress_s = str_compress; + std::string compressed = compress_string(str_compress_s); + // Copy pointer (compressed string may contain 0 byte) + const char * str_compressed = &*compressed.begin(); + return PyBytes_FromStringAndSize(str_compressed, compressed.length()); +} + +static PyObject* z_uncompress(PyObject *self, PyObject *args) { + const char * str_uncompress; + int str_uncompress_len; + // according to https://docs.python.org/3/c-api/arg.html + if (!PyArg_ParseTuple(args, "y#", &str_uncompress, &str_uncompress_len)) { + return NULL; + } + + std::string uncompressed = decompress_string(std::string (str_uncompress, str_uncompress_len)); + + return PyUnicode_FromString(uncompressed.c_str()); +} + +static PyObject* hello(PyObject *self, PyObject *args) { + const char* name; + if (!PyArg_ParseTuple(args, "s", &name)) { + return NULL; + } + + printf("Hello, %s!\n", name); + Py_RETURN_NONE; +} + +// Method definition object for this extension, these argumens mean: +// ml_name: The name of the method +// ml_meth: Function pointer to the method implementation +// ml_flags: Flags indicating special features of this method, such as +// accepting arguments, accepting keyword arguments, being a +// class method, or being a static method of a class. +// ml_doc: Contents of this method's docstring +static PyMethodDef hello_methods[] = { + { + "hello_world", hello_world, METH_NOARGS, + "Print 'hello world' from a method defined in a C extension." + }, + { + "hello", hello, METH_VARARGS, + "Print 'hello xxx' from a method defined in a C extension." + }, + { + "z_compress", z_compress, METH_VARARGS, + "Compresses a string using C's libz.so" + }, + { + "z_uncompress", z_uncompress, METH_VARARGS, + "Unompresses a string using C's libz.so" + }, + {NULL, NULL, 0, NULL} +}; + +// Module definition +// The arguments of this structure tell Python what to call your extension, +// what it's methods are and where to look for it's method definitions +static struct PyModuleDef hello_definition = { + PyModuleDef_HEAD_INIT, + "_hello", + "A Python module that prints 'hello world' from C code.", + -1, + hello_methods +}; + +// Module initialization +// Python calls this function when importing your extension. It is important +// that this function is named PyInit_[[your_module_name]] exactly, and matches +// the name keyword argument in setup.py's setup() call. +PyMODINIT_FUNC PyInit__hello(void) { + Py_Initialize(); + return PyModule_Create(&hello_definition); +} diff --git a/tests/pr134/hello_module/hello/__init__.py b/tests/pr134/hello_module/hello/__init__.py new file mode 100644 index 00000000..ef20328b --- /dev/null +++ b/tests/pr134/hello_module/hello/__init__.py @@ -0,0 +1 @@ +from ._hello import z_compress, z_uncompress diff --git a/tests/pr134/hello_module/setup.py b/tests/pr134/hello_module/setup.py new file mode 100755 index 00000000..1d3adae1 --- /dev/null +++ b/tests/pr134/hello_module/setup.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +import platform +import setuptools.command.build_ext +from setuptools import setup, find_packages, Distribution +from setuptools.extension import Extension, Library +import os + +# despite its name, setuptools.command.build_ext.link_shared_object won't +# link a shared object on Linux, but a static library and patches distutils +# for this ... We're patching this back now. + + +def always_link_shared_object( + self, objects, output_libname, output_dir=None, libraries=None, + library_dirs=None, runtime_library_dirs=None, export_symbols=None, + debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, + target_lang=None): + self.link( + self.SHARED_LIBRARY, objects, output_libname, + output_dir, libraries, library_dirs, runtime_library_dirs, + export_symbols, debug, extra_preargs, extra_postargs, + build_temp, target_lang + ) + + +setuptools.command.build_ext.libtype = "shared" +setuptools.command.build_ext.link_shared_object = always_link_shared_object + +libtype = setuptools.command.build_ext.libtype +build_ext_cmd = Distribution().get_command_obj('build_ext') +build_ext_cmd.initialize_options() +build_ext_cmd.setup_shlib_compiler() + + +def libname(name): + ''' gets 'name' and returns something like libname.cpython-37m-darwin.so''' + filename = build_ext_cmd.get_ext_filename(name) + fn, ext = os.path.splitext(filename) + return build_ext_cmd.shlib_compiler.library_filename(fn, libtype) + + +pkg_name = 'hello' +zlib_name = '_zlibexample' +zlib_soname = libname(zlib_name) + +build_cmd = Distribution().get_command_obj('build') +build_cmd.finalize_options() +build_platlib = build_cmd.build_platlib + + +def link_args(soname=None): + args = [] + if platform.system() == "Linux": + if soname: + args += ['-Wl,-soname,' + soname] + loader_path = '$ORIGIN' + args += ['-Wl,-rpath,' + loader_path] + elif platform.system() == "Darwin": + if soname: + args += ["-Wl,-dylib", + '-Wl,-install_name,@rpath/%s' % soname] + args += ['-Wl,-rpath,@loader_path/'] + return args + + +hello_module = Extension(pkg_name + '._hello', + language='c++', + sources=['hello.cpp'], + extra_link_args=link_args(), + extra_objects=[build_platlib + '/hello/' + zlib_soname]) +zlib_example = Library(pkg_name + '.' + zlib_name, + language='c++', + extra_compile_args=['-lz'], + extra_link_args=link_args(zlib_soname) + ['-lz'], + sources=['extensions/testzlib.cpp'] + ) + +setup(name='hello', + version='0.1.0', + packages=find_packages(), + description='Hello world module written in C', + ext_modules=[zlib_example, hello_module]) diff --git a/tests/pr134/hello_module/tests/manual_test.py b/tests/pr134/hello_module/tests/manual_test.py new file mode 100644 index 00000000..cb325722 --- /dev/null +++ b/tests/pr134/hello_module/tests/manual_test.py @@ -0,0 +1,3 @@ +if __name__ == '__main__': + from hello import z_compress, z_uncompress + assert z_uncompress(z_compress('test')) == 'test' diff --git a/tests/test_hello.py b/tests/test_hello.py new file mode 100644 index 00000000..d2e0c859 --- /dev/null +++ b/tests/test_hello.py @@ -0,0 +1,97 @@ +import os +import os.path as op +import shutil +from test_manylinux import \ + docker_container, \ + docker_exec, \ + WHEEL_CACHE_FOLDER + +HELLO_WHEEL = 'hello-0.1.0-cp35-cp35m-linux_x86_64.whl' + + +def build_hello_wheel(docker_container): + policy, manylinux_id, python_id, io_folder = docker_container + + docker_exec(manylinux_id, 'yum install -y zlib-devel') + + if op.exists(op.join(WHEEL_CACHE_FOLDER, HELLO_WHEEL)): + # If hello has already been built and put in cache, let's reuse this. + shutil.copy2(op.join(WHEEL_CACHE_FOLDER, HELLO_WHEEL), + op.join(io_folder, HELLO_WHEEL)) + else: + docker_exec(manylinux_id, + 'pip wheel -w /io /auditwheel_src/tests/pr134/hello_module/') + shutil.copy2(op.join(io_folder, HELLO_WHEEL), + op.join(WHEEL_CACHE_FOLDER, HELLO_WHEEL)) + filenames = os.listdir(io_folder) + assert filenames == [HELLO_WHEEL] + orig_wheel = filenames[0] + assert 'manylinux' not in orig_wheel + return orig_wheel + + +def repair_hello_wheel(orig_wheel, docker_container): + policy, manylinux_id, python_id, io_folder = docker_container + # Repair the wheel using the manylinux container + repair_command = ( + 'auditwheel repair --plat {policy}_x86_64 -w /io /io/{orig_wheel}' + ).format(policy=policy, orig_wheel=orig_wheel) + docker_exec(manylinux_id, repair_command) + filenames = os.listdir(io_folder) + + # Regardless of build environment, wheel only needs manylinux1 symbols + repaired_wheels = [fn for fn in filenames if policy in fn] + assert repaired_wheels == ['hello-0.1.0-cp35-cp35m-{policy}_x86_64.whl'.format(policy=policy)] + repaired_wheel = repaired_wheels[0] + + return repaired_wheel + + +def test_repair_reccurent_dependency(docker_container): + # tests https://github.com/pypa/auditwheel/issues/136 + policy, manylinux_id, python_id, io_folder = docker_container + orig_wheel = build_hello_wheel(docker_container) + + # attempting repair of the hello wheel + repaired_wheel = repair_hello_wheel(orig_wheel, docker_container) + + output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel) + # because this wheel is eligible to the manylinux1 tag, it will + # actually prioritize manylinux1 instead of manylinux2010 + assert ( + 'hello-0.1.0-cp35-cp35m-{policy}_x86_64.whl is consistent with the' + 'following platform tag: "manylinux1_x86_64"' + ).format(policy=policy) in output.replace('\n', '') + + +def test_correct_rpath_hello_wheel(docker_container): + # this tests https://github.com/pypa/auditwheel/issues/137 + policy, manylinux_id, python_id, io_folder = docker_container + orig_wheel = build_hello_wheel(docker_container) + + # attempting repair of the hello wheel + repaired_wheel = repair_hello_wheel(orig_wheel, docker_container) + + # Test whether repaired wheel is functioning. + + # TODO: Remove once pip supports manylinux2010 + docker_exec( + python_id, + "pip install git+https://github.com/wtolson/pip.git@manylinux2010", + ) + + test_commands = [ + 'pip install -U /io/' + repaired_wheel, + 'python /auditwheel_src/tests/pr134/hello_module/tests/manual_test.py', + ] + for cmd in test_commands: + docker_exec(python_id, cmd) + + +# from auditwheel.wheel_abi import analyze_wheel_abi +# def test_analyze_wheel_abi_hello(): +# winfo = analyze_wheel_abi( +# 'tests/python_snappy-0.5.2-pp260-pypy_41-linux_x86_64.whl') +# external_libs = winfo.external_refs['manylinux1_x86_64']['libs'] +# assert len(external_libs) > 0 +# assert set(external_libs) == {'libsnappy.so.1'} From 6c6575c40323e1633bba1081881008793f4c3395 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 3 Feb 2021 16:01:38 -0600 Subject: [PATCH 03/16] Move test code into integration tests directory, rename to nonpy_rpath. --- tests/{pr134 => integration/nonpy_rpath}/hello_module/.gitignore | 0 tests/{pr134 => integration/nonpy_rpath}/hello_module/README.md | 0 .../nonpy_rpath}/hello_module/extensions/.gitignore | 0 .../nonpy_rpath}/hello_module/extensions/testzlib.cpp | 0 .../nonpy_rpath}/hello_module/extensions/testzlib.h | 0 .../nonpy_rpath}/hello_module/extensions/testzlib.sh | 0 tests/{pr134 => integration/nonpy_rpath}/hello_module/hello.cpp | 0 .../nonpy_rpath}/hello_module/hello/__init__.py | 0 tests/{pr134 => integration/nonpy_rpath}/hello_module/setup.py | 0 .../nonpy_rpath}/hello_module/tests/manual_test.py | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename tests/{pr134 => integration/nonpy_rpath}/hello_module/.gitignore (100%) rename tests/{pr134 => integration/nonpy_rpath}/hello_module/README.md (100%) rename tests/{pr134 => integration/nonpy_rpath}/hello_module/extensions/.gitignore (100%) rename tests/{pr134 => integration/nonpy_rpath}/hello_module/extensions/testzlib.cpp (100%) rename tests/{pr134 => integration/nonpy_rpath}/hello_module/extensions/testzlib.h (100%) rename tests/{pr134 => integration/nonpy_rpath}/hello_module/extensions/testzlib.sh (100%) rename tests/{pr134 => integration/nonpy_rpath}/hello_module/hello.cpp (100%) rename tests/{pr134 => integration/nonpy_rpath}/hello_module/hello/__init__.py (100%) rename tests/{pr134 => integration/nonpy_rpath}/hello_module/setup.py (100%) rename tests/{pr134 => integration/nonpy_rpath}/hello_module/tests/manual_test.py (100%) diff --git a/tests/pr134/hello_module/.gitignore b/tests/integration/nonpy_rpath/hello_module/.gitignore similarity index 100% rename from tests/pr134/hello_module/.gitignore rename to tests/integration/nonpy_rpath/hello_module/.gitignore diff --git a/tests/pr134/hello_module/README.md b/tests/integration/nonpy_rpath/hello_module/README.md similarity index 100% rename from tests/pr134/hello_module/README.md rename to tests/integration/nonpy_rpath/hello_module/README.md diff --git a/tests/pr134/hello_module/extensions/.gitignore b/tests/integration/nonpy_rpath/hello_module/extensions/.gitignore similarity index 100% rename from tests/pr134/hello_module/extensions/.gitignore rename to tests/integration/nonpy_rpath/hello_module/extensions/.gitignore diff --git a/tests/pr134/hello_module/extensions/testzlib.cpp b/tests/integration/nonpy_rpath/hello_module/extensions/testzlib.cpp similarity index 100% rename from tests/pr134/hello_module/extensions/testzlib.cpp rename to tests/integration/nonpy_rpath/hello_module/extensions/testzlib.cpp diff --git a/tests/pr134/hello_module/extensions/testzlib.h b/tests/integration/nonpy_rpath/hello_module/extensions/testzlib.h similarity index 100% rename from tests/pr134/hello_module/extensions/testzlib.h rename to tests/integration/nonpy_rpath/hello_module/extensions/testzlib.h diff --git a/tests/pr134/hello_module/extensions/testzlib.sh b/tests/integration/nonpy_rpath/hello_module/extensions/testzlib.sh similarity index 100% rename from tests/pr134/hello_module/extensions/testzlib.sh rename to tests/integration/nonpy_rpath/hello_module/extensions/testzlib.sh diff --git a/tests/pr134/hello_module/hello.cpp b/tests/integration/nonpy_rpath/hello_module/hello.cpp similarity index 100% rename from tests/pr134/hello_module/hello.cpp rename to tests/integration/nonpy_rpath/hello_module/hello.cpp diff --git a/tests/pr134/hello_module/hello/__init__.py b/tests/integration/nonpy_rpath/hello_module/hello/__init__.py similarity index 100% rename from tests/pr134/hello_module/hello/__init__.py rename to tests/integration/nonpy_rpath/hello_module/hello/__init__.py diff --git a/tests/pr134/hello_module/setup.py b/tests/integration/nonpy_rpath/hello_module/setup.py similarity index 100% rename from tests/pr134/hello_module/setup.py rename to tests/integration/nonpy_rpath/hello_module/setup.py diff --git a/tests/pr134/hello_module/tests/manual_test.py b/tests/integration/nonpy_rpath/hello_module/tests/manual_test.py similarity index 100% rename from tests/pr134/hello_module/tests/manual_test.py rename to tests/integration/nonpy_rpath/hello_module/tests/manual_test.py From 40d5dfbb21d7e51afc8b6430751147cab1050659 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sat, 6 Feb 2021 14:00:01 -0600 Subject: [PATCH 04/16] Move test module out of the subdirectory. --- tests/integration/nonpy_rpath/{hello_module => }/.gitignore | 0 tests/integration/nonpy_rpath/{hello_module => }/README.md | 0 .../nonpy_rpath/{hello_module => }/extensions/.gitignore | 0 .../nonpy_rpath/{hello_module => }/extensions/testzlib.cpp | 0 .../nonpy_rpath/{hello_module => }/extensions/testzlib.h | 0 .../nonpy_rpath/{hello_module => }/extensions/testzlib.sh | 0 tests/integration/nonpy_rpath/{hello_module => }/hello.cpp | 0 .../integration/nonpy_rpath/{hello_module => }/hello/__init__.py | 0 tests/integration/nonpy_rpath/{hello_module => }/setup.py | 0 .../nonpy_rpath/{hello_module => }/tests/manual_test.py | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename tests/integration/nonpy_rpath/{hello_module => }/.gitignore (100%) rename tests/integration/nonpy_rpath/{hello_module => }/README.md (100%) rename tests/integration/nonpy_rpath/{hello_module => }/extensions/.gitignore (100%) rename tests/integration/nonpy_rpath/{hello_module => }/extensions/testzlib.cpp (100%) rename tests/integration/nonpy_rpath/{hello_module => }/extensions/testzlib.h (100%) rename tests/integration/nonpy_rpath/{hello_module => }/extensions/testzlib.sh (100%) rename tests/integration/nonpy_rpath/{hello_module => }/hello.cpp (100%) rename tests/integration/nonpy_rpath/{hello_module => }/hello/__init__.py (100%) rename tests/integration/nonpy_rpath/{hello_module => }/setup.py (100%) rename tests/integration/nonpy_rpath/{hello_module => }/tests/manual_test.py (100%) diff --git a/tests/integration/nonpy_rpath/hello_module/.gitignore b/tests/integration/nonpy_rpath/.gitignore similarity index 100% rename from tests/integration/nonpy_rpath/hello_module/.gitignore rename to tests/integration/nonpy_rpath/.gitignore diff --git a/tests/integration/nonpy_rpath/hello_module/README.md b/tests/integration/nonpy_rpath/README.md similarity index 100% rename from tests/integration/nonpy_rpath/hello_module/README.md rename to tests/integration/nonpy_rpath/README.md diff --git a/tests/integration/nonpy_rpath/hello_module/extensions/.gitignore b/tests/integration/nonpy_rpath/extensions/.gitignore similarity index 100% rename from tests/integration/nonpy_rpath/hello_module/extensions/.gitignore rename to tests/integration/nonpy_rpath/extensions/.gitignore diff --git a/tests/integration/nonpy_rpath/hello_module/extensions/testzlib.cpp b/tests/integration/nonpy_rpath/extensions/testzlib.cpp similarity index 100% rename from tests/integration/nonpy_rpath/hello_module/extensions/testzlib.cpp rename to tests/integration/nonpy_rpath/extensions/testzlib.cpp diff --git a/tests/integration/nonpy_rpath/hello_module/extensions/testzlib.h b/tests/integration/nonpy_rpath/extensions/testzlib.h similarity index 100% rename from tests/integration/nonpy_rpath/hello_module/extensions/testzlib.h rename to tests/integration/nonpy_rpath/extensions/testzlib.h diff --git a/tests/integration/nonpy_rpath/hello_module/extensions/testzlib.sh b/tests/integration/nonpy_rpath/extensions/testzlib.sh similarity index 100% rename from tests/integration/nonpy_rpath/hello_module/extensions/testzlib.sh rename to tests/integration/nonpy_rpath/extensions/testzlib.sh diff --git a/tests/integration/nonpy_rpath/hello_module/hello.cpp b/tests/integration/nonpy_rpath/hello.cpp similarity index 100% rename from tests/integration/nonpy_rpath/hello_module/hello.cpp rename to tests/integration/nonpy_rpath/hello.cpp diff --git a/tests/integration/nonpy_rpath/hello_module/hello/__init__.py b/tests/integration/nonpy_rpath/hello/__init__.py similarity index 100% rename from tests/integration/nonpy_rpath/hello_module/hello/__init__.py rename to tests/integration/nonpy_rpath/hello/__init__.py diff --git a/tests/integration/nonpy_rpath/hello_module/setup.py b/tests/integration/nonpy_rpath/setup.py similarity index 100% rename from tests/integration/nonpy_rpath/hello_module/setup.py rename to tests/integration/nonpy_rpath/setup.py diff --git a/tests/integration/nonpy_rpath/hello_module/tests/manual_test.py b/tests/integration/nonpy_rpath/tests/manual_test.py similarity index 100% rename from tests/integration/nonpy_rpath/hello_module/tests/manual_test.py rename to tests/integration/nonpy_rpath/tests/manual_test.py From f9e9aa53ceff185e4c4cab3e0034c0b9afc734fc Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sat, 6 Feb 2021 15:22:17 -0600 Subject: [PATCH 05/16] Update Python C API to use Py_ssize_t instead of int. --- tests/integration/nonpy_rpath/hello.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/integration/nonpy_rpath/hello.cpp b/tests/integration/nonpy_rpath/hello.cpp index c080f7c8..771b5554 100755 --- a/tests/integration/nonpy_rpath/hello.cpp +++ b/tests/integration/nonpy_rpath/hello.cpp @@ -1,3 +1,4 @@ +#define PY_SSIZE_T_CLEAN #include #include "extensions/testzlib.h" @@ -27,7 +28,7 @@ static PyObject* z_compress(PyObject *self, PyObject *args) { static PyObject* z_uncompress(PyObject *self, PyObject *args) { const char * str_uncompress; - int str_uncompress_len; + Py_ssize_t str_uncompress_len; // according to https://docs.python.org/3/c-api/arg.html if (!PyArg_ParseTuple(args, "y#", &str_uncompress, &str_uncompress_len)) { return NULL; @@ -55,15 +56,15 @@ static PyObject* hello(PyObject *self, PyObject *args) { // accepting arguments, accepting keyword arguments, being a // class method, or being a static method of a class. // ml_doc: Contents of this method's docstring -static PyMethodDef hello_methods[] = { - { +static PyMethodDef hello_methods[] = { + { "hello_world", hello_world, METH_NOARGS, "Print 'hello world' from a method defined in a C extension." - }, - { + }, + { "hello", hello, METH_VARARGS, "Print 'hello xxx' from a method defined in a C extension." - }, + }, { "z_compress", z_compress, METH_VARARGS, "Compresses a string using C's libz.so" @@ -78,11 +79,11 @@ static PyMethodDef hello_methods[] = { // Module definition // The arguments of this structure tell Python what to call your extension, // what it's methods are and where to look for it's method definitions -static struct PyModuleDef hello_definition = { +static struct PyModuleDef hello_definition = { PyModuleDef_HEAD_INIT, "_hello", "A Python module that prints 'hello world' from C code.", - -1, + -1, hello_methods }; From 6871346191ef90cdbfd81ea8fdaf9f6afd6d482f Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sat, 6 Feb 2021 15:22:54 -0600 Subject: [PATCH 06/16] Remove old test file. --- tests/test_hello.py | 97 --------------------------------------------- 1 file changed, 97 deletions(-) delete mode 100644 tests/test_hello.py diff --git a/tests/test_hello.py b/tests/test_hello.py deleted file mode 100644 index d2e0c859..00000000 --- a/tests/test_hello.py +++ /dev/null @@ -1,97 +0,0 @@ -import os -import os.path as op -import shutil -from test_manylinux import \ - docker_container, \ - docker_exec, \ - WHEEL_CACHE_FOLDER - -HELLO_WHEEL = 'hello-0.1.0-cp35-cp35m-linux_x86_64.whl' - - -def build_hello_wheel(docker_container): - policy, manylinux_id, python_id, io_folder = docker_container - - docker_exec(manylinux_id, 'yum install -y zlib-devel') - - if op.exists(op.join(WHEEL_CACHE_FOLDER, HELLO_WHEEL)): - # If hello has already been built and put in cache, let's reuse this. - shutil.copy2(op.join(WHEEL_CACHE_FOLDER, HELLO_WHEEL), - op.join(io_folder, HELLO_WHEEL)) - else: - docker_exec(manylinux_id, - 'pip wheel -w /io /auditwheel_src/tests/pr134/hello_module/') - shutil.copy2(op.join(io_folder, HELLO_WHEEL), - op.join(WHEEL_CACHE_FOLDER, HELLO_WHEEL)) - filenames = os.listdir(io_folder) - assert filenames == [HELLO_WHEEL] - orig_wheel = filenames[0] - assert 'manylinux' not in orig_wheel - return orig_wheel - - -def repair_hello_wheel(orig_wheel, docker_container): - policy, manylinux_id, python_id, io_folder = docker_container - # Repair the wheel using the manylinux container - repair_command = ( - 'auditwheel repair --plat {policy}_x86_64 -w /io /io/{orig_wheel}' - ).format(policy=policy, orig_wheel=orig_wheel) - docker_exec(manylinux_id, repair_command) - filenames = os.listdir(io_folder) - - # Regardless of build environment, wheel only needs manylinux1 symbols - repaired_wheels = [fn for fn in filenames if policy in fn] - assert repaired_wheels == ['hello-0.1.0-cp35-cp35m-{policy}_x86_64.whl'.format(policy=policy)] - repaired_wheel = repaired_wheels[0] - - return repaired_wheel - - -def test_repair_reccurent_dependency(docker_container): - # tests https://github.com/pypa/auditwheel/issues/136 - policy, manylinux_id, python_id, io_folder = docker_container - orig_wheel = build_hello_wheel(docker_container) - - # attempting repair of the hello wheel - repaired_wheel = repair_hello_wheel(orig_wheel, docker_container) - - output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel) - # because this wheel is eligible to the manylinux1 tag, it will - # actually prioritize manylinux1 instead of manylinux2010 - assert ( - 'hello-0.1.0-cp35-cp35m-{policy}_x86_64.whl is consistent with the' - 'following platform tag: "manylinux1_x86_64"' - ).format(policy=policy) in output.replace('\n', '') - - -def test_correct_rpath_hello_wheel(docker_container): - # this tests https://github.com/pypa/auditwheel/issues/137 - policy, manylinux_id, python_id, io_folder = docker_container - orig_wheel = build_hello_wheel(docker_container) - - # attempting repair of the hello wheel - repaired_wheel = repair_hello_wheel(orig_wheel, docker_container) - - # Test whether repaired wheel is functioning. - - # TODO: Remove once pip supports manylinux2010 - docker_exec( - python_id, - "pip install git+https://github.com/wtolson/pip.git@manylinux2010", - ) - - test_commands = [ - 'pip install -U /io/' + repaired_wheel, - 'python /auditwheel_src/tests/pr134/hello_module/tests/manual_test.py', - ] - for cmd in test_commands: - docker_exec(python_id, cmd) - - -# from auditwheel.wheel_abi import analyze_wheel_abi -# def test_analyze_wheel_abi_hello(): -# winfo = analyze_wheel_abi( -# 'tests/python_snappy-0.5.2-pp260-pypy_41-linux_x86_64.whl') -# external_libs = winfo.external_refs['manylinux1_x86_64']['libs'] -# assert len(external_libs) > 0 -# assert set(external_libs) == {'libsnappy.so.1'} From a5ae0730aaa715dd669aa18d61fbed9558e68718 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sat, 6 Feb 2021 15:23:22 -0600 Subject: [PATCH 07/16] Add test for issue 136. --- tests/integration/test_manylinux.py | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index d6ffeb59..09153e97 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -657,3 +657,39 @@ def test_strip_wheel(any_manylinux_container, docker_python, io_folder): ["python", "-c", "from sample_extension import test_func; print(test_func(1))"] ) assert output.strip() == "2" + + +def test_nonpy_rpath(any_manylinux_container, docker_python, io_folder): + # Tests https://github.com/pypa/auditwheel/issues/136 + policy, manylinux_ctr = any_manylinux_container + docker_exec(manylinux_ctr, 'yum install -y zlib-devel') + docker_exec( + manylinux_ctr, + ['bash', '-c', 'cd /auditwheel_src/tests/integration/nonpy_rpath ' + '&& python -m pip wheel --no-deps -w /io .'] + ) + + orig_wheel, *_ = os.listdir(io_folder) + assert orig_wheel.startswith("hello-0.1.0") + assert 'manylinux' not in orig_wheel + + # Repair the wheel using the appropriate manylinux container + repair_command = \ + f'auditwheel repair --plat {policy} --strip -w /io /io/{orig_wheel}' + docker_exec(manylinux_ctr, repair_command) + + repaired_wheel, *_ = glob.glob(f"{io_folder}/*{policy}*.whl") + repaired_wheel = os.path.basename(repaired_wheel) + + docker_exec(docker_python, "pip install /io/" + repaired_wheel) + output = docker_exec( + docker_python, + ["python", "-c", "import hello; print(hello.z_uncompress(hello.z_compress(\"sample_string\")))"] + ) + assert output.strip() == "sample_string" + + output = docker_exec(manylinux_ctr, 'auditwheel show /io/' + repaired_wheel) + assert ( + f'hello-0.1.0-{PYTHON_ABI}-{policy}.whl is consistent' + f' with the following platform tag: "{policy}"' + ) in output.replace('\n', ' ') From 501ab18549b196e299a8a3e85ac7eba73fd68767 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sat, 6 Feb 2021 15:27:19 -0600 Subject: [PATCH 08/16] Fix lint. --- auditwheel/wheel_abi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auditwheel/wheel_abi.py b/auditwheel/wheel_abi.py index ba8fbee2..14fe3dfd 100644 --- a/auditwheel/wheel_abi.py +++ b/auditwheel/wheel_abi.py @@ -122,12 +122,12 @@ def get_wheel_elfdata(wheel_fn: str): # Even if a non-pyextension ELF file is not needed, we # should include it as an external references, because # they might also require external libraries. - full_external_refs[fn] = lddtree_external_references(nonpy_elftree[fn], - ctx.path) + full_external_refs[fn] = lddtree_external_references( + nonpy_elftree[fn], ctx.path) log.debug('full_elftree:\n%s', json.dumps(full_elftree, indent=4)) log.debug('full_external_refs (will be repaired):\n%s', - json.dumps(full_external_refs, indent=4)) + json.dumps(full_external_refs, indent=4)) return (full_elftree, full_external_refs, versioned_symbols, uses_ucs2_symbols, uses_PyFPE_jbuf) From 4a394a0e1036ac83c7f1b9d12bcf21bc17311eb9 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sat, 6 Feb 2021 17:19:02 -0600 Subject: [PATCH 09/16] Improve .gitignore and README. --- tests/integration/nonpy_rpath/.gitignore | 168 ----------------------- tests/integration/nonpy_rpath/README.md | 7 +- 2 files changed, 6 insertions(+), 169 deletions(-) delete mode 100644 tests/integration/nonpy_rpath/.gitignore diff --git a/tests/integration/nonpy_rpath/.gitignore b/tests/integration/nonpy_rpath/.gitignore deleted file mode 100644 index 414521ef..00000000 --- a/tests/integration/nonpy_rpath/.gitignore +++ /dev/null @@ -1,168 +0,0 @@ - -# Created by https://www.gitignore.io/api/python,linux,macos -# Edit at https://www.gitignore.io/?templates=python,linux,macos - -### Linux ### -*~ - -# temporary files which can be created if a process still has a handle open of a deleted file -.fuse_hidden* - -# KDE directory preferences -.directory - -# Linux trash folder which might appear on any partition or disk -.Trash-* - -# .nfs files are created when an open file is removed but is still being accessed -.nfs* - -### macOS ### -# General -.DS_Store -.AppleDouble -.LSOverride - -# Icon must end with two \r -Icon - -# Thumbnails -._* - -# Files that might appear in the root of a volume -.DocumentRevisions-V100 -.fseventsd -.Spotlight-V100 -.TemporaryItems -.Trashes -.VolumeIcon.icns -.com.apple.timemachine.donotpresent - -# Directories potentially created on remote AFP share -.AppleDB -.AppleDesktop -Network Trash Folder -Temporary Items -.apdisk - -### Python ### -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -### Python Patch ### -.venv/ - -# End of https://www.gitignore.io/api/python,linux,macos diff --git a/tests/integration/nonpy_rpath/README.md b/tests/integration/nonpy_rpath/README.md index c9c1c965..b2df547c 100755 --- a/tests/integration/nonpy_rpath/README.md +++ b/tests/integration/nonpy_rpath/README.md @@ -1,4 +1,9 @@ -# Python 3 extension example +# Python 3 extension with non-Python library dependency This example was inspired from https://gist.github.com/physacco/2e1b52415f3a964ad2a542a99bebed8f +This test extension builds two libraries: `_hello.*.so` and `lib_zlibexample.*.so`, where the `*` is a string composed of Python ABI versions and platform tags. + +The extension `lib_zlibexample.*.so` should be repaired by auditwheel because it is a needed library, even though it is not a Python extension. + +[Issue #136](https://github.com/pypa/auditwheel/issues/136) documents the underlying problem that this test case is designed to solve. From 24454957d32cfc22e4cd84ac53195e3cfc10a4f5 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sat, 6 Feb 2021 17:21:51 -0600 Subject: [PATCH 10/16] Format test extension Python code. --- tests/integration/nonpy_rpath/setup.py | 95 ++++++++++++------- .../nonpy_rpath/tests/manual_test.py | 5 +- 2 files changed, 62 insertions(+), 38 deletions(-) diff --git a/tests/integration/nonpy_rpath/setup.py b/tests/integration/nonpy_rpath/setup.py index 1d3adae1..7340c07b 100755 --- a/tests/integration/nonpy_rpath/setup.py +++ b/tests/integration/nonpy_rpath/setup.py @@ -13,15 +13,34 @@ def always_link_shared_object( - self, objects, output_libname, output_dir=None, libraries=None, - library_dirs=None, runtime_library_dirs=None, export_symbols=None, - debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, - target_lang=None): + self, + objects, + output_libname, + output_dir=None, + libraries=None, + library_dirs=None, + runtime_library_dirs=None, + export_symbols=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + build_temp=None, + target_lang=None, +): self.link( - self.SHARED_LIBRARY, objects, output_libname, - output_dir, libraries, library_dirs, runtime_library_dirs, - export_symbols, debug, extra_preargs, extra_postargs, - build_temp, target_lang + self.SHARED_LIBRARY, + objects, + output_libname, + output_dir, + libraries, + library_dirs, + runtime_library_dirs, + export_symbols, + debug, + extra_preargs, + extra_postargs, + build_temp, + target_lang, ) @@ -29,23 +48,23 @@ def always_link_shared_object( setuptools.command.build_ext.link_shared_object = always_link_shared_object libtype = setuptools.command.build_ext.libtype -build_ext_cmd = Distribution().get_command_obj('build_ext') +build_ext_cmd = Distribution().get_command_obj("build_ext") build_ext_cmd.initialize_options() build_ext_cmd.setup_shlib_compiler() def libname(name): - ''' gets 'name' and returns something like libname.cpython-37m-darwin.so''' + """ gets 'name' and returns something like libname.cpython-37m-darwin.so""" filename = build_ext_cmd.get_ext_filename(name) fn, ext = os.path.splitext(filename) return build_ext_cmd.shlib_compiler.library_filename(fn, libtype) -pkg_name = 'hello' -zlib_name = '_zlibexample' +pkg_name = "hello" +zlib_name = "_zlibexample" zlib_soname = libname(zlib_name) -build_cmd = Distribution().get_command_obj('build') +build_cmd = Distribution().get_command_obj("build") build_cmd.finalize_options() build_platlib = build_cmd.build_platlib @@ -54,31 +73,35 @@ def link_args(soname=None): args = [] if platform.system() == "Linux": if soname: - args += ['-Wl,-soname,' + soname] - loader_path = '$ORIGIN' - args += ['-Wl,-rpath,' + loader_path] + args += ["-Wl,-soname," + soname] + loader_path = "$ORIGIN" + args += ["-Wl,-rpath," + loader_path] elif platform.system() == "Darwin": if soname: - args += ["-Wl,-dylib", - '-Wl,-install_name,@rpath/%s' % soname] - args += ['-Wl,-rpath,@loader_path/'] + args += ["-Wl,-dylib", "-Wl,-install_name,@rpath/%s" % soname] + args += ["-Wl,-rpath,@loader_path/"] return args -hello_module = Extension(pkg_name + '._hello', - language='c++', - sources=['hello.cpp'], - extra_link_args=link_args(), - extra_objects=[build_platlib + '/hello/' + zlib_soname]) -zlib_example = Library(pkg_name + '.' + zlib_name, - language='c++', - extra_compile_args=['-lz'], - extra_link_args=link_args(zlib_soname) + ['-lz'], - sources=['extensions/testzlib.cpp'] - ) - -setup(name='hello', - version='0.1.0', - packages=find_packages(), - description='Hello world module written in C', - ext_modules=[zlib_example, hello_module]) +hello_module = Extension( + pkg_name + "._hello", + language="c++", + sources=["hello.cpp"], + extra_link_args=link_args(), + extra_objects=[build_platlib + "/hello/" + zlib_soname], +) +zlib_example = Library( + pkg_name + "." + zlib_name, + language="c++", + extra_compile_args=["-lz"], + extra_link_args=link_args(zlib_soname) + ["-lz"], + sources=["extensions/testzlib.cpp"], +) + +setup( + name="hello", + version="0.1.0", + packages=find_packages(), + description="Hello world module written in C", + ext_modules=[zlib_example, hello_module], +) diff --git a/tests/integration/nonpy_rpath/tests/manual_test.py b/tests/integration/nonpy_rpath/tests/manual_test.py index cb325722..3f9c1ea9 100644 --- a/tests/integration/nonpy_rpath/tests/manual_test.py +++ b/tests/integration/nonpy_rpath/tests/manual_test.py @@ -1,3 +1,4 @@ -if __name__ == '__main__': +if __name__ == "__main__": from hello import z_compress, z_uncompress - assert z_uncompress(z_compress('test')) == 'test' + + assert z_uncompress(z_compress("test")) == "test" From e14a99f17029b8ee8210a7672296f710e51f7f72 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sat, 6 Feb 2021 17:25:56 -0600 Subject: [PATCH 11/16] Fix file modes of non-executable files. --- tests/integration/nonpy_rpath/README.md | 0 tests/integration/nonpy_rpath/hello.cpp | 0 tests/integration/nonpy_rpath/setup.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 tests/integration/nonpy_rpath/README.md mode change 100755 => 100644 tests/integration/nonpy_rpath/hello.cpp mode change 100755 => 100644 tests/integration/nonpy_rpath/setup.py diff --git a/tests/integration/nonpy_rpath/README.md b/tests/integration/nonpy_rpath/README.md old mode 100755 new mode 100644 diff --git a/tests/integration/nonpy_rpath/hello.cpp b/tests/integration/nonpy_rpath/hello.cpp old mode 100755 new mode 100644 diff --git a/tests/integration/nonpy_rpath/setup.py b/tests/integration/nonpy_rpath/setup.py old mode 100755 new mode 100644 From eaa0ddc11cabde2b5e177ccb9386040e43b065fc Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sat, 6 Feb 2021 17:26:02 -0600 Subject: [PATCH 12/16] Define __all__. --- tests/integration/nonpy_rpath/hello/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/nonpy_rpath/hello/__init__.py b/tests/integration/nonpy_rpath/hello/__init__.py index ef20328b..f20c5a09 100644 --- a/tests/integration/nonpy_rpath/hello/__init__.py +++ b/tests/integration/nonpy_rpath/hello/__init__.py @@ -1 +1,3 @@ from ._hello import z_compress, z_uncompress + +__all__ = ["z_compress", "z_uncompress"] From d13f36bc2c11f6222d841a561daf7a5d1edbc3e7 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sat, 6 Feb 2021 17:44:58 -0600 Subject: [PATCH 13/16] Test repaired wheel outside container. --- tests/integration/test_manylinux.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index 09153e97..945ce9b6 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -693,3 +693,7 @@ def test_nonpy_rpath(any_manylinux_container, docker_python, io_folder): f'hello-0.1.0-{PYTHON_ABI}-{policy}.whl is consistent' f' with the following platform tag: "{policy}"' ) in output.replace('\n', ' ') + + # Test the resulting wheel outside the manylinux container + docker_exec(docker_python, 'pip install -U /io/' + repaired_wheel) + docker_exec(docker_python, 'python /auditwheel_src/tests/integration/nonpy_rpath/tests/manual_test.py') From 3076485df04ac1e09acb58a53d95e46b05a6fdf6 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 22 Feb 2021 10:36:08 -0600 Subject: [PATCH 14/16] Add tests with new extension. --- tests/integration/test_manylinux.py | 43 ++++++++++++++++++- .../test_nonpy_dependencies/dependency.cpp | 16 +++++++ .../test_nonpy_dependencies/dependency.h | 4 ++ .../nonpy_dependency.cpp | 9 ++++ .../nonpy_dependency.h | 3 ++ .../test_nonpy_dependencies/setup.py | 32 ++++++++++++++ .../testdependencies.cpp | 21 +++++++++ .../tests/manual_test.py | 4 ++ 8 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_nonpy_dependencies/dependency.cpp create mode 100644 tests/integration/test_nonpy_dependencies/dependency.h create mode 100644 tests/integration/test_nonpy_dependencies/nonpy_dependency.cpp create mode 100644 tests/integration/test_nonpy_dependencies/nonpy_dependency.h create mode 100644 tests/integration/test_nonpy_dependencies/setup.py create mode 100644 tests/integration/test_nonpy_dependencies/testdependencies.cpp create mode 100644 tests/integration/test_nonpy_dependencies/tests/manual_test.py diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index 945ce9b6..b0327546 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -556,7 +556,7 @@ def test_build_repair_multiple_top_level_modules_wheel(any_manylinux_container, def test_build_repair_wheel_with_internal_rpath(any_manylinux_container, docker_python, io_folder): - + policy, manylinux_ctr = any_manylinux_container @@ -576,7 +576,7 @@ def test_build_repair_wheel_with_internal_rpath(any_manylinux_container, docker_ assert filenames == [f'internal_rpath-1.0-{PYTHON_ABI}-linux_{PLATFORM}.whl'] orig_wheel = filenames[0] assert 'manylinux' not in orig_wheel - + # Repair the wheel using the appropriate manylinux container repair_command = f'auditwheel repair --plat {policy} -w /io /io/{orig_wheel}' docker_exec( @@ -697,3 +697,42 @@ def test_nonpy_rpath(any_manylinux_container, docker_python, io_folder): # Test the resulting wheel outside the manylinux container docker_exec(docker_python, 'pip install -U /io/' + repaired_wheel) docker_exec(docker_python, 'python /auditwheel_src/tests/integration/nonpy_rpath/tests/manual_test.py') + + +def test_nonpy_rpath2(any_manylinux_container, docker_python, io_folder): + # Tests https://github.com/pypa/auditwheel/issues/136 + policy, manylinux_ctr = any_manylinux_container + docker_exec( + manylinux_ctr, + ['bash', '-c', 'cd /auditwheel_src/tests/integration/test_nonpy_dependencies ' + '&& python -m pip wheel --no-deps -w /io .'] + ) + + orig_wheel, *_ = os.listdir(io_folder) + assert orig_wheel.startswith("test_nonpy_dependencies-0.0.1") + assert 'manylinux' not in orig_wheel + + # Repair the wheel using the appropriate manylinux container + repair_command = \ + f'auditwheel repair --plat {policy} --strip -w /io /io/{orig_wheel}' + docker_exec(manylinux_ctr, repair_command) + + repaired_wheel, *_ = glob.glob(f"{io_folder}/*{policy}*.whl") + repaired_wheel = os.path.basename(repaired_wheel) + + docker_exec(docker_python, "pip install /io/" + repaired_wheel) + output = docker_exec( + docker_python, + ["python", "-c", "import test_nonpy_dependencies; print(test_nonpy_dependencies.reverse_string(\"banana\"))"] + ) + assert output.strip() == "ananab" + + output = docker_exec(manylinux_ctr, 'auditwheel show /io/' + repaired_wheel) + assert ( + f'test_nonpy_dependencies-0.0.1-{PYTHON_ABI}-{policy}.whl is consistent' + f' with the following platform tag: "{policy}"' + ) in output.replace('\n', ' ') + + # Test the resulting wheel outside the manylinux container + docker_exec(docker_python, 'pip install -U /io/' + repaired_wheel) + docker_exec(docker_python, 'python /auditwheel_src/tests/integration/test_nonpy_dependencies/tests/manual_test.py') diff --git a/tests/integration/test_nonpy_dependencies/dependency.cpp b/tests/integration/test_nonpy_dependencies/dependency.cpp new file mode 100644 index 00000000..03fa019e --- /dev/null +++ b/tests/integration/test_nonpy_dependencies/dependency.cpp @@ -0,0 +1,16 @@ +#define PY_SSIZE_T_CLEAN +#include +#include "nonpy_dependency.h" +#include + +PyObject* reverse_string(PyObject *self, PyObject *args) { + const char* str_to_reverse; + if (!PyArg_ParseTuple(args, "s", &str_to_reverse)) { + return NULL; + } + + std::string str_to_reverse_s(str_to_reverse); + std::string reversed = make_reversed(str_to_reverse_s); + const char * str_reversed = &*reversed.begin(); + return PyUnicode_FromStringAndSize(str_reversed, reversed.length()); +} diff --git a/tests/integration/test_nonpy_dependencies/dependency.h b/tests/integration/test_nonpy_dependencies/dependency.h new file mode 100644 index 00000000..a89dc3d3 --- /dev/null +++ b/tests/integration/test_nonpy_dependencies/dependency.h @@ -0,0 +1,4 @@ +#define PY_SSIZE_T_CLEAN +#include + +PyObject* reverse_string(PyObject *self, PyObject *args); \ No newline at end of file diff --git a/tests/integration/test_nonpy_dependencies/nonpy_dependency.cpp b/tests/integration/test_nonpy_dependencies/nonpy_dependency.cpp new file mode 100644 index 00000000..266dbb0d --- /dev/null +++ b/tests/integration/test_nonpy_dependencies/nonpy_dependency.cpp @@ -0,0 +1,9 @@ +#include "nonpy_dependency.h" +#include +#include + +std::string make_reversed(std::string s){ + std::string s_copy(s); + std::reverse(s_copy.begin(), s_copy.end()); + return s_copy; +} diff --git a/tests/integration/test_nonpy_dependencies/nonpy_dependency.h b/tests/integration/test_nonpy_dependencies/nonpy_dependency.h new file mode 100644 index 00000000..a5c161ff --- /dev/null +++ b/tests/integration/test_nonpy_dependencies/nonpy_dependency.h @@ -0,0 +1,3 @@ +#include + +std::string make_reversed(std::string s); diff --git a/tests/integration/test_nonpy_dependencies/setup.py b/tests/integration/test_nonpy_dependencies/setup.py new file mode 100644 index 00000000..20f98ba5 --- /dev/null +++ b/tests/integration/test_nonpy_dependencies/setup.py @@ -0,0 +1,32 @@ +import sysconfig +from setuptools import setup, Extension +import subprocess +from os import path +from os import getenv + +# Build the string reverser library (an example library) +cmd = "gcc -Wall -shared -fPIC nonpy_dependency.cpp -o libnonpy_dependency.so" +subprocess.check_call(cmd.split()) + +# Build the dependency that calls the string reverser +python_include_dir = sysconfig.get_paths()["include"] +current_dir = path.abspath(path.dirname(__file__)) +cmd = f"gcc -Wall -shared -fPIC -I{python_include_dir} -L{current_dir} dependency.cpp -o libdependency.so -Wl,-rpath={current_dir} -lnonpy_dependency" +subprocess.check_call(cmd.split()) + +libraries = ["dependency"] +library_dirs = [current_dir] + +setup( + name="test_nonpy_dependencies", + version="0.0.1", + ext_modules=[ + Extension( + "test_nonpy_dependencies", + sources=["testdependencies.cpp"], + libraries=libraries, + library_dirs=library_dirs, + runtime_library_dirs=["$ORIGIN"], + ) + ], +) diff --git a/tests/integration/test_nonpy_dependencies/testdependencies.cpp b/tests/integration/test_nonpy_dependencies/testdependencies.cpp new file mode 100644 index 00000000..ca7e3504 --- /dev/null +++ b/tests/integration/test_nonpy_dependencies/testdependencies.cpp @@ -0,0 +1,21 @@ +#define PY_SSIZE_T_CLEAN +#include +#include "dependency.h" +#include + +/* Module initialization */ +PyMODINIT_FUNC PyInit_test_nonpy_dependencies(void) +{ + static PyMethodDef module_methods[] = { + {"reverse_string", (PyCFunction)reverse_string, METH_VARARGS, "Reverse a string."}, + {NULL} /* Sentinel */ + }; + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + .m_name = "test_nonpy_dependencies", + .m_doc = "test_nonpy_dependencies module", + .m_size = -1, + .m_methods = module_methods, + }; + return PyModule_Create(&moduledef); +} diff --git a/tests/integration/test_nonpy_dependencies/tests/manual_test.py b/tests/integration/test_nonpy_dependencies/tests/manual_test.py new file mode 100644 index 00000000..bceeaf77 --- /dev/null +++ b/tests/integration/test_nonpy_dependencies/tests/manual_test.py @@ -0,0 +1,4 @@ +if __name__ == "__main__": + from test_nonpy_dependencies import reverse_string + + assert reverse_string("asdf!") == "!fdsa" From 82a255ebb16376e1a6ba0bd4dd252be60a0977ed Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 22 Feb 2021 10:48:53 -0600 Subject: [PATCH 15/16] Rename modules. --- .../tests/{manual_test.py => manual_test_nonpy_rpath.py} | 0 .../tests/{manual_test.py => manual_test_nonpy_rpath.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/integration/nonpy_rpath/tests/{manual_test.py => manual_test_nonpy_rpath.py} (100%) rename tests/integration/test_nonpy_dependencies/tests/{manual_test.py => manual_test_nonpy_rpath.py} (100%) diff --git a/tests/integration/nonpy_rpath/tests/manual_test.py b/tests/integration/nonpy_rpath/tests/manual_test_nonpy_rpath.py similarity index 100% rename from tests/integration/nonpy_rpath/tests/manual_test.py rename to tests/integration/nonpy_rpath/tests/manual_test_nonpy_rpath.py diff --git a/tests/integration/test_nonpy_dependencies/tests/manual_test.py b/tests/integration/test_nonpy_dependencies/tests/manual_test_nonpy_rpath.py similarity index 100% rename from tests/integration/test_nonpy_dependencies/tests/manual_test.py rename to tests/integration/test_nonpy_dependencies/tests/manual_test_nonpy_rpath.py From b75e8ed6dae95fe69a889d6a168d390885d6e8be Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 22 Feb 2021 12:25:21 -0600 Subject: [PATCH 16/16] Update tests. --- tests/integration/test_manylinux.py | 15 +++++++++------ ...rpath.py => manual_test_nonpy_dependencies.py} | 0 2 files changed, 9 insertions(+), 6 deletions(-) rename tests/integration/test_nonpy_dependencies/tests/{manual_test_nonpy_rpath.py => manual_test_nonpy_dependencies.py} (100%) diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index b0327546..0a7f67db 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -696,16 +696,16 @@ def test_nonpy_rpath(any_manylinux_container, docker_python, io_folder): # Test the resulting wheel outside the manylinux container docker_exec(docker_python, 'pip install -U /io/' + repaired_wheel) - docker_exec(docker_python, 'python /auditwheel_src/tests/integration/nonpy_rpath/tests/manual_test.py') + docker_exec(docker_python, 'python /auditwheel_src/tests/integration/nonpy_rpath/tests/manual_test_nonpy_rpath.py') -def test_nonpy_rpath2(any_manylinux_container, docker_python, io_folder): +def test_nonpy_dependencies(any_manylinux_container, docker_python, io_folder): # Tests https://github.com/pypa/auditwheel/issues/136 policy, manylinux_ctr = any_manylinux_container docker_exec( manylinux_ctr, ['bash', '-c', 'cd /auditwheel_src/tests/integration/test_nonpy_dependencies ' - '&& python -m pip wheel --no-deps -w /io .'] + '&& python setup.py bdist_wheel -d /io'] ) orig_wheel, *_ = os.listdir(io_folder) @@ -715,7 +715,10 @@ def test_nonpy_rpath2(any_manylinux_container, docker_python, io_folder): # Repair the wheel using the appropriate manylinux container repair_command = \ f'auditwheel repair --plat {policy} --strip -w /io /io/{orig_wheel}' - docker_exec(manylinux_ctr, repair_command) + docker_exec( + manylinux_ctr, + ['bash', '-c', 'LD_LIBRARY_PATH=/auditwheel_src/tests/integration/test_nonpy_dependencies:$LD_LIBRARY_PATH ' + repair_command], + ) repaired_wheel, *_ = glob.glob(f"{io_folder}/*{policy}*.whl") repaired_wheel = os.path.basename(repaired_wheel) @@ -730,9 +733,9 @@ def test_nonpy_rpath2(any_manylinux_container, docker_python, io_folder): output = docker_exec(manylinux_ctr, 'auditwheel show /io/' + repaired_wheel) assert ( f'test_nonpy_dependencies-0.0.1-{PYTHON_ABI}-{policy}.whl is consistent' - f' with the following platform tag: "{policy}"' + f' with the following platform tag: "manylinux1_x86_64"' ) in output.replace('\n', ' ') # Test the resulting wheel outside the manylinux container docker_exec(docker_python, 'pip install -U /io/' + repaired_wheel) - docker_exec(docker_python, 'python /auditwheel_src/tests/integration/test_nonpy_dependencies/tests/manual_test.py') + docker_exec(docker_python, 'python /auditwheel_src/tests/integration/test_nonpy_dependencies/tests/manual_test_nonpy_dependencies.py') diff --git a/tests/integration/test_nonpy_dependencies/tests/manual_test_nonpy_rpath.py b/tests/integration/test_nonpy_dependencies/tests/manual_test_nonpy_dependencies.py similarity index 100% rename from tests/integration/test_nonpy_dependencies/tests/manual_test_nonpy_rpath.py rename to tests/integration/test_nonpy_dependencies/tests/manual_test_nonpy_dependencies.py