From 96e2bd2324efa8a8a43985d7dca44f2cc5c9fcca Mon Sep 17 00:00:00 2001 From: Amine Khaldi Date: Thu, 15 Jun 2023 21:22:23 +0100 Subject: [PATCH] Streamline the processes of building libbls, testing it, and creating the blspy wheels (#367) * Speedup the process of creating wheels by skipping the compiling and linking of libbls' tests and benchmarks. * We no longer need Windows-specific CMake version checking in setup.py. * We no longer need Windows-specific architecture flag passing in setup.py. * We no longer treat Windows in a special way w.r.t. compiling and linking in setup.py. * We no longer need Windows-specific preparations before starting the process of creating wheels. * We no longer need Windows-specific repairs after wheels creation. * We no longer need the relic_ietf_64 repository and its related tasks. Dedicated to @wjblanke --- .github/workflows/build-wheels.yml | 20 ---- CMakeLists.txt | 34 +++++- python-bindings/CMakeLists.txt | 3 +- setup.py | 178 +++-------------------------- src/CMakeLists.txt | 4 + 5 files changed, 53 insertions(+), 186 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 22c67980a..57e9defcc 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -154,26 +154,6 @@ jobs: CIBW_BEFORE_BUILD_MACOS: > python -m pip install --upgrade pip CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=10.14" - CIBW_BEFORE_ALL_WINDOWS: > - curl -L https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable-msvc.zip > libsodium-1.0.18-stable-msvc.zip - && 7z x libsodium-1.0.18-stable-msvc.zip - && git clone https://github.com/Chia-Network/relic_ietf_64.git - && ls -l relic_ietf_64 - && git clone https://github.com/Chia-Network/mpir_gc_x64.git - && ls -l mpir_gc_x64 - CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: > - ls -l mpir_gc_x64 && pip uninstall -y delocate - && pip install git+https://github.com/Chia-Network/delocate.git - && delocate-wheel -v -i mpir_gc_x64/mpir.dll {wheel} - && delocate-wheel -v -i mpir_gc_x64/mpir_gc.dll {wheel} - && delocate-wheel -v -i mpir_gc_x64/mpir_broadwell.dll {wheel} - && delocate-wheel -v -i mpir_gc_x64/mpir_broadwell_avx.dll {wheel} - && delocate-wheel -v -i mpir_gc_x64/mpir_bulldozer.dll {wheel} - && delocate-wheel -v -i mpir_gc_x64/mpir_haswell.dll {wheel} - && delocate-wheel -v -i mpir_gc_x64/mpir_piledriver.dll {wheel} - && delocate-wheel -v -i mpir_gc_x64/mpir_sandybridge.dll {wheel} - && delocate-wheel -v -i mpir_gc_x64/mpir_skylake_avx.dll {wheel} - && cp {wheel} {dest_dir} CIBW_TEST_REQUIRES: pytest CIBW_TEST_COMMAND: py.test -v {project}/python-bindings/test.py run: diff --git a/CMakeLists.txt b/CMakeLists.txt index 57a235132..1d8d1e3af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,7 +47,7 @@ if(DEFINED ENV{RELIC_MAIN}) set(RELIC_REPOSITORY "https://github.com/relic-toolkit/relic.git") else() # This is currently anchored to upstream aecdcae7956f542fbee2392c1f0feb0a8ac41dc5 - set(RELIC_GIT_TAG "215c69966cb78b255995f0ee9c86bbbb41c3c42b") + set(RELIC_GIT_TAG "2a4ed8ded3a9a5ac5e39f3877f357264b2e16f11") set(RELIC_REPOSITORY "https://github.com/Chia-Network/relic.git") endif() @@ -63,12 +63,25 @@ FetchContent_Declare( set(STBIN "off" CACHE STRING "Relic - Build static binaries") -find_package(gmp) -if (GMP_FOUND) - message(STATUS "Found libgmp") +if(MSVC) set(ARITH "gmp" CACHE STRING "") + # We'll be using our custom MPIR build instead of GMP for MSVC builds + FetchContent_Declare( + chia_mpir + GIT_REPOSITORY https://github.com/Chia-Network/mpir_gc_x64.git + GIT_SHALLOW true + ) + FetchContent_MakeAvailable(chia_mpir) + set(MPIR_INCLUDE_DIR ${chia_mpir_SOURCE_DIR} CACHE STRING "") + set(MPIR_IMPLIB "${chia_mpir_SOURCE_DIR}/mpir.lib" CACHE STRING "") else() - set(ARITH "easy" CACHE STRING "") + find_package(gmp) + if(GMP_FOUND) + message(STATUS "Found libgmp") + set(ARITH "gmp" CACHE STRING "") + else() + set(ARITH "easy" CACHE STRING "") + endif() endif() if(CMAKE_SIZEOF_VOID_P EQUAL 4) @@ -141,6 +154,17 @@ set(PP_METHD "LAZYR;OATEP" CACHE STRING "") FetchContent_MakeAvailable(relic) +function(handle_mpir_dlls _target) + file(GLOB chia_mpir_dlls ${chia_mpir_SOURCE_DIR}/*.dll) + foreach(mpir_dll IN LISTS chia_mpir_dlls) + add_custom_command( + TARGET ${_target} + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${mpir_dll} $ + ) + endforeach() +endfunction() + add_subdirectory(src) if(EMSCRIPTEN) diff --git a/python-bindings/CMakeLists.txt b/python-bindings/CMakeLists.txt index 2e331c3d6..364386696 100644 --- a/python-bindings/CMakeLists.txt +++ b/python-bindings/CMakeLists.txt @@ -4,7 +4,8 @@ FetchContent_Declare( GIT_REPOSITORY https://github.com/pybind/pybind11.git GIT_TAG v2.10.0 ) -FetchContent_MakeAvailable(pybind11 relic) +FetchContent_MakeAvailable(pybind11) pybind11_add_module(blspy ${CMAKE_CURRENT_SOURCE_DIR}/pythonbindings.cpp) target_link_libraries(blspy PRIVATE bls) +handle_mpir_dlls(blspy) diff --git a/setup.py b/setup.py index 7b4c0bfbb..83767963f 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,9 @@ #!/usr/bin/python3 import os import platform -import re import subprocess import sys -from distutils.version import LooseVersion - -from setuptools import Extension, setup, setuptools +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext @@ -19,7 +16,7 @@ def __init__(self, name, sourcedir=""): class CMakeBuild(build_ext): def run(self): try: - out = subprocess.check_output(["cmake", "--version"]) + subprocess.check_output(["cmake", "--version"]) except OSError: raise RuntimeError( "CMake must be installed to build" @@ -27,13 +24,6 @@ def run(self): + ", ".join(e.name for e in self.extensions) ) - if platform.system() == "Windows": - cmake_version = LooseVersion( - re.search(r"version\s*([\d.]+)", out.decode()).group(1) - ) - if cmake_version < "3.1.0": - raise RuntimeError("CMake >= 3.1.0 is required on Windows") - for ext in self.extensions: self.build_extension(ext) @@ -42,6 +32,8 @@ def build_extension(self, ext): cmake_args = [ "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=" + extdir, "-DPYTHON_EXECUTABLE=" + sys.executable, + "-DBUILD_BLS_TESTS=0", + "-DBUILD_BLS_BENCHMARKS=0", ] cfg = "Debug" if self.debug else "Release" @@ -51,9 +43,6 @@ def build_extension(self, ext): cmake_args += [ "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(cfg.upper(), extdir) ] - if sys.maxsize > 2 ** 32: - cmake_args += ["-A", "x64"] - build_args += ["--", "/m"] else: cmake_args += ["-DCMAKE_BUILD_TYPE=" + cfg] build_args += ["--", "-j", "6"] @@ -72,148 +61,17 @@ def build_extension(self, ext): ) -class get_pybind_include(object): - """Helper class to determine the pybind11 include path - - The purpose of this class is to postpone importing pybind11 - until it is actually installed, so that the ``get_include()`` - method can be invoked.""" - - def __init__(self, user=False): - self.user = user - - def __str__(self): - import pybind11 - - return pybind11.get_include(self.user) - - -ext_modules = [ - Extension( - "blspy", - [ - "src/elements.cpp", - "src/schemes.cpp", - "src/privatekey.cpp", - "src/bls.cpp", - "python-bindings/pythonbindings.cpp", - ], - include_dirs=[ - # Path to pybind11 headers - get_pybind_include(), - get_pybind_include(user=True), - "relic_ietf_64/include", - "mpir_gc_x64", - "libsodium/include", - ], - library_dirs=[ - "relic_ietf_64", - "mpir_gc_x64", - "libsodium/x64/Release/v142/static", - ], - libraries=["relic_s", "Advapi32", "mpir", "libsodium"], - language="c++", - ), -] - - -# As of Python 3.6, CCompiler has a `has_flag` method. -# cf http://bugs.python.org/issue26689 -def has_flag(compiler, flagname): - """Return a boolean indicating whether a flag name is supported on - the specified compiler. - """ - import tempfile - - with tempfile.NamedTemporaryFile("w", suffix=".cpp") as f: - f.write("int main (int argc, char **argv) { return 0; }") - try: - compiler.compile([f.name], extra_postargs=[flagname]) - except setuptools.distutils.errors.CompileError: - return False - return True - - -def cpp_flag(compiler): - """Return the -std=c++[11/14/17] compiler flag. - - The newer version is prefered over c++11 (when it is available). - """ - flags = ["-std=c++17", "-std=c++14", "-std=c++11"] - - for flag in flags: - if has_flag(compiler, flag): - return flag - - raise RuntimeError("Unsupported compiler -- at least C++11 support " "is needed!") - - -class BuildExt(build_ext): - """A custom build extension for adding compiler-specific options.""" - - c_opts = { - "msvc": ["/EHsc", "/std:c++17", "/DBLSALLOC_SODIUM=1", "/DSODIUM_STATIC"], - "unix": [], - } - l_opts = { - "msvc": [], - "unix": [], - } - - if sys.platform == "darwin": - darwin_opts = ["-stdlib=libc++", "-mmacosx-version-min=10.14"] - c_opts["unix"] += darwin_opts - l_opts["unix"] += darwin_opts - - def build_extensions(self): - ct = self.compiler.compiler_type - opts = self.c_opts.get(ct, []) - link_opts = self.l_opts.get(ct, []) - if ct == "unix": - opts.append('-DVERSION_INFO="%s"' % self.distribution.get_version()) - opts.append(cpp_flag(self.compiler)) - if has_flag(self.compiler, "-fvisibility=hidden"): - opts.append("-fvisibility=hidden") - elif ct == "msvc": - if sys.version_info < (3, 9): - ver_flag = '/DVERSION_INFO=\"%s\"' - else: - ver_flag = '-DVERSION_INFO="%s"' - opts.append(ver_flag % self.distribution.get_version()) - for ext in self.extensions: - ext.extra_compile_args = opts - ext.extra_link_args = link_opts - build_ext.build_extensions(self) - - -if platform.system() == "Windows": - setup( - name="blspy", - author="Mariano Sorgente", - author_email="mariano@chia.net", - description="BLS signatures in c++ (with python bindings)", - long_description=open("README.md").read(), - long_description_content_type="text/markdown", - url="https://github.com/Chia-Network/bls-signatures", - python_requires=">=3.7", - setup_requires=["pybind11>=2.10.0"], - install_requires=["pybind11>=2.10.0"], - ext_modules=ext_modules, - cmdclass={"build_ext": BuildExt}, - zip_safe=False, - ) -else: - setup( - name="blspy", - author="Mariano Sorgente", - author_email="mariano@chia.net", - description="BLS signatures in c++ (python bindings)", - python_requires=">=3.7", - install_requires=["wheel"], - long_description=open("README.md").read(), - long_description_content_type="text/markdown", - url="https://github.com/Chia-Network/bls-signatures", - ext_modules=[CMakeExtension("blspy", ".")], - cmdclass=dict(build_ext=CMakeBuild), - zip_safe=False, - ) +setup( + name="blspy", + author="Mariano Sorgente", + author_email="mariano@chia.net", + description="BLS signatures in c++ (python bindings)", + python_requires=">=3.7", + install_requires=["wheel"], + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + url="https://github.com/Chia-Network/bls-signatures", + ext_modules=[CMakeExtension("blspy", ".")], + cmdclass=dict(build_ext=CMakeBuild), + zip_safe=False, +) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2512c0036..cd99ba976 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -39,6 +39,10 @@ install(DIRECTORY ${relic_BINARY_DIR}/include/ DESTINATION include/chiabls) install(FILES ${HEADERS} DESTINATION include/chiabls) install(FILES $ DESTINATION lib) +if(MSVC AND (BUILD_BLS_TESTS OR BUILD_BLS_BENCHMARKS)) + handle_mpir_dlls(bls) +endif() + if(BUILD_BLS_TESTS) FetchContent_Declare( Catch2