Merge pull request #96 from jokva/new-python-buildsystem

New python buildsystem

.travis.yml

@@ -70,15 +70,17 @@ before_install:
     - before_install
 
 install:
-    - pip install bandit
-                  numpy
-                  setuptools
-                  setuptools_scm
-                  pytest
-                  pytest-runner
-                  pybind11
-                  hypothesis
-                  sphinx
+    - pip install --upgrade
+        bandit
+        numpy
+        setuptools
+        setuptools_scm
+        pytest
+        pytest-runner
+        pybind11
+        hypothesis
+        sphinx
+        scikit-build
     - bandit -c bandit.yml -r python/
 
 before_script:
@@ -92,19 +94,15 @@ before_script:
                   ..
     - popd
 
-# distutils/setuptools on macos ignores the --rpath argument, so set
-# DYLD_LIBRARY_PATH so that the freshly-built image is picked up on for the
-# tests.
-#
-# the build is configured with CMAKE_INSTALL_NAME_DIR so that in the wheel
-# build on OS X, the delocate tool can figure out what dylib to include.
 script:
     - pushd build
-    - export DYLD_LIBRARY_PATH=$PWD/lib
     - make
     - ctest --output-on-failure
     - sudo make install
     - popd
+    # install works, so remove the _skbuild because it having root permissions
+    # from make install breaks build_wheel
+    - sudo rm -rf python/_skbuild
     - build_wheel python $PLAT
     - install_run $PLAT
     - mv wheelhouse python/dist

appveyor.yml

@@ -27,18 +27,20 @@ matrix:
 install:
     - IF DEFINED PYTHON (IF "%platform%" == "x64" SET PYTHON=%PYTHON%-x64)
     - IF DEFINED PYTHON SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH%
-    - IF DEFINED PYTHON pip install numpy
-                                    pytest
-                                    pytest-runner
-                                    setuptools
-                                    setuptools_scm
-                                    pybind11
-                                    hypothesis
-                                    twine
-                                    wheel
+    - IF DEFINED PYTHON pip install --upgrade
+        numpy
+        pytest
+        setuptools
+        setuptools_scm
+        pybind11
+        hypothesis
+        twine
+        wheel
+        git+https://github.com/jokva/scikit-build
 
 before_build:
-    - IF "%platform%" == "x64" set W64="-GVisual Studio 14 2015 Win64"
+    - set generator="Visual Studio 14 2015"
+    - IF "%platform%" == "x64" set generator="Visual Studio 14 2015 Win64"
     - git submodule update --init --recursive
     - git fetch --tags
 
@@ -47,16 +49,14 @@ build_script:
           mkdir build
           pushd build
     - cmake %APPVEYOR_BUILD_FOLDER%
-            %W64%
+            -G %generator%
             -DCMAKE_CXX_FLAGS="/D_CRT_SECURE_NO_WARNINGS /EHsc"
-    - cmake --build .
-            --config %configuration%
-    - ctest --build-config %configuration%
-            --output-on-failure
+    - cmake --build .  --config %configuration%
+    - ctest --build-config %configuration% --output-on-failure
     - ps: popd
     - ps: pushd python
     - git describe
-    - IF DEFINED PYTHON python setup.py bdist_wheel
+    - IF DEFINED PYTHON python setup.py bdist_wheel -G %generator%
     - ps: popd
 
 before_deploy:

config.sh

@@ -8,7 +8,7 @@ function run_tests {
 function pre_build {
     if [ -d build-centos5 ]; then return; fi
 
-    python -m pip install cmake pybind11
+    python -m pip install cmake pybind11 scikit-build
 
     mkdir build-centos5
     pushd build-centos5

lib/CMakeLists.txt

@@ -1,3 +1,5 @@
+set(DLISIO_LIB_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR} CACHE PATH "")
+
 add_library(dlisio src/dlisio.cpp
                    src/types.cpp
 )
@@ -21,7 +23,6 @@ install(TARGETS dlisio
 )
 install(DIRECTORY include/ DESTINATION include)
 install(EXPORT dlisio DESTINATION share/dlisio/cmake FILE dlisio-config.cmake)
-export(TARGETS dlisio FILE dlisio-config.cmake)
 
 add_library(dlisio-extension src/parse.cpp
                              src/io.cpp
@@ -51,6 +52,16 @@ install(TARGETS dlisio-extension
         RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
 )
 install(DIRECTORY extension/ DESTINATION include)
+export(
+    TARGETS
+        dlisio
+        dlisio-extension
+        mpark-variant
+        mio
+        fmt-header-only
+    FILE
+        dlisio-config.cmake
+)
 
 if (BUILD_SHARED_LIBS)
     target_compile_definitions(dlisio

python/CMakeLists.txt

@@ -1,3 +1,13 @@
+if (SKBUILD)
+    # invoked as a part of scikit-build, so this is just a proxy for the python
+    # extension cmake. this works around the fundamental limitation in cmake
+    # that it looks only for directories with a CMakeLists.txt in it, not for a
+    # named file
+    include(setup-CMakeLists.txt)
+    return ()
+endif ()
+
+cmake_minimum_required(VERSION 3.5.0)
 project(dlisio-python)
 
 if(NOT BUILD_PYTHON)
@@ -9,22 +19,21 @@ find_package(PythonInterp REQUIRED)
 if(NOT PYTHON_EXECUTABLE)
     message(WARNING "Could not find python - skipping python bindings")
     message(WARNING "Select specific python distribution with "
-                    "-DPYTHON_EXECUTABLE=bin/python"
-    )
+                    "-DPYTHON_EXECUTABLE=bin/python")
     return()
 endif()
 
 set(python ${PYTHON_EXECUTABLE})
 set(setup.py ${CMAKE_CURRENT_SOURCE_DIR}/setup.py)
 
-if (NOT WIN32)
-    # setuptools on microsoft compilers doesn't support the --library-dir or
-    # --build-dir flag and crashes, so only pass it on non-microsoft platforms
-    set(build_ext_args --library-dirs ${CMAKE_CURRENT_SOURCE_DIR}
-                       --rpath ${CMAKE_CURRENT_SOURCE_DIR}
+if (CMAKE_BUILD_TYPE)
+    # use the cmake_build_type of the source project, unless it has been
+    # specifically overriden
+    set(DLISIO_PYTHON_BUILD_TYPE
+        --build-type=${CMAKE_BUILD_TYPE}
+        CACHE STRING "override CMAKE_BUILD_TYPE in python extension"
     )
-endif()
-
+endif ()
 
 add_custom_target(
     dlisio-python ALL
@@ -34,41 +43,48 @@ add_custom_target(
     VERBATIM
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 
-    # setuptools on windows breaks spectacularly when the library isn't
-    # available in the same directory, and build_ext --library-dirs is not
-    # support on msvc is not supported, so we must copy out the libsegyio core
-    # object and put it here
-    COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:dlisio>
-                                     $<TARGET_FILE_NAME:dlisio>
-
-    COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_LINKER_FILE:dlisio>
-                                     $<TARGET_LINKER_FILE_NAME:dlisio>
-
-    COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:dlisio-extension>
-                                     $<TARGET_FILE_NAME:dlisio-extension>
-
-    COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_LINKER_FILE:dlisio-extension>
-                                     $<TARGET_LINKER_FILE_NAME:dlisio-extension>
-
-    COMMAND ${python} ${setup.py} build_ext ${build_ext_args} build
+    COMMAND ${python} ${setup.py}
+        # build the extension inplace (really, once its built, copy it to the
+        # source tree) so that post-build, the directory can be used to run
+        # tests against
+        build_ext --inplace
+        build # setup.py build args
+            --cmake-executable ${CMAKE_COMMAND}
+            --generator ${CMAKE_GENERATOR}
+            ${DLISIO_PYTHON_BUILD_TYPE}
+        -- # cmake to the extension
+            -Ddlisio_DIR=${DLISIO_LIB_BINARY_DIR}
+            # "install" to the python/dlisio dir with rpath, so there's no need
+            # to fiddle with environment in ctest to load the core library from
+            # the build tree
+            -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON
+            -DCMAKE_INSTALL_RPATH=$<TARGET_FILE_DIR:dlisio>
+            -DCMAKE_INSTALL_NAME_DIR=$<TARGET_FILE_DIR:dlisio>
 )
 
 add_dependencies(dlisio-python dlisio dlisio-extension)
 
 install(CODE "
-if (DEFINED ENV{DESTDIR})
-    get_filename_component(abs-destdir \"\$ENV{DESTDIR}\" ABSOLUTE)
-    set(root_destdir --root=\${abs-destdir})
-endif()
+    if (DEFINED ENV{DESTDIR})
+        get_filename_component(abs-destdir \"\$ENV{DESTDIR}\" ABSOLUTE)
+        set(root_destdir --root \${abs-destdir})
+    endif()
 
-execute_process(COMMAND
-    ${python} ${setup.py}
-        install --prefix=${CMAKE_INSTALL_PREFIX}
-                --single-version-externally-managed
-                --record installed-files
+    execute_process(
+        COMMAND ${python} ${setup.py}
+            install
                 \${root_destdir}
+                --single-version-externally-managed
+                --record record.txt
+                --cmake-executable \"${CMAKE_COMMAND}\"
+                --generator \"${CMAKE_GENERATOR}\"
+                ${DLISIO_PYTHON_BUILD_TYPE}
+            --
+                -Ddlisio_DIR=${DLISIO_LIB_BINARY_DIR}
+                -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-)")
+    )"
+)
 
 option(BUILD_PYDOC "Build python documentation" OFF)
 
@@ -107,10 +123,15 @@ if(BUILD_PYDOC AND sphinx)
         DESTINATION ${CMAKE_INSTALL_DATADIR}/doc/dlisio
     )
 endif()
+
 # run tests with setup.py test
 # this is very slow compared to invoking pytest directly, but setuptools will
 # copy the built extension into the tree as it sees fit
+#
+# use --skip-cmake, otherwise running the tests would trigger a build with
+# different args to setup.py, rebuilding the python lib (and wrongly so as it
+# either won't find dlisio or picked up on a system installed one)
 add_test(NAME python.unit
-         COMMAND ${python} ${setup.py} test
-         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND ${python} ${setup.py} --skip-cmake test
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )

python/setup-CMakeLists.txt

@@ -0,0 +1,27 @@
+cmake_minimum_required(VERSION 3.5.0)
+project(dlisio-python-extension LANGUAGES C CXX)
+
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_VISIBILITY_PRESET "hidden")
+set(CMAKE_C_VISIBILITY_PRESET   "hidden")
+
+find_package(PythonExtensions REQUIRED)
+find_package(dlisio REQUIRED)
+
+add_library(core MODULE dlisio/ext/core.cpp)
+target_include_directories(core
+    PRIVATE
+        ${PYBIND11_INCLUDE_DIRS}
+)
+python_extension_module(core)
+target_link_libraries(core dlisio dlisio-extension)
+
+if (MSVC)
+    target_compile_options(core
+        BEFORE
+        PRIVATE
+            /EHsc
+    )
+endif ()
+
+install(TARGETS core LIBRARY DESTINATION dlisio)