From 8d32bed7fb9d5a13704cf0b91c4a7097acf37972 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Tue, 28 Nov 2023 18:44:57 -0700 Subject: [PATCH 1/8] Start working on support for 'pip install adios2' --- CMakeLists.txt | 8 ++++-- bindings/Python/CMakeLists.txt | 11 +++++++-- cmake/DetectOptions.cmake | 36 ++++++++++++++++++--------- pyproject.toml | 45 ++++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 16 deletions(-) create mode 100644 pyproject.toml diff --git a/CMakeLists.txt b/CMakeLists.txt index a26c9b34b6..bbf6895741 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,6 +109,10 @@ set(ADIOS2_EXECUTABLE_SUFFIX "${ADIOS2_LIBRARY_SUFFIX}" CACHE STRING "Suffix to append to executable names") mark_as_advanced(ADIOS2_LIBRARY_SUFFIX ADIOS2_EXECUTABLE_SUFFIX) +# Advanced option to tell CMake the build is on behalf of pip +set(ADIOS2_PIP_INSTALL OFF CACHE BOOL "Turn on for pip install") +mark_as_advanced(ADIOS2_PIP_INSTALL) + # Use meta-compile features if available, otherwise use specific language # features if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Intel|Clang|AppleClang|MSVC)$") @@ -371,8 +375,8 @@ endif() message("") message("ADIOS2 build configuration:") message(" ADIOS Version: ${ADIOS2_VERSION}") -message(" C++ Compiler : ${CMAKE_CXX_COMPILER_ID} " - "${CMAKE_CXX_COMPILER_VERSION} " +message(" C++ Compiler : ${CMAKE_CXX_COMPILER_ID} " + "${CMAKE_CXX_COMPILER_VERSION} " "${CMAKE_CXX_COMPILER_WRAPPER}") message(" ${CMAKE_CXX_COMPILER}") message("") diff --git a/bindings/Python/CMakeLists.txt b/bindings/Python/CMakeLists.txt index 5c6121e8f6..782cd509d3 100644 --- a/bindings/Python/CMakeLists.txt +++ b/bindings/Python/CMakeLists.txt @@ -47,11 +47,18 @@ set_target_properties(adios2_py PROPERTIES COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 ) -string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2") + if(CMAKE_SYSTEM_NAME MATCHES "Linux") + if (ADIOS2_PIP_INSTALL) set_target_properties(adios2_py PROPERTIES - INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}" + INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}" ) + else() + string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2") + set_target_properties(adios2_py PROPERTIES + INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}" + ) + endif() endif() install(TARGETS adios2_py diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index efecb57624..fd63ede3d9 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -52,7 +52,9 @@ endfunction() # Extract the common prefix from a collection of variables function(lists_get_prefix listVars outVar) foreach(l IN LISTS listVars) + message(" l: ${l}") foreach(d IN LISTS ${l}) + message(" d: ${d}") if(NOT prefix) set(prefix "${d}") continue() @@ -64,6 +66,7 @@ function(lists_get_prefix listVars outVar) endif() endforeach() endforeach() + message("setting ${outVar} = ${prefix} in the PARENT_SCOPE") set(${outVar} "${prefix}" PARENT_SCOPE) endfunction() @@ -423,21 +426,30 @@ endif() if(Python_Interpreter_FOUND) # Setup output directories - if(Python_Development_FOUND) - lists_get_prefix("Python_INCLUDE_DIRS;Python_LIBRARIES;Python_SITEARCH" _Python_DEVPREFIX) + if (ADIOS2_PIP_INSTALL) + # do it the pip way + set(CMAKE_INSTALL_PYTHONDIR ${Python_SITEARCH}) + set(CMAKE_PYTHON_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/${Python_SITEARCH} + ) else() - lists_get_prefix("Python_EXECUTABLE;Python_SITEARCH" _Python_DEVPREFIX) + if(Python_Development_FOUND) + lists_get_prefix("Python_INCLUDE_DIRS;Python_LIBRARIES;Python_SITEARCH" _Python_DEVPREFIX) + else() + lists_get_prefix("Python_EXECUTABLE;Python_SITEARCH" _Python_DEVPREFIX) + endif() + message("This is borked! _Python_DEVPREFIX = ${_Python_DEVPREFIX}, Python_SITEARCH = ${Python_SITEARCH}") + string_strip_prefix( + "${_Python_DEVPREFIX}" "${Python_SITEARCH}" CMAKE_INSTALL_PYTHONDIR_DEFAULT + ) + set(CMAKE_INSTALL_PYTHONDIR "${CMAKE_INSTALL_PYTHONDIR_DEFAULT}" + CACHE PATH "Install directory for python modules" + ) + set(CMAKE_PYTHON_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR} + ) endif() - string_strip_prefix( - "${_Python_DEVPREFIX}" "${Python_SITEARCH}" CMAKE_INSTALL_PYTHONDIR_DEFAULT - ) - set(CMAKE_INSTALL_PYTHONDIR "${CMAKE_INSTALL_PYTHONDIR_DEFAULT}" - CACHE PATH "Install directory for python modules" - ) mark_as_advanced(CMAKE_INSTALL_PYTHONDIR) - set(CMAKE_PYTHON_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR} - ) endif() # Sst diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..3f02b358fc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,45 @@ +[build-system] +requires = ["scikit-build-core", "numpy"] +build-backend = "scikit_build_core.build" + +[project] +name = "adios2_scottwittenburg" +version = "2.9.2" +authors = [ + { name="Chuck Atkins", email="chuck.atkins@kitware.com" }, + { name="Greg S. Eisenhauer", email="eisen@cc.gatech.edu" }, + { name="William F. Godoy", email="godoywf@ornl.gov" }, + { name="Junmin Gu", email="jgu@lbl.gov" }, + { name="Norbert Podhorszki", email="pnorbert@ornl.gov" }, + { name="Ruonan (Jason) Wang", email="wangr1@ornl.gov" }, +] +description = "The Adaptable Input Output System version 2" +readme = "ReadMe.md" +requires-python = ">=3.8" +classifiers = [ + "Programming Language :: C++", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://github.com/ornladios/adios2" +Issues = "https://github.com/ornladios/adios2/issues" + +[tool.scikit-build] +wheel.packages = ["adios2"] +build-dir = "/data/scott/Documents/adios2/pip_build" + +[tool.scikit-build.cmake.define] +ADIOS2_USE_Python = "ON" +ADIOS2_USE_Fortran = "OFF" +ADIOS2_USE_MPI = "OFF" +ADIOS2_USE_HDF5 = "OFF" +ADIOS2_USE_HDF5_VOL = "OFF" +ADIOS2_USE_BZip2 = "OFF" +ADIOS2_USE_Blosc = "OFF" +ADIOS2_USE_DataMan = "OFF" +ADIOS2_USE_SZ = "OFF" +ADIOS2_USE_ZeroMQ = "OFF" +ADIOS2_USE_ZFP = "OFF" +ADIOS2_PIP_INSTALL = "ON" From d30e2dc92d52b8863f540d9935ab05ca66a389d4 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Fri, 1 Dec 2023 17:57:34 -0700 Subject: [PATCH 2/8] Fix install locations and rpaths for pip installs --- CMakeLists.txt | 41 +++- bindings/Python/CMakeLists.txt | 31 +-- cmake/DetectOptions.cmake | 38 ++-- pyproject.toml | 8 +- source/adios2/toolkit/remote/CMakeLists.txt | 26 +-- source/adios2/toolkit/sst/util/CMakeLists.txt | 36 ++-- source/utils/CMakeLists.txt | 187 +++++++++--------- source/utils/adios_iotest/CMakeLists.txt | 17 +- thirdparty/EVPath/CMakeLists.txt | 3 + 9 files changed, 216 insertions(+), 171 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bbf6895741..075f38e689 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,39 @@ setup_version(2.9.0) project(ADIOS2 VERSION ${ADIOS2_VERSION}) +# Advanced option to tell CMake the build is on behalf of pip +set(ADIOS2_PIP_INSTALL OFF CACHE BOOL "Turn on for pip install") +mark_as_advanced(ADIOS2_PIP_INSTALL) + +if (ADIOS2_PIP_INSTALL) + # Required for shared library placement. + if (WIN32) + # Defaults are fine; handled by `delvewheel`. + elseif (APPLE) + set(CMAKE_INSTALL_LIBDIR + # Store libraries in a subdirectory here. + "adios2/.dylibs") + else () + set(CMAKE_INSTALL_LIBDIR + # Linux bundles what libraries we have when they're put beside the modules. + "adios2") + endif () + + if (APPLE) + # macOS loader settings. + set(CMAKE_BUILD_WITH_INSTALL_NAME_DIR ON) + set(CMAKE_BUILD_WITH_INSTALL_RPATH ON) + set(CMAKE_INSTALL_NAME_DIR "@rpath") + list(APPEND CMAKE_INSTALL_RPATH + "@loader_path") + elseif (UNIX) + # ELF loader settings. + set(CMAKE_BUILD_WITH_INSTALL_RPATH ON) + list(APPEND CMAKE_INSTALL_RPATH + "$ORIGIN") + endif () +endif () + #------------------------------------------------------------------------------# # Some boilerplate to setup nice output directories #------------------------------------------------------------------------------# @@ -109,10 +142,6 @@ set(ADIOS2_EXECUTABLE_SUFFIX "${ADIOS2_LIBRARY_SUFFIX}" CACHE STRING "Suffix to append to executable names") mark_as_advanced(ADIOS2_LIBRARY_SUFFIX ADIOS2_EXECUTABLE_SUFFIX) -# Advanced option to tell CMake the build is on behalf of pip -set(ADIOS2_PIP_INSTALL OFF CACHE BOOL "Turn on for pip install") -mark_as_advanced(ADIOS2_PIP_INSTALL) - # Use meta-compile features if available, otherwise use specific language # features if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Intel|Clang|AppleClang|MSVC)$") @@ -287,12 +316,12 @@ if(BUILD_SHARED_LIBS AND ADIOS2_RUN_INSTALL_TEST) set(ADIOS2_RUN_INSTALL_TEST FALSE) endif() else() - set(CMAKE_INSTALL_RPATH "@loader_path/${relative_base}/${CMAKE_INSTALL_LIBDIR}") + list(APPEND CMAKE_INSTALL_RPATH "@loader_path/${relative_base}/${CMAKE_INSTALL_LIBDIR}") endif() elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") # Linux needs some specialized RPATH handling - set(CMAKE_INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}") + list(APPEND CMAKE_INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}") endif() endif() diff --git a/bindings/Python/CMakeLists.txt b/bindings/Python/CMakeLists.txt index 782cd509d3..9e2f530e5c 100644 --- a/bindings/Python/CMakeLists.txt +++ b/bindings/Python/CMakeLists.txt @@ -47,12 +47,11 @@ set_target_properties(adios2_py PROPERTIES COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 ) - if(CMAKE_SYSTEM_NAME MATCHES "Linux") if (ADIOS2_PIP_INSTALL) - set_target_properties(adios2_py PROPERTIES - INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}" - ) + set_target_properties(adios2_py PROPERTIES + INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" + ) else() string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2") set_target_properties(adios2_py PROPERTIES @@ -61,11 +60,21 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux") endif() endif() -install(TARGETS adios2_py - DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2 - COMPONENT adios2_python-python -) -install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py - DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2 - COMPONENT adios2_python-python +if (ADIOS2_PIP_INSTALL) + install(TARGETS adios2_py + DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT adios2_python-python) + install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py + DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT adios2_python-python) +else () + install(TARGETS adios2_py + DESTINATION ${CMAKE_INSTALL_PYTHONDIR} + COMPONENT adios2_python-python + ) + install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py + DESTINATION ${CMAKE_INSTALL_PYTHONDIR} + COMPONENT adios2_python-python ) +endif () + diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index fd63ede3d9..d335dbb4e1 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -398,7 +398,9 @@ if(NOT SHARED_LIBS_SUPPORTED) endif() endif() -if(ADIOS2_USE_Python STREQUAL AUTO) +if(ADIOS2_PIP_INSTALL) + find_package(Python 3 REQUIRED COMPONENTS Interpreter Development.Module NumPy) +elseif(ADIOS2_USE_Python STREQUAL AUTO) find_package(Python 3 COMPONENTS Interpreter Development NumPy) if(Python_FOUND AND ADIOS2_HAVE_MPI) find_package(PythonModule COMPONENTS mpi4py mpi4py/mpi4py.h) @@ -426,29 +428,21 @@ endif() if(Python_Interpreter_FOUND) # Setup output directories - if (ADIOS2_PIP_INSTALL) - # do it the pip way - set(CMAKE_INSTALL_PYTHONDIR ${Python_SITEARCH}) - set(CMAKE_PYTHON_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/${Python_SITEARCH} - ) + if(Python_Development_FOUND) + lists_get_prefix("Python_INCLUDE_DIRS;Python_LIBRARIES;Python_SITEARCH" _Python_DEVPREFIX) else() - if(Python_Development_FOUND) - lists_get_prefix("Python_INCLUDE_DIRS;Python_LIBRARIES;Python_SITEARCH" _Python_DEVPREFIX) - else() - lists_get_prefix("Python_EXECUTABLE;Python_SITEARCH" _Python_DEVPREFIX) - endif() - message("This is borked! _Python_DEVPREFIX = ${_Python_DEVPREFIX}, Python_SITEARCH = ${Python_SITEARCH}") - string_strip_prefix( - "${_Python_DEVPREFIX}" "${Python_SITEARCH}" CMAKE_INSTALL_PYTHONDIR_DEFAULT - ) - set(CMAKE_INSTALL_PYTHONDIR "${CMAKE_INSTALL_PYTHONDIR_DEFAULT}" - CACHE PATH "Install directory for python modules" - ) - set(CMAKE_PYTHON_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR} - ) + lists_get_prefix("Python_EXECUTABLE;Python_SITEARCH" _Python_DEVPREFIX) endif() + message("This is borked! _Python_DEVPREFIX = ${_Python_DEVPREFIX}, Python_SITEARCH = ${Python_SITEARCH}") + string_strip_prefix( + "${_Python_DEVPREFIX}" "${Python_SITEARCH}" CMAKE_INSTALL_PYTHONDIR_DEFAULT + ) + set(CMAKE_INSTALL_PYTHONDIR "${CMAKE_INSTALL_PYTHONDIR_DEFAULT}" + CACHE PATH "Install directory for python modules" + ) + set(CMAKE_PYTHON_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR} + ) mark_as_advanced(CMAKE_INSTALL_PYTHONDIR) endif() diff --git a/pyproject.toml b/pyproject.toml index 3f02b358fc..5d60e04485 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,9 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ] +dependencies = [ + "numpy", +] [project.urls] Homepage = "https://github.com/ornladios/adios2" @@ -28,7 +31,6 @@ Issues = "https://github.com/ornladios/adios2/issues" [tool.scikit-build] wheel.packages = ["adios2"] -build-dir = "/data/scott/Documents/adios2/pip_build" [tool.scikit-build.cmake.define] ADIOS2_USE_Python = "ON" @@ -37,9 +39,11 @@ ADIOS2_USE_MPI = "OFF" ADIOS2_USE_HDF5 = "OFF" ADIOS2_USE_HDF5_VOL = "OFF" ADIOS2_USE_BZip2 = "OFF" -ADIOS2_USE_Blosc = "OFF" +ADIOS2_USE_Blosc2 = "OFF" ADIOS2_USE_DataMan = "OFF" ADIOS2_USE_SZ = "OFF" ADIOS2_USE_ZeroMQ = "OFF" ADIOS2_USE_ZFP = "OFF" ADIOS2_PIP_INSTALL = "ON" +BUILD_TESTING = "OFF" +ADIOS2_INSTALL_GENERATE_CONFIG = "OFF" diff --git a/source/adios2/toolkit/remote/CMakeLists.txt b/source/adios2/toolkit/remote/CMakeLists.txt index b3b49d6159..9971799724 100644 --- a/source/adios2/toolkit/remote/CMakeLists.txt +++ b/source/adios2/toolkit/remote/CMakeLists.txt @@ -3,19 +3,21 @@ # accompanying file Copyright.txt for details. #------------------------------------------------------------------------------# -add_executable(remote_server ./remote_server.cpp remote_common.cpp) +if (NOT ADIOS2_PIP_INSTALL) + add_executable(remote_server ./remote_server.cpp remote_common.cpp) -target_link_libraries(remote_server PUBLIC EVPath::EVPath adios2_core adios2sys - PRIVATE $<$:shlwapi>) + target_link_libraries(remote_server PUBLIC EVPath::EVPath adios2_core adios2sys + PRIVATE $<$:shlwapi>) -get_property(pugixml_headers_path - TARGET pugixml - PROPERTY INTERFACE_INCLUDE_DIRECTORIES -) + get_property(pugixml_headers_path + TARGET pugixml + PROPERTY INTERFACE_INCLUDE_DIRECTORIES + ) -target_include_directories(remote_server PRIVATE ${PROJECT_BINARY_DIR} ${pugixml_headers_path}) + target_include_directories(remote_server PRIVATE ${PROJECT_BINARY_DIR} ${pugixml_headers_path}) -set_property(TARGET remote_server PROPERTY OUTPUT_NAME remote_server${ADIOS2_EXECUTABLE_SUFFIX}) -install(TARGETS remote_server EXPORT adios2 - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime -) + set_property(TARGET remote_server PROPERTY OUTPUT_NAME remote_server${ADIOS2_EXECUTABLE_SUFFIX}) + install(TARGETS remote_server EXPORT adios2 + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime + ) +endif () diff --git a/source/adios2/toolkit/sst/util/CMakeLists.txt b/source/adios2/toolkit/sst/util/CMakeLists.txt index 7f7ec99db9..8e2700fa95 100644 --- a/source/adios2/toolkit/sst/util/CMakeLists.txt +++ b/source/adios2/toolkit/sst/util/CMakeLists.txt @@ -3,25 +3,27 @@ # accompanying file Copyright.txt for details. #------------------------------------------------------------------------------# -if(ADIOS2_HAVE_MPI) - set(maybe_adios2_core_mpi adios2_core_mpi) -else() - set(maybe_adios2_core_mpi) -endif() +if (NOT ADIOS2_PIP_INSTALL) + if(ADIOS2_HAVE_MPI) + set(maybe_adios2_core_mpi adios2_core_mpi) + else() + set(maybe_adios2_core_mpi) + endif() -# stt_conn_tool -add_executable(sst_conn_tool sst_conn_tool.c sst_conn_tool.cxx) -target_link_libraries(sst_conn_tool ${maybe_adios2_core_mpi} adios2_core) -set_property(TARGET sst_conn_tool PROPERTY OUTPUT_NAME sst_conn_tool${ADIOS2_EXECUTABLE_SUFFIX}) + # stt_conn_tool + add_executable(sst_conn_tool sst_conn_tool.c sst_conn_tool.cxx) + target_link_libraries(sst_conn_tool ${maybe_adios2_core_mpi} adios2_core) + set_property(TARGET sst_conn_tool PROPERTY OUTPUT_NAME sst_conn_tool${ADIOS2_EXECUTABLE_SUFFIX}) -if(WIN32) - target_link_libraries(sst_conn_tool getopt) -endif() + if(WIN32) + target_link_libraries(sst_conn_tool getopt) + endif() -target_link_libraries(sst_conn_tool sst adios2::thirdparty::EVPath) + target_link_libraries(sst_conn_tool sst adios2::thirdparty::EVPath) -target_include_directories(sst_conn_tool PRIVATE .. ../cp ) + target_include_directories(sst_conn_tool PRIVATE .. ../cp ) -install(TARGETS sst_conn_tool EXPORT adios2 - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime -) + install(TARGETS sst_conn_tool EXPORT adios2 + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime + ) +endif () diff --git a/source/utils/CMakeLists.txt b/source/utils/CMakeLists.txt index 5d46d91c23..3ac709099c 100644 --- a/source/utils/CMakeLists.txt +++ b/source/utils/CMakeLists.txt @@ -3,115 +3,116 @@ # accompanying file Copyright.txt for details. #------------------------------------------------------------------------------# -configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/verinfo.h.in - ${PROJECT_BINARY_DIR}/verinfo.h - @ONLY -) +if (NOT ADIOS2_PIP_INSTALL) + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/verinfo.h.in + ${PROJECT_BINARY_DIR}/verinfo.h + @ONLY + ) -# BPLS -add_executable(bpls ./bpls/bpls.cpp) -target_link_libraries(bpls - PUBLIC adios2_core adios2sys - PRIVATE $<$:shlwapi>) + # BPLS + add_executable(bpls ./bpls/bpls.cpp) + target_link_libraries(bpls + PUBLIC adios2_core adios2sys + PRIVATE $<$:shlwapi>) -get_property(pugixml_headers_path - TARGET pugixml - PROPERTY INTERFACE_INCLUDE_DIRECTORIES -) + get_property(pugixml_headers_path + TARGET pugixml + PROPERTY INTERFACE_INCLUDE_DIRECTORIES + ) -target_include_directories(bpls PRIVATE - ${PROJECT_BINARY_DIR} - ${PROJECT_SOURCE_DIR}/bindings/C - ${pugixml_headers_path} -) + target_include_directories(bpls PRIVATE + ${PROJECT_BINARY_DIR} + ${PROJECT_SOURCE_DIR}/bindings/C + ${pugixml_headers_path} + ) -set_property(TARGET bpls PROPERTY OUTPUT_NAME bpls${ADIOS2_EXECUTABLE_SUFFIX}) -install(TARGETS bpls EXPORT adios2 - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime -) -configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/bpls/bpls.cmake.gen.in - ${PROJECT_BINARY_DIR}/bpls.cmake.gen - @ONLY -) -file(GENERATE - OUTPUT ${PROJECT_BINARY_DIR}/$/bpls.cmake - INPUT ${PROJECT_BINARY_DIR}/bpls.cmake.gen -) + set_property(TARGET bpls PROPERTY OUTPUT_NAME bpls${ADIOS2_EXECUTABLE_SUFFIX}) + install(TARGETS bpls EXPORT adios2 + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime + ) + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/bpls/bpls.cmake.gen.in + ${PROJECT_BINARY_DIR}/bpls.cmake.gen + @ONLY + ) + file(GENERATE + OUTPUT ${PROJECT_BINARY_DIR}/$/bpls.cmake + INPUT ${PROJECT_BINARY_DIR}/bpls.cmake.gen + ) + # BPSPLIT + #add_executable(bpsplit ./bpsplit/bpsplit.cpp) + #target_link_libraries(bpsplit adios2::cxx11 adios2sys_interface adios2::thirdparty::pugixml) + #install(TARGETS bpsplit EXPORT adios2 + # RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + #) -# BPSPLIT -#add_executable(bpsplit ./bpsplit/bpsplit.cpp) -#target_link_libraries(bpsplit adios2::cxx11 adios2sys_interface adios2::thirdparty::pugixml) -#install(TARGETS bpsplit EXPORT adios2 -# RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} -#) + # ADIOS_REORGANIZE + set(adios_reorganize_srcs + adios_reorganize/main.cpp + adios_reorganize/Reorganize.cpp + Utils.cpp + ) + if(ADIOS2_HAVE_MGARD) + set(maybe_mgard mgard::mgard) + else() + set(maybe_mgard) + endif() + add_executable(adios_reorganize ${adios_reorganize_srcs}) + target_link_libraries(adios_reorganize PRIVATE adios2_core ${maybe_mgard}) + set_property(TARGET adios_reorganize PROPERTY OUTPUT_NAME adios2_reorganize${ADIOS2_EXECUTABLE_SUFFIX}) -# ADIOS_REORGANIZE -set(adios_reorganize_srcs - adios_reorganize/main.cpp - adios_reorganize/Reorganize.cpp - Utils.cpp - ) -if(ADIOS2_HAVE_MGARD) - set(maybe_mgard mgard::mgard) -else() - set(maybe_mgard) -endif() -add_executable(adios_reorganize ${adios_reorganize_srcs}) -target_link_libraries(adios_reorganize PRIVATE adios2_core ${maybe_mgard}) -set_property(TARGET adios_reorganize PROPERTY OUTPUT_NAME adios2_reorganize${ADIOS2_EXECUTABLE_SUFFIX}) + if(ADIOS2_HAVE_MPI) + add_executable(adios_reorganize_mpi ${adios_reorganize_srcs}) + target_link_libraries(adios_reorganize_mpi PRIVATE adios2_core_mpi ${maybe_mgard}) + set_property(TARGET adios_reorganize_mpi PROPERTY OUTPUT_NAME adios2_reorganize_mpi${ADIOS2_EXECUTABLE_SUFFIX}) + set(maybe_adios_reorganize_mpi adios_reorganize_mpi) + else() + set(maybe_adios_reorganize_mpi) + endif() -if(ADIOS2_HAVE_MPI) - add_executable(adios_reorganize_mpi ${adios_reorganize_srcs}) - target_link_libraries(adios_reorganize_mpi PRIVATE adios2_core_mpi ${maybe_mgard}) - set_property(TARGET adios_reorganize_mpi PROPERTY OUTPUT_NAME adios2_reorganize_mpi${ADIOS2_EXECUTABLE_SUFFIX}) - set(maybe_adios_reorganize_mpi adios_reorganize_mpi) -else() - set(maybe_adios_reorganize_mpi) -endif() + install(TARGETS adios_reorganize + ${maybe_adios_reorganize_mpi} + EXPORT adios2 + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime + ) -install(TARGETS adios_reorganize - ${maybe_adios_reorganize_mpi} - EXPORT adios2 - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime -) + if(ADIOS2_HAVE_MPI) + add_subdirectory(adios_iotest) + endif() -if(ADIOS2_HAVE_MPI) - add_subdirectory(adios_iotest) -endif() + if(Python_Interpreter_FOUND) + add_subdirectory(bp4dbg) + add_subdirectory(bp5dbg) + install(PROGRAMS adios2_json_pp.py + RENAME adios2_json_pp + DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT adios2_scripts-runtime) + endif() -if(Python_Interpreter_FOUND) - add_subdirectory(bp4dbg) - add_subdirectory(bp5dbg) - install(PROGRAMS adios2_json_pp.py - RENAME adios2_json_pp + install(PROGRAMS adios2_deactivate_bp + RENAME adios2_deactivate_bp DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_scripts-runtime) -endif() -install(PROGRAMS adios2_deactivate_bp - RENAME adios2_deactivate_bp - DESTINATION ${CMAKE_INSTALL_BINDIR} - COMPONENT adios2_scripts-runtime) - -# Simplified wrappers for adios2_reorganize -configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/adios_reorganize/adios2_reorganize_wrapper - ${PROJECT_BINARY_DIR}/adios2_reorganize_wrapper - @ONLY -) - -if(ADIOS2_HAVE_HDF5) - install(PROGRAMS ${PROJECT_BINARY_DIR}/adios2_reorganize_wrapper - RENAME bp2h5${ADIOS2_EXECUTABLE_SUFFIX} - DESTINATION ${CMAKE_INSTALL_BINDIR} + # Simplified wrappers for adios2_reorganize + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/adios_reorganize/adios2_reorganize_wrapper + ${PROJECT_BINARY_DIR}/adios2_reorganize_wrapper + @ONLY ) - if(ADIOS2_HAVE_MPI) + + if(ADIOS2_HAVE_HDF5) install(PROGRAMS ${PROJECT_BINARY_DIR}/adios2_reorganize_wrapper - RENAME bp2h5_mpi${ADIOS2_EXECUTABLE_SUFFIX} + RENAME bp2h5${ADIOS2_EXECUTABLE_SUFFIX} DESTINATION ${CMAKE_INSTALL_BINDIR} ) + if(ADIOS2_HAVE_MPI) + install(PROGRAMS ${PROJECT_BINARY_DIR}/adios2_reorganize_wrapper + RENAME bp2h5_mpi${ADIOS2_EXECUTABLE_SUFFIX} + DESTINATION ${CMAKE_INSTALL_BINDIR} + ) + endif() endif() -endif() +endif () diff --git a/source/utils/adios_iotest/CMakeLists.txt b/source/utils/adios_iotest/CMakeLists.txt index fc490e63b4..d607dcd75d 100644 --- a/source/utils/adios_iotest/CMakeLists.txt +++ b/source/utils/adios_iotest/CMakeLists.txt @@ -29,11 +29,12 @@ if(ADIOS2_HAVE_HDF5 AND HDF5_IS_PARALLEL) endif() - -install(TARGETS adios_iotest EXPORT adios2 - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime -) - -install(DIRECTORY iotest-config/ - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/iotest-config COMPONENT adios2_tools-data -) +if (NOT ADIOS2_PIP_INSTALL) + install(TARGETS adios_iotest EXPORT adios2 + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime + ) + + install(DIRECTORY iotest-config/ + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/iotest-config COMPONENT adios2_tools-data + ) +endif() diff --git a/thirdparty/EVPath/CMakeLists.txt b/thirdparty/EVPath/CMakeLists.txt index 42de1109e6..0ea63102cf 100644 --- a/thirdparty/EVPath/CMakeLists.txt +++ b/thirdparty/EVPath/CMakeLists.txt @@ -17,6 +17,9 @@ set(EVPATH_LIBRARY_COMPONENT adios2_evpath-libraries) set(EVPATH_ARCHIVE_COMPONENT adios2_evpath-development) set(EVPATH_HEADER_COMPONENT adios2_evpath-development) +string(REGEX REPLACE "[^/]+" ".." relative_base "${EVPATH_INSTALL_MODULE_DIR}") +list(APPEND CMAKE_INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}") + add_subdirectory(EVPath) set(EVPath_DIR ${CMAKE_CURRENT_BINARY_DIR}/EVPath CACHE INTERNAL "") setup_libversion_dir(EVPath) From 9ffd7e4fe2f3ba0640ddb9e41226572497b52b06 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Mon, 4 Dec 2023 16:36:04 -0700 Subject: [PATCH 3/8] add packaging workflow, remove some cmake messages --- .github/workflows/pypackaging.yml | 65 +++++++++++++++++++++++++++++++ cmake/DetectOptions.cmake | 4 -- 2 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/pypackaging.yml diff --git a/.github/workflows/pypackaging.yml b/.github/workflows/pypackaging.yml new file mode 100644 index 0000000000..92d2f68d62 --- /dev/null +++ b/.github/workflows/pypackaging.yml @@ -0,0 +1,65 @@ +name: Python Packaging + +on: + workflow_dispatch: + pull_request: + # push: + # branches: + # - main + release: + types: + - published + +jobs: + make_sdist: + name: Make SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build SDist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz + + build_wheels: + name: Wheel on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: true + + - uses: pypa/cibuildwheel@v2.16 + env: + CIBW_BUILD: cp*-manylinux_x86_64 + + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + path: wheelhouse/*.whl + + upload_all: + needs: [build_wheels, make_sdist] + environment: pypi + permissions: + id-token: write + runs-on: ubuntu-latest + if: github.event_name == 'release' && github.event.action == 'published' + steps: + - uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - uses: pypa/gh-action-pypi-publish@release/v1 + diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index d335dbb4e1..0eda58e7d2 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -52,9 +52,7 @@ endfunction() # Extract the common prefix from a collection of variables function(lists_get_prefix listVars outVar) foreach(l IN LISTS listVars) - message(" l: ${l}") foreach(d IN LISTS ${l}) - message(" d: ${d}") if(NOT prefix) set(prefix "${d}") continue() @@ -66,7 +64,6 @@ function(lists_get_prefix listVars outVar) endif() endforeach() endforeach() - message("setting ${outVar} = ${prefix} in the PARENT_SCOPE") set(${outVar} "${prefix}" PARENT_SCOPE) endfunction() @@ -433,7 +430,6 @@ if(Python_Interpreter_FOUND) else() lists_get_prefix("Python_EXECUTABLE;Python_SITEARCH" _Python_DEVPREFIX) endif() - message("This is borked! _Python_DEVPREFIX = ${_Python_DEVPREFIX}, Python_SITEARCH = ${Python_SITEARCH}") string_strip_prefix( "${_Python_DEVPREFIX}" "${Python_SITEARCH}" CMAKE_INSTALL_PYTHONDIR_DEFAULT ) From a0a2f350894783427226c15fae949d0e90d8b0ce Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Mon, 4 Dec 2023 18:02:53 -0700 Subject: [PATCH 4/8] Try to detect python package version dynamically --- pyproject.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5d60e04485..ac9c2d2965 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [build-system] -requires = ["scikit-build-core", "numpy"] +requires = ["scikit-build-core", "numpy", "setuptools_scm>=8"] build-backend = "scikit_build_core.build" [project] name = "adios2_scottwittenburg" -version = "2.9.2" +dynamic = ["version"] authors = [ { name="Chuck Atkins", email="chuck.atkins@kitware.com" }, { name="Greg S. Eisenhauer", email="eisen@cc.gatech.edu" }, @@ -31,6 +31,10 @@ Issues = "https://github.com/ornladios/adios2/issues" [tool.scikit-build] wheel.packages = ["adios2"] +metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" + +[tool.setuptools_scm] # Section required +write_to = "_version.py" [tool.scikit-build.cmake.define] ADIOS2_USE_Python = "ON" From f748acb91ab5cd78f3325971ebdbf6839dc8fce3 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Thu, 7 Dec 2023 16:33:00 -0700 Subject: [PATCH 5/8] Update project metadata, and test publishing from gha --- .github/workflows/pypackaging.yml | 40 ++++++++++++++++++++++++------- pyproject.toml | 11 +++++++-- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pypackaging.yml b/.github/workflows/pypackaging.yml index 92d2f68d62..65498f61fa 100644 --- a/.github/workflows/pypackaging.yml +++ b/.github/workflows/pypackaging.yml @@ -3,9 +3,9 @@ name: Python Packaging on: workflow_dispatch: pull_request: - # push: - # branches: - # - main + push: + branches: + - master release: types: - published @@ -19,6 +19,9 @@ jobs: - name: Build SDist run: pipx run build --sdist + env: + # TODO: debug only, prevents upload failure due to "local" version + SETUPTOOLS_SCM_PRETEND_VERSION: "2.9.2.post431" - uses: actions/upload-artifact@v3 with: @@ -35,20 +38,19 @@ jobs: steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: true - uses: pypa/cibuildwheel@v2.16 env: CIBW_BUILD: cp*-manylinux_x86_64 + # TODO: debug only, prevents upload failure due to "local" version + SETUPTOOLS_SCM_PRETEND_VERSION: "2.9.2.post431" - name: Upload wheels uses: actions/upload-artifact@v3 with: path: wheelhouse/*.whl - upload_all: + upload_pypi: needs: [build_wheels, make_sdist] environment: pypi permissions: @@ -61,5 +63,27 @@ jobs: name: artifact path: dist - - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + upload_test_pypi: + needs: [build_wheels, make_sdist] + environment: pypi + permissions: + id-token: write + runs-on: ubuntu-latest + # Upload to Test PyPI for every commit on main branch + # if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' + # TODO: debug only, test publishing to testpypi on this PR + if: github.event_name == 'pull_request' && github.event.action == 'synchronize' && github.event.pull_request.head.ref == 'add-pip-packaging' + steps: + - uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - name: Publish package distributions to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ diff --git a/pyproject.toml b/pyproject.toml index ac9c2d2965..e77adaf33a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["scikit-build-core", "numpy", "setuptools_scm>=8"] build-backend = "scikit_build_core.build" [project] -name = "adios2_scottwittenburg" +name = "adios2" dynamic = ["version"] authors = [ { name="Chuck Atkins", email="chuck.atkins@kitware.com" }, @@ -27,7 +27,10 @@ dependencies = [ [project.urls] Homepage = "https://github.com/ornladios/adios2" -Issues = "https://github.com/ornladios/adios2/issues" +Documentation = "https://adios2.readthedocs.io/" +"Bug Tracker" = "https://github.com/ornladios/adios2/issues" +Discussions = "https://github.com/ornladios/ADIOS2/discussions" +Changelog = "https://github.com/ornladios/ADIOS2/releases" [tool.scikit-build] wheel.packages = ["adios2"] @@ -35,6 +38,10 @@ metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" [tool.setuptools_scm] # Section required write_to = "_version.py" +version_scheme = "post-release" + +[tool.cibuildwheel.linux] +environment-pass = ["SETUPTOOLS_SCM_PRETEND_VERSION"] [tool.scikit-build.cmake.define] ADIOS2_USE_Python = "ON" From 036226e1d37d44f97797b6962c6b73e6fd9f7f21 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Thu, 7 Dec 2023 17:04:12 -0700 Subject: [PATCH 6/8] try to get more info about trusted publish failure --- .github/workflows/pypackaging.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypackaging.yml b/.github/workflows/pypackaging.yml index 65498f61fa..c74e4d85c8 100644 --- a/.github/workflows/pypackaging.yml +++ b/.github/workflows/pypackaging.yml @@ -86,4 +86,5 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ + verbose: true From f213f2c4ac1c215f67e18cfe5a4f99ca644c7c7b Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Fri, 8 Dec 2023 09:11:34 -0700 Subject: [PATCH 7/8] Try different repository url key README mentions repository-url, but then I see this in my gha output: Run pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ verbose: true user: __token__ repository_url: https://upload.pypi.org/legacy/ packages_dir: dist verify_metadata: true skip_existing: false print_hash: false --- .github/workflows/pypackaging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypackaging.yml b/.github/workflows/pypackaging.yml index c74e4d85c8..4fa82016ca 100644 --- a/.github/workflows/pypackaging.yml +++ b/.github/workflows/pypackaging.yml @@ -85,6 +85,6 @@ jobs: - name: Publish package distributions to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: - repository-url: https://test.pypi.org/legacy/ + repository_url: https://test.pypi.org/legacy/ verbose: true From ff8fd815c3b1299992c76dd20f49347d27e9b4eb Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Fri, 8 Dec 2023 09:43:33 -0700 Subject: [PATCH 8/8] try a separate environment for testpypi --- .github/workflows/pypackaging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypackaging.yml b/.github/workflows/pypackaging.yml index 4fa82016ca..bac3f04bb6 100644 --- a/.github/workflows/pypackaging.yml +++ b/.github/workflows/pypackaging.yml @@ -68,7 +68,7 @@ jobs: upload_test_pypi: needs: [build_wheels, make_sdist] - environment: pypi + environment: testpypi permissions: id-token: write runs-on: ubuntu-latest