From dc39625389594be33eaa6a1ce7ccf82422f8ca07 Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Fri, 3 Nov 2023 11:21:10 -0500 Subject: [PATCH 01/18] Fix issue with HDF5_VOL_ALLOW_EXTERNAL CMake variable --- CMakeVOL.cmake | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/CMakeVOL.cmake b/CMakeVOL.cmake index 79afa59a2fb..d14ac5b2800 100644 --- a/CMakeVOL.cmake +++ b/CMakeVOL.cmake @@ -26,14 +26,10 @@ function (get_generated_cmake_targets out_var dir) set (${out_var} "${dir_targets}" PARENT_SCOPE) endfunction () -# For now, only support building of external VOL connectors with FetchContent -option (HDF5_VOL_ALLOW_EXTERNAL "Allow building of external HDF5 VOL connectors with FetchContent" "NO") +set (HDF5_VOL_ALLOW_EXTERNAL "NO" CACHE STRING "Allow building of external HDF5 VOL connectors with FetchContent") +set_property (CACHE HDF5_VOL_ALLOW_EXTERNAL PROPERTY STRINGS NO GIT LOCAL_DIR) mark_as_advanced (HDF5_VOL_ALLOW_EXTERNAL) -if (HDF5_VOL_ALLOW_EXTERNAL) - if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "NO" OR (NOT HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT" AND NOT HDF5_VOL_ALLOW_EXTERNAL MATCHES "LOCAL_DIR")) - message (FATAL_ERROR "HDF5_VOL_ALLOW_EXTERNAL must be set to 'GIT' or 'LOCAL_DIR' to allow building of external HDF5 VOL connectors") - endif() - +if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT" OR HDF5_VOL_ALLOW_EXTERNAL MATCHES "LOCAL_DIR") # For compatibility, set some variables that projects would # typically look for after calling find_package(HDF5) set (HDF5_FOUND 1) From be75361e233a59865a162935331bc77eb2ca0c53 Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Fri, 3 Nov 2023 11:21:55 -0500 Subject: [PATCH 02/18] Add initial API test workflow Add support for testing external passthrough VOL connector with API tests --- .github/workflows/vol.yml | 45 ++++++++++++++++ .github/workflows/vol_ext_passthru.yml | 75 ++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 .github/workflows/vol.yml create mode 100644 .github/workflows/vol_ext_passthru.yml diff --git a/.github/workflows/vol.yml b/.github/workflows/vol.yml new file mode 100644 index 00000000000..3f4769ff5f1 --- /dev/null +++ b/.github/workflows/vol.yml @@ -0,0 +1,45 @@ +name: hdf5 VOL connectors CI + +# Run VOL connector CI daily at 06:00 CDT (11:00 UTC) +on: + workflow_dispatch: + # Only for now + push: + # Only for now + # Not yet + #schedule: + # - cron: "0 11 * * *" + # Not yet + +permissions: + contents: read + +jobs: + # Build and test individual VOL connectors by using HDF5's + # CMake FetchContent functionality. + #fetchcontent_hdf5_vol_daos: + # uses: ./.github/workflows/vol_daos.yml + + #fetchcontent_hdf5_vol_rest: + # uses: ./.github/workflows/vol_rest.yml + + fetchcontent_hdf5_vol_ext_passthru: + uses: ./.github/workflows/vol_ext_passthru.yml + with: + build_mode: "Release" + + #fetchcontent_hdf5_vol_async: + # uses: ./.github/workflows/vol_async.yml + + #fetchcontent_hdf5_vol_cache: + # uses: ./.github/workflows/vol_cache.yml + + #fetchcontent_hdf5_vol_adios2: + # uses: ./.github/workflows/vol_adios2.yml + + #fetchcontent_hdf5_vol_log: + # uses: ./.github/workflows/vol_log.yml + + # Build and install HDF5 and test individual VOL connectors + # against the installed API test executables + diff --git a/.github/workflows/vol_ext_passthru.yml b/.github/workflows/vol_ext_passthru.yml new file mode 100644 index 00000000000..cb22d3bbf70 --- /dev/null +++ b/.github/workflows/vol_ext_passthru.yml @@ -0,0 +1,75 @@ +name: Test HDF5 external pass-through VOL + +on: + workflow_call: + inputs: + build_mode: + description: "CMake Build type" + required: true + type: string + +permissions: + contents: read + +jobs: + build_and_test: + name: Test HDF5 external passthrough VOL connector + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: | + sudo apt update + sudo apt-get install ninja-build automake autoconf libtool libtool-bin libopenmpi-dev + + # TODO + #- name: Checkout HDF5 + # uses: actions/checkout@v4 + # with: + # repository: HDFGroup/hdf5 + # path: hdf5 + # TODO + + # TODO + - name: Checkout HDF5 + uses: actions/checkout@v4 + with: + repository: jhendersonHDF/hdf5 + path: hdf5 + ref: api_tests_dev + # TODO + + - name: Checkout vol-external-passthrough + uses: actions/checkout@v4 + with: + repository: hpc-io/vol-external-passthrough + path: vol-external-passthrough + + - name: Configure HDF5 with external passthrough VOL connector + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build + cd ${{ github.workspace }}/hdf5/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_TEST_API:BOOL=ON \ + -DHDF5_ENABLE_PARALLEL:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ + -DHDF5_VOL_URL01:STRING="https://github.com/hpc-io/vol-external-passthrough.git" \ + -DHDF5_VOL_VOL-EXTERNAL-PASSTHROUGH_BRANCH:STRING="develop" \ + -DHDF5_VOL_VOL-EXTERNAL-PASSTHROUGH_NAME:STRING="pass_through_ext under_vol=0\;under_info={}\;" \ + -DHDF5_VOL_VOL-EXTERNAL-PASSTHROUGH_TEST_PARALLEL:BOOL=ON \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Build HDF5 and external passthrough VOL connector + shell: bash + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + + - name: Test HDF5 external passthrough VOL connector + working-directory: ${{ github.workspace }}/hdf5/build + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-external-passthrough" . From 1f942a98d3316523138c1f585b7850b90ebc7cce Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Fri, 3 Nov 2023 12:58:41 -0500 Subject: [PATCH 03/18] Add testing of Async VOL connector --- .github/workflows/vol.yml | 18 +++-- .github/workflows/vol_async.yml | 101 +++++++++++++++++++++++++ .github/workflows/vol_ext_passthru.yml | 4 +- 3 files changed, 113 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/vol_async.yml diff --git a/.github/workflows/vol.yml b/.github/workflows/vol.yml index 3f4769ff5f1..86df5114106 100644 --- a/.github/workflows/vol.yml +++ b/.github/workflows/vol.yml @@ -17,27 +17,29 @@ permissions: jobs: # Build and test individual VOL connectors by using HDF5's # CMake FetchContent functionality. - #fetchcontent_hdf5_vol_daos: + #hdf5_vol_daos_fetchcontent: # uses: ./.github/workflows/vol_daos.yml - #fetchcontent_hdf5_vol_rest: + #hdf5_vol_rest_fetchcontent: # uses: ./.github/workflows/vol_rest.yml - fetchcontent_hdf5_vol_ext_passthru: + hdf5_vol_ext_passthru_fetchcontent: uses: ./.github/workflows/vol_ext_passthru.yml with: build_mode: "Release" - #fetchcontent_hdf5_vol_async: - # uses: ./.github/workflows/vol_async.yml + hdf5_vol_async_fetchcontent: + uses: ./.github/workflows/vol_async.yml + with: + build_mode: "Release" - #fetchcontent_hdf5_vol_cache: + #hdf5_vol_cache_fetchcontent: # uses: ./.github/workflows/vol_cache.yml - #fetchcontent_hdf5_vol_adios2: + #hdf5_vol_adios2_fetchcontent: # uses: ./.github/workflows/vol_adios2.yml - #fetchcontent_hdf5_vol_log: + #hdf5_vol_log_fetchcontent: # uses: ./.github/workflows/vol_log.yml # Build and install HDF5 and test individual VOL connectors diff --git a/.github/workflows/vol_async.yml b/.github/workflows/vol_async.yml new file mode 100644 index 00000000000..b0a6e4ed7da --- /dev/null +++ b/.github/workflows/vol_async.yml @@ -0,0 +1,101 @@ +name: Test HDF5 async VOL + +on: + workflow_call: + inputs: + build_mode: + description: "CMake Build type" + required: true + type: string + +permissions: + contents: read + +jobs: + build_and_test: + name: Test HDF5 asynchronous I/O VOL connector + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: | + sudo apt update + sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev + + # TODO + #- name: Checkout HDF5 + # uses: actions/checkout@v4 + # with: + # repository: HDFGroup/hdf5 + # path: hdf5 + # TODO + + # TODO + - name: Checkout HDF5 + uses: actions/checkout@v4 + with: + repository: jhendersonHDF/hdf5 + path: hdf5 + ref: api_tests_dev + # TODO + + - name: Checkout vol-async + uses: actions/checkout@v4 + with: + repository: hpc-io/vol-async + path: vol-async + + - name: Checkout Argobots + uses: actions/checkout@v3 + with: + repository: pmodels/argobots + path: abt + + # Argobots builds and installs fairly quickly, + # so no caching is currently performed here + - name: Install Argobots + working-directory: ${{ github.workspace }}/abt + run: | + ./autogen.sh + ./configure --prefix=/usr/local + make -j2 + sudo make -j2 install + + - name: Configure HDF5 with asynchronous I/O VOL connector + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build + cd ${{ github.workspace }}/hdf5/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_TEST_API:BOOL=ON \ + -DHDF5_ENABLE_PARALLEL:BOOL=ON \ + -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ + -DALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ + -DHDF5_VOL_URL01:STRING="https://github.com/hpc-io/vol-async.git" \ + -DHDF5_VOL_VOL-ASYNC_BRANCH:STRING="develop" \ + -DHDF5_VOL_VOL-ASYNC_NAME:STRING="async under_vol=0\;under_info={}" \ + -DHDF5_VOL_VOL-ASYNC_TEST_PARALLEL:BOOL=ON \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Build HDF5 and asynchronous I/O VOL connector + shell: bash + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + echo "LD_LIBRARY_PATH=/usr/local/lib:${{ github.workspace }}/hdf5/build/bin" >> $GITHUB_ENV + + - name: Test HDF5 asynchronous I/O VOL connector with external tests + working-directory: ${{ github.workspace }}/hdf5/build + run: | + # Workaround for vol-async CMake issue + cp bin/async_test* ./_deps/hdf5_vol_vol-async-build/test + ctest --build-config ${{ inputs.build_mode }} -VV -R "async_test" . + + - name: Test HDF5 asynchronous I/O VOL connector with HDF5 API tests + working-directory: ${{ github.workspace }}/hdf5/build + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-async" . diff --git a/.github/workflows/vol_ext_passthru.yml b/.github/workflows/vol_ext_passthru.yml index cb22d3bbf70..caafeb32715 100644 --- a/.github/workflows/vol_ext_passthru.yml +++ b/.github/workflows/vol_ext_passthru.yml @@ -19,7 +19,7 @@ jobs: - name: Install dependencies run: | sudo apt update - sudo apt-get install ninja-build automake autoconf libtool libtool-bin libopenmpi-dev + sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev # TODO #- name: Checkout HDF5 @@ -69,7 +69,7 @@ jobs: run: | cmake --build . --parallel 3 --config ${{ inputs.build_mode }} - - name: Test HDF5 external passthrough VOL connector + - name: Test HDF5 external passthrough VOL connector with HDF5 API tests working-directory: ${{ github.workspace }}/hdf5/build run: | ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-external-passthrough" . From 8bf1f31f76260f5ff6b3ca3392679ce3ea2a23af Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Mon, 6 Nov 2023 15:01:31 -0600 Subject: [PATCH 04/18] Initialize parallel testing with MPI_THREAD_MULTIPLE when testing API --- test/API/CMakeLists.txt | 2 +- test/CMakeLists.txt | 78 ++++++++++++-------------------------- testpar/API/CMakeLists.txt | 2 +- testpar/CMakeLists.txt | 19 ++++++---- testpar/t_bigio.c | 39 +++++++++++++++++-- testpar/t_pshutdown.c | 39 +++++++++++++++++-- testpar/t_shapesame.c | 39 +++++++++++++++++-- testpar/testphdf5.c | 39 +++++++++++++++++-- 8 files changed, 181 insertions(+), 76 deletions(-) diff --git a/test/API/CMakeLists.txt b/test/API/CMakeLists.txt index 6f6af47c305..9495acd43f8 100644 --- a/test/API/CMakeLists.txt +++ b/test/API/CMakeLists.txt @@ -141,7 +141,7 @@ target_compile_options ( target_compile_definitions ( h5_api_test PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" + "${HDF5_TEST_COMPILE_DEFS_PRIVATE}" ) # Always prefer linking the shared HDF5 library by default if (BUILD_SHARED_LIBS) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 246f1df73e1..fa73a0f7fc2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -27,6 +27,15 @@ set (TEST_LIB_HEADERS ${HDF5_TEST_SOURCE_DIR}/swmr_common.h ) +################################################################################# +# Set private compile-time definitions added when +# compiling test source files +################################################################################# +set (HDF5_TEST_COMPILE_DEFS_PRIVATE + "$<$:${HDF5_DEVELOPER_DEFS}>" + "$<$:H5_HAVE_TEST_API>" +) + if (BUILD_STATIC_LIBS) add_library (${HDF5_TEST_LIB_TARGET} STATIC ${TEST_LIB_SOURCES} ${TEST_LIB_HEADERS}) target_include_directories (${HDF5_TEST_LIB_TARGET} @@ -37,7 +46,7 @@ if (BUILD_STATIC_LIBS) target_compile_definitions(${HDF5_TEST_LIB_TARGET} PRIVATE "H5_TEST_EXPRESS_LEVEL_DEFAULT=${H5_TEST_EXPRESS_LEVEL_DEFAULT}" - "$<$:${HDF5_DEVELOPER_DEFS}>" + "${HDF5_TEST_COMPILE_DEFS_PRIVATE}" ) TARGET_C_PROPERTIES (${HDF5_TEST_LIB_TARGET} STATIC) target_link_libraries (${HDF5_TEST_LIB_TARGET} @@ -79,7 +88,7 @@ if (BUILD_SHARED_LIBS) "H5_BUILT_AS_DYNAMIC_LIB" PRIVATE "H5_TEST_EXPRESS_LEVEL_DEFAULT=${H5_TEST_EXPRESS_LEVEL_DEFAULT}" - "$<$:${HDF5_DEVELOPER_DEFS}>" + "${HDF5_TEST_COMPILE_DEFS_PRIVATE}" ) TARGET_C_PROPERTIES (${HDF5_TEST_LIBSH_TARGET} SHARED) target_link_libraries (${HDF5_TEST_LIBSH_TARGET} @@ -431,10 +440,7 @@ macro (ADD_H5_EXE file) add_executable (${file} ${HDF5_TEST_SOURCE_DIR}/${file}.c) target_include_directories (${file} PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};${HDF5_TEST_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(${file} PRIVATE "${HDF5_CMAKE_C_FLAGS}") - target_compile_definitions(${file} - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" - ) + target_compile_definitions(${file} PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (${file} STATIC) target_link_libraries (${file} PRIVATE ${HDF5_TEST_LIB_TARGET}) @@ -475,10 +481,7 @@ endforeach () #-- Adding test for chunk_info add_executable (chunk_info ${HDF5_TEST_SOURCE_DIR}/chunk_info.c) target_compile_options(chunk_info PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_compile_definitions(chunk_info - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" -) +target_compile_definitions(chunk_info PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") target_include_directories (chunk_info PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};${HDF5_TEST_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (chunk_info STATIC) @@ -499,10 +502,7 @@ endif () #-- Adding test for direct_chunk add_executable (direct_chunk ${HDF5_TEST_SOURCE_DIR}/direct_chunk.c) target_compile_options(direct_chunk PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_compile_definitions(direct_chunk - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" -) +target_compile_definitions(direct_chunk PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") target_include_directories (direct_chunk PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};${HDF5_TEST_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (direct_chunk STATIC) @@ -524,10 +524,7 @@ endif () #-- Adding test for testhdf5 add_executable (testhdf5 ${testhdf5_SOURCES}) target_compile_options(testhdf5 PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_compile_definitions(testhdf5 - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" -) +target_compile_definitions(testhdf5 PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") target_include_directories (testhdf5 PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (testhdf5 STATIC) @@ -548,10 +545,7 @@ endif () #-- Adding test for cache_image add_executable (cache_image ${cache_image_SOURCES}) target_compile_options(cache_image PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_compile_definitions(cache_image - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" -) +target_compile_definitions(cache_image PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") target_include_directories (cache_image PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (cache_image STATIC) @@ -572,10 +566,7 @@ endif () #-- Adding test for ttsafe add_executable (ttsafe ${ttsafe_SOURCES}) target_compile_options(ttsafe PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_compile_definitions(ttsafe - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" -) +target_compile_definitions(ttsafe PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") target_include_directories (ttsafe PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (ttsafe STATIC) @@ -602,10 +593,7 @@ endif () #-- Adding test for thread_id add_executable (thread_id ${HDF5_TEST_SOURCE_DIR}/thread_id.c) target_compile_options(thread_id PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_compile_definitions(thread_id - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" -) +target_compile_definitions(thread_id PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") target_include_directories (thread_id PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (thread_id STATIC) @@ -712,10 +700,7 @@ macro (ADD_H5_VDS_EXE file) add_executable (${file} ${HDF5_TEST_SOURCE_DIR}/${file}.c ${HDF5_TEST_SOURCE_DIR}/vds_swmr.h) target_include_directories (${file} PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_BINARY_DIR};${HDF5_TEST_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(${file} PRIVATE "${HDF5_CMAKE_C_FLAGS}") - target_compile_definitions(${file} - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" - ) + target_compile_definitions(${file} PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (${file} STATIC) target_link_libraries (${file} PRIVATE ${HDF5_TEST_LIB_TARGET}) @@ -742,10 +727,7 @@ endforeach () # and it can't be renamed (i.e., no -shared). add_executable (accum_swmr_reader ${HDF5_TEST_SOURCE_DIR}/accum_swmr_reader.c) target_compile_options(accum_swmr_reader PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_compile_definitions(accum_swmr_reader - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" -) +target_compile_definitions(accum_swmr_reader PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") target_include_directories (accum_swmr_reader PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (accum_swmr_reader STATIC) @@ -816,10 +798,7 @@ endif () set (use_append_chunk_SOURCES ${HDF5_TEST_SOURCE_DIR}/use_append_chunk.c ${HDF5_TEST_SOURCE_DIR}/use_common.c ${HDF5_TEST_SOURCE_DIR}/use.h) add_executable (use_append_chunk ${use_append_chunk_SOURCES}) target_compile_options(use_append_chunk PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_compile_definitions(use_append_chunk - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" -) +target_compile_definitions(use_append_chunk PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") target_include_directories (use_append_chunk PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (use_append_chunk STATIC) @@ -841,10 +820,7 @@ if (HDF5_BUILD_UTILS) # requires mirror server set (use_append_chunk_mirror_SOURCES ${HDF5_TEST_SOURCE_DIR}/use_append_chunk_mirror.c ${HDF5_TEST_SOURCE_DIR}/use_common.c ${HDF5_TEST_SOURCE_DIR}/use.h) add_executable (use_append_chunk_mirror ${use_append_chunk_mirror_SOURCES}) target_compile_options(use_append_chunk_mirror PRIVATE "${HDF5_CMAKE_C_FLAGS}") - target_compile_definitions(use_append_chunk_mirror - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" - ) + target_compile_definitions(use_append_chunk_mirror PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") target_include_directories (use_append_chunk_mirror PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (use_append_chunk_mirror STATIC) @@ -866,10 +842,7 @@ endif () set (use_append_mchunks_SOURCES ${HDF5_TEST_SOURCE_DIR}/use_append_mchunks.c ${HDF5_TEST_SOURCE_DIR}/use_common.c ${HDF5_TEST_SOURCE_DIR}/use.h) add_executable (use_append_mchunks ${use_append_mchunks_SOURCES}) target_compile_options(use_append_mchunks PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_compile_definitions(use_append_mchunks - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" -) +target_compile_definitions(use_append_mchunks PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") target_include_directories (use_append_mchunks PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (use_append_mchunks STATIC) @@ -890,10 +863,7 @@ endif () set (use_disable_mdc_flushes_SOURCES ${HDF5_TEST_SOURCE_DIR}/use_disable_mdc_flushes.c) add_executable (use_disable_mdc_flushes ${use_disable_mdc_flushes_SOURCES}) target_compile_options(use_disable_mdc_flushes PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_compile_definitions(use_disable_mdc_flushes - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" -) +target_compile_definitions(use_disable_mdc_flushes PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") target_include_directories (use_disable_mdc_flushes PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (use_disable_mdc_flushes STATIC) diff --git a/testpar/API/CMakeLists.txt b/testpar/API/CMakeLists.txt index 818bee665d5..e5cb5779b31 100644 --- a/testpar/API/CMakeLists.txt +++ b/testpar/API/CMakeLists.txt @@ -99,7 +99,7 @@ target_compile_options ( target_compile_definitions ( h5_api_test_parallel PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" + "${HDF5_TESTPAR_COMPILE_DEFS_PRIVATE}" ) # Always prefer linking the shared HDF5 library by default if (BUILD_SHARED_LIBS) diff --git a/testpar/CMakeLists.txt b/testpar/CMakeLists.txt index 7894cffa1da..106f79eba1e 100644 --- a/testpar/CMakeLists.txt +++ b/testpar/CMakeLists.txt @@ -21,13 +21,19 @@ set (testphdf5_SOURCES ${HDF5_TEST_PAR_SOURCE_DIR}/t_oflush.c ) +################################################################################# +# Set private compile-time definitions added when +# compiling test source files +################################################################################# +set (HDF5_TESTPAR_COMPILE_DEFS_PRIVATE + "$<$:${HDF5_DEVELOPER_DEFS}>" + "$<$:H5_HAVE_TEST_API>" +) + #-- Adding test for testhdf5 add_executable (testphdf5 ${testphdf5_SOURCES}) target_compile_options(testphdf5 PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_compile_definitions(testphdf5 - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" -) +target_compile_definitions(testphdf5 PRIVATE "${HDF5_TESTPAR_COMPILE_DEFS_PRIVATE}") target_include_directories (testphdf5 PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>" ) @@ -54,10 +60,7 @@ endif () macro (ADD_H5P_EXE file) add_executable (${file} ${HDF5_TEST_PAR_SOURCE_DIR}/${file}.c) target_compile_options(${file} PRIVATE "${HDF5_CMAKE_C_FLAGS}") - target_compile_definitions(${file} - PRIVATE - "$<$:${HDF5_DEVELOPER_DEFS}>" - ) + target_compile_definitions(${file} PRIVATE "${HDF5_TESTPAR_COMPILE_DEFS_PRIVATE}") target_include_directories (${file} PRIVATE "${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>" ) diff --git a/testpar/t_bigio.c b/testpar/t_bigio.c index 910c7a2612b..068e3aa0f50 100644 --- a/testpar/t_bigio.c +++ b/testpar/t_bigio.c @@ -1867,6 +1867,11 @@ main(int argc, char **argv) /* Set the bigio processing limit to be 'newsize' bytes */ hsize_t oldsize = H5_mpi_set_bigio_count(newsize); hid_t acc_plist = H5I_INVALID_HID; +#ifdef H5_HAVE_TEST_API + int required = MPI_THREAD_MULTIPLE; + int provided; +#endif + int mpi_code; /* Having set the bigio handling to a size that is manageable, * we'll set our 'bigcount' variable to be 2X that limit so @@ -1876,9 +1881,37 @@ main(int argc, char **argv) if (newsize != oldsize) bigcount = newsize * 2; - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &mpi_size_g); - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank_g); +#ifdef H5_HAVE_TEST_API + /* Attempt to initialize with MPI_THREAD_MULTIPLE if possible */ + if (MPI_SUCCESS != (mpi_code = MPI_Init_thread(&argc, &argv, required, &provided))) { + printf("MPI_Init_thread failed with error code %d\n", mpi_code); + return -1; + } +#else + if (MPI_SUCCESS != (mpi_code = MPI_Init(&argc, &argv))) { + printf("MPI_Init failed with error code %d\n", mpi_code); + return -1; + } +#endif + + if (MPI_SUCCESS != (mpi_code = MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank_g))) { + printf("MPI_Comm_rank failed with error code %d\n", mpi_code); + MPI_Finalize(); + return -1; + } + +#ifdef H5_HAVE_TEST_API + /* Warn about missing MPI_THREAD_MULTIPLE support */ + if ((provided < required) && MAIN_PROCESS) + printf("** MPI doesn't support MPI_Init_thread with MPI_THREAD_MULTIPLE **\n"); +#endif + + if (MPI_SUCCESS != (mpi_code = MPI_Comm_size(MPI_COMM_WORLD, &mpi_size_g))) { + if (MAIN_PROCESS) + printf("MPI_Comm_size failed with error code %d\n", mpi_code); + MPI_Finalize(); + return -1; + } /* Attempt to turn off atexit post processing so that in case errors * happen during the test and the process is aborted, it will not get diff --git a/testpar/t_pshutdown.c b/testpar/t_pshutdown.c index 47c78d08d62..92f0bf17429 100644 --- a/testpar/t_pshutdown.c +++ b/testpar/t_pshutdown.c @@ -41,10 +41,43 @@ main(int argc, char **argv) hsize_t stride[RANK]; hsize_t block[RANK]; DATATYPE *data_array = NULL; /* data buffer */ + int mpi_code; +#ifdef H5_HAVE_TEST_API + int required = MPI_THREAD_MULTIPLE; + int provided; +#endif + +#ifdef H5_HAVE_TEST_API + /* Attempt to initialize with MPI_THREAD_MULTIPLE if possible */ + if (MPI_SUCCESS != (mpi_code = MPI_Init_thread(&argc, &argv, required, &provided))) { + printf("MPI_Init_thread failed with error code %d\n", mpi_code); + return -1; + } +#else + if (MPI_SUCCESS != (mpi_code = MPI_Init(&argc, &argv))) { + printf("MPI_Init failed with error code %d\n", mpi_code); + return -1; + } +#endif + + if (MPI_SUCCESS != (mpi_code = MPI_Comm_rank(comm, &mpi_rank))) { + printf("MPI_Comm_rank failed with error code %d\n", mpi_code); + MPI_Finalize(); + return -1; + } + +#ifdef H5_HAVE_TEST_API + /* Warn about missing MPI_THREAD_MULTIPLE support */ + if ((provided < required) && MAINPROCESS) + printf("** MPI doesn't support MPI_Init_thread with MPI_THREAD_MULTIPLE **\n"); +#endif - MPI_Init(&argc, &argv); - MPI_Comm_size(comm, &mpi_size); - MPI_Comm_rank(comm, &mpi_rank); + if (MPI_SUCCESS != (mpi_code = MPI_Comm_size(comm, &mpi_size))) { + if (MAINPROCESS) + printf("MPI_Comm_size failed with error code %d\n", mpi_code); + MPI_Finalize(); + return -1; + } if (MAINPROCESS) TESTING("proper shutdown of HDF5 library"); diff --git a/testpar/t_shapesame.c b/testpar/t_shapesame.c index 4f48f931c93..2a2cfe8fd8b 100644 --- a/testpar/t_shapesame.c +++ b/testpar/t_shapesame.c @@ -4271,6 +4271,11 @@ int main(int argc, char **argv) { int mpi_size, mpi_rank; /* mpi variables */ + int mpi_code; +#ifdef H5_HAVE_TEST_API + int required = MPI_THREAD_MULTIPLE; + int provided; +#endif #ifndef H5_HAVE_WIN32_API /* Un-buffer the stdout and stderr */ @@ -4278,9 +4283,37 @@ main(int argc, char **argv) HDsetbuf(stdout, NULL); #endif - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); +#ifdef H5_HAVE_TEST_API + /* Attempt to initialize with MPI_THREAD_MULTIPLE if possible */ + if (MPI_SUCCESS != (mpi_code = MPI_Init_thread(&argc, &argv, required, &provided))) { + printf("MPI_Init_thread failed with error code %d\n", mpi_code); + return -1; + } +#else + if (MPI_SUCCESS != (mpi_code = MPI_Init(&argc, &argv))) { + printf("MPI_Init failed with error code %d\n", mpi_code); + return -1; + } +#endif + + if (MPI_SUCCESS != (mpi_code = MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank))) { + printf("MPI_Comm_rank failed with error code %d\n", mpi_code); + MPI_Finalize(); + return -1; + } + +#ifdef H5_HAVE_TEST_API + /* Warn about missing MPI_THREAD_MULTIPLE support */ + if ((provided < required) && MAINPROCESS) + printf("** MPI doesn't support MPI_Init_thread with MPI_THREAD_MULTIPLE **\n"); +#endif + + if (MPI_SUCCESS != (mpi_code = MPI_Comm_size(MPI_COMM_WORLD, &mpi_size))) { + if (MAINPROCESS) + printf("MPI_Comm_size failed with error code %d\n", mpi_code); + MPI_Finalize(); + return -1; + } mpi_rank_framework_g = mpi_rank; diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c index 57ef5c9bbd4..0b0a67c0a65 100644 --- a/testpar/testphdf5.c +++ b/testpar/testphdf5.c @@ -301,10 +301,15 @@ int main(int argc, char **argv) { int mpi_size, mpi_rank; /* mpi variables */ + int mpi_code; H5Ptest_param_t ndsets_params, ngroups_params; H5Ptest_param_t collngroups_params; H5Ptest_param_t io_mode_confusion_params; H5Ptest_param_t rr_obj_flush_confusion_params; +#ifdef H5_HAVE_TEST_API + int required = MPI_THREAD_MULTIPLE; + int provided; +#endif #ifndef H5_HAVE_WIN32_API /* Un-buffer the stdout and stderr */ @@ -312,9 +317,37 @@ main(int argc, char **argv) HDsetbuf(stdout, NULL); #endif - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); +#ifdef H5_HAVE_TEST_API + /* Attempt to initialize with MPI_THREAD_MULTIPLE if possible */ + if (MPI_SUCCESS != (mpi_code = MPI_Init_thread(&argc, &argv, required, &provided))) { + printf("MPI_Init_thread failed with error code %d\n", mpi_code); + return -1; + } +#else + if (MPI_SUCCESS != (mpi_code = MPI_Init(&argc, &argv))) { + printf("MPI_Init failed with error code %d\n", mpi_code); + return -1; + } +#endif + + if (MPI_SUCCESS != (mpi_code = MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank))) { + printf("MPI_Comm_rank failed with error code %d\n", mpi_code); + MPI_Finalize(); + return -1; + } + +#ifdef H5_HAVE_TEST_API + /* Warn about missing MPI_THREAD_MULTIPLE support */ + if ((provided < required) && MAINPROCESS) + printf("** MPI doesn't support MPI_Init_thread with MPI_THREAD_MULTIPLE **\n"); +#endif + + if (MPI_SUCCESS != (mpi_code = MPI_Comm_size(MPI_COMM_WORLD, &mpi_size))) { + if (MAINPROCESS) + printf("MPI_Comm_size failed with error code %d\n", mpi_code); + MPI_Finalize(); + return -1; + } mpi_rank_framework_g = mpi_rank; From f37242ff306eed9452e800798ba64029af0cad23 Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Mon, 6 Nov 2023 15:45:17 -0600 Subject: [PATCH 05/18] Add testing of Cache VOL connector --- .github/workflows/vol.yml | 6 +- .github/workflows/vol_async.yml | 3 +- .github/workflows/vol_cache.yml | 181 ++++++++++++++++++++++++++++++++ CMakeVOL.cmake | 11 ++ 4 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/vol_cache.yml diff --git a/.github/workflows/vol.yml b/.github/workflows/vol.yml index 86df5114106..a69bfa3f9f6 100644 --- a/.github/workflows/vol.yml +++ b/.github/workflows/vol.yml @@ -33,8 +33,10 @@ jobs: with: build_mode: "Release" - #hdf5_vol_cache_fetchcontent: - # uses: ./.github/workflows/vol_cache.yml + hdf5_vol_cache_fetchcontent: + uses: ./.github/workflows/vol_cache.yml + with: + build_mode: "Release" #hdf5_vol_adios2_fetchcontent: # uses: ./.github/workflows/vol_adios2.yml diff --git a/.github/workflows/vol_async.yml b/.github/workflows/vol_async.yml index b0a6e4ed7da..eb30a555d12 100644 --- a/.github/workflows/vol_async.yml +++ b/.github/workflows/vol_async.yml @@ -45,7 +45,7 @@ jobs: path: vol-async - name: Checkout Argobots - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: pmodels/argobots path: abt @@ -69,6 +69,7 @@ jobs: -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ -DBUILD_STATIC_LIBS=OFF \ -DHDF5_TEST_API:BOOL=ON \ + -DHDF5_TEST_API_ENABLE_ASYNC:BOOL=ON \ -DHDF5_ENABLE_PARALLEL:BOOL=ON \ -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ -DALLOW_UNSUPPORTED:BOOL=ON \ diff --git a/.github/workflows/vol_cache.yml b/.github/workflows/vol_cache.yml new file mode 100644 index 00000000000..6835d4f88a1 --- /dev/null +++ b/.github/workflows/vol_cache.yml @@ -0,0 +1,181 @@ +name: Test HDF5 cache VOL + +on: + workflow_call: + inputs: + build_mode: + description: "CMake Build type" + required: true + type: string + +permissions: + contents: read + +jobs: + build_and_test: + name: Test HDF5 cache VOL connector + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: | + sudo apt update + sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev + + # TODO + #- name: Checkout HDF5 + # uses: actions/checkout@v4 + # with: + # repository: HDFGroup/hdf5 + # path: hdf5 + # TODO + + # TODO + - name: Checkout HDF5 + uses: actions/checkout@v4 + with: + repository: jhendersonHDF/hdf5 + path: hdf5 + ref: api_tests_dev + # TODO + + - name: Checkout vol-cache + uses: actions/checkout@v4 + with: + repository: hpc-io/vol-cache + path: vol-cache + + - name: Checkout vol-async + uses: actions/checkout@v4 + with: + repository: hpc-io/vol-async + path: vol-async + + - name: Checkout Argobots + uses: actions/checkout@v4 + with: + repository: pmodels/argobots + path: abt + + # Argobots builds and installs fairly quickly, + # so no caching is currently performed here + - name: Install Argobots + working-directory: ${{ github.workspace }}/abt + run: | + ./autogen.sh + ./configure --prefix=/usr/local + make -j2 + sudo make -j2 install + + # TODO: It should be possible to build both the async VOL + # and cache VOL at the same time, but the cache VOL looks + # for the async VOL by a different name than 'vol-async' + # and there are dependency issues where the cache VOL tries + # to link to a built async VOL. CMake's FetchContent + # OVERRIDE_FIND_PACKAGE may come in handy here, but our + # current CMake code declares VOL connector targets with + # a prefixed name (HDF5_VOL_vol_name_lower), so this also + # doesn't work yet. For now, just install the async VOL + # manually. +# - name: Install vol-async +# working-directory: ${{ github.workspace }}/vol-async +# run: | +# mkdir build +# cd build +# cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. +# sudo make -j2 install + + - name: Configure HDF5 with cache VOL connector + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build + cd ${{ github.workspace }}/hdf5/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_TEST_API:BOOL=ON \ + -DHDF5_TEST_API_ENABLE_ASYNC:BOOL=ON \ + -DHDF5_ENABLE_PARALLEL:BOOL=ON \ + -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ + -DALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ + -DHDF5_VOL_URL01:STRING="https://github.com/hpc-io/vol-async.git" \ + -DHDF5_VOL_VOL-ASYNC_BRANCH:STRING="develop" \ + -DHDF5_VOL_VOL-ASYNC_NAME:STRING="async under_vol=0\;under_info={}" \ + -DHDF5_VOL_VOL-ASYNC_TEST_PARALLEL:BOOL=ON \ + -DHDF5_VOL_URL02:STRING="https://github.com/hpc-io/vol-cache.git" \ + -DHDF5_VOL_VOL-CACHE_BRANCH:STRING="develop" \ + -DHDF5_VOL_VOL-CACHE_NAME:STRING="cache_ext config=$GITHUB_WORKSPACE/config1.cfg\;under_vol=0\;under_info={}\;" \ + -DHDF5_VOL_VOL-CACHE_TEST_PARALLEL:BOOL=ON \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Configure HDF5 with cache VOL connector stacked atop async VOL connector + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build2 + cd ${{ github.workspace }}/hdf5/build2 + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build2 \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_TEST_API:BOOL=ON \ + -DHDF5_TEST_API_ENABLE_ASYNC:BOOL=ON \ + -DHDF5_ENABLE_PARALLEL:BOOL=ON \ + -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ + -DALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ + -DHDF5_VOL_URL01:STRING="https://github.com/hpc-io/vol-async.git" \ + -DHDF5_VOL_VOL-ASYNC_BRANCH:STRING="develop" \ + -DHDF5_VOL_VOL-ASYNC_NAME:STRING="async under_vol=0\;under_info={}" \ + -DHDF5_VOL_VOL-ASYNC_TEST_PARALLEL:BOOL=ON \ + -DHDF5_VOL_URL02:STRING="https://github.com/hpc-io/vol-cache.git" \ + -DHDF5_VOL_VOL-CACHE_BRANCH:STRING="develop" \ + -DHDF5_VOL_VOL-CACHE_NAME:STRING="cache_ext config=$GITHUB_WORKSPACE/config2.cfg\;under_vol=512\;under_info={under_vol=0\;under_info={}}\;" \ + -DHDF5_VOL_VOL-CACHE_TEST_PARALLEL:BOOL=ON \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Build HDF5 and cache VOL connector + shell: bash + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + echo "LD_LIBRARY_PATH=/usr/local/lib:${{ github.workspace }}/hdf5/build/bin" >> $GITHUB_ENV + + - name: Create cache VOL connector configuration files for testing + shell: bash + run: | + mkdir -p $GITHUB_WORKSPACE/scratch + touch $GITHUB_WORKSPACE/config1.cfg + touch $GITHUB_WORKSPACE/config2.cfg + echo "HDF5_CACHE_STORAGE_SCOPE: LOCAL" >> $GITHUB_WORKSPACE/config1.cfg + echo "HDF5_CACHE_STORAGE_PATH: $GITHUB_WORKSPACE/scratch" >> $GITHUB_WORKSPACE/config1.cfg + echo "HDF5_CACHE_STORAGE_SIZE: 4294967296" >> $GITHUB_WORKSPACE/config1.cfg + echo "HDF5_CACHE_STORAGE_TYPE: SSD" >> $GITHUB_WORKSPACE/config1.cfg + echo "HDF5_CACHE_REPLACEMENT_POLICY: LRU" >> $GITHUB_WORKSPACE/config1.cfg + echo "HDF5_CACHE_STORAGE_SCOPE: LOCAL" >> $GITHUB_WORKSPACE/config2.cfg + echo "HDF5_CACHE_STORAGE_PATH: $GITHUB_WORKSPACE/scratch" >> $GITHUB_WORKSPACE/config2.cfg + echo "HDF5_CACHE_STORAGE_SIZE: 4294967296" >> $GITHUB_WORKSPACE/config2.cfg + echo "HDF5_CACHE_STORAGE_TYPE: SSD" >> $GITHUB_WORKSPACE/config2.cfg + echo "HDF5_CACHE_REPLACEMENT_POLICY: LRU" >> $GITHUB_WORKSPACE/config2.cfg + + - name: Test HDF5 cache VOL connector with external tests + working-directory: ${{ github.workspace }}/hdf5/build + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "test_*.exe$" . + + - name: Test HDF5 cache VOL connector with HDF5 API tests + working-directory: ${{ github.workspace }}/hdf5/build + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-cache" . + + - name: Test HDF5 cache VOL connector atop async VOL connector with external tests + working-directory: ${{ github.workspace }}/hdf5/build2 + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "test_*.exe$" . + + - name: Test HDF5 cache VOL connector atop async VOL connector with HDF5 API tests + working-directory: ${{ github.workspace }}/hdf5/build2 + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-cache" . diff --git a/CMakeVOL.cmake b/CMakeVOL.cmake index d14ac5b2800..6e6d2c7418a 100644 --- a/CMakeVOL.cmake +++ b/CMakeVOL.cmake @@ -124,6 +124,7 @@ if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT" OR HDF5_VOL_ALLOW_EXTERNAL MATCHES "LO FetchContent_Declare (HDF5_VOL_${hdf5_vol_name_lower} GIT_REPOSITORY "${HDF5_VOL_SOURCE}" GIT_TAG "${HDF5_VOL_${hdf5_vol_name_upper}_BRANCH}" + FIND_PACKAGE_ARGS NAMES ${hdf5_vol_name_lower} ) elseif(HDF5_VOL_ALLOW_EXTERNAL MATCHES "LOCAL_DIR") FetchContent_Declare (HDF5_VOL_${hdf5_vol_name_lower} @@ -157,6 +158,16 @@ if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT" OR HDF5_VOL_ALLOW_EXTERNAL MATCHES "LO file (WRITE "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/src/CMakeLists.txt" "${vol_cmake_contents}") endif () + # Work around issue with Cache VOL's CMake code related + # to its Async VOL dependency + if ("${hdf5_vol_name_lower}" MATCHES "vol-cache") + if (EXISTS "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/CMakeLists.txt") + file (READ "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/CMakeLists.txt" vol_cmake_contents) + string (REGEX REPLACE "[ \t]*find_package[ \t]*\\([ \t]*ASYNC[^\r\n\\)]*\\)[ \t]*[\r\n]+" "find_package\(vol-async\)\n" vol_cmake_contents "${vol_cmake_contents}") + file (WRITE "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/CMakeLists.txt" "${vol_cmake_contents}") + endif () + endif () + add_subdirectory (${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR} ${hdf5_vol_${hdf5_vol_name_lower}_BINARY_DIR}) # Get list of targets generated by build of connector From b48db3b7ec7bdd2d44f7a08f5d77f30020517892 Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Mon, 6 Nov 2023 18:30:57 -0600 Subject: [PATCH 06/18] Add CMake variable to allow specifying a VOL connector's package name --- CMakeVOL.cmake | 68 +++++++++++++++++++++++----------- doc/cmake-vols-fetchcontent.md | 26 +++++++++---- 2 files changed, 65 insertions(+), 29 deletions(-) diff --git a/CMakeVOL.cmake b/CMakeVOL.cmake index 6e6d2c7418a..4992cdf8b4a 100644 --- a/CMakeVOL.cmake +++ b/CMakeVOL.cmake @@ -99,6 +99,13 @@ if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT" OR HDF5_VOL_ALLOW_EXTERNAL MATCHES "LO mark_as_advanced ("HDF5_VOL_${hdf5_vol_name_upper}_BRANCH") endif() + set ("HDF5_VOL_${hdf5_vol_name_upper}_CMAKE_PACKAGE_NAME" + "${hdf5_vol_name_lower}" + CACHE + STRING + "CMake package name used by find_package(...) calls for VOL connector '${hdf5_vol_name}'" + ) + set ("HDF5_VOL_${hdf5_vol_name_upper}_NAME" "" CACHE STRING "Name of VOL connector to set for the HDF5_VOL_CONNECTOR environment variable") option ("HDF5_VOL_${hdf5_vol_name_upper}_TEST_PARALLEL" "Whether to test VOL connector '${hdf5_vol_name}' against the parallel API tests" OFF) @@ -120,23 +127,40 @@ if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT" OR HDF5_VOL_ALLOW_EXTERNAL MATCHES "LO message (FATAL_ERROR "HDF5_VOL_PATH${vol_idx_fixed} must be an absolute path to a valid directory") endif () + # Set internal convenience variables for FetchContent dependency name + set (hdf5_vol_depname "${HDF5_VOL_${hdf5_vol_name_upper}_CMAKE_PACKAGE_NAME}") + string (TOLOWER "${hdf5_vol_depname}" hdf5_vol_depname_lower) + if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT") - FetchContent_Declare (HDF5_VOL_${hdf5_vol_name_lower} - GIT_REPOSITORY "${HDF5_VOL_SOURCE}" - GIT_TAG "${HDF5_VOL_${hdf5_vol_name_upper}_BRANCH}" - FIND_PACKAGE_ARGS NAMES ${hdf5_vol_name_lower} - ) + if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24") + FetchContent_Declare (${hdf5_vol_depname} + GIT_REPOSITORY "${HDF5_VOL_SOURCE}" + GIT_TAG "${HDF5_VOL_${hdf5_vol_name_upper}_BRANCH}" + OVERRIDE_FIND_PACKAGE + ) + else () + FetchContent_Declare (${hdf5_vol_depname} + GIT_REPOSITORY "${HDF5_VOL_SOURCE}" + GIT_TAG "${HDF5_VOL_${hdf5_vol_name_upper}_BRANCH}" + FIND_PACKAGE_ARGS NAMES ${hdf5_vol_name_lower} + ) + endif () elseif(HDF5_VOL_ALLOW_EXTERNAL MATCHES "LOCAL_DIR") - FetchContent_Declare (HDF5_VOL_${hdf5_vol_name_lower} - SOURCE_DIR "${HDF5_VOL_SOURCE}" + FetchContent_Declare (${hdf5_vol_depname} + SOURCE_DIR "${HDF5_VOL_SOURCE}" ) endif() - FetchContent_GetProperties(HDF5_VOL_${hdf5_vol_name_lower}) - if (NOT hdf5_vol_${hdf5_vol_name_lower}_POPULATED) - FetchContent_Populate(HDF5_VOL_${hdf5_vol_name_lower}) + FetchContent_GetProperties(${hdf5_vol_depname}) + if (NOT ${hdf5_vol_depname}_POPULATED) + FetchContent_Populate(${hdf5_vol_depname}) + + # Now that content has been populated, set other internal + # convenience variables for FetchContent dependency + set (hdf5_vol_depname_source_dir "${${hdf5_vol_depname_lower}_SOURCE_DIR}") + set (hdf5_vol_depname_binary_dir "${${hdf5_vol_depname_lower}_BINARY_DIR}") - if (NOT EXISTS "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/CMakeLists.txt") + if (NOT EXISTS "${hdf5_vol_depname_source_dir}/CMakeLists.txt") if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT") message (SEND_ERROR "The git repository branch '${HDF5_VOL_${hdf5_vol_name_upper}_BRANCH}' for VOL connector '${hdf5_vol_name}' does not appear to contain a CMakeLists.txt file") elseif (HDF5_VOL_ALLOW_EXTERNAL MATCHES "LOCAL_DIR") @@ -147,31 +171,31 @@ if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT" OR HDF5_VOL_ALLOW_EXTERNAL MATCHES "LO # If there are any calls to find_package(HDF5) in the connector's # CMakeLists.txt files, remove those since any found HDF5 targets # will conflict with targets being generated by this build of HDF5 - if (EXISTS "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/CMakeLists.txt") - file (READ "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/CMakeLists.txt" vol_cmake_contents) + if (EXISTS "${hdf5_vol_depname_source_dir}/CMakeLists.txt") + file (READ "${hdf5_vol_depname_source_dir}/CMakeLists.txt" vol_cmake_contents) string (REGEX REPLACE "[ \t]*find_package[ \t]*\\([ \t]*HDF5[^\r\n\\)]*\\)[ \t]*[\r\n]+" "" vol_cmake_contents "${vol_cmake_contents}") - file (WRITE "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/CMakeLists.txt" "${vol_cmake_contents}") + file (WRITE "${hdf5_vol_depname_source_dir}/CMakeLists.txt" "${vol_cmake_contents}") endif () - if (EXISTS "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/src/CMakeLists.txt") - file (READ "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/src/CMakeLists.txt" vol_cmake_contents) + if (EXISTS "${hdf5_vol_depname_source_dir}/src/CMakeLists.txt") + file (READ "${hdf5_vol_depname_source_dir}/src/CMakeLists.txt" vol_cmake_contents) string (REGEX REPLACE "[ \t]*find_package[ \t]*\\([ \t]*HDF5[^\r\n\\)]*\\)[ \t]*[\r\n]+" "" vol_cmake_contents "${vol_cmake_contents}") - file (WRITE "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/src/CMakeLists.txt" "${vol_cmake_contents}") + file (WRITE "${hdf5_vol_depname_source_dir}/src/CMakeLists.txt" "${vol_cmake_contents}") endif () # Work around issue with Cache VOL's CMake code related # to its Async VOL dependency if ("${hdf5_vol_name_lower}" MATCHES "vol-cache") - if (EXISTS "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/CMakeLists.txt") - file (READ "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/CMakeLists.txt" vol_cmake_contents) + if (EXISTS "${hdf5_vol_depname_source_dir}/CMakeLists.txt") + file (READ "${hdf5_vol_depname_source_dir}/CMakeLists.txt" vol_cmake_contents) string (REGEX REPLACE "[ \t]*find_package[ \t]*\\([ \t]*ASYNC[^\r\n\\)]*\\)[ \t]*[\r\n]+" "find_package\(vol-async\)\n" vol_cmake_contents "${vol_cmake_contents}") - file (WRITE "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/CMakeLists.txt" "${vol_cmake_contents}") + file (WRITE "${hdf5_vol_depname_source_dir}/CMakeLists.txt" "${vol_cmake_contents}") endif () endif () - add_subdirectory (${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR} ${hdf5_vol_${hdf5_vol_name_lower}_BINARY_DIR}) + add_subdirectory (${hdf5_vol_depname_source_dir} ${hdf5_vol_depname_binary_dir}) # Get list of targets generated by build of connector - get_generated_cmake_targets (connector_targets ${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}) + get_generated_cmake_targets (connector_targets ${hdf5_vol_depname_source_dir}) # Create a custom target for the connector to encompass all its # targets and other custom properties set by us for later use diff --git a/doc/cmake-vols-fetchcontent.md b/doc/cmake-vols-fetchcontent.md index 9d3c1ba85ac..ff0591311c4 100644 --- a/doc/cmake-vols-fetchcontent.md +++ b/doc/cmake-vols-fetchcontent.md @@ -97,12 +97,21 @@ After the VOL's internal name is generated, the following new variables get crea variable must be set in order for the VOL connector to be testable with HDF5's tests. + HDF5_VOL__CMAKE_PACKAGE_NAME (Default: ">") + This variable specifies the exact name that would be passed to CMake + find_package(...) calls for the VOL connector in question. It is used as + the dependency name when making CMake FetchContent calls to try to ensure + that any other VOL connectors to be built which depend on this VOL connector + can make find_package(...) calls for this VOL connector at configure time. + By default, this variable is set to a lowercased version of the internal + name generated for the VOL connector (described above). + HDF5_VOL__TEST_PARALLEL (Default: OFF) This variable determines whether the VOL connector with the CMake-internal name '' should be tested against HDF5's parallel tests. If the source was retrieved from a Git URL, then the following variable will additionally be created: - + HDF5_VOL__BRANCH (Default: "main") This variable specifies the git branch name or tag to use when fetching the source code for the VOL connector with the CMake-internal name @@ -111,9 +120,10 @@ If the source was retrieved from a Git URL, then the following variable will add As an example, this would create the following variables for the previously-mentioned VOL connector if it is retrieved from a URL: - HDF5_VOL_VOL-ASYNC_BRANCH - HDF5_VOL_VOL-ASYNC_NAME - HDF5_VOL_VOL-ASYNC_TEST_PARALLEL + HDF5_VOL_VOL-ASYNC_NAME "" + HDF5_VOL_VOL-ASYNC_CMAKE_PACKAGE_NAME "vol-async" + HDF5_VOL_VOL-ASYNC_BRANCH "main" + HDF5_VOL_VOL-ASYNC_TEST_PARALLEL OFF **NOTE** If a VOL connector requires extra information to be passed in its @@ -139,9 +149,10 @@ would typically be passed when building HDF5, such as `CMAKE_INSTALL_PREFIX`, -DHDF5_TEST_API=ON -DHDF5_VOL_ALLOW_EXTERNAL="GIT" -DHDF5_VOL_URL01=https://github.com/hpc-io/vol-async.git - -DHDF5_VOL_VOL-ASYNC_BRANCH=develop + -DHDF5_VOL_VOL-ASYNC_BRANCH=develop -DHDF5_VOL_VOL-ASYNC_NAME="async under_vol=0\;under_info={}" - -DHDF5_VOL_VOL-ASYNC_TEST_PARALLEL=ON .. + -DHDF5_VOL_VOL-ASYNC_TEST_PARALLEL=ON + .. Here, we are specifying that: @@ -156,7 +167,8 @@ Here, we are specifying that: variable should be set to "async under_vol=0\;under_info={}", which specifies that the VOL connector with the canonical name "async" should be loaded and it should be passed the string "under_vol=0;under_info={}" - for its configuration + for its configuration (note the backslash-escaping of semicolons in the string + provided) * The Asynchronous I/O VOL connector should be tested against HDF5's parallel API tests Note that this also assumes that the Asynchronous I/O VOL connector's From 1e384e19b646d75cf743faafa6028d0ebb3123e4 Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Mon, 6 Nov 2023 19:59:26 -0600 Subject: [PATCH 07/18] Add workaround for testing Cache VOL connector --- .github/workflows/vol_async.yml | 8 +++-- .github/workflows/vol_cache.yml | 61 ++++++++++++++++++++++----------- CMakeVOL.cmake | 50 +++++++++++++++++++++------ 3 files changed, 87 insertions(+), 32 deletions(-) diff --git a/.github/workflows/vol_async.yml b/.github/workflows/vol_async.yml index eb30a555d12..214dedf4927 100644 --- a/.github/workflows/vol_async.yml +++ b/.github/workflows/vol_async.yml @@ -89,11 +89,15 @@ jobs: cmake --build . --parallel 3 --config ${{ inputs.build_mode }} echo "LD_LIBRARY_PATH=/usr/local/lib:${{ github.workspace }}/hdf5/build/bin" >> $GITHUB_ENV + # Workaround for asynchronous I/O VOL CMake issue + - name: Copy testing files + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cp bin/async_test* ./_deps/vol-async-build/test + - name: Test HDF5 asynchronous I/O VOL connector with external tests working-directory: ${{ github.workspace }}/hdf5/build run: | - # Workaround for vol-async CMake issue - cp bin/async_test* ./_deps/hdf5_vol_vol-async-build/test ctest --build-config ${{ inputs.build_mode }} -VV -R "async_test" . - name: Test HDF5 asynchronous I/O VOL connector with HDF5 API tests diff --git a/.github/workflows/vol_cache.yml b/.github/workflows/vol_cache.yml index 6835d4f88a1..9416e8ea15d 100644 --- a/.github/workflows/vol_cache.yml +++ b/.github/workflows/vol_cache.yml @@ -66,24 +66,11 @@ jobs: make -j2 sudo make -j2 install - # TODO: It should be possible to build both the async VOL - # and cache VOL at the same time, but the cache VOL looks - # for the async VOL by a different name than 'vol-async' - # and there are dependency issues where the cache VOL tries - # to link to a built async VOL. CMake's FetchContent - # OVERRIDE_FIND_PACKAGE may come in handy here, but our - # current CMake code declares VOL connector targets with - # a prefixed name (HDF5_VOL_vol_name_lower), so this also - # doesn't work yet. For now, just install the async VOL - # manually. -# - name: Install vol-async -# working-directory: ${{ github.workspace }}/vol-async -# run: | -# mkdir build -# cd build -# cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. -# sudo make -j2 install - + # Define ASYNC_INCLUDE_DIR, ASYNC_INCLUDE_DIRS and ASYNC_LIBRARIES to + # patch around having the cache VOL find the async VOL when they're built + # at the same time. Once the Async and Cache VOLs create CMake .config + # files, this should no longer be needed with CMake 3.24 and newer (see + # FetchContent's OVERRIDE_FIND_PACKAGE) - name: Configure HDF5 with cache VOL connector shell: bash run: | @@ -107,9 +94,17 @@ jobs: -DHDF5_VOL_VOL-CACHE_BRANCH:STRING="develop" \ -DHDF5_VOL_VOL-CACHE_NAME:STRING="cache_ext config=$GITHUB_WORKSPACE/config1.cfg\;under_vol=0\;under_info={}\;" \ -DHDF5_VOL_VOL-CACHE_TEST_PARALLEL:BOOL=ON \ + -DASYNC_INCLUDE_DIR=${{ github.workspace }}/hdf5/build/_deps/vol-async-src/src \ + -DASYNC_INCLUDE_DIRS=${{ github.workspace }}/hdf5/build/_deps/vol-async-src/src \ + -DASYNC_LIBRARIES=${{ github.workspace }}/hdf5/build/bin/libasynchdf5.a\;${{ github.workspace }}/hdf5/build/bin/libh5async.so \ ${{ github.workspace }}/hdf5 cat src/libhdf5.settings + # Define ASYNC_INCLUDE_DIR, ASYNC_INCLUDE_DIRS and ASYNC_LIBRARIES to + # patch around having the cache VOL find the async VOL when they're built + # at the same time. Once the Async and Cache VOLs create CMake .config + # files, this should no longer be needed with CMake 3.24 and newer (see + # FetchContent's OVERRIDE_FIND_PACKAGE) - name: Configure HDF5 with cache VOL connector stacked atop async VOL connector shell: bash run: | @@ -133,6 +128,9 @@ jobs: -DHDF5_VOL_VOL-CACHE_BRANCH:STRING="develop" \ -DHDF5_VOL_VOL-CACHE_NAME:STRING="cache_ext config=$GITHUB_WORKSPACE/config2.cfg\;under_vol=512\;under_info={under_vol=0\;under_info={}}\;" \ -DHDF5_VOL_VOL-CACHE_TEST_PARALLEL:BOOL=ON \ + -DASYNC_INCLUDE_DIR=${{ github.workspace }}/hdf5/build/_deps/vol-async-src/src \ + -DASYNC_INCLUDE_DIRS=${{ github.workspace }}/hdf5/build/_deps/vol-async-src/src \ + -DASYNC_LIBRARIES=${{ github.workspace }}/hdf5/build/bin/libasynchdf5.a\;${{ github.workspace }}/hdf5/build/bin/libh5async.so \ ${{ github.workspace }}/hdf5 cat src/libhdf5.settings @@ -160,20 +158,43 @@ jobs: echo "HDF5_CACHE_STORAGE_TYPE: SSD" >> $GITHUB_WORKSPACE/config2.cfg echo "HDF5_CACHE_REPLACEMENT_POLICY: LRU" >> $GITHUB_WORKSPACE/config2.cfg + # Workaround for cache VOL CMake issue + - name: Copy testing files + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cp bin/test_file.exe ./_deps/vol-cache-build/tests + cp bin/test_group.exe ./_deps/vol-cache-build/tests + cp bin/test_dataset.exe ./_deps/vol-cache-build/tests + cp bin/test_dataset_async_api.exe ./_deps/vol-cache-build/tests + cp bin/test_write_multi.exe ./_deps/vol-cache-build/tests + cp bin/test_multdset.exe ./_deps/vol-cache-build/tests + + # Until cache VOL tests are namespaced properly, run them directly - name: Test HDF5 cache VOL connector with external tests working-directory: ${{ github.workspace }}/hdf5/build run: | - ctest --build-config ${{ inputs.build_mode }} -VV -R "test_*.exe$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_file$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_group$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_dataset$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_dataset_async_api$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_write_multi$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_multdset$" . - name: Test HDF5 cache VOL connector with HDF5 API tests working-directory: ${{ github.workspace }}/hdf5/build run: | ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-cache" . + # Until cache VOL tests are namespaced properly, run them directly - name: Test HDF5 cache VOL connector atop async VOL connector with external tests working-directory: ${{ github.workspace }}/hdf5/build2 run: | - ctest --build-config ${{ inputs.build_mode }} -VV -R "test_*.exe$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_file$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_group$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_dataset$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_dataset_async_api$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_write_multi$" . + ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_multdset$" . - name: Test HDF5 cache VOL connector atop async VOL connector with HDF5 API tests working-directory: ${{ github.workspace }}/hdf5/build2 diff --git a/CMakeVOL.cmake b/CMakeVOL.cmake index 4992cdf8b4a..748f21400e8 100644 --- a/CMakeVOL.cmake +++ b/CMakeVOL.cmake @@ -26,6 +26,21 @@ function (get_generated_cmake_targets out_var dir) set (${out_var} "${dir_targets}" PARENT_SCOPE) endfunction () +# Function to apply connector-specify workarounds to build +# code once a connector has been populated through FetchContent +function (apply_connector_workarounds connector_name source_dir) + # For the cache VOL, remove the call to find_package(ASYNC). + # Eventually, the FetchContent OVERRIDE_FIND_PACKAGE should be + # able to fulfill this dependency when building the cache VOL, + # but for now we have to hack around this until the async and + # cache VOLs create CMake .config files + if ("${connector_name}" MATCHES "vol-cache") + file (READ "${source_dir}/CMakeLists.txt" vol_cmake_contents) + string (REGEX REPLACE "[ \t]*find_package[ \t]*\\([ \t]*ASYNC[^\r\n\\)]*\\)[ \t]*[\r\n]+" "" vol_cmake_contents "${vol_cmake_contents}") + file (WRITE "${source_dir}/CMakeLists.txt" "${vol_cmake_contents}") + endif () +endfunction () + set (HDF5_VOL_ALLOW_EXTERNAL "NO" CACHE STRING "Allow building of external HDF5 VOL connectors with FetchContent") set_property (CACHE HDF5_VOL_ALLOW_EXTERNAL PROPERTY STRINGS NO GIT LOCAL_DIR) mark_as_advanced (HDF5_VOL_ALLOW_EXTERNAL) @@ -182,15 +197,8 @@ if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT" OR HDF5_VOL_ALLOW_EXTERNAL MATCHES "LO file (WRITE "${hdf5_vol_depname_source_dir}/src/CMakeLists.txt" "${vol_cmake_contents}") endif () - # Work around issue with Cache VOL's CMake code related - # to its Async VOL dependency - if ("${hdf5_vol_name_lower}" MATCHES "vol-cache") - if (EXISTS "${hdf5_vol_depname_source_dir}/CMakeLists.txt") - file (READ "${hdf5_vol_depname_source_dir}/CMakeLists.txt" vol_cmake_contents) - string (REGEX REPLACE "[ \t]*find_package[ \t]*\\([ \t]*ASYNC[^\r\n\\)]*\\)[ \t]*[\r\n]+" "find_package\(vol-async\)\n" vol_cmake_contents "${vol_cmake_contents}") - file (WRITE "${hdf5_vol_depname_source_dir}/CMakeLists.txt" "${vol_cmake_contents}") - endif () - endif () + # Apply any connector-specific workarounds + apply_connector_workarounds ("${hdf5_vol_name_lower}" "${hdf5_vol_depname_source_dir}") add_subdirectory (${hdf5_vol_depname_source_dir} ${hdf5_vol_depname_binary_dir}) @@ -248,8 +256,30 @@ if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT" OR HDF5_VOL_ALLOW_EXTERNAL MATCHES "LO HDF5_VOL_TEST_PARALLEL ${HDF5_VOL_${hdf5_vol_name_upper}_TEST_PARALLEL} ) - # Add this connector's target to the list of external connector targets + # Add this VOL connector's target to the list of external connector targets list (APPEND HDF5_EXTERNAL_VOL_TARGETS "HDF5_VOL_${hdf5_vol_name_lower}") + + # Get the list of library targets from this VOL connector + unset (connector_lib_targets) + foreach (connector_target ${connector_targets}) + get_target_property (target_type ${connector_target} TYPE) + if (target_type STREQUAL "SHARED_LIBRARY" OR target_type STREQUAL "STATIC_LIBRARY") + list (APPEND connector_lib_targets "${connector_target}") + endif () + endforeach () + + # Add all of the previous VOL connector's library targets as + # dependencies for the current VOL connector to ensure that + # VOL connectors get built serially in case there are dependencies + if (DEFINED last_vol_lib_targets) + foreach (connector_target ${connector_targets}) + add_dependencies (${connector_target} ${last_vol_lib_targets}) + endforeach () + endif () + + # Use this connector's library targets as dependencies + # for the next connector that is built + set (last_vol_lib_targets "${connector_lib_targets}") endif () endif () endforeach () From 7022efe24cd8ff88f9ff5428785ea64386bc7b91 Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Mon, 6 Nov 2023 23:28:33 -0600 Subject: [PATCH 08/18] Remove call to MPI_Init in serial API tests While previously necessary, it now interferes with VOL connectors that may need to be initialized with MPI_THREAD_MULTIPLE --- test/API/H5_api_test.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/test/API/H5_api_test.c b/test/API/H5_api_test.c index 654eb4024ac..ff7ede34038 100644 --- a/test/API/H5_api_test.c +++ b/test/API/H5_api_test.c @@ -136,15 +136,6 @@ main(int argc, char **argv) } } -#ifdef H5_HAVE_PARALLEL - /* If HDF5 was built with parallel enabled, go ahead and call MPI_Init before - * running these tests. Even though these are meant to be serial tests, they will - * likely be run using mpirun (or similar) and we cannot necessarily expect HDF5 or - * an HDF5 VOL connector to call MPI_Init. - */ - MPI_Init(&argc, &argv); -#endif - H5open(); n_tests_run_g = 0; @@ -304,9 +295,5 @@ main(int argc, char **argv) H5close(); -#ifdef H5_HAVE_PARALLEL - MPI_Finalize(); -#endif - exit(((err_occurred || n_tests_failed_g > 0) ? EXIT_FAILURE : EXIT_SUCCESS)); } From f9be9bfec07ab744edeec679bebae46d928b6565 Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Tue, 7 Nov 2023 01:36:29 -0600 Subject: [PATCH 09/18] Add more workarounds for cache VOL testing --- .github/workflows/vol_cache.yml | 10 ++++++++++ CMakeVOL.cmake | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/.github/workflows/vol_cache.yml b/.github/workflows/vol_cache.yml index 9416e8ea15d..fbbc209f7a9 100644 --- a/.github/workflows/vol_cache.yml +++ b/.github/workflows/vol_cache.yml @@ -169,6 +169,11 @@ jobs: cp bin/test_write_multi.exe ./_deps/vol-cache-build/tests cp bin/test_multdset.exe ./_deps/vol-cache-build/tests + - name: Set environment variables for testing + run: | + echo "HDF5_PLUGIN_PATH=${{ github.workspace }}/hdf5/build/bin/" >> $GITHUB_ENV + echo "HDF5_VOL_CONNECTOR=cache_ext config=$GITHUB_WORKSPACE/config1.cfg;under_vol=0;under_info={};" >> $GITHUB_ENV + # Until cache VOL tests are namespaced properly, run them directly - name: Test HDF5 cache VOL connector with external tests working-directory: ${{ github.workspace }}/hdf5/build @@ -185,6 +190,11 @@ jobs: run: | ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-cache" . + - name: Set environment variables for testing + run: | + echo "HDF5_PLUGIN_PATH=${{ github.workspace }}/hdf5/build/bin/" >> $GITHUB_ENV + echo "HDF5_VOL_CONNECTOR=cache_ext config=$GITHUB_WORKSPACE/config2.cfg;under_vol=512;under_info={under_vol=0;under_info={}};" >> $GITHUB_ENV + # Until cache VOL tests are namespaced properly, run them directly - name: Test HDF5 cache VOL connector atop async VOL connector with external tests working-directory: ${{ github.workspace }}/hdf5/build2 diff --git a/CMakeVOL.cmake b/CMakeVOL.cmake index 748f21400e8..8b2207ae1dc 100644 --- a/CMakeVOL.cmake +++ b/CMakeVOL.cmake @@ -35,9 +35,35 @@ function (apply_connector_workarounds connector_name source_dir) # but for now we have to hack around this until the async and # cache VOLs create CMake .config files if ("${connector_name}" MATCHES "vol-cache") + # Remove find_package(ASYNC) call from connector's CMake code file (READ "${source_dir}/CMakeLists.txt" vol_cmake_contents) string (REGEX REPLACE "[ \t]*find_package[ \t]*\\([ \t]*ASYNC[^\r\n\\)]*\\)[ \t]*[\r\n]+" "" vol_cmake_contents "${vol_cmake_contents}") file (WRITE "${source_dir}/CMakeLists.txt" "${vol_cmake_contents}") + + # Remove setting of HDF5_VOL_CONNECTOR and HDF5_PLUGIN_PATH + # in connector's external tests CMake code + file (STRINGS "${source_dir}/tests/CMakeLists.txt" file_lines) + file (WRITE "${source_dir}/tests/CMakeLists.txt" "") + foreach (line IN LISTS file_lines) + set (stripped_line "${line}") + string (REGEX MATCH "^[ \t]*set_tests_properties\\([ \t]*[\r\n]?" match_string "${line}") + if (NOT "${match_string}" STREQUAL "") + string (REGEX REPLACE "^[ \t]*set_tests_properties\\([ \t]*[\r\n]?" "" stripped_line "${line}") + endif () + string (REGEX MATCH "^[ \t]*.\\{test\\}[ \t]*[\r\n]?" match_string "${line}") + if (NOT "${match_string}" STREQUAL "") + string (REGEX REPLACE "^[ \t]*.\\{[A-Za-z]*\\}[ \t]*[\r\n]?" "" stripped_line "${line}") + endif () + string (REGEX MATCH "^[ \t]*PROPERTIES[ \t]*[\r\n]?" match_string "${line}") + if (NOT "${match_string}" STREQUAL "") + string (REGEX REPLACE "^[ \t]*PROPERTIES[ \t]*[\r\n]?" "" stripped_line "${line}") + endif () + string (REGEX MATCH "^[ \t]*ENVIRONMENT[ \t]*.*[\r\n]?" match_string "${line}") + if (NOT "${match_string}" STREQUAL "") + string (REGEX REPLACE "^[ \t]*ENVIRONMENT[ \t]*.*[\r\n]?" "" stripped_line "${line}") + endif () + file (APPEND "${source_dir}/tests/CMakeLists.txt" "${stripped_line}\n") + endforeach () endif () endfunction () From 6b79e0a6d048445e27026271fae6af801d010f1d Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Wed, 8 Nov 2023 12:26:30 -0600 Subject: [PATCH 10/18] Reorganize cache VOL .yml file --- .github/workflows/vol_async.yml | 6 -- .github/workflows/vol_cache.yml | 113 ++++++++++---------------------- 2 files changed, 35 insertions(+), 84 deletions(-) diff --git a/.github/workflows/vol_async.yml b/.github/workflows/vol_async.yml index 214dedf4927..87ca7815d65 100644 --- a/.github/workflows/vol_async.yml +++ b/.github/workflows/vol_async.yml @@ -38,12 +38,6 @@ jobs: ref: api_tests_dev # TODO - - name: Checkout vol-async - uses: actions/checkout@v4 - with: - repository: hpc-io/vol-async - path: vol-async - - name: Checkout Argobots uses: actions/checkout@v4 with: diff --git a/.github/workflows/vol_cache.yml b/.github/workflows/vol_cache.yml index fbbc209f7a9..cad2dfde56d 100644 --- a/.github/workflows/vol_cache.yml +++ b/.github/workflows/vol_cache.yml @@ -13,7 +13,20 @@ permissions: jobs: build_and_test: - name: Test HDF5 cache VOL connector + strategy: + matrix: + name: + - "Test HDF5 cache VOL connector" + - "Test HDF5 cache VOL connector atop async VOL connector" + async: [false, true] + exclude: + - name: "Test HDF5 cache VOL connector" + async: true + + - name: "Test HDF5 cache VOL connector atop async VOL connector" + async: false + + runs-on: ubuntu-latest steps: - name: Install dependencies @@ -38,18 +51,6 @@ jobs: ref: api_tests_dev # TODO - - name: Checkout vol-cache - uses: actions/checkout@v4 - with: - repository: hpc-io/vol-cache - path: vol-cache - - - name: Checkout vol-async - uses: actions/checkout@v4 - with: - repository: hpc-io/vol-async - path: vol-async - - name: Checkout Argobots uses: actions/checkout@v4 with: @@ -66,6 +67,16 @@ jobs: make -j2 sudo make -j2 install + - name: Set environment variables for configuration (cache VOL only) + run: | + echo "HDF5_VOL_CACHE_TEST_NAME=cache_ext config=$GITHUB_WORKSPACE/config1.cfg\;under_vol=0\;under_info={}\;" >> $GITHUB_ENV + if: ${{ ! matrix.async }} + + - name: Set environment variables for configuration (cache VOL atop async VOL) + run: | + echo "HDF5_VOL_CACHE_TEST_NAME=cache_ext config=$GITHUB_WORKSPACE/config1.cfg\;under_vol=512\;under_info={under_vol=0\;under_info={}}\;" >> $GITHUB_ENV + if: ${{ matrix.async }} + # Define ASYNC_INCLUDE_DIR, ASYNC_INCLUDE_DIRS and ASYNC_LIBRARIES to # patch around having the cache VOL find the async VOL when they're built # at the same time. Once the Async and Cache VOLs create CMake .config @@ -90,43 +101,9 @@ jobs: -DHDF5_VOL_VOL-ASYNC_BRANCH:STRING="develop" \ -DHDF5_VOL_VOL-ASYNC_NAME:STRING="async under_vol=0\;under_info={}" \ -DHDF5_VOL_VOL-ASYNC_TEST_PARALLEL:BOOL=ON \ - -DHDF5_VOL_URL02:STRING="https://github.com/hpc-io/vol-cache.git" \ + -DHDF5_VOL_URL02:STRING="https://github.com/HDFGroup/vol-cache.git" \ -DHDF5_VOL_VOL-CACHE_BRANCH:STRING="develop" \ - -DHDF5_VOL_VOL-CACHE_NAME:STRING="cache_ext config=$GITHUB_WORKSPACE/config1.cfg\;under_vol=0\;under_info={}\;" \ - -DHDF5_VOL_VOL-CACHE_TEST_PARALLEL:BOOL=ON \ - -DASYNC_INCLUDE_DIR=${{ github.workspace }}/hdf5/build/_deps/vol-async-src/src \ - -DASYNC_INCLUDE_DIRS=${{ github.workspace }}/hdf5/build/_deps/vol-async-src/src \ - -DASYNC_LIBRARIES=${{ github.workspace }}/hdf5/build/bin/libasynchdf5.a\;${{ github.workspace }}/hdf5/build/bin/libh5async.so \ - ${{ github.workspace }}/hdf5 - cat src/libhdf5.settings - - # Define ASYNC_INCLUDE_DIR, ASYNC_INCLUDE_DIRS and ASYNC_LIBRARIES to - # patch around having the cache VOL find the async VOL when they're built - # at the same time. Once the Async and Cache VOLs create CMake .config - # files, this should no longer be needed with CMake 3.24 and newer (see - # FetchContent's OVERRIDE_FIND_PACKAGE) - - name: Configure HDF5 with cache VOL connector stacked atop async VOL connector - shell: bash - run: | - mkdir ${{ github.workspace }}/hdf5/build2 - cd ${{ github.workspace }}/hdf5/build2 - cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ - -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build2 \ - -DBUILD_STATIC_LIBS=OFF \ - -DHDF5_TEST_API:BOOL=ON \ - -DHDF5_TEST_API_ENABLE_ASYNC:BOOL=ON \ - -DHDF5_ENABLE_PARALLEL:BOOL=ON \ - -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ - -DALLOW_UNSUPPORTED:BOOL=ON \ - -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ - -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ - -DHDF5_VOL_URL01:STRING="https://github.com/hpc-io/vol-async.git" \ - -DHDF5_VOL_VOL-ASYNC_BRANCH:STRING="develop" \ - -DHDF5_VOL_VOL-ASYNC_NAME:STRING="async under_vol=0\;under_info={}" \ - -DHDF5_VOL_VOL-ASYNC_TEST_PARALLEL:BOOL=ON \ - -DHDF5_VOL_URL02:STRING="https://github.com/hpc-io/vol-cache.git" \ - -DHDF5_VOL_VOL-CACHE_BRANCH:STRING="develop" \ - -DHDF5_VOL_VOL-CACHE_NAME:STRING="cache_ext config=$GITHUB_WORKSPACE/config2.cfg\;under_vol=512\;under_info={under_vol=0\;under_info={}}\;" \ + -DHDF5_VOL_VOL-CACHE_NAME:STRING="$HDF5_VOL_CACHE_TEST_NAME" \ -DHDF5_VOL_VOL-CACHE_TEST_PARALLEL:BOOL=ON \ -DASYNC_INCLUDE_DIR=${{ github.workspace }}/hdf5/build/_deps/vol-async-src/src \ -DASYNC_INCLUDE_DIRS=${{ github.workspace }}/hdf5/build/_deps/vol-async-src/src \ @@ -141,22 +118,16 @@ jobs: cmake --build . --parallel 3 --config ${{ inputs.build_mode }} echo "LD_LIBRARY_PATH=/usr/local/lib:${{ github.workspace }}/hdf5/build/bin" >> $GITHUB_ENV - - name: Create cache VOL connector configuration files for testing + - name: Create cache VOL connector configuration file for testing shell: bash run: | mkdir -p $GITHUB_WORKSPACE/scratch touch $GITHUB_WORKSPACE/config1.cfg - touch $GITHUB_WORKSPACE/config2.cfg echo "HDF5_CACHE_STORAGE_SCOPE: LOCAL" >> $GITHUB_WORKSPACE/config1.cfg echo "HDF5_CACHE_STORAGE_PATH: $GITHUB_WORKSPACE/scratch" >> $GITHUB_WORKSPACE/config1.cfg echo "HDF5_CACHE_STORAGE_SIZE: 4294967296" >> $GITHUB_WORKSPACE/config1.cfg echo "HDF5_CACHE_STORAGE_TYPE: SSD" >> $GITHUB_WORKSPACE/config1.cfg echo "HDF5_CACHE_REPLACEMENT_POLICY: LRU" >> $GITHUB_WORKSPACE/config1.cfg - echo "HDF5_CACHE_STORAGE_SCOPE: LOCAL" >> $GITHUB_WORKSPACE/config2.cfg - echo "HDF5_CACHE_STORAGE_PATH: $GITHUB_WORKSPACE/scratch" >> $GITHUB_WORKSPACE/config2.cfg - echo "HDF5_CACHE_STORAGE_SIZE: 4294967296" >> $GITHUB_WORKSPACE/config2.cfg - echo "HDF5_CACHE_STORAGE_TYPE: SSD" >> $GITHUB_WORKSPACE/config2.cfg - echo "HDF5_CACHE_REPLACEMENT_POLICY: LRU" >> $GITHUB_WORKSPACE/config2.cfg # Workaround for cache VOL CMake issue - name: Copy testing files @@ -169,10 +140,17 @@ jobs: cp bin/test_write_multi.exe ./_deps/vol-cache-build/tests cp bin/test_multdset.exe ./_deps/vol-cache-build/tests - - name: Set environment variables for testing + - name: Set environment variables for external tests (cache VOL only) run: | echo "HDF5_PLUGIN_PATH=${{ github.workspace }}/hdf5/build/bin/" >> $GITHUB_ENV echo "HDF5_VOL_CONNECTOR=cache_ext config=$GITHUB_WORKSPACE/config1.cfg;under_vol=0;under_info={};" >> $GITHUB_ENV + if: ${{ ! matrix.async }} + + - name: Set environment variables for external tests (cache VOL atop async VOL) + run: | + echo "HDF5_PLUGIN_PATH=${{ github.workspace }}/hdf5/build/bin/" >> $GITHUB_ENV + echo "HDF5_VOL_CONNECTOR=cache_ext config=$GITHUB_WORKSPACE/config1.cfg;under_vol=512;under_info={under_vol=0;under_info={}};" >> $GITHUB_ENV + if: ${{ matrix.async }} # Until cache VOL tests are namespaced properly, run them directly - name: Test HDF5 cache VOL connector with external tests @@ -189,24 +167,3 @@ jobs: working-directory: ${{ github.workspace }}/hdf5/build run: | ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-cache" . - - - name: Set environment variables for testing - run: | - echo "HDF5_PLUGIN_PATH=${{ github.workspace }}/hdf5/build/bin/" >> $GITHUB_ENV - echo "HDF5_VOL_CONNECTOR=cache_ext config=$GITHUB_WORKSPACE/config2.cfg;under_vol=512;under_info={under_vol=0;under_info={}};" >> $GITHUB_ENV - - # Until cache VOL tests are namespaced properly, run them directly - - name: Test HDF5 cache VOL connector atop async VOL connector with external tests - working-directory: ${{ github.workspace }}/hdf5/build2 - run: | - ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_file$" . - ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_group$" . - ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_dataset$" . - ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_dataset_async_api$" . - ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_write_multi$" . - ctest --build-config ${{ inputs.build_mode }} -VV -R "^test_multdset$" . - - - name: Test HDF5 cache VOL connector atop async VOL connector with HDF5 API tests - working-directory: ${{ github.workspace }}/hdf5/build2 - run: | - ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-cache" . From 440852a6dfa71f2fa4a823ab5e50a3cde2d6157d Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Wed, 8 Nov 2023 17:49:15 -0600 Subject: [PATCH 11/18] Add testing of ADIOS2 VOL connector --- .github/workflows/vol.yml | 32 ++++---- .github/workflows/vol_adios2.yml | 125 +++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/vol_adios2.yml diff --git a/.github/workflows/vol.yml b/.github/workflows/vol.yml index a69bfa3f9f6..5c823b62886 100644 --- a/.github/workflows/vol.yml +++ b/.github/workflows/vol.yml @@ -23,24 +23,26 @@ jobs: #hdf5_vol_rest_fetchcontent: # uses: ./.github/workflows/vol_rest.yml - hdf5_vol_ext_passthru_fetchcontent: - uses: ./.github/workflows/vol_ext_passthru.yml +# hdf5_vol_ext_passthru_fetchcontent: +# uses: ./.github/workflows/vol_ext_passthru.yml +# with: +# build_mode: "Release" + +# hdf5_vol_async_fetchcontent: +# uses: ./.github/workflows/vol_async.yml +# with: +# build_mode: "Release" + +# hdf5_vol_cache_fetchcontent: +# uses: ./.github/workflows/vol_cache.yml +# with: +# build_mode: "Release" + + hdf5_vol_adios2: + uses: ./.github/workflows/vol_adios2.yml with: build_mode: "Release" - hdf5_vol_async_fetchcontent: - uses: ./.github/workflows/vol_async.yml - with: - build_mode: "Release" - - hdf5_vol_cache_fetchcontent: - uses: ./.github/workflows/vol_cache.yml - with: - build_mode: "Release" - - #hdf5_vol_adios2_fetchcontent: - # uses: ./.github/workflows/vol_adios2.yml - #hdf5_vol_log_fetchcontent: # uses: ./.github/workflows/vol_log.yml diff --git a/.github/workflows/vol_adios2.yml b/.github/workflows/vol_adios2.yml new file mode 100644 index 00000000000..7dc94bf0df5 --- /dev/null +++ b/.github/workflows/vol_adios2.yml @@ -0,0 +1,125 @@ +name: Test HDF5 ADIOS2 VOL + +on: + workflow_call: + inputs: + build_mode: + description: "CMake Build type" + required: true + type: string + +permissions: + contents: read + +env: + ADIOS2_COMMIT: 3adf20a929b69c23312a6b5f3cccc49376df77e8 + ADIOS2_COMMIT_SHORT: 3adf20a + +jobs: + build_and_test: + name: Test HDF5 ADIOS2 VOL connector + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: | + sudo apt update + sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev + + # TODO + #- name: Checkout HDF5 + # uses: actions/checkout@v4 + # with: + # repository: HDFGroup/hdf5 + # path: hdf5 + # TODO + + # TODO + - name: Checkout HDF5 + uses: actions/checkout@v4 + with: + repository: jhendersonHDF/hdf5 + path: hdf5 + ref: api_tests_dev + # TODO + + - name: Configure HDF5 + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build + cd ${{ github.workspace }}/hdf5/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_TEST_API:BOOL=ON \ + -DHDF5_TEST_API_ENABLE_ASYNC:BOOL=ON \ + -DHDF5_ENABLE_PARALLEL:BOOL=ON \ + -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ + -DALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Build and install HDF5 + shell: bash + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + cmake --install . + echo "LD_LIBRARY_PATH=${{ github.workspace }}/hdf5/build/bin" >> $GITHUB_ENV + echo "PATH=${{ runner.workspace }}/hdf5_build/bin:${PATH}" >> $GITHUB_ENV + + # Since the HDF5 ADIOS2 VOL connector is part of the ADIOS2 repository, + # it is difficult to use CMake's FetchContent functionality to fetch + # and build the ADIOS2 connector. Also, since building of ADIOS2 takes + # a while, it isn't ideal to have to rebuild it every time we want to + # test against changes in HDF5 or the VOL connector. Therefore, just + # use a fixed commit for the build of ADIOS2 so we can cache that and + # still test the connector against changes in HDF5. + - name: Restore ADIOS2 (${{ env.ADIOS2_COMMIT_SHORT }}) installation cache + id: cache-adios2 + uses: actions/cache@v3 + with: + path: ${{ runner.workspace }}/adios2-${{ env.ADIOS2_COMMIT_SHORT }}-install + key: ${{ runner.os }}-${{ runner.arch }}-adios2-${{ env.ADIOS2_COMMIT }}-${{ inputs.build_mode }}-cache + + - if: ${{ steps.cache-adios2.outputs.cache-hit != 'true' }} + name: Checkout ADIOS2 (${{ env.ADIOS2_COMMIT_SHORT }}) + uses: actions/checkout@v4 + with: + repository: ornladios/ADIOS2 + ref: ${{ env.ADIOS2_COMMIT }} + path: adios2 + + - if: ${{ steps.cache-adios2.outputs.cache-hit != 'true' }} + name: Install ADIOS2 (${{ env.ADIOS2_COMMIT_SHORT }}) + env: + CXX: mpic++ + CC: mpicc + run: | + mkdir adios2/build + cd adios2/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/adios2-${{ env.ADIOS2_COMMIT_SHORT }}-install \ + -DADIOS2_USE_HDF5:BOOL=ON \ + -DHDF5_ROOT=${{ runner.workspace }}/hdf5_build/ \ + .. + make -j2 + sudo make -j2 install + + - name: Cache ADIOS2 (${{ env.ADIOS2_COMMIT_SHORT }}) installation + uses: actions/cache/save@v3 + if: ${{ steps.cache-adios2.outputs.cache-hit != 'true' }} + with: + path: ${{ runner.workspace }}/adios2-${{ env.ADIOS2_COMMIT_SHORT }}-install + key: ${{ runner.os }}-${{ runner.arch }}-adios2-${{ env.ADIOS2_COMMIT }}-${{ inputs.build_mode }}-cache + + - name: Set environment variables for tests + run: | + echo "HDF5_PLUGIN_PATH=${{ runner.workspace }}/adios2-${{ env.ADIOS2_COMMIT_SHORT }}-install/lib" >> $GITHUB_ENV + echo "HDF5_VOL_CONNECTOR=ADIOS2_VOL" >> $GITHUB_ENV + + # Skip parallel testing for now as it appears to hang + - name: Test HDF5 ADIOS2 VOL connector with HDF5 API tests + working-directory: ${{ github.workspace }}/hdf5/build + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "h5_api" -E "parallel" . From ad4933c07ce08b2e67687e5a0690debd06c091f4 Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Wed, 8 Nov 2023 21:34:10 -0600 Subject: [PATCH 12/18] Add status badge for VOL connectors CI --- test/API/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/API/README.md b/test/API/README.md index aec6eaa0fa4..9a8c15c29fc 100644 --- a/test/API/README.md +++ b/test/API/README.md @@ -1,5 +1,7 @@ # HDF5 API Tests +[![HDF5 VOL connectors CI](https://github.com/jhendersonHDF/hdf5/actions/workflows/vol.yml/badge.svg)](https://github.com/jhendersonHDF/hdf5/actions/workflows/vol.yml) + This directory contains several test applications that exercise HDF5's public API and serve as regression tests for HDF5 [VOL Connectors](https://portal.hdfgroup.org/display/HDF5/Virtual+Object+Layer). From 962f15f9b5a0595bd82f001470588c5b27c4cdb4 Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Wed, 8 Nov 2023 22:14:10 -0600 Subject: [PATCH 13/18] Add testing of Log-based VOL connector --- .github/workflows/vol.yml | 12 ++-- .github/workflows/vol_adios2.yml | 2 +- .github/workflows/vol_log.yml | 95 ++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/vol_log.yml diff --git a/.github/workflows/vol.yml b/.github/workflows/vol.yml index 5c823b62886..e6171970369 100644 --- a/.github/workflows/vol.yml +++ b/.github/workflows/vol.yml @@ -38,14 +38,16 @@ jobs: # with: # build_mode: "Release" - hdf5_vol_adios2: - uses: ./.github/workflows/vol_adios2.yml +# hdf5_vol_adios2: +# uses: ./.github/workflows/vol_adios2.yml +# with: +# build_mode: "Release" + + hdf5_vol_log: + uses: ./.github/workflows/vol_log.yml with: build_mode: "Release" - #hdf5_vol_log_fetchcontent: - # uses: ./.github/workflows/vol_log.yml - # Build and install HDF5 and test individual VOL connectors # against the installed API test executables diff --git a/.github/workflows/vol_adios2.yml b/.github/workflows/vol_adios2.yml index 7dc94bf0df5..1de5b039851 100644 --- a/.github/workflows/vol_adios2.yml +++ b/.github/workflows/vol_adios2.yml @@ -104,7 +104,7 @@ jobs: -DHDF5_ROOT=${{ runner.workspace }}/hdf5_build/ \ .. make -j2 - sudo make -j2 install + make -j2 install - name: Cache ADIOS2 (${{ env.ADIOS2_COMMIT_SHORT }}) installation uses: actions/cache/save@v3 diff --git a/.github/workflows/vol_log.yml b/.github/workflows/vol_log.yml new file mode 100644 index 00000000000..2983100e888 --- /dev/null +++ b/.github/workflows/vol_log.yml @@ -0,0 +1,95 @@ +name: Test HDF5 Log-based VOL + +on: + workflow_call: + inputs: + build_mode: + description: "CMake Build type" + required: true + type: string + +permissions: + contents: read + +jobs: + build_and_test: + name: Test HDF5 Log-based VOL connector + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: | + sudo apt update + sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev zlib1g-dev + #mpich + + # TODO + #- name: Checkout HDF5 + # uses: actions/checkout@v4 + # with: + # repository: HDFGroup/hdf5 + # path: hdf5 + # TODO + + # TODO + - name: Checkout HDF5 + uses: actions/checkout@v4 + with: + repository: jhendersonHDF/hdf5 + path: hdf5 + ref: api_tests_dev + # TODO + + # Log-based VOL currently doesn't have CMake support + - name: Configure HDF5 + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build + cd ${{ github.workspace }}/hdf5/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_TEST_API:BOOL=ON \ + -DHDF5_TEST_API_ENABLE_ASYNC:BOOL=ON \ + -DHDF5_ENABLE_PARALLEL:BOOL=ON \ + -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ + -DALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Build and install HDF5 + shell: bash + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + cmake --install . + echo "LD_LIBRARY_PATH=${{ github.workspace }}/hdf5/build/bin" >> $GITHUB_ENV + echo "PATH=${{ runner.workspace }}/hdf5_build/bin:${PATH}" >> $GITHUB_ENV + + - name: Checkout Log-based VOL + uses: actions/checkout@v4 + with: + repository: DataLib-ECP/vol-log-based + path: vol-log-based + + - name: Build HDF5 Log-based VOL connector and test with external tests + env: + CXX: mpic++ + CC: mpicc + LD_LIBRARY_PATH: ${{ runner.workspace }}/hdf5_build/lib + run: | + cd vol-log-based + autoreconf -i + ./configure --prefix=${{ runner.workspace }}/vol-log-based-build --with-hdf5=${{ runner.workspace }}/hdf5_build/ --enable-shared --enable-zlib + make -j2 && make install + export HDF5_PLUGIN_PATH="${{ runner.workspace }}/vol-log-based-build/lib" + export HDF5_VOL_CONNECTOR="LOG under_vol=0;under_info={}" + make check + echo "HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" >> $GITHUB_ENV + echo "HDF5_VOL_CONNECTOR=${HDF5_VOL_CONNECTOR}" >> $GITHUB_ENV + + # Skip parallel testing for now as it appears to hang + - name: Test HDF5 Log-based VOL connector with HDF5 API tests + working-directory: ${{ github.workspace }}/hdf5/build + run: | + ctest --build-config ${{ inputs.build_mode }} -VV -R "h5_api" -E "parallel" . From 6de5f537e302816157a5f24cfef5e71af12eec5a Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Wed, 8 Nov 2023 23:21:55 -0600 Subject: [PATCH 14/18] Add testing of REST VOL connector --- .github/workflows/vol.yml | 14 ++- .github/workflows/vol_rest.yml | 201 +++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/vol_rest.yml diff --git a/.github/workflows/vol.yml b/.github/workflows/vol.yml index e6171970369..96811f273b5 100644 --- a/.github/workflows/vol.yml +++ b/.github/workflows/vol.yml @@ -20,8 +20,10 @@ jobs: #hdf5_vol_daos_fetchcontent: # uses: ./.github/workflows/vol_daos.yml - #hdf5_vol_rest_fetchcontent: - # uses: ./.github/workflows/vol_rest.yml + hdf5_vol_rest_fetchcontent: + uses: ./.github/workflows/vol_rest.yml + with: + build_mode: "Release" # hdf5_vol_ext_passthru_fetchcontent: # uses: ./.github/workflows/vol_ext_passthru.yml @@ -43,10 +45,10 @@ jobs: # with: # build_mode: "Release" - hdf5_vol_log: - uses: ./.github/workflows/vol_log.yml - with: - build_mode: "Release" +# hdf5_vol_log: +# uses: ./.github/workflows/vol_log.yml +# with: +# build_mode: "Release" # Build and install HDF5 and test individual VOL connectors # against the installed API test executables diff --git a/.github/workflows/vol_rest.yml b/.github/workflows/vol_rest.yml new file mode 100644 index 00000000000..975b449f821 --- /dev/null +++ b/.github/workflows/vol_rest.yml @@ -0,0 +1,201 @@ +name: Test HDF5 REST VOL + +on: + workflow_call: + inputs: + build_mode: + description: "CMake Build type" + required: true + type: string + +permissions: + contents: read + +env: + ADMIN_PASSWORD: admin + ADMIN_USERNAME: admin + USER_NAME: test_user1 + USER_PASSWORD: test + USER2_NAME: test_user2 + USER2_PASSWORD: test + HSDS_USERNAME: test_user1 + HSDS_PASSWORD: test + HSDS_PATH: /home/test_user1/ + HDF5_API_TEST_PATH_PREFIX: /home/test_user1/ + HSDS_ENDPOINT: http+unix://%2Ftmp%2Fhs%2Fsn_1.sock + HDF5_VOL_CONNECTOR: REST + ROOT_DIR: ${{github.workspace}}/hsdsdata + BUCKET_NAME: hsdstest + +jobs: + build_and_test: + name: Test HDF5 REST VOL connector + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10"] + + steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install automake autoconf libtool libtool-bin libcurl4-openssl-dev libyajl-dev + + # TODO + #- name: Checkout HDF5 + # uses: actions/checkout@v4 + # with: + # repository: HDFGroup/hdf5 + # path: hdf5 + # TODO + + # TODO + - name: Checkout HDF5 + uses: actions/checkout@v4 + with: + repository: jhendersonHDF/hdf5 + path: hdf5 + ref: api_tests_dev + # TODO + + - name: Configure HDF5 with REST VOL connector + shell: bash + run: | + mkdir ${{ github.workspace }}/hdf5/build + cd ${{ github.workspace }}/hdf5/build + cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/hdf5_build \ + -DBUILD_STATIC_LIBS=OFF \ + -DHDF5_BUILD_HL_LIB:BOOL=ON \ + -DHDF5_TEST_API:BOOL=ON \ + -DALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + -DHDF5_ENABLE_Z_LIB_SUPPORT:BOOL=OFF \ + -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ + -DHDF5_VOL_URL01:STRING="https://github.com/HDFGroup/vol-rest.git" \ + -DHDF5_VOL_VOL-REST_BRANCH:STRING="master" \ + -DHDF5_VOL_VOL-REST_NAME:STRING="REST" \ + -DHDF5_VOL_VOL-REST_TEST_PARALLEL:BOOL=OFF \ + -DHDF5_VOL_REST_ENABLE_EXAMPLES=ON \ + ${{ github.workspace }}/hdf5 + cat src/libhdf5.settings + + - name: Build and install HDF5 and REST VOL connector + shell: bash + working-directory: ${{ github.workspace }}/hdf5/build + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + cmake --install . + echo "LD_LIBRARY_PATH=${{ github.workspace }}/hdf5/build/bin" >> $GITHUB_ENV + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Checkout HSDS + uses: actions/checkout@v4 + with: + repository: HDFGroup/hsds + path: ${{github.workspace}}/hsds + + - name: Get HSDS HEAD commit SHA + shell: bash + working-directory: ${{github.workspace}}/hsds + run: | + export HSDS_COMMIT=`git rev-parse HEAD` + export HSDS_COMMIT_SHORT=`git rev-parse --short HEAD` + echo "HSDS_COMMIT=${HSDS_COMMIT}" >> $GITHUB_ENV + echo "HSDS_COMMIT_SHORT=${HSDS_COMMIT_SHORT}" >> $GITHUB_ENV + + # Note that we don't currently cache HSDS, as we would need + # to pick a fixed commit/tag in order to generate a reasonable + # key to use for caching/restoring from the cache + #- name: Restore HSDS (${{ env.HSDS_COMMIT_SHORT }}) installation cache + # id: restore-hsds + # uses: actions/cache@v3 + # with: + # path: ${{ runner.workspace }}/hsds-${{ env.HSDS_COMMIT_SHORT }}-install + # key: ${{ runner.os }}-${{ runner.arch }}-hsds-${{ env.HSDS_COMMIT }}-${{ inputs.build_mode }}-cache + + - name: Install HSDS (${{ env.HSDS_COMMIT_SHORT }}) dependencies + shell: bash + working-directory: ${{github.workspace}}/hsds + run: | + python -m pip install --upgrade pip + python -m pip install pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Install HSDS (${{ env.HSDS_COMMIT_SHORT }}) package + #if: ${{ ! steps.restore-hsds.outputs.cache-hit }} + shell: bash + run: | + cd ${{github.workspace}}/hsds + pip install -e . + + #- name: Cache HSDS (${{ env.HSDS_COMMIT_SHORT }}) installation + # uses: actions/cache/save@v3 + # if: ${{ ! steps.restore-hsds.outputs.cache-hit }} + # with: + # path: ${{ runner.workspace }}/hsds-${{ env.HSDS_COMMIT_SHORT }}-install + # key: ${{ runner.os }}-${{ runner.arch }}-hsds-${{ env.HSDS_COMMIT }}-${{ inputs.build_mode }}-cache + + - name: Run HSDS unit tests + shell: bash + working-directory: ${{github.workspace}}/hsds + run: | + pytest + + - name: Start HSDS + working-directory: ${{github.workspace}}/hsds + run: | + mkdir ${{github.workspace}}/hsdsdata && + mkdir ${{github.workspace}}/hsdsdata/hsdstest && + cp admin/config/groups.default admin/config/groups.txt && + cp admin/config/passwd.default admin/config/passwd.txt && + cp admin/config/groups.default admin/config/groups.txt && + cp admin/config/passwd.default admin/config/passwd.txt + ROOT_DIR=${{github.workspace}}/hsdsdata ./runall.sh --no-docker 1 & + sleep 10 + + - name: Test HSDS + working-directory: ${{github.workspace}}/hsds + run: | + python tests/integ/setup_test.py + + - name: Test HDF5 REST VOL connector with external tests + working-directory: ${{github.workspace}}/hdf5/build/ + run: | + sudo \ + HDF5_PLUGIN_PATH="${{ runner.workspace }}/hdf5_build/lib" \ + HDF5_VOL_CONNECTOR=REST \ + ADMIN_USERNAME=admin ADMIN_PASSWORD=admin \ + USER_NAME=test_user1 USER_PASSWORD=test \ + USER2_NAME=test_user2 USER2_PASSWORD=test \ + HSDS_USERNAME=test_user1 HSDS_PASSWORD=test \ + HSDS_PATH=/home/test_user1/ HDF5_API_TEST_PATH_PREFIX=/home/test_user1/ \ + HSDS_ENDPOINT=http+unix://%2Ftmp%2Fhs%2Fsn_1.sock \ + ROOT_DIR=${{github.workspace}}/hsdsdata \ + BUCKET_NAME=hsdstest \ + ctest --build-config ${{ inputs.build_mode }} -VV -R "test_rest_vol" . + + - name: Test HDF5 REST VOL connector with HDF5 API tests + working-directory: ${{github.workspace}}/hdf5/build/ + run: | + sudo \ + HDF5_PLUGIN_PATH="${{ runner.workspace }}/hdf5_build/lib" \ + HDF5_VOL_CONNECTOR=REST \ + ADMIN_USERNAME=admin ADMIN_PASSWORD=admin \ + USER_NAME=test_user1 USER_PASSWORD=test \ + USER2_NAME=test_user2 USER2_PASSWORD=test \ + HSDS_USERNAME=test_user1 HSDS_PASSWORD=test \ + HSDS_PATH=/home/test_user1/ HDF5_API_TEST_PATH_PREFIX=/home/test_user1/ \ + HSDS_ENDPOINT=http+unix://%2Ftmp%2Fhs%2Fsn_1.sock \ + ROOT_DIR=${{github.workspace}}/hsdsdata \ + BUCKET_NAME=hsdstest \ + ctest --build-config ${{ inputs.build_mode }} -VV -R "h5_api" . + + - name: Stop HSDS + working-directory: ${{github.workspace}}/hsds + run: | + ./stopall.sh From b48574714572e21628e10c14c34c7d6cdec4e07e Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Thu, 9 Nov 2023 00:17:40 -0600 Subject: [PATCH 15/18] Re-enable testing of all connectors --- .github/workflows/vol.yml | 55 +++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/.github/workflows/vol.yml b/.github/workflows/vol.yml index 96811f273b5..542e4da5d2f 100644 --- a/.github/workflows/vol.yml +++ b/.github/workflows/vol.yml @@ -25,31 +25,30 @@ jobs: with: build_mode: "Release" -# hdf5_vol_ext_passthru_fetchcontent: -# uses: ./.github/workflows/vol_ext_passthru.yml -# with: -# build_mode: "Release" - -# hdf5_vol_async_fetchcontent: -# uses: ./.github/workflows/vol_async.yml -# with: -# build_mode: "Release" - -# hdf5_vol_cache_fetchcontent: -# uses: ./.github/workflows/vol_cache.yml -# with: -# build_mode: "Release" - -# hdf5_vol_adios2: -# uses: ./.github/workflows/vol_adios2.yml -# with: -# build_mode: "Release" - -# hdf5_vol_log: -# uses: ./.github/workflows/vol_log.yml -# with: -# build_mode: "Release" - - # Build and install HDF5 and test individual VOL connectors - # against the installed API test executables - + hdf5_vol_ext_passthru_fetchcontent: + uses: ./.github/workflows/vol_ext_passthru.yml + with: + build_mode: "Release" + + hdf5_vol_async_fetchcontent: + uses: ./.github/workflows/vol_async.yml + with: + build_mode: "Release" + + hdf5_vol_cache_fetchcontent: + uses: ./.github/workflows/vol_cache.yml + with: + build_mode: "Release" + + hdf5_vol_adios2: + uses: ./.github/workflows/vol_adios2.yml + with: + build_mode: "Release" + + hdf5_vol_log: + uses: ./.github/workflows/vol_log.yml + with: + build_mode: "Release" + + # TODO: Build and install HDF5 and test individual VOL + # connectors against the installed API test executables From c7dd280c88b86460c4c71a509e07bf184a8cd0f6 Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Mon, 13 Nov 2023 13:59:41 -0600 Subject: [PATCH 16/18] Move VOL connector testing status badge to main README --- README.md | 1 + test/API/README.md | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 28d8d7a1b4d..e76e4c63e6d 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ HDF5 version 1.15.0 currently under development [![netCDF build status](https://img.shields.io/github/actions/workflow/status/HDFGroup/hdf5/netcdf.yml?branch=develop&label=netCDF)](https://github.com/HDFGroup/hdf5/actions?query=branch%3Adevelop) [![h5py build status](https://img.shields.io/github/actions/workflow/status/HDFGroup/hdf5/h5py.yml?branch=develop&label=h5py)](https://github.com/HDFGroup/hdf5/actions?query=branch%3Adevelop) [![CVE regression](https://img.shields.io/github/actions/workflow/status/HDFGroup/hdf5/cve.yml?branch=develop&label=CVE)](https://github.com/HDFGroup/hdf5/actions?query=branch%3Adevelop) +[![HDF5 VOL connectors build status](https://github.com/jhendersonHDF/hdf5/actions/workflows/vol.yml/badge.svg)](https://github.com/jhendersonHDF/hdf5/actions/workflows/vol.yml) [![1.14 build status](https://img.shields.io/github/actions/workflow/status/HDFGroup/hdf5/main.yml?branch=hdf5_1_14&label=1.14)](https://github.com/HDFGroup/hdf5/actions?query=branch%3Ahdf5_1_14) [![BSD](https://img.shields.io/badge/License-BSD-blue.svg)](https://github.com/HDFGroup/hdf5/blob/develop/COPYING) diff --git a/test/API/README.md b/test/API/README.md index 9a8c15c29fc..aec6eaa0fa4 100644 --- a/test/API/README.md +++ b/test/API/README.md @@ -1,7 +1,5 @@ # HDF5 API Tests -[![HDF5 VOL connectors CI](https://github.com/jhendersonHDF/hdf5/actions/workflows/vol.yml/badge.svg)](https://github.com/jhendersonHDF/hdf5/actions/workflows/vol.yml) - This directory contains several test applications that exercise HDF5's public API and serve as regression tests for HDF5 [VOL Connectors](https://portal.hdfgroup.org/display/HDF5/Virtual+Object+Layer). From 9d35ec558b0598cac76773db7263629266976fbb Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Mon, 13 Nov 2023 14:09:50 -0600 Subject: [PATCH 17/18] Disable testing of most connectors against API tests --- .github/workflows/vol_adios2.yml | 5 +++++ .github/workflows/vol_async.yml | 5 +++++ .github/workflows/vol_cache.yml | 6 ++++++ .github/workflows/vol_log.yml | 5 +++++ .github/workflows/vol_rest.yml | 5 +++++ 5 files changed, 26 insertions(+) diff --git a/.github/workflows/vol_adios2.yml b/.github/workflows/vol_adios2.yml index 1de5b039851..a817614fca2 100644 --- a/.github/workflows/vol_adios2.yml +++ b/.github/workflows/vol_adios2.yml @@ -121,5 +121,10 @@ jobs: # Skip parallel testing for now as it appears to hang - name: Test HDF5 ADIOS2 VOL connector with HDF5 API tests working-directory: ${{ github.workspace }}/hdf5/build + # Don't test the ADIOS2 VOL connector with the HDF5 API tests yet, + # as it doesn't currently pass all the tests. Leave the step in, + # but skip it to leave an indication that this should be re-enabled + # in the future. + if: false run: | ctest --build-config ${{ inputs.build_mode }} -VV -R "h5_api" -E "parallel" . diff --git a/.github/workflows/vol_async.yml b/.github/workflows/vol_async.yml index 87ca7815d65..a0479d47288 100644 --- a/.github/workflows/vol_async.yml +++ b/.github/workflows/vol_async.yml @@ -96,5 +96,10 @@ jobs: - name: Test HDF5 asynchronous I/O VOL connector with HDF5 API tests working-directory: ${{ github.workspace }}/hdf5/build + # Don't test the Async VOL connector with the HDF5 API tests yet, + # as it doesn't currently pass all the tests. Leave the step in, + # but skip it to leave an indication that this should be re-enabled + # in the future. + if: false run: | ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-async" . diff --git a/.github/workflows/vol_cache.yml b/.github/workflows/vol_cache.yml index cad2dfde56d..50572e42917 100644 --- a/.github/workflows/vol_cache.yml +++ b/.github/workflows/vol_cache.yml @@ -165,5 +165,11 @@ jobs: - name: Test HDF5 cache VOL connector with HDF5 API tests working-directory: ${{ github.workspace }}/hdf5/build + # Don't test the Cache VOL connector with the HDF5 API tests yet + # when it's stacked on top of the Async connector, as it doesn't + # currently pass all the tests due to the Async connector not passing + # all the tests. Leave the step in, but skip it to leave an indication + # that this should be re-enabled in the future. + if: ${{ ! matrix.async }} run: | ctest --build-config ${{ inputs.build_mode }} -VV -R "HDF5_VOL_vol-cache" . diff --git a/.github/workflows/vol_log.yml b/.github/workflows/vol_log.yml index 2983100e888..787f143fff9 100644 --- a/.github/workflows/vol_log.yml +++ b/.github/workflows/vol_log.yml @@ -91,5 +91,10 @@ jobs: # Skip parallel testing for now as it appears to hang - name: Test HDF5 Log-based VOL connector with HDF5 API tests working-directory: ${{ github.workspace }}/hdf5/build + # Don't test the Log-based VOL connector with the HDF5 API tests yet, + # as it doesn't currently pass all the tests. Leave the step in, + # but skip it to leave an indication that this should be re-enabled + # in the future. + if: false run: | ctest --build-config ${{ inputs.build_mode }} -VV -R "h5_api" -E "parallel" . diff --git a/.github/workflows/vol_rest.yml b/.github/workflows/vol_rest.yml index 975b449f821..1056147b57a 100644 --- a/.github/workflows/vol_rest.yml +++ b/.github/workflows/vol_rest.yml @@ -181,6 +181,11 @@ jobs: - name: Test HDF5 REST VOL connector with HDF5 API tests working-directory: ${{github.workspace}}/hdf5/build/ + # Don't test the REST VOL connector with the HDF5 API tests yet, + # as it doesn't currently pass all the tests. Leave the step in, + # but skip it to leave an indication that this should be re-enabled + # in the future. + if: false run: | sudo \ HDF5_PLUGIN_PATH="${{ runner.workspace }}/hdf5_build/lib" \ From 076ade39b4cd02dc9d6f54f608211f9cf28c31c0 Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Mon, 13 Nov 2023 14:37:36 -0600 Subject: [PATCH 18/18] Replace some paths --- .github/workflows/vol.yml | 14 ++++---------- .github/workflows/vol_adios2.yml | 13 +------------ .github/workflows/vol_async.yml | 13 +------------ .github/workflows/vol_cache.yml | 13 +------------ .github/workflows/vol_ext_passthru.yml | 13 +------------ .github/workflows/vol_log.yml | 13 +------------ .github/workflows/vol_rest.yml | 13 +------------ README.md | 2 +- 8 files changed, 11 insertions(+), 83 deletions(-) diff --git a/.github/workflows/vol.yml b/.github/workflows/vol.yml index 542e4da5d2f..ac2a9f5c553 100644 --- a/.github/workflows/vol.yml +++ b/.github/workflows/vol.yml @@ -3,13 +3,8 @@ name: hdf5 VOL connectors CI # Run VOL connector CI daily at 06:00 CDT (11:00 UTC) on: workflow_dispatch: - # Only for now - push: - # Only for now - # Not yet - #schedule: - # - cron: "0 11 * * *" - # Not yet + schedule: + - cron: "0 11 * * *" permissions: contents: read @@ -19,6 +14,8 @@ jobs: # CMake FetchContent functionality. #hdf5_vol_daos_fetchcontent: # uses: ./.github/workflows/vol_daos.yml + # with: + # build_mode: "Release" hdf5_vol_rest_fetchcontent: uses: ./.github/workflows/vol_rest.yml @@ -49,6 +46,3 @@ jobs: uses: ./.github/workflows/vol_log.yml with: build_mode: "Release" - - # TODO: Build and install HDF5 and test individual VOL - # connectors against the installed API test executables diff --git a/.github/workflows/vol_adios2.yml b/.github/workflows/vol_adios2.yml index a817614fca2..35fde5e3dc2 100644 --- a/.github/workflows/vol_adios2.yml +++ b/.github/workflows/vol_adios2.yml @@ -25,22 +25,11 @@ jobs: sudo apt update sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev - # TODO - #- name: Checkout HDF5 - # uses: actions/checkout@v4 - # with: - # repository: HDFGroup/hdf5 - # path: hdf5 - # TODO - - # TODO - name: Checkout HDF5 uses: actions/checkout@v4 with: - repository: jhendersonHDF/hdf5 + repository: HDFGroup/hdf5 path: hdf5 - ref: api_tests_dev - # TODO - name: Configure HDF5 shell: bash diff --git a/.github/workflows/vol_async.yml b/.github/workflows/vol_async.yml index a0479d47288..bb4c3a18953 100644 --- a/.github/workflows/vol_async.yml +++ b/.github/workflows/vol_async.yml @@ -21,22 +21,11 @@ jobs: sudo apt update sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev - # TODO - #- name: Checkout HDF5 - # uses: actions/checkout@v4 - # with: - # repository: HDFGroup/hdf5 - # path: hdf5 - # TODO - - # TODO - name: Checkout HDF5 uses: actions/checkout@v4 with: - repository: jhendersonHDF/hdf5 + repository: HDFGroup/hdf5 path: hdf5 - ref: api_tests_dev - # TODO - name: Checkout Argobots uses: actions/checkout@v4 diff --git a/.github/workflows/vol_cache.yml b/.github/workflows/vol_cache.yml index 50572e42917..1a8c40cfdf2 100644 --- a/.github/workflows/vol_cache.yml +++ b/.github/workflows/vol_cache.yml @@ -34,22 +34,11 @@ jobs: sudo apt update sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev - # TODO - #- name: Checkout HDF5 - # uses: actions/checkout@v4 - # with: - # repository: HDFGroup/hdf5 - # path: hdf5 - # TODO - - # TODO - name: Checkout HDF5 uses: actions/checkout@v4 with: - repository: jhendersonHDF/hdf5 + repository: HDFGroup/hdf5 path: hdf5 - ref: api_tests_dev - # TODO - name: Checkout Argobots uses: actions/checkout@v4 diff --git a/.github/workflows/vol_ext_passthru.yml b/.github/workflows/vol_ext_passthru.yml index caafeb32715..337130bc263 100644 --- a/.github/workflows/vol_ext_passthru.yml +++ b/.github/workflows/vol_ext_passthru.yml @@ -21,22 +21,11 @@ jobs: sudo apt update sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev - # TODO - #- name: Checkout HDF5 - # uses: actions/checkout@v4 - # with: - # repository: HDFGroup/hdf5 - # path: hdf5 - # TODO - - # TODO - name: Checkout HDF5 uses: actions/checkout@v4 with: - repository: jhendersonHDF/hdf5 + repository: HDFGroup/hdf5 path: hdf5 - ref: api_tests_dev - # TODO - name: Checkout vol-external-passthrough uses: actions/checkout@v4 diff --git a/.github/workflows/vol_log.yml b/.github/workflows/vol_log.yml index 787f143fff9..0a355782eef 100644 --- a/.github/workflows/vol_log.yml +++ b/.github/workflows/vol_log.yml @@ -22,22 +22,11 @@ jobs: sudo apt-get install automake autoconf libtool libtool-bin libopenmpi-dev zlib1g-dev #mpich - # TODO - #- name: Checkout HDF5 - # uses: actions/checkout@v4 - # with: - # repository: HDFGroup/hdf5 - # path: hdf5 - # TODO - - # TODO - name: Checkout HDF5 uses: actions/checkout@v4 with: - repository: jhendersonHDF/hdf5 + repository: HDFGroup/hdf5 path: hdf5 - ref: api_tests_dev - # TODO # Log-based VOL currently doesn't have CMake support - name: Configure HDF5 diff --git a/.github/workflows/vol_rest.yml b/.github/workflows/vol_rest.yml index 1056147b57a..188e80dca79 100644 --- a/.github/workflows/vol_rest.yml +++ b/.github/workflows/vol_rest.yml @@ -41,22 +41,11 @@ jobs: sudo apt-get update sudo apt-get install automake autoconf libtool libtool-bin libcurl4-openssl-dev libyajl-dev - # TODO - #- name: Checkout HDF5 - # uses: actions/checkout@v4 - # with: - # repository: HDFGroup/hdf5 - # path: hdf5 - # TODO - - # TODO - name: Checkout HDF5 uses: actions/checkout@v4 with: - repository: jhendersonHDF/hdf5 + repository: HDFGroup/hdf5 path: hdf5 - ref: api_tests_dev - # TODO - name: Configure HDF5 with REST VOL connector shell: bash diff --git a/README.md b/README.md index e76e4c63e6d..eb413bfa7a5 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ HDF5 version 1.15.0 currently under development [![netCDF build status](https://img.shields.io/github/actions/workflow/status/HDFGroup/hdf5/netcdf.yml?branch=develop&label=netCDF)](https://github.com/HDFGroup/hdf5/actions?query=branch%3Adevelop) [![h5py build status](https://img.shields.io/github/actions/workflow/status/HDFGroup/hdf5/h5py.yml?branch=develop&label=h5py)](https://github.com/HDFGroup/hdf5/actions?query=branch%3Adevelop) [![CVE regression](https://img.shields.io/github/actions/workflow/status/HDFGroup/hdf5/cve.yml?branch=develop&label=CVE)](https://github.com/HDFGroup/hdf5/actions?query=branch%3Adevelop) -[![HDF5 VOL connectors build status](https://github.com/jhendersonHDF/hdf5/actions/workflows/vol.yml/badge.svg)](https://github.com/jhendersonHDF/hdf5/actions/workflows/vol.yml) +[![HDF5 VOL connectors build status](https://img.shields.io/github/actions/workflow/status/HDFGroup/hdf5/vol.yml?branch=develop&label=HDF5-VOL)](https://github.com/HDFGroup/hdf5/actions/workflows/vol.yml?query=branch%3Adevelop) [![1.14 build status](https://img.shields.io/github/actions/workflow/status/HDFGroup/hdf5/main.yml?branch=hdf5_1_14&label=1.14)](https://github.com/HDFGroup/hdf5/actions?query=branch%3Ahdf5_1_14) [![BSD](https://img.shields.io/badge/License-BSD-blue.svg)](https://github.com/HDFGroup/hdf5/blob/develop/COPYING)