Merge remote-tracking branch 'origin/develop' into threading_updates

qkoziol · May 3, 2024 · bc932b2 · bc932b2
2 parents 081c478 + e4f079e
commit bc932b2
Show file tree

Hide file tree

Showing 13 changed files with 258 additions and 202 deletions.
diff --git a/.github/workflows/nvhpc-cmake.yml b/.github/workflows/nvhpc-cmake.yml
@@ -59,6 +59,7 @@ jobs:
           -DHDF5_BUILD_FORTRAN:BOOL=ON \
           -DHDF5_BUILD_JAVA:BOOL=OFF \
           -DMPIEXEC_MAX_NUMPROCS:STRING="2" \
+          -DMPIEXEC_PREFLAGS:STRING="--mca;opal_warn_on_missing_libcuda;0" \
           $GITHUB_WORKSPACE
 
       - name: CMake Build

diff --git a/config/cmake/grepTest.cmake b/config/cmake/grepTest.cmake
@@ -11,6 +11,7 @@
 #
 # grepTest.cmake executes a command and captures the output in a file. File is then compared
 # against a reference file. Exit status of command can also be compared.
+cmake_policy(SET CMP0007 NEW)
 
 # arguments checking
 if (NOT TEST_PROGRAM)
@@ -67,6 +68,11 @@ execute_process (
 
 message (STATUS "COMMAND Result: ${TEST_RESULT}")
 
+# append the test result status with a predefined text
+if (TEST_APPEND)
+  file (APPEND ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_APPEND} ${TEST_RESULT}\n")
+endif ()
+
 message (STATUS "COMMAND Error: ${TEST_ERROR}")
 
 # remove special output
@@ -75,15 +81,22 @@ if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}")
   string (FIND "${TEST_STREAM}" "_pmi_alps" TEST_FIND_RESULT)
   if (TEST_FIND_RESULT GREATER -1)
     string (REGEX REPLACE "^.*_pmi_alps[^\n]+\n" "" TEST_STREAM "${TEST_STREAM}")
-    file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}")
+    file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_STREAM})
   endif ()
   string (FIND "${TEST_STREAM}" "ulimit -s" TEST_FIND_RESULT)
   if (TEST_FIND_RESULT GREATER -1)
     string (REGEX REPLACE "^.*ulimit -s[^\n]+\n" "" TEST_STREAM "${TEST_STREAM}")
-    file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}")
+    file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_STREAM})
   endif ()
 endif ()
 
+if (TEST_REF_FILTER)
+  #message (STATUS "TEST_REF_FILTER: ${TEST_APPEND}${TEST_REF_FILTER}")
+  file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM)
+  string (REGEX REPLACE "${TEST_REF_APPEND}" "${TEST_REF_FILTER}" TEST_STREAM "${TEST_STREAM}")
+  file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}")
+endif ()
+
 # if the TEST_ERRREF exists grep the error output with the error reference
 set (TEST_ERRREF_RESULT 0)
 if (TEST_ERRREF)
@@ -96,26 +109,25 @@ if (TEST_ERRREF)
       string (REGEX MATCH "${TEST_ERRREF}" TEST_MATCH ${TEST_ERR_STREAM})
       string (COMPARE EQUAL "${TEST_ERRREF}" "${TEST_MATCH}" TEST_ERRREF_RESULT)
       if (NOT TEST_ERRREF_RESULT)
+        # dump the output unless nodisplay option is set
+        if (NOT TEST_NO_DISPLAY)
+          execute_process (
+              COMMAND ${CMAKE_COMMAND} -E echo ${TEST_ERR_STREAM}
+              RESULT_VARIABLE TEST_ERRREF_RESULT
+          )
+        endif ()
         message (FATAL_ERROR "Failed: The error output of ${TEST_PROGRAM} did not contain ${TEST_ERRREF}")
       endif ()
     endif ()
   endif ()
 
-  #always compare output file to reference unless this must be skipped
+  # compare output files to references unless this must be skipped
   set (TEST_COMPARE_RESULT 0)
   if (NOT TEST_SKIP_COMPARE)
     if (EXISTS "${TEST_FOLDER}/${TEST_REFERENCE}")
       file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM)
       list (LENGTH TEST_STREAM test_len)
       if (test_len GREATER 0)
-        if (WIN32)
-          configure_file(${TEST_FOLDER}/${TEST_REFERENCE} ${TEST_FOLDER}/${TEST_REFERENCE}.tmp NEWLINE_STYLE CRLF)
-          if (EXISTS "${TEST_FOLDER}/${TEST_REFERENCE}.tmp")
-            file(RENAME ${TEST_FOLDER}/${TEST_REFERENCE}.tmp ${TEST_FOLDER}/${TEST_REFERENCE})
-          endif ()
-          #file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM)
-          #file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}")
-        endif ()
         if (NOT TEST_SORT_COMPARE)
           # now compare the output with the reference
           execute_process (
@@ -138,7 +150,14 @@ if (TEST_ERRREF)
           list (LENGTH test_act len_act)
           file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} test_ref)
           list (LENGTH test_ref len_ref)
+          if (NOT len_act EQUAL len_ref)
+            set (TEST_COMPARE_RESULT 1)
+          endif ()
           if (len_act GREATER 0 AND len_ref GREATER 0)
+            if (TEST_SORT_COMPARE)
+              list (SORT test_act)
+              list (SORT test_ref)
+            endif ()
             math (EXPR _FP_LEN "${len_ref} - 1")
             foreach (line RANGE 0 ${_FP_LEN})
               list (GET test_act ${line} str_act)
@@ -200,7 +219,7 @@ if (TEST_FILTER)
 endif ()
 
 if (NOT DEFINED ENV{HDF5_NOCLEANUP})
-  if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}")
+  if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}" AND NOT TEST_SAVE)
     file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT})
   endif ()
 

diff --git a/config/cmake/runTest.cmake b/config/cmake/runTest.cmake
@@ -49,7 +49,7 @@ endif ()
 
 if (TEST_ENV_VAR)
   set (ENV{${TEST_ENV_VAR}} "${TEST_ENV_VALUE}")
-  #message (STATUS "ENV:${TEST_ENV_VAR}=$ENV{${TEST_ENV_VAR}}")
+  message (TRACE "ENV:${TEST_ENV_VAR}=$ENV{${TEST_ENV_VAR}}")
 endif ()
 
 if (NOT TEST_INPUT)
@@ -233,7 +233,6 @@ if (NOT TEST_SKIP_COMPARE)
     file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM)
     list (LENGTH TEST_STREAM test_len)
     if (test_len GREATER 0)
-
       if (NOT TEST_SORT_COMPARE)
         # now compare the output with the reference
         execute_process (

diff --git a/fortran/src/H5Rff.F90 b/fortran/src/H5Rff.F90
@@ -1111,7 +1111,7 @@ SUBROUTINE h5rget_obj_name_f( ref_ptr, name, hdferr, rapl_id, name_len)
     INTEGER(SIZE_T) :: l
 
     INTERFACE
-       INTEGER FUNCTION H5Rget_obj_name(ref_ptr, rapl_id, name, size_default) &
+       INTEGER(SIZE_T) FUNCTION H5Rget_obj_name(ref_ptr, rapl_id, name, size_default) &
             BIND(C, NAME='H5Rget_obj_name')
          IMPORT :: c_char, c_ptr
          IMPORT :: HID_T, SIZE_T
@@ -1164,7 +1164,7 @@ SUBROUTINE h5rget_attr_name_f(ref_ptr, name, hdferr, name_len)
     INTEGER(SIZE_T) :: l
 
     INTERFACE
-       INTEGER FUNCTION H5Rget_attr_name(ref_ptr, name, size_default) &
+       INTEGER(SIZE_T) FUNCTION H5Rget_attr_name(ref_ptr, name, size_default) &
             BIND(C, NAME='H5Rget_attr_name')
          IMPORT :: c_char, c_ptr
          IMPORT :: HID_T, SIZE_T
@@ -1217,7 +1217,7 @@ SUBROUTINE h5rget_file_name_f(ref_ptr, name, hdferr, name_len)
     INTEGER(SIZE_T) :: l
 
     INTERFACE
-       INTEGER FUNCTION H5Rget_file_name(ref_ptr, name, size_default) &
+       INTEGER(SIZE_T) FUNCTION H5Rget_file_name(ref_ptr, name, size_default) &
             BIND(C, NAME='H5Rget_file_name')
          IMPORT :: c_char, c_ptr
          IMPORT :: HID_T, SIZE_T

diff --git a/fortran/test/tH5R.F90 b/fortran/test/tH5R.F90
@@ -650,7 +650,7 @@ SUBROUTINE v3reftest(cleanup, total_error)
 
   CALL h5rget_obj_name_f(C_LOC(ref_ptr(3)), "", error, H5P_DEFAULT_F, buf_size)
   CALL check("h5rget_obj_name_f", error, total_error)
-  CALL verify("h5rget_obj_name_f", buf_size, LEN(dsetnamei,KIND=SIZE_T)+1, total_error)
+  CALL verify("h5rget_obj_name_f", buf_size, LEN(dsetnamei,KIND=SIZE_T)+1_SIZE_T, total_error)
   CALL h5rget_obj_name_f(C_LOC(ref_ptr(1)), "", error, H5P_DEFAULT_F, buf_size)
   CALL check("h5rget_obj_name_f", error, total_error)
   CALL verify("h5rget_obj_name_f", buf_size, 7_SIZE_T, total_error)

diff --git a/tools/test/h5copy/CMakeTests.cmake b/tools/test/h5copy/CMakeTests.cmake
@@ -30,7 +30,6 @@
 
   set (LIST_OTHER_TEST_FILES
       h5copy_misc1.out
-      h5copy_misc1.err
       tudfilter.h5.txt
       tudfilter.h5_ERR.txt
       h5copy_plugin_fail_ERR.out.h5.txt

diff --git a/tools/test/h5diff/CMakeTests.cmake b/tools/test/h5diff/CMakeTests.cmake
@@ -192,7 +192,6 @@
       h5diff_452.txt
       h5diff_453.txt
       h5diff_454.txt
-      dangling_link.err
       h5diff_455.txt
       h5diff_456.txt
       h5diff_457.txt
@@ -254,7 +253,6 @@
       h5diff_63.txt
       h5diff_600.txt
       h5diff_601.txt
-      h5diff_601_ERR.err
       h5diff_603.txt
       h5diff_604.txt
       h5diff_605.txt
@@ -313,7 +311,6 @@
       h5diff_8639.txt
       h5diff_reg.txt
       h5diff_ud.txt
-      h5diff_udfail.err
       h5diff_udfail.txt
       h5diff_v1.txt
       h5diff_v2.txt
@@ -477,6 +474,82 @@
     endif ()
   endmacro ()
 
+  macro (ADD_H5_CMP_TEST resultfile resultcode result_errcheck)
+    if (HDF5_TEST_SERIAL)
+      ADD_SH5_CMP_TEST (${resultfile} ${resultcode} ${result_errcheck} ${ARGN})
+    endif ()
+    if (H5_HAVE_PARALLEL AND HDF5_TEST_PARALLEL)
+      ADD_PH5_CMP_TEST (${resultfile} ${resultcode} ${result_errcheck} ${ARGN})
+    endif ()
+  endmacro ()
+
+  macro (ADD_SH5_CMP_TEST resultfile resultcode result_errcheck)
+    # If using memchecker add tests without using scripts
+    if (HDF5_USING_ANALYSIS_TOOL)
+      add_test (NAME H5DIFF-${resultfile} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5diff> ${ARGN})
+      if (${resultcode})
+        set_tests_properties (H5DIFF-${resultfile} PROPERTIES WILL_FAIL "true")
+      endif ()
+    else ()
+      add_test (
+          NAME H5DIFF-${resultfile}
+          COMMAND "${CMAKE_COMMAND}"
+              -D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}"
+              -D "TEST_PROGRAM=$<TARGET_FILE:h5diff>"
+              -D "TEST_ARGS:STRING=${ARGN}"
+              -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles"
+              -D "TEST_OUTPUT=${resultfile}.out"
+              -D "TEST_EXPECT=${resultcode}"
+              -D "TEST_REFERENCE=${resultfile}.txt"
+              -D "TEST_ERRREF=${result_errcheck}"
+              -D "TEST_APPEND=EXIT CODE:"
+              -P "${HDF_RESOURCES_DIR}/grepTest.cmake"
+      )
+    endif ()
+    set_tests_properties (H5DIFF-${resultfile} PROPERTIES
+        WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles"
+    )
+    if ("H5DIFF-${resultfile}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
+      set_tests_properties (H5DIFF-${resultfile} PROPERTIES DISABLED true)
+    endif ()
+  endmacro ()
+
+  macro (ADD_PH5_CMP_TEST resultfile resultcode result_errcheck)
+    # If using memchecker add tests without using scripts
+    if (HDF5_USING_ANALYSIS_TOOL)
+      add_test (NAME MPI_TEST_H5DIFF-${resultfile} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} $<TARGET_FILE:ph5diff> ${MPIEXEC_POSTFLAGS} ${ARGN})
+      set_tests_properties (MPI_TEST_H5DIFF-${resultfile} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/PAR/testfiles")
+      if (${resultcode})
+        set_tests_properties (MPI_TEST_H5DIFF-${resultfile} PROPERTIES WILL_FAIL "true")
+      endif ()
+    else ()
+      add_test (
+          NAME MPI_TEST_H5DIFF-${resultfile}
+          COMMAND "${CMAKE_COMMAND}"
+              -D "TEST_PROGRAM=${MPIEXEC_EXECUTABLE}"
+              -D "TEST_ARGS:STRING=${MPIEXEC_NUMPROC_FLAG};${MPIEXEC_MAX_NUMPROCS};${MPIEXEC_PREFLAGS};$<TARGET_FILE:ph5diff>;${MPIEXEC_POSTFLAGS};${ARGN}"
+              -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/PAR/testfiles"
+              -D "TEST_OUTPUT=${resultfile}.out"
+              #-D "TEST_EXPECT=${resultcode}"
+              -D "TEST_EXPECT=0" # ph5diff currently always exits with a zero status code due to
+                                 # output from some MPI implementations from a non-zero exit code
+              -D "TEST_REFERENCE=${resultfile}.txt"
+              -D "TEST_ERRREF=${result_errcheck}"
+              -D "TEST_APPEND=EXIT CODE:"
+              -D "TEST_REF_APPEND=EXIT CODE: [0-9]"
+              -D "TEST_REF_FILTER=EXIT CODE: 0"
+              -D "TEST_SORT_COMPARE=TRUE"
+              -P "${HDF_RESOURCES_DIR}/grepTest.cmake"
+      )
+    endif ()
+    set_tests_properties (MPI_TEST_H5DIFF-${resultfile} PROPERTIES
+        WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/PAR/testfiles"
+    )
+    if ("MPI_TEST_H5DIFF-${resultfile}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
+      set_tests_properties (MPI_TEST_H5DIFF-${resultfile} PROPERTIES DISABLED true)
+    endif ()
+  endmacro ()
+
   macro (ADD_H5_UD_TEST testname resultcode resultfile)
     if (NOT HDF5_USING_ANALYSIS_TOOL)
       if ("${resultcode}" STREQUAL "2")
@@ -490,11 +563,12 @@
                 -D "TEST_OUTPUT=${resultfile}.out"
                 -D "TEST_EXPECT=${resultcode}"
                 -D "TEST_REFERENCE=${resultfile}.txt"
+                -D "TEST_ERRREF=user defined filter is not available"
                 -D "TEST_APPEND=EXIT CODE:"
                 -D "TEST_ENV_VAR=HDF5_PLUGIN_PATH"
                 -D "TEST_ENV_VALUE=${CMAKE_BINARY_DIR}"
                 -D "TEST_LIBRARY_DIRECTORY=${CMAKE_TEST_OUTPUT_DIRECTORY}"
-                -P "${HDF_RESOURCES_DIR}/runTest.cmake"
+                -P "${HDF_RESOURCES_DIR}/grepTest.cmake"
         )
       else ()
         add_test (
@@ -734,7 +808,7 @@ ADD_H5_TEST (h5diff_63 1 -v ${STRINGS1} ${STRINGS2} string4 string4)
 ADD_H5_TEST (h5diff_600 1 ${FILE1})
 
 # 6.1: Check if non-exist object name is specified
-ADD_H5_TEST (h5diff_601 2 ${FILE1} ${FILE1} nono_obj)
+ADD_H5_CMP_TEST (h5diff_601 2 "Object could not be found" ${FILE1} ${FILE1} nono_obj)
 
 # ##############################################################################
 # # -d

diff --git a/tools/test/h5dump/CMakeTests.cmake b/tools/test/h5dump/CMakeTests.cmake
@@ -370,33 +370,6 @@
       tst_onion_dset_1d.h5
       tst_onion_dset_1d.h5.onion
   )
-  set (HDF5_ERROR_REFERENCE_TEST_FILES
-      filter_fail.err
-      non_existing.err
-      infinite_loop.err
-      tall-1.err
-      tall-2A.err
-      tall-2A0.err
-      tall-2B.err
-      tarray1_big.err
-      tattrregR.err
-      tattr-3.err
-      tcomp-3.err
-      tdataregR.err
-      tdset-2.err
-      texceedsubblock.err
-      texceedsubcount.err
-      texceedsubstart.err
-      texceedsubstride.err
-      textlink.err
-      textlinkfar.err
-      textlinksrc.err
-      torderlinks1.err
-      torderlinks2.err
-      tgroup-2.err
-      tperror.err
-      tslink-D.err
-  )
 
   # make test dir
   file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles")
@@ -420,10 +393,6 @@
     HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/expected/${tst_h5N_file}" "${PROJECT_BINARY_DIR}/testfiles/std/${tst_h5N_file}-N" "h5dump_std_files")
   endforeach ()
 
-  foreach (tst_error_file ${HDF5_ERROR_REFERENCE_TEST_FILES})
-    HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/errfiles/${tst_error_file}" "${PROJECT_BINARY_DIR}/testfiles/std/${tst_error_file}" "h5dump_std_files")
-  endforeach ()
-
   # --------------------------------------------------------------------
   # Special file handling
   # --------------------------------------------------------------------
@@ -1269,10 +1238,10 @@
   ADD_H5_TEST (tindicessub4 0 --enable-error-stack -d 4d -s 0,0,1,2  -c 2,2,3,2 -S 1,1,3,3 -k 1,1,2,2  taindices.h5)
 
   # Exceed the dimensions for subsetting
-  ADD_H5_TEST (texceedsubstart 1 --enable-error-stack -d 1d -s 1,3 taindices.h5)
-  ADD_H5_TEST (texceedsubcount 1 --enable-error-stack -d 1d -c 1,3 taindices.h5)
-  ADD_H5_TEST (texceedsubstride 1 --enable-error-stack -d 1d -S 1,3 taindices.h5)
-  ADD_H5_TEST (texceedsubblock 1 --enable-error-stack -d 1d -k 1,3 taindices.h5)
+  ADD_H5ERR_MASK_TEST (texceedsubstart 1 "exceed dataset dims" --enable-error-stack -d 1d -s 1,3 taindices.h5)
+  ADD_H5ERR_MASK_TEST (texceedsubcount 1 "exceed dataset dims" --enable-error-stack -d 1d -c 1,3 taindices.h5)
+  ADD_H5ERR_MASK_TEST (texceedsubstride 1 "exceed dataset dims" --enable-error-stack -d 1d -S 1,3 taindices.h5)
+  ADD_H5ERR_MASK_TEST (texceedsubblock 1 "exceed dataset dims" --enable-error-stack -d 1d -k 1,3 taindices.h5)
 
   # tests for filters
   # SZIP
@@ -1424,24 +1393,24 @@
   ADD_H5_TEST_EXPORT (tall-6 tall.h5 0 --enable-error-stack -d /g1/g1.1/dset1.1.1 -y -o)
 
   # test for non-existing file
-  ADD_H5_TEST (non_existing 1 --enable-error-stack tgroup.h5 non_existing.h5)
+  ADD_H5ERR_MASK_TEST (non_existing 1 "unable to open file" --enable-error-stack tgroup.h5 non_existing.h5)
 
   # test to verify github issue#3790: infinite loop closing library
-  ADD_H5_TEST (infinite_loop 1 3790_infinite_loop.h5)
+  ADD_H5ERR_MASK_TEST (infinite_loop 1 "unable to open file" 3790_infinite_loop.h5)
 
   # test to verify HDFFV-10333: error similar to H5O_attr_decode in the jira issue
-  ADD_H5_TEST (err_attr_dspace 1 err_attr_dspace.h5)
+  ADD_H5ERR_MASK_TEST (err_attr_dspace 1 "error getting attribute information" err_attr_dspace.h5)
 
   # test to verify HDFFV-9407: long double full precision
 #  ADD_H5_GREP_TEST (t128bit_float 1 "1.123456789012345" -m %.35Lg t128bit_float.h5)
 
   # test to verify HDFFV-10480: out of bounds read in H5O_fill_new[old]_decode
-  ADD_H5_TEST (tCVE_2018_11206_fill_old 1 tCVE_2018_11206_fill_old.h5)
-  ADD_H5_TEST (tCVE_2018_11206_fill_new 1 tCVE_2018_11206_fill_new.h5)
+  ADD_H5ERR_MASK_TEST (tCVE_2018_11206_fill_old 1 "" tCVE_2018_11206_fill_old.h5)
+  ADD_H5ERR_MASK_TEST (tCVE_2018_11206_fill_new 1 "" tCVE_2018_11206_fill_new.h5)
 
   # test to verify fix for CVE-2021-37501: multiplication overflow in H5O__attr_decode()
   # https://github.com/ST4RF4LL/Something_Found/blob/main/HDF5_v1.13.0_h5dump_heap_overflow.assets/poc
-  ADD_H5_TEST (tCVE-2021-37501_attr_decode 1 tCVE-2021-37501_attr_decode.h5)
+  ADD_H5ERR_MASK_TEST (tCVE-2021-37501_attr_decode 1 "error getting attribute information" tCVE-2021-37501_attr_decode.h5)
 
   # onion VFD tests
   ADD_H5_TEST (tst_onion_objs 0 --enable-error-stack --vfd-name onion --vfd-info 3 tst_onion_objs.h5)