From e01ea706e21b323d17b7b5c6cdaad22602b5573c Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Fri, 20 Oct 2023 12:46:26 -0700
Subject: [PATCH 01/52] Add missing test files to distclean target (#3734)
Cleans up new files in Autotools `make distclean` in the test directory
---
test/Makefile.am | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/test/Makefile.am b/test/Makefile.am
index 291907ca0f9..9fd7b94625c 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -192,7 +192,7 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 mdset.h5 compact_dataset.h5 dataset.h5 d
storage_size.h5 dls_01_strings.h5 power2up.h5 version_bounds.h5 \
alloc_0sized.h5 h5s_block.h5 h5s_plist.h5 \
extend.h5 istore.h5 extlinks*.h5 frspace.h5 links*.h5 \
- sys_file1 tfile[1-7].h5 th5s[1-4].h5 lheap.h5 fheap.h5 ohdr.h5 \
+ sys_file1 tfile[1-8].h5 th5s[1-4].h5 lheap.h5 fheap.h5 ohdr.h5 \
stab.h5 extern_[1-5].h5 extern_[1-4][rw].raw gheap[0-4].h5 \
ohdr_min_a.h5 ohdr_min_b.h5 min_dset_ohdr_testfile.h5 \
dt_arith[1-2] links.h5 links[0-6]*.h5 extlinks[0-15].h5 \
@@ -226,7 +226,10 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 mdset.h5 compact_dataset.h5 dataset.h5 d
test_swmr*.h5 cache_logging.h5 cache_logging.out vds_swmr.h5 vds_swmr_src_*.h5 \
swmr[0-2].h5 swmr_writer.out swmr_writer.log.* swmr_reader.out.* swmr_reader.log.* \
tbogus.h5.copy cache_image_test.h5 direct_chunk.h5 native_vol_test.h5 \
- splitter*.h5 splitter.log mirror_rw mirror_ro event_set_[0-9].h5
+ splitter*.h5 splitter.log mirror_rw mirror_ro event_set_[0-9].h5 \
+ cmpd_dtransform.h5 single_latest.h5 source_file.h5 stdio_file.h5 \
+ tfile_is_accessible.h5 tfile_is_accessible_non_hdf5.h5 tverbounds_dtype.h5 \
+ virtual_file1.h5
# Sources for testhdf5 executable
testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \
From 98b5779ea54edd011d1d3197117ed0571ec7ad6c Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Fri, 20 Oct 2023 13:50:57 -0700
Subject: [PATCH 02/52] Add tools/libtest to Autotools builds (#3735)
This was only added to CMake many years ago and tests the tools
library.
---
hl/test/Makefile.am | 2 +-
tools/Makefile.am | 2 +-
tools/libtest/Makefile.am | 8 ++++----
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/hl/test/Makefile.am b/hl/test/Makefile.am
index 1d1cb0f92fb..6f66291b25d 100644
--- a/hl/test/Makefile.am
+++ b/hl/test/Makefile.am
@@ -20,7 +20,7 @@ include $(top_srcdir)/config/commence.am
# Add include directories to C preprocessor flags
AM_CPPFLAGS+=-I. -I$(srcdir) -I$(top_builddir)/src -I$(top_srcdir)/src -I$(top_builddir)/test -I$(top_srcdir)/test -I$(top_srcdir)/hl/src
-# The tests depend on the hdf5, hdf5 test, and hdf5_hl libraries
+# The tests depend on the hdf5, hdf5 test, and hdf5_hl libraries
LDADD=$(LIBH5_HL) $(LIBH5TEST) $(LIBHDF5)
# Test programs. These are our main targets. They should be listed in the
diff --git a/tools/Makefile.am b/tools/Makefile.am
index 7db4040d0eb..d0a6c5c5bc4 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -19,7 +19,7 @@
include $(top_srcdir)/config/commence.am
if BUILD_TESTS_CONDITIONAL
- TESTSERIAL_DIR =test
+ TESTSERIAL_DIR=libtest test
else
TESTSERIAL_DIR=
endif
diff --git a/tools/libtest/Makefile.am b/tools/libtest/Makefile.am
index 835667c74b0..45b3f476df7 100644
--- a/tools/libtest/Makefile.am
+++ b/tools/libtest/Makefile.am
@@ -19,11 +19,11 @@
include $(top_srcdir)/config/commence.am
-# Include src and tools/lib directories
-AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib
+# Include src, test, and tools/lib directories
+AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib
-# All programs depend on the hdf5 and h5tools libraries
-LDADD=$(LIBH5TOOLS) $(LIBHDF5)
+# All programs depend on the hdf5, hdf5 test, and h5tools libraries
+LDADD=$(LIBH5TOOLS) $(LIBH5TEST) $(LIBHDF5)
# main target
From 7843db55ff046ba2d044647add5f1861bfdd33bc Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Sat, 21 Oct 2023 11:44:03 -0700
Subject: [PATCH 03/52] Clean up onion VFD files in tools `make clean` (#3739)
Cleans up h5dump and h5diff *.onion files in the Autotools when
runing `make clean`.
---
tools/test/h5diff/Makefile.am | 3 +--
tools/test/h5dump/Makefile.am | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/tools/test/h5diff/Makefile.am b/tools/test/h5diff/Makefile.am
index b561d722275..f920afab74a 100644
--- a/tools/test/h5diff/Makefile.am
+++ b/tools/test/h5diff/Makefile.am
@@ -60,8 +60,7 @@ endif
# Temporary files. *.h5 are generated by h5diff. They should
# be copied to the testfiles/ directory if update is required
-CHECK_CLEANFILES+=*.h5 expect_sorted actual_sorted
-
+CHECK_CLEANFILES+=*.h5 *.onion expect_sorted actual_sorted
DISTCLEANFILES=testh5diff.sh h5diff_plugin.sh
include $(top_srcdir)/config/conclude.am
diff --git a/tools/test/h5dump/Makefile.am b/tools/test/h5dump/Makefile.am
index a79b0fe8b73..619647c670e 100644
--- a/tools/test/h5dump/Makefile.am
+++ b/tools/test/h5dump/Makefile.am
@@ -45,7 +45,7 @@ endif
# Temporary files. *.h5 are generated by h5dumpgentest. They should
# copied to the testfiles/ directory if update is required.
-CHECK_CLEANFILES+=*.h5 *.bin
+CHECK_CLEANFILES+=*.h5 *.bin *.onion
DISTCLEANFILES=testh5dump.sh testh5dumppbits.sh testh5dumpxml.sh h5dump_plugin.sh
include $(top_srcdir)/config/conclude.am
From 29b27b77619d7260aa39e45e1c04369f32f094ca Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Sat, 21 Oct 2023 11:44:16 -0700
Subject: [PATCH 04/52] Clean Java test files on Autotools (#3740)
Removes generated HDF5 and text output files when running `make clean`.
---
java/test/Makefile.am | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/java/test/Makefile.am b/java/test/Makefile.am
index 9f39be9ca1c..7f6ab0169a8 100644
--- a/java/test/Makefile.am
+++ b/java/test/Makefile.am
@@ -90,7 +90,8 @@ noinst_DATA = $(jarfile)
check_SCRIPTS = junit.sh
TEST_SCRIPT = $(check_SCRIPTS)
-CLEANFILES = classnoinst.stamp $(jarfile) $(JAVAROOT)/$(pkgpath)/*.class junit.sh
+CLEANFILES = classnoinst.stamp $(jarfile) $(JAVAROOT)/$(pkgpath)/*.class junit.sh \
+ *.h5 testExport*.txt
clean:
rm -rf $(JAVAROOT)/*
From 4b2d2eccdd8be50814ca2ed5e37e6d77b5a50340 Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Sat, 21 Oct 2023 11:44:33 -0700
Subject: [PATCH 05/52] Clean the flushrefresh test dir on Autotools (#3741)
The flushrefresh_test directory was not being cleaned up w/
`make clean` under the Autotools
---
test/Makefile.am | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/test/Makefile.am b/test/Makefile.am
index 9fd7b94625c..3146e0174d0 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -229,7 +229,7 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 mdset.h5 compact_dataset.h5 dataset.h5 d
splitter*.h5 splitter.log mirror_rw mirror_ro event_set_[0-9].h5 \
cmpd_dtransform.h5 single_latest.h5 source_file.h5 stdio_file.h5 \
tfile_is_accessible.h5 tfile_is_accessible_non_hdf5.h5 tverbounds_dtype.h5 \
- virtual_file1.h5
+ virtual_file1.h5 flushrefresh_test
# Sources for testhdf5 executable
testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \
From 4dfde6bac7395386379072c532560fbe8a5b6f79 Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Sat, 21 Oct 2023 17:08:12 -0700
Subject: [PATCH 06/52] Fix file names in tfile.c (#3743)
Some tests in tfile.c use h5_fileaccess to get a VFD-dependent file
name but use the scheme from testhdf5, reusing the FILE1 and FILE8
names. This leads to files like test1.h5.h5 which are unintended
and not cleaned up.
This changes the filename scheme for a few tests to work with h5test,
resulting in more informative names and allowing the files to
be cleaned up at the end of the test. The test files have also
been added to the `make clean` target for the Autotools.
---
test/Makefile.am | 2 +-
test/tfile.c | 21 +++++++++++++++++----
2 files changed, 18 insertions(+), 5 deletions(-)
diff --git a/test/Makefile.am b/test/Makefile.am
index 3146e0174d0..fdd83e5bdeb 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -229,7 +229,7 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 mdset.h5 compact_dataset.h5 dataset.h5 d
splitter*.h5 splitter.log mirror_rw mirror_ro event_set_[0-9].h5 \
cmpd_dtransform.h5 single_latest.h5 source_file.h5 stdio_file.h5 \
tfile_is_accessible.h5 tfile_is_accessible_non_hdf5.h5 tverbounds_dtype.h5 \
- virtual_file1.h5 flushrefresh_test
+ virtual_file1.h5 tfile_double_open.h5 tfile_incr_filesize.h5 flushrefresh_test
# Sources for testhdf5 executable
testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \
diff --git a/test/tfile.c b/test/tfile.c
index 1c5196acf60..24cc7ce000e 100644
--- a/test/tfile.c
+++ b/test/tfile.c
@@ -138,9 +138,15 @@
#define NGROUPS 2
#define NDSETS 4
-/* Declaration for test_incr_filesize() */
+/* Declaration for libver bounds tests */
#define FILE8 "tfile8.h5" /* Test file */
+/* Declaration for test_file_double_file_dataset_open() */
+#define FILE_DOUBLE_OPEN "tfile_double_open"
+
+/* Declaration for test_incr_filesize() */
+#define FILE_INCR_FILESIZE "tfile_incr_filesize"
+
/* Files created under 1.6 branch and 1.8 branch--used in test_filespace_compatible() */
static const char *OLD_FILENAME[] = {
"filespace_1_6.h5", /* 1.6 HDF5 file */
@@ -2623,8 +2629,8 @@ test_file_double_file_dataset_open(bool new_format)
if (new_format) {
ret = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
CHECK(ret, FAIL, "H5Pset_libver_bounds");
- } /* end if */
- h5_fixname(FILE1, fapl, filename, sizeof filename);
+ }
+ h5_fixname(FILE_DOUBLE_OPEN, fapl, filename, sizeof filename);
/* Create the test file */
fid1 = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
@@ -2934,6 +2940,9 @@ test_file_double_file_dataset_open(bool new_format)
ret = H5Tclose(tid1);
CHECK(ret, FAIL, "H5Tclose");
+ /* Delete the test file */
+ h5_delete_test_file(filename, fapl);
+
/* Close FAPL */
ret = H5Pclose(fapl);
CHECK(ret, FAIL, "H5Pclose");
@@ -7650,7 +7659,7 @@ test_incr_filesize(void)
MESSAGE(5, ("Testing H5Fincrement_filesize() and H5Fget_eoa())\n"));
fapl = h5_fileaccess();
- h5_fixname(FILE8, fapl, filename, sizeof filename);
+ h5_fixname(FILE_INCR_FILESIZE, fapl, filename, sizeof filename);
/* Get the VFD feature flags */
driver_id = H5Pget_driver(fapl);
@@ -7735,6 +7744,9 @@ test_incr_filesize(void)
/* Verify the filesize is the previous stored_eoa + 512 */
VERIFY(filesize, stored_eoa + 512, "file size");
+ /* Delete the test file */
+ h5_delete_test_file(FILE_INCR_FILESIZE, fapl);
+
/* Close the file access property list */
ret = H5Pclose(fapl);
CHECK(ret, FAIL, "H5Pclose");
@@ -8225,6 +8237,7 @@ cleanup_file(void)
H5Fdelete(FILE5, H5P_DEFAULT);
H5Fdelete(FILE6, H5P_DEFAULT);
H5Fdelete(FILE7, H5P_DEFAULT);
+ H5Fdelete(FILE8, H5P_DEFAULT);
H5Fdelete(DST_FILE, H5P_DEFAULT);
}
H5E_END_TRY
From 21ec3730c5d625995cd406e29ae5cb84dc45a11c Mon Sep 17 00:00:00 2001
From: bmribler <39579120+bmribler@users.noreply.github.com>
Date: Sat, 21 Oct 2023 20:15:12 -0400
Subject: [PATCH 07/52] Additional update to the library version matrix for
H5Pset_libver_bounds() (#3742)
This is the follow-up of PR #3702.
---
src/H5Ppublic.h | 232 ++++++++++++++++++------------------------------
1 file changed, 87 insertions(+), 145 deletions(-)
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index d79c7d785fd..3a059105a11 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -4849,14 +4849,13 @@ H5_DLL herr_t H5Pset_gc_references(hid_t fapl_id, unsigned gc_ref);
* \p low=#H5F_LIBVER_EARLIEST
* \p high=#H5F_LIBVER_V18 |
*
- * \li The library will create objects with the earliest
- * possible format versions.
- * \li The library will allow objects to be created with the
- * latest format versions available to library release 1.8.x.
- * \li API calls that create objects or features that are
- * available to versions of the library greater than 1.8.x
- * release will fail.
- * |
+ * \li The library will create objects with the earliest
+ * possible format versions.
+ * \li The library will allow objects to be created with the
+ * latest format versions available to library release 1.8.x.
+ * \li API calls that create objects or features that are
+ * available to versions of the library greater than 1.8.x
+ * release will fail.
*
*
* \p low=#H5F_LIBVER_EARLIEST
@@ -4865,12 +4864,10 @@ H5_DLL herr_t H5Pset_gc_references(hid_t fapl_id, unsigned gc_ref);
* \li The library will create objects with the earliest possible
* format versions.
* \li The library will allow objects to be created with the latest
- * format versions available to library release 1.10.x. Note
- * that as 1.10.11 is the last release of the 1.10 series.
+ * format versions available to library release 1.10.x.
* \li API calls that create objects or features that are
* available to versions of the library greater than 1.10.x
- * release will fail.
- * |
+ * release will fail.
*
*
* \p low=#H5F_LIBVER_EARLIEST
@@ -4879,12 +4876,10 @@ H5_DLL herr_t H5Pset_gc_references(hid_t fapl_id, unsigned gc_ref);
* \li The library will create objects with the earliest possible
* format versions.
* \li The library will allow objects to be created with the latest
- * format versions available to library release 1.12.x. Note
- * that as 1.12.3 is the last release of the 1.12 series.
+ * format versions available to library release 1.12.x.
* \li API calls that create objects or features that are
* available to versions of the library greater than 1.12.x
- * release will fail.
- * |
+ * release will fail.
*
*
* \p low=#H5F_LIBVER_EARLIEST
@@ -4896,8 +4891,7 @@ H5_DLL herr_t H5Pset_gc_references(hid_t fapl_id, unsigned gc_ref);
* format versions available to library release 1.14.x.
* \li API calls that create objects or features that are
* available to versions of the library greater than 1.14.x
- * release will fail.
- * |
+ * release will fail.
*
*
* \p low=#H5F_LIBVER_EARLIEST
@@ -4906,15 +4900,10 @@ H5_DLL herr_t H5Pset_gc_references(hid_t fapl_id, unsigned gc_ref);
* \li The library will create objects with the earliest possible
* format versions.
* \li The library will allow objects to be created with the latest
- * format versions available to library release 1.16.x.
- * Since 1.16.x is also #H5F_LIBVER_LATEST, there is no upper
- * limit on the format versions to use. That is, if a
- * newer format version is required to support a feature
- * in 1.16.x series, this setting will allow the object to be
- * created.
+ * format versions available to library release 1.16.x. See
+ * note *H5F_LIBVER_LATEST* below the table.
* \li This is the library default setting and provides the greatest
- * format compatibility.
- * |
+ * format compatibility.
*
*
* \p low=#H5F_LIBVER_V18
@@ -4922,6 +4911,11 @@ H5_DLL herr_t H5Pset_gc_references(hid_t fapl_id, unsigned gc_ref);
* |
* \li The library will create objects with the latest format
* versions available to library release 1.8.x.
+ * \li The library will allow objects to be created with the latest
+ * format versions available to library release 1.8.x.
+ * \li The objects written with this setting may be
+ * accessible to a smaller range of library versions than
+ * would be the case if low is set to #H5F_LIBVER_EARLIEST.
* \li API calls that create objects or features that are available
* to versions of the library greater than 1.8.x release will
* fail.
@@ -4932,71 +4926,66 @@ H5_DLL herr_t H5Pset_gc_references(hid_t fapl_id, unsigned gc_ref);
* | \p low=#H5F_LIBVER_V18
* \p high=#H5F_LIBVER_V110 |
*
- * \li The library will create objects with the latest format
- * versions available to library release 1.8.x.
- * \li The library will allow objects to be created with the latest
- * format versions available to library release 1.10.x. Note
- * that 1.10.11 is the last release of the 1.10 series.
- * \li API calls that create objects or features that are
- * available to versions of the library greater than 1.10.x
- * release will fail.
- * \li Earlier versions of the library may not be able to access
- * objects created with this setting. |
+ * \li The library will create objects with the latest format
+ * versions available to library release 1.8.x.
+ * \li The library will allow objects to be created with the latest
+ * format versions available to library release 1.10.x.
+ * \li API calls that create objects or features that are
+ * available to versions of the library greater than 1.10.x
+ * release will fail.
+ * \li Earlier versions of the library may not be able to access
+ * objects created with this setting.
*
*
* \p low=#H5F_LIBVER_V18
* \p high=#H5F_LIBVER_V112 |
*
- * \li The library will create objects with the latest format
- * versions available to library release 1.8.x.
- * \li The library will allow objects to be created with the latest
- * format versions available to library release 1.12.x.
- * \li API calls that create objects or features that are
- * available to versions of the library greater than 1.12.x
- * release will fail.
- * \li Earlier versions of the library may not be able to access
- * objects created with this setting. |
+ * \li The library will create objects with the latest format
+ * versions available to library release 1.8.x.
+ * \li The library will allow objects to be created with the latest
+ * format versions available to library release 1.12.x.
+ * \li API calls that create objects or features that are
+ * available to versions of the library greater than 1.12.x
+ * release will fail.
+ * \li Earlier versions of the library may not be able to access
+ * objects created with this setting.
*
*
* \p low=#H5F_LIBVER_V18
* \p high=#H5F_LIBVER_V114 |
*
- * \li The library will create objects with the latest format
- * versions available to library release 1.8.x.
- * \li The library will allow objects to be created with the latest
- * format versions available to library release 1.14.x.
- * \li API calls that create objects or features that are
- * available to versions of the library greater than 1.14.x
- * release will fail.
- * \li Earlier versions of the library may not be able to access
- * objects created with this setting. |
+ * \li The library will create objects with the latest format
+ * versions available to library release 1.8.x.
+ * \li The library will allow objects to be created with the latest
+ * format versions available to library release 1.14.x.
+ * \li API calls that create objects or features that are
+ * available to versions of the library greater than 1.14.x
+ * release will fail.
+ * \li Earlier versions of the library may not be able to access
+ * objects created with this setting.
*
*
* \p low=#H5F_LIBVER_V18
* \p high=#H5F_LIBVER_V116 |
*
- * \li The library will create objects with the latest format
- * versions available to library release 1.8.x.
- * \li The library will allow objects to be created with the latest
- * format versions available to library release 1.16.x.
- * Since 1.16.x is also #H5F_LIBVER_LATEST, there is no upper
- * limit on the format versions to use. That is, if a
- * newer format version is required to support a feature
- * in 1.16.x series, this setting will allow the object to be
- * created.
- * \li Earlier versions of the library may not be able to access
- * objects created with this setting. |
+ * \li The library will create objects with the latest format
+ * versions available to library release 1.8.x.
+ * \li The library will allow objects to be created with the latest
+ * format versions available to library release 1.16.x. See
+ * note *H5F_LIBVER_LATEST* below the table.
+ * \li This setting allows users to take advantage of the latest
+ * features and performance enhancements in the library.
+ * \li Earlier versions of the library may not be able to access
+ * objects created with this setting.
*
*
* \p low=#H5F_LIBVER_V110
- * \p high=#H5F_LIBVER_V110
- * |
+ * \p high=#H5F_LIBVER_V110
*
* \li The library will create objects with the latest format
* versions available to library release 1.10.x.
* \li The library will allow objects to be created with the latest
- * format versions available to library release 1.10.x. Note
- * that 1.10.11 is the last release of the 1.10 series.
+ * format versions available to library release 1.10.x.
* \li The objects written with this setting may be
* accessible to a smaller range of library versions than
* would be the case if low is set to #H5F_LIBVER_EARLIEST.
@@ -5005,72 +4994,52 @@ H5_DLL herr_t H5Pset_gc_references(hid_t fapl_id, unsigned gc_ref);
* fail.
* \li Earlier versions of the library may not be able to access
* objects created with this setting. |
- *
*
*
* \p low=#H5F_LIBVER_V110
- * \p high=#H5F_LIBVER_V112
- * |
+ * \p high=#H5F_LIBVER_V112
*
* \li The library will create objects with the latest format
* versions available to library release 1.10.x.
* \li The library will allow objects to be created with the latest
* format versions available to library release 1.12.x.
- * \li The objects written with this setting may be
- * accessible to a smaller range of library versions than
- * would be the case if low is set to #H5F_LIBVER_EARLIEST.
* \li API calls that create objects or features that are available
* to versions of the library greater than 1.12.x release will
* fail.
* \li Earlier versions of the library may not be able to access
* objects created with this setting. |
- *
*
*
* \p low=#H5F_LIBVER_V110
- * \p high=#H5F_LIBVER_V114
- * |
+ * \p high=#H5F_LIBVER_V114
*
* \li The library will create objects with the latest format
* versions available to library release 1.10.x.
* \li The library will allow objects to be created with the latest
* format versions available to library release 1.14.x.
- * \li The objects written with this setting may be
- * accessible to a smaller range of library versions than
- * would be the case if low is set to #H5F_LIBVER_EARLIEST.
* \li API calls that create objects or features that are available
* to versions of the library greater than 1.14.x release will
* fail.
* \li Earlier versions of the library may not be able to access
* objects created with this setting. |
- *
*
*
* \p low=#H5F_LIBVER_V110
- * \p high=#H5F_LIBVER_V116
- * |
+ * \p high=#H5F_LIBVER_V116
*
* \li The library will create objects with the latest format
* versions available to library release 1.10.x.
* \li The library will allow objects to be created with the latest
- * format versions available to library release 1.16.x.
- * Since 1.16.x is also #H5F_LIBVER_LATEST, there is no upper
- * limit on the format versions to use. That is, if a
- * newer format version is required to support a feature
- * in 1.16.x series, this setting will allow the object to be
- * created.
+ * format versions available to library release 1.16.x. See
+ * note *H5F_LIBVER_LATEST* below the table.
* \li This setting allows users to take advantage of the latest
* features and performance enhancements in the library.
- * However, objects written with this setting may be
- * accessible to a smaller range of library versions than
- * would be the case if low is set to #H5F_LIBVER_EARLIEST.
* \li Earlier versions of the library may not be able to access
- * objects created with this setting.
- * |
+ * objects created with this setting.
+ *
*
* \p low=#H5F_LIBVER_V112
- * \p high=#H5F_LIBVER_V112
- * |
+ * \p high=#H5F_LIBVER_V112
*
* \li The library will create objects with the latest format
* versions available to library release 1.12.x.
@@ -5084,55 +5053,38 @@ H5_DLL herr_t H5Pset_gc_references(hid_t fapl_id, unsigned gc_ref);
* fail.
* \li Earlier versions of the library may not be able to access
* objects created with this setting. |
- *
*
*
* \p low=#H5F_LIBVER_V112
- * \p high=#H5F_LIBVER_V114
- * |
+ * \p high=#H5F_LIBVER_V114
*
* \li The library will create objects with the latest format
* versions available to library release 1.12.x.
* \li The library will allow objects to be created with the latest
* format versions available to library release 1.14.x.
- * \li The objects written with this setting may be
- * accessible to a smaller range of library versions than
- * would be the case if low is set to #H5F_LIBVER_EARLIEST.
* \li API calls that create objects or features that are available
* to versions of the library greater than 1.14.x release will
* fail.
* \li Earlier versions of the library may not be able to access
* objects created with this setting. |
- *
- *
*
*
* \p low=#H5F_LIBVER_V112
- * \p high=#H5F_LIBVER_V116
- * |
+ * \p high=#H5F_LIBVER_V116
*
* \li The library will create objects with the latest format
* versions available to library release 1.12.x.
* \li The library will allow objects to be created with the latest
- * format versions available to library release 1.16.x.
- * Since 1.16.x is also #H5F_LIBVER_LATEST, there is no upper
- * limit on the format versions to use. That is, if a
- * newer format version is required to support a feature
- * in 1.16.x series, this setting will allow the object to be
- * created.
+ * format versions available to library release 1.16.x. See
+ * note *H5F_LIBVER_LATEST* below the table.
* \li This setting allows users to take advantage of the latest
* features and performance enhancements in the library.
- * However, objects written with this setting may be
- * accessible to a smaller range of library versions than
- * would be the case if low is set to #H5F_LIBVER_EARLIEST.
* \li Earlier versions of the library may not be able to access
- * objects created with this setting.
- * |
+ * objects created with this setting.
*
*
* \p low=#H5F_LIBVER_V114
- * \p high=#H5F_LIBVER_V114
- * |
+ * \p high=#H5F_LIBVER_V114
*
* \li The library will create objects with the latest format
* versions available to library release 1.14.x.
@@ -5146,57 +5098,47 @@ H5_DLL herr_t H5Pset_gc_references(hid_t fapl_id, unsigned gc_ref);
* fail.
* \li Earlier versions of the library may not be able to access
* objects created with this setting. |
- *
- *
*
*
* \p low=#H5F_LIBVER_V114
- * \p high=#H5F_LIBVER_V116
- * |
+ * \p high=#H5F_LIBVER_V116
*
* \li The library will create objects with the latest format
* versions available to library release 1.14.x.
* \li The library will allow objects to be created with the latest
- * format versions available to library release 1.16.x.
- * Since 1.16.x is also #H5F_LIBVER_LATEST, there is no upper
- * limit on the format versions to use. That is, if a
- * newer format version is required to support a feature
- * in 1.16.x series, this setting will allow the object to be
- * created.
+ * format versions available to library release 1.16.x. See
+ * note *H5F_LIBVER_LATEST* below the table.
* \li This setting allows users to take advantage of the latest
* features and performance enhancements in the library.
- * However, objects written with this setting may be
- * accessible to a smaller range of library versions than
- * would be the case if low is set to #H5F_LIBVER_EARLIEST.
* \li Earlier versions of the library may not be able to access
- * objects created with this setting.
- * |
+ * objects created with this setting.
*
*
* \p low=#H5F_LIBVER_V116
- * \p high=#H5F_LIBVER_V116
- * |
+ * \p high=#H5F_LIBVER_V116
*
* \li The library will create objects with the latest format
* versions available to library release 1.16.x.
* \li The library will allow objects to be created with the latest
- * format versions available to library release 1.16.x.
- * Since 1.16.x is also #H5F_LIBVER_LATEST, there is no upper
- * limit on the format versions to use. That is, if a
- * newer format version is required to support a feature
- * in 1.16.x series, this setting will allow the object to be
- * created.
+ * format versions available to library release 1.16.x. See
+ * note *H5F_LIBVER_LATEST* below the table.
* \li This setting allows users to take advantage of the latest
* features and performance enhancements in the library.
* However, objects written with this setting may be
* accessible to a smaller range of library versions than
* would be the case if low is set to #H5F_LIBVER_EARLIEST.
* \li Earlier versions of the library may not be able to access
- * objects created with this setting.
- * |
+ * objects created with this setting.
*
*
*
+ * \note *H5F_LIBVER_LATEST*:
+ * Since 1.16.x is also #H5F_LIBVER_LATEST, there is no upper
+ * limit on the format versions to use. That is, if a
+ * newer format version is required to support a feature
+ * in 1.16.x series, this setting will allow the object to be
+ * created.
+ *
* \version 1.10.2 #H5F_LIBVER_V18 added to the enumerated defines in
* #H5F_libver_t.
*
From e4fb67c84aa4a4983e2cee4994235600d813a9ee Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Sun, 22 Oct 2023 18:45:22 -0700
Subject: [PATCH 08/52] Clean Autotools files in parallel tests (#3744)
Adds missing files to `make clean` for parallel, including Fortran.
---
fortran/testpar/Makefile.am | 2 +-
testpar/Makefile.am | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/fortran/testpar/Makefile.am b/fortran/testpar/Makefile.am
index 7f9f2846928..afdda980c5c 100644
--- a/fortran/testpar/Makefile.am
+++ b/fortran/testpar/Makefile.am
@@ -36,7 +36,7 @@ TEST_PROG_PARA=parallel_test subfiling_test async_test
check_PROGRAMS=$(TEST_PROG_PARA)
# Temporary files
-CHECK_CLEANFILES+=parf[12].h5 subf.h5*
+CHECK_CLEANFILES+=parf[12].h5 h5*_tests.h5 subf.h5* test_async_apis.mod
# Test source files
parallel_test_SOURCES=ptest.F90 hyper.F90 mdset.F90 multidsetrw.F90
diff --git a/testpar/Makefile.am b/testpar/Makefile.am
index 59d47e15ebf..4a8cb826f49 100644
--- a/testpar/Makefile.am
+++ b/testpar/Makefile.am
@@ -58,6 +58,7 @@ LDADD = $(LIBH5TEST) $(LIBHDF5)
# after_mpi_fin.h5 is from t_init_term
# go is used for debugging. See testphdf5.c.
CHECK_CLEANFILES+=MPItest.h5 Para*.h5 bigio_test.h5 CacheTestDummy.h5 \
- ShapeSameTest.h5 shutdown.h5 pmulti_dset.h5 after_mpi_fin.h5 go
+ ShapeSameTest.h5 shutdown.h5 pmulti_dset.h5 after_mpi_fin.h5 go noflush.h5 \
+ mpio_select_test_file.h5 *.btr
include $(top_srcdir)/config/conclude.am
From 5a00539e0dbec38894fe2e7ef7cdbef6d09f8dc2 Mon Sep 17 00:00:00 2001
From: mattjala <124107509+mattjala@users.noreply.github.com>
Date: Mon, 23 Oct 2023 14:32:54 -0500
Subject: [PATCH 09/52] Add native VOL checks to deprecated functions (#3647)
* Add native VOL checks to deprecated functions
* Remove unneeded native VOL checks
* Move native checks to top level calls
---
src/H5Odeprec.c | 75 +++++++++++++++++++++++-----
src/H5Rdeprec.c | 128 +++++++++++++++++++++++++++++++-----------------
2 files changed, 145 insertions(+), 58 deletions(-)
diff --git a/src/H5Odeprec.c b/src/H5Odeprec.c
index 6e8b34e789b..3de58185675 100644
--- a/src/H5Odeprec.c
+++ b/src/H5Odeprec.c
@@ -116,9 +116,10 @@ static herr_t
H5O__iterate1_adapter(hid_t obj_id, const char *name, const H5O_info2_t *oinfo2, void *op_data)
{
H5O_visit1_adapter_t *shim_data = (H5O_visit1_adapter_t *)op_data;
- H5O_info1_t oinfo; /* Deprecated object info struct */
- unsigned dm_fields; /* Fields for data model query */
- unsigned nat_fields; /* Fields for native query */
+ H5O_info1_t oinfo; /* Deprecated object info struct */
+ unsigned dm_fields; /* Fields for data model query */
+ unsigned nat_fields; /* Fields for native query */
+ H5VL_object_t *vol_obj;
herr_t ret_value = H5_ITER_CONT; /* Return value */
FUNC_ENTER_PACKAGE
@@ -158,7 +159,6 @@ H5O__iterate1_adapter(hid_t obj_id, const char *name, const H5O_info2_t *oinfo2,
/* Check for retrieving native information */
nat_fields = shim_data->fields & (H5O_INFO_HDR | H5O_INFO_META_SIZE);
if (nat_fields) {
- H5VL_object_t *vol_obj; /* Object of obj_id */
H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */
H5VL_native_object_optional_args_t obj_opt_args; /* Arguments for optional operation */
H5VL_loc_params_t loc_params; /* Location parameters for VOL callback */
@@ -401,7 +401,8 @@ H5Oget_info1(hid_t loc_id, H5O_info1_t *oinfo /*out*/)
{
H5VL_object_t *vol_obj = NULL; /* Object of loc_id */
H5VL_loc_params_t loc_params;
- herr_t ret_value = SUCCEED; /* Return value */
+ bool is_native_vol_obj = false;
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
H5TRACE2("e", "ix", loc_id, oinfo);
@@ -418,6 +419,15 @@ H5Oget_info1(hid_t loc_id, H5O_info1_t *oinfo /*out*/)
if (NULL == (vol_obj = H5VL_vol_object(loc_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier");
+ /* Check if using native VOL connector */
+ if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine if VOL object is native connector object");
+
+ /* Must use native VOL connector for this operation */
+ if (!is_native_vol_obj)
+ HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL,
+ "Deprecated H5Oget_info1 is only meant to be used with the native VOL connector");
+
/* Retrieve the object's information */
if (H5O__get_info_old(vol_obj, &loc_params, oinfo, H5O_INFO_ALL) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't get deprecated info for object");
@@ -441,7 +451,8 @@ H5Oget_info_by_name1(hid_t loc_id, const char *name, H5O_info1_t *oinfo /*out*/,
{
H5VL_object_t *vol_obj = NULL; /* object of loc_id */
H5VL_loc_params_t loc_params;
- herr_t ret_value = SUCCEED; /* Return value */
+ bool is_native_vol_obj = false;
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
H5TRACE4("e", "i*sxi", loc_id, name, oinfo, lapl_id);
@@ -468,6 +479,15 @@ H5Oget_info_by_name1(hid_t loc_id, const char *name, H5O_info1_t *oinfo /*out*/,
if (NULL == (vol_obj = H5VL_vol_object(loc_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier");
+ /* Check if using native VOL connector */
+ if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine if VOL object is native connector object");
+
+ /* Must use native VOL connector for this operation */
+ if (!is_native_vol_obj)
+ HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL,
+ "Deprecated H5Oget_info_by_name1 is only meant to be used with the native VOL connector");
+
/* Retrieve the object's information */
if (H5O__get_info_old(vol_obj, &loc_params, oinfo, H5O_INFO_ALL) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't get deprecated info for object");
@@ -493,7 +513,8 @@ H5Oget_info_by_idx1(hid_t loc_id, const char *group_name, H5_index_t idx_type, H
{
H5VL_object_t *vol_obj = NULL; /* object of loc_id */
H5VL_loc_params_t loc_params;
- herr_t ret_value = SUCCEED; /* Return value */
+ bool is_native_vol_obj = false;
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
H5TRACE7("e", "i*sIiIohxi", loc_id, group_name, idx_type, order, n, oinfo, lapl_id);
@@ -524,6 +545,15 @@ H5Oget_info_by_idx1(hid_t loc_id, const char *group_name, H5_index_t idx_type, H
if (NULL == (vol_obj = H5VL_vol_object(loc_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier");
+ /* Check if using native VOL connector */
+ if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine if VOL object is native connector object");
+
+ /* Must use native VOL connector for this operation */
+ if (!is_native_vol_obj)
+ HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL,
+ "Deprecated H5Oget_info_by_idx1 is only meant to be used with the native VOL connector");
+
/* Retrieve the object's information */
if (H5O__get_info_old(vol_obj, &loc_params, oinfo, H5O_INFO_ALL) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't get deprecated info for object");
@@ -574,7 +604,7 @@ H5Oget_info2(hid_t loc_id, H5O_info1_t *oinfo /*out*/, unsigned fields)
"can't determine if VOL object is native connector object");
if (!is_native_vol_obj)
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, H5I_INVALID_HID,
- "H5Oget_info2 is only meant to be used with the native VOL connector");
+ "Deprecated H5Oget_info2 is only meant to be used with the native VOL connector");
/* Retrieve deprecated info struct */
if (H5O__get_info_old(vol_obj, &loc_params, oinfo, fields) < 0)
@@ -637,7 +667,7 @@ H5Oget_info_by_name2(hid_t loc_id, const char *name, H5O_info1_t *oinfo /*out*/,
"can't determine if VOL object is native connector object");
if (!is_native_vol_obj)
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, H5I_INVALID_HID,
- "H5Oget_info_by_name2 is only meant to be used with the native VOL connector");
+ "Deprecated H5Oget_info_by_name2 is only meant to be used with the native VOL connector");
/* Retrieve deprecated info struct */
if (H5O__get_info_old(vol_obj, &loc_params, oinfo, fields) < 0)
@@ -706,7 +736,7 @@ H5Oget_info_by_idx2(hid_t loc_id, const char *group_name, H5_index_t idx_type, H
"can't determine if VOL object is native connector object");
if (!is_native_vol_obj)
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, H5I_INVALID_HID,
- "H5Oget_info_by_idx2 is only meant to be used with the native VOL connector");
+ "Deprecated H5Oget_info_by_idx2 is only meant to be used with the native VOL connector");
/* Retrieve deprecated info struct */
if (H5O__get_info_old(vol_obj, &loc_params, oinfo, fields) < 0)
@@ -753,6 +783,7 @@ H5Ovisit1(hid_t obj_id, H5_index_t idx_type, H5_iter_order_t order, H5O_iterate1
H5VL_loc_params_t loc_params; /* Location parameters for object access */
H5O_visit1_adapter_t shim_data; /* Adapter for passing app callback & user data */
herr_t ret_value; /* Return value */
+ bool is_native_vol_obj = false;
FUNC_ENTER_API(FAIL)
H5TRACE5("e", "iIiIoOi*x", obj_id, idx_type, order, op, op_data);
@@ -769,6 +800,15 @@ H5Ovisit1(hid_t obj_id, H5_index_t idx_type, H5_iter_order_t order, H5O_iterate1
if (NULL == (vol_obj = H5VL_vol_object(obj_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier");
+ /* Check if using native VOL connector */
+ if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine if VOL object is native connector object");
+
+ /* Must use native VOL connector for this operation */
+ if (!is_native_vol_obj)
+ HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL,
+ "Deprecated H5Ovisit1 is only meant to be used with the native VOL connector");
+
/* Set location parameters */
loc_params.type = H5VL_OBJECT_BY_SELF;
loc_params.obj_type = H5I_get_type(obj_id);
@@ -833,6 +873,7 @@ H5Ovisit_by_name1(hid_t loc_id, const char *obj_name, H5_index_t idx_type, H5_it
H5VL_loc_params_t loc_params; /* Location parameters for object access */
H5O_visit1_adapter_t shim_data; /* Adapter for passing app callback & user data */
herr_t ret_value; /* Return value */
+ bool is_native_vol_obj = false;
FUNC_ENTER_API(FAIL)
H5TRACE7("e", "i*sIiIoOi*xi", loc_id, obj_name, idx_type, order, op, op_data, lapl_id);
@@ -857,6 +898,15 @@ H5Ovisit_by_name1(hid_t loc_id, const char *obj_name, H5_index_t idx_type, H5_it
if (NULL == (vol_obj = H5VL_vol_object(loc_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier");
+ /* Check if using native VOL connector */
+ if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine if VOL object is native connector object");
+
+ /* Must use native VOL connector for this operation */
+ if (!is_native_vol_obj)
+ HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL,
+ "Deprecated H5Ovisit_by_name1 is only meant to be used with the native VOL connector");
+
/* Set location parameters */
loc_params.type = H5VL_OBJECT_BY_NAME;
loc_params.loc_data.loc_by_name.name = obj_name;
@@ -949,9 +999,10 @@ H5Ovisit2(hid_t obj_id, H5_index_t idx_type, H5_iter_order_t order, H5O_iterate1
if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, H5I_INVALID_HID,
"can't determine if VOL object is native connector object");
+
if (!is_native_vol_obj)
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, H5I_INVALID_HID,
- "H5Ovisit2 is only meant to be used with the native VOL connector");
+ "Deprecated H5Ovisit2 is only meant to be used with the native VOL connector");
/* Set location parameters */
loc_params.type = H5VL_OBJECT_BY_SELF;
@@ -1053,7 +1104,7 @@ H5Ovisit_by_name2(hid_t loc_id, const char *obj_name, H5_index_t idx_type, H5_it
"can't determine if VOL object is native connector object");
if (!is_native_vol_obj)
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, H5I_INVALID_HID,
- "H5Ovisit_by_name2 is only meant to be used with the native VOL connector");
+ "Deprecated H5Ovisit_by_name2 is only meant to be used with the native VOL connector");
/* Set location parameters */
loc_params.type = H5VL_OBJECT_BY_NAME;
diff --git a/src/H5Rdeprec.c b/src/H5Rdeprec.c
index 773d8b0c706..1d12ebae25a 100644
--- a/src/H5Rdeprec.c
+++ b/src/H5Rdeprec.c
@@ -101,14 +101,14 @@ H5R__decode_token_compat(H5VL_object_t *vol_obj, H5I_type_t type, H5R_type_t ref
#ifndef NDEBUG
{
- bool is_native = false; /* Whether the src file is using the native VOL connector */
+ bool is_native_vol_obj = false; /* Whether the src file is using the native VOL connector */
/* Check if using native VOL connector */
- if (H5VL_object_is_native(vol_obj, &is_native) < 0)
+ if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL, "can't query if file uses native VOL connector");
/* Must use native VOL connector for this operation */
- assert(is_native);
+ assert(is_native_vol_obj);
}
#endif /* NDEBUG */
@@ -251,7 +251,8 @@ H5Rget_obj_type1(hid_t id, H5R_type_t ref_type, const void *ref)
H5O_token_t obj_token = {0}; /* Object token */
const unsigned char *buf = (const unsigned char *)ref; /* Reference buffer */
H5O_type_t obj_type = H5O_TYPE_UNKNOWN; /* Type of the referenced object */
- H5G_obj_t ret_value; /* Return value */
+ bool is_native_vol_obj; /* Whether the native VOL connector is in use */
+ H5G_obj_t ret_value; /* Return value */
FUNC_ENTER_API(H5G_UNKNOWN)
H5TRACE3("Go", "iRt*x", id, ref_type, ref);
@@ -266,6 +267,16 @@ H5Rget_obj_type1(hid_t id, H5R_type_t ref_type, const void *ref)
if (NULL == (vol_obj = H5VL_vol_object(id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5G_UNKNOWN, "invalid location identifier");
+ /* Check if using native VOL connector */
+ if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
+ HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL,
+ "can't determine if VOL object is native connector object");
+
+ /* Must use native VOL connector for this operation */
+ if (!is_native_vol_obj)
+ HGOTO_ERROR(H5E_REFERENCE, H5E_VOL, FAIL,
+ "H5Rget_obj_type1 is only meant to be used with the native VOL connector");
+
/* Get object type */
if ((vol_obj_type = H5I_get_type(id)) < 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5G_UNKNOWN, "invalid location identifier");
@@ -315,7 +326,8 @@ H5Rdereference1(hid_t obj_id, H5R_type_t ref_type, const void *ref)
H5I_type_t opened_type; /* Opened object type */
void *opened_obj = NULL; /* Opened object */
const unsigned char *buf = (const unsigned char *)ref; /* Reference buffer */
- hid_t ret_value = H5I_INVALID_HID; /* Return value */
+ bool is_native_vol_obj; /* Whether the native VOL connector is in use */
+ hid_t ret_value = H5I_INVALID_HID; /* Return value */
FUNC_ENTER_API(H5I_INVALID_HID)
H5TRACE3("i", "iRt*x", obj_id, ref_type, ref);
@@ -330,6 +342,16 @@ H5Rdereference1(hid_t obj_id, H5R_type_t ref_type, const void *ref)
if (NULL == (vol_obj = H5VL_vol_object(obj_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid location identifier");
+ /* Check if using native VOL connector */
+ if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
+ HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL,
+ "can't determine if VOL object is native connector object");
+
+ /* Must use native VOL connector for this operation */
+ if (!is_native_vol_obj)
+ HGOTO_ERROR(H5E_REFERENCE, H5E_VOL, FAIL,
+ "H5Rdereference1 is only meant to be used with the native VOL connector");
+
/* Get object type */
if ((vol_obj_type = H5I_get_type(obj_id)) < 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid location identifier");
@@ -382,8 +404,9 @@ H5Rcreate(void *ref, hid_t loc_id, const char *name, H5R_type_t ref_type, hid_t
H5VL_file_get_args_t file_get_vol_cb_args; /* Arguments to VOL callback */
hid_t file_id = H5I_INVALID_HID; /* File ID for region reference */
void *vol_obj_file = NULL;
- unsigned char *buf = (unsigned char *)ref; /* Return reference pointer */
- herr_t ret_value = SUCCEED; /* Return value */
+ bool is_native_vol_obj = false; /* Whether the src file is using the native VOL connector */
+ unsigned char *buf = (unsigned char *)ref; /* Return reference pointer */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
H5TRACE5("e", "*xi*sRti", ref, loc_id, name, ref_type, space_id);
@@ -404,18 +427,13 @@ H5Rcreate(void *ref, hid_t loc_id, const char *name, H5R_type_t ref_type, hid_t
if (NULL == (vol_obj = H5VL_vol_object(loc_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier");
-#ifndef NDEBUG
- {
- bool is_native = false; /* Whether the src file is using the native VOL connector */
+ /* Check if using native VOL connector */
+ if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
+ HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL, "can't query if file uses native VOL connector");
- /* Check if using native VOL connector */
- if (H5VL_object_is_native(vol_obj, &is_native) < 0)
- HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL, "can't query if file uses native VOL connector");
-
- /* Must use native VOL connector for this operation */
- assert(is_native);
- }
-#endif /* NDEBUG */
+ /* Must use native VOL connector for this operation */
+ if (!is_native_vol_obj)
+ HGOTO_ERROR(H5E_REFERENCE, H5E_VOL, FAIL, "must use native VOL connector to create reference");
/* Get object type */
if ((vol_obj_type = H5I_get_type(loc_id)) < 0)
@@ -500,13 +518,14 @@ H5Rcreate(void *ref, hid_t loc_id, const char *name, H5R_type_t ref_type, hid_t
herr_t
H5Rget_obj_type2(hid_t id, H5R_type_t ref_type, const void *ref, H5O_type_t *obj_type /*out*/)
{
- H5VL_object_t *vol_obj = NULL; /* Object of loc_id */
- H5I_type_t vol_obj_type = H5I_BADID; /* Object type of loc_id */
- H5VL_object_get_args_t vol_cb_args; /* Arguments to VOL callback */
- H5VL_loc_params_t loc_params; /* Location parameters */
- H5O_token_t obj_token = {0}; /* Object token */
- const unsigned char *buf = (const unsigned char *)ref; /* Reference pointer */
- herr_t ret_value = SUCCEED; /* Return value */
+ H5VL_object_t *vol_obj = NULL; /* Object of loc_id */
+ H5I_type_t vol_obj_type = H5I_BADID; /* Object type of loc_id */
+ H5VL_object_get_args_t vol_cb_args; /* Arguments to VOL callback */
+ H5VL_loc_params_t loc_params; /* Location parameters */
+ H5O_token_t obj_token = {0}; /* Object token */
+ const unsigned char *buf = (const unsigned char *)ref; /* Reference pointer */
+ bool is_native_vol_obj = false; /* Whether the native VOL connector is in use */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
H5TRACE4("e", "iRt*xx", id, ref_type, ref, obj_type);
@@ -521,6 +540,16 @@ H5Rget_obj_type2(hid_t id, H5R_type_t ref_type, const void *ref, H5O_type_t *obj
if (NULL == (vol_obj = H5VL_vol_object(id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier");
+ /* Check if using native VOL connector */
+ if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
+ HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL,
+ "can't determine if VOL object is native connector object");
+
+ /* Must use native VOL connector for this operation */
+ if (!is_native_vol_obj)
+ HGOTO_ERROR(H5E_REFERENCE, H5E_VOL, FAIL,
+ "H5Rget_obj_type2 is only meant to be used with the native VOL connector");
+
/* Get object type */
if ((vol_obj_type = H5I_get_type(id)) < 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier");
@@ -560,14 +589,15 @@ H5Rget_obj_type2(hid_t id, H5R_type_t ref_type, const void *ref, H5O_type_t *obj
hid_t
H5Rdereference2(hid_t obj_id, hid_t oapl_id, H5R_type_t ref_type, const void *ref)
{
- H5VL_object_t *vol_obj = NULL; /* Object of loc_id */
- H5I_type_t vol_obj_type = H5I_BADID; /* Object type of loc_id */
- H5VL_loc_params_t loc_params; /* Location parameters */
- H5O_token_t obj_token = {0}; /* Object token */
- H5I_type_t opened_type; /* Opened object type */
- void *opened_obj = NULL; /* Opened object */
- const unsigned char *buf = (const unsigned char *)ref; /* Reference pointer */
- hid_t ret_value = H5I_INVALID_HID; /* Return value */
+ H5VL_object_t *vol_obj = NULL; /* Object of loc_id */
+ H5I_type_t vol_obj_type = H5I_BADID; /* Object type of loc_id */
+ H5VL_loc_params_t loc_params; /* Location parameters */
+ H5O_token_t obj_token = {0}; /* Object token */
+ H5I_type_t opened_type; /* Opened object type */
+ void *opened_obj = NULL; /* Opened object */
+ const unsigned char *buf = (const unsigned char *)ref; /* Reference pointer */
+ bool is_native_vol_obj = false; /* Whether the native VOL connector is in use */
+ hid_t ret_value = H5I_INVALID_HID; /* Return value */
FUNC_ENTER_API(H5I_INVALID_HID)
H5TRACE4("i", "iiRt*x", obj_id, oapl_id, ref_type, ref);
@@ -588,6 +618,16 @@ H5Rdereference2(hid_t obj_id, hid_t oapl_id, H5R_type_t ref_type, const void *re
if (NULL == (vol_obj = H5VL_vol_object(obj_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid file identifier");
+ /* Check if using native VOL connector */
+ if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
+ HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL,
+ "can't determine if VOL object is native connector object");
+
+ /* Must use native VOL connector for this operation */
+ if (!is_native_vol_obj)
+ HGOTO_ERROR(H5E_REFERENCE, H5E_VOL, FAIL,
+ "H5Rdereference2 is only meant to be used with the native VOL connector");
+
/* Get object type */
if ((vol_obj_type = H5I_get_type(obj_id)) < 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid location identifier");
@@ -639,7 +679,8 @@ H5Rget_region(hid_t id, H5R_type_t ref_type, const void *ref)
H5S_t *space = NULL; /* Dataspace object */
hid_t file_id = H5I_INVALID_HID; /* File ID for region reference */
const unsigned char *buf = (const unsigned char *)ref; /* Reference pointer */
- hid_t ret_value; /* Return value */
+ bool is_native_vol_obj = false; /* Whether the src file is using the native VOL connector */
+ hid_t ret_value; /* Return value */
FUNC_ENTER_API(H5I_INVALID_HID)
H5TRACE3("i", "iRt*x", id, ref_type, ref);
@@ -654,19 +695,14 @@ H5Rget_region(hid_t id, H5R_type_t ref_type, const void *ref)
if (NULL == (vol_obj = H5VL_vol_object(id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid file identifier");
-#ifndef NDEBUG
- {
- bool is_native = false; /* Whether the src file is using the native VOL connector */
-
- /* Check if using native VOL connector */
- if (H5VL_object_is_native(vol_obj, &is_native) < 0)
- HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, H5I_INVALID_HID,
- "can't query if file uses native VOL connector");
+ /* Check if using native VOL connector */
+ if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0)
+ HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, H5I_INVALID_HID,
+ "can't query if file uses native VOL connector");
- /* Must use native VOL connector for this operation */
- assert(is_native);
- }
-#endif /* NDEBUG */
+ if (!is_native_vol_obj)
+ HGOTO_ERROR(H5E_REFERENCE, H5E_VOL, FAIL,
+ "H5Rget_region is only meant to be used with the native VOL connector");
/* Get object type */
if ((vol_obj_type = H5I_get_type(id)) < 0)
From a4c318de285e60d58a71ac5c3d7ee146f7e43824 Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Mon, 23 Oct 2023 14:33:37 -0500
Subject: [PATCH 10/52] Fix buffer overflow in cache debugging code (#3691)
---
src/H5Cmpio.c | 104 ++++++++++++++++++++++++++++++++++++++++----------
1 file changed, 84 insertions(+), 20 deletions(-)
diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c
index d7bf5b1dbda..643bbc80207 100644
--- a/src/H5Cmpio.c
+++ b/src/H5Cmpio.c
@@ -169,7 +169,7 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha
haddr_t last_addr;
#endif /* H5C_DO_SANITY_CHECKS */
#if H5C_APPLY_CANDIDATE_LIST__DEBUG
- char tbl_buf[1024];
+ char *tbl_buf = NULL;
#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
unsigned m, n;
unsigned u; /* Local index variable */
@@ -190,16 +190,48 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha
memset(entries_to_clear, 0, sizeof(entries_to_clear));
#if H5C_APPLY_CANDIDATE_LIST__DEBUG
- fprintf(stdout, "%s:%d: setting up candidate assignment table.\n", __func__, mpi_rank);
+ {
+ const char *const table_header = "candidate list = ";
+ size_t tbl_buf_size;
+ size_t tbl_buf_left;
+ size_t entry_nchars;
+ int bytes_printed;
- memset(tbl_buf, 0, sizeof(tbl_buf));
+ fprintf(stdout, "%s:%d: setting up candidate assignment table.\n", __func__, mpi_rank);
- snprintf(tbl_buf, sizeof(tbl_buf), "candidate list = ");
- for (u = 0; u < num_candidates; u++)
- sprintf(&(tbl_buf[strlen(tbl_buf)]), " 0x%llx", (long long)(*(candidates_list_ptr + u)));
- sprintf(&(tbl_buf[strlen(tbl_buf)]), "\n");
+ /* Calculate maximum number of characters printed for each
+ * candidate entry, including the leading space and "0x"
+ */
+ entry_nchars = (sizeof(long long) * CHAR_BIT / 4) + 3;
+
+ tbl_buf_size = strlen(table_header) + (num_candidates * entry_nchars) + 1;
+ if (NULL == (tbl_buf = H5MM_malloc(tbl_buf_size)))
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "can't allocate debug buffer");
+ tbl_buf_left = tbl_buf_size;
+
+ if ((bytes_printed = snprintf(tbl_buf, tbl_buf_left, table_header)) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSERRSTR, FAIL, "can't add to candidate list");
+ assert((size_t)bytes_printed < tbl_buf_left);
+ tbl_buf_left -= (size_t)bytes_printed;
+
+ for (u = 0; u < num_candidates; u++) {
+ if ((bytes_printed = snprintf(&(tbl_buf[tbl_buf_size - tbl_buf_left]), tbl_buf_left, " 0x%llx",
+ (long long)(*(candidates_list_ptr + u)))) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSERRSTR, FAIL, "can't add to candidate list");
+ assert((size_t)bytes_printed < tbl_buf_left);
+ tbl_buf_left -= (size_t)bytes_printed;
+ }
+
+ if ((bytes_printed = snprintf(&(tbl_buf[tbl_buf_size - tbl_buf_left]), tbl_buf_left, "\n")) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSERRSTR, FAIL, "can't add to candidate list");
+ assert((size_t)bytes_printed < tbl_buf_left);
+ tbl_buf_left -= (size_t)bytes_printed + 1; /* NUL terminator */
- fprintf(stdout, "%s", tbl_buf);
+ fprintf(stdout, "%s", tbl_buf);
+
+ H5MM_free(tbl_buf);
+ tbl_buf = NULL;
+ }
#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
if (f->shared->coll_md_write) {
@@ -258,18 +290,50 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha
last_entry_to_flush = candidate_assignment_table[mpi_rank + 1] - 1;
#if H5C_APPLY_CANDIDATE_LIST__DEBUG
- for (u = 0; u < 1024; u++)
- tbl_buf[u] = '\0';
- snprintf(tbl_buf, sizeof(tbl_buf), "candidate assignment table = ");
- for (u = 0; u <= (unsigned)mpi_size; u++)
- sprintf(&(tbl_buf[strlen(tbl_buf)]), " %u", candidate_assignment_table[u]);
- sprintf(&(tbl_buf[strlen(tbl_buf)]), "\n");
- fprintf(stdout, "%s", tbl_buf);
-
- fprintf(stdout, "%s:%d: flush entries [%u, %u].\n", __func__, mpi_rank, first_entry_to_flush,
- last_entry_to_flush);
-
- fprintf(stdout, "%s:%d: marking entries.\n", __func__, mpi_rank);
+ {
+ const char *const table_header = "candidate assignment table = ";
+ unsigned umax = UINT_MAX;
+ size_t tbl_buf_size;
+ size_t tbl_buf_left;
+ size_t entry_nchars;
+ int bytes_printed;
+
+ /* Calculate the maximum number of characters printed for each entry */
+ entry_nchars = (size_t)(log10(umax) + 1) + 1;
+
+ tbl_buf_size = strlen(table_header) + ((size_t)mpi_size * entry_nchars) + 1;
+ if (NULL == (tbl_buf = H5MM_malloc(tbl_buf_size)))
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "can't allocate debug buffer");
+ tbl_buf_left = tbl_buf_size;
+
+ if ((bytes_printed = snprintf(tbl_buf, tbl_buf_left, table_header)) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSERRSTR, FAIL, "can't add to candidate list");
+ assert((size_t)bytes_printed < tbl_buf_left);
+ tbl_buf_left -= (size_t)bytes_printed;
+
+ for (u = 0; u <= (unsigned)mpi_size; u++) {
+ if ((bytes_printed = snprintf(&(tbl_buf[tbl_buf_size - tbl_buf_left]), tbl_buf_left, " %u",
+ candidate_assignment_table[u])) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSERRSTR, FAIL, "can't add to candidate list");
+ assert((size_t)bytes_printed < tbl_buf_left);
+ tbl_buf_left -= (size_t)bytes_printed;
+ }
+
+ if ((bytes_printed = snprintf(&(tbl_buf[tbl_buf_size - tbl_buf_left]), tbl_buf_left, "\n")) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSERRSTR, FAIL, "can't add to candidate list");
+ assert((size_t)bytes_printed < tbl_buf_left);
+ tbl_buf_left -= (size_t)bytes_printed + 1; /* NUL terminator */
+
+ fprintf(stdout, "%s", tbl_buf);
+
+ H5MM_free(tbl_buf);
+ tbl_buf = NULL;
+
+ fprintf(stdout, "%s:%d: flush entries [%u, %u].\n", __func__, mpi_rank, first_entry_to_flush,
+ last_entry_to_flush);
+
+ fprintf(stdout, "%s:%d: marking entries.\n", __func__, mpi_rank);
+ }
#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
for (u = 0; u < num_candidates; u++) {
From 34f4569cdd65c51476fe4d295b87de7a9c90f184 Mon Sep 17 00:00:00 2001
From: Scot Breitenfeld
Date: Mon, 23 Oct 2023 14:35:42 -0500
Subject: [PATCH 11/52] update stat arg for apple (#3726)
* update stat arg for apple
* use H5_HAVE_DARWIN for Apple ifdef
* fix typo
* removed duplicate H5_ih_info_t
* added fortran async test to cmake
---
fortran/src/H5Fff.F90 | 6 ------
fortran/src/H5Off.F90 | 6 ------
fortran/src/H5config_f.inc.cmake | 8 +++++++-
fortran/src/H5config_f.inc.in | 3 +++
fortran/src/H5f90global.F90 | 6 ++++++
fortran/testpar/CMakeTests.cmake | 1 +
fortran/testpar/subfiling.F90 | 11 ++++++++---
7 files changed, 25 insertions(+), 16 deletions(-)
diff --git a/fortran/src/H5Fff.F90 b/fortran/src/H5Fff.F90
index fee4d3c7cf8..d31117784e5 100644
--- a/fortran/src/H5Fff.F90
+++ b/fortran/src/H5Fff.F90
@@ -72,12 +72,6 @@ END FUNCTION h5fis_accessible
INTEGER(HSIZE_T) :: tot_space !< Amount of free space in the file
END TYPE H5F_info_free_t
-!> @brief H5_ih_info_t derived type.
- TYPE, BIND(C) :: H5_ih_info_t
- INTEGER(HSIZE_T) :: index_size !< btree and/or list
- INTEGER(HSIZE_T) :: heap_size !< Heap size
- END TYPE H5_ih_info_t
-
!> @brief H5F_info_t_sohm derived type.
TYPE, BIND(C) :: H5F_info_sohm_t
INTEGER(C_INT) :: version !< Version # of shared object header info
diff --git a/fortran/src/H5Off.F90 b/fortran/src/H5Off.F90
index 4a0a1632e78..b705ba324d7 100644
--- a/fortran/src/H5Off.F90
+++ b/fortran/src/H5Off.F90
@@ -110,12 +110,6 @@ MODULE H5O
TYPE(mesg_t) :: mesg
END TYPE c_hdr_t
-!> @brief Extra metadata storage for obj & attributes
- TYPE, BIND(C) :: H5_ih_info_t
- INTEGER(hsize_t) :: index_size !< btree and/or list
- INTEGER(hsize_t) :: heap_size !< heap
- END TYPE H5_ih_info_t
-
!> @brief meta_size_t derived type
TYPE, BIND(C) :: meta_size_t
TYPE(H5_ih_info_t) :: obj !< v1/v2 B-tree & local/fractal heap for groups, B-tree for chunked datasets
diff --git a/fortran/src/H5config_f.inc.cmake b/fortran/src/H5config_f.inc.cmake
index 34fb091c787..71bce0e18c2 100644
--- a/fortran/src/H5config_f.inc.cmake
+++ b/fortran/src/H5config_f.inc.cmake
@@ -23,6 +23,12 @@
#undef H5_HAVE_SUBFILING_VFD
#endif
+! Define if on APPLE
+#cmakedefine01 H5_HAVE_DARWIN
+#if H5_HAVE_DARWIN == 0
+#undef H5_HAVE_DARWIN
+#endif
+
! Define if the intrinsic function STORAGE_SIZE exists
#define H5_FORTRAN_HAVE_STORAGE_SIZE @H5_FORTRAN_HAVE_STORAGE_SIZE@
@@ -81,4 +87,4 @@
#cmakedefine01 H5_NO_DEPRECATED_SYMBOLS
#if H5_NO_DEPRECATED_SYMBOLS == 0
#undef H5_NO_DEPRECATED_SYMBOLS
-#endif
\ No newline at end of file
+#endif
diff --git a/fortran/src/H5config_f.inc.in b/fortran/src/H5config_f.inc.in
index 7fb76e12449..991e4b0750b 100644
--- a/fortran/src/H5config_f.inc.in
+++ b/fortran/src/H5config_f.inc.in
@@ -20,6 +20,9 @@
! Define if we have subfiling support
#undef HAVE_SUBFILING_VFD
+! Define if on APPLE
+#undef HAVE_DARWIN
+
! Define if the intrinsic function STORAGE_SIZE exists
#undef FORTRAN_HAVE_STORAGE_SIZE
diff --git a/fortran/src/H5f90global.F90 b/fortran/src/H5f90global.F90
index e60f1e83320..f6c06cbc056 100644
--- a/fortran/src/H5f90global.F90
+++ b/fortran/src/H5f90global.F90
@@ -25,6 +25,12 @@ MODULE H5GLOBAL
IMPLICIT NONE
+!> @brief H5_ih_info_t derived type.
+ TYPE, BIND(C) :: H5_ih_info_t
+ INTEGER(HSIZE_T) :: index_size !< btree and/or list
+ INTEGER(HSIZE_T) :: heap_size !< Heap size
+ END TYPE H5_ih_info_t
+
!> \addtogroup FH5
!> @{
! Parameters used in the function 'h5kind_to_type' located in H5_ff.F90.
diff --git a/fortran/testpar/CMakeTests.cmake b/fortran/testpar/CMakeTests.cmake
index 8c157241500..473049fb976 100644
--- a/fortran/testpar/CMakeTests.cmake
+++ b/fortran/testpar/CMakeTests.cmake
@@ -17,3 +17,4 @@
##############################################################################
add_test (NAME MPI_TEST_FORT_parallel_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS})
add_test (NAME MPI_TEST_FORT_subfiling_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS})
+add_test (NAME MPI_TEST_FORT_async_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS})
diff --git a/fortran/testpar/subfiling.F90 b/fortran/testpar/subfiling.F90
index 043ac6cb771..a677bea6121 100644
--- a/fortran/testpar/subfiling.F90
+++ b/fortran/testpar/subfiling.F90
@@ -54,6 +54,7 @@ PROGRAM subfiling_test
INTEGER(HID_T) :: driver_id
CHARACTER(len=8) :: hex1, hex2
+ CHARACTER(len=1) :: arg
!
! initialize MPI
@@ -336,10 +337,14 @@ PROGRAM subfiling_test
WRITE(*,"(A,A)") "Failed to find the stub subfile ",TRIM(filename)
nerrors = nerrors + 1
ENDIF
-
- CALL EXECUTE_COMMAND_LINE("stat --format='%i' "//filename//" >> tmp_inode", EXITSTAT=i)
+#ifdef H5_HAVE_DARWIN
+ arg(1:1)="f"
+#else
+ arg(1:1)="c"
+#endif
+ CALL EXECUTE_COMMAND_LINE("stat -"//arg(1:1)//" %i "//filename//" >> tmp_inode", EXITSTAT=i)
IF(i.ne.0)THEN
- WRITE(*,"(A,A)") "Failed to stat the stub subfile ",TRIM(filename)
+ WRITE(*,"(A,A)") "Failed to stat the stub subfile ",TRIM(filename)
nerrors = nerrors + 1
ENDIF
From 66396cadb14faf892907b1b7fd2f21854b7655f4 Mon Sep 17 00:00:00 2001
From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com>
Date: Mon, 23 Oct 2023 14:39:43 -0500
Subject: [PATCH 12/52] Fix windows cpack error in WiX package. (#3747)
---
doxygen/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doxygen/CMakeLists.txt b/doxygen/CMakeLists.txt
index 8fe3b771103..7dd7660621d 100644
--- a/doxygen/CMakeLists.txt
+++ b/doxygen/CMakeLists.txt
@@ -39,7 +39,7 @@ if (DOXYGEN_FOUND)
install (
DIRECTORY ${HDF5_BINARY_DIR}/hdf5lib_docs/html
DESTINATION ${HDF5_INSTALL_DOC_DIR}
- COMPONENT Documents
+ COMPONENT hdfdocuments
)
if (NOT TARGET doxygen)
From d76d591555c57f95460e23a12baff66c6150a124 Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Mon, 23 Oct 2023 12:45:41 -0700
Subject: [PATCH 13/52] Add a simple cache to the ros3 VFD (#3753)
Adds a small cache of the first N bytes of a file opened with the
read-only S3 (ros3) VFD, where N is 4kiB or the size of the file,
whichever is smaller. This avoids a lot of small I/O operations
on file open.
Addresses GitHub issue #3381
---
release_docs/RELEASE.txt | 10 ++++++
src/H5FDros3.c | 73 ++++++++++++++++++++++++++++------------
2 files changed, 61 insertions(+), 22 deletions(-)
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 0239a9e356f..19c7831862a 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -228,6 +228,16 @@ New Features
Library:
--------
+ - Added a simple cache to the read-only S3 VFD
+
+ The read-only S3 VFD now caches the first N bytes of a file stored
+ in S3 to avoid a lot of small I/O operations when opening files.
+ This cache is per-file and created when the file is opened.
+
+ N is currently 4kiB or the size of the file, whichever is smaller.
+
+ Addresses GitHub issue #3381
+
- Added new API function H5Pget_actual_selection_io_mode()
This function allows the user to determine if the library performed
diff --git a/src/H5FDros3.c b/src/H5FDros3.c
index 3f3413c6d0e..f704eff1efa 100644
--- a/src/H5FDros3.c
+++ b/src/H5FDros3.c
@@ -43,6 +43,9 @@
*/
#define ROS3_STATS 0
+/* Max size of the cache, in bytes */
+#define ROS3_MAX_CACHE_SIZE 4096
+
/* The driver identification number, initialized at runtime
*/
static hid_t H5FD_ROS3_g = 0;
@@ -189,6 +192,8 @@ typedef struct H5FD_ros3_t {
H5FD_ros3_fapl_t fa;
haddr_t eoa;
s3r_t *s3r_handle;
+ uint8_t *cache;
+ size_t cache_size;
#if ROS3_STATS
ros3_statsbin meta[ROS3_STATS_BIN_COUNT + 1];
ros3_statsbin raw[ROS3_STATS_BIN_COUNT + 1];
@@ -1000,6 +1005,18 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics");
#endif /* ROS3_STATS */
+ /* Cache the initial bytes of the file */
+ {
+ size_t filesize = H5FD_s3comms_s3r_get_filesize(file->s3r_handle);
+
+ file->cache_size = (filesize < ROS3_MAX_CACHE_SIZE) ? filesize : ROS3_MAX_CACHE_SIZE;
+
+ if (NULL == (file->cache = (uint8_t *)H5MM_calloc(file->cache_size)))
+ HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, NULL, "unable to allocate cache memory");
+ if (H5FD_s3comms_s3r_read(file->s3r_handle, 0, file->cache_size, file->cache) == FAIL)
+ HGOTO_ERROR(H5E_VFL, H5E_READERROR, NULL, "unable to execute read");
+ }
+
ret_value = (H5FD_t *)file;
done:
@@ -1007,8 +1024,10 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
if (handle != NULL)
if (FAIL == H5FD_s3comms_s3r_close(handle))
HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle");
- if (file != NULL)
+ if (file != NULL) {
+ H5MM_xfree(file->cache);
file = H5FL_FREE(H5FD_ros3_t, file);
+ }
curl_global_cleanup(); /* early cleanup because open failed */
} /* end if null return value (error) */
@@ -1335,6 +1354,7 @@ H5FD__ros3_close(H5FD_t H5_ATTR_UNUSED *_file)
#endif /* ROS3_STATS */
/* Release the file info */
+ H5MM_xfree(file->cache);
file = H5FL_FREE(H5FD_ros3_t, file);
done:
@@ -1666,41 +1686,50 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU
fprintf(stdout, "H5FD__ros3_read() called.\n");
#endif
- assert(file != NULL);
- assert(file->s3r_handle != NULL);
- assert(buf != NULL);
+ assert(file);
+ assert(file->cache);
+ assert(file->s3r_handle);
+ assert(buf);
filesize = H5FD_s3comms_s3r_get_filesize(file->s3r_handle);
if ((addr > filesize) || ((addr + size) > filesize))
HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address");
- if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL)
- HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read");
+ /* Copy from the cache when accessing the first N bytes of the file.
+ * Saves network I/O operations when opening files.
+ */
+ if (addr + size < file->cache_size) {
+ memcpy(buf, file->cache + addr, size);
+ }
+ else {
+ if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL)
+ HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read");
#if ROS3_STATS
- /* Find which "bin" this read fits in. Can be "overflow" bin. */
- for (bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++)
- if ((unsigned long long)size < ros3_stats_boundaries[bin_i])
- break;
- bin = (type == H5FD_MEM_DRAW) ? &file->raw[bin_i] : &file->meta[bin_i];
+ /* Find which "bin" this read fits in. Can be "overflow" bin. */
+ for (bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++)
+ if ((unsigned long long)size < ros3_stats_boundaries[bin_i])
+ break;
+ bin = (type == H5FD_MEM_DRAW) ? &file->raw[bin_i] : &file->meta[bin_i];
- /* Store collected stats in appropriate bin */
- if (bin->count == 0) {
- bin->min = size;
- bin->max = size;
- }
- else {
- if (size < bin->min)
+ /* Store collected stats in appropriate bin */
+ if (bin->count == 0) {
bin->min = size;
- if (size > bin->max)
bin->max = size;
- }
- bin->count++;
- bin->bytes += (unsigned long long)size;
+ }
+ else {
+ if (size < bin->min)
+ bin->min = size;
+ if (size > bin->max)
+ bin->max = size;
+ }
+ bin->count++;
+ bin->bytes += (unsigned long long)size;
#endif /* ROS3_STATS */
+ }
done:
FUNC_LEAVE_NOAPI(ret_value)
From 744a20fc21be7437bb2449e6b6165702703a1349 Mon Sep 17 00:00:00 2001
From: bmribler <39579120+bmribler@users.noreply.github.com>
Date: Mon, 23 Oct 2023 15:46:56 -0400
Subject: [PATCH 14/52] Add helpful text to h5clear (#3754)
Added text to the usage of h5clear to explain that this tool is not
for fixing corrupted files but simply for helping in the inspection
of the damage..
Fixed expected output for testing the modified usage.
---
tools/src/misc/h5clear.c | 11 +++++++++--
tools/test/misc/expected/h5clear_missing_file.ddl | 11 +++++++++--
tools/test/misc/expected/h5clear_usage.ddl | 11 +++++++++--
3 files changed, 27 insertions(+), 6 deletions(-)
diff --git a/tools/src/misc/h5clear.c b/tools/src/misc/h5clear.c
index 31f7c1744f8..ea3e07249b1 100644
--- a/tools/src/misc/h5clear.c
+++ b/tools/src/misc/h5clear.c
@@ -61,6 +61,13 @@ static struct h5_long_options l_opts[] = {
static void
usage(const char *prog)
{
+ fprintf(stdout, "h5clear clears superblock status flag field, removes metadata cache image, prints\n");
+ fprintf(stdout, "EOA and EOF, or sets EOA of a file. It is not a general repair tool and should not\n");
+ fprintf(stdout, "be used to fix file corruption. If a process doesn't shut down cleanly, the\n");
+ fprintf(stdout, "superblock mark can be left that prevents opening a file without SWMR. Then,\n");
+ fprintf(stdout, "h5clear can be used to remove this superblock mark so that the file can be inspected\n");
+ fprintf(stdout, "and appropriate actions can be taken.\n");
+ fprintf(stdout, "\n");
fprintf(stdout, "usage: %s [OPTIONS] file_name\n", prog);
fprintf(stdout, " OPTIONS\n");
fprintf(stdout, " -h, --help Print a usage message and exit\n");
@@ -73,8 +80,8 @@ usage(const char *prog)
fprintf(stdout,
" C is >= 0; C is optional and will default to 1M when not set.\n");
fprintf(stdout,
- " This option helps to repair a crashed file where the stored EOA\n");
- fprintf(stdout, " in the superblock is different from the actual EOF.\n");
+ " This option helps to repair a crashed SWMR file when the stored\n");
+ fprintf(stdout, " EOA in the superblock is different from the actual EOF.\n");
fprintf(stdout, " The file's EOA and EOF will be the same after applying\n");
fprintf(stdout, " this option to the file.\n");
fprintf(stdout, "\n");
diff --git a/tools/test/misc/expected/h5clear_missing_file.ddl b/tools/test/misc/expected/h5clear_missing_file.ddl
index c7a21189ea7..fe659af9593 100644
--- a/tools/test/misc/expected/h5clear_missing_file.ddl
+++ b/tools/test/misc/expected/h5clear_missing_file.ddl
@@ -1,3 +1,10 @@
+h5clear clears superblock status flag field, removes metadata cache image, prints
+EOA and EOF, or sets EOA of a file. It is not a general repair tool and should not
+be used to fix file corruption. If a process doesn't shut down cleanly, the
+superblock mark can be left that prevents opening a file without SWMR. Then,
+h5clear can be used to remove this superblock mark so that the file can be inspected
+and appropriate actions can be taken.
+
usage: h5clear [OPTIONS] file_name
OPTIONS
-h, --help Print a usage message and exit
@@ -8,8 +15,8 @@ usage: h5clear [OPTIONS] file_name
--increment=C Set the file's EOA to the maximum of (EOA, EOF) + C for
the file .
C is >= 0; C is optional and will default to 1M when not set.
- This option helps to repair a crashed file where the stored EOA
- in the superblock is different from the actual EOF.
+ This option helps to repair a crashed SWMR file when the stored
+ EOA in the superblock is different from the actual EOF.
The file's EOA and EOF will be the same after applying
this option to the file.
diff --git a/tools/test/misc/expected/h5clear_usage.ddl b/tools/test/misc/expected/h5clear_usage.ddl
index c7a21189ea7..fe659af9593 100644
--- a/tools/test/misc/expected/h5clear_usage.ddl
+++ b/tools/test/misc/expected/h5clear_usage.ddl
@@ -1,3 +1,10 @@
+h5clear clears superblock status flag field, removes metadata cache image, prints
+EOA and EOF, or sets EOA of a file. It is not a general repair tool and should not
+be used to fix file corruption. If a process doesn't shut down cleanly, the
+superblock mark can be left that prevents opening a file without SWMR. Then,
+h5clear can be used to remove this superblock mark so that the file can be inspected
+and appropriate actions can be taken.
+
usage: h5clear [OPTIONS] file_name
OPTIONS
-h, --help Print a usage message and exit
@@ -8,8 +15,8 @@ usage: h5clear [OPTIONS] file_name
--increment=C Set the file's EOA to the maximum of (EOA, EOF) + C for
the file .
C is >= 0; C is optional and will default to 1M when not set.
- This option helps to repair a crashed file where the stored EOA
- in the superblock is different from the actual EOF.
+ This option helps to repair a crashed SWMR file when the stored
+ EOA in the superblock is different from the actual EOF.
The file's EOA and EOF will be the same after applying
this option to the file.
From a6d1bda6d2da6121a2d8f66cd7a1e0d2cafe2da6 Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Mon, 23 Oct 2023 12:50:15 -0700
Subject: [PATCH 15/52] Update Autotools to correctly configure oneAPI (#3751)
* Update Autotools to correctly configure oneAPI
Splits the Intel config files under the Autotools into 'classic'
Intel and oneAPI versions, fixing 'unsupported option' messages.
Also turns off `-check uninit` (new in 2023) in Fortran, which kills
the H5_buildiface program due to false positives.
* Enable Fortran in oneAPI CI workflow
* Turn on Fortran in CMake, update LD_LIBRARY_PATH
* Go back to disabling Fortran w/ Intel
For some reason there's a linking problem w/ Fortran
error while loading shared libraries: libifport.so.5: cannot open shared object file: No such file or directory
---
config/apple | 42 +++++++----
config/freebsd | 16 +++-
config/linux-gnulibc1 | 33 ++++++++-
config/netbsd | 13 +++-
config/oneapi-cxxflags | 155 +++++++++++++++++++++++++++++++++++++++
config/oneapi-fflags | 145 ++++++++++++++++++++++++++++++++++++
config/oneapi-flags | 151 ++++++++++++++++++++++++++++++++++++++
release_docs/RELEASE.txt | 8 ++
8 files changed, 539 insertions(+), 24 deletions(-)
create mode 100644 config/oneapi-cxxflags
create mode 100644 config/oneapi-fflags
create mode 100644 config/oneapi-flags
diff --git a/config/apple b/config/apple
index a8a219b6798..39ed454a11f 100644
--- a/config/apple
+++ b/config/apple
@@ -55,30 +55,19 @@ fi
# Figure out C compiler flags
. $srcdir/config/gnu-flags
. $srcdir/config/clang-flags
+. $srcdir/config/oneapi-flags
. $srcdir/config/intel-flags
-# temp patch: if GCC 4.2.1 is used in Lion or Mountain Lion systems, do not
-# use -O option as it causes failures in test/dt_arith.
-case "$host_os" in
- darwin1[12].*) # lion & mountain lion
- #echo cc_vendor=$cc_vendor'-'cc_version=$cc_version
- case "$cc_vendor-$cc_version" in
- gcc-4.2.1)
- # Remove any -O flags
- #echo PROD_CFLAGS=$PROD_CFLAGS
- PROD_CFLAGS="`echo $PROD_CFLAGS | sed -e 's/-O[0-3]*//'`"
- #echo new PROD_CFLAGS=$PROD_CFLAGS
- ;;
- esac
- ;;
-esac
-
if test "X-" = "X-$FC"; then
case $CC_BASENAME in
gcc*)
FC=gfortran
FC_BASENAME=gfortran
;;
+ icx*)
+ FC=ifx
+ FC_BASENAME=ifx
+ ;;
icc*)
FC=ifort
FC_BASENAME=ifort
@@ -97,6 +86,7 @@ fi
# Figure out FORTRAN compiler flags
. $srcdir/config/gnu-fflags
+. $srcdir/config/oneapi-fflags
. $srcdir/config/intel-fflags
@@ -107,6 +97,10 @@ if test "X-" = "X-$CXX"; then
CXX=g++
CXX_BASENAME=g++
;;
+ icx)
+ CXX=icpx
+ CXX_BASENAME=icpx
+ ;;
icc)
CXX=icpc
CXX_BASENAME=icpc
@@ -123,6 +117,7 @@ if test "X-" = "X-$CXX"; then
fi
# Figure out C++ compiler flags
+. $srcdir/config/oneapi-cxxflags
. $srcdir/config/intel-cxxflags # Do this ahead of GNU to avoid icpc being detected as g++
. $srcdir/config/gnu-cxxflags
. $srcdir/config/clang-cxxflags
@@ -139,6 +134,11 @@ case $CC in
grep 'GCC' | sed 's/.*\((GCC) [-a-z0-9\. ]*.*\)/\1/'`
;;
+ *icx*)
+ cc_version_info=`$CC $CCFLAGS $H5_CCFLAGS -V 2>&1 | grep 'Version' |\
+ sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'`
+ ;;
+
*icc*)
cc_version_info=`$CC $CCFLAGS $H5_CCFLAGS -V 2>&1 | grep 'Version' |\
sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'`
@@ -156,6 +156,11 @@ case $FC in
grep 'GCC' | sed 's/\(.*(GCC) [-a-z0-9\. ]*\).*/\1/'`
;;
+ *ifx*)
+ fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'Version' |\
+ sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'`
+ ;;
+
*ifc*|*ifort*)
fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'Version' |\
sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'`
@@ -179,6 +184,11 @@ case $CXX in
grep 'GCC' | sed 's/.*\((GCC) [-a-z0-9\. ]*.*\)/\1/'`
;;
+ *icpx*)
+ cxx_version_info=`$CXX $CXXFLAGS $H5_CXXFLAGS -V 2>&1 | grep 'Version' |\
+ sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'`
+ ;;
+
*icpc*)
cxx_version_info=`$CXX $CXXFLAGS $H5_CXXFLAGS -V 2>&1 | grep 'Version' |\
sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'`
diff --git a/config/freebsd b/config/freebsd
index 2fb962fe308..b0e825a9e26 100644
--- a/config/freebsd
+++ b/config/freebsd
@@ -29,7 +29,10 @@ fi
# Figure out GNU C compiler flags
. $srcdir/config/gnu-flags
-# Figure out Intel C compiler flags
+# Figure out Intel oneAPI C compiler flags
+. $srcdir/config/oneapi-flags
+
+# Figure out Intel classic C compiler flags
. $srcdir/config/intel-flags
# The default Fortran 90 compiler
@@ -43,6 +46,10 @@ if test "X-" = "X-$FC"; then
FC=gfortran
FC_BASENAME=gfortran
;;
+ icx*)
+ FC=ifx
+ FC_BASENAME=ifx
+ ;;
icc*)
FC=ifort
FC_BASENAME=ifort
@@ -57,8 +64,11 @@ fi
# Figure out FORTRAN compiler flags
. $srcdir/config/gnu-fflags
-# Figure out Intel F90 compiler flags
-. $srcdir/config/intel-fflags
+# Figure out Intel oneAPI FC compiler flags
+. $srcdir/config/oneapi-fflags
+
+# Figure out Intel classic FC compiler flags
+. $srcdir/config/classic-fflags
# The default C++ compiler
diff --git a/config/linux-gnulibc1 b/config/linux-gnulibc1
index 7614b07852f..328f8d3cec3 100644
--- a/config/linux-gnulibc1
+++ b/config/linux-gnulibc1
@@ -38,7 +38,10 @@ fi
# Figure out CCE C compiler flags
. $srcdir/config/cce-flags
-# Figure out Intel C compiler flags
+# Figure out Intel oneAPI C compiler flags
+. $srcdir/config/oneapi-flags
+
+# Figure out Intel classic C compiler flags
. $srcdir/config/intel-flags
# Figure out Clang C compiler flags
@@ -55,6 +58,10 @@ if test "X-" = "X-$FC"; then
FC=pgf90
FC_BASENAME=pgf90
;;
+ icx*)
+ FC=ifx
+ FC_BASENAME=ifx
+ ;;
icc*)
FC=ifort
FC_BASENAME=ifort
@@ -119,7 +126,10 @@ fi
# Figure out CCE FC compiler flags
. $srcdir/config/cce-fflags
-# Figure out Intel FC compiler flags
+# Figure out Intel oneAPI FC compiler flags
+. $srcdir/config/oneapi-fflags
+
+# Figure out Intel classic FC compiler flags
. $srcdir/config/intel-fflags
# Figure out Clang FC compiler flags
@@ -200,7 +210,10 @@ if test -z "$CXX"; then
CXX_BASENAME=g++
fi
-# Figure out Intel CXX compiler flags
+# Figure out Intel oneAPI CXX compiler flags
+. $srcdir/config/oneapi-cxxflags
+
+# Figure out Intel classic CXX compiler flags
# Do this ahead of GNU to avoid icpc being detected as g++
. $srcdir/config/intel-cxxflags
@@ -237,6 +250,11 @@ case $CC in
cc_version_info=`echo $cc_version_info`
;;
+ *icx*)
+ cc_version_info=`$CC $CCFLAGS $H5_CCFLAGS -V 2>&1 | grep 'Version' |\
+ sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'`
+ ;;
+
*icc*)
cc_version_info=`$CC $CCFLAGS $H5_CCFLAGS -V 2>&1 | grep 'Version' |\
sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'`
@@ -271,6 +289,11 @@ case $FC in
fc_version_info=`echo $fc_version_info`
;;
+ *ifx*)
+ fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'Version' |\
+ sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'`
+ ;;
+
*ifc*|*ifort*)
fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'Version' |\
sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'`
@@ -333,6 +356,10 @@ case $CXX in
cxx_version_info=`$CXX $CXXFLAGS $H5_CXXFLAGS --version 2>&1 |\
grep 'GCC' | sed 's/\(.*(GCC) [-a-z0-9\. ]*\).*/\1/'`
;;
+ *icpx*)
+ cxx_version_info=`$CXX $CXXFLAGS $H5_CXXFLAGS -V 2>&1 | grep 'Version' |\
+ sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'`
+ ;;
*icpc*)
cxx_version_info=`$CXX $CXXFLAGS $H5_CXXFLAGS -V 2>&1 | grep 'Version' |\
sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'`
diff --git a/config/netbsd b/config/netbsd
index 04761f294a8..0ed84f7b3d2 100644
--- a/config/netbsd
+++ b/config/netbsd
@@ -26,7 +26,10 @@ fi
# Figure out C compiler flags
. $srcdir/config/gnu-flags
-# Figure out Intel C compiler flags
+# Figure out Intel oneAPI C compiler flags
+. $srcdir/config/oneapi-flags
+
+# Figure out Intel classic C compiler flags
. $srcdir/config/intel-flags
# The default Fortran 90 compiler
@@ -36,6 +39,10 @@ if test "X-" = "X-$FC"; then
FC=gfortran
FC_BASENAME=gfortran
;;
+ icx*)
+ FC=ifx
+ FC_BASENAME=ifx
+ ;;
icc*)
FC=ifort
FC_BASENAME=ifort
@@ -50,6 +57,8 @@ fi
# Figure out FORTRAN compiler flags
. $srcdir/config/gnu-fflags
-# Figure out Intel F90 compiler flags
+# Figure out Intel oneAPI FC compiler flags
. $srcdir/config/intel-fflags
+# Figure out Intel classic FC compiler flags
+. $srcdir/config/oneapi-fflags
diff --git a/config/oneapi-cxxflags b/config/oneapi-cxxflags
new file mode 100644
index 00000000000..d9819b94c44
--- /dev/null
+++ b/config/oneapi-cxxflags
@@ -0,0 +1,155 @@
+# -*- shell-script -*-
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://www.hdfgroup.org/licenses.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+
+
+# This file should be sourced into configure if the compiler is the
+# Intel icpx compiler or a derivative. It is careful not to do anything
+# if the compiler is not Intel; otherwise `cxx_flags_set' is set to `yes'
+#
+
+#
+# Prepend `$srcdir/config/intel-warnings/` to the filename suffix(es) given as
+# subroutine argument(s), remove comments starting with # and ending
+# at EOL, replace spans of whitespace (including newlines) with spaces,
+# and re-emit the file(s) thus filtered on the standard output stream.
+#
+load_intel_arguments()
+{
+ set -- $(for arg; do
+ sed 's,#.*$,,' $srcdir/config/intel-warnings/${arg}
+ done)
+ IFS=' ' echo "$*"
+}
+
+# Get the compiler version in a way that works for icpx
+# icpx unless a compiler version is already known
+#
+# cxx_vendor: The compiler name: icpx
+# cxx_version: Version number: 2023.2.0
+#
+if test X = "X$cxx_flags_set"; then
+ cxx_version="`$CXX $CXXFLAGS $H5_CXXFLAGS -V 2>&1 |grep 'oneAPI'`"
+ if test X != "X$cxx_version"; then
+ cxx_vendor=icpx
+ cxx_version=`echo $cxx_version |sed 's/.*Version \([-a-z0-9\.\-]*\).*/\1/'`
+ echo "compiler '$CXX' is Intel oneAPI $cxx_vendor-$cxx_version"
+
+ # Some version numbers
+ # Intel oneAPI version numbers are of the form: "major.minor.patch"
+ cxx_vers_major=`echo $cxx_version | cut -f1 -d.`
+ cxx_vers_minor=`echo $cxx_version | cut -f2 -d.`
+ cxx_vers_patch=`echo $cxx_version | cut -f2 -d.`
+ test -n "$cxx_vers_major" || cxx_vers_major=0
+ test -n "$cxx_vers_minor" || cxx_vers_minor=0
+ test -n "$cxx_vers_patch" || cxx_vers_patch=0
+ cxx_vers_all=`expr $cxx_vers_major '*' 1000000 + $cxx_vers_minor '*' 1000 + $cxx_vers_patch`
+ fi
+fi
+
+# Common Intel flags for various situations
+if test "X-icpx" = "X-$cxx_vendor"; then
+ # Insert section about version specific problems from compiler flags here,
+ # if necessary.
+
+ arch=
+ # Architecture-specific flags
+ # Nothing currently. (Uncomment code below and modify to add any)
+ #case "$host_os-$host_cpu" in
+ # *-i686)
+ # arch="-march=i686"
+ # ;;
+ #esac
+
+ # Host-specific flags
+ # Nothing currently. (Uncomment code below and modify to add any)
+ #case "`hostname`" in
+ # sleipnir.ncsa.uiuc.edu)
+ # arch="$arch -pipe"
+ # ;;
+ #esac
+
+ ###########
+ # General #
+ ###########
+
+ # Default to C++11 standard
+ H5_CXXFLAGS="$H5_CXXFLAGS $arch -std=c++11"
+
+ ##############
+ # Production #
+ ##############
+
+ PROD_CXXFLAGS=
+
+ #########
+ # Debug #
+ #########
+
+ # NDEBUG is handled explicitly in configure
+ # -g is handled by the symbols flags
+ DEBUG_CXXFLAGS=
+
+ ###########
+ # Symbols #
+ ###########
+
+ NO_SYMBOLS_CXXFLAGS="-Wl,-s"
+ SYMBOLS_CXXFLAGS="-g"
+
+ #############
+ # Profiling #
+ #############
+
+ PROFILE_CXXFLAGS="-p"
+
+ ################
+ # Optimization #
+ ################
+
+ HIGH_OPT_CXXFLAGS="-O3"
+ DEBUG_OPT_CXXFLAGS="-O0"
+ NO_OPT_CXXFLAGS="-O0"
+
+ ############
+ # Warnings #
+ ############
+
+ ###########
+ # General #
+ ###########
+
+ # Add various general warning flags in intel-warnings.
+ # Use the C warnings as CXX warnings are the same
+ H5_CXXFLAGS="$H5_CXXFLAGS $(load_intel_arguments oneapi/general)"
+
+ ######################
+ # Developer warnings #
+ ######################
+
+ # Use the C warnings as CXX warnings are the same
+ DEVELOPER_WARNING_CXXFLAGS=$(load_intel_arguments oneapi/developer-general)
+
+ #############################
+ # Version-specific warnings #
+ #############################
+
+ #################
+ # Flags are set #
+ #################
+ cxx_flags_set=yes
+fi
+
+# Clear cxx info if no flags set
+if test "X-$cxx_flags_set" = "X-"; then
+ cxx_vendor=
+ cxx_version=
+fi
diff --git a/config/oneapi-fflags b/config/oneapi-fflags
new file mode 100644
index 00000000000..a63108d0b99
--- /dev/null
+++ b/config/oneapi-fflags
@@ -0,0 +1,145 @@
+# -*- shell-script -*-
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://www.hdfgroup.org/licenses.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+
+
+# This file should be sourced into configure if the compiler is the
+# Intel oneAPI ifx compiler or a derivative. It is careful not to do anything
+# if the compiler is not Intel; otherwise `f9x_flags_set' is set to `yes'
+#
+
+#
+# Prepend `$srcdir/config/intel-warnings/` to the filename suffix(es) given as
+# subroutine argument(s), remove comments starting with # and ending
+# at EOL, replace spans of whitespace (including newlines) with spaces,
+# and re-emit the file(s) thus filtered on the standard output stream.
+#
+load_intel_arguments()
+{
+ set -- $(for arg; do
+ sed 's,#.*$,,' $srcdir/config/intel-warnings/${arg}
+ done)
+ IFS=' ' echo "$*"
+}
+
+# Get the compiler version in a way that works for ifx
+# ifx unless a compiler version is already known
+#
+# f9x_vendor: The compiler name: ifx
+# f9x_version: Version number: 2023.2.0
+#
+if test X = "X$f9x_flags_set"; then
+ f9x_version="`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 |grep '^Intel'`"
+ if test X != "X$f9x_version"; then
+ f9x_vendor=ifx
+ f9x_version="`echo $f9x_version |sed 's/.*Version \([-a-z0-9\.\-]*\).*/\1/'`"
+ echo "compiler '$FC' is Intel oneAPI $f9x_vendor-$f9x_version"
+
+ # Some version numbers
+ # Intel oneAPI version numbers are of the form: "major.minor.patch"
+ f9x_vers_major=`echo $f9x_version | cut -f1 -d.`
+ f9x_vers_minor=`echo $f9x_version | cut -f2 -d.`
+ f9x_vers_patch=`echo $f9x_version | cut -f2 -d.`
+ test -n "$f9x_vers_major" || f9x_vers_major=0
+ test -n "$f9x_vers_minor" || f9x_vers_minor=0
+ test -n "$f9x_vers_patch" || f9x_vers_patch=0
+ f9x_vers_all=`expr $f9x_vers_major '*' 1000000 + $f9x_vers_minor '*' 1000 + $f9x_vers_patch`
+ fi
+fi
+
+if test "X-ifx" = "X-$f9x_vendor"; then
+
+ FC_BASENAME=ifx
+ F9XSUFFIXFLAG=""
+ FSEARCH_DIRS=""
+
+ ###############################
+ # Architecture-specific flags #
+ ###############################
+
+ arch=
+ # Nothing currently. (Uncomment code below and modify to add any)
+ #case "$host_os-$host_cpu" in
+ # *-i686)
+ # arch="-march=i686"
+ # ;;
+ #esac
+
+ # Host-specific flags
+ # Nothing currently. (Uncomment code below and modify to add any)
+ #case "`hostname`" in
+ # sleipnir.ncsa.uiuc.edu)
+ # arch="$arch -pipe"
+ # ;;
+ #esac
+
+ ##############
+ # Production #
+ ##############
+
+ PROD_FCFLAGS=
+
+ #########
+ # Debug #
+ #########
+
+ # Don't use -check uninit or you'll get false positives from H5_buildiface
+ DEBUG_FCFLAGS="-check all,nouninit"
+
+ ###########
+ # Symbols #
+ ###########
+
+ NO_SYMBOLS_FCFLAGS=
+ SYMBOLS_FCFLAGS="-g"
+
+ #############
+ # Profiling #
+ #############
+
+ PROFILE_FCFLAGS="-p"
+
+ ################
+ # Optimization #
+ ################
+
+ HIGH_OPT_FCFLAGS="-O3"
+ DEBUG_OPT_FCFLAGS="-O0"
+ NO_OPT_FCFLAGS="-O0"
+
+ ############
+ # Warnings #
+ ############
+
+ ###########
+ # General #
+ ###########
+
+ H5_FCFLAGS="$H5_FCFLAGS -free"
+ H5_FCFLAGS="$H5_FCFLAGS $(load_intel_arguments oneapi/ifort-general)"
+
+ #############################
+ # Version-specific warnings #
+ #############################
+
+
+ #################
+ # Flags are set #
+ #################
+ f9x_flags_set=yes
+fi
+
+# Clear f9x info if no flags set
+if test "X-$f9x_flags_set" = "X-"; then
+ f9x_vendor=
+ f9x_version=
+fi
+
diff --git a/config/oneapi-flags b/config/oneapi-flags
new file mode 100644
index 00000000000..629e93f02f1
--- /dev/null
+++ b/config/oneapi-flags
@@ -0,0 +1,151 @@
+# -*- shell-script -*-
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://www.hdfgroup.org/licenses.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+
+
+# This file should be sourced into configure if the compiler is the
+# Intel icx compiler or a derivative. It is careful not to do anything
+# if the compiler is not Intel; otherwise `cc_flags_set' is set to `yes'
+#
+
+#
+# Prepend `$srcdir/config/intel-warnings/` to the filename suffix(es) given as
+# subroutine argument(s), remove comments starting with # and ending
+# at EOL, replace spans of whitespace (including newlines) with spaces,
+# and re-emit the file(s) thus filtered on the standard output stream.
+#
+load_intel_arguments()
+{
+ set -- $(for arg; do
+ sed 's,#.*$,,' $srcdir/config/intel-warnings/${arg}
+ done)
+ IFS=' ' echo "$*"
+}
+
+# Get the compiler version in a way that works for icx
+# icx unless a compiler version is already known
+# cc_vendor: The compiler name: icx
+# cc_version: Version number: 2023.2.0
+#
+if test X = "X$cc_flags_set"; then
+ cc_version="`$CC $CFLAGS $H5_CFLAGS -V 2>&1 |grep 'oneAPI'`"
+ if test X != "X$cc_version"; then
+ cc_vendor=icx
+ cc_version=`echo $cc_version |sed 's/.*Version \([-a-z0-9\.\-]*\).*/\1/'`
+ echo "compiler '$CC' is Intel oneAPI $cc_vendor-$cc_version"
+
+ # Some version numbers
+ # Intel oneAPI version numbers are of the form: "major.minor.patch"
+ cc_vers_major=`echo $cc_version | cut -f1 -d.`
+ cc_vers_minor=`echo $cc_version | cut -f2 -d.`
+ cc_vers_patch=`echo $cc_version | cut -f2 -d.`
+ test -n "$cc_vers_major" || cc_vers_major=0
+ test -n "$cc_vers_minor" || cc_vers_minor=0
+ test -n "$cc_vers_patch" || cc_vers_patch=0
+ cc_vers_all=`expr $cc_vers_major '*' 1000000 + $cc_vers_minor '*' 1000 + $cc_vers_patch`
+ fi
+fi
+
+# Common Intel flags for various situations
+if test "X-icx" = "X-$cc_vendor"; then
+ # Insert section about version specific problems from compiler flags here,
+ # if necessary.
+
+ arch=
+ # Architecture-specific flags
+ # Nothing currently. (Uncomment code below and modify to add any)
+ #case "$host_os-$host_cpu" in
+ # *-i686)
+ # arch="-march=i686"
+ # ;;
+ #esac
+
+ # Host-specific flags
+ # Nothing currently. (Uncomment code below and modify to add any)
+ #case "`hostname`" in
+ # sleipnir.ncsa.uiuc.edu)
+ # arch="$arch -pipe"
+ # ;;
+ #esac
+
+ ###########
+ # General #
+ ###########
+
+ # Default to C99 standard.
+ H5_CFLAGS="$H5_CFLAGS $arch -std=c99"
+
+ ##############
+ # Production #
+ ##############
+
+ PROD_CFLAGS=
+
+ #########
+ # Debug #
+ #########
+
+ # NDEBUG is handled explicitly in configure
+ DEBUG_CFLAGS=
+
+ ###########
+ # Symbols #
+ ###########
+
+ NO_SYMBOLS_CFLAGS="-Wl,-s"
+ SYMBOLS_CFLAGS="-g"
+
+ #############
+ # Profiling #
+ #############
+
+ PROFILE_CFLAGS="-p"
+
+ ################
+ # Optimization #
+ ################
+
+ HIGH_OPT_CFLAGS="-O3"
+ DEBUG_OPT_CFLAGS="-O0"
+ NO_OPT_CFLAGS="-O0"
+
+ ############
+ # Warnings #
+ ############
+
+ ###########
+ # General #
+ ###########
+
+ # Add various general warning flags in intel-warnings.
+ H5_CFLAGS="$H5_CFLAGS $(load_intel_arguments oneapi/general)"
+
+ ######################
+ # Developer warnings #
+ ######################
+
+ DEVELOPER_WARNING_CFLAGS=$(load_intel_arguments oneapi/developer-general)
+
+ #############################
+ # Version-specific warnings #
+ #############################
+
+ #################
+ # Flags are set #
+ #################
+ cc_flags_set=yes
+fi
+
+# Clear cc info if no flags set
+if test "X-$cc_flags_set" = "X-"; then
+ cc_vendor=
+ cc_version=
+fi
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 19c7831862a..9643cf9ecf6 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -47,6 +47,14 @@ New Features
Configuration:
-------------
+ - Improved support for Intel oneAPI
+
+ * Separates the old 'classic' Intel compiler settings and warnings
+ from the oneAPI settings
+ * Uses `-check nouninit` in debug builds to avoid false positives
+ when building H5_buildiface with `-check all`
+ * Both Autotools and CMake
+
- Added new options for CMake and Autotools to control the Doxygen
warnings as errors setting.
From da3b7ff945c931878df85bca49b9c21a89e5a824 Mon Sep 17 00:00:00 2001
From: Scot Breitenfeld
Date: Mon, 23 Oct 2023 14:55:47 -0500
Subject: [PATCH 16/52] Add h5pget_actual_selection_io_mode fortran wrapper
(#3746)
* added h5pget_actual_selection_io_mode_f test
* added tests for h5pget_actual_selection_io_mode_f
* fixed int_f type conversion
---
fortran/src/H5Pff.F90 | 38 ++++++++++++++++++++++++++++++++++++-
fortran/src/H5_f.c | 4 ++++
fortran/src/H5_ff.F90 | 5 ++++-
fortran/src/H5f90global.F90 | 10 ++++++++++
fortran/test/tH5P.F90 | 5 +++++
fortran/testpar/hyper.F90 | 15 +++++++++++++++
release_docs/RELEASE.txt | 3 ++-
7 files changed, 77 insertions(+), 3 deletions(-)
diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90
index bbc7a9dc066..5821889c3e9 100644
--- a/fortran/src/H5Pff.F90
+++ b/fortran/src/H5Pff.F90
@@ -6405,7 +6405,7 @@ END SUBROUTINE H5Pset_file_space_strategy_f
!! \brief Gets the file space handling strategy and persisting free-space values for a file creation property list.
!!
!! \param plist_id File creation property list identifier
-!! \param strategy The file space handling strategy to be used.
+!! \param strategy The file space handling strategy to be used
!! \param persist Indicate whether free space should be persistent or not
!! \param threshold The free-space section size threshold value
!! \param hdferr \fortran_error
@@ -6507,6 +6507,42 @@ END FUNCTION H5Pget_file_space_page_size
hdferr = INT(h5pget_file_space_page_size(prp_id, fsp_size))
END SUBROUTINE h5pget_file_space_page_size_f
+!>
+!! \ingroup FH5P
+!!
+!! \brief Retrieves the type(s) of I/O that HDF5 actually performed on raw data
+!! during the last I/O call.
+!!
+!! \param plist_id File creation property list identifier
+!! \param actual_selection_io_mode A bitwise set value indicating the type(s) of I/O performed
+!! \param hdferr \fortran_error
+!!
+!! See C API: @ref H5Pget_actual_selection_io_mode()
+!!
+ SUBROUTINE h5pget_actual_selection_io_mode_f(plist_id, actual_selection_io_mode, hdferr)
+
+ IMPLICIT NONE
+ INTEGER(HID_T), INTENT(IN) :: plist_id
+ INTEGER , INTENT(OUT) :: actual_selection_io_mode
+ INTEGER , INTENT(OUT) :: hdferr
+
+ INTEGER(C_INT32_T) :: c_actual_selection_io_mode
+
+ INTERFACE
+ INTEGER(C_INT) FUNCTION H5Pget_actual_selection_io_mode(plist_id, actual_selection_io_mode) &
+ BIND(C, NAME='H5Pget_actual_selection_io_mode')
+ IMPORT :: HID_T, C_INT32_T, C_INT
+ IMPLICIT NONE
+ INTEGER(HID_T), VALUE :: plist_id
+ INTEGER(C_INT32_T) :: actual_selection_io_mode
+ END FUNCTION H5Pget_actual_selection_io_mode
+ END INTERFACE
+
+ hdferr = INT(H5Pget_actual_selection_io_mode(plist_id, c_actual_selection_io_mode))
+
+ actual_selection_io_mode = INT(c_actual_selection_io_mode)
+
+ END SUBROUTINE h5pget_actual_selection_io_mode_f
END MODULE H5P
diff --git a/fortran/src/H5_f.c b/fortran/src/H5_f.c
index 181047b5454..0392c2bdfa6 100644
--- a/fortran/src/H5_f.c
+++ b/fortran/src/H5_f.c
@@ -477,6 +477,10 @@ h5init_flags_c(int_f *h5d_flags, size_t_f *h5d_size_flags, int_f *h5e_flags, hid
h5d_flags[55] = (int_f)H5D_MPIO_LINK_CHUNK;
h5d_flags[56] = (int_f)H5D_MPIO_MULTI_CHUNK;
+ h5d_flags[57] = (int_f)H5D_SCALAR_IO;
+ h5d_flags[58] = (int_f)H5D_VECTOR_IO;
+ h5d_flags[59] = (int_f)H5D_SELECTION_IO;
+
/*
* H5E flags
*/
diff --git a/fortran/src/H5_ff.F90 b/fortran/src/H5_ff.F90
index 68b3dd874bb..53156731946 100644
--- a/fortran/src/H5_ff.F90
+++ b/fortran/src/H5_ff.F90
@@ -74,7 +74,7 @@ MODULE H5LIB
!
! H5D flags declaration
!
- INTEGER, PARAMETER :: H5D_FLAGS_LEN = 57
+ INTEGER, PARAMETER :: H5D_FLAGS_LEN = 60
INTEGER, DIMENSION(1:H5D_FLAGS_LEN) :: H5D_flags
INTEGER, PARAMETER :: H5D_SIZE_FLAGS_LEN = 2
INTEGER(SIZE_T), DIMENSION(1:H5D_SIZE_FLAGS_LEN) :: H5D_size_flags
@@ -467,6 +467,9 @@ END FUNCTION h5init1_flags_c
H5D_MPIO_NO_CHUNK_OPTIMIZATION_F = H5D_flags(55)
H5D_MPIO_LINK_CHUNK_F = H5D_flags(56)
H5D_MPIO_MULTI_CHUNK_F = H5D_flags(57)
+ H5D_SCALAR_IO_F = H5D_flags(58)
+ H5D_VECTOR_IO_F = H5D_flags(59)
+ H5D_SELECTION_IO_F = H5D_flags(60)
H5D_CHUNK_CACHE_NSLOTS_DFLT_F = H5D_size_flags(1)
H5D_CHUNK_CACHE_NBYTES_DFLT_F = H5D_size_flags(2)
diff --git a/fortran/src/H5f90global.F90 b/fortran/src/H5f90global.F90
index f6c06cbc056..aa046235eb3 100644
--- a/fortran/src/H5f90global.F90
+++ b/fortran/src/H5f90global.F90
@@ -374,6 +374,12 @@ MODULE H5GLOBAL
!DEC$ATTRIBUTES DLLEXPORT :: H5D_MPIO_NO_CHUNK_OPTIMIZATION_F
!DEC$ATTRIBUTES DLLEXPORT :: H5D_MPIO_LINK_CHUNK_F
!DEC$ATTRIBUTES DLLEXPORT :: H5D_MPIO_MULTI_CHUNK_F
+
+ !DEC$ATTRIBUTES DLLEXPORT :: H5D_SCALAR_IO_F
+ !DEC$ATTRIBUTES DLLEXPORT :: H5D_VECTOR_IO_F
+ !DEC$ATTRIBUTES DLLEXPORT :: H5D_SELECTION_IO_F
+
+
!DEC$endif
!> \addtogroup FH5D
!> @{
@@ -450,6 +456,10 @@ MODULE H5GLOBAL
INTEGER :: H5D_MPIO_NO_CHUNK_OPTIMIZATION_F !< H5D_MPIO_NO_CHUNK_OPTIMIZATION
INTEGER :: H5D_MPIO_LINK_CHUNK_F !< H5D_MPIO_LINK_CHUNK
INTEGER :: H5D_MPIO_MULTI_CHUNK_F !< H5D_MPIO_MULTI_CHUNK
+
+ INTEGER :: H5D_SCALAR_IO_F !< Scalar (or legacy MPIO) I/O was performed
+ INTEGER :: H5D_VECTOR_IO_F !< Vector I/O was performed
+ INTEGER :: H5D_SELECTION_IO_F !< Selection I/O was performed
!
! H5E flags declaration
!
diff --git a/fortran/test/tH5P.F90 b/fortran/test/tH5P.F90
index c73016bc7cf..78d665f0aa1 100644
--- a/fortran/test/tH5P.F90
+++ b/fortran/test/tH5P.F90
@@ -869,6 +869,7 @@ SUBROUTINE test_in_place_conversion(cleanup, total_error)
REAL(KIND=C_DOUBLE), DIMENSION(1:array_len) :: wbuf_d_org
REAL(KIND=C_FLOAT), DIMENSION(1:array_len), TARGET :: rbuf
INTEGER :: i
+ INTEGER :: actual_selection_io_mode
TYPE(C_PTR) :: f_ptr
! create the data
@@ -919,6 +920,10 @@ SUBROUTINE test_in_place_conversion(cleanup, total_error)
! Should not be equal for in-place buffer use
CALL VERIFY("h5dwrite_f -- in-place", wbuf_d(1), wbuf_d_org(1), total_error, .FALSE.)
+ CALL h5pget_actual_selection_io_mode_f(plist_id, actual_selection_io_mode, error)
+ CALL check("h5pget_actual_selection_io_mode_f", error, total_error)
+ CALL VERIFY("h5pget_actual_selection_io_mode_f", actual_selection_io_mode, H5D_SCALAR_IO_F, total_error)
+
f_ptr = C_LOC(rbuf)
CALL h5dread_f(dset_id, h5kind_to_type(KIND(rbuf(1)), H5_REAL_KIND), f_ptr, error)
CALL check("h5dread_f", error, total_error)
diff --git a/fortran/testpar/hyper.F90 b/fortran/testpar/hyper.F90
index edd93cf9b8f..ec3a657afbb 100644
--- a/fortran/testpar/hyper.F90
+++ b/fortran/testpar/hyper.F90
@@ -55,6 +55,7 @@ SUBROUTINE hyper(length,do_collective,do_chunk, mpi_size, mpi_rank, nerrors)
INTEGER :: local_no_collective_cause
INTEGER :: global_no_collective_cause
INTEGER :: no_selection_io_cause
+ INTEGER :: actual_selection_io_mode
!
! initialize the array data between the processes (3)
@@ -236,6 +237,20 @@ SUBROUTINE hyper(length,do_collective,do_chunk, mpi_size, mpi_rank, nerrors)
CALL h5dwrite_f(dset_id,H5T_NATIVE_INTEGER,wbuf,dims,hdferror,file_space_id=fspace_id,mem_space_id=mspace_id,xfer_prp=dxpl_id)
CALL check("h5dwrite_f", hdferror, nerrors)
+ CALL h5pget_actual_selection_io_mode_f(dxpl_id, actual_selection_io_mode, hdferror)
+ CALL check("h5pget_actual_selection_io_mode_f", hdferror, nerrors)
+ IF(do_collective)THEN
+ IF(actual_selection_io_mode .NE. H5D_SELECTION_IO_F)THEN
+ PRINT*, "Incorrect actual selection io mode"
+ nerrors = nerrors + 1
+ ENDIF
+ ELSE
+ IF(actual_selection_io_mode .NE. IOR(H5D_SELECTION_IO_F, H5D_SCALAR_IO_F))THEN
+ PRINT*, "Incorrect actual selection io mode"
+ nerrors = nerrors + 1
+ ENDIF
+ ENDIF
+
! Check h5pget_mpio_actual_io_mode_f function
CALL h5pget_mpio_actual_io_mode_f(dxpl_id, actual_io_mode, hdferror)
CALL check("h5pget_mpio_actual_io_mode_f", hdferror, nerrors)
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 9643cf9ecf6..a5bcb94e807 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -332,7 +332,8 @@ New Features
- Fortran async APIs H5A, H5D, H5ES, H5G, H5F, H5L and H5O were added.
- Added Fortran APIs:
- h5pset_selection_io_f, h5pget_selection_io_f
+ h5pset_selection_io_f, h5pget_selection_io_f,
+ h5pget_actual_selection_io_mode_f,
h5pset_modify_write_buf_f, h5pget_modify_write_buf_f
- Added Fortran APIs:
From b411852e982e849ffa0602d640b5af4d47026fbb Mon Sep 17 00:00:00 2001
From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com>
Date: Mon, 23 Oct 2023 15:01:10 -0500
Subject: [PATCH 17/52] Update fortran action step (#3748)
---
.github/workflows/cmake-ctest.yml | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/cmake-ctest.yml b/.github/workflows/cmake-ctest.yml
index da99e7e02cc..ee4bcff4de3 100644
--- a/.github/workflows/cmake-ctest.yml
+++ b/.github/workflows/cmake-ctest.yml
@@ -191,8 +191,12 @@ jobs:
# symlinks the compiler executables to a common location
- name: Setup GNU Fortran
- uses: modflowpy/install-gfortran-action@v1
-
+ uses: fortran-lang/setup-fortran@v1
+ id: setup-fortran
+ with:
+ compiler: gcc
+ version: 12
+
- name: Run ctest (MacOS)
id: run-ctest
run: |
From 207d749a2810bd30ecc4321609c83fe51adbd0c3 Mon Sep 17 00:00:00 2001
From: Scot Breitenfeld
Date: Mon, 23 Oct 2023 19:02:16 -0500
Subject: [PATCH 18/52] Added missing DLL for H5PGET_ACTUAL_SELECTION_IO_MODE_F
(#3760)
* add missing H5PGET_ACTUAL_SELECTION_IO_MODE_F dll
---
fortran/src/hdf5_fortrandll.def.in | 1 +
1 file changed, 1 insertion(+)
diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in
index 3b6600c061e..2ded00222ba 100644
--- a/fortran/src/hdf5_fortrandll.def.in
+++ b/fortran/src/hdf5_fortrandll.def.in
@@ -417,6 +417,7 @@ H5P_mp_H5PSET_FILE_SPACE_STRATEGY_F
H5P_mp_H5PGET_FILE_SPACE_STRATEGY_F
H5P_mp_H5PSET_FILE_SPACE_PAGE_SIZE_F
H5P_mp_H5PGET_FILE_SPACE_PAGE_SIZE_F
+H5P_mp_H5PGET_ACTUAL_SELECTION_IO_MODE_F
; Parallel
@H5_NOPAREXP@H5P_mp_H5PSET_FAPL_MPIO_F
@H5_NOPAREXP@H5P_mp_H5PGET_FAPL_MPIO_F
From ceb03358a1d713078ae36bfff07be62b433d970a Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Mon, 23 Oct 2023 19:06:18 -0700
Subject: [PATCH 19/52] Bump the ros3 VFD cache to 16 MiB (#3759)
---
release_docs/RELEASE.txt | 4 ++--
src/H5FDros3.c | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index a5bcb94e807..222c2774057 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -236,13 +236,13 @@ New Features
Library:
--------
- - Added a simple cache to the read-only S3 VFD
+ - Added a simple cache to the read-only S3 (ros3) VFD
The read-only S3 VFD now caches the first N bytes of a file stored
in S3 to avoid a lot of small I/O operations when opening files.
This cache is per-file and created when the file is opened.
- N is currently 4kiB or the size of the file, whichever is smaller.
+ N is currently 16 MiB or the size of the file, whichever is smaller.
Addresses GitHub issue #3381
diff --git a/src/H5FDros3.c b/src/H5FDros3.c
index f704eff1efa..c6aea0e327a 100644
--- a/src/H5FDros3.c
+++ b/src/H5FDros3.c
@@ -44,7 +44,7 @@
#define ROS3_STATS 0
/* Max size of the cache, in bytes */
-#define ROS3_MAX_CACHE_SIZE 4096
+#define ROS3_MAX_CACHE_SIZE 16777216
/* The driver identification number, initialized at runtime
*/
From af49eb5b8647e8d9ffb527fd533def0910eb535c Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Mon, 23 Oct 2023 21:06:44 -0500
Subject: [PATCH 20/52] Fix hangs during collective I/O with independent
metadata writes (#3693)
---
release_docs/RELEASE.txt | 19 ++++++++
src/H5Cmpio.c | 38 +++++++++++++--
src/H5Pfapl.c | 2 +-
testpar/t_coll_md.c | 103 +++++++++++++++++++++++++++++++++++++++
testpar/testphdf5.c | 2 +
testpar/testphdf5.h | 1 +
6 files changed, 160 insertions(+), 5 deletions(-)
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 222c2774057..e5d53e41cbf 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -392,6 +392,25 @@ Bug Fixes since HDF5-1.14.0 release
===================================
Library
-------
+ - Fixed potential hangs in parallel library during collective I/O with
+ independent metadata writes
+
+ When performing collective parallel writes to a dataset where metadata
+ writes are requested as (or left as the default setting of) independent,
+ hangs could potentially occur during metadata cache sync points. This
+ was due to incorrect management of the internal state tracking whether
+ an I/O operation should be collective or not, causing the library to
+ attempt collective writes of metadata when they were meant to be
+ independent writes. During the metadata cache sync points, if the number
+ of cache entries being flushed was a multiple of the number of MPI ranks
+ in the MPI communicator used to access the HDF5 file, an equal amount of
+ collective MPI I/O calls were made and the dataset write call would be
+ successful. However, when the number of cache entries being flushed was
+ NOT a multiple of the number of MPI ranks, the ranks with more entries
+ than others would get stuck in an MPI_File_set_view call, while other
+ ranks would get stuck in a post-write MPI_Barrier call. This issue has
+ been fixed by correctly switching to independent I/O temporarily when
+ writing metadata independently during collective dataset I/O.
- Dropped support for MPI-2
diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c
index 643bbc80207..c8db5352ff6 100644
--- a/src/H5Cmpio.c
+++ b/src/H5Cmpio.c
@@ -154,8 +154,9 @@ herr_t
H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, haddr_t *candidates_list_ptr,
int mpi_rank, int mpi_size)
{
- unsigned first_entry_to_flush;
- unsigned last_entry_to_flush;
+ H5FD_mpio_xfer_t orig_xfer_mode;
+ unsigned first_entry_to_flush;
+ unsigned last_entry_to_flush;
#ifndef NDEBUG
unsigned total_entries_to_clear = 0;
unsigned total_entries_to_flush = 0;
@@ -172,8 +173,9 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha
char *tbl_buf = NULL;
#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
unsigned m, n;
- unsigned u; /* Local index variable */
- herr_t ret_value = SUCCEED; /* Return value */
+ unsigned u; /* Local index variable */
+ bool restore_io_mode = false;
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
@@ -185,6 +187,10 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha
assert(0 <= mpi_rank);
assert(mpi_rank < mpi_size);
+ /* Get I/O transfer mode */
+ if (H5CX_get_io_xfer_mode(&orig_xfer_mode) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode");
+
/* Initialize the entries_to_flush and entries_to_clear arrays */
memset(entries_to_flush, 0, sizeof(entries_to_flush));
memset(entries_to_clear, 0, sizeof(entries_to_clear));
@@ -418,6 +424,19 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha
num_candidates, total_entries_to_clear, total_entries_to_flush);
#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
+ /*
+ * If collective I/O was requested, but collective metadata
+ * writes were not requested, temporarily disable collective
+ * I/O while flushing candidate entries so that we don't cause
+ * a hang in the case where the number of candidate entries
+ * to flush isn't a multiple of mpi_size.
+ */
+ if ((orig_xfer_mode == H5FD_MPIO_COLLECTIVE) && !f->shared->coll_md_write) {
+ if (H5CX_set_io_xfer_mode(H5FD_MPIO_INDEPENDENT) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "can't set MPI-I/O transfer mode");
+ restore_io_mode = true;
+ }
+
/* We have now marked all the entries on the candidate list for
* either flush or clear -- now scan the LRU and the pinned list
* for these entries and do the deed. Do this via a call to
@@ -431,6 +450,13 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha
if (H5C__flush_candidate_entries(f, entries_to_flush, entries_to_clear) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush candidates failed");
+ /* Restore collective I/O if we temporarily disabled it */
+ if (restore_io_mode) {
+ if (H5CX_set_io_xfer_mode(orig_xfer_mode) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "can't set MPI-I/O transfer mode");
+ restore_io_mode = false;
+ }
+
/* If we've deferred writing to do it collectively, take care of that now */
if (f->shared->coll_md_write) {
/* Sanity check */
@@ -442,6 +468,10 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha
} /* end if */
done:
+ /* Restore collective I/O if we temporarily disabled it */
+ if (restore_io_mode && (H5CX_set_io_xfer_mode(orig_xfer_mode) < 0))
+ HDONE_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "can't set MPI-I/O transfer mode");
+
if (candidate_assignment_table != NULL)
candidate_assignment_table = (unsigned *)H5MM_xfree((void *)candidate_assignment_table);
if (cache_ptr->coll_write_list) {
diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c
index 5f5782cae3b..dc122af9393 100644
--- a/src/H5Pfapl.c
+++ b/src/H5Pfapl.c
@@ -5174,7 +5174,7 @@ H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collective /*out*/)
* Function: H5Pset_coll_metadata_write
*
* Purpose: Tell the library whether the metadata write operations will
- * be done collectively (1) or not (0). Default is collective.
+ * be done collectively (1) or not (0). Default is independent.
*
* Return: Non-negative on success/Negative on failure
*
diff --git a/testpar/t_coll_md.c b/testpar/t_coll_md.c
index 1220111a56d..9c6fc7120cf 100644
--- a/testpar/t_coll_md.c
+++ b/testpar/t_coll_md.c
@@ -43,6 +43,11 @@
#define COLL_GHEAP_WRITE_ATTR_NAME "coll_gheap_write_attr"
#define COLL_GHEAP_WRITE_ATTR_DIMS 1
+#define COLL_IO_IND_MD_WRITE_NDIMS 2
+#define COLL_IO_IND_MD_WRITE_CHUNK0 4
+#define COLL_IO_IND_MD_WRITE_CHUNK1 256
+#define COLL_IO_IND_MD_WRITE_NCHUNK1 16384
+
/*
* A test for issue HDFFV-10501. A parallel hang was reported which occurred
* in linked-chunk I/O when collective metadata reads are enabled and some ranks
@@ -569,3 +574,101 @@ test_collective_global_heap_write(void)
VRFY((H5Pclose(fapl_id) >= 0), "H5Pclose succeeded");
VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded");
}
+
+/*
+ * A test to ensure that hangs don't occur when collective I/O
+ * is requested at the interface level (by a call to
+ * H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE)), while
+ * collective metadata writes are NOT requested.
+ */
+void
+test_coll_io_ind_md_write(void)
+{
+ const char *filename;
+ long long *data = NULL;
+ hsize_t dset_dims[COLL_IO_IND_MD_WRITE_NDIMS];
+ hsize_t chunk_dims[COLL_IO_IND_MD_WRITE_NDIMS];
+ hsize_t sel_dims[COLL_IO_IND_MD_WRITE_NDIMS];
+ hsize_t offset[COLL_IO_IND_MD_WRITE_NDIMS];
+ hid_t file_id = H5I_INVALID_HID;
+ hid_t fapl_id = H5I_INVALID_HID;
+ hid_t dset_id = H5I_INVALID_HID;
+ hid_t dset_id2 = H5I_INVALID_HID;
+ hid_t dcpl_id = H5I_INVALID_HID;
+ hid_t dxpl_id = H5I_INVALID_HID;
+ hid_t fspace_id = H5I_INVALID_HID;
+ int mpi_rank, mpi_size;
+
+ MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
+
+ filename = GetTestParameters();
+
+ fapl_id = create_faccess_plist(MPI_COMM_WORLD, MPI_INFO_NULL, facc_type);
+ VRFY((fapl_id >= 0), "create_faccess_plist succeeded");
+
+ VRFY((H5Pset_all_coll_metadata_ops(fapl_id, false) >= 0), "Unset collective metadata reads succeeded");
+ VRFY((H5Pset_coll_metadata_write(fapl_id, false) >= 0), "Unset collective metadata writes succeeded");
+
+ file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+ VRFY((file_id >= 0), "H5Fcreate succeeded");
+
+ dset_dims[0] = (hsize_t)(mpi_size * COLL_IO_IND_MD_WRITE_CHUNK0);
+ dset_dims[1] = (hsize_t)(COLL_IO_IND_MD_WRITE_CHUNK1 * COLL_IO_IND_MD_WRITE_NCHUNK1);
+
+ fspace_id = H5Screate_simple(COLL_IO_IND_MD_WRITE_NDIMS, dset_dims, NULL);
+ VRFY((fspace_id >= 0), "H5Screate_simple succeeded");
+
+ dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
+ VRFY((dcpl_id >= 0), "H5Pcreate succeeded");
+
+ chunk_dims[0] = (hsize_t)(COLL_IO_IND_MD_WRITE_CHUNK0);
+ chunk_dims[1] = (hsize_t)(COLL_IO_IND_MD_WRITE_CHUNK1);
+
+ VRFY((H5Pset_chunk(dcpl_id, COLL_IO_IND_MD_WRITE_NDIMS, chunk_dims) >= 0), "H5Pset_chunk succeeded");
+
+ VRFY((H5Pset_shuffle(dcpl_id) >= 0), "H5Pset_shuffle succeeded");
+
+ dset_id = H5Dcreate2(file_id, "dset1", H5T_NATIVE_LLONG, fspace_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT);
+ VRFY((dset_id >= 0), "H5Dcreate2 succeeded");
+
+ sel_dims[0] = (hsize_t)(COLL_IO_IND_MD_WRITE_CHUNK0);
+ sel_dims[1] = (hsize_t)(COLL_IO_IND_MD_WRITE_CHUNK1 * COLL_IO_IND_MD_WRITE_NCHUNK1);
+
+ offset[0] = (hsize_t)mpi_rank * sel_dims[0];
+ offset[1] = 0;
+
+ VRFY((H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, offset, NULL, sel_dims, NULL) >= 0),
+ "H5Sselect_hyperslab succeeded");
+
+ dxpl_id = H5Pcreate(H5P_DATASET_XFER);
+ VRFY((dxpl_id >= 0), "H5Pcreate succeeded");
+
+ VRFY((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE) >= 0), "H5Pset_dxpl_mpio succeeded");
+
+ data = malloc(sel_dims[0] * sel_dims[1] * sizeof(long long));
+ for (size_t i = 0; i < sel_dims[0] * sel_dims[1]; i++)
+ data[i] = rand();
+
+ VRFY((H5Dwrite(dset_id, H5T_NATIVE_LLONG, H5S_BLOCK, fspace_id, dxpl_id, data) >= 0),
+ "H5Dwrite succeeded");
+
+ dset_id2 = H5Dcreate2(file_id, "dset2", H5T_NATIVE_LLONG, fspace_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT);
+ VRFY((dset_id2 >= 0), "H5Dcreate2 succeeded");
+
+ for (size_t i = 0; i < sel_dims[0] * sel_dims[1]; i++)
+ data[i] = rand();
+
+ VRFY((H5Dwrite(dset_id2, H5T_NATIVE_LLONG, H5S_BLOCK, fspace_id, dxpl_id, data) >= 0),
+ "H5Dwrite succeeded");
+
+ free(data);
+
+ VRFY((H5Sclose(fspace_id) >= 0), "H5Sclose succeeded");
+ VRFY((H5Dclose(dset_id) >= 0), "H5Dclose succeeded");
+ VRFY((H5Dclose(dset_id2) >= 0), "H5Dclose succeeded");
+ VRFY((H5Pclose(dcpl_id) >= 0), "H5Pclose succeeded");
+ VRFY((H5Pclose(dxpl_id) >= 0), "H5Pclose succeeded");
+ VRFY((H5Pclose(fapl_id) >= 0), "H5Pclose succeeded");
+ VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded");
+}
diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c
index 584ca1f6107..2d85e1ae289 100644
--- a/testpar/testphdf5.c
+++ b/testpar/testphdf5.c
@@ -521,6 +521,8 @@ main(int argc, char **argv)
"Collective MD read with link chunk I/O (H5D__sort_chunk)", PARATESTFILE);
AddTest("GH_coll_MD_wr", test_collective_global_heap_write, NULL,
"Collective MD write of global heap data", PARATESTFILE);
+ AddTest("COLLIO_INDMDWR", test_coll_io_ind_md_write, NULL,
+ "Collective I/O with Independent metadata writes", PARATESTFILE);
/* Display testing information */
TestInfo(argv[0]);
diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h
index 6ac8080c82a..5699760c61b 100644
--- a/testpar/testphdf5.h
+++ b/testpar/testphdf5.h
@@ -296,6 +296,7 @@ void test_partial_no_selection_coll_md_read(void);
void test_multi_chunk_io_addrmap_issue(void);
void test_link_chunk_io_sort_chunk_issue(void);
void test_collective_global_heap_write(void);
+void test_coll_io_ind_md_write(void);
void test_oflush(void);
/* commonly used prototypes */
From 8cff88c5073b0bbd66c5876e0ce3da16214af39c Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Mon, 23 Oct 2023 21:08:06 -0500
Subject: [PATCH 21/52] Fix some issues with collective metadata reads for
chunked datasets (#3716)
Add functions/callbacks for explicit control over chunk index open/close
Add functions/callbacks to check if chunk index is open or not so
that it can be opened if necessary before temporarily disabling
collective metadata reads in the library
Add functions/callbacks for requesting loading of additional chunk
index metadata beyond the chunk index itself
---
release_docs/RELEASE.txt | 40 ++++
src/H5Dbtree.c | 442 +++++++++++++++++++++---------------
src/H5Dbtree2.c | 467 ++++++++++++++++++++++++---------------
src/H5Dchunk.c | 63 ++++--
src/H5Dearray.c | 271 ++++++++++++++++-------
src/H5Dfarray.c | 234 ++++++++++++++------
src/H5Dmpio.c | 40 ++--
src/H5Dnone.c | 170 ++++++++++----
src/H5Dpkg.h | 21 +-
src/H5Dsingle.c | 168 ++++++++++----
10 files changed, 1292 insertions(+), 624 deletions(-)
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index e5d53e41cbf..e5cb09707bd 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -392,6 +392,46 @@ Bug Fixes since HDF5-1.14.0 release
===================================
Library
-------
+ - Fixed some issues with chunk index metadata not getting read
+ collectively when collective metadata reads are enabled
+
+ When looking up dataset chunks during I/O, the parallel library
+ temporarily disables collective metadata reads since it's generally
+ unlikely that the application will read the same chunks from all
+ MPI ranks. Leaving collective metadata reads enabled during
+ chunk lookups can lead to hangs or other bad behavior depending
+ on the chunk indexing structure used for the dataset in question.
+ However, due to the way that dataset chunk index metadata was
+ previously loaded in a deferred manner, this could mean that
+ the metadata for the main chunk index structure or its
+ accompanying pieces of metadata (e.g., fixed array data blocks)
+ could end up being read independently if these chunk lookup
+ operations are the first chunk index-related operation that
+ occurs on a dataset. This behavior is generally observed when
+ opening a dataset for which the metadata isn't in the metadata
+ cache yet and then immediately performing I/O on that dataset.
+ This behavior is not generally observed when creating a dataset
+ and then performing I/O on it, as the relevant metadata will
+ usually be in the metadata cache as a side effect of creating
+ the chunk index structures during dataset creation.
+
+ This issue has been fixed by adding callbacks to the different
+ chunk indexing structure classes that allow more explicit control
+ over when chunk index metadata gets loaded. When collective
+ metadata reads are enabled, the necessary index metadata will now
+ get loaded collectively by all MPI ranks at the start of dataset
+ I/O to ensure that the ranks don't unintentionally read this
+ metadata independently further on. These changes fix collective
+ loading of the main chunk index structure, as well as v2 B-tree
+ root nodes, extensible array index blocks and fixed array data
+ blocks. There are still pieces of metadata that cannot currently
+ be loaded collectively, however, such as extensible array data
+ blocks, data block pages and super blocks, as well as fixed array
+ data block pages. These pieces of metadata are not necessarily
+ read in by all MPI ranks since this depends on which chunks the
+ ranks have selected in the dataset. Therefore, reading of these
+ pieces of metadata remains an independent operation.
+
- Fixed potential hangs in parallel library during collective I/O with
independent metadata writes
diff --git a/src/H5Dbtree.c b/src/H5Dbtree.c
index d79f7d0b031..4f8a867974e 100644
--- a/src/H5Dbtree.c
+++ b/src/H5Dbtree.c
@@ -24,30 +24,32 @@
/***********/
/* Headers */
/***********/
-#include "H5private.h" /* Generic Functions */
-#include "H5Bprivate.h" /* B-link trees */
-#include "H5Dpkg.h" /* Datasets */
-#include "H5Eprivate.h" /* Error handling */
-#include "H5Fprivate.h" /* Files */
-#include "H5FDprivate.h" /* File drivers */
+#include "H5private.h" /* Generic Functions */
+#include "H5Bprivate.h" /* B-link trees */
+#include "H5Dpkg.h" /* Datasets */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Fprivate.h" /* Files */
+#include "H5FDprivate.h" /* File drivers */
#include "H5FLprivate.h" /* Free Lists */
-#include "H5Iprivate.h" /* IDs */
-#include "H5MFprivate.h" /* File space management */
-#include "H5MMprivate.h" /* Memory management */
-#include "H5Oprivate.h" /* Object headers */
+#include "H5Iprivate.h" /* IDs */
+#include "H5MFprivate.h" /* File space management */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5Oprivate.h" /* Object headers */
#include "H5Sprivate.h" /* Dataspaces */
-#include "H5VMprivate.h" /* Vector and array functions */
+#include "H5VMprivate.h" /* Vector and array functions */
/****************/
/* Local Macros */
/****************/
+#define H5D_BTREE_IDX_IS_OPEN(idx_info) (NULL != (idx_info)->storage->u.btree.shared)
+
/******************/
/* Local Typedefs */
/******************/
/*
- * B-tree key. A key contains the minimum logical N-dimensional coordinates and
+ * B-tree key. A key contains the minimum logical N-dimensional coordinates and
* the logical size of the chunk to which this key refers. The
* fastest-varying dimension is assumed to reference individual bytes of the
* array, so a 100-element 1-d array of 4-byte integers would really be a 2-d
@@ -61,9 +63,9 @@
* The chunk's file address is part of the B-tree and not part of the key.
*/
typedef struct H5D_btree_key_t {
- hsize_t scaled[H5O_LAYOUT_NDIMS]; /*logical offset to start*/
- uint32_t nbytes; /*size of stored data */
- unsigned filter_mask; /*excluded filters */
+ hsize_t scaled[H5O_LAYOUT_NDIMS]; /*logical offset to start */
+ uint32_t nbytes; /*size of stored data */
+ unsigned filter_mask; /*excluded filters */
} H5D_btree_key_t;
/* B-tree callback info for iteration over chunks */
@@ -111,10 +113,14 @@ static herr_t H5D__btree_debug_key(FILE *stream, int indent, int fwidth, const v
static herr_t H5D__btree_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t *space,
haddr_t dset_ohdr_addr);
static herr_t H5D__btree_idx_create(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__btree_idx_open(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__btree_idx_close(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__btree_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open);
static bool H5D__btree_idx_is_space_alloc(const H5O_storage_chunk_t *storage);
static herr_t H5D__btree_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata,
const H5D_t *dset);
static herr_t H5D__btree_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata);
+static herr_t H5D__btree_idx_load_metadata(const H5D_chk_idx_info_t *idx_info);
static int H5D__btree_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb,
void *chunk_udata);
static herr_t H5D__btree_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata);
@@ -137,9 +143,13 @@ const H5D_chunk_ops_t H5D_COPS_BTREE[1] = {{
false, /* v1 B-tree indices does not support SWMR access */
H5D__btree_idx_init, /* insert */
H5D__btree_idx_create, /* create */
+ H5D__btree_idx_open, /* open */
+ H5D__btree_idx_close, /* close */
+ H5D__btree_idx_is_open, /* is_open */
H5D__btree_idx_is_space_alloc, /* is_space_alloc */
H5D__btree_idx_insert, /* insert */
H5D__btree_idx_get_addr, /* get_addr */
+ H5D__btree_idx_load_metadata, /* load_metadata */
NULL, /* resize */
H5D__btree_idx_iterate, /* iterate */
H5D__btree_idx_remove, /* remove */
@@ -158,21 +168,21 @@ const H5D_chunk_ops_t H5D_COPS_BTREE[1] = {{
/* inherits B-tree like properties from H5B */
static H5B_class_t H5B_BTREE[1] = {{
- H5B_CHUNK_ID, /*id */
- sizeof(H5D_btree_key_t), /*sizeof_nkey */
- H5D__btree_get_shared, /*get_shared */
- H5D__btree_new_node, /*new */
- H5D__btree_cmp2, /*cmp2 */
- H5D__btree_cmp3, /*cmp3 */
- H5D__btree_found, /*found */
- H5D__btree_insert, /*insert */
- false, /*follow min branch? */
- false, /*follow max branch? */
- H5B_LEFT, /*critical key */
- H5D__btree_remove, /*remove */
- H5D__btree_decode_key, /*decode */
- H5D__btree_encode_key, /*encode */
- H5D__btree_debug_key /*debug */
+ H5B_CHUNK_ID, /* id */
+ sizeof(H5D_btree_key_t), /* sizeof_nkey */
+ H5D__btree_get_shared, /* get_shared */
+ H5D__btree_new_node, /* new */
+ H5D__btree_cmp2, /* cmp2 */
+ H5D__btree_cmp3, /* cmp3 */
+ H5D__btree_found, /* found */
+ H5D__btree_insert, /* insert */
+ false, /* follow min branch? */
+ false, /* follow max branch? */
+ H5B_LEFT, /* critical key */
+ H5D__btree_remove, /* remove */
+ H5D__btree_decode_key, /* decode */
+ H5D__btree_encode_key, /* encode */
+ H5D__btree_debug_key /* debug */
}};
/*******************/
@@ -183,13 +193,13 @@ static H5B_class_t H5B_BTREE[1] = {{
H5FL_DEFINE_STATIC(H5O_layout_chunk_t);
/*-------------------------------------------------------------------------
- * Function: H5D__btree_get_shared
+ * Function: H5D__btree_get_shared
*
- * Purpose: Returns the shared B-tree info for the specified UDATA.
+ * Purpose: Returns the shared B-tree info for the specified UDATA.
*
- * Return: Success: Pointer to the raw B-tree page for this dataset
+ * Return: Success: Pointer to the raw B-tree page for this dataset
*
- * Failure: Can't fail
+ * Failure: Can't fail
*
*-------------------------------------------------------------------------
*/
@@ -210,17 +220,17 @@ H5D__btree_get_shared(const H5F_t H5_ATTR_UNUSED *f, const void *_udata)
} /* end H5D__btree_get_shared() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_new_node
+ * Function: H5D__btree_new_node
*
- * Purpose: Adds a new entry to an i-storage B-tree. We can assume that
- * the domain represented by UDATA doesn't intersect the domain
- * already represented by the B-tree.
+ * Purpose: Adds a new entry to an i-storage B-tree. We can assume
+ * that the domain represented by UDATA doesn't intersect the
+ * domain already represented by the B-tree.
*
- * Return: Success: Non-negative. The address of leaf is returned
- * through the ADDR argument. It is also added
- * to the UDATA.
+ * Return: Success: Non-negative. The address of leaf is returned
+ * through the ADDR argument. It is also added
+ * to the UDATA.
*
- * Failure: Negative
+ * Failure: Negative
*
*-------------------------------------------------------------------------
*/
@@ -275,18 +285,18 @@ H5D__btree_new_node(H5F_t H5_ATTR_NDEBUG_UNUSED *f, H5B_ins_t op, void *_lt_key,
} /* end H5D__btree_new_node() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_cmp2
+ * Function: H5D__btree_cmp2
*
- * Purpose: Compares two keys sort of like strcmp(). The UDATA pointer
- * is only to supply extra information not carried in the keys
- * (in this case, the dimensionality) and is not compared
- * against the keys.
+ * Purpose: Compares two keys sort of like strcmp(). The UDATA pointer
+ * is only to supply extra information not carried in the keys
+ * (in this case, the dimensionality) and is not compared
+ * against the keys.
*
- * Return: Success: -1 if LT_KEY is less than RT_KEY;
- * 1 if LT_KEY is greater than RT_KEY;
- * 0 if LT_KEY and RT_KEY are equal.
+ * Return: Success: -1 if LT_KEY is less than RT_KEY;
+ * 1 if LT_KEY is greater than RT_KEY;
+ * 0 if LT_KEY and RT_KEY are equal.
*
- * Failure: FAIL (same as LT_KEYstorage);
+ assert(H5D_CHUNK_IDX_BTREE == idx_info->storage->idx_type);
+ assert(is_open);
+
+ *is_open = H5D_BTREE_IDX_IS_OPEN(idx_info);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5D__btree_idx_is_open() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__btree_idx_is_space_alloc
+ *
+ * Purpose: Query if space is allocated for index method
+ *
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -886,11 +960,11 @@ H5D__btree_idx_is_space_alloc(const H5O_storage_chunk_t *storage)
} /* end H5D__btree_idx_is_space_alloc() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_idx_insert
+ * Function: H5D__btree_idx_insert
*
- * Purpose: Insert chunk entry into the indexing structure.
+ * Purpose: Insert chunk entry into the indexing structure.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -922,13 +996,13 @@ H5D__btree_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata,
} /* H5D__btree_idx_insert() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_idx_get_addr
+ * Function: H5D__btree_idx_get_addr
*
- * Purpose: Get the file address of a chunk if file space has been
- * assigned. Save the retrieved information in the udata
- * supplied.
+ * Purpose: Get the file address of a chunk if file space has been
+ * assigned. Save the retrieved information in the udata
+ * supplied.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -959,14 +1033,34 @@ H5D__btree_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat
} /* H5D__btree_idx_get_addr() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_idx_iterate_cb
+ * Function: H5D__btree_idx_load_metadata
+ *
+ * Purpose: Load additional chunk index metadata beyond the chunk index
+ * itself. Currently a no-op.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__btree_idx_load_metadata(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* NO OP */
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5D__btree_idx_load_metadata() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__btree_idx_iterate_cb
*
- * Purpose: Translate the B-tree specific chunk record into a generic
+ * Purpose: Translate the B-tree specific chunk record into a generic
* form and make the callback to the generic chunk callback
* routine.
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
*-------------------------------------------------------------------------
*/
@@ -1001,12 +1095,12 @@ H5D__btree_idx_iterate_cb(H5F_t H5_ATTR_UNUSED *f, const void *_lt_key, haddr_t
} /* H5D__btree_idx_iterate_cb() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_idx_iterate
+ * Function: H5D__btree_idx_iterate
*
- * Purpose: Iterate over the chunks in an index, making a callback
+ * Purpose: Iterate over the chunks in an index, making a callback
* for each one.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1043,11 +1137,11 @@ H5D__btree_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t c
} /* end H5D__btree_idx_iterate() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_idx_remove
+ * Function: H5D__btree_idx_remove
*
- * Purpose: Remove chunk from index.
+ * Purpose: Remove chunk from index.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1077,13 +1171,13 @@ H5D__btree_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t
} /* H5D__btree_idx_remove() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_idx_delete
+ * Function: H5D__btree_idx_delete
*
- * Purpose: Delete index and raw data storage for entire dataset
+ * Purpose: Delete index and raw data storage for entire dataset
* (i.e. all chunks)
*
- * Return: Success: Non-negative
- * Failure: negative
+ * Return: Success: Non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -1134,11 +1228,11 @@ H5D__btree_idx_delete(const H5D_chk_idx_info_t *idx_info)
} /* end H5D__btree_idx_delete() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_idx_copy_setup
+ * Function: H5D__btree_idx_copy_setup
*
- * Purpose: Set up any necessary information for copying chunks
+ * Purpose: Set up any necessary information for copying chunks
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1178,11 +1272,11 @@ H5D__btree_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk_
} /* end H5D__btree_idx_copy_setup() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_idx_copy_shutdown
+ * Function: H5D__btree_idx_copy_shutdown
*
- * Purpose: Shutdown any information from copying chunks
+ * Purpose: Shutdown any information from copying chunks
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1250,11 +1344,11 @@ H5D__btree_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size)
} /* end H5D__btree_idx_size() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_idx_reset
+ * Function: H5D__btree_idx_reset
*
- * Purpose: Reset indexing information.
+ * Purpose: Reset indexing information.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1274,11 +1368,11 @@ H5D__btree_idx_reset(H5O_storage_chunk_t *storage, bool reset_addr)
} /* end H5D__btree_idx_reset() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_idx_dump
+ * Function: H5D__btree_idx_dump
*
- * Purpose: Dump indexing information to a stream.
+ * Purpose: Dump indexing information to a stream.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1296,11 +1390,11 @@ H5D__btree_idx_dump(const H5O_storage_chunk_t *storage, FILE *stream)
} /* end H5D__btree_idx_dump() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree_idx_dest
+ * Function: H5D__btree_idx_dest
*
- * Purpose: Release indexing information in memory.
+ * Purpose: Release indexing information in memory.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1328,11 +1422,11 @@ H5D__btree_idx_dest(const H5D_chk_idx_info_t *idx_info)
} /* end H5D__btree_idx_dest() */
/*-------------------------------------------------------------------------
- * Function: H5D_btree_debug
+ * Function: H5D_btree_debug
*
- * Purpose: Debugs a B-tree node for indexed raw data storage.
+ * Purpose: Debugs a B-tree node for indexed raw data storage.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
diff --git a/src/H5Dbtree2.c b/src/H5Dbtree2.c
index 4da9555d0c0..7a26b6d016c 100644
--- a/src/H5Dbtree2.c
+++ b/src/H5Dbtree2.c
@@ -27,16 +27,18 @@
/* Headers */
/***********/
#include "H5private.h" /* Generic Functions */
-#include "H5Dpkg.h" /* Datasets */
+#include "H5Dpkg.h" /* Datasets */
#include "H5FLprivate.h" /* Free Lists */
#include "H5MFprivate.h" /* File space management */
-#include "H5MMprivate.h" /* Memory management */
-#include "H5VMprivate.h" /* Vector and array functions */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5VMprivate.h" /* Vector and array functions */
/****************/
/* Local Macros */
/****************/
+#define H5D_BT2_IDX_IS_OPEN(idx_info) (NULL != (idx_info)->storage->u.btree2.bt2)
+
/******************/
/* Local Typedefs */
/******************/
@@ -92,7 +94,6 @@ static herr_t H5D__bt2_filt_debug(FILE *stream, int indent, int fwidth, const vo
const void *u_ctx);
/* Helper routine */
-static herr_t H5D__bt2_idx_open(const H5D_chk_idx_info_t *idx_info);
static herr_t H5D__btree2_idx_depend(const H5D_chk_idx_info_t *idx_info);
/* Callback for H5B2_iterate() which is called in H5D__bt2_idx_iterate() */
@@ -114,10 +115,14 @@ static herr_t H5D__bt2_mod_cb(void *_record, void *_op_data, bool *changed);
static herr_t H5D__bt2_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t *space,
haddr_t dset_ohdr_addr);
static herr_t H5D__bt2_idx_create(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__bt2_idx_open(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__bt2_idx_close(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__bt2_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open);
static bool H5D__bt2_idx_is_space_alloc(const H5O_storage_chunk_t *storage);
static herr_t H5D__bt2_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata,
const H5D_t *dset);
static herr_t H5D__bt2_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata);
+static herr_t H5D__bt2_idx_load_metadata(const H5D_chk_idx_info_t *idx_info);
static int H5D__bt2_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb,
void *chunk_udata);
static herr_t H5D__bt2_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata);
@@ -139,9 +144,13 @@ const H5D_chunk_ops_t H5D_COPS_BT2[1] = {{
true, /* Fixed array indices support SWMR access */
H5D__bt2_idx_init, /* init */
H5D__bt2_idx_create, /* create */
+ H5D__bt2_idx_open, /* open */
+ H5D__bt2_idx_close, /* close */
+ H5D__bt2_idx_is_open, /* is_open */
H5D__bt2_idx_is_space_alloc, /* is_space_alloc */
H5D__bt2_idx_insert, /* insert */
H5D__bt2_idx_get_addr, /* get_addr */
+ H5D__bt2_idx_load_metadata, /* load_metadata */
NULL, /* resize */
H5D__bt2_idx_iterate, /* iterate */
H5D__bt2_idx_remove, /* remove */
@@ -203,8 +212,8 @@ H5FL_ARR_DEFINE_STATIC(uint32_t, H5O_LAYOUT_NDIMS);
*
* Purpose: Create client callback context
*
- * Return: Success: non-NULL
- * Failure: NULL
+ * Return: Success: non-NULL
+ * Failure: NULL
*
*-------------------------------------------------------------------------
*/
@@ -258,8 +267,8 @@ H5D__bt2_crt_context(void *_udata)
*
* Purpose: Destroy client callback context
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -286,10 +295,10 @@ H5D__bt2_dst_context(void *_ctx)
* Function: H5D__bt2_store
*
* Purpose: Store native information into record for v2 B-tree
- * (non-filtered)
+ * (non-filtered)
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -308,8 +317,8 @@ H5D__bt2_store(void *record, const void *_udata)
/*-------------------------------------------------------------------------
* Function: H5D__bt2_compare
*
- * Purpose: Compare two native information records, according to some key
- * (non-filtered)
+ * Purpose: Compare two native information records, according to some
+ * key (non-filtered)
*
* Return: <0 if rec1 < rec2
* =0 if rec1 == rec2
@@ -341,10 +350,10 @@ H5D__bt2_compare(const void *_udata, const void *_rec2, int *result)
* Function: H5D__bt2_unfilt_encode
*
* Purpose: Encode native information into raw form for storing on disk
- * (non-filtered)
+ * (non-filtered)
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -373,10 +382,10 @@ H5D__bt2_unfilt_encode(uint8_t *raw, const void *_record, void *_ctx)
* Function: H5D__bt2_unfilt_decode
*
* Purpose: Decode raw disk form of record into native form
- * (non-filtered)
+ * (non-filtered)
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -403,12 +412,12 @@ H5D__bt2_unfilt_decode(const uint8_t *raw, void *_record, void *_ctx)
} /* H5D__bt2_unfilt_decode() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_unfilt_debug
+ * Function: H5D__bt2_unfilt_debug
*
- * Purpose: Debug native form of record (non-filtered)
+ * Purpose: Debug native form of record (non-filtered)
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -440,10 +449,10 @@ H5D__bt2_unfilt_debug(FILE *stream, int indent, int fwidth, const void *_record,
* Function: H5D__bt2_filt_encode
*
* Purpose: Encode native information into raw form for storing on disk
- * (filtered)
+ * (filtered)
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -473,13 +482,13 @@ H5D__bt2_filt_encode(uint8_t *raw, const void *_record, void *_ctx)
} /* H5D__bt2_filt_encode() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_filt_decode
+ * Function: H5D__bt2_filt_decode
*
- * Purpose: Decode raw disk form of record into native form
- * (filtered)
+ * Purpose: Decode raw disk form of record into native form
+ * (filtered)
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -511,12 +520,12 @@ H5D__bt2_filt_decode(const uint8_t *raw, void *_record, void *_ctx)
} /* H5D__bt2_filt_decode() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_filt_debug
+ * Function: H5D__bt2_filt_debug
*
- * Purpose: Debug native form of record (filtered)
+ * Purpose: Debug native form of record (filtered)
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -570,13 +579,13 @@ H5D__bt2_idx_init(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, const H5S_t
} /* end H5D__bt2_idx_init() */
/*-------------------------------------------------------------------------
- * Function: H5D__btree2_idx_depend
+ * Function: H5D__btree2_idx_depend
*
- * Purpose: Create flush dependency between v2 B-tree and dataset's
+ * Purpose: Create flush dependency between v2 B-tree and dataset's
* object header.
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -629,63 +638,9 @@ H5D__btree2_idx_depend(const H5D_chk_idx_info_t *idx_info)
} /* end H5D__btree2_idx_depend() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_open()
- *
- * Purpose: Opens an existing v2 B-tree.
- *
- * Note: This information is passively initialized from each index
- * operation callback because those abstract chunk index operations
- * are designed to work with the v2 B-tree chunk indices also,
- * which don't require an 'open' for the data structure.
+ * Function: H5D__bt2_idx_create
*
- * Return: Success: non-negative
- * Failure: negative
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-H5D__bt2_idx_open(const H5D_chk_idx_info_t *idx_info)
-{
- H5D_bt2_ctx_ud_t u_ctx; /* user data for creating context */
- herr_t ret_value = SUCCEED; /* Return value */
-
- FUNC_ENTER_PACKAGE
-
- /* Check args */
- assert(idx_info);
- assert(idx_info->f);
- assert(idx_info->pline);
- assert(idx_info->layout);
- assert(H5D_CHUNK_IDX_BT2 == idx_info->layout->idx_type);
- assert(idx_info->storage);
- assert(H5_addr_defined(idx_info->storage->idx_addr));
- assert(NULL == idx_info->storage->u.btree2.bt2);
-
- /* Set up the user data */
- u_ctx.f = idx_info->f;
- u_ctx.ndims = idx_info->layout->ndims - 1;
- u_ctx.chunk_size = idx_info->layout->size;
- u_ctx.dim = idx_info->layout->dim;
-
- /* Open v2 B-tree for the chunk index */
- if (NULL ==
- (idx_info->storage->u.btree2.bt2 = H5B2_open(idx_info->f, idx_info->storage->idx_addr, &u_ctx)))
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open v2 B-tree for tracking chunked dataset");
-
- /* Check for SWMR writes to the file */
- if (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)
- if (H5D__btree2_idx_depend(idx_info) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL,
- "unable to create flush dependency on object header");
-
-done:
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5D__bt2_idx_open() */
-
-/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_create
- *
- * Purpose: Create the v2 B-tree for tracking dataset chunks
+ * Purpose: Create the v2 B-tree for tracking dataset chunks
*
* Return: SUCCEED/FAIL
*
@@ -758,11 +713,120 @@ H5D__bt2_idx_create(const H5D_chk_idx_info_t *idx_info)
} /* end H5D__bt2_idx_create() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_is_space_alloc
+ * Function: H5D__bt2_idx_open()
+ *
+ * Purpose: Opens an existing v2 B-tree.
*
- * Purpose: Query if space is allocated for index method
+ * Note: This information is passively initialized from each index
+ * operation callback because those abstract chunk index
+ * operations are designed to work with the v2 B-tree chunk
+ * indices also, which don't require an 'open' for the data
+ * structure.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__bt2_idx_open(const H5D_chk_idx_info_t *idx_info)
+{
+ H5D_bt2_ctx_ud_t u_ctx; /* user data for creating context */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args */
+ assert(idx_info);
+ assert(idx_info->f);
+ assert(idx_info->pline);
+ assert(idx_info->layout);
+ assert(H5D_CHUNK_IDX_BT2 == idx_info->layout->idx_type);
+ assert(idx_info->storage);
+ assert(H5_addr_defined(idx_info->storage->idx_addr));
+ assert(NULL == idx_info->storage->u.btree2.bt2);
+
+ /* Set up the user data */
+ u_ctx.f = idx_info->f;
+ u_ctx.ndims = idx_info->layout->ndims - 1;
+ u_ctx.chunk_size = idx_info->layout->size;
+ u_ctx.dim = idx_info->layout->dim;
+
+ /* Open v2 B-tree for the chunk index */
+ if (NULL ==
+ (idx_info->storage->u.btree2.bt2 = H5B2_open(idx_info->f, idx_info->storage->idx_addr, &u_ctx)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open v2 B-tree for tracking chunked dataset");
+
+ /* Check for SWMR writes to the file */
+ if (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)
+ if (H5D__btree2_idx_depend(idx_info) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL,
+ "unable to create flush dependency on object header");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__bt2_idx_open() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__bt2_idx_close()
+ *
+ * Purpose: Closes an existing v2 B-tree.
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__bt2_idx_close(const H5D_chk_idx_info_t *idx_info)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ assert(idx_info);
+ assert(idx_info->storage);
+ assert(H5D_CHUNK_IDX_BT2 == idx_info->storage->idx_type);
+ assert(idx_info->storage->u.btree2.bt2);
+
+ if (H5B2_close(idx_info->storage->u.btree2.bt2) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close v2 B-tree");
+ idx_info->storage->u.btree2.bt2 = NULL;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__bt2_idx_close() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__bt2_idx_is_open
+ *
+ * Purpose: Query if the index is opened or not
+ *
+ * Return: SUCCEED (can't fail)
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__bt2_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ assert(idx_info);
+ assert(idx_info->storage);
+ assert(H5D_CHUNK_IDX_BT2 == idx_info->storage->idx_type);
+ assert(is_open);
+
+ *is_open = H5D_BT2_IDX_IS_OPEN(idx_info);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5D__bt2_idx_is_open() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__bt2_idx_is_space_alloc
+ *
+ * Purpose: Query if space is allocated for index method
+ *
+ * Return: true/false
*
*-------------------------------------------------------------------------
*/
@@ -778,14 +842,14 @@ H5D__bt2_idx_is_space_alloc(const H5O_storage_chunk_t *storage)
} /* end H5D__bt2_idx_is_space_alloc() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_mod_cb
+ * Function: H5D__bt2_mod_cb
*
- * Purpose: Modify record for dataset chunk when it is found in a v2 B-tree.
- * This is the callback for H5B2_update() which is called in
- * H5D__bt2_idx_insert().
+ * Purpose: Modify record for dataset chunk when it is found in a v2
+ * B-tree. This is the callback for H5B2_update() which is
+ * called in H5D__bt2_idx_insert().
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -817,18 +881,21 @@ H5D__bt2_mod_cb(void *_record, void *_op_data, bool *changed)
} /* end H5D__bt2_mod_cb() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_insert
+ * Function: H5D__bt2_idx_insert
+ *
+ * Purpose: Insert chunk address into the indexing structure.
+ * A non-filtered chunk:
+ * Should not exist
+ * Allocate the chunk and pass chunk address back up
+ * A filtered chunk:
+ * If it was not found, create the chunk and pass chunk
+ * address back up
+ * If it was found but its size changed, reallocate the chunk
+ * and pass chunk address back up
+ * If it was found but its size was the same, pass chunk
+ * address back up
*
- * Purpose: Insert chunk address into the indexing structure.
- * A non-filtered chunk:
- * Should not exist
- * Allocate the chunk and pass chunk address back up
- * A filtered chunk:
- * If it was not found, create the chunk and pass chunk address back up
- * If it was found but its size changed, reallocate the chunk and pass chunk address back up
- * If it was found but its size was the same, pass chunk address back up
- *
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -854,7 +921,7 @@ H5D__bt2_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata,
assert(H5_addr_defined(udata->chunk_block.offset));
/* Check if the v2 B-tree is open yet */
- if (NULL == idx_info->storage->u.btree2.bt2) {
+ if (!H5D_BT2_IDX_IS_OPEN(idx_info)) {
/* Open existing v2 B-tree */
if (H5D__bt2_idx_open(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open v2 B-tree");
@@ -889,14 +956,14 @@ H5D__bt2_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata,
} /* H5D__bt2_idx_insert() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_found_cb
+ * Function: H5D__bt2_found_cb
*
- * Purpose: Retrieve record for dataset chunk when it is found in a v2 B-tree.
- * This is the callback for H5B2_find() which is called in
- * H5D__bt2_idx_get_addr() and H5D__bt2_idx_insert().
+ * Purpose: Retrieve record for dataset chunk when it is found in a v2
+ * B-tree. This is the callback for H5B2_find() which is called
+ * in H5D__bt2_idx_get_addr() and H5D__bt2_idx_insert().
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -911,13 +978,13 @@ H5D__bt2_found_cb(const void *nrecord, void *op_data)
} /* H5D__bt2_found_cb() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_get_addr
+ * Function: H5D__bt2_idx_get_addr
*
- * Purpose: Get the file address of a chunk if file space has been
- * assigned. Save the retrieved information in the udata
- * supplied.
+ * Purpose: Get the file address of a chunk if file space has been
+ * assigned. Save the retrieved information in the udata
+ * supplied.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -944,7 +1011,7 @@ H5D__bt2_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata)
assert(udata);
/* Check if the v2 B-tree is open yet */
- if (NULL == idx_info->storage->u.btree2.bt2) {
+ if (!H5D_BT2_IDX_IS_OPEN(idx_info)) {
/* Open existing v2 B-tree */
if (H5D__bt2_idx_open(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open v2 B-tree");
@@ -1003,16 +1070,59 @@ H5D__bt2_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata)
} /* H5D__bt2_idx_get_addr() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_iterate_cb
+ * Function: H5D__bt2_idx_load_metadata
+ *
+ * Purpose: Load additional chunk index metadata beyond the chunk index
+ * itself.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__bt2_idx_load_metadata(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info)
+{
+ H5D_chunk_ud_t chunk_ud;
+ hsize_t scaled[H5O_LAYOUT_NDIMS] = {0};
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /*
+ * After opening a dataset that uses a v2 Btree, the root
+ * node will generally not be read in until an element is
+ * looked up for the first time. Since there isn't currently
+ * a good way of controlling that explicitly, perform a fake
+ * lookup of a chunk to cause it to be read in.
+ */
+ chunk_ud.common.layout = idx_info->layout;
+ chunk_ud.common.storage = idx_info->storage;
+ chunk_ud.common.scaled = scaled;
+
+ chunk_ud.chunk_block.offset = HADDR_UNDEF;
+ chunk_ud.chunk_block.length = 0;
+ chunk_ud.filter_mask = 0;
+ chunk_ud.new_unfilt_chunk = false;
+ chunk_ud.idx_hint = UINT_MAX;
+
+ if (H5D__bt2_idx_get_addr(idx_info, &chunk_ud) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't load v2 B-tree root node");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5D__bt2_idx_load_metadata() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__bt2_idx_iterate_cb
*
- * Purpose: Translate the B-tree specific chunk record into a generic
+ * Purpose: Translate the B-tree specific chunk record into a generic
* form and make the callback to the generic chunk callback
* routine.
- * This is the callback for H5B2_iterate() which is called in
- * H5D__bt2_idx_iterate().
+ * This is the callback for H5B2_iterate() which is called in
+ * H5D__bt2_idx_iterate().
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
*-------------------------------------------------------------------------
*/
@@ -1033,12 +1143,12 @@ H5D__bt2_idx_iterate_cb(const void *_record, void *_udata)
} /* H5D__bt2_idx_iterate_cb() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_iterate
+ * Function: H5D__bt2_idx_iterate
*
- * Purpose: Iterate over the chunks in an index, making a callback
+ * Purpose: Iterate over the chunks in an index, making a callback
* for each one.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1062,7 +1172,7 @@ H5D__bt2_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chu
assert(chunk_udata);
/* Check if the v2 B-tree is open yet */
- if (NULL == idx_info->storage->u.btree2.bt2) {
+ if (!H5D_BT2_IDX_IS_OPEN(idx_info)) {
/* Open existing v2 B-tree */
if (H5D__bt2_idx_open(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open v2 B-tree");
@@ -1087,15 +1197,16 @@ H5D__bt2_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chu
} /* end H5D__bt2_idx_iterate() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_remove_cb()
+ * Function: H5D__bt2_remove_cb()
*
- * Purpose: Free space for 'dataset chunk' object as v2 B-tree
- * is being deleted or v2 B-tree node is removed.
- * This is the callback for H5B2_remove() and H5B2_delete() which
- * which are called in H5D__bt2_idx_remove() and H5D__bt2_idx_delete().
+ * Purpose: Free space for 'dataset chunk' object as v2 B-tree
+ * is being deleted or v2 B-tree node is removed.
+ * This is the callback for H5B2_remove() and H5B2_delete()
+ * which are called in H5D__bt2_idx_remove() and
+ * H5D__bt2_idx_delete().
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -1121,11 +1232,11 @@ H5D__bt2_remove_cb(const void *_record, void *_udata)
} /* H5D__bt2_remove_cb() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_remove
+ * Function: H5D__bt2_idx_remove
*
- * Purpose: Remove chunk from index.
+ * Purpose: Remove chunk from index.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1149,7 +1260,7 @@ H5D__bt2_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *u
assert(udata);
/* Check if the v2 B-tree is open yet */
- if (NULL == idx_info->storage->u.btree2.bt2) {
+ if (!H5D_BT2_IDX_IS_OPEN(idx_info)) {
/* Open existing v2 B-tree */
if (H5D__bt2_idx_open(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open v2 B-tree");
@@ -1180,13 +1291,13 @@ H5D__bt2_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *u
} /* H5D__bt2_idx_remove() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_delete
+ * Function: H5D__bt2_idx_delete
*
- * Purpose: Delete index and raw data storage for entire dataset
+ * Purpose: Delete index and raw data storage for entire dataset
* (i.e. all chunks)
*
- * Return: Success: Non-negative
- * Failure: negative
+ * Return: Success: Non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -1233,11 +1344,11 @@ H5D__bt2_idx_delete(const H5D_chk_idx_info_t *idx_info)
} /* end H5D__bt2_idx_delete() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_copy_setup
+ * Function: H5D__bt2_idx_copy_setup
*
- * Purpose: Set up any necessary information for copying chunks
+ * Purpose: Set up any necessary information for copying chunks
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1264,7 +1375,7 @@ H5D__bt2_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk_id
assert(!H5_addr_defined(idx_info_dst->storage->idx_addr));
/* Check if the source v2 B-tree is open yet */
- if (NULL == idx_info_src->storage->u.btree2.bt2)
+ if (!H5D_BT2_IDX_IS_OPEN(idx_info_src))
if (H5D__bt2_idx_open(idx_info_src) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open v2 B-tree");
@@ -1284,11 +1395,11 @@ H5D__bt2_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk_id
} /* end H5D__bt2_idx_copy_setup() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_copy_shutdown
+ * Function: H5D__bt2_idx_copy_shutdown
*
- * Purpose: Shutdown any information from copying chunks
+ * Purpose: Shutdown any information from copying chunks
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1324,8 +1435,8 @@ H5D__bt2_idx_copy_shutdown(H5O_storage_chunk_t *storage_src, H5O_storage_chunk_t
*
* Purpose: Retrieve the amount of index storage for chunked dataset
*
- * Return: Success: Non-negative
- * Failure: negative
+ * Return: Success: Non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -1355,23 +1466,23 @@ H5D__bt2_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size)
/* Get v2 B-tree size for indexing chunked dataset */
if (H5B2_size(bt2_cdset, index_size) < 0)
- HGOTO_ERROR(H5E_SYM, H5E_CANTGET, FAIL, "can't retrieve v2 B-tree storage info for chunked dataset");
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL,
+ "can't retrieve v2 B-tree storage info for chunked dataset");
done:
/* Close v2 B-tree index */
- if (bt2_cdset && H5B2_close(bt2_cdset) < 0)
- HDONE_ERROR(H5E_SYM, H5E_CLOSEERROR, FAIL, "can't close v2 B-tree for tracking chunked dataset");
- idx_info->storage->u.btree2.bt2 = NULL;
+ if (H5D__bt2_idx_close(idx_info) < 0)
+ HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close v2 B-tree for tracking chunked dataset");
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5D__bt2_idx_size() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_reset
+ * Function: H5D__bt2_idx_reset
*
- * Purpose: Reset indexing information.
+ * Purpose: Reset indexing information.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1392,11 +1503,11 @@ H5D__bt2_idx_reset(H5O_storage_chunk_t *storage, bool reset_addr)
} /* end H5D__bt2_idx_reset() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_dump
+ * Function: H5D__bt2_idx_dump
*
- * Purpose: Dump indexing information to a stream.
+ * Purpose: Dump indexing information to a stream.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1415,11 +1526,11 @@ H5D__bt2_idx_dump(const H5O_storage_chunk_t *storage, FILE *stream)
} /* end H5D__bt2_idx_dump() */
/*-------------------------------------------------------------------------
- * Function: H5D__bt2_idx_dest
+ * Function: H5D__bt2_idx_dest
*
- * Purpose: Release indexing information in memory.
+ * Purpose: Release indexing information in memory.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -1436,16 +1547,14 @@ H5D__bt2_idx_dest(const H5D_chk_idx_info_t *idx_info)
assert(idx_info->storage);
/* Check if the v2-btree is open */
- if (idx_info->storage->u.btree2.bt2) {
-
+ if (H5D_BT2_IDX_IS_OPEN(idx_info)) {
/* Patch the top level file pointer contained in bt2 if needed */
if (H5B2_patch_file(idx_info->storage->u.btree2.bt2, idx_info->f) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't patch v2 B-tree file pointer");
/* Close v2 B-tree */
- if (H5B2_close(idx_info->storage->u.btree2.bt2) < 0)
+ if (H5D__bt2_idx_close(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "can't close v2 B-tree");
- idx_info->storage->u.btree2.bt2 = NULL;
} /* end if */
done:
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index 9f4bd90b68a..41d774d0d3e 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -1124,18 +1124,33 @@ H5D__chunk_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo)
if (H5F_SHARED_HAS_FEATURE(io_info->f_sh, H5FD_FEAT_HAS_MPI) &&
H5F_shared_get_coll_metadata_reads(io_info->f_sh) &&
H5D__chunk_is_space_alloc(&dataset->shared->layout.storage)) {
- H5D_chunk_ud_t udata;
- hsize_t scaled[H5O_LAYOUT_NDIMS] = {0};
+ H5O_storage_chunk_t *sc = &(dataset->shared->layout.storage.u.chunk);
+ H5D_chk_idx_info_t idx_info;
+ bool index_is_open;
+
+ idx_info.f = dataset->oloc.file;
+ idx_info.pline = &dataset->shared->dcpl_cache.pline;
+ idx_info.layout = &dataset->shared->layout.u.chunk;
+ idx_info.storage = sc;
+
+ assert(sc && sc->ops && sc->ops->is_open);
+ if (sc->ops->is_open(&idx_info, &index_is_open) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to check if dataset chunk index is open");
+
+ if (!index_is_open) {
+ assert(sc->ops->open);
+ if (sc->ops->open(&idx_info) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to open dataset chunk index");
+ }
/*
- * TODO: Until the dataset chunk index callback structure has
- * callbacks for checking if an index is opened and also for
- * directly opening the index, the following fake chunk lookup
- * serves the purpose of forcing a chunk index open operation
- * on all ranks
+ * Load any other chunk index metadata that we can,
+ * such as fixed array data blocks, while we know all
+ * MPI ranks will do so with collective metadata reads
+ * enabled
*/
- if (H5D__chunk_lookup(dataset, scaled, &udata) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to collectively open dataset chunk index");
+ if (sc->ops->load_metadata && sc->ops->load_metadata(&idx_info) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to load additional chunk index metadata");
}
#endif
@@ -3827,15 +3842,29 @@ H5D__chunk_lookup(const H5D_t *dset, const hsize_t *scaled, H5D_chunk_ud_t *udat
idx_info.storage = sc;
#ifdef H5_HAVE_PARALLEL
- /* Disable collective metadata read for chunk indexes as it is
- * highly unlikely that users would read the same chunks from all
- * processes.
- */
if (H5F_HAS_FEATURE(idx_info.f, H5FD_FEAT_HAS_MPI)) {
- md_reads_file_flag = H5P_FORCE_FALSE;
- md_reads_context_flag = false;
- H5F_set_coll_metadata_reads(idx_info.f, &md_reads_file_flag, &md_reads_context_flag);
- restore_md_reads_state = true;
+ /* Disable collective metadata read for chunk indexes as it is
+ * highly unlikely that users would read the same chunks from all
+ * processes.
+ */
+ if (H5F_get_coll_metadata_reads(idx_info.f)) {
+#ifndef NDEBUG
+ bool index_is_open;
+
+ /*
+ * The dataset's chunk index should be open at this point.
+ * Otherwise, we will end up reading it in independently,
+ * which may not be desired.
+ */
+ sc->ops->is_open(&idx_info, &index_is_open);
+ assert(index_is_open);
+#endif
+
+ md_reads_file_flag = H5P_FORCE_FALSE;
+ md_reads_context_flag = false;
+ H5F_set_coll_metadata_reads(idx_info.f, &md_reads_file_flag, &md_reads_context_flag);
+ restore_md_reads_state = true;
+ }
}
#endif /* H5_HAVE_PARALLEL */
diff --git a/src/H5Dearray.c b/src/H5Dearray.c
index c713b6f18bb..965eaacaca3 100644
--- a/src/H5Dearray.c
+++ b/src/H5Dearray.c
@@ -26,19 +26,21 @@
/***********/
/* Headers */
/***********/
-#include "H5private.h" /* Generic Functions */
-#include "H5Dpkg.h" /* Datasets */
-#include "H5Eprivate.h" /* Error handling */
-#include "H5EAprivate.h" /* Extensible arrays */
+#include "H5private.h" /* Generic Functions */
+#include "H5Dpkg.h" /* Datasets */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5EAprivate.h" /* Extensible arrays */
#include "H5FLprivate.h" /* Free Lists */
-#include "H5MFprivate.h" /* File space management */
-#include "H5MMprivate.h" /* Memory management */
-#include "H5VMprivate.h" /* Vector functions */
+#include "H5MFprivate.h" /* File space management */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5VMprivate.h" /* Vector functions */
/****************/
/* Local Macros */
/****************/
+#define H5D_EARRAY_IDX_IS_OPEN(idx_info) (NULL != (idx_info)->storage->u.earray.ea)
+
/* Value to fill unset array elements with */
#define H5D_EARRAY_FILL HADDR_UNDEF
#define H5D_EARRAY_FILT_FILL \
@@ -106,10 +108,14 @@ static herr_t H5D__earray_filt_debug(FILE *stream, int indent, int fwidth, hsize
static herr_t H5D__earray_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t *space,
haddr_t dset_ohdr_addr);
static herr_t H5D__earray_idx_create(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__earray_idx_open(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__earray_idx_close(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__earray_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open);
static bool H5D__earray_idx_is_space_alloc(const H5O_storage_chunk_t *storage);
static herr_t H5D__earray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata,
const H5D_t *dset);
static herr_t H5D__earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata);
+static herr_t H5D__earray_idx_load_metadata(const H5D_chk_idx_info_t *idx_info);
static herr_t H5D__earray_idx_resize(H5O_layout_chunk_t *layout);
static int H5D__earray_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb,
void *chunk_udata);
@@ -125,7 +131,6 @@ static herr_t H5D__earray_idx_dump(const H5O_storage_chunk_t *storage, FILE *str
static herr_t H5D__earray_idx_dest(const H5D_chk_idx_info_t *idx_info);
/* Generic extensible array routines */
-static herr_t H5D__earray_idx_open(const H5D_chk_idx_info_t *idx_info);
static herr_t H5D__earray_idx_depend(const H5D_chk_idx_info_t *idx_info);
/*********************/
@@ -137,9 +142,13 @@ const H5D_chunk_ops_t H5D_COPS_EARRAY[1] = {{
true, /* Extensible array indices support SWMR access */
H5D__earray_idx_init, /* init */
H5D__earray_idx_create, /* create */
+ H5D__earray_idx_open, /* open */
+ H5D__earray_idx_close, /* close */
+ H5D__earray_idx_is_open, /* is_open */
H5D__earray_idx_is_space_alloc, /* is_space_alloc */
H5D__earray_idx_insert, /* insert */
H5D__earray_idx_get_addr, /* get_addr */
+ H5D__earray_idx_load_metadata, /* load_metadata */
H5D__earray_idx_resize, /* resize */
H5D__earray_idx_iterate, /* iterate */
H5D__earray_idx_remove, /* remove */
@@ -270,10 +279,10 @@ H5D__earray_dst_context(void *_ctx)
/*-------------------------------------------------------------------------
* Function: H5D__earray_fill
*
- * Purpose: Fill "missing elements" in block of elements
+ * Purpose: Fill "missing elements" in block of elements
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -705,59 +714,6 @@ H5D__earray_idx_depend(const H5D_chk_idx_info_t *idx_info)
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5D__earray_idx_depend() */
-/*-------------------------------------------------------------------------
- * Function: H5D__earray_idx_open
- *
- * Purpose: Opens an existing extensible array.
- *
- * Note: This information is passively initialized from each index
- * operation callback because those abstract chunk index operations
- * are designed to work with the v1 B-tree chunk indices also,
- * which don't require an 'open' for the data structure.
- *
- * Return: Success: non-negative
- * Failure: negative
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-H5D__earray_idx_open(const H5D_chk_idx_info_t *idx_info)
-{
- H5D_earray_ctx_ud_t udata; /* User data for extensible array open call */
- herr_t ret_value = SUCCEED; /* Return value */
-
- FUNC_ENTER_PACKAGE
-
- /* Check args */
- assert(idx_info);
- assert(idx_info->f);
- assert(idx_info->pline);
- assert(idx_info->layout);
- assert(H5D_CHUNK_IDX_EARRAY == idx_info->layout->idx_type);
- assert(idx_info->storage);
- assert(H5D_CHUNK_IDX_EARRAY == idx_info->storage->idx_type);
- assert(H5_addr_defined(idx_info->storage->idx_addr));
- assert(NULL == idx_info->storage->u.earray.ea);
-
- /* Set up the user data */
- udata.f = idx_info->f;
- udata.chunk_size = idx_info->layout->size;
-
- /* Open the extensible array for the chunk index */
- if (NULL ==
- (idx_info->storage->u.earray.ea = H5EA_open(idx_info->f, idx_info->storage->idx_addr, &udata)))
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open extensible array");
-
- /* Check for SWMR writes to the file */
- if (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)
- if (H5D__earray_idx_depend(idx_info) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL,
- "unable to create flush dependency on object header");
-
-done:
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5D__earray_idx_open() */
-
/*-------------------------------------------------------------------------
* Function: H5D__earray_idx_init
*
@@ -905,12 +861,120 @@ H5D__earray_idx_create(const H5D_chk_idx_info_t *idx_info)
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5D__earray_idx_create() */
+/*-------------------------------------------------------------------------
+ * Function: H5D__earray_idx_open
+ *
+ * Purpose: Opens an existing extensible array.
+ *
+ * Note: This information is passively initialized from each index
+ * operation callback because those abstract chunk index
+ * operations are designed to work with the v1 B-tree chunk
+ * indices also, which don't require an 'open' for the data
+ * structure.
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__earray_idx_open(const H5D_chk_idx_info_t *idx_info)
+{
+ H5D_earray_ctx_ud_t udata; /* User data for extensible array open call */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args */
+ assert(idx_info);
+ assert(idx_info->f);
+ assert(idx_info->pline);
+ assert(idx_info->layout);
+ assert(H5D_CHUNK_IDX_EARRAY == idx_info->layout->idx_type);
+ assert(idx_info->storage);
+ assert(H5D_CHUNK_IDX_EARRAY == idx_info->storage->idx_type);
+ assert(H5_addr_defined(idx_info->storage->idx_addr));
+ assert(NULL == idx_info->storage->u.earray.ea);
+
+ /* Set up the user data */
+ udata.f = idx_info->f;
+ udata.chunk_size = idx_info->layout->size;
+
+ /* Open the extensible array for the chunk index */
+ if (NULL ==
+ (idx_info->storage->u.earray.ea = H5EA_open(idx_info->f, idx_info->storage->idx_addr, &udata)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open extensible array");
+
+ /* Check for SWMR writes to the file */
+ if (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)
+ if (H5D__earray_idx_depend(idx_info) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL,
+ "unable to create flush dependency on object header");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__earray_idx_open() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__earray_idx_close
+ *
+ * Purpose: Closes an existing extensible array.
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__earray_idx_close(const H5D_chk_idx_info_t *idx_info)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ assert(idx_info);
+ assert(idx_info->storage);
+ assert(H5D_CHUNK_IDX_EARRAY == idx_info->storage->idx_type);
+ assert(idx_info->storage->u.earray.ea);
+
+ if (H5EA_close(idx_info->storage->u.earray.ea) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array");
+ idx_info->storage->u.earray.ea = NULL;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__earray_idx_close() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__earray_idx_is_open
+ *
+ * Purpose: Query if the index is opened or not
+ *
+ * Return: SUCCEED (can't fail)
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__earray_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ assert(idx_info);
+ assert(idx_info->storage);
+ assert(H5D_CHUNK_IDX_EARRAY == idx_info->storage->idx_type);
+ assert(is_open);
+
+ *is_open = H5D_EARRAY_IDX_IS_OPEN(idx_info);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5D__earray_idx_is_open() */
+
/*-------------------------------------------------------------------------
* Function: H5D__earray_idx_is_space_alloc
*
* Purpose: Query if space is allocated for index method
*
- * Return: Non-negative on success/Negative on failure
+ * Return: true/false
*
*-------------------------------------------------------------------------
*/
@@ -953,7 +1017,7 @@ H5D__earray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata
assert(udata);
/* Check if the extensible array is open yet */
- if (NULL == idx_info->storage->u.earray.ea) {
+ if (!H5D_EARRAY_IDX_IS_OPEN(idx_info)) {
/* Open the extensible array in file */
if (H5D__earray_idx_open(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array");
@@ -1021,7 +1085,7 @@ H5D__earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *uda
assert(udata);
/* Check if the extensible array is open yet */
- if (NULL == idx_info->storage->u.earray.ea) {
+ if (!H5D_EARRAY_IDX_IS_OPEN(idx_info)) {
/* Open the extensible array in file */
if (H5D__earray_idx_open(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array");
@@ -1086,6 +1150,51 @@ H5D__earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *uda
FUNC_LEAVE_NOAPI(ret_value)
} /* H5D__earray_idx_get_addr() */
+/*-------------------------------------------------------------------------
+ * Function: H5D__earray_idx_load_metadata
+ *
+ * Purpose: Load additional chunk index metadata beyond the chunk index
+ * itself.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__earray_idx_load_metadata(const H5D_chk_idx_info_t *idx_info)
+{
+ H5D_chunk_ud_t chunk_ud;
+ hsize_t scaled[H5O_LAYOUT_NDIMS] = {0};
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /*
+ * After opening a dataset that uses an extensible array,
+ * the extensible array header index block will generally
+ * not be read in until an element is looked up for the
+ * first time. Since there isn't currently a good way of
+ * controlling that explicitly, perform a fake lookup of
+ * a chunk to cause it to be read in or created if it
+ * doesn't exist yet.
+ */
+ chunk_ud.common.layout = idx_info->layout;
+ chunk_ud.common.storage = idx_info->storage;
+ chunk_ud.common.scaled = scaled;
+
+ chunk_ud.chunk_block.offset = HADDR_UNDEF;
+ chunk_ud.chunk_block.length = 0;
+ chunk_ud.filter_mask = 0;
+ chunk_ud.new_unfilt_chunk = false;
+ chunk_ud.idx_hint = UINT_MAX;
+
+ if (H5D__earray_idx_get_addr(idx_info, &chunk_ud) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't load extensible array header index block");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5D__earray_idx_load_metadata() */
+
/*-------------------------------------------------------------------------
* Function: H5D__earray_idx_resize
*
@@ -1195,10 +1304,6 @@ H5D__earray_idx_iterate_cb(hsize_t H5_ATTR_UNUSED idx, const void *_elmt, void *
* Purpose: Iterate over the chunks in an index, making a callback
* for each one.
*
- * Note: This implementation is slow, particularly for sparse
- * extensible arrays, replace it with call to H5EA_iterate()
- * when that's available.
- *
* Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
@@ -1223,10 +1328,10 @@ H5D__earray_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t
assert(chunk_udata);
/* Check if the extensible array is open yet */
- if (NULL == idx_info->storage->u.earray.ea) {
+ if (!H5D_EARRAY_IDX_IS_OPEN(idx_info)) {
/* Open the extensible array in file */
if (H5D__earray_idx_open(idx_info) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array");
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, H5_ITER_ERROR, "can't open extensible array");
}
else /* Patch the top level file pointer contained in ea if needed */
H5EA_patch_file(idx_info->storage->u.earray.ea, idx_info->f);
@@ -1236,7 +1341,7 @@ H5D__earray_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t
/* Get the extensible array statistics */
if (H5EA_get_stats(ea, &ea_stat) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query extensible array statistics");
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, H5_ITER_ERROR, "can't query extensible array statistics");
if (ea_stat.stored.max_idx_set > 0) {
H5D_earray_it_ud_t udata; /* User data for iteration callback */
@@ -1291,7 +1396,7 @@ H5D__earray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t
assert(udata);
/* Check if the extensible array is open yet */
- if (NULL == idx_info->storage->u.earray.ea) {
+ if (!H5D_EARRAY_IDX_IS_OPEN(idx_info)) {
/* Open the extensible array in file */
if (H5D__earray_idx_open(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array");
@@ -1444,9 +1549,8 @@ H5D__earray_idx_delete(const H5D_chk_idx_info_t *idx_info)
HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to iterate over chunk addresses");
/* Close extensible array */
- if (H5EA_close(idx_info->storage->u.earray.ea) < 0)
+ if (H5D__earray_idx_close(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array");
- idx_info->storage->u.earray.ea = NULL;
/* Set up the context user data */
ctx_udata.f = idx_info->f;
@@ -1494,7 +1598,7 @@ H5D__earray_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk
assert(!H5_addr_defined(idx_info_dst->storage->idx_addr));
/* Check if the source extensible array is open yet */
- if (NULL == idx_info_src->storage->u.earray.ea)
+ if (!H5D_EARRAY_IDX_IS_OPEN(idx_info_src))
/* Open the extensible array in file */
if (H5D__earray_idx_open(idx_info_src) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array");
@@ -1593,9 +1697,8 @@ H5D__earray_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size)
done:
if (idx_info->storage->u.earray.ea) {
- if (H5EA_close(idx_info->storage->u.earray.ea) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array");
- idx_info->storage->u.earray.ea = NULL;
+ if (H5D__earray_idx_close(idx_info) < 0)
+ HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array");
} /* end if */
FUNC_LEAVE_NOAPI(ret_value)
@@ -1673,16 +1776,14 @@ H5D__earray_idx_dest(const H5D_chk_idx_info_t *idx_info)
assert(idx_info->storage);
/* Check if the extensible array is open */
- if (idx_info->storage->u.earray.ea) {
-
+ if (H5D_EARRAY_IDX_IS_OPEN(idx_info)) {
/* Patch the top level file pointer contained in ea if needed */
if (H5EA_patch_file(idx_info->storage->u.earray.ea, idx_info->f) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't patch earray file pointer");
/* Close extensible array */
- if (H5EA_close(idx_info->storage->u.earray.ea) < 0)
+ if (H5D__earray_idx_close(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array");
- idx_info->storage->u.earray.ea = NULL;
} /* end if */
done:
diff --git a/src/H5Dfarray.c b/src/H5Dfarray.c
index 450d466755c..8d06de47b02 100644
--- a/src/H5Dfarray.c
+++ b/src/H5Dfarray.c
@@ -37,6 +37,8 @@
/* Local Macros */
/****************/
+#define H5D_FARRAY_IDX_IS_OPEN(idx_info) (NULL != (idx_info)->storage->u.btree2.bt2)
+
/* Value to fill unset array elements with */
#define H5D_FARRAY_FILL HADDR_UNDEF
#define H5D_FARRAY_FILT_FILL \
@@ -105,10 +107,14 @@ static herr_t H5D__farray_filt_debug(FILE *stream, int indent, int fwidth, hsize
static herr_t H5D__farray_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t *space,
haddr_t dset_ohdr_addr);
static herr_t H5D__farray_idx_create(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__farray_idx_open(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__farray_idx_close(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__farray_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open);
static bool H5D__farray_idx_is_space_alloc(const H5O_storage_chunk_t *storage);
static herr_t H5D__farray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata,
const H5D_t *dset);
static herr_t H5D__farray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata);
+static herr_t H5D__farray_idx_load_metadata(const H5D_chk_idx_info_t *idx_info);
static int H5D__farray_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb,
void *chunk_udata);
static herr_t H5D__farray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata);
@@ -123,7 +129,6 @@ static herr_t H5D__farray_idx_dump(const H5O_storage_chunk_t *storage, FILE *str
static herr_t H5D__farray_idx_dest(const H5D_chk_idx_info_t *idx_info);
/* Generic fixed array routines */
-static herr_t H5D__farray_idx_open(const H5D_chk_idx_info_t *idx_info);
static herr_t H5D__farray_idx_depend(const H5D_chk_idx_info_t *idx_info);
/*********************/
@@ -135,9 +140,13 @@ const H5D_chunk_ops_t H5D_COPS_FARRAY[1] = {{
true, /* Fixed array indices support SWMR access */
H5D__farray_idx_init, /* init */
H5D__farray_idx_create, /* create */
+ H5D__farray_idx_open, /* open */
+ H5D__farray_idx_close, /* close */
+ H5D__farray_idx_is_open, /* is_open */
H5D__farray_idx_is_space_alloc, /* is_space_alloc */
H5D__farray_idx_insert, /* insert */
H5D__farray_idx_get_addr, /* get_addr */
+ H5D__farray_idx_load_metadata, /* load_metadata */
NULL, /* resize */
H5D__farray_idx_iterate, /* iterate */
H5D__farray_idx_remove, /* remove */
@@ -726,55 +735,6 @@ H5D__farray_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t H5_ATTR_UNU
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5D__farray_idx_init() */
-/*-------------------------------------------------------------------------
- * Function: H5D__farray_idx_open
- *
- * Purpose: Opens an existing fixed array and initializes
- * the layout struct with information about the storage.
- *
- * Return: Success: non-negative
- * Failure: negative
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-H5D__farray_idx_open(const H5D_chk_idx_info_t *idx_info)
-{
- H5D_farray_ctx_ud_t udata; /* User data for fixed array open call */
- herr_t ret_value = SUCCEED; /* Return value */
-
- FUNC_ENTER_PACKAGE
-
- /* Check args */
- assert(idx_info);
- assert(idx_info->f);
- assert(idx_info->pline);
- assert(idx_info->layout);
- assert(H5D_CHUNK_IDX_FARRAY == idx_info->layout->idx_type);
- assert(idx_info->storage);
- assert(H5D_CHUNK_IDX_FARRAY == idx_info->storage->idx_type);
- assert(H5_addr_defined(idx_info->storage->idx_addr));
- assert(NULL == idx_info->storage->u.farray.fa);
-
- /* Set up the user data */
- udata.f = idx_info->f;
- udata.chunk_size = idx_info->layout->size;
-
- /* Open the fixed array for the chunk index */
- if (NULL ==
- (idx_info->storage->u.farray.fa = H5FA_open(idx_info->f, idx_info->storage->idx_addr, &udata)))
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open fixed array");
-
- /* Check for SWMR writes to the file */
- if (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)
- if (H5D__farray_idx_depend(idx_info) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL,
- "unable to create flush dependency on object header");
-
-done:
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5D__farray_idx_open() */
-
/*-------------------------------------------------------------------------
* Function: H5D__farray_idx_create
*
@@ -853,12 +813,115 @@ H5D__farray_idx_create(const H5D_chk_idx_info_t *idx_info)
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5D__farray_idx_create() */
+/*-------------------------------------------------------------------------
+ * Function: H5D__farray_idx_open
+ *
+ * Purpose: Opens an existing fixed array and initializes
+ * the layout struct with information about the storage.
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__farray_idx_open(const H5D_chk_idx_info_t *idx_info)
+{
+ H5D_farray_ctx_ud_t udata; /* User data for fixed array open call */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Check args */
+ assert(idx_info);
+ assert(idx_info->f);
+ assert(idx_info->pline);
+ assert(idx_info->layout);
+ assert(H5D_CHUNK_IDX_FARRAY == idx_info->layout->idx_type);
+ assert(idx_info->storage);
+ assert(H5D_CHUNK_IDX_FARRAY == idx_info->storage->idx_type);
+ assert(H5_addr_defined(idx_info->storage->idx_addr));
+ assert(NULL == idx_info->storage->u.farray.fa);
+
+ /* Set up the user data */
+ udata.f = idx_info->f;
+ udata.chunk_size = idx_info->layout->size;
+
+ /* Open the fixed array for the chunk index */
+ if (NULL ==
+ (idx_info->storage->u.farray.fa = H5FA_open(idx_info->f, idx_info->storage->idx_addr, &udata)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open fixed array");
+
+ /* Check for SWMR writes to the file */
+ if (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)
+ if (H5D__farray_idx_depend(idx_info) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL,
+ "unable to create flush dependency on object header");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__farray_idx_open() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__farray_idx_close
+ *
+ * Purpose: Closes an existing fixed array.
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__farray_idx_close(const H5D_chk_idx_info_t *idx_info)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ assert(idx_info);
+ assert(idx_info->storage);
+ assert(H5D_CHUNK_IDX_FARRAY == idx_info->storage->idx_type);
+ assert(idx_info->storage->u.farray.fa);
+
+ if (H5FA_close(idx_info->storage->u.farray.fa) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close fixed array");
+ idx_info->storage->u.farray.fa = NULL;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__farray_idx_close() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__farray_idx_is_open
+ *
+ * Purpose: Query if the index is opened or not
+ *
+ * Return: SUCCEED (can't fail)
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__farray_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ assert(idx_info);
+ assert(idx_info->storage);
+ assert(H5D_CHUNK_IDX_FARRAY == idx_info->storage->idx_type);
+ assert(is_open);
+
+ *is_open = H5D_FARRAY_IDX_IS_OPEN(idx_info);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5D__farray_idx_is_open() */
+
/*-------------------------------------------------------------------------
* Function: H5D__farray_idx_is_space_alloc
*
* Purpose: Query if space is allocated for index method
*
- * Return: Non-negative on success/Negative on failure
+ * Return: true/false
*
*-------------------------------------------------------------------------
*/
@@ -901,7 +964,7 @@ H5D__farray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata
assert(udata);
/* Check if the fixed array is open yet */
- if (NULL == idx_info->storage->u.farray.fa) {
+ if (!H5D_FARRAY_IDX_IS_OPEN(idx_info)) {
/* Open the fixed array in file */
if (H5D__farray_idx_open(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open fixed array");
@@ -969,7 +1032,7 @@ H5D__farray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *uda
assert(udata);
/* Check if the fixed array is open yet */
- if (NULL == idx_info->storage->u.farray.fa) {
+ if (!H5D_FARRAY_IDX_IS_OPEN(idx_info)) {
/* Open the fixed array in file */
if (H5D__farray_idx_open(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open fixed array");
@@ -1016,6 +1079,50 @@ H5D__farray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *uda
FUNC_LEAVE_NOAPI(ret_value)
} /* H5D__farray_idx_get_addr() */
+/*-------------------------------------------------------------------------
+ * Function: H5D__farray_idx_load_metadata
+ *
+ * Purpose: Load additional chunk index metadata beyond the chunk index
+ * itself.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__farray_idx_load_metadata(const H5D_chk_idx_info_t *idx_info)
+{
+ H5D_chunk_ud_t chunk_ud;
+ hsize_t scaled[H5O_LAYOUT_NDIMS] = {0};
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_PACKAGE
+
+ /*
+ * After opening a dataset that uses a fixed array, the
+ * fixed array data block will generally not be read in
+ * until an element is looked up for the first time. Since
+ * there isn't currently a good way of controlling that
+ * explicitly, perform a fake lookup of a chunk to cause
+ * it to be read in.
+ */
+ chunk_ud.common.layout = idx_info->layout;
+ chunk_ud.common.storage = idx_info->storage;
+ chunk_ud.common.scaled = scaled;
+
+ chunk_ud.chunk_block.offset = HADDR_UNDEF;
+ chunk_ud.chunk_block.length = 0;
+ chunk_ud.filter_mask = 0;
+ chunk_ud.new_unfilt_chunk = false;
+ chunk_ud.idx_hint = UINT_MAX;
+
+ if (H5D__farray_idx_get_addr(idx_info, &chunk_ud) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't load fixed array data block");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5D__farray_idx_load_metadata() */
+
/*-------------------------------------------------------------------------
* Function: H5D__farray_idx_iterate_cb
*
@@ -1102,7 +1209,7 @@ H5D__farray_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t
assert(chunk_udata);
/* Check if the fixed array is open yet */
- if (NULL == idx_info->storage->u.farray.fa) {
+ if (!H5D_FARRAY_IDX_IS_OPEN(idx_info)) {
/* Open the fixed array in file */
if (H5D__farray_idx_open(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open fixed array");
@@ -1171,7 +1278,7 @@ H5D__farray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t
assert(udata);
/* Check if the fixed array is open yet */
- if (NULL == idx_info->storage->u.farray.fa) {
+ if (!H5D_FARRAY_IDX_IS_OPEN(idx_info)) {
/* Open the fixed array in file */
if (H5D__farray_idx_open(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open fixed array");
@@ -1302,9 +1409,8 @@ H5D__farray_idx_delete(const H5D_chk_idx_info_t *idx_info)
HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to iterate over chunk addresses");
/* Close fixed array */
- if (H5FA_close(idx_info->storage->u.farray.fa) < 0)
+ if (H5D__farray_idx_close(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close fixed array");
- idx_info->storage->u.farray.fa = NULL;
/* Set up the user data */
ctx_udata.f = idx_info->f;
@@ -1352,10 +1458,11 @@ H5D__farray_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk
assert(!H5_addr_defined(idx_info_dst->storage->idx_addr));
/* Check if the source fixed array is open yet */
- if (NULL == idx_info_src->storage->u.farray.fa)
+ if (!H5D_FARRAY_IDX_IS_OPEN(idx_info_src)) {
/* Open the fixed array in file */
if (H5D__farray_idx_open(idx_info_src) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open fixed array");
+ }
/* Set copied metadata tag */
H5_BEGIN_TAG(H5AC__COPIED_TAG)
@@ -1450,9 +1557,8 @@ H5D__farray_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size)
done:
if (idx_info->storage->u.farray.fa) {
- if (H5FA_close(idx_info->storage->u.farray.fa) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close fixed array");
- idx_info->storage->u.farray.fa = NULL;
+ if (H5D__farray_idx_close(idx_info) < 0)
+ HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close fixed array");
} /* end if */
FUNC_LEAVE_NOAPI(ret_value)
@@ -1528,16 +1634,14 @@ H5D__farray_idx_dest(const H5D_chk_idx_info_t *idx_info)
assert(idx_info->storage);
/* Check if the fixed array is open */
- if (idx_info->storage->u.farray.fa) {
-
+ if (H5D_FARRAY_IDX_IS_OPEN(idx_info)) {
/* Patch the top level file pointer contained in fa if needed */
if (H5FA_patch_file(idx_info->storage->u.farray.fa, idx_info->f) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't patch fixed array file pointer");
/* Close fixed array */
- if (H5FA_close(idx_info->storage->u.farray.fa) < 0)
+ if (H5D__farray_idx_close(idx_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close fixed array");
- idx_info->storage->u.farray.fa = NULL;
} /* end if */
done:
diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c
index 0ef6542fcdb..16243fadf92 100644
--- a/src/H5Dmpio.c
+++ b/src/H5Dmpio.c
@@ -3024,6 +3024,26 @@ H5D__obtain_mpio_mode(H5D_io_info_t *io_info, H5D_dset_io_info_t *di, uint8_t as
* metadata reads are enabled.
*/
if (H5F_get_coll_metadata_reads(di->dset->oloc.file)) {
+#ifndef NDEBUG
+ {
+ H5D_chk_idx_info_t idx_info;
+ bool index_is_open;
+
+ idx_info.f = di->dset->oloc.file;
+ idx_info.pline = &di->dset->shared->dcpl_cache.pline;
+ idx_info.layout = &di->dset->shared->layout.u.chunk;
+ idx_info.storage = &di->dset->shared->layout.storage.u.chunk;
+
+ /*
+ * The dataset's chunk index should be open at this point.
+ * Otherwise, we will end up reading it in independently,
+ * which may not be desired.
+ */
+ idx_info.storage->ops->is_open(&idx_info, &index_is_open);
+ assert(index_is_open);
+ }
+#endif
+
md_reads_file_flag = H5P_FORCE_FALSE;
md_reads_context_flag = false;
H5F_set_coll_metadata_reads(di->dset->oloc.file, &md_reads_file_flag, &md_reads_context_flag);
@@ -3446,26 +3466,6 @@ H5D__mpio_collective_filtered_chunk_io_setup(const H5D_io_info_t *io_info, const
chunk_node = H5SL_next(chunk_node);
}
}
- else if (H5F_get_coll_metadata_reads(di[dset_idx].dset->oloc.file)) {
- hsize_t scaled[H5O_LAYOUT_NDIMS] = {0};
-
- /*
- * If this rank has no selection in the dataset and collective
- * metadata reads are enabled, do a fake lookup of a chunk to
- * ensure that this rank has the chunk index opened. Otherwise,
- * only the ranks that had a selection will have opened the
- * chunk index and they will have done so independently. Therefore,
- * when ranks with no selection participate in later collective
- * metadata reads, they will try to open the chunk index collectively
- * and issues will occur since other ranks won't participate.
- *
- * In the future, we should consider having a chunk index "open"
- * callback that can be used to ensure collectivity between ranks
- * in a more natural way, but this hack should suffice for now.
- */
- if (H5D__chunk_lookup(di[dset_idx].dset, scaled, &udata) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address");
- }
/* Reset metadata tagging */
H5AC_tag(prev_tag, NULL);
diff --git a/src/H5Dnone.c b/src/H5Dnone.c
index 472a2214dc7..d4eb9188840 100644
--- a/src/H5Dnone.c
+++ b/src/H5Dnone.c
@@ -14,9 +14,9 @@
* Purpose: Implicit (Non Index) chunked I/O functions.
*
* This is used when the dataset is:
- * - extendible but with fixed max. dims
- * - with early allocation
- * - without filter
+ * - extendible but with fixed max. dims
+ * - with early allocation
+ * - without filter
*
* The chunk coordinate is mapped into the actual disk addresses
* for the chunk without indexing.
@@ -31,12 +31,12 @@
/***********/
/* Headers */
/***********/
-#include "H5private.h" /* Generic Functions */
-#include "H5Dpkg.h" /* Datasets */
-#include "H5Eprivate.h" /* Error handling */
+#include "H5private.h" /* Generic Functions */
+#include "H5Dpkg.h" /* Datasets */
+#include "H5Eprivate.h" /* Error handling */
#include "H5FLprivate.h" /* Free Lists */
-#include "H5MFprivate.h" /* File space management */
-#include "H5VMprivate.h" /* Vector functions */
+#include "H5MFprivate.h" /* File space management */
+#include "H5VMprivate.h" /* Vector functions */
/****************/
/* Local Macros */
@@ -52,8 +52,12 @@
/* Non Index chunking I/O ops */
static herr_t H5D__none_idx_create(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__none_idx_open(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__none_idx_close(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__none_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open);
static bool H5D__none_idx_is_space_alloc(const H5O_storage_chunk_t *storage);
static herr_t H5D__none_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata);
+static herr_t H5D__none_idx_load_metadata(const H5D_chk_idx_info_t *idx_info);
static int H5D__none_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb,
void *chunk_udata);
static herr_t H5D__none_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata);
@@ -73,9 +77,13 @@ const H5D_chunk_ops_t H5D_COPS_NONE[1] = {{
false, /* Non-indexed chunking don't current support SWMR access */
NULL, /* init */
H5D__none_idx_create, /* create */
+ H5D__none_idx_open, /* open */
+ H5D__none_idx_close, /* close */
+ H5D__none_idx_is_open, /* is_open */
H5D__none_idx_is_space_alloc, /* is_space_alloc */
NULL, /* insert */
H5D__none_idx_get_addr, /* get_addr */
+ H5D__none_idx_load_metadata, /* load_metadata */
NULL, /* resize */
H5D__none_idx_iterate, /* iterate */
H5D__none_idx_remove, /* remove */
@@ -97,12 +105,12 @@ const H5D_chunk_ops_t H5D_COPS_NONE[1] = {{
/*******************/
/*-------------------------------------------------------------------------
- * Function: H5D__none_idx_create
+ * Function: H5D__none_idx_create
*
- * Purpose: Allocate memory for the maximum # of chunks in the dataset.
+ * Purpose: Allocate memory for the maximum # of chunks in the dataset.
*
- * Return: Non-negative on success
- * Negative on failure.
+ * Return: Non-negative on success
+ * Negative on failure.
*
*-------------------------------------------------------------------------
*/
@@ -141,11 +149,73 @@ H5D__none_idx_create(const H5D_chk_idx_info_t *idx_info)
} /* end H5D__none_idx_create() */
/*-------------------------------------------------------------------------
- * Function: H5D__none_idx_is_space_alloc
+ * Function: H5D__none_idx_open
*
- * Purpose: Query if space for the dataset chunks is allocated
+ * Purpose: Opens an existing "none" index. Currently a no-op.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: SUCCEED (can't fail)
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__none_idx_open(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* NO OP */
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5D__none_idx_open() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__none_idx_close
+ *
+ * Purpose: Closes an existing "none" index. Currently a no-op.
+ *
+ * Return: SUCCEED (can't fail)
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__none_idx_close(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* NO OP */
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5D__none_idx_close() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__none_idx_is_open
+ *
+ * Purpose: Query if the index is opened or not
+ *
+ * Return: SUCCEED (can't fail)
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__none_idx_is_open(const H5D_chk_idx_info_t H5_ATTR_NDEBUG_UNUSED *idx_info, bool *is_open)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ assert(idx_info);
+ assert(idx_info->storage);
+ assert(H5D_CHUNK_IDX_NONE == idx_info->storage->idx_type);
+ assert(is_open);
+
+ *is_open = true;
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5D__none_idx_is_open() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__none_idx_is_space_alloc
+ *
+ * Purpose: Query if space for the dataset chunks is allocated
+ *
+ * Return: true/false
*
*-------------------------------------------------------------------------
*/
@@ -161,12 +231,12 @@ H5D__none_idx_is_space_alloc(const H5O_storage_chunk_t *storage)
} /* end H5D__none_idx_is_space_alloc() */
/*-------------------------------------------------------------------------
- * Function: H5D__none_idx_get_addr
+ * Function: H5D__none_idx_get_addr
*
- * Purpose: Get the file address of a chunk.
- * Save the retrieved information in the udata supplied.
+ * Purpose: Get the file address of a chunk.
+ * Save the retrieved information in the udata supplied.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -200,12 +270,32 @@ H5D__none_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata
} /* H5D__none_idx_get_addr() */
/*-------------------------------------------------------------------------
- * Function: H5D__none_idx_iterate
+ * Function: H5D__none_idx_load_metadata
+ *
+ * Purpose: Load additional chunk index metadata beyond the chunk index
+ * itself. Currently a no-op.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__none_idx_load_metadata(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* NO OP */
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5D__none_idx_load_metadata() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__none_idx_iterate
*
- * Purpose: Iterate over the chunks in an index, making a callback
+ * Purpose: Iterate over the chunks in an index, making a callback
* for each one.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -275,13 +365,13 @@ H5D__none_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t ch
} /* end H5D__none_idx_iterate() */
/*-------------------------------------------------------------------------
- * Function: H5D__none_idx_remove
+ * Function: H5D__none_idx_remove
*
- * Purpose: Remove chunk from index.
+ * Purpose: Remove chunk from index.
*
- * Note: Chunks can't be removed (or added) to datasets with this
- * form of index - all the space for all the chunks is always
- * allocated in the file.
+ * Note: Chunks can't be removed (or added) to datasets with this
+ * form of index - all the space for all the chunks is always
+ * allocated in the file.
*
* Return: Non-negative on success/Negative on failure
*
@@ -299,12 +389,12 @@ H5D__none_idx_remove(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info,
} /* H5D__none_idx_remove() */
/*-------------------------------------------------------------------------
- * Function: H5D__none_idx_delete
+ * Function: H5D__none_idx_delete
*
- * Purpose: Delete raw data storage for entire dataset (i.e. all chunks)
+ * Purpose: Delete raw data storage for entire dataset (i.e. all chunks)
*
- * Return: Success: Non-negative
- * Failure: negative
+ * Return: Success: Non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -337,11 +427,11 @@ H5D__none_idx_delete(const H5D_chk_idx_info_t *idx_info)
} /* end H5D__none_idx_delete() */
/*-------------------------------------------------------------------------
- * Function: H5D__none_idx_copy_setup
+ * Function: H5D__none_idx_copy_setup
*
- * Purpose: Set up any necessary information for copying chunks
+ * Purpose: Set up any necessary information for copying chunks
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -407,11 +497,11 @@ H5D__none_idx_size(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, hsize_t *i
} /* end H5D__none_idx_size() */
/*-------------------------------------------------------------------------
- * Function: H5D__none_idx_reset
+ * Function: H5D__none_idx_reset
*
- * Purpose: Reset indexing information.
+ * Purpose: Reset indexing information.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -431,11 +521,11 @@ H5D__none_idx_reset(H5O_storage_chunk_t *storage, bool reset_addr)
} /* end H5D__none_idx_reset() */
/*-------------------------------------------------------------------------
- * Function: H5D__none_idx_dump
+ * Function: H5D__none_idx_dump
*
- * Purpose: Dump
+ * Purpose: Dump
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h
index 82fec0ea1ff..a3695ae8544 100644
--- a/src/H5Dpkg.h
+++ b/src/H5Dpkg.h
@@ -393,10 +393,14 @@ typedef int (*H5D_chunk_cb_func_t)(const H5D_chunk_rec_t *chunk_rec, void *udata
typedef herr_t (*H5D_chunk_init_func_t)(const H5D_chk_idx_info_t *idx_info, const H5S_t *space,
haddr_t dset_ohdr_addr);
typedef herr_t (*H5D_chunk_create_func_t)(const H5D_chk_idx_info_t *idx_info);
+typedef herr_t (*H5D_chunk_open_func_t)(const H5D_chk_idx_info_t *idx_info);
+typedef herr_t (*H5D_chunk_close_func_t)(const H5D_chk_idx_info_t *idx_info);
+typedef herr_t (*H5D_chunk_is_open_func_t)(const H5D_chk_idx_info_t *idx_info, bool *is_open);
typedef bool (*H5D_chunk_is_space_alloc_func_t)(const H5O_storage_chunk_t *storage);
typedef herr_t (*H5D_chunk_insert_func_t)(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata,
const H5D_t *dset);
typedef herr_t (*H5D_chunk_get_addr_func_t)(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata);
+typedef herr_t (*H5D_chunk_load_metadata_func_t)(const H5D_chk_idx_info_t *idx_info);
typedef herr_t (*H5D_chunk_resize_func_t)(H5O_layout_chunk_t *layout);
typedef int (*H5D_chunk_iterate_func_t)(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb,
void *chunk_udata);
@@ -413,13 +417,18 @@ typedef herr_t (*H5D_chunk_dest_func_t)(const H5D_chk_idx_info_t *idx_info);
/* Typedef for grouping chunk I/O routines */
typedef struct H5D_chunk_ops_t {
- bool can_swim; /* Flag to indicate that the index supports SWMR access */
- H5D_chunk_init_func_t init; /* Routine to initialize indexing information in memory */
- H5D_chunk_create_func_t create; /* Routine to create chunk index */
+ bool can_swim; /* Flag to indicate that the index supports SWMR access */
+ H5D_chunk_init_func_t init; /* Routine to initialize indexing information in memory */
+ H5D_chunk_create_func_t create; /* Routine to create chunk index */
+ H5D_chunk_open_func_t open; /* Routine to open chunk index */
+ H5D_chunk_close_func_t close; /* Routine to close chunk index */
+ H5D_chunk_is_open_func_t is_open; /* Query routine to determine if index is open or not */
H5D_chunk_is_space_alloc_func_t
- is_space_alloc; /* Query routine to determine if storage/index is allocated */
- H5D_chunk_insert_func_t insert; /* Routine to insert a chunk into an index */
- H5D_chunk_get_addr_func_t get_addr; /* Routine to retrieve address of chunk in file */
+ is_space_alloc; /* Query routine to determine if storage/index is allocated */
+ H5D_chunk_insert_func_t insert; /* Routine to insert a chunk into an index */
+ H5D_chunk_get_addr_func_t get_addr; /* Routine to retrieve address of chunk in file */
+ H5D_chunk_load_metadata_func_t
+ load_metadata; /* Routine to load additional chunk index metadata, such as fixed array data blocks */
H5D_chunk_resize_func_t resize; /* Routine to update chunk index info after resizing dataset */
H5D_chunk_iterate_func_t iterate; /* Routine to iterate over chunks */
H5D_chunk_remove_func_t remove; /* Routine to remove a chunk from an index */
diff --git a/src/H5Dsingle.c b/src/H5Dsingle.c
index 9cb18d35278..dd9f2353d7b 100644
--- a/src/H5Dsingle.c
+++ b/src/H5Dsingle.c
@@ -27,12 +27,12 @@
/***********/
/* Headers */
/***********/
-#include "H5private.h" /* Generic Functions */
-#include "H5Dpkg.h" /* Datasets */
-#include "H5Eprivate.h" /* Error handling */
+#include "H5private.h" /* Generic Functions */
+#include "H5Dpkg.h" /* Datasets */
+#include "H5Eprivate.h" /* Error handling */
#include "H5FLprivate.h" /* Free Lists */
-#include "H5MFprivate.h" /* File space management */
-#include "H5VMprivate.h" /* Vector functions */
+#include "H5MFprivate.h" /* File space management */
+#include "H5VMprivate.h" /* Vector functions */
/****************/
/* Local Macros */
@@ -50,10 +50,14 @@
static herr_t H5D__single_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t *space,
haddr_t dset_ohdr_addr);
static herr_t H5D__single_idx_create(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__single_idx_open(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__single_idx_close(const H5D_chk_idx_info_t *idx_info);
+static herr_t H5D__single_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open);
static bool H5D__single_idx_is_space_alloc(const H5O_storage_chunk_t *storage);
static herr_t H5D__single_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata,
const H5D_t *dset);
static herr_t H5D__single_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata);
+static herr_t H5D__single_idx_load_metadata(const H5D_chk_idx_info_t *idx_info);
static int H5D__single_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb,
void *chunk_udata);
static herr_t H5D__single_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata);
@@ -73,9 +77,13 @@ const H5D_chunk_ops_t H5D_COPS_SINGLE[1] = {{
false, /* Single Chunk indexing doesn't current support SWMR access */
H5D__single_idx_init, /* init */
H5D__single_idx_create, /* create */
+ H5D__single_idx_open, /* open */
+ H5D__single_idx_close, /* close */
+ H5D__single_idx_is_open, /* is_open */
H5D__single_idx_is_space_alloc, /* is_space_alloc */
H5D__single_idx_insert, /* insert */
H5D__single_idx_get_addr, /* get_addr */
+ H5D__single_idx_load_metadata, /* load_metadata */
NULL, /* resize */
H5D__single_idx_iterate, /* iterate */
H5D__single_idx_remove, /* remove */
@@ -133,12 +141,12 @@ H5D__single_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t H5_ATTR_UNU
} /* end H5D__single_idx_init() */
/*-------------------------------------------------------------------------
- * Function: H5D__single_idx_create
+ * Function: H5D__single_idx_create
*
- * Purpose: Set up Single Chunk Index: filtered or non-filtered
+ * Purpose: Set up Single Chunk Index: filtered or non-filtered
*
- * Return: Non-negative on success
- * Negative on failure.
+ * Return: Non-negative on success
+ * Negative on failure.
*
*-------------------------------------------------------------------------
*/
@@ -166,11 +174,73 @@ H5D__single_idx_create(const H5D_chk_idx_info_t *idx_info)
} /* end H5D__single_idx_create() */
/*-------------------------------------------------------------------------
- * Function: H5D__single_idx_is_space_alloc
+ * Function: H5D__single_idx_open
*
- * Purpose: Query if space is allocated for the single chunk
+ * Purpose: Opens an existing "single" index. Currently a no-op.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: SUCCEED (can't fail)
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__single_idx_open(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* NO OP */
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5D__single_idx_open() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__single_idx_close
+ *
+ * Purpose: Closes an existing "single" index. Currently a no-op.
+ *
+ * Return: SUCCEED (can't fail)
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__single_idx_close(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* NO OP */
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5D__single_idx_close() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__single_idx_is_open
+ *
+ * Purpose: Query if the index is opened or not
+ *
+ * Return: SUCCEED (can't fail)
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__single_idx_is_open(const H5D_chk_idx_info_t H5_ATTR_NDEBUG_UNUSED *idx_info, bool *is_open)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ assert(idx_info);
+ assert(idx_info->storage);
+ assert(H5D_CHUNK_IDX_SINGLE == idx_info->storage->idx_type);
+ assert(is_open);
+
+ *is_open = true;
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5D__single_idx_is_open() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__single_idx_is_space_alloc
+ *
+ * Purpose: Query if space is allocated for the single chunk
+ *
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -186,11 +256,11 @@ H5D__single_idx_is_space_alloc(const H5O_storage_chunk_t *storage)
} /* end H5D__single_idx_is_space_alloc() */
/*-------------------------------------------------------------------------
- * Function: H5D__single_idx_insert
+ * Function: H5D__single_idx_insert
*
- * Purpose: Allocate space for the single chunk
+ * Purpose: Allocate space for the single chunk
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -231,12 +301,12 @@ H5D__single_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata
} /* H5D__single_idx_insert() */
/*-------------------------------------------------------------------------
- * Function: H5D__single_idx_get_addr
+ * Function: H5D__single_idx_get_addr
*
- * Purpose: Get the file address of a chunk.
- * Save the retrieved information in the udata supplied.
+ * Purpose: Get the file address of a chunk.
+ * Save the retrieved information in the udata supplied.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -271,11 +341,31 @@ H5D__single_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *uda
} /* H5D__single_idx_get_addr() */
/*-------------------------------------------------------------------------
- * Function: H5D__single_idx_iterate
+ * Function: H5D__single_idx_load_metadata
+ *
+ * Purpose: Load additional chunk index metadata beyond the chunk index
+ * itself. Currently a no-op.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5D__single_idx_load_metadata(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info)
+{
+ FUNC_ENTER_PACKAGE_NOERR
+
+ /* NO OP */
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5D__single_idx_load_metadata() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__single_idx_iterate
*
- * Purpose: Make callback for the single chunk
+ * Purpose: Make callback for the single chunk
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -318,11 +408,11 @@ H5D__single_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t
} /* end H5D__single_idx_iterate() */
/*-------------------------------------------------------------------------
- * Function: H5D__single_idx_remove
+ * Function: H5D__single_idx_remove
*
- * Purpose: Remove the single chunk
+ * Purpose: Remove the single chunk
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -357,12 +447,13 @@ H5D__single_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t
} /* H5D__single_idx_remove() */
/*-------------------------------------------------------------------------
- * Function: H5D__single_idx_delete
+ * Function: H5D__single_idx_delete
*
- * Purpose: Delete raw data storage for entire dataset (i.e. the only chunk)
+ * Purpose: Delete raw data storage for entire dataset (i.e. the only
+ * chunk)
*
- * Return: Success: Non-negative
- * Failure: negative
+ * Return: Success: Non-negative
+ * Failure: negative
*
*-------------------------------------------------------------------------
*/
@@ -389,11 +480,12 @@ H5D__single_idx_delete(const H5D_chk_idx_info_t *idx_info)
} /* end H5D__single_idx_delete() */
/*-------------------------------------------------------------------------
- * Function: H5D__single_idx_copy_setup
+ * Function: H5D__single_idx_copy_setup
*
- * Purpose: Set up any necessary information for copying the single chunk
+ * Purpose: Set up any necessary information for copying the single
+ * chunk
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -457,11 +549,11 @@ H5D__single_idx_size(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, hsize_t
} /* end H5D__single_idx_size() */
/*-------------------------------------------------------------------------
- * Function: H5D__single_idx_reset
+ * Function: H5D__single_idx_reset
*
- * Purpose: Reset indexing information.
+ * Purpose: Reset indexing information.
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
@@ -481,11 +573,11 @@ H5D__single_idx_reset(H5O_storage_chunk_t *storage, bool reset_addr)
} /* end H5D__single_idx_reset() */
/*-------------------------------------------------------------------------
- * Function: H5D__single_idx_dump
+ * Function: H5D__single_idx_dump
*
- * Purpose: Dump the address of the single chunk
+ * Purpose: Dump the address of the single chunk
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
From 1900cc63eb14a240ce988a0cd99d1e6babd528f1 Mon Sep 17 00:00:00 2001
From: Neil Fortner
Date: Mon, 23 Oct 2023 21:08:48 -0500
Subject: [PATCH 22/52] Fix failure in t_select_io_dset when run with more than
10 ranks (#3758)
---
testpar/t_select_io_dset.c | 30 ++++++++++++++++++++++++++++--
1 file changed, 28 insertions(+), 2 deletions(-)
diff --git a/testpar/t_select_io_dset.c b/testpar/t_select_io_dset.c
index 2be2b407236..9d3f1205051 100644
--- a/testpar/t_select_io_dset.c
+++ b/testpar/t_select_io_dset.c
@@ -221,6 +221,26 @@ check_actual_selection_io_mode(hid_t dxpl, uint32_t sel_io_mode_expected)
}
}
+/*
+ * Helper routine to check actual selection I/O mode on a dxpl
+ */
+static void
+check_actual_selection_io_mode_either(hid_t dxpl, uint32_t sel_io_mode_expected1,
+ uint32_t sel_io_mode_expected2)
+{
+ uint32_t actual_sel_io_mode;
+
+ if (H5Pget_actual_selection_io_mode(dxpl, &actual_sel_io_mode) < 0)
+ P_TEST_ERROR;
+ if (actual_sel_io_mode != sel_io_mode_expected1 && actual_sel_io_mode != sel_io_mode_expected2) {
+ if (MAINPROCESS)
+ printf("\n Failed: Incorrect selection I/O mode (expected/actual) %u or %u : %u",
+ (unsigned)sel_io_mode_expected1, (unsigned)sel_io_mode_expected2,
+ (unsigned)actual_sel_io_mode);
+ P_TEST_ERROR;
+ }
+}
+
/*
* Case 1: single dataset read/write, no type conversion (null case)
*/
@@ -327,8 +347,14 @@ test_no_type_conv(hid_t fid, unsigned chunked, unsigned dtrans, unsigned select,
exp_io_mode = chunked ? H5D_MPIO_CHUNK_COLLECTIVE : H5D_MPIO_CONTIGUOUS_COLLECTIVE;
testing_check_io_mode(dxpl, exp_io_mode);
- if (chunked && !dtrans)
- check_actual_selection_io_mode(dxpl, H5D_VECTOR_IO);
+ if (chunked && !dtrans) {
+ /* If there are more ranks than chunks, then some ranks will not perform vector I/O due to how the
+ * parallel compression code redistributes data */
+ if ((hsize_t)mpi_size > (dims[0] / cdims[0]))
+ check_actual_selection_io_mode_either(dxpl, H5D_VECTOR_IO, 0);
+ else
+ check_actual_selection_io_mode(dxpl, H5D_VECTOR_IO);
+ }
else
check_actual_selection_io_mode(dxpl, select ? H5D_SELECTION_IO : H5D_SCALAR_IO);
From ea1714b3035b9de6cac885508c9f4d882c817b3f Mon Sep 17 00:00:00 2001
From: Glenn Song <43005495+glennsong09@users.noreply.github.com>
Date: Tue, 24 Oct 2023 12:51:55 -0500
Subject: [PATCH 23/52] Fix H5Pset_evict_on_close failing regardless of actual
parallel use (#3761)
Allow H5Pset_evict_on_close to be called regardless of whether a parallel build of HDF5 is being used
Fail during file opens if H5Pset_evict_on_close has been set to true on the given File Access Property List and the size of the MPI communicator being used is greater than 1
---
src/H5Fint.c | 18 ++++++++-
src/H5Pfapl.c | 7 +---
test/evict_on_close.c | 92 -------------------------------------------
testpar/t_file.c | 59 +++++++++++++++++++++++++++
testpar/testphdf5.c | 3 ++
testpar/testphdf5.h | 1 +
6 files changed, 80 insertions(+), 100 deletions(-)
diff --git a/src/H5Fint.c b/src/H5Fint.c
index 014f619d8a9..4093b4b7c45 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -1968,6 +1968,22 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get minimum raw data fraction of page buffer");
} /* end if */
+ /* Get the evict on close setting */
+ if (H5P_get(a_plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get evict on close value");
+
+#ifdef H5_HAVE_PARALLEL
+ /* Check for evict on close in parallel (currently unsupported) */
+ assert(file->shared);
+ if (H5F_SHARED_HAS_FEATURE(file->shared, H5FD_FEAT_HAS_MPI)) {
+ int mpi_size = H5F_shared_mpi_get_size(file->shared);
+
+ if ((mpi_size > 1) && evict_on_close)
+ HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, NULL,
+ "evict on close is currently not supported in parallel HDF5");
+ }
+#endif
+
/*
* Read or write the file superblock, depending on whether the file is
* empty or not.
@@ -2046,8 +2062,6 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
* or later, verify that the access property list value matches the value
* in shared file structure.
*/
- if (H5P_get(a_plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0)
- HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get evict on close value");
if (shared->nrefs == 1)
shared->evict_on_close = evict_on_close;
else if (shared->nrefs > 1) {
diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c
index dc122af9393..e7c1fb3acb8 100644
--- a/src/H5Pfapl.c
+++ b/src/H5Pfapl.c
@@ -4848,7 +4848,7 @@ H5P__facc_mdc_log_location_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR
*-------------------------------------------------------------------------
*/
herr_t
-H5Pset_evict_on_close(hid_t fapl_id, hbool_t H5_ATTR_PARALLEL_UNUSED evict_on_close)
+H5Pset_evict_on_close(hid_t fapl_id, hbool_t evict_on_close)
{
H5P_genplist_t *plist; /* property list pointer */
herr_t ret_value = SUCCEED; /* return value */
@@ -4864,14 +4864,9 @@ H5Pset_evict_on_close(hid_t fapl_id, hbool_t H5_ATTR_PARALLEL_UNUSED evict_on_cl
if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id)))
HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
-#ifndef H5_HAVE_PARALLEL
/* Set value */
if (H5P_set(plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set evict on close property");
-#else
- HGOTO_ERROR(H5E_PLIST, H5E_UNSUPPORTED, FAIL,
- "evict on close is currently not supported in parallel HDF5");
-#endif /* H5_HAVE_PARALLEL */
done:
FUNC_LEAVE_API(ret_value)
diff --git a/test/evict_on_close.c b/test/evict_on_close.c
index 9ca7f9f9cf3..db2a96282ef 100644
--- a/test/evict_on_close.c
+++ b/test/evict_on_close.c
@@ -32,12 +32,6 @@
#include "H5Ipkg.h"
#include "H5VLprivate.h" /* Virtual Object Layer */
-/* Evict on close is not supported under parallel at this time.
- * In the meantime, we just run a simple check that EoC can't be
- * enabled in parallel HDF5.
- */
-#ifndef H5_HAVE_PARALLEL
-
/* Uncomment to manually inspect cache states */
/* (Requires debug build of the library) */
/* #define EOC_MANUAL_INSPECTION */
@@ -974,89 +968,3 @@ main(void)
exit(EXIT_FAILURE);
} /* end main() */
-
-#else
-
-/*-------------------------------------------------------------------------
- * Function: check_evict_on_close_parallel_fail()
- *
- * Purpose: Verify that the H5Pset_evict_on_close() call fails in
- * parallel HDF5.
- *
- * Return: SUCCEED/FAIL
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-check_evict_on_close_parallel_fail(void)
-{
- hid_t fapl_id = H5I_INVALID_HID;
- bool evict_on_close;
- herr_t status;
-
- TESTING("evict on close fails in parallel");
-
- /* Create a fapl */
- if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
- TEST_ERROR;
-
- /* Set the evict on close property (should fail)*/
- evict_on_close = true;
- H5E_BEGIN_TRY
- {
- status = H5Pset_evict_on_close(fapl_id, evict_on_close);
- }
- H5E_END_TRY
- if (status >= 0)
- FAIL_PUTS_ERROR("H5Pset_evict_on_close() did not fail in parallel HDF5.");
-
- /* close fapl */
- if (H5Pclose(fapl_id) < 0)
- TEST_ERROR;
-
- PASSED();
- return SUCCEED;
-
-error:
- H5_FAILED();
- return FAIL;
-
-} /* check_evict_on_close_parallel_fail() */
-
-/*-------------------------------------------------------------------------
- * Function: main (parallel version)
- *
- * Return: EXIT_FAILURE/EXIT_SUCCESS
- *
- *-------------------------------------------------------------------------
- */
-int
-main(void)
-{
- unsigned nerrors = 0; /* number of test errors */
-
- printf("Testing evict-on-close cache behavior\n");
-
- /* Initialize */
- h5_reset();
-
- /* Test that EoC fails in parallel HDF5 */
- nerrors += check_evict_on_close_parallel_fail() < 0 ? 1 : 0;
-
- if (nerrors)
- goto error;
-
- printf("All evict-on-close tests passed.\n");
- printf("Note that EoC is not supported under parallel so most tests are skipped.\n");
-
- exit(EXIT_SUCCESS);
-
-error:
-
- printf("***** %u evict-on-close test%s FAILED! *****\n", nerrors, nerrors > 1 ? "S" : "");
-
- exit(EXIT_FAILURE);
-
-} /* main() - parallel */
-
-#endif /* H5_HAVE_PARALLEL */
diff --git a/testpar/t_file.c b/testpar/t_file.c
index a6a541becf3..700ccc2256f 100644
--- a/testpar/t_file.c
+++ b/testpar/t_file.c
@@ -1060,3 +1060,62 @@ test_invalid_libver_bounds_file_close_assert(void)
ret = H5Pclose(fcpl_id);
VRFY((SUCCEED == ret), "H5Pclose");
}
+
+/*
+ * Tests that H5Pevict_on_close properly succeeds in serial/one rank and fails when
+ * called by multiple ranks.
+ */
+void
+test_evict_on_close_parallel_unsupp(void)
+{
+ const char *filename = NULL;
+ MPI_Comm comm = MPI_COMM_WORLD;
+ MPI_Info info = MPI_INFO_NULL;
+ hid_t fid = H5I_INVALID_HID;
+ hid_t fapl_id = H5I_INVALID_HID;
+ herr_t ret;
+
+ filename = (const char *)GetTestParameters();
+
+ /* set up MPI parameters */
+ MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
+ MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+
+ /* setup file access plist */
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ VRFY((fapl_id != H5I_INVALID_HID), "H5Pcreate");
+ ret = H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_V18);
+ VRFY((SUCCEED == ret), "H5Pset_libver_bounds");
+
+ ret = H5Pset_evict_on_close(fapl_id, true);
+ VRFY((SUCCEED == ret), "H5Pset_evict_on_close");
+
+ /* test on 1 rank */
+ ret = H5Pset_fapl_mpio(fapl_id, MPI_COMM_SELF, info);
+ VRFY((SUCCEED == ret), "H5Pset_fapl_mpio");
+
+ if (mpi_rank == 0) {
+ fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+ VRFY((SUCCEED == ret), "H5Fcreate");
+ ret = H5Fclose(fid);
+ VRFY((SUCCEED == ret), "H5Fclose");
+ }
+
+ VRFY((MPI_SUCCESS == MPI_Barrier(MPI_COMM_WORLD)), "MPI_Barrier");
+
+ /* test on multiple ranks if we have them */
+ if (mpi_size > 1) {
+ ret = H5Pset_fapl_mpio(fapl_id, comm, info);
+ VRFY((SUCCEED == ret), "H5Pset_fapl_mpio");
+
+ H5E_BEGIN_TRY
+ {
+ fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+ }
+ H5E_END_TRY
+ VRFY((fid == H5I_INVALID_HID), "H5Fcreate");
+ }
+
+ ret = H5Pclose(fapl_id);
+ VRFY((SUCCEED == ret), "H5Pclose");
+}
diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c
index 2d85e1ae289..2428c719a1d 100644
--- a/testpar/testphdf5.c
+++ b/testpar/testphdf5.c
@@ -366,6 +366,9 @@ main(int argc, char **argv)
AddTest("invlibverassert", test_invalid_libver_bounds_file_close_assert, NULL,
"Invalid libver bounds assertion failure", PARATESTFILE);
+ AddTest("evictparassert", test_evict_on_close_parallel_unsupp, NULL, "Evict on close in parallel failure",
+ PARATESTFILE);
+
AddTest("idsetw", dataset_writeInd, NULL, "dataset independent write", PARATESTFILE);
AddTest("idsetr", dataset_readInd, NULL, "dataset independent read", PARATESTFILE);
diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h
index 5699760c61b..6bbdb0d23d5 100644
--- a/testpar/testphdf5.h
+++ b/testpar/testphdf5.h
@@ -233,6 +233,7 @@ void zero_dim_dset(void);
void test_file_properties(void);
void test_delete(void);
void test_invalid_libver_bounds_file_close_assert(void);
+void test_evict_on_close_parallel_unsupp(void);
void multiple_dset_write(void);
void multiple_group_write(void);
void multiple_group_read(void);
From fbf77a8b1620cd23e707294e22693caaf79c60ab Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Tue, 24 Oct 2023 16:28:22 -0500
Subject: [PATCH 24/52] Add release note for H5Pset_evict_on_close change for
parallel HDF5 (#3765)
---
release_docs/RELEASE.txt | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index e5cb09707bd..291f72a8701 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -320,6 +320,18 @@ New Features
performing I/O on all the filtered datasets at once and then performing
I/O on all the unfiltered datasets at once.
+ - Changed H5Pset_evict_on_close so that it can be called with a parallel
+ build of HDF5
+
+ Previously, H5Pset_evict_on_close would always fail when called from a
+ parallel build of HDF5, stating that the feature is not supported with
+ parallel HDF5. This failure would occur even if a parallel build of HDF5
+ was used with a serial HDF5 application. H5Pset_evict_on_close can now
+ be called regardless of the library build type and the library will
+ instead fail during H5Fcreate/H5Fopen if the "evict on close" property
+ has been set to true and the file is being opened for parallel access
+ with more than 1 MPI process.
+
Fortran Library:
----------------
From 6578c452b5d7b0db82d3cf4e58f8a2a11e909b52 Mon Sep 17 00:00:00 2001
From: vchoi-hdfgroup <55293060+vchoi-hdfgroup@users.noreply.github.com>
Date: Tue, 24 Oct 2023 21:48:28 -0500
Subject: [PATCH 25/52] Preserve MPI-I/O file hints when fapl is closed (#3755)
* Fix for issue #3025: Save the MPI info in the file struct so H5Fget_access_plist() can retrieve it from there.
---
src/H5Fint.c | 23 +++++++---
src/H5Fpkg.h | 1 +
testpar/t_file.c | 102 ++++++++++++++++++++++++++++++++++++++++++++
testpar/testphdf5.c | 2 +
testpar/testphdf5.h | 1 +
5 files changed, 124 insertions(+), 5 deletions(-)
diff --git a/src/H5Fint.c b/src/H5Fint.c
index 4093b4b7c45..439fa4f35a6 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -402,7 +402,6 @@ H5F_get_access_plist(H5F_t *f, bool app_ref)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set collective metadata read flag");
if (H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) {
MPI_Comm mpi_comm;
- MPI_Info mpi_info;
/* Retrieve and set MPI communicator */
if (MPI_COMM_NULL == (mpi_comm = H5F_mpi_get_comm(f)))
@@ -410,10 +409,8 @@ H5F_get_access_plist(H5F_t *f, bool app_ref)
if (H5P_set(new_plist, H5F_ACS_MPI_PARAMS_COMM_NAME, &mpi_comm) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set MPI communicator");
- /* Retrieve and set MPI info object */
- if (H5P_get(old_plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &mpi_info) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, H5I_INVALID_HID, "can't get MPI info object");
- if (H5P_set(new_plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &mpi_info) < 0)
+ /* Retrieve MPI info object */
+ if (H5P_set(new_plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &(f->shared->mpi_info)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set MPI info object");
}
#endif /* H5_HAVE_PARALLEL */
@@ -1133,6 +1130,12 @@ H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F
/* initialize point of no return */
f->shared->point_of_no_return = false;
+#ifdef H5_HAVE_PARALLEL
+ /* Initialize this just in case we fail before setting this field and */
+ /* we try to call H5_mpi_info_free() on uninitialized memory in H5F__dest() */
+ f->shared->mpi_info = MPI_INFO_NULL;
+#endif /* H5_HAVE_PARALLEL */
+
/* Copy the file creation and file access property lists into the
* new file handle. We do this early because some values might need
* to change as the file is being opened.
@@ -1209,6 +1212,8 @@ H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get collective metadata read flag");
if (H5P_get(plist, H5F_ACS_COLL_MD_WRITE_FLAG_NAME, &(f->shared->coll_md_write)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get collective metadata write flag");
+ if (H5P_get(plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &(f->shared->mpi_info)) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't set MPI info object");
#endif /* H5_HAVE_PARALLEL */
if (H5P_get(plist, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, &(f->shared->mdc_initCacheImageCfg)) <
0)
@@ -1414,6 +1419,14 @@ H5F__dest(H5F_t *f, bool flush, bool free_on_failure)
f->shared->efc = NULL;
} /* end if */
+#ifdef H5_HAVE_PARALLEL
+ if (f->shared->mpi_info != MPI_INFO_NULL) {
+ /* Free MPI info saved in the file struct */
+ if (H5_mpi_info_free(&f->shared->mpi_info) < 0)
+ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't free MPI info");
+ }
+#endif
+
/* With the shutdown modifications, the contents of the metadata cache
* should be clean at this point, with the possible exception of the
* the superblock and superblock extension.
diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h
index bc5c90bd5da..e81b25072eb 100644
--- a/src/H5Fpkg.h
+++ b/src/H5Fpkg.h
@@ -359,6 +359,7 @@ struct H5F_shared_t {
#ifdef H5_HAVE_PARALLEL
H5P_coll_md_read_flag_t coll_md_read; /* Do all metadata reads collectively */
bool coll_md_write; /* Do all metadata writes collectively */
+ MPI_Info mpi_info; /* MPI info */
#endif /* H5_HAVE_PARALLEL */
};
diff --git a/testpar/t_file.c b/testpar/t_file.c
index 700ccc2256f..ce55270cdd2 100644
--- a/testpar/t_file.c
+++ b/testpar/t_file.c
@@ -1119,3 +1119,105 @@ test_evict_on_close_parallel_unsupp(void)
ret = H5Pclose(fapl_id);
VRFY((SUCCEED == ret), "H5Pclose");
}
+
+/*
+ * Verify that MPI I/O hints are preserved after closing the file access property list
+ * as described in issue #3025
+ * This is a test program from the user.
+ */
+void
+test_fapl_preserve_hints(void)
+{
+ hid_t fid = H5I_INVALID_HID; /* HDF5 file ID */
+ hid_t fapl_id = H5I_INVALID_HID; /* File access plist */
+ const char *filename;
+
+ int nkeys_used;
+ bool same = false;
+
+ MPI_Info info = MPI_INFO_NULL;
+ const char *key = "hdf_info_fapl";
+ const char *value = "xyz";
+
+ MPI_Info info_used = MPI_INFO_NULL;
+ int flag = -1;
+ char value_used[20];
+ char key_used[20];
+
+ int i;
+ herr_t ret; /* Generic return value */
+ int mpi_ret; /* MPI return value */
+
+ filename = (const char *)GetTestParameters();
+
+ /* set up MPI parameters */
+ mpi_ret = MPI_Info_create(&info);
+ VRFY((mpi_ret >= 0), "MPI_Info_create succeeded");
+
+ mpi_ret = MPI_Info_set(info, key, value);
+ VRFY((mpi_ret == MPI_SUCCESS), "MPI_Info_set succeeded");
+
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ VRFY((fapl_id != H5I_INVALID_HID), "H5Pcreate");
+
+ ret = H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, info);
+ VRFY((ret >= 0), "H5Pset_fapl_mpio");
+
+ fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+ VRFY((fid != H5I_INVALID_HID), "H5Fcreate succeeded");
+
+ ret = H5Pclose(fapl_id);
+ VRFY((ret >= 0), "H5Pclose succeeded");
+
+ fapl_id = H5Fget_access_plist(fid);
+ VRFY((fapl_id != H5I_INVALID_HID), "H5Fget_access_plist succeeded");
+
+ ret = H5Pget_fapl_mpio(fapl_id, NULL, &info_used);
+ VRFY((ret >= 0), "H5Pget_fapl_mpio succeeded");
+
+ VRFY((info_used != MPI_INFO_NULL), "H5Pget_fapl_mpio");
+
+ mpi_ret = MPI_Info_get_nkeys(info_used, &nkeys_used);
+ VRFY((mpi_ret == MPI_SUCCESS), "MPI_Info_get_nkeys succeeded");
+
+ /* Loop over the # of keys */
+ for (i = 0; i < nkeys_used; i++) {
+
+ /* Memset the buffers to zero */
+ memset(key_used, 0, 20);
+ memset(value_used, 0, 20);
+
+ /* Get the nth key */
+ mpi_ret = MPI_Info_get_nthkey(info_used, i, key_used);
+ VRFY((mpi_ret == MPI_SUCCESS), "MPI_Info_get_nthkey succeeded");
+
+ if (!strcmp(key_used, key)) {
+
+ mpi_ret = MPI_Info_get(info_used, key_used, 20, value_used, &flag);
+ VRFY((mpi_ret == MPI_SUCCESS), "MPI_Info_get succeeded");
+
+ if (!strcmp(value_used, value)) {
+
+ /* Both key_used and value_used are the same */
+ same = true;
+ break;
+ }
+ }
+ } /* end for */
+
+ VRFY((same == true), "key_used and value_used are the same");
+
+ ret = H5Pclose(fapl_id);
+ VRFY((ret >= 0), "H5Pclose succeeded");
+
+ ret = H5Fclose(fid);
+ VRFY((ret >= 0), "H5Fclose succeeded");
+
+ /* Free the MPI info object */
+ mpi_ret = MPI_Info_free(&info);
+ VRFY((mpi_ret >= 0), "MPI_Info_free succeeded");
+
+ mpi_ret = MPI_Info_free(&info_used);
+ VRFY((mpi_ret >= 0), "MPI_Info_free succeeded");
+
+} /* end test_fapl_preserve_hints() */
diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c
index 2428c719a1d..985c3de734c 100644
--- a/testpar/testphdf5.c
+++ b/testpar/testphdf5.c
@@ -368,6 +368,8 @@ main(int argc, char **argv)
AddTest("evictparassert", test_evict_on_close_parallel_unsupp, NULL, "Evict on close in parallel failure",
PARATESTFILE);
+ AddTest("fapl_preserve", test_fapl_preserve_hints, NULL, "preserve MPI I/O hints after fapl closed",
+ PARATESTFILE);
AddTest("idsetw", dataset_writeInd, NULL, "dataset independent write", PARATESTFILE);
AddTest("idsetr", dataset_readInd, NULL, "dataset independent read", PARATESTFILE);
diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h
index 6bbdb0d23d5..45f1a945d07 100644
--- a/testpar/testphdf5.h
+++ b/testpar/testphdf5.h
@@ -234,6 +234,7 @@ void test_file_properties(void);
void test_delete(void);
void test_invalid_libver_bounds_file_close_assert(void);
void test_evict_on_close_parallel_unsupp(void);
+void test_fapl_preserve_hints(void);
void multiple_dset_write(void);
void multiple_group_write(void);
void multiple_group_read(void);
From 1d8513f034864e690d6006d5b6a324e2edc2d3ea Mon Sep 17 00:00:00 2001
From: Scot Breitenfeld
Date: Wed, 25 Oct 2023 13:49:30 -0500
Subject: [PATCH 26/52] Add compression tests for subfiling (#3769)
---
testpar/t_subfiling_vfd.c | 142 +++++++++++++++++++++++++++++++++-----
1 file changed, 125 insertions(+), 17 deletions(-)
diff --git a/testpar/t_subfiling_vfd.c b/testpar/t_subfiling_vfd.c
index 7c565997b3d..72613a3bef1 100644
--- a/testpar/t_subfiling_vfd.c
+++ b/testpar/t_subfiling_vfd.c
@@ -40,6 +40,8 @@
#define PATH_MAX 4096
#endif
+#define DEFAULT_DEFLATE_LEVEL 9
+
#define ARRAY_SIZE(a) sizeof(a) / sizeof(a[0])
#define CHECK_PASSED() \
@@ -82,12 +84,15 @@ static char *config_dir = NULL;
int nerrors = 0;
int curr_nerrors = 0;
+bool enable_compression = false;
+
/* Function pointer typedef for test functions */
typedef void (*test_func)(void);
/* Utility functions */
static hid_t create_subfiling_ioc_fapl(MPI_Comm comm, MPI_Info info, bool custom_config,
H5FD_subfiling_params_t *custom_cfg, int32_t thread_pool_size);
+static hid_t create_dcpl_id(int rank, const hsize_t dims[], hid_t dxpl_id);
/* Test functions */
static void test_create_and_close(void);
@@ -182,7 +187,47 @@ create_subfiling_ioc_fapl(MPI_Comm comm, MPI_Info info, bool custom_config,
return H5I_INVALID_HID;
}
+/* ---------------------------------------------------------------------------
+ * Function: create_dcpl_id
+ *
+ * Purpose: Creates dataset creation property list identifier with
+ * chunking and compression, and enforces the
+ * required collective IO.
+ *
+ * Return: Success: HID Dataset creation property list identifier,
+ * a non-negative value.
+ * Failure: H5I_INVALID_HID, a negative value.
+ * ---------------------------------------------------------------------------
+ */
+static hid_t
+create_dcpl_id(int rank, const hsize_t dset_dims[], hid_t dxpl_id)
+{
+ hsize_t chunk_dims[1];
+ hid_t ret_value = H5I_INVALID_HID;
+
+ if ((ret_value = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ TEST_ERROR;
+
+ if (enable_compression) {
+ if (H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE) < 0)
+ TEST_ERROR;
+ chunk_dims[0] = dset_dims[0] / 2;
+ if (H5Pset_chunk(ret_value, rank, chunk_dims) < 0)
+ TEST_ERROR;
+ if (H5Pset_deflate(ret_value, DEFAULT_DEFLATE_LEVEL) < 0)
+ TEST_ERROR;
+ }
+
+ return ret_value;
+error:
+ if ((H5I_INVALID_HID != ret_value) && (H5Pclose(ret_value) < 0)) {
+ H5_FAILED();
+ AT();
+ }
+
+ return H5I_INVALID_HID;
+}
/*
* A simple test that creates and closes a file with the
* subfiling VFD
@@ -1060,6 +1105,7 @@ test_read_different_stripe_size(void)
hid_t fapl_id = H5I_INVALID_HID;
hid_t dset_id = H5I_INVALID_HID;
hid_t dxpl_id = H5I_INVALID_HID;
+ hid_t dcpl_id = H5I_INVALID_HID;
hid_t fspace_id = H5I_INVALID_HID;
char *tmp_filename = NULL;
void *buf = NULL;
@@ -1106,7 +1152,10 @@ test_read_different_stripe_size(void)
fspace_id = H5Screate_simple(1, dset_dims, NULL);
VRFY((fspace_id >= 0), "H5Screate_simple succeeded");
- dset_id = H5Dcreate2(file_id, "DSET", SUBF_HDF5_TYPE, fspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ dcpl_id = create_dcpl_id(1, dset_dims, dxpl_id);
+ VRFY((dcpl_id >= 0), "DCPL creation succeeded");
+
+ dset_id = H5Dcreate2(file_id, "DSET", SUBF_HDF5_TYPE, fspace_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT);
VRFY((dset_id >= 0), "Dataset creation succeeded");
/* Select hyperslab */
@@ -1129,6 +1178,7 @@ test_read_different_stripe_size(void)
VRFY((H5Sclose(fspace_id) >= 0), "File dataspace close succeeded");
VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded");
+ VRFY((H5Pclose(dcpl_id) >= 0), "DCPL close succeeded");
VRFY((H5Fclose(file_id) >= 0), "File close succeeded");
/* Ensure all the subfiles are present */
@@ -1153,10 +1203,12 @@ test_read_different_stripe_size(void)
VRFY((fclose(subfile_ptr) >= 0), "fclose on subfile succeeded");
/* Check file size */
- VRFY((HDstat(tmp_filename, &subfile_info) >= 0), "HDstat succeeded");
- subfile_size = (h5_stat_size_t)subfile_info.st_size;
+ if (!enable_compression) {
+ VRFY((HDstat(tmp_filename, &subfile_info) >= 0), "HDstat succeeded");
+ subfile_size = (h5_stat_size_t)subfile_info.st_size;
- VRFY((subfile_size >= cfg.stripe_size), "File size verification succeeded");
+ VRFY((subfile_size >= cfg.stripe_size), "File size verification succeeded");
+ }
}
}
@@ -1376,10 +1428,12 @@ test_subfiling_precreate_rank_0(void)
VRFY((fclose(subfile_ptr) >= 0), "fclose on subfile succeeded");
/* Check file size */
- VRFY((HDstat(tmp_filename, &subfile_info) >= 0), "HDstat succeeded");
- file_size = (h5_stat_size_t)subfile_info.st_size;
+ if (!enable_compression) {
+ VRFY((HDstat(tmp_filename, &subfile_info) >= 0), "HDstat succeeded");
+ file_size = (h5_stat_size_t)subfile_info.st_size;
- VRFY((file_size >= cfg.stripe_size), "File size verification succeeded");
+ VRFY((file_size >= cfg.stripe_size), "File size verification succeeded");
+ }
}
/* Verify that there aren't too many subfiles */
@@ -1470,6 +1524,7 @@ test_subfiling_write_many_read_one(void)
hid_t fapl_id = H5I_INVALID_HID;
hid_t dset_id = H5I_INVALID_HID;
hid_t dxpl_id = H5I_INVALID_HID;
+ hid_t dcpl_id = H5I_INVALID_HID;
hid_t fspace_id = H5I_INVALID_HID;
void *buf = NULL;
@@ -1517,7 +1572,10 @@ test_subfiling_write_many_read_one(void)
fspace_id = H5Screate_simple(1, dset_dims, NULL);
VRFY((fspace_id >= 0), "H5Screate_simple succeeded");
- dset_id = H5Dcreate2(file_id, "DSET", SUBF_HDF5_TYPE, fspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ dcpl_id = create_dcpl_id(1, dset_dims, dxpl_id);
+ VRFY((dcpl_id >= 0), "DCPL creation succeeded");
+
+ dset_id = H5Dcreate2(file_id, "DSET", SUBF_HDF5_TYPE, fspace_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT);
VRFY((dset_id >= 0), "Dataset creation succeeded");
/* Select hyperslab */
@@ -1539,6 +1597,7 @@ test_subfiling_write_many_read_one(void)
buf = NULL;
VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded");
+ VRFY((H5Pclose(dcpl_id) >= 0), "DCPL close succeeded");
VRFY((H5Fclose(file_id) >= 0), "File close succeeded");
mpi_code_g = MPI_Barrier(comm_g);
@@ -1616,6 +1675,7 @@ test_subfiling_write_many_read_few(void)
hid_t fapl_id = H5I_INVALID_HID;
hid_t dset_id = H5I_INVALID_HID;
hid_t dxpl_id = H5I_INVALID_HID;
+ hid_t dcpl_id = H5I_INVALID_HID;
hid_t fspace_id = H5I_INVALID_HID;
void *buf = NULL;
@@ -1673,7 +1733,10 @@ test_subfiling_write_many_read_few(void)
fspace_id = H5Screate_simple(1, dset_dims, NULL);
VRFY((fspace_id >= 0), "H5Screate_simple succeeded");
- dset_id = H5Dcreate2(file_id, "DSET", SUBF_HDF5_TYPE, fspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ dcpl_id = create_dcpl_id(1, dset_dims, dxpl_id);
+ VRFY((dcpl_id >= 0), "DCPL creation succeeded");
+
+ dset_id = H5Dcreate2(file_id, "DSET", SUBF_HDF5_TYPE, fspace_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT);
VRFY((dset_id >= 0), "Dataset creation succeeded");
/* Select hyperslab */
@@ -1695,6 +1758,7 @@ test_subfiling_write_many_read_few(void)
buf = NULL;
VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded");
+ VRFY((H5Pclose(dcpl_id) >= 0), "DCPL close succeeded");
VRFY((H5Fclose(file_id) >= 0), "File close succeeded");
/*
@@ -1808,6 +1872,7 @@ test_subfiling_h5fuse(void)
hid_t fapl_id = H5I_INVALID_HID;
hid_t dset_id = H5I_INVALID_HID;
hid_t dxpl_id = H5I_INVALID_HID;
+ hid_t dcpl_id = H5I_INVALID_HID;
hid_t fspace_id = H5I_INVALID_HID;
void *buf = NULL;
int skip_test = 0;
@@ -1898,7 +1963,10 @@ test_subfiling_h5fuse(void)
fspace_id = H5Screate_simple(1, dset_dims, NULL);
VRFY((fspace_id >= 0), "H5Screate_simple succeeded");
- dset_id = H5Dcreate2(file_id, "DSET", SUBF_HDF5_TYPE, fspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ dcpl_id = create_dcpl_id(1, dset_dims, dxpl_id);
+ VRFY((dcpl_id >= 0), "DCPL creation succeeded");
+
+ dset_id = H5Dcreate2(file_id, "DSET", SUBF_HDF5_TYPE, fspace_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT);
VRFY((dset_id >= 0), "Dataset creation succeeded");
/* Select hyperslab */
@@ -1919,8 +1987,11 @@ test_subfiling_h5fuse(void)
free(buf);
buf = NULL;
+ VRFY((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_INDEPENDENT) >= 0), "H5Pset_dxpl_mpio succeeded");
+
VRFY((H5Sclose(fspace_id) >= 0), "File dataspace close succeeded");
VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded");
+ VRFY((H5Pclose(dcpl_id) >= 0), "DCPL close succeeded");
VRFY((H5Fclose(file_id) >= 0), "File close succeeded");
if (MAINPROCESS) {
@@ -1973,8 +2044,10 @@ test_subfiling_h5fuse(void)
}
/* Verify the size of the fused file */
- VRFY((HDstat(SUBF_FILENAME, &file_info) >= 0), "HDstat succeeded");
- VRFY(((size_t)file_info.st_size >= target_size), "File size verification succeeded");
+ if (!enable_compression) {
+ VRFY((HDstat(SUBF_FILENAME, &file_info) >= 0), "HDstat succeeded");
+ VRFY(((size_t)file_info.st_size >= target_size), "File size verification succeeded");
+ }
/* Re-open file with sec2 driver and verify the data */
file_id = H5Fopen(SUBF_FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT);
@@ -2414,9 +2487,28 @@ main(int argc, char **argv)
if (num_iocs_g > mpi_size)
num_iocs_g = mpi_size;
- if (MAINPROCESS) {
- printf("Re-running tests with environment variables set\n");
+ if (MAINPROCESS)
+ printf(" Re-running tests with compression enabled\n");
+
+#ifdef H5_HAVE_FILTER_DEFLATE
+ enable_compression = true;
+ for (size_t i = 0; i < ARRAY_SIZE(tests); i++) {
+ if (MPI_SUCCESS == (mpi_code_g = MPI_Barrier(comm_g))) {
+ (*tests[i])();
+ }
+ else {
+ if (MAINPROCESS)
+ MESG("MPI_Barrier failed");
+ nerrors++;
+ }
}
+ enable_compression = false;
+#else
+ if (MAINPROCESS)
+ SKIPPED();
+#endif
+ if (MAINPROCESS)
+ printf("\nRe-running tests with environment variables set\n");
for (size_t i = 0; i < ARRAY_SIZE(tests); i++) {
if (MPI_SUCCESS == (mpi_code_g = MPI_Barrier(comm_g))) {
@@ -2430,13 +2522,29 @@ main(int argc, char **argv)
}
if (MAINPROCESS)
- puts("");
-
+ printf("\n Re-running tests with compression enabled\n");
+#ifdef H5_HAVE_FILTER_DEFLATE
+ enable_compression = true;
+ for (size_t i = 0; i < ARRAY_SIZE(tests); i++) {
+ if (MPI_SUCCESS == (mpi_code_g = MPI_Barrier(comm_g))) {
+ (*tests[i])();
+ }
+ else {
+ if (MAINPROCESS)
+ MESG("MPI_Barrier failed");
+ nerrors++;
+ }
+ }
+ enable_compression = false;
+#else
+ if (MAINPROCESS)
+ SKIPPED();
+#endif
if (nerrors)
goto exit;
if (MAINPROCESS)
- puts("All Subfiling VFD tests passed\n");
+ puts("\nAll Subfiling VFD tests passed\n");
exit:
if (must_unset_stripe_size_env)
From cbd5a8d3ce28fa409ce27db167691278a4d60852 Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Wed, 25 Oct 2023 16:37:19 -0500
Subject: [PATCH 27/52] Cache installation of OpenMPI for AOCC action (#3774)
---
.github/workflows/linux-auto-aocc-ompi.yml | 25 ++++++++++++++--------
1 file changed, 16 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/linux-auto-aocc-ompi.yml b/.github/workflows/linux-auto-aocc-ompi.yml
index 76b6452226c..e4aa25a4e94 100644
--- a/.github/workflows/linux-auto-aocc-ompi.yml
+++ b/.github/workflows/linux-auto-aocc-ompi.yml
@@ -21,7 +21,7 @@ concurrency:
permissions:
contents: read
-
+
jobs:
build:
runs-on: ubuntu-latest
@@ -42,24 +42,31 @@ jobs:
which clang
which flang
clang -v
- - name: Install OpenMPI 4.1.5
+ - name: Cache OpenMPI 4.1.5 installation
+ id: cache-openmpi-4_1_5
+ uses: actions/cache@v3
+ with:
+ path: /home/runner/work/hdf5/hdf5/openmpi-4.1.5-install
+ key: ${{ runner.os }}-${{ runner.arch }}-openmpi-4_1_5-cache
+ - if: ${{ steps.cache-openmpi-4_1_5.outputs.cache-hit != 'true' }}
+ name: Install OpenMPI 4.1.5
run: |
export LD_LIBRARY_PATH=/home/runner/work/hdf5/hdf5/aocc-compiler-4.1.0/lib:/usr/local/lib
wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.5.tar.gz
tar zxvf openmpi-4.1.5.tar.gz
cd openmpi-4.1.5
- ./configure CC=/home/runner/work/hdf5/hdf5/aocc-compiler-4.1.0/bin/clang FC=/home/runner/work/hdf5/hdf5/aocc-compiler-4.1.0/bin/flang --prefix=/usr/local
+ ./configure CC=/home/runner/work/hdf5/hdf5/aocc-compiler-4.1.0/bin/clang FC=/home/runner/work/hdf5/hdf5/aocc-compiler-4.1.0/bin/flang --prefix=/home/runner/work/hdf5/hdf5/openmpi-4.1.5-install
make
- sudo make install
+ make install
- name: Install HDF5
env:
- NPROCS: 2
+ NPROCS: 2
run: |
- export LD_LIBRARY_PATH=/home/runner/work/hdf5/hdf5/aocc-compiler-4.1.0/lib:/usr/local/lib/openmpi:/usr/local/lib
- export LD_RUN_PATH=/home/runner/work/hdf5/hdf5/aocc-compiler-4.1.0/lib:/usr/local/lib/openmpi:/usr/local/lib
- export PATH=/usr/local/bin:$PATH
+ export LD_LIBRARY_PATH=/home/runner/work/hdf5/hdf5/aocc-compiler-4.1.0/lib:/home/runner/work/hdf5/hdf5/openmpi-4.1.5-install/lib:/usr/local/lib
+ export LD_RUN_PATH=/home/runner/work/hdf5/hdf5/aocc-compiler-4.1.0/lib:/home/runner/work/hdf5/hdf5/openmpi-4.1.5-install/lib:/usr/local/lib
+ export PATH=/home/runner/work/hdf5/hdf5/openmpi-4.1.5-install/bin:/usr/local/bin:$PATH
./autogen.sh
- ./configure --prefix=/tmp --enable-parallel --enable-shared CC=/usr/local/bin/mpicc LDFLAGS="-L/home/runner/work/hdf5/hdf5/aocc-compiler-4.1.0/lib -L/usr/local/lib/openmpi"
+ ./configure --prefix=/tmp --enable-parallel --enable-shared CC=mpicc LDFLAGS="-L/home/runner/work/hdf5/hdf5/aocc-compiler-4.1.0/lib -L/home/runner/work/hdf5/hdf5/openmpi-4.1.5-install/lib"
make -j
make check -j
make install
From ebeb642fcdc3c82dec38c046948559119cd4451b Mon Sep 17 00:00:00 2001
From: tbeu
Date: Thu, 26 Oct 2023 05:08:41 +0200
Subject: [PATCH 28/52] Fix typo in comment (#3775)
---
src/H5Tvlen.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/H5Tvlen.c b/src/H5Tvlen.c
index 68087a8e134..651ff8f7267 100644
--- a/src/H5Tvlen.c
+++ b/src/H5Tvlen.c
@@ -503,7 +503,7 @@ H5T__vlen_mem_seq_write(H5VL_object_t H5_ATTR_UNUSED *file, const H5T_vlen_alloc
if (seq_len) {
size_t len = seq_len * base_size; /* Sequence size */
- /* Use the user's memory allocation routine is one is defined */
+ /* Use the user's memory allocation routine if one is defined */
if (vl_alloc_info->alloc_func != NULL) {
if (NULL == (vl.p = (vl_alloc_info->alloc_func)(len, vl_alloc_info->alloc_info)))
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTALLOC, FAIL,
From 6ccace572e99561adf820af686da19f9a31efee3 Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Wed, 25 Oct 2023 21:16:32 -0700
Subject: [PATCH 29/52] Fixed a file handle leak in the core VFD (#3779)
When opening a file with the core VFD and a file image, if the file
already exists, the file check would leak the POSIX file handle.
Fixes GitHub issue #635
---
release_docs/RELEASE.txt | 7 +++++++
src/H5FDcore.c | 4 +++-
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 291f72a8701..7266de3eea9 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -404,6 +404,13 @@ Bug Fixes since HDF5-1.14.0 release
===================================
Library
-------
+ - Fixed a file handle leak in the core VFD
+
+ When opening a file with the core VFD and a file image, if the file
+ already exists, the file check would leak the POSIX file handle.
+
+ Fixes GitHub issue #635
+
- Fixed some issues with chunk index metadata not getting read
collectively when collective metadata reads are enabled
diff --git a/src/H5FDcore.c b/src/H5FDcore.c
index 08b714dfa34..1aa8d4bf19c 100644
--- a/src/H5FDcore.c
+++ b/src/H5FDcore.c
@@ -754,8 +754,10 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
((file_image_info.buffer == NULL) && (file_image_info.size == 0)));
memset(&sb, 0, sizeof(sb));
if ((file_image_info.buffer != NULL) && !(H5F_ACC_CREAT & flags)) {
- if (HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW) >= 0)
+ if ((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW)) >= 0) {
+ HDclose(fd);
HGOTO_ERROR(H5E_FILE, H5E_FILEEXISTS, NULL, "file already exists");
+ }
/* If backing store is requested, create and stat the file
* Note: We are forcing the O_CREAT flag here, even though this is
From 74cc1cf59eae121c1d15caf4fb0420a18a45da13 Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Wed, 25 Oct 2023 21:31:21 -0700
Subject: [PATCH 30/52] Fix a format string warning in the C++ examples (#3776)
---
hl/c++/examples/ptExampleFL.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hl/c++/examples/ptExampleFL.cpp b/hl/c++/examples/ptExampleFL.cpp
index 0e280f0355b..cb407e21f65 100644
--- a/hl/c++/examples/ptExampleFL.cpp
+++ b/hl/c++/examples/ptExampleFL.cpp
@@ -72,7 +72,7 @@ main(void)
if (err < 0)
fprintf(stderr, "Error getting packet count.");
- printf("Number of packets in packet table after five appends: %llu\n", count);
+ printf("Number of packets in packet table after five appends: %" PRIuHSIZE "\n", count);
/* Initialize packet table's "current record" */
ptable.ResetIndex();
From 119e694f971470b26876d227ceedbaab36eef275 Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Thu, 26 Oct 2023 10:49:50 -0500
Subject: [PATCH 31/52] Cancel running GitHub workflows on push to same PR
(#3772)
* Cancel running GitHub workflows on push to same PR
* Remove github.sha from workflow concurrency groups
---
.github/workflows/cve.yml | 2 +-
.github/workflows/hdfeos5.yml | 2 +-
.github/workflows/linux-auto-aocc-ompi.yml | 2 +-
.github/workflows/main.yml | 2 +-
.github/workflows/netcdf.yml | 2 +-
5 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/cve.yml b/.github/workflows/cve.yml
index 6756840981d..372518ade12 100644
--- a/.github/workflows/cve.yml
+++ b/.github/workflows/cve.yml
@@ -16,7 +16,7 @@ on:
# Using concurrency to cancel any in-progress job or run
concurrency:
- group: ${{ github.workflow }}-${{ github.sha || github.event.pull_request.number }}
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
permissions:
diff --git a/.github/workflows/hdfeos5.yml b/.github/workflows/hdfeos5.yml
index 5faf74a9ac4..dad262d426f 100644
--- a/.github/workflows/hdfeos5.yml
+++ b/.github/workflows/hdfeos5.yml
@@ -16,7 +16,7 @@ on:
# Using concurrency to cancel any in-progress job or run
concurrency:
- group: ${{ github.workflow }}-${{ github.sha || github.event.pull_request.number }}
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
permissions:
diff --git a/.github/workflows/linux-auto-aocc-ompi.yml b/.github/workflows/linux-auto-aocc-ompi.yml
index e4aa25a4e94..435d93f95c7 100644
--- a/.github/workflows/linux-auto-aocc-ompi.yml
+++ b/.github/workflows/linux-auto-aocc-ompi.yml
@@ -16,7 +16,7 @@ on:
# Using concurrency to cancel any in-progress job or run
concurrency:
- group: ${{ github.workflow }}-${{ github.sha || github.event.pull_request.number }}
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
permissions:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index e472b8cff56..43513c51a26 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -17,7 +17,7 @@ on:
# Using concurrency to cancel any in-progress job or run
concurrency:
- group: ${{ github.workflow }}-${{ github.sha || github.event.pull_request.number }}
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
permissions:
diff --git a/.github/workflows/netcdf.yml b/.github/workflows/netcdf.yml
index f38608012c0..f08361f6ea0 100644
--- a/.github/workflows/netcdf.yml
+++ b/.github/workflows/netcdf.yml
@@ -19,7 +19,7 @@ permissions:
# Using concurrency to cancel any in-progress job or run
concurrency:
- group: ${{ github.workflow }}-${{ github.sha || github.event.pull_request.number }}
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
jobs:
From 1bcef50b864fb1ea6e00900b3982e71d183cfda5 Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Fri, 27 Oct 2023 23:18:24 -0500
Subject: [PATCH 32/52] Print some messages in parallel tests on MPI rank 0
only (#3785)
Avoids overly verbose output from all processes emitting progress, etc. info.
---
test/h5test.c | 28 +++++++++------
test/testframe.c | 84 ++++++++++++++++++++++++-------------------
testpar/t_2Gio.c | 6 ++--
testpar/t_mpi.c | 46 +++++++++++-------------
testpar/t_shapesame.c | 17 +++++----
5 files changed, 99 insertions(+), 82 deletions(-)
diff --git a/test/h5test.c b/test/h5test.c
index 5348e1b4c8d..304637580f6 100644
--- a/test/h5test.c
+++ b/test/h5test.c
@@ -864,22 +864,23 @@ h5_show_hostname(void)
WSADATA wsaData;
int err;
#endif
+#ifdef H5_HAVE_PARALLEL
+ int mpi_rank, mpi_initialized, mpi_finalized;
+#endif
/* try show the process or thread id in multiple processes cases*/
#ifdef H5_HAVE_PARALLEL
- {
- int mpi_rank, mpi_initialized, mpi_finalized;
-
- MPI_Initialized(&mpi_initialized);
- MPI_Finalized(&mpi_finalized);
+ MPI_Initialized(&mpi_initialized);
+ MPI_Finalized(&mpi_finalized);
- if (mpi_initialized && !mpi_finalized) {
- MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
- printf("MPI-process %d.", mpi_rank);
- }
- else
- printf("thread 0.");
+ if (mpi_initialized && !mpi_finalized) {
+ /* Prevent output here from getting mixed with later output */
+ MPI_Barrier(MPI_COMM_WORLD);
+ MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+ printf("MPI-process %d.", mpi_rank);
}
+ else
+ printf("thread 0.");
#else
printf("thread %" PRIu64 ".", H5TS_thread_id());
#endif
@@ -915,6 +916,11 @@ h5_show_hostname(void)
#ifdef H5_HAVE_WIN32_API
WSACleanup();
#endif
+#ifdef H5_HAVE_PARALLEL
+ /* Prevent output here from getting mixed with later output */
+ if (mpi_initialized && !mpi_finalized)
+ MPI_Barrier(MPI_COMM_WORLD);
+#endif
}
#ifdef H5_HAVE_PARALLEL
diff --git a/test/testframe.c b/test/testframe.c
index 2b650270f3a..5cb25ed8148 100644
--- a/test/testframe.c
+++ b/test/testframe.c
@@ -155,35 +155,37 @@ TestUsage(void)
{
unsigned i;
- print_func("Usage: %s [-v[erbose] (l[ow]|m[edium]|h[igh]|0-9)] %s\n", TestProgName,
- (TestPrivateUsage ? "" : ""));
- print_func(" [-[e]x[clude] name]+ \n");
- print_func(" [-o[nly] name]+ \n");
- print_func(" [-b[egin] name] \n");
- print_func(" [-s[ummary]] \n");
- print_func(" [-c[leanoff]] \n");
- print_func(" [-h[elp]] \n");
- print_func("\n\n");
- print_func("verbose controls the amount of information displayed\n");
- print_func("exclude to exclude tests by name\n");
- print_func("only to name tests which should be run\n");
- print_func("begin start at the name of the test given\n");
- print_func("summary prints a summary of test results at the end\n");
- print_func("cleanoff does not delete *.hdf files after execution of tests\n");
- print_func("help print out this information\n");
- if (TestPrivateUsage) {
- print_func("\nExtra options\n");
- TestPrivateUsage();
- }
- print_func("\n\n");
- print_func("This program currently tests the following: \n\n");
- print_func("%16s %s\n", "Name", "Description");
- print_func("%16s %s\n", "----", "-----------");
+ if (mpi_rank_framework_g == 0) {
+ print_func("Usage: %s [-v[erbose] (l[ow]|m[edium]|h[igh]|0-9)] %s\n", TestProgName,
+ (TestPrivateUsage ? "" : ""));
+ print_func(" [-[e]x[clude] name]+ \n");
+ print_func(" [-o[nly] name]+ \n");
+ print_func(" [-b[egin] name] \n");
+ print_func(" [-s[ummary]] \n");
+ print_func(" [-c[leanoff]] \n");
+ print_func(" [-h[elp]] \n");
+ print_func("\n\n");
+ print_func("verbose controls the amount of information displayed\n");
+ print_func("exclude to exclude tests by name\n");
+ print_func("only to name tests which should be run\n");
+ print_func("begin start at the name of the test given\n");
+ print_func("summary prints a summary of test results at the end\n");
+ print_func("cleanoff does not delete *.hdf files after execution of tests\n");
+ print_func("help print out this information\n");
+ if (TestPrivateUsage) {
+ print_func("\nExtra options\n");
+ TestPrivateUsage();
+ }
+ print_func("\n\n");
+ print_func("This program currently tests the following: \n\n");
+ print_func("%16s %s\n", "Name", "Description");
+ print_func("%16s %s\n", "----", "-----------");
- for (i = 0; i < Index; i++)
- print_func("%16s %s\n", Test[i].Name, Test[i].Description);
+ for (i = 0; i < Index; i++)
+ print_func("%16s %s\n", Test[i].Name, Test[i].Description);
- print_func("\n\n");
+ print_func("\n\n");
+ }
}
/*
@@ -192,12 +194,14 @@ TestUsage(void)
void
TestInfo(const char *ProgName)
{
- unsigned major, minor, release;
+ if (mpi_rank_framework_g == 0) {
+ unsigned major, minor, release;
- H5get_libversion(&major, &minor, &release);
+ H5get_libversion(&major, &minor, &release);
- print_func("\nFor help use: %s -help\n", ProgName);
- print_func("Linked with hdf5 version %u.%u release %u\n", major, minor, release);
+ print_func("\nFor help use: %s -help\n", ProgName);
+ print_func("Linked with hdf5 version %u.%u release %u\n", major, minor, release);
+ }
}
/*
@@ -301,20 +305,24 @@ PerformTests(void)
for (Loop = 0; Loop < Index; Loop++)
if (Test[Loop].SkipFlag) {
- MESSAGE(2, ("Skipping -- %s (%s) \n", Test[Loop].Description, Test[Loop].Name));
+ if (mpi_rank_framework_g == 0)
+ MESSAGE(2, ("Skipping -- %s (%s) \n", Test[Loop].Description, Test[Loop].Name));
}
else {
if (mpi_rank_framework_g == 0)
MESSAGE(2, ("Testing -- %s (%s) \n", Test[Loop].Description, Test[Loop].Name));
- MESSAGE(5, ("===============================================\n"));
+ if (mpi_rank_framework_g == 0)
+ MESSAGE(5, ("===============================================\n"));
Test[Loop].NumErrors = num_errs;
Test_parameters = Test[Loop].Parameters;
TestAlarmOn();
Test[Loop].Call();
TestAlarmOff();
Test[Loop].NumErrors = num_errs - Test[Loop].NumErrors;
- MESSAGE(5, ("===============================================\n"));
- MESSAGE(5, ("There were %d errors detected.\n\n", (int)Test[Loop].NumErrors));
+ if (mpi_rank_framework_g == 0) {
+ MESSAGE(5, ("===============================================\n"));
+ MESSAGE(5, ("There were %d errors detected.\n\n", (int)Test[Loop].NumErrors));
+ }
}
Test_parameters = NULL; /* clear it. */
@@ -358,7 +366,8 @@ TestCleanup(void)
{
unsigned Loop;
- MESSAGE(2, ("\nCleaning Up temp files...\n\n"));
+ if (mpi_rank_framework_g == 0)
+ MESSAGE(2, ("\nCleaning Up temp files...\n\n"));
/* call individual cleanup routines in each source module */
for (Loop = 0; Loop < Index; Loop++)
@@ -619,7 +628,8 @@ SetTest(const char *testname, int action)
break;
default:
/* error */
- printf("*** ERROR: Unknown action (%d) for SetTest\n", action);
+ if (mpi_rank_framework_g == 0)
+ printf("*** ERROR: Unknown action (%d) for SetTest\n", action);
break;
}
}
diff --git a/testpar/t_2Gio.c b/testpar/t_2Gio.c
index c2aac771b29..48abf8ed9dd 100644
--- a/testpar/t_2Gio.c
+++ b/testpar/t_2Gio.c
@@ -4291,9 +4291,10 @@ main(int argc, char **argv)
printf("2 GByte IO TESTS START\n");
printf("2 MPI ranks will run the tests...\n");
printf("===================================\n");
- h5_show_hostname();
}
+ h5_show_hostname();
+
if (H5dont_atexit() < 0) {
printf("Failed to turn off atexit processing. Continue.\n");
};
@@ -4345,8 +4346,7 @@ main(int argc, char **argv)
#endif /* H5_HAVE_FILTER_DEFLATE */
/* Display testing information */
- if (MAINPROCESS)
- TestInfo(argv[0]);
+ TestInfo(argv[0]);
/* setup file access property list */
fapl = H5Pcreate(H5P_FILE_ACCESS);
diff --git a/testpar/t_mpi.c b/testpar/t_mpi.c
index eff39d057e0..0f1e27b506e 100644
--- a/testpar/t_mpi.c
+++ b/testpar/t_mpi.c
@@ -53,14 +53,14 @@ test_mpio_overlap_writes(char *filename)
MPI_Offset mpi_off;
MPI_Status mpi_stat;
- if (VERBOSE_MED)
- printf("MPIO independent overlapping writes test on file %s\n", filename);
-
nerrs = 0;
/* set up MPI parameters */
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+ if (VERBOSE_MED && MAINPROCESS)
+ printf("MPIO independent overlapping writes test on file %s\n", filename);
+
/* Need at least 2 processes */
if (mpi_size < 2) {
if (MAINPROCESS)
@@ -211,7 +211,7 @@ test_mpio_gb_file(char *filename)
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
- if (VERBOSE_MED)
+ if (VERBOSE_MED && MAINPROCESS)
printf("MPI_Offset range test\n");
/* figure out the signness and sizeof MPI_Offset */
@@ -274,12 +274,13 @@ test_mpio_gb_file(char *filename)
/*
* Verify if we can write to a file of multiple GB sizes.
*/
- if (VERBOSE_MED)
+ if (VERBOSE_MED && MAINPROCESS)
printf("MPIO GB file test %s\n", filename);
if (sizeof_mpi_offset <= 4) {
- printf("Skipped GB file range test "
- "because MPI_Offset cannot support it\n");
+ if (MAINPROCESS)
+ printf("Skipped GB file range test "
+ "because MPI_Offset cannot support it\n");
}
else {
buf = (char *)malloc(MB);
@@ -294,7 +295,8 @@ test_mpio_gb_file(char *filename)
mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh);
VRFY((mrc == MPI_SUCCESS), "MPI_FILE_OPEN");
- printf("MPIO GB file write test %s\n", filename);
+ if (MAINPROCESS)
+ printf("MPIO GB file write test %s\n", filename);
/* instead of writing every bytes of the file, we will just write
* some data around the 2 and 4 GB boundaries. That should cover
@@ -333,7 +335,8 @@ test_mpio_gb_file(char *filename)
*/
/* open it again to verify the data written */
/* but only if there was no write errors */
- printf("MPIO GB file read test %s\n", filename);
+ if (MAINPROCESS)
+ printf("MPIO GB file read test %s\n", filename);
if (errors_sum(writerrs) > 0) {
printf("proc %d: Skip read test due to previous write errors\n", mpi_rank);
goto finish;
@@ -377,7 +380,8 @@ test_mpio_gb_file(char *filename)
mrc = MPI_Barrier(MPI_COMM_WORLD);
VRFY((mrc == MPI_SUCCESS), "Sync before leaving test");
- printf("Test if MPI_File_get_size works correctly with %s\n", filename);
+ if (MAINPROCESS)
+ printf("Test if MPI_File_get_size works correctly with %s\n", filename);
mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, info, &fh);
VRFY((mrc == MPI_SUCCESS), "");
@@ -432,7 +436,6 @@ test_mpio_gb_file(char *filename)
static int
test_mpio_1wMr(char *filename, int special_request)
{
- char hostname[128];
int mpi_size, mpi_rank;
MPI_File fh;
char mpi_err_str[MPI_MAX_ERROR_STRING];
@@ -456,19 +459,8 @@ test_mpio_1wMr(char *filename, int special_request)
}
/* show the hostname so that we can tell where the processes are running */
- if (VERBOSE_DEF) {
-#ifdef H5_HAVE_GETHOSTNAME
- if (gethostname(hostname, sizeof(hostname)) < 0) {
- printf("gethostname failed\n");
- hostname[0] = '\0';
- }
-#else
- printf("gethostname unavailable\n");
- hostname[0] = '\0';
-#endif
- PRINTID;
- printf("hostname=%s\n", hostname);
- }
+ if (VERBOSE_DEF)
+ h5_show_hostname();
/* Delete any old file in order to start anew. */
/* Must delete because MPI_File_open does not have a Truncate mode. */
@@ -1005,6 +997,10 @@ test_mpio_special_collective(char *filename)
static int
parse_options(int argc, char **argv)
{
+ int mpi_rank;
+
+ MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+
while (--argc) {
if (**(++argv) != '-') {
break;
@@ -1053,7 +1049,7 @@ parse_options(int argc, char **argv)
return (1);
}
H5Pclose(plist);
- if (VERBOSE_MED) {
+ if (VERBOSE_MED && MAINPROCESS) {
printf("Test filenames are:\n");
for (i = 0; i < n; i++)
printf(" %s\n", filenames[i]);
diff --git a/testpar/t_shapesame.c b/testpar/t_shapesame.c
index 98e307772a9..0a3d3d0a49e 100644
--- a/testpar/t_shapesame.c
+++ b/testpar/t_shapesame.c
@@ -4089,7 +4089,8 @@ parse_options(int argc, char **argv)
case 'h': /* print help message--return with nerrors set */
return (1);
default:
- printf("Illegal option(%s)\n", *argv);
+ if (MAINPROCESS)
+ printf("Illegal option(%s)\n", *argv);
nerrors++;
return (1);
}
@@ -4098,12 +4099,14 @@ parse_options(int argc, char **argv)
/* check validity of dimension and chunk sizes */
if (dim0 <= 0 || dim1 <= 0) {
- printf("Illegal dim sizes (%d, %d)\n", dim0, dim1);
+ if (MAINPROCESS)
+ printf("Illegal dim sizes (%d, %d)\n", dim0, dim1);
nerrors++;
return (1);
}
if (chunkdim0 <= 0 || chunkdim1 <= 0) {
- printf("Illegal chunkdim sizes (%d, %d)\n", chunkdim0, chunkdim1);
+ if (MAINPROCESS)
+ printf("Illegal chunkdim sizes (%d, %d)\n", chunkdim0, chunkdim1);
nerrors++;
return (1);
}
@@ -4128,9 +4131,11 @@ parse_options(int argc, char **argv)
nerrors++;
return (1);
}
- printf("Test filenames are:\n");
- for (i = 0; i < n; i++)
- printf(" %s\n", filenames[i]);
+ if (MAINPROCESS) {
+ printf("Test filenames are:\n");
+ for (i = 0; i < n; i++)
+ printf(" %s\n", filenames[i]);
+ }
}
return (0);
From edb5cffdb20ad959c80b665cb8c2589f4c71414d Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Fri, 27 Oct 2023 23:20:29 -0500
Subject: [PATCH 33/52] Avoid attempted use of NULL pointer in parallel
compression code (#3786)
The parallel compression test code tests for the case where all MPI ranks have no selection in a dataset when writing to it. Add an early exit to the code to avoid attempting to use a NULL pointer due to there being no work to do.
---
src/H5Dmpio.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c
index 16243fadf92..b40ab4b2012 100644
--- a/src/H5Dmpio.c
+++ b/src/H5Dmpio.c
@@ -3782,6 +3782,10 @@ H5D__mpio_redistribute_shared_chunks_int(H5D_filtered_collective_io_info_t *chun
counts_disps_array = H5MM_xfree(counts_disps_array);
}
+ /* No useful work to do - exit */
+ if (coll_chunk_list_num_entries == 0)
+ HGOTO_DONE(SUCCEED);
+
/*
* Phase 2 - Involved ranks now redistribute any shared chunks to new
* owners as necessary.
From 556add35b78673a8a402841a7ef7e03fdb53daba Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Mon, 30 Oct 2023 11:12:10 -0500
Subject: [PATCH 34/52] Don't install h5tools_test_utils test program on system
(#3793)
---
release_docs/RELEASE.txt | 10 ++++++++++
tools/libtest/Makefile.am | 2 +-
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 7266de3eea9..0f18f6fa276 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -815,6 +815,16 @@ Bug Fixes since HDF5-1.14.0 release
Configuration
-------------
+ - Fixed an issue where the h5tools_test_utils test program was being
+ installed on the system for Autotools builds of HDF5
+
+ The h5tools_test_utils test program was mistakenly added to bin_PROGRAMS
+ in its Makefile.am configuration file, causing the executable to be
+ installed on the system. The executable is now added to noinst_PROGRAMS
+ instead and will no longer be installed on the system for Autotools builds
+ of HDF5. The CMake configuration code already avoids installing the
+ executable on the system.
+
- Fixed a configuration issue that prevented building of the Subfiling VFD on macOS
Checks were added to the CMake and Autotools code to verify that CLOCK_MONOTONIC_COARSE,
diff --git a/tools/libtest/Makefile.am b/tools/libtest/Makefile.am
index 45b3f476df7..8a503d033b2 100644
--- a/tools/libtest/Makefile.am
+++ b/tools/libtest/Makefile.am
@@ -27,7 +27,7 @@ LDADD=$(LIBH5TOOLS) $(LIBH5TEST) $(LIBHDF5)
# main target
-bin_PROGRAMS=h5tools_test_utils
+noinst_PROGRAMS=h5tools_test_utils
# check_PROGRAMS=$(TEST_PROG)
From 87636d685594e804c0bfb59933e3a56b2dabcd28 Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Mon, 30 Oct 2023 09:13:01 -0700
Subject: [PATCH 35/52] Add Doxygen to H5FDsplitter.h (#3794)
* H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION
* H5FD_SPLITTER_PATH_MAX
* H5FD_SPLITTER_MAGIC
* H5FD_splitter_vfd_config_t
* H5Pset_fapl_splitter()
* H5Pget_fapl_splitter()
---
src/H5FDsplitter.h | 109 ++++++++++++++++++++++-----------------------
1 file changed, 54 insertions(+), 55 deletions(-)
diff --git a/src/H5FDsplitter.h b/src/H5FDsplitter.h
index c8751c82349..8e790e212b6 100644
--- a/src/H5FDsplitter.h
+++ b/src/H5FDsplitter.h
@@ -20,70 +20,37 @@
#define H5FD_SPLITTER (H5FDperform_init(H5FD_splitter_init))
#define H5FD_SPLITTER_VALUE H5_VFD_SPLITTER
-/* The version of the H5FD_splitter_vfd_config_t structure used */
+/** The version of the H5FD_splitter_vfd_config_t structure used */
#define H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION 1
-/* Maximum length of a filename/path string in the Write-Only channel,
+/**
+ * Maximum length of a filename/path string in the Write-Only channel,
* including the NULL-terminator.
*/
#define H5FD_SPLITTER_PATH_MAX 4096
-/* Semi-unique constant used to help identify structure pointers */
+/** Semi-unique constant used to help identify structure pointers */
#define H5FD_SPLITTER_MAGIC 0x2B916880
-/* ----------------------------------------------------------------------------
- * Structure: H5FD_spliiter_vfd_config_t
- *
- * One-stop shopping for configuring a Splitter VFD (rather than many
- * parameters passed into H5Pset/get functions).
- *
- * magic (int32_t)
- * Semi-unique number, used to sanity-check that a given pointer is
- * likely (or not) to be this structure type. MUST be first.
- * If magic is not H5FD_SPLITTER_MAGIC, the structure (and/or pointer to)
- * must be considered invalid.
- *
- * version (unsigned int)
- * Version number of this structure -- informs component membership.
- * If not H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION, the structure (and/or
- * pointer to) must be considered invalid.
- *
- * rw_fapl_id (hid_t)
- * Library-given identification number of the Read/Write channel driver
- * File Access Property List.
- * The driver must support read/write access.
- * Must be set to H5P_DEFAULT or a valid FAPL ID.
- *
- * wo_fapl_id (hid_t)
- * Library-given identification number of the Read/Write channel driver
- * File Access Property List.
- * The driver feature flags must include H5FD_FEAT_DEFAULT_VFD_COMPAITBLE.
- * Must be set to H5P_DEFAULT or a valid FAPL ID.
- *
- * wo_file_path (char[H5FD_SPLITTER_PATH_MAX + 1])
- * String buffer for the Write-Only channel target file.
- * Must be null-terminated, cannot be empty.
- *
- * log_file_path (char[H5FD_SPLITTER_PATH_MAX + 1])
- * String buffer for the Splitter VFD logging output.
- * Must be null-terminated.
- * If null, no logfile is created.
- *
- * ignore_wo_errors (hbool_t)
- * Toggle flag for how judiciously to respond to errors on the Write-Only
- * channel.
- *
- * ----------------------------------------------------------------------------
+//!
+/**
+ * Configuration options for setting up the Splitter VFD
*/
typedef struct H5FD_splitter_vfd_config_t {
- int32_t magic;
- unsigned int version;
- hid_t rw_fapl_id;
- hid_t wo_fapl_id;
- char wo_path[H5FD_SPLITTER_PATH_MAX + 1];
- char log_file_path[H5FD_SPLITTER_PATH_MAX + 1];
- hbool_t ignore_wo_errs;
+ int32_t magic; /**< Magic number to identify this struct. Must be \p H5FD_SPLITTER_MAGIC. */
+ unsigned int version; /**< Version number of this struct. Currently must be \p
+ H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION. */
+ hid_t rw_fapl_id; /**< File-access property list for setting up the read/write channel. Can be \p
+ H5P_DEFAULT. */
+ hid_t wo_fapl_id; /**< File-access property list for setting up the read-only channel. The selected VFD
+ must support the \p H5FD_FEAT_DEFAULT_VFD_COMPATIBLE flag. Can be \p H5P_DEFAULT. */
+ char wo_path[H5FD_SPLITTER_PATH_MAX + 1]; /**< Path to the write-only file */
+ char log_file_path[H5FD_SPLITTER_PATH_MAX + 1]; /**< Path to the log file, which will be created on HDF5
+ file open (existing files will be clobbered). Can be
+ NULL, in which case no logging output is generated. */
+ hbool_t ignore_wo_errs; /**< Whether to ignore errors on the write-only channel */
} H5FD_splitter_vfd_config_t;
+//!
#ifdef __cplusplus
extern "C" {
@@ -93,14 +60,46 @@ H5_DLL hid_t H5FD_splitter_init(void);
/**
* \ingroup FAPL
*
- * \todo Add missing documentation
+ * \brief Sets the file access property list to use the splitter driver
+ *
+ * \fapl_id
+ * \param[in] config_ptr Configuration options for the VFD
+ * \returns \herr_t
+ *
+ * \details H5Pset_fapl_splitter() sets the file access property list identifier,
+ * \p fapl_id, to use the splitter driver.
+ *
+ * The splitter VFD echoes file manipulation (e.g. create, truncate)
+ * and write calls to a second, write-only file.
+ *
+ * \note The splitter VFD should not be confused with the split VFD,
+ * which is a simplification of the multi VFD and creates separate
+ * files for metadata and data.
+ *
+ * \since 1.10.7, 1.12.1
*/
H5_DLL herr_t H5Pset_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_ptr);
/**
* \ingroup FAPL
*
- * \todo Add missing documentation
+ * \brief Gets splitter driver properties from the the file access property list
+ *
+ * \fapl_id
+ * \param[out] config_ptr Configuration options for the VFD
+ * \returns \herr_t
+ *
+ * \details H5Pset_fapl_splitter() sets the file access property list identifier,
+ * \p fapl_id, to use the splitter driver.
+ *
+ * The splitter VFD echoes file manipulation (e.g. create, truncate)
+ * and write calls to a second file.
+ *
+ * \note The splitter VFD should not be confused with the split VFD,
+ * which is a simplification of the multi VFD and creates separate
+ * files for metadata and data.
+ *
+ * \since 1.10.7, 1.12.1
*/
H5_DLL herr_t H5Pget_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_ptr);
From 58ef7dcdc9bfbfc65fe69d782914671f66322d85 Mon Sep 17 00:00:00 2001
From: Dana Robinson <43805+derobins@users.noreply.github.com>
Date: Mon, 30 Oct 2023 09:13:54 -0700
Subject: [PATCH 36/52] Update Doxygen initializers & identifiers in VFDs
(#3795)
* Add Doxygen for all H5FD_ initializers
* Add Doxygen for all H5FD__VALUE values
* Mark H5FD__init() calls private in Doxygen
---
src/H5FDcore.h | 12 ++++++++++--
src/H5FDdirect.h | 33 +++++++++++++++++++++++++--------
src/H5FDfamily.h | 11 +++++++++--
src/H5FDhdfs.h | 21 ++++++++++++++++++---
src/H5FDlog.h | 11 +++++++++--
src/H5FDmirror.h | 11 +++++++++--
src/H5FDmpio.h | 25 ++++++++++++++++---------
src/H5FDmulti.h | 8 +++++++-
src/H5FDonion.h | 17 ++++++++++-------
src/H5FDros3.h | 20 ++++++++++++--------
src/H5FDsec2.h | 13 ++++++++++---
src/H5FDsplitter.h | 12 ++++++++++--
src/H5FDstdio.h | 14 ++++++++++++--
src/H5FDwindows.h | 7 ++++++-
14 files changed, 163 insertions(+), 52 deletions(-)
diff --git a/src/H5FDcore.h b/src/H5FDcore.h
index 235d6fcaaf7..cd45c8d6061 100644
--- a/src/H5FDcore.h
+++ b/src/H5FDcore.h
@@ -11,17 +11,25 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose: The public header file for the core driver.
+ * Purpose: The public header file for the core virtual file driver (VFD)
*/
#ifndef H5FDcore_H
#define H5FDcore_H
-#define H5FD_CORE (H5FDperform_init(H5FD_core_init))
+/** Initializer for the core VFD */
+#define H5FD_CORE (H5FDperform_init(H5FD_core_init))
+
+/** Identifier for the core VFD */
#define H5FD_CORE_VALUE H5_VFD_CORE
#ifdef __cplusplus
extern "C" {
#endif
+
+/** @private
+ *
+ * \brief Private initializer for the core VFD
+ */
H5_DLL hid_t H5FD_core_init(void);
/**
diff --git a/src/H5FDdirect.h b/src/H5FDdirect.h
index e47ac37cdea..1e60bb08119 100644
--- a/src/H5FDdirect.h
+++ b/src/H5FDdirect.h
@@ -11,30 +11,47 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose: The public header file for the direct driver.
+ * Purpose: The public header file for the direct virtual file driver (VFD)
*/
#ifndef H5FDdirect_H
#define H5FDdirect_H
#ifdef H5_HAVE_DIRECT
-#define H5FD_DIRECT (H5FDperform_init(H5FD_direct_init))
+
+/** Initializer for the direct VFD */
+#define H5FD_DIRECT (H5FDperform_init(H5FD_direct_init))
+
+/** Identifier for the direct VFD */
#define H5FD_DIRECT_VALUE H5_VFD_DIRECT
+
#else
+
+/** Initializer for the direct VFD (disabled) */
#define H5FD_DIRECT (H5I_INVALID_HID)
+
+/** Identifier for the direct VFD (disabled) */
#define H5FD_DIRECT_VALUE H5_VFD_INVALID
+
#endif /* H5_HAVE_DIRECT */
+/** Default value for memory boundary */
+#define MBOUNDARY_DEF 4096
+
+/** Default value for file block size */
+#define FBSIZE_DEF 4096
+
+/** Default value for maximum copy buffer size */
+#define CBSIZE_DEF (16 * 1024 * 1024)
+
#ifdef H5_HAVE_DIRECT
#ifdef __cplusplus
extern "C" {
#endif
-/* Default values for memory boundary, file block size, and maximal copy buffer size.
- * Application can set these values through the function H5Pset_fapl_direct. */
-#define MBOUNDARY_DEF 4096
-#define FBSIZE_DEF 4096
-#define CBSIZE_DEF 16 * 1024 * 1024
-
+/** @private
+ *
+ * \brief Private initializer for the direct VFD
+ */
H5_DLL hid_t H5FD_direct_init(void);
/**
diff --git a/src/H5FDfamily.h b/src/H5FDfamily.h
index 76020f0a268..32e885c422d 100644
--- a/src/H5FDfamily.h
+++ b/src/H5FDfamily.h
@@ -11,18 +11,25 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose: The public header file for the family driver.
+ * Purpose: The public header file for the family virtual file driver (VFD)
*/
#ifndef H5FDfamily_H
#define H5FDfamily_H
-#define H5FD_FAMILY (H5FDperform_init(H5FD_family_init))
+/** Initializer for the family VFD */
+#define H5FD_FAMILY (H5FDperform_init(H5FD_family_init))
+
+/** Identifier for the family VFD */
#define H5FD_FAMILY_VALUE H5_VFD_FAMILY
#ifdef __cplusplus
extern "C" {
#endif
+/** @private
+ *
+ * \brief Private initializer for the family VFD
+ */
H5_DLL hid_t H5FD_family_init(void);
/**
diff --git a/src/H5FDhdfs.h b/src/H5FDhdfs.h
index c8c2c37f1b5..e5f7173fce2 100644
--- a/src/H5FDhdfs.h
+++ b/src/H5FDhdfs.h
@@ -11,18 +11,29 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose: The public header file for the hdfs driver.
+ * Purpose: The public header file for the Hadoop Distributed File System
+ * (hdfs) virtual file driver (VFD)
*/
#ifndef H5FDhdfs_H
#define H5FDhdfs_H
#ifdef H5_HAVE_LIBHDFS
-#define H5FD_HDFS (H5FDperform_init(H5FD_hdfs_init))
+
+/** Initializer for the hdfs VFD */
+#define H5FD_HDFS (H5FDperform_init(H5FD_hdfs_init))
+
+/** Identifier for the hdfs VFD */
#define H5FD_HDFS_VALUE H5_VFD_HDFS
-#else /* H5_HAVE_LIBHDFS */
+
+#else
+
+/** Initializer for the hdfs VFD (disabled) */
#define H5FD_HDFS (H5I_INVALID_HID)
+
+/** Identifier for the hdfs VFD (disabled) */
#define H5FD_HDFS_VALUE H5_VFD_INVALID
+
#endif /* H5_HAVE_LIBHDFS */
#ifdef H5_HAVE_LIBHDFS
@@ -104,6 +115,10 @@ typedef struct H5FD_hdfs_fapl_t {
int32_t stream_buffer_size;
} H5FD_hdfs_fapl_t;
+/** @private
+ *
+ * \brief Private initializer for the hdfs VFD
+ */
H5_DLL hid_t H5FD_hdfs_init(void);
/**
diff --git a/src/H5FDlog.h b/src/H5FDlog.h
index ae4e2d05e72..b4af2050a62 100644
--- a/src/H5FDlog.h
+++ b/src/H5FDlog.h
@@ -11,12 +11,15 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose: The public header file for the log driver.
+ * Purpose: The public header file for the log virtual file driver (VFD)
*/
#ifndef H5FDlog_H
#define H5FDlog_H
-#define H5FD_LOG (H5FDperform_init(H5FD_log_init))
+/** Initializer for the log VFD */
+#define H5FD_LOG (H5FDperform_init(H5FD_log_init))
+
+/** Identifier for the log VFD */
#define H5FD_LOG_VALUE H5_VFD_LOG
/* Flags for H5Pset_fapl_log() */
@@ -62,6 +65,10 @@
extern "C" {
#endif
+/** @private
+ *
+ * \brief Private initializer for the log VFD
+ */
H5_DLL hid_t H5FD_log_init(void);
/**
diff --git a/src/H5FDmirror.h b/src/H5FDmirror.h
index b196b2b1adc..6c98e1a8a6f 100644
--- a/src/H5FDmirror.h
+++ b/src/H5FDmirror.h
@@ -11,7 +11,7 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose: Public, shared definitions for Mirror VFD & remote Writer.
+ * Purpose: The public header file for the mirror virtual file driver (VFD)
*/
#ifndef H5FDmirror_H
@@ -19,7 +19,10 @@
#ifdef H5_HAVE_MIRROR_VFD
-#define H5FD_MIRROR (H5FDperform_init(H5FD_mirror_init))
+/** Initializer for the mirror VFD */
+#define H5FD_MIRROR (H5FDperform_init(H5FD_mirror_init))
+
+/** Identifier for the mirror VFD */
#define H5FD_MIRROR_VALUE H5_VFD_MIRROR
#ifdef __cplusplus
@@ -62,6 +65,10 @@ typedef struct H5FD_mirror_fapl_t {
char remote_ip[H5FD_MIRROR_MAX_IP_LEN + 1];
} H5FD_mirror_fapl_t;
+/** @private
+ *
+ * \brief Private initializer for the mirror VFD
+ */
H5_DLL hid_t H5FD_mirror_init(void);
/**
diff --git a/src/H5FDmpio.h b/src/H5FDmpio.h
index 60deec2c07b..5e7ecf30353 100644
--- a/src/H5FDmpio.h
+++ b/src/H5FDmpio.h
@@ -11,35 +11,42 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose: The public header file for the mpio driver.
+ * Purpose: The public header file for the MPI-I/O (mpio) virtual file driver (VFD)
*/
#ifndef H5FDmpio_H
#define H5FDmpio_H
-/* Macros */
-
#ifdef H5_HAVE_PARALLEL
+
+/** Initializer for the mpio VFD */
#define H5FD_MPIO (H5FDperform_init(H5FD_mpio_init))
+
#else
+
+/** Initializer for the mpio VFD (disabled) */
#define H5FD_MPIO (H5I_INVALID_HID)
-#endif /* H5_HAVE_PARALLEL */
+
+#endif
#ifdef H5_HAVE_PARALLEL
-/*Turn on H5FDmpio_debug if H5F_DEBUG is on */
-#ifdef H5F_DEBUG
-#ifndef H5FDmpio_DEBUG
+
+#if defined(H5F_DEBUG) && !defined(H5FDmpio_DEBUG)
+/** Turn mpio VFD debugging on (requires H5F_DEBUG) */
#define H5FDmpio_DEBUG
#endif
-#endif
/* Global var whose value comes from environment variable */
/* (Defined in H5FDmpio.c) */
H5_DLLVAR hbool_t H5FD_mpi_opt_types_g;
-/* Function prototypes */
#ifdef __cplusplus
extern "C" {
#endif
+
+/** @private
+ *
+ * \brief Private initializer for the mpio VFD
+ */
H5_DLL hid_t H5FD_mpio_init(void);
/**
diff --git a/src/H5FDmulti.h b/src/H5FDmulti.h
index a85f2dfed25..d89a3e27cce 100644
--- a/src/H5FDmulti.h
+++ b/src/H5FDmulti.h
@@ -11,16 +11,22 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose: The public header file for the "multi" driver.
+ * Purpose: The public header file for the multi virtual file driver (VFD)
*/
#ifndef H5FDmulti_H
#define H5FDmulti_H
+/** Initializer for the multi VFD */
#define H5FD_MULTI (H5FDperform_init(H5FD_multi_init))
#ifdef __cplusplus
extern "C" {
#endif
+
+/** @private
+ *
+ * \brief Private initializer for the multi VFD
+ */
H5_DLL hid_t H5FD_multi_init(void);
/**
diff --git a/src/H5FDonion.h b/src/H5FDonion.h
index 09b290e2a9e..4aaab6d3c3e 100644
--- a/src/H5FDonion.h
+++ b/src/H5FDonion.h
@@ -11,19 +11,18 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Onion Virtual File Driver (VFD)
- *
- * Purpose: The public header file for the Onion VFD.
+ * Purpose: The public header file for the onion virtual file driver (VFD)
*/
#ifndef H5FDonion_H
#define H5FDonion_H
-#define H5FD_ONION (H5FDperform_init(H5FD_onion_init))
+/** Initializer for the onion VFD */
+#define H5FD_ONION (H5FDperform_init(H5FD_onion_init))
+
+/** Identifier for the onion VFD */
#define H5FD_ONION_VALUE H5_VFD_ONION
-/**
- * Current version of the onion VFD fapl info struct.
- */
+/** Current version of the onion VFD fapl info struct */
#define H5FD_ONION_FAPL_INFO_VERSION_CURR 1
#define H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT \
@@ -114,6 +113,10 @@ typedef struct H5FD_onion_fapl_info_t {
extern "C" {
#endif
+/** @private
+ *
+ * \brief Private initializer for the onion VFD
+ */
H5_DLL hid_t H5FD_onion_init(void);
/**
diff --git a/src/H5FDros3.h b/src/H5FDros3.h
index ecd26789cc8..217af2d01b6 100644
--- a/src/H5FDros3.h
+++ b/src/H5FDros3.h
@@ -11,20 +11,24 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Read-Only S3 Virtual File Driver (VFD)
- *
- * Purpose: The public header file for the ros3 driver.
+ * Purpose: The public header file for the read-only S3 (ros3) virtual file driver (VFD)
*/
#ifndef H5FDros3_H
#define H5FDros3_H
#ifdef H5_HAVE_ROS3_VFD
-#define H5FD_ROS3 (H5FDperform_init(H5FD_ros3_init))
+/** Initializer for the ros3 VFD */
+#define H5FD_ROS3 (H5FDperform_init(H5FD_ros3_init))
+
+/** Identifier for the ros3 VFD */
#define H5FD_ROS3_VALUE H5_VFD_ROS3
#else
+/** Initializer for the ros3 VFD (disabled) */
#define H5FD_ROS3 (H5I_INVALID_HID)
+
+/** Identifier for the ros3 VFD (disabled) */
#define H5FD_ROS3_VALUE H5_VFD_INVALID
-#endif /* H5_HAVE_ROS3_VFD */
+#endif
#ifdef H5_HAVE_ROS3_VFD
@@ -139,9 +143,9 @@ typedef struct H5FD_ros3_fapl_t {
extern "C" {
#endif
-/**
- * \brief Internal routine to initialize #H5FD_ROS3 driver. Not meant to be
- * called directly by an HDF5 application.
+/** @private
+ *
+ * \brief Private initializer for the ros3 VFD
*/
H5_DLL hid_t H5FD_ros3_init(void);
diff --git a/src/H5FDsec2.h b/src/H5FDsec2.h
index a2590aee968..dd0a4d8918d 100644
--- a/src/H5FDsec2.h
+++ b/src/H5FDsec2.h
@@ -11,18 +11,26 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose: The public header file for the sec2 driver
+ * Purpose: The public header file for the POSOX I/O (sec2 - "POSIX section 2")
+ * virtual file driver (VFD)
*/
#ifndef H5FDsec2_H
#define H5FDsec2_H
-#define H5FD_SEC2 (H5FDperform_init(H5FD_sec2_init))
+/** Initializer for the sec2 VFD */
+#define H5FD_SEC2 (H5FDperform_init(H5FD_sec2_init))
+
+/** Identifier for the sec2 VFD */
#define H5FD_SEC2_VALUE H5_VFD_SEC2
#ifdef __cplusplus
extern "C" {
#endif
+/** @private
+ *
+ * \brief Private initializer for the sec2 VFD
+ */
H5_DLL hid_t H5FD_sec2_init(void);
/**
@@ -38,7 +46,6 @@ H5_DLL hid_t H5FD_sec2_init(void);
* #H5FD_SEC2 driver.
*
* \since 1.4.0
- *
*/
H5_DLL herr_t H5Pset_fapl_sec2(hid_t fapl_id);
diff --git a/src/H5FDsplitter.h b/src/H5FDsplitter.h
index 8e790e212b6..99a471e5ce3 100644
--- a/src/H5FDsplitter.h
+++ b/src/H5FDsplitter.h
@@ -11,13 +11,16 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose: The public header file for the "splitter" driver.
+ * Purpose: The public header file for the splitter virtual file driver (VFD)
*/
#ifndef H5FDsplitter_H
#define H5FDsplitter_H
-#define H5FD_SPLITTER (H5FDperform_init(H5FD_splitter_init))
+/** Initializer for the splitter VFD */
+#define H5FD_SPLITTER (H5FDperform_init(H5FD_splitter_init))
+
+/** Identifier for the splitter VFD */
#define H5FD_SPLITTER_VALUE H5_VFD_SPLITTER
/** The version of the H5FD_splitter_vfd_config_t structure used */
@@ -55,6 +58,11 @@ typedef struct H5FD_splitter_vfd_config_t {
#ifdef __cplusplus
extern "C" {
#endif
+
+/** @private
+ *
+ * \brief Private initializer for the splitter VFD
+ */
H5_DLL hid_t H5FD_splitter_init(void);
/**
diff --git a/src/H5FDstdio.h b/src/H5FDstdio.h
index e2e05a77d64..794fe31bf61 100644
--- a/src/H5FDstdio.h
+++ b/src/H5FDstdio.h
@@ -11,20 +11,26 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose: The public header file for the C stdio driver
+ * Purpose: The public header file for the C stdio virtual file driver (VFD)
*/
#ifndef H5FDstdio_H
#define H5FDstdio_H
#include "H5Ipublic.h"
+/** Initializer for the stdio VFD */
#define H5FD_STDIO (H5FDperform_init(H5FD_stdio_init))
#ifdef __cplusplus
extern "C" {
#endif
+/** @private
+ *
+ * \brief Private initializer for the stdio VFD
+ */
H5_DLL hid_t H5FD_stdio_init(void);
+
/**
* \ingroup FAPL
*
@@ -34,7 +40,11 @@ H5_DLL hid_t H5FD_stdio_init(void);
* \returns \herr_t
*
* \details H5Pset_fapl_stdio() modifies the file access property list to use
- * the standard I/O driver, H5FDstdio().
+ * the stdio VFD, which uses I/O calls from stdio.h.
+ *
+ * \note This VFD was designed to be a "demo" VFD that shows how to write
+ * your own VFD. Most applications should not use this VFD and should instead
+ * use the POSIX I/O VFD (sec2).
*
* \since 1.4.0
*
diff --git a/src/H5FDwindows.h b/src/H5FDwindows.h
index 14f698580f8..673d1c93b59 100644
--- a/src/H5FDwindows.h
+++ b/src/H5FDwindows.h
@@ -11,11 +11,16 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Purpose: The public header file for the Windows driver
+ * Purpose: The public header file for the Windows virtual file driver (VFD)
+ *
+ * This VFD uses no Win32 API calls directly (though it may be
+ * rewritten to do so in the future). It is currently defined to
+ * be the sec2 VFD.
*/
#ifndef H5FDwindows_H
#define H5FDwindows_H
+/** Initializer for the Windows VFD */
#define H5FD_WINDOWS (H5FD_sec2_init())
#ifdef __cplusplus
From 44ec53ec485029fc964d67c811c5d3e227f99372 Mon Sep 17 00:00:00 2001
From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com>
Date: Mon, 30 Oct 2023 12:11:07 -0500
Subject: [PATCH 37/52] Remove CDASH_LOCAL variable reference (#3796)
---
CMakeInstallation.cmake | 6 +++---
CTestConfig.cmake | 12 ++----------
config/cmake/LIBAEC/CMakeLists.txt | 6 +++---
config/cmake/ZLIB/CMakeLists.txt | 6 +++---
config/cmake/examples/HDF5_Examples.cmake.in | 8 --------
5 files changed, 11 insertions(+), 27 deletions(-)
diff --git a/CMakeInstallation.cmake b/CMakeInstallation.cmake
index bb244764de2..74ab8b15fd2 100644
--- a/CMakeInstallation.cmake
+++ b/CMakeInstallation.cmake
@@ -295,10 +295,10 @@ endif ()
if (NOT HDF5_EXTERNALLY_CONFIGURED AND NOT HDF5_NO_PACKAGES)
set (CPACK_PACKAGE_VENDOR "HDF_Group")
set (CPACK_PACKAGE_NAME "${HDF5_PACKAGE_NAME}")
- if (CDASH_LOCAL)
- set (CPACK_PACKAGE_VERSION "${HDF5_PACKAGE_VERSION}")
- else ()
+ if (NOT WIN32 OR HDF5_VERS_SUBRELEASE MATCHES "^[0-9]+$")
set (CPACK_PACKAGE_VERSION "${HDF5_PACKAGE_VERSION_STRING}")
+ else ()
+ set (CPACK_PACKAGE_VERSION "${HDF5_PACKAGE_VERSION}")
endif ()
set (CPACK_PACKAGE_VERSION_MAJOR "${HDF5_PACKAGE_VERSION_MAJOR}")
set (CPACK_PACKAGE_VERSION_MINOR "${HDF5_PACKAGE_VERSION_MINOR}")
diff --git a/CTestConfig.cmake b/CTestConfig.cmake
index b780b86edae..62beafc0c12 100644
--- a/CTestConfig.cmake
+++ b/CTestConfig.cmake
@@ -22,20 +22,12 @@ set (CTEST_DROP_METHOD "https")
if (CTEST_DROP_SITE_INIT)
set (CTEST_DROP_SITE "${CTEST_DROP_SITE_INIT}")
else ()
- if (CDASH_LOCAL)
- set (CTEST_DROP_SITE "cdash-internal.hdfgroup.org")
- else ()
- set (CTEST_DROP_SITE "cdash.hdfgroup.org")
- endif ()
+ set (CTEST_DROP_SITE "cdash.hdfgroup.org")
endif ()
if (CTEST_DROP_LOCATION_INIT)
set (CTEST_DROP_LOCATION "${CTEST_DROP_LOCATION_INIT}")
else ()
- if (CDASH_LOCAL)
- set (CTEST_DROP_LOCATION "/submit.php?project=HDF5Trunk")
- else ()
- set (CTEST_DROP_LOCATION "/submit.php?project=HDF5")
- endif ()
+ set (CTEST_DROP_LOCATION "/submit.php?project=HDF5")
endif ()
set (CTEST_DROP_SITE_CDASH TRUE)
diff --git a/config/cmake/LIBAEC/CMakeLists.txt b/config/cmake/LIBAEC/CMakeLists.txt
index 53950d8942a..9ffb68d28b7 100644
--- a/config/cmake/LIBAEC/CMakeLists.txt
+++ b/config/cmake/LIBAEC/CMakeLists.txt
@@ -369,10 +369,10 @@ configure_file (${LIBAEC_SOURCE_DIR}/README.md ${LIBAEC_BINARY_DIR}/LIBAEC_READM
if (NOT LIBAEC_EXTERNALLY_CONFIGURED)
set (CPACK_PACKAGE_VENDOR "HDF_Group")
set (CPACK_PACKAGE_NAME "${LIBAEC_PACKAGE_NAME}")
- if (CDASH_LOCAL)
- set (CPACK_PACKAGE_VERSION "${LIBAEC_PACKAGE_VERSION}")
- else ()
+ if (NOT WIN32 OR LIBAEC_VERS_SUBRELEASE MATCHES "^[0-9]+$")
set (CPACK_PACKAGE_VERSION "${LIBAEC_PACKAGE_VERSION_STRING}")
+ else ()
+ set (CPACK_PACKAGE_VERSION "${LIBAEC_PACKAGE_VERSION}")
endif ()
set (CPACK_PACKAGE_VERSION_MAJOR "${LIBAEC_PACKAGE_VERSION_MAJOR}")
set (CPACK_PACKAGE_VERSION_MINOR "${LIBAEC_PACKAGE_VERSION_MINOR}")
diff --git a/config/cmake/ZLIB/CMakeLists.txt b/config/cmake/ZLIB/CMakeLists.txt
index 12411ac9e07..5c06a544ce2 100644
--- a/config/cmake/ZLIB/CMakeLists.txt
+++ b/config/cmake/ZLIB/CMakeLists.txt
@@ -428,10 +428,10 @@ configure_file (${ZLIB_SOURCE_DIR}/README ${ZLIB_BINARY_DIR}/ZLIB_README @ONLY)
if (NOT ZLIB_EXTERNALLY_CONFIGURED)
set (CPACK_PACKAGE_VENDOR "HDF_Group")
set (CPACK_PACKAGE_NAME "${ZLIB_PACKAGE_NAME}")
- if (CDASH_LOCAL)
- set (CPACK_PACKAGE_VERSION "${ZLIB_PACKAGE_VERSION}")
- else ()
+ if (NOT WIN32 OR ZLIB_VERS_SUBRELEASE MATCHES "^[0-9]+$")
set (CPACK_PACKAGE_VERSION "${ZLIB_PACKAGE_VERSION_STRING}")
+ else ()
+ set (CPACK_PACKAGE_VERSION "${ZLIB_PACKAGE_VERSION}")
endif ()
set (CPACK_PACKAGE_VERSION_MAJOR "${ZLIB_PACKAGE_VERSION_MAJOR}")
set (CPACK_PACKAGE_VERSION_MINOR "${ZLIB_PACKAGE_VERSION_MINOR}")
diff --git a/config/cmake/examples/HDF5_Examples.cmake.in b/config/cmake/examples/HDF5_Examples.cmake.in
index d77c16d49ec..f45e1ed6dbe 100644
--- a/config/cmake/examples/HDF5_Examples.cmake.in
+++ b/config/cmake/examples/HDF5_Examples.cmake.in
@@ -62,11 +62,6 @@ if(NOT DEFINED CTEST_SOURCE_NAME)
set(CTEST_SOURCE_NAME "HDF5Examples")
endif()
-if(NOT DEFINED HDF_LOCAL)
- set(CDASH_LOCAL "NO")
-else()
- set(CDASH_LOCAL "YES")
-endif()
if(NOT DEFINED CTEST_SITE)
set(CTEST_SITE "local")
endif()
@@ -100,9 +95,6 @@ else()
endif()
### default HDF5_PLUGIN_PATH to where the filter libraries are located
set(ENV{HDF5_PLUGIN_PATH} "${INSTALLDIR}/lib/plugin")
-if(${CDASH_LOCAL})
- set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCDASH_LOCAL:BOOL=ON")
-endif()
set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACKAGE_NAME:STRING=@HDF5_PACKAGE@@HDF_PACKAGE_EXT@")
###############################################################################################################
From 87b3e7c8211d9c587589f4d8d2965efb119fbf6d Mon Sep 17 00:00:00 2001
From: Robert Adam
Date: Mon, 30 Oct 2023 18:47:06 +0100
Subject: [PATCH 38/52] Don't build util tests when
HDF5_EXTERNALLY_CONFIGURED=ON (#3781)
Fixes #3780
---
c++/CMakeLists.txt | 2 +-
fortran/CMakeLists.txt | 2 +-
hl/CMakeLists.txt | 2 +-
java/CMakeLists.txt | 2 +-
tools/CMakeLists.txt | 2 +-
utils/CMakeLists.txt | 2 +-
6 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/c++/CMakeLists.txt b/c++/CMakeLists.txt
index 2c0275bc586..b419c805880 100644
--- a/c++/CMakeLists.txt
+++ b/c++/CMakeLists.txt
@@ -13,6 +13,6 @@ endif ()
#-----------------------------------------------------------------------------
# Build the CPP unit tests
#-----------------------------------------------------------------------------
-if (BUILD_TESTING)
+if (NOT HDF5_EXTERNALLY_CONFIGURED AND BUILD_TESTING)
add_subdirectory (test)
endif ()
diff --git a/fortran/CMakeLists.txt b/fortran/CMakeLists.txt
index cf9b427b3a6..efaf963c108 100644
--- a/fortran/CMakeLists.txt
+++ b/fortran/CMakeLists.txt
@@ -22,7 +22,7 @@ endif ()
#-----------------------------------------------------------------------------
# Testing
#-----------------------------------------------------------------------------
-if (BUILD_TESTING)
+if (NOT HDF5_EXTERNALLY_CONFIGURED AND BUILD_TESTING)
add_subdirectory (test)
if (MPI_Fortran_FOUND)
add_subdirectory (testpar)
diff --git a/hl/CMakeLists.txt b/hl/CMakeLists.txt
index a777b72f540..45a9a22dded 100644
--- a/hl/CMakeLists.txt
+++ b/hl/CMakeLists.txt
@@ -26,6 +26,6 @@ if (HDF5_BUILD_EXAMPLES)
endif ()
#-- Build the Unit testing if requested
-if (BUILD_TESTING AND HDF5_TEST_SERIAL)
+if (HDF5_EXTERNALLY_CONFIGURED AND BUILD_TESTING AND HDF5_TEST_SERIAL)
add_subdirectory (test)
endif ()
diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt
index ae37ceb9ad8..4965f2c9584 100644
--- a/java/CMakeLists.txt
+++ b/java/CMakeLists.txt
@@ -49,7 +49,7 @@ endif ()
#-----------------------------------------------------------------------------
# Testing
#-----------------------------------------------------------------------------
-if (BUILD_TESTING)
+if (NOT HDF5_EXTERNALLY_CONFIGURED AND BUILD_TESTING)
add_subdirectory (test)
endif ()
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
index 55f2c2a576f..91c57c9cf08 100644
--- a/tools/CMakeLists.txt
+++ b/tools/CMakeLists.txt
@@ -7,7 +7,7 @@ add_subdirectory (lib)
add_subdirectory (src)
#-- Add the tests
-if (BUILD_TESTING)
+if (NOT HDF5_EXTERNALLY_CONFIGURED AND BUILD_TESTING)
add_subdirectory (test)
# --------------------------------------------------------------------
diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
index 718f88e1907..b7e4630c07e 100644
--- a/utils/CMakeLists.txt
+++ b/utils/CMakeLists.txt
@@ -1,7 +1,7 @@
cmake_minimum_required (VERSION 3.18)
project (HDF5_UTILS C)
-if (BUILD_TESTING)
+if (NOT HDF5_EXTERNALLY_CONFIGURED AND BUILD_TESTING)
add_subdirectory (test)
endif ()
From e3f060852fdd1036453a715697effdf61c73b360 Mon Sep 17 00:00:00 2001
From: "H. Joe Lee"
Date: Tue, 31 Oct 2023 08:04:54 -0500
Subject: [PATCH 39/52] Fix h5py CI failure. (#3805)
---
.github/workflows/h5py.yml | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/h5py.yml b/.github/workflows/h5py.yml
index 10958441ab1..316a71d99b1 100644
--- a/.github/workflows/h5py.yml
+++ b/.github/workflows/h5py.yml
@@ -11,10 +11,11 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- - name: Install gfortran
- run: |
- sudo apt-get update
- sudo apt-get install -y gfortran-12
+ - name: Install Fortran
+ uses: fortran-lang/setup-fortran@v1
+ with:
+ compiler: gcc
+ version: 13
- name: Checkout Spack
uses: actions/checkout@v4
with:
From 8df0008f0718d1152f41c66447f77934a0de66af Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Tue, 31 Oct 2023 08:05:12 -0500
Subject: [PATCH 40/52] Fix memory corruption in 'MPI I/O FAPL preserve' test
(#3806)
---
testpar/t_file.c | 44 ++++++++++++++++++++++----------------------
1 file changed, 22 insertions(+), 22 deletions(-)
diff --git a/testpar/t_file.c b/testpar/t_file.c
index ce55270cdd2..8f8b2914a70 100644
--- a/testpar/t_file.c
+++ b/testpar/t_file.c
@@ -1128,28 +1128,27 @@ test_evict_on_close_parallel_unsupp(void)
void
test_fapl_preserve_hints(void)
{
- hid_t fid = H5I_INVALID_HID; /* HDF5 file ID */
- hid_t fapl_id = H5I_INVALID_HID; /* File access plist */
const char *filename;
-
- int nkeys_used;
- bool same = false;
-
- MPI_Info info = MPI_INFO_NULL;
- const char *key = "hdf_info_fapl";
- const char *value = "xyz";
-
- MPI_Info info_used = MPI_INFO_NULL;
- int flag = -1;
- char value_used[20];
- char key_used[20];
-
- int i;
- herr_t ret; /* Generic return value */
- int mpi_ret; /* MPI return value */
+ const char *key = "hdf_info_fapl";
+ const char *value = "xyz";
+ MPI_Info info_used = MPI_INFO_NULL;
+ MPI_Info info = MPI_INFO_NULL;
+ hid_t fid = H5I_INVALID_HID; /* HDF5 file ID */
+ hid_t fapl_id = H5I_INVALID_HID; /* File access plist */
+ char key_used[MPI_MAX_INFO_KEY + 1];
+ char *value_used = NULL;
+ bool same = false;
+ int flag = -1;
+ int nkeys_used;
+ int i;
+ int mpi_ret; /* MPI return value */
+ herr_t ret; /* Generic return value */
filename = (const char *)GetTestParameters();
+ value_used = malloc(MPI_MAX_INFO_VAL + 1);
+ VRFY(value_used, "malloc succeeded");
+
/* set up MPI parameters */
mpi_ret = MPI_Info_create(&info);
VRFY((mpi_ret >= 0), "MPI_Info_create succeeded");
@@ -1184,16 +1183,15 @@ test_fapl_preserve_hints(void)
for (i = 0; i < nkeys_used; i++) {
/* Memset the buffers to zero */
- memset(key_used, 0, 20);
- memset(value_used, 0, 20);
+ memset(key_used, 0, MPI_MAX_INFO_KEY + 1);
+ memset(value_used, 0, MPI_MAX_INFO_VAL + 1);
/* Get the nth key */
mpi_ret = MPI_Info_get_nthkey(info_used, i, key_used);
VRFY((mpi_ret == MPI_SUCCESS), "MPI_Info_get_nthkey succeeded");
if (!strcmp(key_used, key)) {
-
- mpi_ret = MPI_Info_get(info_used, key_used, 20, value_used, &flag);
+ mpi_ret = MPI_Info_get(info_used, key_used, MPI_MAX_INFO_VAL, value_used, &flag);
VRFY((mpi_ret == MPI_SUCCESS), "MPI_Info_get succeeded");
if (!strcmp(value_used, value)) {
@@ -1220,4 +1218,6 @@ test_fapl_preserve_hints(void)
mpi_ret = MPI_Info_free(&info_used);
VRFY((mpi_ret >= 0), "MPI_Info_free succeeded");
+ free(value_used);
+
} /* end test_fapl_preserve_hints() */
From ea3b6fd12b4fb50c1d891e1408e99060b8afacd1 Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Tue, 31 Oct 2023 08:05:25 -0500
Subject: [PATCH 41/52] Fix usage of h5_clean_files in t_pflush2.c (#3807)
---
testpar/t_pflush2.c | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/testpar/t_pflush2.c b/testpar/t_pflush2.c
index 95ad1257801..e1dce1bbfd7 100644
--- a/testpar/t_pflush2.c
+++ b/testpar/t_pflush2.c
@@ -20,7 +20,8 @@
#include "h5test.h"
-static const char *FILENAME[] = {"flush", "noflush", NULL};
+static const char *FLUSH_FILENAME[] = {"flush", NULL};
+static const char *NOFLUSH_FILENAME[] = {"noflush", NULL};
static int *data_g = NULL;
@@ -173,7 +174,7 @@ main(int argc, char *argv[])
goto error;
/* Check the case where the file was flushed */
- h5_fixname(FILENAME[0], fapl_id1, name, sizeof(name));
+ h5_fixname(FLUSH_FILENAME[0], fapl_id1, name, sizeof(name));
if (check_test_file(name, sizeof(name), fapl_id1)) {
H5_FAILED();
goto error;
@@ -190,7 +191,7 @@ main(int argc, char *argv[])
H5Eget_auto2(H5E_DEFAULT, &func, NULL);
H5Eset_auto2(H5E_DEFAULT, NULL, NULL);
- h5_fixname(FILENAME[1], fapl_id2, name, sizeof(name));
+ h5_fixname(NOFLUSH_FILENAME[0], fapl_id2, name, sizeof(name));
if (check_test_file(name, sizeof(name), fapl_id2)) {
if (mpi_rank == 0)
PASSED();
@@ -202,8 +203,8 @@ main(int argc, char *argv[])
H5Eset_auto2(H5E_DEFAULT, func, NULL);
- h5_clean_files(&FILENAME[0], fapl_id1);
- h5_clean_files(&FILENAME[1], fapl_id2);
+ h5_clean_files(FLUSH_FILENAME, fapl_id1);
+ h5_clean_files(NOFLUSH_FILENAME, fapl_id2);
if (data_g) {
free(data_g);
From ebc1651425e6052824bae82b2a8016f9a8af4ff2 Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Tue, 31 Oct 2023 08:05:40 -0500
Subject: [PATCH 42/52] Fix parallel driver check in h5_fixname_real (#3808)
---
test/h5test.c | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/test/h5test.c b/test/h5test.c
index 304637580f6..ef580cf3072 100644
--- a/test/h5test.c
+++ b/test/h5test.c
@@ -457,7 +457,7 @@ h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, char *fu
const char *suffix = _suffix;
size_t i, j;
hid_t driver = H5I_INVALID_HID;
- int isppdriver = 0; /* if the driver is MPI parallel */
+ bool isppdriver = false; /* if the driver is MPI parallel */
if (!base_name || !fullname || size < 1)
return NULL;
@@ -516,10 +516,8 @@ h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, char *fu
}
}
- /* Must first check fapl is not H5P_DEFAULT (-1) because H5FD_XXX
- * could be of value -1 if it is not defined.
- */
- isppdriver = ((H5P_DEFAULT != fapl) || driver_env_var) && (H5FD_MPIO == driver);
+ if (h5_using_parallel_driver(fapl, &isppdriver) < 0)
+ return NULL;
/* Check HDF5_NOCLEANUP environment setting.
* (The #ifdef is needed to prevent compile failure in case MPI is not
From 39c0284c38395aac9eb43364b14931fc2826b7f6 Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Tue, 31 Oct 2023 08:05:52 -0500
Subject: [PATCH 43/52] Fix a couple usages of MPI_Info_get (#3809)
---
src/H5FDmpio.c | 4 ++--
src/H5mpi.c | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c
index 7141550f40a..83a5ad45a9a 100644
--- a/src/H5FDmpio.c
+++ b/src/H5FDmpio.c
@@ -906,7 +906,7 @@ H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t H5_ATTR
/* copy over each hint */
for (i = 0; i < nkeys; i++) {
- char key[MPI_MAX_INFO_KEY], value[MPI_MAX_INFO_VAL];
+ char key[MPI_MAX_INFO_KEY], value[MPI_MAX_INFO_VAL + 1];
int valuelen, flag;
/* retrieve the nth hint */
@@ -916,7 +916,7 @@ H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t H5_ATTR
if (MPI_SUCCESS != (mpi_code = MPI_Info_get_valuelen(info_used, key, &valuelen, &flag)))
HMPI_GOTO_ERROR(NULL, "MPI_Info_get_valuelen failed", mpi_code)
/* retrieve the value of nth hint */
- if (MPI_SUCCESS != (mpi_code = MPI_Info_get(info_used, key, valuelen + 1, value, &flag)))
+ if (MPI_SUCCESS != (mpi_code = MPI_Info_get(info_used, key, valuelen, value, &flag)))
HMPI_GOTO_ERROR(NULL, "MPI_Info_get failed", mpi_code)
/* copy the hint into info */
diff --git a/src/H5mpi.c b/src/H5mpi.c
index 2725ec5bf19..cf7e33d46c9 100644
--- a/src/H5mpi.c
+++ b/src/H5mpi.c
@@ -380,9 +380,9 @@ H5_mpi_info_cmp(MPI_Info info1, MPI_Info info2, int *result)
/* Allocate buffers for iteration */
if (NULL == (key = (char *)H5MM_malloc(MPI_MAX_INFO_KEY * sizeof(char))))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
- if (NULL == (value1 = (char *)H5MM_malloc(MPI_MAX_INFO_VAL * sizeof(char))))
+ if (NULL == (value1 = (char *)H5MM_malloc((MPI_MAX_INFO_VAL + 1) * sizeof(char))))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
- if (NULL == (value2 = (char *)H5MM_malloc(MPI_MAX_INFO_VAL * sizeof(char))))
+ if (NULL == (value2 = (char *)H5MM_malloc((MPI_MAX_INFO_VAL + 1) * sizeof(char))))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
/* Iterate over the keys, comparing them */
From e9e4ffaec73d74075426f61ecdd9e9ea35fa6f50 Mon Sep 17 00:00:00 2001
From: "H. Joe Lee"
Date: Wed, 1 Nov 2023 07:12:03 -0500
Subject: [PATCH 44/52] Remove H5system.c warning on Windows oneAPI. (#3812)
---
src/H5system.c | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/H5system.c b/src/H5system.c
index 30a89a174af..be886ae52f3 100644
--- a/src/H5system.c
+++ b/src/H5system.c
@@ -807,13 +807,12 @@ H5_nanosleep(uint64_t nanosec)
#ifdef H5_HAVE_WIN32_API
DWORD dwMilliseconds = (DWORD)ceil(nanosec / 1.0e6);
- DWORD ignore;
/* Windows can't sleep at a ns resolution. Best we can do is ~1 ms. We
* don't care about the return value since the second parameter
* (bAlertable) is false, so it will always be zero.
*/
- ignore = SleepEx(dwMilliseconds, false);
+ SleepEx(dwMilliseconds, false);
#else
From f8ab865ce75821a2e24127b1d52ce3070980c298 Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Wed, 1 Nov 2023 07:13:22 -0500
Subject: [PATCH 45/52] Add processing of NVHPC flags in linux-gnulibc1 file
(#3804)
* Disable testing as tests are failing the same as in CMake
---
.github/workflows/nvhpc-auto.yml | 10 +++++-----
config/linux-gnulibc1 | 25 +++++++++++++++++++++++--
2 files changed, 28 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/nvhpc-auto.yml b/.github/workflows/nvhpc-auto.yml
index 3e3a323fe1e..2a97ba1fd21 100644
--- a/.github/workflows/nvhpc-auto.yml
+++ b/.github/workflows/nvhpc-auto.yml
@@ -67,11 +67,11 @@ jobs:
# RUN TESTS
# NORMAL
- - name: Autotools Run Tests
- run: |
- export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin:$PATH
- make check -j
- working-directory: ${{ runner.workspace }}/build
+# - name: Autotools Run Tests
+# run: |
+# export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin:$PATH
+# make check -j
+# working-directory: ${{ runner.workspace }}/build
# INSTALL (note that this runs even when we don't run the tests)
- name: Autotools Install
diff --git a/config/linux-gnulibc1 b/config/linux-gnulibc1
index 328f8d3cec3..92f2be63df5 100644
--- a/config/linux-gnulibc1
+++ b/config/linux-gnulibc1
@@ -47,6 +47,9 @@ fi
# Figure out Clang C compiler flags
. $srcdir/config/clang-flags
+# Figure out NVHPC C compiler flags
+. $srcdir/config/nvidia-flags
+
# Use default Fortran 90 compiler according to what C compiler is used.
if test "X-" = "X-$FC"; then
case $CC_BASENAME in
@@ -58,6 +61,10 @@ if test "X-" = "X-$FC"; then
FC=pgf90
FC_BASENAME=pgf90
;;
+ nvc*)
+ FC=nvfortran
+ FC_BASENAME=nvfortran
+ ;;
icx*)
FC=ifx
FC_BASENAME=ifx
@@ -79,7 +86,7 @@ if test "X-" = "X-$FC"; then
else
case $FC in
# The PGI and Intel compilers are automatically detected below
- ifc*|ifort*|pgf90*)
+ ifc*|ifort*|pgf90*|nvfortran*)
;;
*f95*)
@@ -135,6 +142,9 @@ fi
# Figure out Clang FC compiler flags
. $srcdir/config/clang-fflags
+# Figure out NVHPC FC compiler flags
+. $srcdir/config/nvidia-fflags
+
case $FC_BASENAME in
#
# Absoft compiler
@@ -226,6 +236,9 @@ fi
# Figure out Clang CXX compiler flags
. $srcdir/config/clang-cxxflags
+# Figure out NVHPC CXX compiler flags
+. $srcdir/config/nvidia-cxxflags
+
# compiler version strings
# check if the compiler_version_info is already set
@@ -248,7 +261,11 @@ case $CC in
sed 's/\"/\\\"/g' |\
sed 's/^\([a-z]* \)/ built with \1/1'`
cc_version_info=`echo $cc_version_info`
- ;;
+ ;;
+
+ *nvc*)
+ cc_version_info=`$CC $CFLAGS $H5_CFLAGS -V 2>&1 | grep 'nvc'`
+ ;;
*icx*)
cc_version_info=`$CC $CCFLAGS $H5_CCFLAGS -V 2>&1 | grep 'Version' |\
@@ -322,6 +339,10 @@ case $FC in
fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'pgf90'`
;;
+ *nvfortran*)
+ fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'nvfortran'`
+ ;;
+
*nagfor*|*nagftn*)
RM='rm -f'
tmpfile=/tmp/cmpver.$$
From a654b2c1c5670010e14d8fba6d287ce62c87169a Mon Sep 17 00:00:00 2001
From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com>
Date: Wed, 1 Nov 2023 07:15:19 -0500
Subject: [PATCH 46/52] Use the current toolchain for examples as default
(#3810)
---
CMakeInstallation.cmake | 4 ++++
config/cmake/examples/HDF5_Examples.cmake.in | 7 +++++++
config/cmake/examples/HDF5_Examples_options.cmake | 13 +++++++++++++
config/cmake/scripts/HDF5options.cmake | 1 -
4 files changed, 24 insertions(+), 1 deletion(-)
diff --git a/CMakeInstallation.cmake b/CMakeInstallation.cmake
index 74ab8b15fd2..02f8dc35058 100644
--- a/CMakeInstallation.cmake
+++ b/CMakeInstallation.cmake
@@ -141,6 +141,10 @@ install (
#-----------------------------------------------------------------------------
option (HDF5_PACK_EXAMPLES "Package the HDF5 Library Examples Compressed File" OFF)
if (HDF5_PACK_EXAMPLES)
+ if (DEFINED CMAKE_TOOLCHAIN_FILE)
+ get_filename_component(TOOLCHAIN ${CMAKE_TOOLCHAIN_FILE} NAME)
+ set(CTEST_TOOLCHAIN_FILE "\${CTEST_SOURCE_DIRECTORY}/config/toolchain/${TOOLCHAIN}")
+ endif ()
configure_file (
${HDF_RESOURCES_DIR}/examples/HDF5_Examples.cmake.in
${HDF5_BINARY_DIR}/HDF5_Examples.cmake @ONLY
diff --git a/config/cmake/examples/HDF5_Examples.cmake.in b/config/cmake/examples/HDF5_Examples.cmake.in
index f45e1ed6dbe..2f3a6491a90 100644
--- a/config/cmake/examples/HDF5_Examples.cmake.in
+++ b/config/cmake/examples/HDF5_Examples.cmake.in
@@ -29,6 +29,7 @@ set(CTEST_DASHBOARD_ROOT ${CTEST_SCRIPT_DIRECTORY})
#INSTALLDIR - HDF5 root folder
#CTEST_CONFIGURATION_TYPE - Release, Debug, RelWithDebInfo
#CTEST_SOURCE_NAME - name of source folder; HDF5Examples
+#CTEST_TOOLCHAIN_FILE - name and path in source of toolchain file
if(DEFINED CTEST_SCRIPT_ARG)
# transform ctest script arguments of the form
# script.ctest,var1=value1,var2=value2
@@ -96,6 +97,12 @@ endif()
### default HDF5_PLUGIN_PATH to where the filter libraries are located
set(ENV{HDF5_PLUGIN_PATH} "${INSTALLDIR}/lib/plugin")
set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACKAGE_NAME:STRING=@HDF5_PACKAGE@@HDF_PACKAGE_EXT@")
+### use a toolchain file (supported everywhere) ####
+if(NOT DEFINED CTEST_TOOLCHAIN_FILE)
+ set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=@CTEST_TOOLCHAIN_FILE@")
+else()
+ set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=${CTEST_TOOLCHAIN_FILE}")
+endif()
###############################################################################################################
# For any comments please contact cdashhelp@hdfgroup.org
diff --git a/config/cmake/examples/HDF5_Examples_options.cmake b/config/cmake/examples/HDF5_Examples_options.cmake
index cdd49eb13ad..684ec5bf641 100644
--- a/config/cmake/examples/HDF5_Examples_options.cmake
+++ b/config/cmake/examples/HDF5_Examples_options.cmake
@@ -28,6 +28,19 @@
### build with shared libraries
#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DBUILD_SHARED_LIBS:BOOL=ON")
+#############################################################################################
+#### maximum parallel processor count for build and test ####
+#set(MAX_PROC_COUNT 8)
+
+#############################################################################################
+#### alternate toolsets (Windows usually) ####
+#set(CMAKE_GENERATOR_TOOLSET "Intel C++ Compiler 17.0")
+
+#############################################################################################
+### use a toolchain file (supported everywhere) ####
+#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/clang.cmake")
+#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/intel.cmake")
+
#############################################################################################
#### languages ####
### disable C builds
diff --git a/config/cmake/scripts/HDF5options.cmake b/config/cmake/scripts/HDF5options.cmake
index 92bfd37ecbe..5d078461be9 100644
--- a/config/cmake/scripts/HDF5options.cmake
+++ b/config/cmake/scripts/HDF5options.cmake
@@ -26,7 +26,6 @@
#############################################################################################
### use a toolchain file (supported everywhere) ####
-
#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/intel.cmake")
#############################################################################################
From 562c53c44a22c979419deb1cb025b2a74900fbac Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Wed, 1 Nov 2023 08:16:33 -0500
Subject: [PATCH 47/52] Fix misc. warnings from GCC when compiling with
-fsanitize=undefined (#3787)
---
src/H5Dmpio.c | 3 ++-
src/H5FDfamily.c | 3 ++-
src/H5FDlog.c | 2 +-
src/H5FDsec2.c | 2 +-
src/H5T.c | 6 +++---
tools/src/misc/h5repart.c | 8 ++++++++
6 files changed, 17 insertions(+), 7 deletions(-)
diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c
index b40ab4b2012..b6976e6a067 100644
--- a/src/H5Dmpio.c
+++ b/src/H5Dmpio.c
@@ -911,7 +911,8 @@ H5D__mpio_get_no_coll_cause_strings(char *local_cause, size_t local_cause_len, c
case H5D_MPIO_COLLECTIVE:
case H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE:
default:
- assert(0 && "invalid no collective cause reason");
+ cause_str = "invalid or unknown no collective cause reason";
+ assert(0 && "invalid or unknown no collective cause reason");
break;
}
diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c
index 94805a23a8b..3f43ae9cc5b 100644
--- a/src/H5FDfamily.c
+++ b/src/H5FDfamily.c
@@ -1427,7 +1427,8 @@ H5FD__family_delete(const char *filename, hid_t fapl_id)
FUNC_ENTER_PACKAGE
- assert(filename);
+ if (!filename)
+ HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid filename pointer");
/* Get the driver info (for the member fapl)
* The family_open call accepts H5P_DEFAULT, so we'll accept that here, too.
diff --git a/src/H5FDlog.c b/src/H5FDlog.c
index 8d43dc836a6..e35a6a65c4a 100644
--- a/src/H5FDlog.c
+++ b/src/H5FDlog.c
@@ -545,7 +545,7 @@ H5FD__log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
#endif /* H5_HAVE_WIN32_API */
/* Retain a copy of the name used to open the file, for possible error reporting */
- strncpy(file->filename, name, sizeof(file->filename));
+ strncpy(file->filename, name, sizeof(file->filename) - 1);
file->filename[sizeof(file->filename) - 1] = '\0';
/* Get the flags for logging */
diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c
index 29616842c15..15accf76d33 100644
--- a/src/H5FDsec2.c
+++ b/src/H5FDsec2.c
@@ -368,7 +368,7 @@ H5FD__sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
}
/* Retain a copy of the name used to open the file, for possible error reporting */
- strncpy(file->filename, name, sizeof(file->filename));
+ strncpy(file->filename, name, sizeof(file->filename) - 1);
file->filename[sizeof(file->filename) - 1] = '\0';
/* Check for non-default FAPL */
diff --git a/src/H5T.c b/src/H5T.c
index ef94925982f..a02abfc18d0 100644
--- a/src/H5T.c
+++ b/src/H5T.c
@@ -2501,7 +2501,7 @@ H5T__register(H5T_pers_t pers, const char *name, H5T_t *src, H5T_t *dst, H5T_con
H5T_g.asoft = na;
H5T_g.soft = x;
} /* end if */
- strncpy(H5T_g.soft[H5T_g.nsoft].name, name, (size_t)H5T_NAMELEN);
+ strncpy(H5T_g.soft[H5T_g.nsoft].name, name, (size_t)H5T_NAMELEN - 1);
H5T_g.soft[H5T_g.nsoft].name[H5T_NAMELEN - 1] = '\0';
H5T_g.soft[H5T_g.nsoft].src = src->shared->type;
H5T_g.soft[H5T_g.nsoft].dst = dst->shared->type;
@@ -2550,7 +2550,7 @@ H5T__register(H5T_pers_t pers, const char *name, H5T_t *src, H5T_t *dst, H5T_con
/* Create a new conversion path */
if (NULL == (new_path = H5FL_CALLOC(H5T_path_t)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
- strncpy(new_path->name, name, (size_t)H5T_NAMELEN);
+ strncpy(new_path->name, name, (size_t)H5T_NAMELEN - 1);
new_path->name[H5T_NAMELEN - 1] = '\0';
if (NULL == (new_path->src = H5T_copy(old_path->src, H5T_COPY_ALL)) ||
NULL == (new_path->dst = H5T_copy(old_path->dst, H5T_COPY_ALL)))
@@ -4953,7 +4953,7 @@ H5T__path_find_real(const H5T_t *src, const H5T_t *dst, const char *name, H5T_co
if (NULL == (path = H5FL_CALLOC(H5T_path_t)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for type conversion path");
if (name && *name) {
- strncpy(path->name, name, (size_t)H5T_NAMELEN);
+ strncpy(path->name, name, (size_t)H5T_NAMELEN - 1);
path->name[H5T_NAMELEN - 1] = '\0';
} /* end if */
else
diff --git a/tools/src/misc/h5repart.c b/tools/src/misc/h5repart.c
index feb447f7e09..12b293273bc 100644
--- a/tools/src/misc/h5repart.c
+++ b/tools/src/misc/h5repart.c
@@ -227,6 +227,10 @@ main(int argc, char *argv[])
if (argno >= argc)
usage(prog_name);
src_gen_name = argv[argno++];
+ if (!src_gen_name) {
+ fprintf(stderr, "invalid source file name pointer");
+ exit(EXIT_FAILURE);
+ }
snprintf(src_name, NAMELEN, src_gen_name, src_membno);
src_is_family = strcmp(src_name, src_gen_name);
@@ -249,6 +253,10 @@ main(int argc, char *argv[])
if (argno >= argc)
usage(prog_name);
dst_gen_name = argv[argno++];
+ if (!dst_gen_name) {
+ fprintf(stderr, "invalid destination file name pointer");
+ exit(EXIT_FAILURE);
+ }
snprintf(dst_name, NAMELEN, dst_gen_name, dst_membno);
dst_is_family = strcmp(dst_name, dst_gen_name);
From 64e239c4b6adcf597ac5ada6e7f03cbff126e833 Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Wed, 1 Nov 2023 14:41:46 -0500
Subject: [PATCH 48/52] Set NVHPC maximum optimization level to -O1 for now
(#3800)
* Set NVHPC maximum optimization level to -O1 for now
Compiling HDF5 with NVHPC 23.5 - 23.9 results in test failures in
4 different test files that need to be resolved. Since those tests
pass with an optimization level of -O1 (and -O0) and it is currently
unclear whether the test failures are due to issues in HDF5 or issues
in the 'nvc' compiler, set the maximum optimization level for NVHPC
to -O1 until the test failures are resolved.
* Disable nvhpc Java testing in CMake and amend known issues
* Re-enable testing of Autotools nvhpc
---
.github/workflows/nvhpc-auto.yml | 10 ++++----
.github/workflows/nvhpc-cmake.yml | 12 ++++-----
config/cmake/HDFCompilerFlags.cmake | 38 +++++++++++++++++++++++++++++
config/nvidia-flags | 6 +++--
release_docs/RELEASE.txt | 22 +++++++++++++++++
5 files changed, 75 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/nvhpc-auto.yml b/.github/workflows/nvhpc-auto.yml
index 2a97ba1fd21..3e3a323fe1e 100644
--- a/.github/workflows/nvhpc-auto.yml
+++ b/.github/workflows/nvhpc-auto.yml
@@ -67,11 +67,11 @@ jobs:
# RUN TESTS
# NORMAL
-# - name: Autotools Run Tests
-# run: |
-# export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin:$PATH
-# make check -j
-# working-directory: ${{ runner.workspace }}/build
+ - name: Autotools Run Tests
+ run: |
+ export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin:$PATH
+ make check -j
+ working-directory: ${{ runner.workspace }}/build
# INSTALL (note that this runs even when we don't run the tests)
- name: Autotools Install
diff --git a/.github/workflows/nvhpc-cmake.yml b/.github/workflows/nvhpc-cmake.yml
index 489c0bbf3fb..e4a1454f215 100644
--- a/.github/workflows/nvhpc-cmake.yml
+++ b/.github/workflows/nvhpc-cmake.yml
@@ -56,7 +56,7 @@ jobs:
-DLIBAEC_USE_LOCALCONTENT=OFF \
-DZLIB_USE_LOCALCONTENT=OFF \
-DHDF5_BUILD_FORTRAN:BOOL=ON \
- -DHDF5_ENABLE_ASSERTS:BOOL=ON \
+ -DHDF5_BUILD_JAVA:BOOL=OFF \
-DMPIEXEC_MAX_NUMPROCS:STRING="2" \
$GITHUB_WORKSPACE
cat src/libhdf5.settings
@@ -69,8 +69,8 @@ jobs:
working-directory: ${{ runner.workspace }}/build
# RUN TESTS
-# - name: CMake Run Tests
-# shell: bash
-# run: |
-# ctest . --parallel 2 -C ${{ inputs.build_mode }} -V
-# working-directory: ${{ runner.workspace }}/build
+ - name: CMake Run Tests
+ shell: bash
+ run: |
+ ctest . --parallel 2 -C ${{ inputs.build_mode }} -V
+ working-directory: ${{ runner.workspace }}/build
diff --git a/config/cmake/HDFCompilerFlags.cmake b/config/cmake/HDFCompilerFlags.cmake
index e7b9337f39c..7bddad0f776 100644
--- a/config/cmake/HDFCompilerFlags.cmake
+++ b/config/cmake/HDFCompilerFlags.cmake
@@ -56,6 +56,44 @@ if (CMAKE_C_COMPILER_ID STREQUAL "NVHPC" )
else ()
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Mbounds -g")
endif ()
+
+ # With at least NVHPC 23.5 - 23.9, compiling with -O2 or higher and -DNDEBUG
+ # appears to have issues that manifest in the tests as incorrect metadata
+ # checksums being read or memory being corrupted. Compiling without -DNDEBUG
+ # does not appear to have these issues, but is not ideal due to compiling in
+ # asserts and other library debug code. Compiling with -O1 also does not appear
+ # to have these issues, so set maximum optimization level to -O1 for now until
+ # it can be determined whether these issues are compiler-specific or issues
+ # in the library.
+ set (cmake_c_flags_minsizerel_edited "${CMAKE_C_FLAGS_MINSIZEREL}")
+ string (REPLACE "-O2" "" cmake_c_flags_minsizerel_edited "${cmake_c_flags_minsizerel_edited}")
+ string (REPLACE "-O3" "" cmake_c_flags_minsizerel_edited "${cmake_c_flags_minsizerel_edited}")
+ string (REPLACE "-O4" "" cmake_c_flags_minsizerel_edited "${cmake_c_flags_minsizerel_edited}")
+ string (REPLACE "-Ofast" "" cmake_c_flags_minsizerel_edited "${cmake_c_flags_minsizerel_edited}")
+ string (REPLACE "-fast" "" cmake_c_flags_minsizerel_edited "${cmake_c_flags_minsizerel_edited}")
+ string (STRIP "${cmake_c_flags_minsizerel_edited}" cmake_c_flags_minsizerel_edited)
+ string (PREPEND cmake_c_flags_minsizerel_edited "-O1 ")
+ set (CMAKE_C_FLAGS_MINSIZEREL "${cmake_c_flags_minsizerel_edited}")
+
+ set (cmake_c_flags_release_edited "${CMAKE_C_FLAGS_RELEASE}")
+ string (REPLACE "-O2" "" cmake_c_flags_release_edited "${cmake_c_flags_release_edited}")
+ string (REPLACE "-O3" "" cmake_c_flags_release_edited "${cmake_c_flags_release_edited}")
+ string (REPLACE "-O4" "" cmake_c_flags_release_edited "${cmake_c_flags_release_edited}")
+ string (REPLACE "-Ofast" "" cmake_c_flags_release_edited "${cmake_c_flags_release_edited}")
+ string (REPLACE "-fast" "" cmake_c_flags_release_edited "${cmake_c_flags_release_edited}")
+ string (STRIP "${cmake_c_flags_release_edited}" cmake_c_flags_release_edited)
+ string (PREPEND cmake_c_flags_release_edited "-O1 ")
+ set (CMAKE_C_FLAGS_RELEASE "${cmake_c_flags_release_edited}")
+
+ set (cmake_c_flags_relwithdebinfo_edited "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
+ string (REPLACE "-O2" "" cmake_c_flags_relwithdebinfo_edited "${cmake_c_flags_relwithdebinfo_edited}")
+ string (REPLACE "-O3" "" cmake_c_flags_relwithdebinfo_edited "${cmake_c_flags_relwithdebinfo_edited}")
+ string (REPLACE "-O4" "" cmake_c_flags_relwithdebinfo_edited "${cmake_c_flags_relwithdebinfo_edited}")
+ string (REPLACE "-Ofast" "" cmake_c_flags_relwithdebinfo_edited "${cmake_c_flags_relwithdebinfo_edited}")
+ string (REPLACE "-fast" "" cmake_c_flags_relwithdebinfo_edited "${cmake_c_flags_relwithdebinfo_edited}")
+ string (STRIP "${cmake_c_flags_relwithdebinfo_edited}" cmake_c_flags_relwithdebinfo_edited)
+ string (PREPEND cmake_c_flags_relwithdebinfo_edited "-O1 ")
+ set (CMAKE_C_FLAGS_RELWITHDEBINFO "${cmake_c_flags_relwithdebinfo_edited}")
endif ()
if (CMAKE_COMPILER_IS_GNUCC)
diff --git a/config/nvidia-flags b/config/nvidia-flags
index 864c6444114..c140edd9830 100644
--- a/config/nvidia-flags
+++ b/config/nvidia-flags
@@ -76,7 +76,8 @@ if test "X-nvc" = "X-$cc_vendor" -o "X-nvcc" = "X-$cc_vendor"; then
##############
# NDEBUG is handled explicitly by the configure script
- PROD_CFLAGS="-fast"
+ #PROD_CFLAGS="-fast"
+ PROD_CFLAGS="" # -fast implies -O2 and -O2+ currently has test failures.
#########
# Debug #
@@ -106,7 +107,8 @@ if test "X-nvc" = "X-$cc_vendor" -o "X-nvcc" = "X-$cc_vendor"; then
################
HIGH_OPT_CFLAGS="-O1" # -O2+ currently has test failures.
- DEBUG_OPT_CFLAGS="-gopt -O2"
+ #DEBUG_OPT_CFLAGS="-gopt -O2"
+ DEBUG_OPT_CFLAGS="-gopt -O1" # -O2+ currently has test failures.
NO_OPT_CFLAGS="-O0"
#################
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 0f18f6fa276..51a73ce1121 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -1158,6 +1158,28 @@ Platforms Tested
Known Problems
==============
+ When HDF5 is compiled with NVHPC versions 23.5 - 23.9 (additional versions may
+ also be applicable) and with -O2 (or higher) and -DNDEBUG, test failures occur
+ in the following tests:
+
+ H5PLUGIN-filter_plugin
+ H5TEST-flush2
+ H5TEST-testhdf5-base
+ MPI_TEST_t_filters_parallel
+
+ Since these tests pass with an optimization level of -O1 (and -O0) and it is
+ currently unclear whether the test failures are due to issues in HDF5 or issues
+ in the 'nvc' compiler, the maximum optimization level for NVHPC has been set
+ to -O1 until the test failures can be resolved. Note that even at -O1 optimization
+ level, there still appears to be a sporadic test failure in the Java JUnit tests
+ that has occasionally been seen in JUnit-TestH5Pfapl and JUnit-TestH5D. It is also
+ unclear whether this is an issue in HDF5 or with the 'nvc' compiler. Finally, note
+ that NVHPC 23.9 will fail to compile the test/tselect.c test file with a compiler
+ error of 'use of undefined value' when the optimization level is -O2 or higher.
+ Nvidia is aware of this issue and has suggested lowering the optimization level to
+ -O1 for the time being:
+ https://forums.developer.nvidia.com/t/hdf5-no-longer-compiles-with-nv-23-9/269045.
+
IEEE standard arithmetic enables software to raise exceptions such as overflow,
division by zero, and other illegal operations without interrupting or halting
the program flow. The HDF5 C library intentionally performs these exceptions.
From 94a992689e0ff1de71f191f606bdc231211fb2d6 Mon Sep 17 00:00:00 2001
From: "H. Joe Lee"
Date: Thu, 2 Nov 2023 00:11:23 -0500
Subject: [PATCH 49/52] Update NVHPC version from 23.7 to 23.9. (#3736)
Co-authored-by: Larry Knox
---
.github/workflows/nvhpc-auto.yml | 24 ++++++++++++------------
.github/workflows/nvhpc-cmake.yml | 12 ++++++------
2 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/nvhpc-auto.yml b/.github/workflows/nvhpc-auto.yml
index 3e3a323fe1e..0b6f64af0e9 100644
--- a/.github/workflows/nvhpc-auto.yml
+++ b/.github/workflows/nvhpc-auto.yml
@@ -31,21 +31,21 @@ jobs:
curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg
echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list
sudo apt-get update -y
- sudo apt-get install -y nvhpc-23-7
- echo "CC=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/openmpi4/bin/mpicc" >> $GITHUB_ENV
- echo "FC=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/openmpi4/bin/mpifort" >> $GITHUB_ENV
+ sudo apt-get install -y nvhpc-23-9
+ echo "CC=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/comm_libs/openmpi4/bin/mpicc" >> $GITHUB_ENV
+ echo "FC=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/comm_libs/openmpi4/bin/mpifort" >> $GITHUB_ENV
echo "NVHPCSDK=/opt/nvidia/hpc_sdk" >> $GITHUB_ENV
- echo "OMPI_CXX=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin/nvc++" >> $GITHUB_ENV
- echo "OMPI_CC=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin/nvc" >> $GITHUB_ENV
- echo "OMPI_FC=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin/nvfortran" >> $GITHUB_ENV
- echo "LD_LIBRARY_PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/lib" >> $GITHUB_ENV
+ echo "OMPI_CXX=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvc++" >> $GITHUB_ENV
+ echo "OMPI_CC=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvc" >> $GITHUB_ENV
+ echo "OMPI_FC=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvfortran" >> $GITHUB_ENV
+ echo "LD_LIBRARY_PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/lib" >> $GITHUB_ENV
echo "DESTDIR=/tmp" >> $GITHUB_ENV
- name: Autotools Configure
shell: bash
run: |
export RUNPARALLEL="mpiexec -np 2"
- export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin:$PATH
+ export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin:$PATH
sh ./autogen.sh
mkdir "${{ runner.workspace }}/build"
cd "${{ runner.workspace }}/build"
@@ -61,7 +61,7 @@ jobs:
- name: Autotools Build
shell: bash
run: |
- export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin:$PATH
+ export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin:$PATH
make -j3
working-directory: ${{ runner.workspace }}/build
@@ -69,19 +69,19 @@ jobs:
# NORMAL
- name: Autotools Run Tests
run: |
- export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin:$PATH
+ export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin:$PATH
make check -j
working-directory: ${{ runner.workspace }}/build
# INSTALL (note that this runs even when we don't run the tests)
- name: Autotools Install
run: |
- export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin:$PATH
+ export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin:$PATH
make install
working-directory: ${{ runner.workspace }}/build
# - name: Autotools Verify Install
# run: |
-# export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin:$PATH
+# export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin:$PATH
# make check-install
# working-directory: ${{ runner.workspace }}/build
diff --git a/.github/workflows/nvhpc-cmake.yml b/.github/workflows/nvhpc-cmake.yml
index e4a1454f215..1b0dbebc19e 100644
--- a/.github/workflows/nvhpc-cmake.yml
+++ b/.github/workflows/nvhpc-cmake.yml
@@ -32,20 +32,20 @@ jobs:
curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg
echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list
sudo apt-get update -y
- sudo apt-get install -y nvhpc-23-7
+ sudo apt-get install -y nvhpc-23-9
echo "CC=nvc" >> $GITHUB_ENV
echo "FC=nvfortran" >> $GITHUB_ENV
echo "NVHPCSDK=/opt/nvidia/hpc_sdk" >> $GITHUB_ENV
- echo "OMPI_CXX=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin/nvc++" >> $GITHUB_ENV
- echo "OMPI_CC=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin/nvc" >> $GITHUB_ENV
- echo "OMPI_FC=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin/nvfortran" >> $GITHUB_ENV
- echo "LD_LIBRARY_PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda/12.2/lib64:/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/lib" >> $GITHUB_ENV
+ echo "OMPI_CXX=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvc++" >> $GITHUB_ENV
+ echo "OMPI_CC=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvc" >> $GITHUB_ENV
+ echo "OMPI_FC=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvfortran" >> $GITHUB_ENV
+ echo "LD_LIBRARY_PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/cuda/12.2/lib64:/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/lib" >> $GITHUB_ENV
echo "DESTDIR=/tmp" >> $GITHUB_ENV
- name: CMake Configure with nvc
shell: bash
run: |
- export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin:$PATH
+ export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/comm_libs/openmpi4/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin:$PATH
mkdir "${{ runner.workspace }}/build"
cd "${{ runner.workspace }}/build"
cmake -C $GITHUB_WORKSPACE/config/cmake/cacheinit.cmake -G Ninja \
From 61982b60fd5e3a29044a1cdd1931b0ab9f96dec1 Mon Sep 17 00:00:00 2001
From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com>
Date: Thu, 2 Nov 2023 07:55:50 -0500
Subject: [PATCH 50/52] Update some doxygen links to local refs (#3814)
---
doxygen/dox/LearnBasics2.dox | 6 +++---
doxygen/dox/LearnBasics3.dox | 8 ++++----
doxygen/dox/Overview.dox | 4 ++--
doxygen/dox/ViewTools.dox | 8 ++++----
doxygen/dox/ViewTools2.dox | 2 +-
5 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/doxygen/dox/LearnBasics2.dox b/doxygen/dox/LearnBasics2.dox
index 87bbe87fe47..f436a027d81 100644
--- a/doxygen/dox/LearnBasics2.dox
+++ b/doxygen/dox/LearnBasics2.dox
@@ -728,12 +728,12 @@ There are numerous datatype functions that allow a user to alter a pre-defined d
Refer to the \ref H5T in the \ref RM. Example functions are #H5Tset_size and #H5Tset_precision.
\section secLBDtypeSpec Specific Datatypes
-On the Examples by API
-page under Datatypes
+On the \ref ExAPI
+page under \ref sec_exapi_dtypes
you will find many example programs for creating and reading datasets with different datatypes.
Below is additional information on some of the datatypes. See
-the Examples by API
+the \ref ExAPI
page for examples of these datatypes.
\subsection subsecLBDtypeSpec Array Datatype vs Array Dataspace
diff --git a/doxygen/dox/LearnBasics3.dox b/doxygen/dox/LearnBasics3.dox
index 748745827f2..a91368b00fc 100644
--- a/doxygen/dox/LearnBasics3.dox
+++ b/doxygen/dox/LearnBasics3.dox
@@ -152,13 +152,13 @@ To use the compact storage layout, call: #H5Pset_layout
\li Create a dataset with the modified property list. (See #H5Dcreate)
\li Close the property list. (See #H5Pclose)
For example code, see the \ref HDF5Examples page.
-Specifically look at the Examples by API.
+Specifically look at the \ref ExAPI.
There are examples for different languages.
The C example to create a chunked dataset is:
-h5ex_d_chunk.c
+h5ex_d_chunk.c
The C example to create a compact dataset is:
-h5ex_d_compact.c
+h5ex_d_compact.c
\section secLBDsetLayoutChange Changing the Layout after Dataset Creation
The dataset layout is a Dataset Creation Property List. This means that once the dataset has been
@@ -290,7 +290,7 @@ is met, at a certain point in the future.)
\subsection subsecLBContentsProgUsing Using #H5Literate, #H5Lvisit and #H5Ovisit
For example code, see the \ref HDF5Examples page.
-Specifically look at the Examples by API.
+Specifically look at the \ref ExAPI.
There are examples for different languages, where examples of using #H5Literate and #H5Ovisit/#H5Lvisit are included.
The h5ex_g_traverse example traverses a file using H5Literate:
diff --git a/doxygen/dox/Overview.dox b/doxygen/dox/Overview.dox
index 54cc638d9d6..438788eb028 100644
--- a/doxygen/dox/Overview.dox
+++ b/doxygen/dox/Overview.dox
@@ -23,8 +23,8 @@ documents cover a mix of tasks, concepts, and reference, to help a specific
\par Versions
Version-specific documentation (see the version in the title area) can be found
here:
- - HDF5 develop branch (this site)
- - HDF5 1.14.x
+ - HDF5 develop branch (this site)
+ - HDF5 1.14.x
- HDF5 1.12.x
- HDF5 1.10.x
- HDF5 1.8.x
diff --git a/doxygen/dox/ViewTools.dox b/doxygen/dox/ViewTools.dox
index 66b2def0624..951605674be 100644
--- a/doxygen/dox/ViewTools.dox
+++ b/doxygen/dox/ViewTools.dox
@@ -53,7 +53,7 @@ packages, which can be obtained from the HDF-EOS and Tools and
Information Center. Specifically, the following examples are used in this tutorial topic:
\li HDF5 Files created from compiling the \ref LBExamples
-\li HDF5 Files on the Examples by API page
+\li HDF5 Files on the \ref ExAPI page
\li NPP JPSS files, SVM01_npp.. (gzipped)
and SVM09_npp.. (gzipped)
\li HDF-EOS OMI-Aura file
@@ -497,7 +497,7 @@ In the file shown below the dataset /
\endcode
You can obtain the h5ex_d_gzip.c program that created this file, as well as the file created,
-from the Examples by API page.
+from the \ref ExAPI page.
\subsection subsecViewToolsViewDset_h5ls h5ls
Specific datasets can be specified with h5ls by simply adding the dataset path and dataset after the
@@ -925,7 +925,7 @@ The output of the above command is shown below:
\subsection subsecViewToolsViewDtypes_h5dump h5dump
The following datatypes are discussed, using the output of h5dump with HDF5 files from the
-Examples by API page:
+\ref ExAPI page:
- @ref subsubsecViewToolsViewDtypes_array
- @ref subsubsecViewToolsViewDtypes_objref
@@ -1036,7 +1036,7 @@ the elements or slab that is selected. A dataset with a Region Reference datatyp
one or more Region References.
An example of a Region Reference dataset (h5ex_t_regref.h5) can be found on the
-Examples by API page,
+\ref ExAPI page,
under Datatypes. If you examine this dataset with h5dump you will see that /DS1 is a
Region Reference dataset as indicated by its datatype, highlighted in bold below:
\code
diff --git a/doxygen/dox/ViewTools2.dox b/doxygen/dox/ViewTools2.dox
index 4d8788a81fc..71d74652d4a 100644
--- a/doxygen/dox/ViewTools2.dox
+++ b/doxygen/dox/ViewTools2.dox
@@ -727,7 +727,7 @@ GROUP "/" {
\endcode
The second example imports string data. The example program that creates this file can be downloaded
-from the Examples by API page.
+from the \ref ExAPI page.
Note that string data requires use of the h5dump -y
option to exclude indexes and the h5dump --width=1
From 7404b57da68e92bd28c5da2053830e7cbfe032d1 Mon Sep 17 00:00:00 2001
From: jhendersonHDF
Date: Thu, 2 Nov 2023 21:42:28 -0500
Subject: [PATCH 51/52] Rework MPI Info FAPL preserve PR to use VFD 'ctl'
operations (#3782)
---
src/H5FDmpi.c | 48 +++++++++++++++++++++++++++----
src/H5FDmpio.c | 7 +++++
src/H5FDprivate.h | 1 +
src/H5FDpublic.h | 1 +
src/H5FDsubfiling/H5FDsubfiling.c | 6 ++++
src/H5Fint.c | 25 ++++------------
src/H5Fmpi.c | 34 +++++++++++++++++++---
src/H5Fpkg.h | 1 -
src/H5Fprivate.h | 1 +
9 files changed, 95 insertions(+), 29 deletions(-)
diff --git a/src/H5FDmpi.c b/src/H5FDmpi.c
index 127740efe3b..f247c3478f8 100644
--- a/src/H5FDmpi.c
+++ b/src/H5FDmpi.c
@@ -104,13 +104,12 @@ H5FD_mpi_get_size(H5FD_t *file)
} /* end H5FD_mpi_get_size() */
/*-------------------------------------------------------------------------
- * Function: H5FD_mpi_get_comm
+ * Function: H5FD_mpi_get_comm
*
- * Purpose: Retrieves the file's communicator
+ * Purpose: Retrieves the file's MPI_Comm communicator object
*
- * Return: Success: The communicator (non-negative)
- *
- * Failure: Negative
+ * Return: Success: The communicator object
+ * Failure: MPI_COMM_NULL
*
*-------------------------------------------------------------------------
*/
@@ -143,6 +142,45 @@ H5FD_mpi_get_comm(H5FD_t *file)
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_mpi_get_comm() */
+/*-------------------------------------------------------------------------
+ * Function: H5FD_mpi_get_info
+ *
+ * Purpose: Retrieves the file's MPI_Info info object
+ *
+ * Return: Success: The info object
+ * Failure: MPI_INFO_NULL
+ *
+ *-------------------------------------------------------------------------
+ */
+MPI_Info
+H5FD_mpi_get_info(H5FD_t *file)
+{
+ const H5FD_class_t *cls;
+ uint64_t flags = H5FD_CTL_FAIL_IF_UNKNOWN_FLAG | H5FD_CTL_ROUTE_TO_TERMINAL_VFD_FLAG;
+ MPI_Info info = MPI_INFO_NULL;
+ void *info_ptr = (void *)(&info);
+ MPI_Info ret_value;
+
+ FUNC_ENTER_NOAPI(MPI_INFO_NULL)
+
+ assert(file);
+ cls = (const H5FD_class_t *)(file->cls);
+ assert(cls);
+ assert(cls->ctl); /* All MPI drivers must implement this */
+
+ /* Dispatch to driver */
+ if ((cls->ctl)(file, H5FD_CTL_GET_MPI_INFO_OPCODE, flags, NULL, &info_ptr) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTGET, MPI_INFO_NULL, "driver get_info request failed");
+
+ if (info == MPI_INFO_NULL)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTGET, MPI_INFO_NULL, "driver get_info request failed -- bad info object");
+
+ ret_value = info;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_mpi_get_info() */
+
/*-------------------------------------------------------------------------
* Function: H5FD_mpi_MPIOff_to_haddr
*
diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c
index 83a5ad45a9a..d5dd1261178 100644
--- a/src/H5FDmpio.c
+++ b/src/H5FDmpio.c
@@ -3795,6 +3795,7 @@ H5FD__mpio_delete(const char *filename, hid_t fapl_id)
* At present, the supported op codes are:
*
* H5FD_CTL_GET_MPI_COMMUNICATOR_OPCODE
+ * H5FD_CTL_GET_MPI_INFO_OPCODE
* H5FD_CTL_GET_MPI_RANK_OPCODE
* H5FD_CTL_GET_MPI_SIZE_OPCODE
* H5FD_CTL_GET_MPI_FILE_SYNC_OPCODE
@@ -3827,6 +3828,12 @@ H5FD__mpio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_AT
**((MPI_Comm **)output) = file->comm;
break;
+ case H5FD_CTL_GET_MPI_INFO_OPCODE:
+ assert(output);
+ assert(*output);
+ **((MPI_Info **)output) = file->info;
+ break;
+
case H5FD_CTL_GET_MPI_RANK_OPCODE:
assert(output);
assert(*output);
diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h
index 5330077565b..2fe54a588a9 100644
--- a/src/H5FDprivate.h
+++ b/src/H5FDprivate.h
@@ -214,6 +214,7 @@ H5_DLL herr_t H5FD_get_mpio_atomicity(H5FD_t *file, bool *flag);
H5_DLL int H5FD_mpi_get_rank(H5FD_t *file);
H5_DLL int H5FD_mpi_get_size(H5FD_t *file);
H5_DLL MPI_Comm H5FD_mpi_get_comm(H5FD_t *file);
+H5_DLL MPI_Info H5FD_mpi_get_info(H5FD_t *file);
H5_DLL herr_t H5FD_mpi_get_file_sync_required(H5FD_t *file, bool *file_sync_required);
#endif /* H5_HAVE_PARALLEL */
diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h
index 5f40bff6845..d8d77d6534b 100644
--- a/src/H5FDpublic.h
+++ b/src/H5FDpublic.h
@@ -179,6 +179,7 @@
#define H5FD_CTL_INVALID_OPCODE 0
#define H5FD_CTL_TEST_OPCODE 1
#define H5FD_CTL_GET_MPI_COMMUNICATOR_OPCODE 2
+#define H5FD_CTL_GET_MPI_INFO_OPCODE 9
#define H5FD_CTL_GET_MPI_RANK_OPCODE 3
#define H5FD_CTL_GET_MPI_SIZE_OPCODE 4
#define H5FD_CTL_MEM_ALLOC 5
diff --git a/src/H5FDsubfiling/H5FDsubfiling.c b/src/H5FDsubfiling/H5FDsubfiling.c
index a2daba0d01b..461fa16cfe7 100644
--- a/src/H5FDsubfiling/H5FDsubfiling.c
+++ b/src/H5FDsubfiling/H5FDsubfiling.c
@@ -2551,6 +2551,12 @@ H5FD__subfiling_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void
**((MPI_Comm **)output) = file->ext_comm;
break;
+ case H5FD_CTL_GET_MPI_INFO_OPCODE:
+ assert(output);
+ assert(*output);
+ **((MPI_Info **)output) = file->info;
+ break;
+
case H5FD_CTL_GET_MPI_RANK_OPCODE:
assert(output);
assert(*output);
diff --git a/src/H5Fint.c b/src/H5Fint.c
index 439fa4f35a6..8738026d7c9 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -402,6 +402,7 @@ H5F_get_access_plist(H5F_t *f, bool app_ref)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set collective metadata read flag");
if (H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) {
MPI_Comm mpi_comm;
+ MPI_Info mpi_info;
/* Retrieve and set MPI communicator */
if (MPI_COMM_NULL == (mpi_comm = H5F_mpi_get_comm(f)))
@@ -409,9 +410,11 @@ H5F_get_access_plist(H5F_t *f, bool app_ref)
if (H5P_set(new_plist, H5F_ACS_MPI_PARAMS_COMM_NAME, &mpi_comm) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set MPI communicator");
- /* Retrieve MPI info object */
- if (H5P_set(new_plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &(f->shared->mpi_info)) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set MPI info object");
+ /* Retrieve and set MPI info */
+ if (MPI_INFO_NULL == (mpi_info = H5F_mpi_get_info(f)))
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, H5I_INVALID_HID, "can't get MPI info");
+ if (H5P_set(new_plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &mpi_info) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set MPI info");
}
#endif /* H5_HAVE_PARALLEL */
if (H5P_set(new_plist, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, &(f->shared->mdc_initCacheImageCfg)) <
@@ -1130,12 +1133,6 @@ H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F
/* initialize point of no return */
f->shared->point_of_no_return = false;
-#ifdef H5_HAVE_PARALLEL
- /* Initialize this just in case we fail before setting this field and */
- /* we try to call H5_mpi_info_free() on uninitialized memory in H5F__dest() */
- f->shared->mpi_info = MPI_INFO_NULL;
-#endif /* H5_HAVE_PARALLEL */
-
/* Copy the file creation and file access property lists into the
* new file handle. We do this early because some values might need
* to change as the file is being opened.
@@ -1212,8 +1209,6 @@ H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get collective metadata read flag");
if (H5P_get(plist, H5F_ACS_COLL_MD_WRITE_FLAG_NAME, &(f->shared->coll_md_write)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get collective metadata write flag");
- if (H5P_get(plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &(f->shared->mpi_info)) < 0)
- HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't set MPI info object");
#endif /* H5_HAVE_PARALLEL */
if (H5P_get(plist, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, &(f->shared->mdc_initCacheImageCfg)) <
0)
@@ -1419,14 +1414,6 @@ H5F__dest(H5F_t *f, bool flush, bool free_on_failure)
f->shared->efc = NULL;
} /* end if */
-#ifdef H5_HAVE_PARALLEL
- if (f->shared->mpi_info != MPI_INFO_NULL) {
- /* Free MPI info saved in the file struct */
- if (H5_mpi_info_free(&f->shared->mpi_info) < 0)
- HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't free MPI info");
- }
-#endif
-
/* With the shutdown modifications, the contents of the metadata cache
* should be clean at this point, with the possible exception of the
* the superblock and superblock extension.
diff --git a/src/H5Fmpi.c b/src/H5Fmpi.c
index 8a8fdc135c8..7a535e90d7d 100644
--- a/src/H5Fmpi.c
+++ b/src/H5Fmpi.c
@@ -97,11 +97,10 @@ H5F_mpi_get_rank(const H5F_t *f)
/*-------------------------------------------------------------------------
* Function: H5F_mpi_get_comm
*
- * Purpose: Retrieves the file's communicator
+ * Purpose: Retrieves the file's MPI_Comm communicator object
*
- * Return: Success: The communicator (non-negative)
- *
- * Failure: Negative
+ * Return: Success: The communicator object
+ * Failure: MPI_COMM_NULL
*
*-------------------------------------------------------------------------
*/
@@ -122,6 +121,33 @@ H5F_mpi_get_comm(const H5F_t *f)
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F_mpi_get_comm() */
+/*-------------------------------------------------------------------------
+ * Function: H5F_mpi_get_info
+ *
+ * Purpose: Retrieves the file's MPI_Info info object
+ *
+ * Return: Success: The info object
+ * Failure: MPI_INFO_NULL
+ *
+ *-------------------------------------------------------------------------
+ */
+MPI_Info
+H5F_mpi_get_info(const H5F_t *f)
+{
+ MPI_Info ret_value = MPI_INFO_NULL;
+
+ FUNC_ENTER_NOAPI(MPI_INFO_NULL)
+
+ assert(f && f->shared);
+
+ /* Dispatch to driver */
+ if ((ret_value = H5FD_mpi_get_info(f->shared->lf)) == MPI_INFO_NULL)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, MPI_INFO_NULL, "driver get_info request failed");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F_mpi_get_info() */
+
/*-------------------------------------------------------------------------
* Function: H5F_shared_mpi_get_size
*
diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h
index e81b25072eb..bc5c90bd5da 100644
--- a/src/H5Fpkg.h
+++ b/src/H5Fpkg.h
@@ -359,7 +359,6 @@ struct H5F_shared_t {
#ifdef H5_HAVE_PARALLEL
H5P_coll_md_read_flag_t coll_md_read; /* Do all metadata reads collectively */
bool coll_md_write; /* Do all metadata writes collectively */
- MPI_Info mpi_info; /* MPI info */
#endif /* H5_HAVE_PARALLEL */
};
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 9adbf3a0258..682e938120c 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -640,6 +640,7 @@ H5_DLL herr_t H5F_eoa_dirty(H5F_t *f);
#ifdef H5_HAVE_PARALLEL
H5_DLL int H5F_mpi_get_rank(const H5F_t *f);
H5_DLL MPI_Comm H5F_mpi_get_comm(const H5F_t *f);
+H5_DLL MPI_Info H5F_mpi_get_info(const H5F_t *f);
H5_DLL int H5F_shared_mpi_get_size(const H5F_shared_t *f_sh);
H5_DLL int H5F_mpi_get_size(const H5F_t *f);
H5_DLL herr_t H5F_mpi_retrieve_comm(hid_t loc_id, hid_t acspl_id, MPI_Comm *mpi_comm);
From 5e787741cdc991eb0011c9393152c1678ca737a8 Mon Sep 17 00:00:00 2001
From: Scot Breitenfeld
Date: Fri, 3 Nov 2023 08:02:50 -0500
Subject: [PATCH 52/52] Removed the use of C wrappers from H5P APIs. (#3824)
* fix seg fault on frontier/cray
* fix seg fault on frontier/cray
* fix seg fault on frontier/cray
* removed the use of h5pclose_c
* removed the use of h5pclose_c
---
fortran/src/H5Off.F90 | 2 +-
fortran/src/H5Pf.c | 70 ------------------------------
fortran/src/H5Pff.F90 | 55 ++++++++++++-----------
fortran/src/H5f90proto.h | 4 --
fortran/src/hdf5_fortrandll.def.in | 2 -
fortran/testpar/Makefile.am | 2 +-
6 files changed, 32 insertions(+), 103 deletions(-)
diff --git a/fortran/src/H5Off.F90 b/fortran/src/H5Off.F90
index b705ba324d7..9c8b09141b8 100644
--- a/fortran/src/H5Off.F90
+++ b/fortran/src/H5Off.F90
@@ -1263,7 +1263,7 @@ SUBROUTINE h5oget_info_by_idx_f(loc_id, group_name, index_field, order, n, &
INTERFACE
INTEGER FUNCTION h5oget_info_by_idx_c(loc_id, group_name, namelen, &
index_field, order, n, lapl_id_default, object_info, fields) BIND(C, NAME='h5oget_info_by_idx_c')
- IMPORT :: c_char, c_ptr, c_funptr
+ IMPORT :: c_char, c_ptr
IMPORT :: HID_T, SIZE_T, HSIZE_T
INTEGER(HID_T) , INTENT(IN) :: loc_id
CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: group_name
diff --git a/fortran/src/H5Pf.c b/fortran/src/H5Pf.c
index 3a97d7eab4a..87e6bfbebe6 100644
--- a/fortran/src/H5Pf.c
+++ b/fortran/src/H5Pf.c
@@ -69,30 +69,6 @@ h5pcreate_c(hid_t_f *cls, hid_t_f *prp_id)
return ret_value;
}
-/****if* H5Pf/h5pclose_c
- * NAME
- * h5pclose_c
- * PURPOSE
- * Call H5Pclose to close property lis
- * INPUTS
- * prp_id - identifier of the property list to be closed
- * RETURNS
- * 0 on success, -1 on failure
- * SOURCE
- */
-
-int_f
-h5pclose_c(hid_t_f *prp_id)
-/******/
-{
- int_f ret_value = 0;
-
- if (H5Pclose((hid_t)*prp_id) < 0)
- ret_value = -1;
-
- return ret_value;
-}
-
/****if* H5Pf/h5pcopy_c
* NAME
* h5pcopy_c
@@ -2252,52 +2228,6 @@ h5pget_hyper_vector_size_c(hid_t_f *prp_id, size_t_f *size)
return ret_value;
}
-/****if* H5Pf/h5pcreate_class_c
- * NAME
- * h5pcreate_class_c
- * PURPOSE
- * Call H5Pcreate_class ito create a new property class
- * INPUTS
- * parent - property list class identifier
- * name - name of the new class
- * name_len - length of the "name" buffer
- * OUTPUTS
- * class - new class identifier
- * RETURNS
- * 0 on success, -1 on failure
- * SOURCE
- */
-int_f
-h5pcreate_class_c(hid_t_f *parent, _fcd name, int_f *name_len, hid_t_f *cls, H5P_cls_create_func_t create,
- void *create_data, H5P_cls_copy_func_t copy, void *copy_data, H5P_cls_close_func_t close,
- void *close_data)
-/******/
-{
- int ret_value = -1;
- hid_t c_class;
- char *c_name;
-
- c_name = (char *)HD5f2cstring(name, (size_t)*name_len);
- if (c_name == NULL)
- goto DONE;
-
- /*
- * Call H5Pcreate_class function.
- */
- c_class =
- H5Pcreate_class((hid_t)*parent, c_name, create, create_data, copy, copy_data, close, close_data);
-
- if (c_class < 0)
- goto DONE;
- *cls = (hid_t_f)c_class;
- ret_value = 0;
-
-DONE:
- if (c_name != NULL)
- free(c_name);
- return ret_value;
-}
-
/****if* H5Pf/h5pregister_c
* NAME
* h5pregister_c
diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90
index 5821889c3e9..576509534ae 100644
--- a/fortran/src/H5Pff.F90
+++ b/fortran/src/H5Pff.F90
@@ -400,15 +400,16 @@ SUBROUTINE h5pclose_f(prp_id, hdferr)
INTEGER(HID_T), INTENT(IN) :: prp_id
INTEGER, INTENT(OUT) :: hdferr
INTERFACE
- INTEGER FUNCTION h5pclose_c(prp_id) &
- BIND(C,NAME='h5pclose_c')
+ INTEGER(C_INT) FUNCTION H5Pclose(prp_id) &
+ BIND(C,NAME='H5Pclose')
+ IMPORT :: C_INT
IMPORT :: HID_T
IMPLICIT NONE
- INTEGER(HID_T), INTENT(IN) :: prp_id
- END FUNCTION h5pclose_c
+ INTEGER(HID_T), VALUE :: prp_id
+ END FUNCTION H5Pclose
END INTERFACE
- hdferr = h5pclose_c(prp_id)
+ hdferr = INT(H5Pclose(prp_id))
END SUBROUTINE h5pclose_f
!>
@@ -5005,31 +5006,32 @@ SUBROUTINE h5pcreate_class_f(parent, name, class, hdferr, create, create_data, &
INTEGER , INTENT(OUT) :: hdferr
TYPE(C_PTR) , OPTIONAL, INTENT(IN) :: create_data, copy_data, close_data
TYPE(C_FUNPTR) , OPTIONAL, INTENT(IN) :: create, copy, close
- INTEGER :: name_len
- TYPE(C_PTR) :: create_data_default, copy_data_default, close_data_default
+ TYPE(C_PTR) :: create_data_default, copy_data_default, close_data_default
TYPE(C_FUNPTR) :: create_default, copy_default, close_default
+
+ CHARACTER(LEN=LEN_TRIM(name)+1,KIND=C_CHAR) :: c_name
+
INTERFACE
- INTEGER FUNCTION h5pcreate_class_c(parent, name, name_len, class, &
+ INTEGER(HID_T) FUNCTION H5Pcreate_class(parent, name, &
create, create_data, copy, copy_data, close, close_data) &
- BIND(C, NAME='h5pcreate_class_c')
- IMPORT :: c_char, c_ptr, c_funptr
+ BIND(C, NAME='H5Pcreate_class')
+ IMPORT :: C_CHAR, C_PTR, C_FUNPTR
IMPORT :: HID_T
- INTEGER(HID_T), INTENT(IN) :: parent
- CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: name
- INTEGER, INTENT(IN) :: name_len
- INTEGER(HID_T), INTENT(OUT) :: class
- TYPE(C_PTR), VALUE :: create_data, copy_data, close_data
- TYPE(C_FUNPTR), VALUE :: create, copy, close
- END FUNCTION h5pcreate_class_c
+ INTEGER(HID_T), VALUE :: parent
+ CHARACTER(KIND=C_CHAR), DIMENSION(*) :: name
+ TYPE(C_PTR), VALUE :: create_data, copy_data, close_data
+ TYPE(C_FUNPTR), VALUE :: create, copy, close
+ END FUNCTION H5Pcreate_class
END INTERFACE
- name_len = LEN(name)
- create_default = c_null_funptr !fix:scot
- create_data_default = c_null_ptr
- copy_default = c_null_funptr !fix:scot
- copy_data_default = c_null_ptr
- close_default = c_null_funptr !fix:scot
- close_data_default = c_null_ptr
+ c_name = TRIM(name)//C_NULL_CHAR
+
+ create_default = C_NULL_FUNPTR
+ create_data_default = C_NULL_PTR
+ copy_default = C_NULL_FUNPTR
+ copy_data_default = C_NULL_PTR
+ close_default = C_NULL_FUNPTR
+ close_data_default = C_NULL_PTR
IF(PRESENT(create)) create_default = create
IF(PRESENT(create_data)) create_data_default = create_data
@@ -5038,11 +5040,14 @@ END FUNCTION h5pcreate_class_c
IF(PRESENT(close)) close_default = close
IF(PRESENT(close_data)) close_data_default = close_data
- hdferr = h5pcreate_class_c(parent, name , name_len, class, &
+ class = H5Pcreate_class(parent, c_name, &
create_default, create_data_default, &
copy_default, copy_data_default, &
close_default, close_data_default)
+ hdferr = 0
+ IF(class.LT.0) hdferr = -1
+
END SUBROUTINE h5pcreate_class_f
!>
diff --git a/fortran/src/H5f90proto.h b/fortran/src/H5f90proto.h
index 28a4fa66e7b..0fe1b2017a1 100644
--- a/fortran/src/H5f90proto.h
+++ b/fortran/src/H5f90proto.h
@@ -368,7 +368,6 @@ H5_FCDLL int_f h5otoken_cmp_c(hid_t_f *loc_id, H5O_token_t *token1, H5O_token_t
* Functions from H5Pf.c
*/
H5_FCDLL int_f h5pcreate_c(hid_t_f *cls, hid_t_f *prp_id);
-H5_FCDLL int_f h5pclose_c(hid_t_f *prp_id);
H5_FCDLL int_f h5pcopy_c(hid_t_f *prp_id, hid_t_f *new_prp_id);
H5_FCDLL int_f h5pequal_c(hid_t_f *plist1_id, hid_t_f *plist2_id, int_f *c_flag);
H5_FCDLL int_f h5pget_class_c(hid_t_f *prp_id, hid_t_f *classtype);
@@ -451,9 +450,6 @@ H5_FCDLL int_f h5pset_small_data_block_size_c(hid_t_f *plist, hsize_t_f *size);
H5_FCDLL int_f h5pget_small_data_block_size_c(hid_t_f *plist, hsize_t_f *size);
H5_FCDLL int_f h5pset_hyper_vector_size_c(hid_t_f *plist, size_t_f *size);
H5_FCDLL int_f h5pget_hyper_vector_size_c(hid_t_f *plist, size_t_f *size);
-H5_FCDLL int_f h5pcreate_class_c(hid_t_f *parent, _fcd name, int_f *name_len, hid_t_f *cls,
- H5P_cls_create_func_t create, void *create_data, H5P_cls_copy_func_t copy,
- void *copy_data, H5P_cls_close_func_t close, void *close_data);
H5_FCDLL int_f h5pregister_c(hid_t_f *cls, _fcd name, int_f *name_len, size_t_f *size, void *value);
H5_FCDLL int_f h5pinsert_c(hid_t_f *plist, _fcd name, int_f *name_len, size_t_f *size, void *value);
H5_FCDLL int_f h5pset_c(hid_t_f *prp_id, _fcd name, int_f *name_len, void *value);
diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in
index 2ded00222ba..00801776d4d 100644
--- a/fortran/src/hdf5_fortrandll.def.in
+++ b/fortran/src/hdf5_fortrandll.def.in
@@ -258,7 +258,6 @@ H5P_mp_H5PSET_PRESERVE_F
H5P_mp_H5PGET_PRESERVE_F
H5P_mp_H5PGET_CLASS_F
H5P_mp_H5PCOPY_F
-H5P_mp_H5PCLOSE_F
H5P_mp_H5PSET_CHUNK_F
H5P_mp_H5PGET_CHUNK_F
H5P_mp_H5PSET_DEFLATE_F
@@ -331,7 +330,6 @@ H5P_mp_H5PCOPY_PROP_F
H5P_mp_H5PREMOVE_F
H5P_mp_H5PUNREGISTER_F
H5P_mp_H5PCLOSE_CLASS_F
-H5P_mp_H5PCREATE_CLASS_F
H5P_mp_H5PREGISTER_INTEGER
H5P_mp_H5PREGISTER_CHAR
H5P_mp_H5PINSERT_CHAR
diff --git a/fortran/testpar/Makefile.am b/fortran/testpar/Makefile.am
index afdda980c5c..1c374090601 100644
--- a/fortran/testpar/Makefile.am
+++ b/fortran/testpar/Makefile.am
@@ -36,7 +36,7 @@ TEST_PROG_PARA=parallel_test subfiling_test async_test
check_PROGRAMS=$(TEST_PROG_PARA)
# Temporary files
-CHECK_CLEANFILES+=parf[12].h5 h5*_tests.h5 subf.h5* test_async_apis.mod
+CHECK_CLEANFILES+=parf[12].h5 h5*_tests.h5 subf.h5* *.mod
# Test source files
parallel_test_SOURCES=ptest.F90 hyper.F90 mdset.F90 multidsetrw.F90