From 136739b37a9e1ecefe83883f0f422d283ae497e5 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Wed, 27 Mar 2024 13:12:32 -0700 Subject: [PATCH] File format security issues (#4234) --- src/H5Adense.c | 26 +- src/H5Aint.c | 109 ++++--- src/H5Apkg.h | 5 +- src/H5Dint.c | 7 + src/H5Dio.c | 6 +- src/H5Fint.c | 4 + src/H5Fpkg.h | 1 + src/H5Fprivate.h | 6 +- src/H5Fpublic.h | 14 + src/H5Fquery.c | 28 +- src/H5HG.c | 37 ++- src/H5Oainfo.c | 6 + src/H5Oattribute.c | 33 ++- src/H5Odtype.c | 72 ++++- src/H5Oefl.c | 6 +- src/H5Oint.c | 20 ++ src/H5Olayout.c | 8 +- src/H5Olinfo.c | 5 + src/H5Opkg.h | 2 + src/H5Oprivate.h | 1 + src/H5Pencdec.c | 71 +++++ src/H5Pfapl.c | 84 ++++++ src/H5Ppkg.h | 2 + src/H5Ppublic.h | 73 +++++ src/H5Rint.c | 34 ++- src/H5S.c | 4 + src/H5Shyper.c | 22 +- src/H5Spkg.h | 9 - src/H5Spoint.c | 36 ++- src/H5Sselect.c | 2 +- src/H5T.c | 49 +++ src/H5Tcommit.c | 70 +++-- src/H5Tconv.c | 196 +++++++++--- src/H5Tprivate.h | 3 +- src/H5Tvlen.c | 2 +- src/H5Zscaleoffset.c | 2 + src/H5encode.h | 4 +- src/H5private.h | 26 +- test/CMakeTests.cmake | 2 + test/testfiles/tmisc38a.h5 | Bin 0 -> 72368 bytes test/testfiles/tmisc38b.h5 | Bin 0 -> 1400 bytes test/tmisc.c | 280 ++++++++++++++++-- tools/lib/h5tools_str.c | 82 ++--- tools/test/h5dump/CMakeTests.cmake | 2 +- tools/test/h5dump/expected/tnbit.ddl | 4 +- tools/test/h5dump/expected/treadintfilter.ddl | 42 +-- tools/test/h5dump/h5dumpgentest.c | 2 +- tools/test/h5dump/testfiles/tfilters.h5 | Bin 46496 -> 46696 bytes tools/test/h5dump/testh5dump.sh.in | 2 +- .../test/h5stat/expected/h5stat_filters-F.ddl | 6 +- .../h5stat/expected/h5stat_filters-UD.ddl | 2 +- .../test/h5stat/expected/h5stat_filters-d.ddl | 2 +- .../h5stat/expected/h5stat_filters-dT.ddl | 2 +- tools/test/h5stat/expected/h5stat_filters.ddl | 16 +- tools/test/h5stat/testfiles/h5stat_filters.h5 | Bin 46272 -> 46696 bytes 55 files changed, 1234 insertions(+), 295 deletions(-) create mode 100644 test/testfiles/tmisc38a.h5 create mode 100644 test/testfiles/tmisc38b.h5 diff --git a/src/H5Adense.c b/src/H5Adense.c index 80c3c94f733..48004d2aa70 100644 --- a/src/H5Adense.c +++ b/src/H5Adense.c @@ -1104,12 +1104,12 @@ H5A__dense_iterate(H5F_t *f, hid_t loc_id, const H5O_ainfo_t *ainfo, H5_index_t H5_iter_order_t order, hsize_t skip, hsize_t *last_attr, const H5A_attr_iter_op_t *attr_op, void *op_data) { - H5HF_t *fheap = NULL; /* Fractal heap handle */ - H5HF_t *shared_fheap = NULL; /* Fractal heap handle for shared header messages */ - H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */ - H5B2_t *bt2 = NULL; /* v2 B-tree handle for index */ - haddr_t bt2_addr; /* Address of v2 B-tree to use for lookup */ - herr_t ret_value = FAIL; /* Return value */ + H5HF_t *fheap = NULL; /* Fractal heap handle */ + H5HF_t *shared_fheap = NULL; /* Fractal heap handle for shared header messages */ + H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */ + H5B2_t *bt2 = NULL; /* v2 B-tree handle for index */ + haddr_t bt2_addr; /* Address of v2 B-tree to use for lookup */ + herr_t ret_value = FAIL; /* Return value */ FUNC_ENTER_PACKAGE @@ -1499,12 +1499,12 @@ herr_t H5A__dense_remove_by_idx(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type, H5_iter_order_t order, hsize_t n) { - H5HF_t *fheap = NULL; /* Fractal heap handle */ - H5HF_t *shared_fheap = NULL; /* Fractal heap handle for shared header messages */ - H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */ - H5B2_t *bt2 = NULL; /* v2 B-tree handle for index */ - haddr_t bt2_addr; /* Address of v2 B-tree to use for operation */ - herr_t ret_value = SUCCEED; /* Return value */ + H5HF_t *fheap = NULL; /* Fractal heap handle */ + H5HF_t *shared_fheap = NULL; /* Fractal heap handle for shared header messages */ + H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */ + H5B2_t *bt2 = NULL; /* v2 B-tree handle for index */ + haddr_t bt2_addr; /* Address of v2 B-tree to use for operation */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -1586,7 +1586,7 @@ H5A__dense_remove_by_idx(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "error building table of attributes"); /* Check for skipping too many attributes */ - if (n >= atable.nattrs) + if (n >= atable.num_attrs) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid index specified"); /* Delete appropriate attribute from dense storage */ diff --git a/src/H5Aint.c b/src/H5Aint.c index 5ba02163f87..45df0dfbe3e 100644 --- a/src/H5Aint.c +++ b/src/H5Aint.c @@ -51,16 +51,9 @@ typedef struct { H5F_t *f; /* Pointer to file that fractal heap is in */ H5A_attr_table_t *atable; /* Pointer to attribute table to build */ - size_t curr_attr; /* Current attribute to operate on */ bool bogus_crt_idx; /* Whether bogus creation index values need to be set */ } H5A_compact_bt_ud_t; -/* Data exchange structure to use when building table of dense attributes for an object */ -typedef struct { - H5A_attr_table_t *atable; /* Pointer to attribute table to build */ - size_t curr_attr; /* Current attribute to operate on */ -} H5A_dense_bt_ud_t; - /* Data exchange structure to use when copying an attribute from _SRC to _DST */ typedef struct { const H5O_ainfo_t *ainfo; /* dense information */ @@ -1455,30 +1448,31 @@ H5A__compact_build_table_cb(H5O_t H5_ATTR_UNUSED *oh, H5O_mesg_t *mesg /*in,out* assert(mesg); /* Re-allocate the table if necessary */ - if (udata->curr_attr == udata->atable->nattrs) { + if (udata->atable->num_attrs == udata->atable->max_attrs) { H5A_t **new_table; /* New table for attributes */ size_t new_table_size; /* Number of attributes in new table */ /* Allocate larger table */ - new_table_size = MAX(1, 2 * udata->atable->nattrs); + new_table_size = MAX(1, 2 * udata->atable->max_attrs); if (NULL == (new_table = (H5A_t **)H5FL_SEQ_REALLOC(H5A_t_ptr, udata->atable->attrs, new_table_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5_ITER_ERROR, "unable to extend attribute table"); /* Update table information in user data */ - udata->atable->attrs = new_table; - udata->atable->nattrs = new_table_size; + udata->atable->attrs = new_table; + udata->atable->max_attrs = new_table_size; } /* end if */ /* Copy attribute into table */ - if (NULL == (udata->atable->attrs[udata->curr_attr] = H5A__copy(NULL, (const H5A_t *)mesg->native))) + if (NULL == + (udata->atable->attrs[udata->atable->num_attrs] = H5A__copy(NULL, (const H5A_t *)mesg->native))) HGOTO_ERROR(H5E_ATTR, H5E_CANTCOPY, H5_ITER_ERROR, "can't copy attribute"); /* Assign [somewhat arbitrary] creation order value, if requested */ if (udata->bogus_crt_idx) - ((udata->atable->attrs[udata->curr_attr])->shared)->crt_idx = sequence; + ((udata->atable->attrs[udata->atable->num_attrs])->shared)->crt_idx = sequence; - /* Increment current attribute */ - udata->curr_attr++; + /* Increment attribute count */ + udata->atable->num_attrs++; done: FUNC_LEAVE_NOAPI(ret_value) @@ -1501,9 +1495,10 @@ herr_t H5A__compact_build_table(H5F_t *f, H5O_t *oh, H5_index_t idx_type, H5_iter_order_t order, H5A_attr_table_t *atable) { - H5A_compact_bt_ud_t udata; /* User data for iteration callback */ - H5O_mesg_operator_t op; /* Wrapper for operator */ - herr_t ret_value = SUCCEED; /* Return value */ + H5A_compact_bt_ud_t udata; /* User data for iteration callback */ + H5O_mesg_operator_t op; /* Wrapper for operator */ + bool iter_set_up = false; /* Is everything set up for iteration */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -1513,13 +1508,13 @@ H5A__compact_build_table(H5F_t *f, H5O_t *oh, H5_index_t idx_type, H5_iter_order assert(atable); /* Initialize table */ - atable->attrs = NULL; - atable->nattrs = 0; + atable->attrs = NULL; + atable->num_attrs = 0; + atable->max_attrs = 0; /* Set up user data for iteration */ - udata.f = f; - udata.atable = atable; - udata.curr_attr = 0; + udata.f = f; + udata.atable = atable; udata.bogus_crt_idx = (bool)((oh->version == H5O_VERSION_1 || !(oh->flags & H5O_HDR_ATTR_CRT_ORDER_TRACKED)) ? true : false); @@ -1527,20 +1522,23 @@ H5A__compact_build_table(H5F_t *f, H5O_t *oh, H5_index_t idx_type, H5_iter_order /* Iterate over existing attributes, checking for attribute with same name */ op.op_type = H5O_MESG_OP_LIB; op.u.lib_op = H5A__compact_build_table_cb; + iter_set_up = true; if (H5O__msg_iterate_real(f, oh, H5O_MSG_ATTR, &op, &udata) < 0) HGOTO_ERROR(H5E_ATTR, H5E_BADITER, FAIL, "error building attribute table"); - /* Correct # of attributes in table */ - atable->nattrs = udata.curr_attr; - /* Don't sort an empty table. */ - if (atable->nattrs > 0) { + if (atable->num_attrs > 0) /* Sort attribute table in correct iteration order */ if (H5A__attr_sort_table(atable, idx_type, order) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTSORT, FAIL, "error sorting attribute table"); - } /* end if */ done: + if (ret_value < 0) + /* Clean up partially built table on error */ + if (iter_set_up) + if (atable->attrs && H5A__attr_release_table(atable) < 0) + HDONE_ERROR(H5E_ATTR, H5E_CANTFREE, FAIL, "unable to release attribute table"); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5A__compact_build_table() */ @@ -1557,26 +1555,26 @@ H5A__compact_build_table(H5F_t *f, H5O_t *oh, H5_index_t idx_type, H5_iter_order static herr_t H5A__dense_build_table_cb(const H5A_t *attr, void *_udata) { - H5A_dense_bt_ud_t *udata = (H5A_dense_bt_ud_t *)_udata; /* 'User data' passed in */ - herr_t ret_value = H5_ITER_CONT; /* Return value */ + H5A_attr_table_t *atable = (H5A_attr_table_t *)_udata; /* 'User data' passed in */ + herr_t ret_value = H5_ITER_CONT; /* Return value */ FUNC_ENTER_PACKAGE /* check arguments */ assert(attr); - assert(udata); - assert(udata->curr_attr < udata->atable->nattrs); + assert(atable); + assert(atable->num_attrs < atable->max_attrs); /* Allocate attribute for entry in the table */ - if (NULL == (udata->atable->attrs[udata->curr_attr] = H5FL_CALLOC(H5A_t))) + if (NULL == (atable->attrs[atable->num_attrs] = H5FL_CALLOC(H5A_t))) HGOTO_ERROR(H5E_ATTR, H5E_CANTALLOC, H5_ITER_ERROR, "can't allocate attribute"); /* Copy attribute information. Share the attribute object in copying. */ - if (NULL == H5A__copy(udata->atable->attrs[udata->curr_attr], attr)) + if (NULL == H5A__copy(atable->attrs[atable->num_attrs], attr)) HGOTO_ERROR(H5E_ATTR, H5E_CANTCOPY, H5_ITER_ERROR, "can't copy attribute"); /* Increment number of attributes stored */ - udata->curr_attr++; + atable->num_attrs++; done: FUNC_LEAVE_NOAPI(ret_value) @@ -1622,22 +1620,18 @@ H5A__dense_build_table(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type, if (H5B2_get_nrec(bt2_name, &nrec) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "can't retrieve # of records in index"); - /* Set size of table */ - H5_CHECK_OVERFLOW(nrec, /* From: */ hsize_t, /* To: */ size_t); - atable->nattrs = (size_t)nrec; - /* Allocate space for the table entries */ - if (atable->nattrs > 0) { - H5A_dense_bt_ud_t udata; /* User data for iteration callback */ + if (nrec > 0) { H5A_attr_iter_op_t attr_op; /* Attribute operator */ - /* Allocate the table to store the attributes */ - if ((atable->attrs = (H5A_t **)H5FL_SEQ_CALLOC(H5A_t_ptr, atable->nattrs)) == NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed"); + /* Check for overflow on the downcast */ + H5_CHECK_OVERFLOW(nrec, /* From: */ hsize_t, /* To: */ size_t); - /* Set up user data for iteration */ - udata.atable = atable; - udata.curr_attr = 0; + /* Allocate the table to store the attributes */ + if (NULL == (atable->attrs = (H5A_t **)H5FL_SEQ_CALLOC(H5A_t_ptr, (size_t)nrec))) + HGOTO_ERROR(H5E_ATTR, H5E_CANTALLOC, FAIL, "memory allocation failed"); + atable->num_attrs = 0; + atable->max_attrs = (size_t)nrec; /* Build iterator operator */ attr_op.op_type = H5A_ATTR_OP_LIB; @@ -1645,7 +1639,7 @@ H5A__dense_build_table(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type, /* Iterate over the links in the group, building a table of the link messages */ if (H5A__dense_iterate(f, (hid_t)0, ainfo, H5_INDEX_NAME, H5_ITER_NATIVE, (hsize_t)0, NULL, &attr_op, - &udata) < 0) + atable) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, FAIL, "error building attribute table"); /* Sort attribute table in correct iteration order */ @@ -1791,18 +1785,18 @@ H5A__attr_sort_table(H5A_attr_table_t *atable, H5_index_t idx_type, H5_iter_orde /* Pick appropriate comparison routine */ if (idx_type == H5_INDEX_NAME) { if (order == H5_ITER_INC) - qsort(atable->attrs, atable->nattrs, sizeof(H5A_t *), H5A__attr_cmp_name_inc); + qsort(atable->attrs, atable->num_attrs, sizeof(H5A_t *), H5A__attr_cmp_name_inc); else if (order == H5_ITER_DEC) - qsort(atable->attrs, atable->nattrs, sizeof(H5A_t *), H5A__attr_cmp_name_dec); + qsort(atable->attrs, atable->num_attrs, sizeof(H5A_t *), H5A__attr_cmp_name_dec); else assert(order == H5_ITER_NATIVE); } /* end if */ else { assert(idx_type == H5_INDEX_CRT_ORDER); if (order == H5_ITER_INC) - qsort(atable->attrs, atable->nattrs, sizeof(H5A_t *), H5A__attr_cmp_corder_inc); + qsort(atable->attrs, atable->num_attrs, sizeof(H5A_t *), H5A__attr_cmp_corder_inc); else if (order == H5_ITER_DEC) - qsort(atable->attrs, atable->nattrs, sizeof(H5A_t *), H5A__attr_cmp_corder_dec); + qsort(atable->attrs, atable->num_attrs, sizeof(H5A_t *), H5A__attr_cmp_corder_dec); else assert(order == H5_ITER_NATIVE); } /* end else */ @@ -1839,7 +1833,7 @@ H5A__attr_iterate_table(const H5A_attr_table_t *atable, hsize_t skip, hsize_t *l /* Iterate over attribute messages */ H5_CHECKED_ASSIGN(u, size_t, skip, hsize_t); - for (; u < atable->nattrs && !ret_value; u++) { + for (; u < atable->num_attrs && !ret_value; u++) { /* Check which type of callback to make */ switch (attr_op->op_type) { case H5A_ATTR_OP_APP2: { @@ -1906,19 +1900,20 @@ H5A__attr_release_table(H5A_attr_table_t *atable) assert(atable); /* Release attribute info, if any. */ - if (atable->nattrs > 0) { + if (atable->num_attrs > 0) { size_t u; /* Local index variable */ /* Free attribute message information */ - for (u = 0; u < atable->nattrs; u++) + for (u = 0; u < atable->num_attrs; u++) if (atable->attrs[u] && H5A__close(atable->attrs[u]) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTFREE, FAIL, "unable to release attribute"); + + /* Release array */ + atable->attrs = (H5A_t **)H5FL_SEQ_FREE(H5A_t_ptr, atable->attrs); } /* end if */ else assert(atable->attrs == NULL); - atable->attrs = (H5A_t **)H5FL_SEQ_FREE(H5A_t_ptr, atable->attrs); - done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5A__attr_release_table() */ diff --git a/src/H5Apkg.h b/src/H5Apkg.h index 64549c818c0..239d7550c46 100644 --- a/src/H5Apkg.h +++ b/src/H5Apkg.h @@ -145,8 +145,9 @@ typedef struct H5A_bt2_ud_ins_t { /* Data structure to hold table of attributes for an object */ typedef struct { - size_t nattrs; /* # of attributes in table */ - H5A_t **attrs; /* Pointer to array of attribute pointers */ + size_t num_attrs; /* Curr. # of attributes in table */ + size_t max_attrs; /* Max. # of attributes in table */ + H5A_t **attrs; /* Pointer to array of attribute pointers */ } H5A_attr_table_t; /*****************************/ diff --git a/src/H5Dint.c b/src/H5Dint.c index 8f363ebadbe..37c9fe490a8 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -946,6 +946,13 @@ H5D__update_oh_info(H5F_t *file, H5D_t *dset, hid_t dapl_id) if (NULL == (oh = H5O_pin(oloc))) HGOTO_ERROR(H5E_DATASET, H5E_CANTPIN, FAIL, "unable to pin dataset object header"); + /* Check for creating dataset with unusual datatype */ + if (H5T_is_numeric_with_unusual_unused_bits(type) && + !(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "creating dataset with unusual datatype, see documentation for " + "H5Pset_relax_file_integrity_checks for details."); + /* Write the dataspace header message */ if (H5S_append(file, oh, dset->shared->space) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update dataspace header message"); diff --git a/src/H5Dio.c b/src/H5Dio.c index 62bb48c2a26..312c7fd854b 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -395,8 +395,12 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) H5AC_tag(dset_info[i].dset->oloc.addr, &prev_tag); /* Invoke correct "high level" I/O routine */ - if ((*dset_info[i].io_ops.multi_read)(&io_info, &dset_info[i]) < 0) + if ((*dset_info[i].io_ops.multi_read)(&io_info, &dset_info[i]) < 0) { + /* Reset metadata tagging */ + H5AC_tag(prev_tag, NULL); + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data"); + } /* Reset metadata tagging */ H5AC_tag(prev_tag, NULL); diff --git a/src/H5Fint.c b/src/H5Fint.c index 325947656bf..3f5a1379834 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -438,6 +438,8 @@ H5F_get_access_plist(H5F_t *f, bool app_ref) 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set initial metadata cache resize config."); + if (H5P_set(new_plist, H5F_ACS_RFIC_FLAGS_NAME, &(f->shared->rfic_flags)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set RFIC flags value"); /* Prepare the driver property */ driver_prop.driver_id = f->shared->lf->driver_id; @@ -1230,6 +1232,8 @@ H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F if (H5P_get(plist, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, &(f->shared->mdc_initCacheImageCfg)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get initial metadata cache resize config"); + if (H5P_get(plist, H5F_ACS_RFIC_FLAGS_NAME, &(f->shared->rfic_flags)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get RFIC flags value"); /* Get the VFD values to cache */ f->shared->maxaddr = H5FD_get_maxaddr(lf); diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 60de31ebdfd..7e12ff111d0 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -304,6 +304,7 @@ struct H5F_shared_t { bool use_file_locking; /* Whether or not to use file locking */ bool ignore_disabled_locks; /* Whether or not to ignore disabled file locking */ bool closing; /* File is in the process of being closed */ + uint64_t rfic_flags; /* Relaxed file integrity check (RFIC) flags */ /* Cached VOL connector ID & info */ hid_t vol_id; /* ID of VOL connector for the container */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 682e938120c..d2b1b887a7f 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -101,6 +101,7 @@ typedef struct H5F_t H5F_t; #define H5F_VOL_CLS(F) ((F)->shared->vol_cls) #define H5F_VOL_OBJ(F) ((F)->vol_obj) #define H5F_USE_FILE_LOCKING(F) ((F)->shared->use_file_locking) +#define H5F_RFIC_FLAGS(F) ((F)->shared->rfic_flags) #else /* H5F_MODULE */ #define H5F_LOW_BOUND(F) (H5F_get_low_bound(F)) #define H5F_HIGH_BOUND(F) (H5F_get_high_bound(F)) @@ -165,6 +166,7 @@ typedef struct H5F_t H5F_t; #define H5F_VOL_CLS(F) (H5F_get_vol_cls(F)) #define H5F_VOL_OBJ(F) (H5F_get_vol_obj(F)) #define H5F_USE_FILE_LOCKING(F) (H5F_get_use_file_locking(F)) +#define H5F_RFIC_FLAGS(F) (H5F_get_rfic_flags(F)) #endif /* H5F_MODULE */ /* Macros to encode/decode offset/length's for storing in the file */ @@ -282,6 +284,7 @@ typedef struct H5F_t H5F_t; #define H5F_ACS_MPI_PARAMS_COMM_NAME "mpi_params_comm" /* the MPI communicator */ #define H5F_ACS_MPI_PARAMS_INFO_NAME "mpi_params_info" /* the MPI info struct */ #endif /* H5_HAVE_PARALLEL */ +#define H5F_ACS_RFIC_FLAGS_NAME "rfic_flags" /* Relaxed file integrity check (RFIC) flags */ /* ======================== File Mount properties ====================*/ #define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */ @@ -525,7 +528,8 @@ H5_DLL bool H5F_get_min_dset_ohdr(const H5F_t *f); H5_DLL herr_t H5F_set_min_dset_ohdr(H5F_t *f, bool minimize); H5_DLL const H5VL_class_t *H5F_get_vol_cls(const H5F_t *f); H5_DLL H5VL_object_t *H5F_get_vol_obj(const H5F_t *f); -H5_DLL bool H5F_get_file_locking(const H5F_t *f); +H5_DLL bool H5F_get_use_file_locking(const H5F_t *f); +H5_DLL uint64_t H5F_get_rfic_flags(const H5F_t *f); /* Functions than retrieve values set/cached from the superblock/FCPL */ H5_DLL haddr_t H5F_get_base_addr(const H5F_t *f); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index 45580ac41ec..9dad8852a06 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -242,6 +242,20 @@ typedef struct H5F_retry_info_t { */ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata); +/* + * These are the bits that can be passed to the `flags' argument of + * H5Pset_relax_file_integrity_checks(). Use the bit-wise OR operator (|) to + * combine them as needed. + */ +#define H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS \ + (0x0001u) /**< Suppress errors for numeric datatypes with an unusually \ + * high number of unused bits. See documentation for \ + * H5Pset_relax_file_integrity_checks for details. */ +#define H5F_RFIC_ALL \ + (H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS) /**< Suppress all format integrity check errors. See \ + * documentation for H5Pset_relax_file_integrity_checks \ + * for details. */ + /*********************/ /* Public Prototypes */ /*********************/ diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 44a52c8dbfc..89181dad479 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -1356,16 +1356,16 @@ H5F__get_cont_info(const H5F_t *f, H5VL_file_cont_info_t *info) } /* end H5F_get_cont_info */ /*------------------------------------------------------------------------- - * Function: H5F_get_file_locking + * Function: H5F_get_use_file_locking * - * Purpose: Get the file locking flag for the file + * Purpose: Get the 'use file locking' flag for the file * * Return: true/false * *------------------------------------------------------------------------- */ bool -H5F_get_file_locking(const H5F_t *f) +H5F_get_use_file_locking(const H5F_t *f) { FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1373,7 +1373,7 @@ H5F_get_file_locking(const H5F_t *f) assert(f->shared); FUNC_LEAVE_NOAPI(f->shared->use_file_locking) -} /* end H5F_get_file_locking */ +} /* end H5F_get_use_file_locking */ /*------------------------------------------------------------------------- * Function: H5F_has_vector_select_io @@ -1401,3 +1401,23 @@ H5F_has_vector_select_io(const H5F_t *f, bool is_write) FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_has_vector_select_io */ + +/*------------------------------------------------------------------------- + * Function: H5F_get_rfic_flags + * + * Purpose: Get the relaxed file integrity checks (RFIC) flags for the file + * + * Return: RFIC flags for a file on success (which can be 0), can't fail + * + *------------------------------------------------------------------------- + */ +uint64_t +H5F_get_rfic_flags(const H5F_t *f) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + assert(f); + assert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->rfic_flags) +} /* end H5F_get_rfic_flags */ diff --git a/src/H5HG.c b/src/H5HG.c index 7037376118d..3709c705566 100644 --- a/src/H5HG.c +++ b/src/H5HG.c @@ -559,9 +559,13 @@ H5HG_read(H5F_t *f, H5HG_t *hobj, void *object /*out*/, size_t *buf_size) /* Load the heap */ if (NULL == (heap = H5HG__protect(f, hobj->addr, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, NULL, "unable to protect global heap"); + if (hobj->idx >= heap->nused) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad heap index, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); + if (NULL == heap->obj[hobj->idx].begin) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad heap pointer, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); - assert(hobj->idx < heap->nused); - assert(heap->obj[hobj->idx].begin); size = heap->obj[hobj->idx].size; p = heap->obj[hobj->idx].begin + H5HG_SIZEOF_OBJHDR(f); @@ -631,8 +635,12 @@ H5HG_link(H5F_t *f, const H5HG_t *hobj, int adjust) HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, FAIL, "unable to protect global heap"); if (adjust != 0) { - assert(hobj->idx < heap->nused); - assert(heap->obj[hobj->idx].begin); + if (hobj->idx >= heap->nused) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap index, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); + if (NULL == heap->obj[hobj->idx].begin) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap pointer, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); if ((heap->obj[hobj->idx].nrefs + adjust) < 0) HGOTO_ERROR(H5E_HEAP, H5E_BADRANGE, FAIL, "new link count would be out of range"); if ((heap->obj[hobj->idx].nrefs + adjust) > H5HG_MAXLINK) @@ -678,8 +686,13 @@ H5HG_get_obj_size(H5F_t *f, H5HG_t *hobj, size_t *obj_size) if (NULL == (heap = H5HG__protect(f, hobj->addr, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, FAIL, "unable to protect global heap"); - assert(hobj->idx < heap->nused); - assert(heap->obj[hobj->idx].begin); + /* Sanity check the heap object */ + if (hobj->idx >= heap->nused) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap index, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); + if (NULL == heap->obj[hobj->idx].begin) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap pointer, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); /* Set object size */ *obj_size = heap->obj[hobj->idx].size; @@ -722,14 +735,22 @@ H5HG_remove(H5F_t *f, H5HG_t *hobj) if (NULL == (heap = H5HG__protect(f, hobj->addr, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, FAIL, "unable to protect global heap"); - assert(hobj->idx < heap->nused); + /* Sanity check the heap object (split around bugfix below) */ + if (hobj->idx >= heap->nused) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap index, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); /* When the application selects the same location to rewrite the VL element by using H5Sselect_elements, * it can happen that the entry has been removed by first rewrite. Here we simply skip the removal of * the entry and let the second rewrite happen (see HDFFV-10635). In the future, it'd be nice to handle * this situation in H5T_conv_vlen in H5Tconv.c instead of this level (HDFFV-10648). */ if (heap->obj[hobj->idx].nrefs == 0 && heap->obj[hobj->idx].size == 0 && !heap->obj[hobj->idx].begin) - HGOTO_DONE(ret_value); + HGOTO_DONE(SUCCEED); + + /* Finish sanity checking the heap object */ + if (NULL == heap->obj[hobj->idx].begin) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap pointer, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); obj_start = heap->obj[hobj->idx].begin; /* Include object header size */ diff --git a/src/H5Oainfo.c b/src/H5Oainfo.c index 8b82e39e2a6..8b4340d2392 100644 --- a/src/H5Oainfo.c +++ b/src/H5Oainfo.c @@ -138,19 +138,25 @@ H5O__ainfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUS ainfo->max_crt_idx = H5O_MAX_CRT_ORDER_IDX; /* Address of fractal heap to store "dense" attributes */ + H5_GCC_CLANG_DIAG_OFF("type-limits") if (H5_IS_BUFFER_OVERFLOW(p, sizeof_addr, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + H5_GCC_CLANG_DIAG_ON("type-limits") H5F_addr_decode(f, &p, &(ainfo->fheap_addr)); /* Address of v2 B-tree to index names of attributes (names are always indexed) */ + H5_GCC_CLANG_DIAG_OFF("type-limits") if (H5_IS_BUFFER_OVERFLOW(p, sizeof_addr, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + H5_GCC_CLANG_DIAG_ON("type-limits") H5F_addr_decode(f, &p, &(ainfo->name_bt2_addr)); /* Address of v2 B-tree to index creation order of links, if there is one */ if (ainfo->index_corder) { + H5_GCC_CLANG_DIAG_OFF("type-limits") if (H5_IS_BUFFER_OVERFLOW(p, sizeof_addr, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + H5_GCC_CLANG_DIAG_ON("type-limits") H5F_addr_decode(f, &p, &(ainfo->corder_bt2_addr)); } else diff --git a/src/H5Oattribute.c b/src/H5Oattribute.c index 88c595ea213..64634aac405 100644 --- a/src/H5Oattribute.c +++ b/src/H5Oattribute.c @@ -211,6 +211,13 @@ H5O__attr_create(const H5O_loc_t *loc, H5A_t *attr) if (NULL == (oh = H5O_pin(loc))) HGOTO_ERROR(H5E_ATTR, H5E_CANTPIN, FAIL, "unable to pin object header"); + /* Check for creating attribute with unusual datatype */ + if (H5T_is_numeric_with_unusual_unused_bits(attr->shared->dt) && + !(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(loc->file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS))) + HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, FAIL, + "creating attribute with unusual datatype, see documentation for " + "H5Pset_relax_file_integrity_checks for details."); + /* Check if this object already has attribute information */ if (oh->version > H5O_VERSION_1) { bool new_ainfo = false; /* Flag to indicate that the attribute information is new */ @@ -1171,10 +1178,10 @@ herr_t H5O_attr_iterate_real(hid_t loc_id, const H5O_loc_t *loc, H5_index_t idx_type, H5_iter_order_t order, hsize_t skip, hsize_t *last_attr, const H5A_attr_iter_op_t *attr_op, void *op_data) { - H5O_t *oh = NULL; /* Pointer to actual object header */ - H5O_ainfo_t ainfo; /* Attribute information for object */ - H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */ - herr_t ret_value = FAIL; /* Return value */ + H5O_t *oh = NULL; /* Pointer to actual object header */ + H5O_ainfo_t ainfo; /* Attribute information for object */ + H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */ + herr_t ret_value = FAIL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_TAG(loc->addr) @@ -1223,7 +1230,7 @@ H5O_attr_iterate_real(hid_t loc_id, const H5O_loc_t *loc, H5_index_t idx_type, H oh = NULL; /* Check for skipping too many attributes */ - if (skip > 0 && skip >= atable.nattrs) + if (skip > 0 && skip >= atable.num_attrs) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid index specified"); /* Iterate over attributes in table */ @@ -1293,8 +1300,8 @@ H5O__attr_iterate(hid_t loc_id, H5_index_t idx_type, H5_iter_order_t order, hsiz static herr_t H5O__attr_remove_update(const H5O_loc_t *loc, H5O_t *oh, H5O_ainfo_t *ainfo) { - H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */ - herr_t ret_value = SUCCEED; /* Return value */ + H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -1530,11 +1537,11 @@ H5O__attr_remove(const H5O_loc_t *loc, const char *name) herr_t H5O__attr_remove_by_idx(const H5O_loc_t *loc, H5_index_t idx_type, H5_iter_order_t order, hsize_t n) { - H5O_t *oh = NULL; /* Pointer to actual object header */ - H5O_ainfo_t ainfo; /* Attribute information for object */ - htri_t ainfo_exists = false; /* Whether the attribute info exists in the file */ - H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_t *oh = NULL; /* Pointer to actual object header */ + H5O_ainfo_t ainfo; /* Attribute information for object */ + htri_t ainfo_exists = false; /* Whether the attribute info exists in the file */ + H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_TAG(loc->addr) @@ -1568,7 +1575,7 @@ H5O__attr_remove_by_idx(const H5O_loc_t *loc, H5_index_t idx_type, H5_iter_order HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, FAIL, "error building attribute table"); /* Check for skipping too many attributes */ - if (n >= atable.nattrs) + if (n >= atable.num_attrs) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid index specified"); /* Set up user data for callback, to remove the attribute by name */ diff --git a/src/H5Odtype.c b/src/H5Odtype.c index 05652df0fe2..674d8d4ea1c 100644 --- a/src/H5Odtype.c +++ b/src/H5Odtype.c @@ -176,6 +176,14 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT16DECODE(*pp, dt->shared->u.atomic.offset); UINT16DECODE(*pp, dt->shared->u.atomic.prec); + + /* Sanity checks */ + if (dt->shared->u.atomic.offset >= (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "integer offset out of bounds"); + if (0 == dt->shared->u.atomic.prec) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "precision is zero"); + if (((dt->shared->u.atomic.offset + dt->shared->u.atomic.prec) - 1) >= (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "integer offset+precision out of bounds"); break; case H5T_FLOAT: @@ -212,6 +220,8 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t HGOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unknown floating-point normalization"); } dt->shared->u.atomic.u.f.sign = (flags >> 8) & 0xff; + if (dt->shared->u.atomic.u.f.sign >= (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "sign bit position out of bounds"); if (H5_IS_KNOWN_BUFFER_OVERFLOW(skip, *pp, 2 + 2, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); @@ -224,6 +234,11 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t dt->shared->u.atomic.u.f.esize = *(*pp)++; if (dt->shared->u.atomic.u.f.esize == 0) HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "exponent size can't be zero"); + if (dt->shared->u.atomic.u.f.epos >= (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "exponent starting position out of bounds"); + if (((dt->shared->u.atomic.u.f.epos + dt->shared->u.atomic.u.f.esize) - 1) >= + (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "exponent range out of bounds"); if (H5_IS_KNOWN_BUFFER_OVERFLOW(skip, *pp, 1 + 1, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); @@ -231,10 +246,30 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t dt->shared->u.atomic.u.f.msize = *(*pp)++; if (dt->shared->u.atomic.u.f.msize == 0) HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "mantissa size can't be zero"); + if (dt->shared->u.atomic.u.f.mpos >= (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "mantissa starting position out of bounds"); + if (((dt->shared->u.atomic.u.f.mpos + dt->shared->u.atomic.u.f.msize) - 1) >= + (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "mantissa range out of bounds"); if (H5_IS_KNOWN_BUFFER_OVERFLOW(skip, *pp, 4, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT32DECODE(*pp, dt->shared->u.atomic.u.f.ebias); + + /* Sanity check bits don't overlap */ + if (H5_RANGE_OVERLAP(dt->shared->u.atomic.u.f.sign, dt->shared->u.atomic.u.f.sign, + dt->shared->u.atomic.u.f.epos, + ((dt->shared->u.atomic.u.f.epos + dt->shared->u.atomic.u.f.esize) - 1))) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "exponent and sign positions overlap"); + if (H5_RANGE_OVERLAP(dt->shared->u.atomic.u.f.sign, dt->shared->u.atomic.u.f.sign, + dt->shared->u.atomic.u.f.mpos, + ((dt->shared->u.atomic.u.f.mpos + dt->shared->u.atomic.u.f.msize) - 1))) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "mantissa and sign positions overlap"); + if (H5_RANGE_OVERLAP(dt->shared->u.atomic.u.f.epos, + ((dt->shared->u.atomic.u.f.epos + dt->shared->u.atomic.u.f.esize) - 1), + dt->shared->u.atomic.u.f.mpos, + ((dt->shared->u.atomic.u.f.mpos + dt->shared->u.atomic.u.f.msize) - 1))) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "mantissa and exponent positions overlap"); break; case H5T_TIME: @@ -378,9 +413,11 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t /* Decode the field offset */ /* (starting with version 3 of the datatype message, use the minimum # of bytes required) */ if (version >= H5O_DTYPE_VERSION_3) { + H5_GCC_CLANG_DIAG_OFF("type-limits") if (H5_IS_KNOWN_BUFFER_OVERFLOW(skip, *pp, offset_nbytes, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); + H5_GCC_CLANG_DIAG_ON("type-limits") UINT32DECODE_VAR(*pp, dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset, offset_nbytes); } @@ -452,6 +489,13 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t } if (temp_type->shared->size == 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "type size can't be zero"); + if ((dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset + + temp_type->shared->size) > dt->shared->size) { + if (H5T_close_real(temp_type) < 0) + HDONE_ERROR(H5E_DATATYPE, H5E_CANTRELEASE, FAIL, "can't release datatype info"); + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, + "member type extends outside its parent compound type"); + } /* Upgrade the version if we can and it is necessary */ if (can_upgrade && temp_type->shared->version > version) { @@ -770,6 +814,19 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t HGOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unknown datatype class found"); } + /* Check for numeric type w/unusual # of unused bits */ + if (H5T_is_numeric_with_unusual_unused_bits(dt)) + /* Throw an error if the object header is not checksummed, unless the + * H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS flag is set with + * H5Pset_relax_file_integrity_checks() to suppress it. + */ + if (!(*ioflags & H5O_DECODEIO_RFIC_UNUBNT)) + HGOTO_ERROR( + H5E_DATATYPE, H5E_BADVALUE, FAIL, + "datatype has unusually large # of unused bits (prec = %zu bits, size = %zu bytes), possibly " + "corrupted file. See documentation for H5Pset_relax_file_integrity_checks for details.", + dt->shared->u.atomic.prec, dt->shared->size); + done: /* Cleanup on error */ if (ret_value < 0) @@ -1307,8 +1364,8 @@ H5O__dtype_encode_helper(uint8_t **pp, const H5T_t *dt) function using malloc() and is returned to the caller. --------------------------------------------------------------------------*/ static void * -H5O__dtype_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, - unsigned *ioflags /*in,out*/, size_t p_size, const uint8_t *p) +H5O__dtype_decode(H5F_t *f, H5O_t *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, unsigned *ioflags /*in,out*/, + size_t p_size, const uint8_t *p) { bool skip; H5T_t *dt = NULL; @@ -1331,6 +1388,17 @@ H5O__dtype_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsign */ skip = (p_size == SIZE_MAX ? true : false); + /* Indicate if the object header has a checksum, or if the + * H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS flag is set */ + if (open_oh) { + if (H5O_SIZEOF_CHKSUM_OH(open_oh) > 0 || + (f && (H5F_RFIC_FLAGS(f) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS))) + *ioflags |= H5O_DECODEIO_RFIC_UNUBNT; + } + else + /* Decode operations from non-object headers are assumed to be checksummed */ + *ioflags |= H5O_DECODEIO_RFIC_UNUBNT; + /* Perform actual decode of message */ if (H5O__dtype_decode_helper(ioflags, &p, dt, skip, p_end) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, NULL, "can't decode type"); diff --git a/src/H5Oefl.c b/src/H5Oefl.c index ebd92a733ba..57e5e6991df 100644 --- a/src/H5Oefl.c +++ b/src/H5Oefl.c @@ -140,6 +140,9 @@ H5O__efl_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED #endif for (size_t u = 0; u < mesg->nused; u++) { + + hsize_t offset = 0; + /* Name */ if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); @@ -156,7 +159,8 @@ H5O__efl_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED /* File offset */ if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); - H5F_DECODE_LENGTH(f, p, mesg->slot[u].offset); + H5F_DECODE_LENGTH(f, p, offset); /* Decode into an hsize_t to avoid sign warnings */ + mesg->slot[u].offset = (HDoff_t)offset; /* Size */ if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end)) diff --git a/src/H5Oint.c b/src/H5Oint.c index 537563dbbad..a98e22a1821 100644 --- a/src/H5Oint.c +++ b/src/H5Oint.c @@ -2919,3 +2919,23 @@ H5O__reset_info2(H5O_info2_t *oinfo) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5O__reset_info2() */ + +/*------------------------------------------------------------------------- + * Function: H5O_has_chksum + * + * Purpose: Returns true if object header is checksummed + * + * Return: true/false on success, can't fail + * + *------------------------------------------------------------------------- + */ +bool +H5O_has_chksum(const H5O_t *oh) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check args */ + assert(oh); + + FUNC_LEAVE_NOAPI(H5O_SIZEOF_CHKSUM_OH(oh) > 0) +} /* end H5O_has_chksum() */ diff --git a/src/H5Olayout.c b/src/H5Olayout.c index fc0f59ee60d..d14e0009953 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -392,10 +392,16 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU case H5D_CHUNK_IDX_SINGLE: /* Single Chunk Index */ if (mesg->u.chunk.flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER) { + uint64_t nbytes = 0; + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f) + 4, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); - H5F_DECODE_LENGTH(f, p, mesg->storage.u.chunk.u.single.nbytes); + + H5F_DECODE_LENGTH(f, p, nbytes); + H5_CHECKED_ASSIGN(mesg->storage.u.chunk.u.single.nbytes, uint32_t, nbytes, + uint64_t); + UINT32DECODE(p, mesg->storage.u.chunk.u.single.filter_mask); } diff --git a/src/H5Olinfo.c b/src/H5Olinfo.c index 830e4e3113c..6b5dc0b89b5 100644 --- a/src/H5Olinfo.c +++ b/src/H5Olinfo.c @@ -140,6 +140,9 @@ H5O__linfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUS if (H5_IS_BUFFER_OVERFLOW(p, 8, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); INT64DECODE(p, linfo->max_corder); + if (linfo->max_corder < 0) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, + "invalid max creation order value for message: %" PRId64, linfo->max_corder); } else linfo->max_corder = 0; @@ -156,8 +159,10 @@ H5O__linfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUS /* Address of v2 B-tree to index creation order of links, if there is one */ if (linfo->index_corder) { + H5_GCC_CLANG_DIAG_OFF("type-limits") if (H5_IS_BUFFER_OVERFLOW(p, addr_size, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + H5_GCC_CLANG_DIAG_ON("type-limits") H5F_addr_decode(f, &p, &(linfo->corder_bt2_addr)); } else diff --git a/src/H5Opkg.h b/src/H5Opkg.h index 4c719bf340e..8e32f3ae13a 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -150,6 +150,8 @@ /* Input/output flags for decode functions */ #define H5O_DECODEIO_NOCHANGE 0x01u /* IN: do not modify values */ #define H5O_DECODEIO_DIRTY 0x02u /* OUT: message has been changed */ +#define H5O_DECODEIO_RFIC_UNUBNT \ + 0x04u /* IN: Relax file integrity checks for unusual numbers of unused bits in numeric datatypes */ /* Macro to incremend ndecode_dirtied (only if we are debugging) */ #ifndef NDEBUG diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 6f3d39d1e83..968a23caada 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -932,6 +932,7 @@ H5_DLL time_t H5O_get_oh_mtime(const H5O_t *oh); H5_DLL uint8_t H5O_get_oh_version(const H5O_t *oh); H5_DLL herr_t H5O_get_rc_and_type(const H5O_loc_t *oloc, unsigned *rc, H5O_type_t *otype); H5_DLL H5AC_proxy_entry_t *H5O_get_proxy(const H5O_t *oh); +H5_DLL bool H5O_has_chksum(const H5O_t *oh); /* Object header message routines */ H5_DLL herr_t H5O_msg_create(const H5O_loc_t *loc, unsigned type_id, unsigned mesg_flags, diff --git a/src/H5Pencdec.c b/src/H5Pencdec.c index 4d1d8187591..19d453279ee 100644 --- a/src/H5Pencdec.c +++ b/src/H5Pencdec.c @@ -272,6 +272,41 @@ H5P__encode_double(const void *value, void **_pp, size_t *size) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5P__encode_double() */ +/*------------------------------------------------------------------------- + * Function: H5P__encode_uint64_t + * + * Purpose: Generic encoding callback routine for 'uint64_t' properties. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5P__encode_uint64_t(const void *value, void **_pp, size_t *size) +{ + uint8_t **pp = (uint8_t **)_pp; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity checks */ + assert(value); + assert(size); + + if (NULL != *pp) { + /* Encode the size */ + *(*pp)++ = (uint8_t)sizeof(uint64_t); + + /* Encode the value */ + UINT64ENCODE(*pp, *(const unsigned *)value); + } /* end if */ + + /* Set size needed for encoding */ + *size += (1 + sizeof(uint64_t)); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__encode_uint64_t() */ + /*-------------------------------------------------------------------------- NAME H5P__encode_cb @@ -611,6 +646,42 @@ H5P__decode_double(const void **_pp, void *_value) FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__decode_double() */ +/*------------------------------------------------------------------------- + * Function: H5P__decode_uint64_t + * + * Purpose: Generic decoding callback routine for 'uint64_t' properties. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5P__decode_uint64_t(const void **_pp, void *_value) +{ + uint64_t *value = (uint64_t *)_value; /* Property value to return */ + const uint8_t **pp = (const uint8_t **)_pp; + unsigned enc_size; /* Size of encoded property */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity checks */ + assert(pp); + assert(*pp); + assert(value); + + /* Decode the size */ + enc_size = *(*pp)++; + if (enc_size != sizeof(uint64_t)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "uint64_t value can't be decoded"); + + UINT64DECODE(*pp, *value); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__decode_uint64_t() */ + /*------------------------------------------------------------------------- NAME H5P__decode diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 65c21408a3f..80922bd9d1d 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -333,6 +333,11 @@ #endif #define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC H5P__encode_bool #define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC H5P__decode_bool +/* Definition for 'rfic' flags */ +#define H5F_ACS_RFIC_FLAGS_SIZE sizeof(uint64_t) +#define H5F_ACS_RFIC_FLAGS_DEF 0 +#define H5F_ACS_RFIC_FLAGS_ENC H5P__encode_uint64_t +#define H5F_ACS_RFIC_FLAGS_DEC H5P__decode_uint64_t /******************/ /* Local Typedefs */ @@ -529,6 +534,7 @@ static const bool H5F_def_use_file_locking_g = H5F_ACS_USE_FILE_LOCKING_DEF; /* Default use file locking flag */ static const bool H5F_def_ignore_disabled_file_locks_g = H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF; /* Default ignore disabled file locks flag */ +static const uint64_t H5F_def_rfic_flags_g = H5F_ACS_RFIC_FLAGS_DEF; /* Default 'rfic' flags */ /*------------------------------------------------------------------------- * Function: H5P__facc_reg_prop @@ -826,6 +832,12 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class"); + /* Register the 'rfic' flags. */ + if (H5P__register_real(pclass, H5F_ACS_RFIC_FLAGS_NAME, H5F_ACS_RFIC_FLAGS_SIZE, &H5F_def_rfic_flags_g, + NULL, NULL, NULL, H5F_ACS_RFIC_FLAGS_ENC, H5F_ACS_RFIC_FLAGS_DEC, NULL, NULL, NULL, + NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class"); + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_reg_prop() */ @@ -6264,3 +6276,75 @@ H5P__facc_vol_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_vol_close() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_relax_file_integrity_checks + * + * Purpose: Relax certain file integrity checks that may issue errors + * for valid files that have the potential for incorrect library + * behavior when data is incorrect or corrupted. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_relax_file_integrity_checks(hid_t plist_id, uint64_t flags) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "iUL", plist_id, flags); + + /* Check arguments */ + if (H5P_DEFAULT == plist_id) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't modify default property list"); + if (flags & (uint64_t)~H5F_RFIC_ALL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid flags"); + + /* Get the property list structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plist_id is not a file access property list"); + + /* Set value */ + if (H5P_set(plist, H5F_ACS_RFIC_FLAGS_NAME, &flags) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set relaxed file integrity check flags"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_relax_file_integrity_checks() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_relax_file_integrity_checks + * + * Purpose: Retrieve relaxed file integrity check flags + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_relax_file_integrity_checks(hid_t plist_id, uint64_t *flags /*out*/) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*UL", plist_id, flags); + + if (H5P_DEFAULT == plist_id) + plist_id = H5P_FILE_ACCESS_DEFAULT; + + /* Get the property list structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plist_id is not a file access property list"); + + /* Get value */ + if (flags) + if (H5P_get(plist, H5F_ACS_RFIC_FLAGS_NAME, flags) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get relaxed file integrity check flags"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_relax_file_integrity_checks() */ diff --git a/src/H5Ppkg.h b/src/H5Ppkg.h index e249c7f9b2a..63baced637a 100644 --- a/src/H5Ppkg.h +++ b/src/H5Ppkg.h @@ -175,12 +175,14 @@ H5_DLL herr_t H5P__encode_unsigned(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__encode_uint8_t(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__encode_bool(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__encode_double(const void *value, void **_pp, size_t *size); +H5_DLL herr_t H5P__encode_uint64_t(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__decode_hsize_t(const void **_pp, void *value); H5_DLL herr_t H5P__decode_size_t(const void **_pp, void *value); H5_DLL herr_t H5P__decode_unsigned(const void **_pp, void *value); H5_DLL herr_t H5P__decode_uint8_t(const void **_pp, void *value); H5_DLL herr_t H5P__decode_bool(const void **_pp, void *value); H5_DLL herr_t H5P__decode_double(const void **_pp, void *value); +H5_DLL herr_t H5P__decode_uint64_t(const void **_pp, void *value); H5_DLL herr_t H5P__encode_coll_md_read_flag_t(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__decode_coll_md_read_flag_t(const void **_pp, void *value); diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index bf0a3be436c..a73dffb2ca5 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -5634,6 +5634,79 @@ H5_DLL herr_t H5Pset_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t H5_DLL herr_t H5Pset_page_buffer_size(hid_t plist_id, size_t buf_size, unsigned min_meta_per, unsigned min_raw_per); +/** + * \ingroup FAPL + * + * \brief Relax file integrity checks that may issue errors for some valid files + * + * \fapl_id{plist_id} + * \param[in] flags Relaxed integrity checks flag. Valid values are: + * \li #H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS + * suppresses integrity checks for detecting + * unusually high values for the number of unused bits in + * numeric datatype classes (H5T_INTEGER, H5T_FLOAT, and + * H5T_BITFIELD). Integrity checks are triggered when + * the precision for a datatype (i.e. the number of bits + * containing actual data) is less than half of the + * datatype's size and the datatype is greater than + * 1 byte in size. For example, a datatype with a + * precision of 15 bits and a size of 4 bytes (i.e. 32 bits) + * will issue an error, but a datatype with 17 bits of + * precision and a size of 4 bytes will not issue an + * error, nor will a datatype with a precision of 1, 2, or + * 3 bits and a size of 1 byte issue an error. + * \li #H5F_RFIC_ALL relaxes all integrity checks above. + * + * \return \herr_t + * + * \details Incorrectly encoded or corrupted metadata in a native HDF5 + * format file can cause incorrect library behavior when the metadata + * has no checksum. Integrity checks within the library detect these + * circumstances and issue errors when incorrect metadata is found. + * Unfortunately, some of the integrity checks for detecting these + * circumstances may incorrectly issue an error for a valid HDF5 file + * that was intentionally created with these configurations. + * Setting the appropriate flag(s) with this routine will relax the + * file integrity checks for these valid files and suppress errors + * when accessing objects with these configurations. + * + * The library will also issue errors when these configurations are + * used to create objects, preventing applications from unintentionally + * creating them. Setting the appropriate flag with this routine will + * also suppress those errors on creation, although using this routine + * and the appropriate flag(s) will still be required when accessing + * files created with these configurations. + * + * A more complete solution that avoids errors on both object creation + * and access is to use the H5Pset_libver_bounds routine with a low + * bound of at least #H5F_LIBVER_V18 when creating objects with these + * configurations. This will cause the library to checksum a file's + * metadata, allowing accidental data corruption to be correctly + * detected and errors correctly issued without ambiguity. + * + * \since 1.14.4 + * + */ +H5_DLL herr_t H5Pset_relax_file_integrity_checks(hid_t plist_id, uint64_t flags); +/** + * \ingroup FAPL + * + * \brief Retrieve relaxed file integrity check flags + * + * \fapl_id{plist_id} + * \param[out] flags Relaxed file integrity check flags + * + * \return \herr_t + * + * \details H5Pget_relax_file_integrity_checks() retrieves the relaxed file + * integrity check value into \p flags for the file access property + * list specified in \p plist_id. + * + * \since 1.14.4 + * + */ +H5_DLL herr_t H5Pget_relax_file_integrity_checks(hid_t plist_id, uint64_t *flags); + /* Dataset creation property list (DCPL) routines */ /** * \ingroup DCPL diff --git a/src/H5Rint.c b/src/H5Rint.c index 4606a57d3f7..3df70ba48c3 100644 --- a/src/H5Rint.c +++ b/src/H5Rint.c @@ -956,7 +956,10 @@ H5R__decode(const unsigned char *buf, size_t *nbytes, H5R_ref_priv_t *ref) const uint8_t *p = (const uint8_t *)buf; size_t buf_size = 0, decode_size = 0; uint8_t flags; - herr_t ret_value = SUCCEED; + bool decoded_filename = false; /* Whether filename was decoded, for error handling */ + bool decoded_attrname = false; /* Whether attribute name was decoded, for error handling */ + bool decoded_dataspace = false; /* Whether dataspace was decoded, for error handling */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE @@ -990,6 +993,7 @@ H5R__decode(const unsigned char *buf, size_t *nbytes, H5R_ref_priv_t *ref) /* Decode file name */ H5R_DECODE(H5R__decode_string, &ref->info.obj.filename, p, buf_size, decode_size, "Cannot decode filename"); + decoded_filename = true; } else ref->info.obj.filename = NULL; @@ -997,22 +1001,28 @@ H5R__decode(const unsigned char *buf, size_t *nbytes, H5R_ref_priv_t *ref) switch (ref->type) { case H5R_OBJECT2: break; + case H5R_DATASET_REGION2: /* Decode dataspace */ H5R_DECODE(H5R__decode_region, &ref->info.reg.space, p, buf_size, decode_size, "Cannot decode region"); + decoded_dataspace = true; break; + case H5R_ATTR: /* Decode attribute name */ H5R_DECODE(H5R__decode_string, &ref->info.attr.name, p, buf_size, decode_size, "Cannot decode attribute name"); + decoded_attrname = true; break; + case H5R_OBJECT1: case H5R_DATASET_REGION1: case H5R_BADTYPE: case H5R_MAXTYPE: assert("invalid reference type" && 0); HGOTO_ERROR(H5E_REFERENCE, H5E_UNSUPPORTED, FAIL, "internal error (invalid reference type)"); + default: assert("unknown reference type" && 0); HGOTO_ERROR(H5E_REFERENCE, H5E_UNSUPPORTED, FAIL, "internal error (unknown reference type)"); @@ -1031,6 +1041,22 @@ H5R__decode(const unsigned char *buf, size_t *nbytes, H5R_ref_priv_t *ref) *nbytes = decode_size; done: + if (ret_value < 0) { + if (decoded_filename) { + H5MM_xfree(ref->info.obj.filename); + ref->info.obj.filename = NULL; + } + if (decoded_attrname) { + H5MM_xfree(ref->info.attr.name); + ref->info.attr.name = NULL; + } + if (decoded_dataspace) { + if (H5S_close(ref->info.reg.space) < 0) + HDONE_ERROR(H5E_REFERENCE, H5E_CLOSEERROR, FAIL, "unable to release dataspace"); + ref->info.reg.space = NULL; + } + } + FUNC_LEAVE_NOAPI(ret_value) } /* end H5R__decode() */ @@ -1175,7 +1201,7 @@ H5R__decode_region(const unsigned char *buf, size_t *nbytes, H5S_t **space_ptr) const uint8_t *p_end = p + *nbytes - 1; size_t buf_size = 0; unsigned rank; - H5S_t *space; + H5S_t *space = NULL; herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE @@ -1216,6 +1242,10 @@ H5R__decode_region(const unsigned char *buf, size_t *nbytes, H5S_t **space_ptr) *space_ptr = space; done: + if (ret_value < 0) + if (space && H5S_close(space) < 0) + HDONE_ERROR(H5E_REFERENCE, H5E_CLOSEERROR, FAIL, "unable to release dataspace"); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5R__decode_region() */ diff --git a/src/H5S.c b/src/H5S.c index c58176e3243..722eac8b563 100644 --- a/src/H5S.c +++ b/src/H5S.c @@ -615,6 +615,10 @@ H5S__extent_copy_real(H5S_extent_t *dst, const H5S_extent_t *src, bool copy_max) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOPY, FAIL, "can't copy shared information"); done: + if (ret_value < 0) + if (dst->size) + dst->size = H5FL_ARR_FREE(hsize_t, dst->size); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5S__extent_copy_real() */ diff --git a/src/H5Shyper.c b/src/H5Shyper.c index 344351e05be..6b07529df96 100644 --- a/src/H5Shyper.c +++ b/src/H5Shyper.c @@ -9178,8 +9178,8 @@ H5S__check_spans_overlap(const H5S_hyper_span_info_t *spans1, const H5S_hyper_sp assert(spans2); /* Use low & high bounds to try to avoid spinning through the span lists */ - if (H5S_RANGE_OVERLAP(spans1->low_bounds[0], spans1->high_bounds[0], spans2->low_bounds[0], - spans2->high_bounds[0])) { + if (H5_RANGE_OVERLAP(spans1->low_bounds[0], spans1->high_bounds[0], spans2->low_bounds[0], + spans2->high_bounds[0])) { H5S_hyper_span_t *span1, *span2; /* Hyperslab spans */ /* Walk over spans, comparing them for overlap */ @@ -9187,7 +9187,7 @@ H5S__check_spans_overlap(const H5S_hyper_span_info_t *spans1, const H5S_hyper_sp span2 = spans2->head; while (span1 && span2) { /* Check current two spans for overlap */ - if (H5S_RANGE_OVERLAP(span1->low, span1->high, span2->low, span2->high)) { + if (H5_RANGE_OVERLAP(span1->low, span1->high, span2->low, span2->high)) { /* Check for spans in lowest dimension already */ if (span1->down) { /* Sanity check */ @@ -9765,8 +9765,8 @@ H5S__hyper_regular_and_single_block(H5S_t *space, const hsize_t start[], const h block_end = (start[u] + block[u]) - 1; /* Check for overlap */ - if (!H5S_RANGE_OVERLAP(space->select.sel_info.hslab->diminfo.opt[u].start, select_end, start[u], - block_end)) { + if (!H5_RANGE_OVERLAP(space->select.sel_info.hslab->diminfo.opt[u].start, select_end, start[u], + block_end)) { overlap = false; break; } /* end if */ @@ -9812,8 +9812,8 @@ H5S__hyper_regular_and_single_block(H5S_t *space, const hsize_t start[], const h block_end = (start[u] + block[u]) - 1; /* Check for overlap */ - if (!H5S_RANGE_OVERLAP(space->select.sel_info.hslab->diminfo.opt[u].start, select_end, start[u], - block_end)) { + if (!H5_RANGE_OVERLAP(space->select.sel_info.hslab->diminfo.opt[u].start, select_end, start[u], + block_end)) { overlap = false; break; } /* end if */ @@ -10446,7 +10446,7 @@ H5S_combine_hyperslab(const H5S_t *old_space, H5S_seloper_t op, const hsize_t st } /* end for */ /* Check bound box of both spaces to see if they overlap */ - if (H5S_RANGE_OVERLAP(old_low_bounds[0], old_high_bounds[0], new_low_bounds[0], new_high_bounds[0])) + if (H5_RANGE_OVERLAP(old_low_bounds[0], old_high_bounds[0], new_low_bounds[0], new_high_bounds[0])) overlapped = true; /* Non-overlapping situations can be handled in special ways */ @@ -11415,8 +11415,8 @@ H5S__hyper_proj_int_iterate(H5S_hyper_span_info_t *ss_span_info, const H5S_hyper /* Check for non-overlapping bounds */ check_intersect = true; for (u = 0; u < (udata->ss_rank - depth); u++) - if (!H5S_RANGE_OVERLAP(ss_span_info->low_bounds[u], ss_span_info->high_bounds[u], - sis_span_info->low_bounds[u], sis_span_info->high_bounds[u])) { + if (!H5_RANGE_OVERLAP(ss_span_info->low_bounds[u], ss_span_info->high_bounds[u], + sis_span_info->low_bounds[u], sis_span_info->high_bounds[u])) { check_intersect = false; break; } /* end if */ @@ -11441,7 +11441,7 @@ H5S__hyper_proj_int_iterate(H5S_hyper_span_info_t *ss_span_info, const H5S_hyper /* Main loop */ do { /* Check if spans overlap */ - if (H5S_RANGE_OVERLAP(ss_low, ss_span->high, sis_low, sis_span->high)) { + if (H5_RANGE_OVERLAP(ss_low, ss_span->high, sis_low, sis_span->high)) { high = MIN(ss_span->high, sis_span->high); if (ss_span->down) { /* Add skipped elements if there's a pre-gap */ diff --git a/src/H5Spkg.h b/src/H5Spkg.h index fddd5e3c5fa..e851f548f94 100644 --- a/src/H5Spkg.h +++ b/src/H5Spkg.h @@ -89,15 +89,6 @@ * H5S_UNLIMITED) */ #define H5S_MAX_SIZE ((hsize_t)(hssize_t)(-2)) -/* Macro for checking if two ranges overlap one another */ -/* - * Check for the inverse of whether the ranges are disjoint. If they are - * disjoint, then the low bound of one of the ranges must be greater than the - * high bound of the other. - */ -/* (Assumes that low & high bounds are _inclusive_) */ -#define H5S_RANGE_OVERLAP(L1, H1, L2, H2) (!((L1) > (H2) || (L2) > (H1))) - /* * Dataspace extent information */ diff --git a/src/H5Spoint.c b/src/H5Spoint.c index c95429dcfee..7e66a03a5f6 100644 --- a/src/H5Spoint.c +++ b/src/H5Spoint.c @@ -576,8 +576,14 @@ H5S__point_add(H5S_t *space, H5S_seloper_t op, size_t num_elem, const hsize_t *c for (u = 0; u < num_elem; u++) { unsigned dim; /* Counter for dimensions */ + /* The following allocation relies on the size of an hcoords_t being + * the same as an 'H5S_pnt_node_t *', so fail now if that's not true + */ + HDcompile_assert(sizeof(hcoords_t) >= sizeof(H5S_pnt_node_t *)); + /* Allocate space for the new node */ - if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, space->extent.rank))) + /* Note: allocating "rank + 1" to allow for 'next' pointer */ + if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, space->extent.rank + 1))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate point node"); /* Initialize fields in node */ @@ -796,7 +802,7 @@ H5S__copy_pnt_list(const H5S_pnt_list_t *src, unsigned rank) assert(rank > 0); /* Allocate room for the head of the point list */ - if (NULL == (dst = H5FL_MALLOC(H5S_pnt_list_t))) + if (NULL == (dst = H5FL_CALLOC(H5S_pnt_list_t))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, NULL, "can't allocate point list node"); curr = src->head; @@ -804,8 +810,14 @@ H5S__copy_pnt_list(const H5S_pnt_list_t *src, unsigned rank) while (curr) { H5S_pnt_node_t *new_node; /* New point information node */ + /* The following allocation relies on the size of an hcoords_t being + * the same as an 'H5S_pnt_node_t *', so fail now if that's not true + */ + HDcompile_assert(sizeof(hcoords_t) >= sizeof(H5S_pnt_node_t *)); + /* Create new point */ - if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, rank))) + /* Note: allocating "rank + 1" to allow for 'next' pointer */ + if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, rank + 1))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, NULL, "can't allocate point node"); new_node->next = NULL; @@ -2275,7 +2287,7 @@ H5S__point_project_simple(const H5S_t *base_space, H5S_t *new_space, hsize_t *of HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't release selection"); /* Allocate room for the head of the point list */ - if (NULL == (new_space->select.sel_info.pnt_lst = H5FL_MALLOC(H5S_pnt_list_t))) + if (NULL == (new_space->select.sel_info.pnt_lst = H5FL_CALLOC(H5S_pnt_list_t))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate point list node"); /* Check if the new space's rank is < or > base space's rank */ @@ -2294,8 +2306,14 @@ H5S__point_project_simple(const H5S_t *base_space, H5S_t *new_space, hsize_t *of base_node = base_space->select.sel_info.pnt_lst->head; prev_node = NULL; while (base_node) { + /* The following allocation relies on the size of an hcoords_t being + * the same as an 'H5S_pnt_node_t *', so fail now if that's not true + */ + HDcompile_assert(sizeof(hcoords_t) >= sizeof(H5S_pnt_node_t *)); + /* Create new point */ - if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, new_space->extent.rank))) + /* Note: allocating "rank + 1" to allow for 'next' pointer */ + if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, new_space->extent.rank + 1))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate point node"); new_node->next = NULL; @@ -2336,8 +2354,14 @@ H5S__point_project_simple(const H5S_t *base_space, H5S_t *new_space, hsize_t *of base_node = base_space->select.sel_info.pnt_lst->head; prev_node = NULL; while (base_node) { + /* The following allocation relies on the size of an hcoords_t being + * the same as an 'H5S_pnt_node_t *', so fail now if that's not true + */ + HDcompile_assert(sizeof(hcoords_t) >= sizeof(H5S_pnt_node_t *)); + /* Create new point */ - if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, new_space->extent.rank))) + /* Note: allocating "rank + 1" to allow for 'next' pointer */ + if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, new_space->extent.rank + 1))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate point node"); new_node->next = NULL; diff --git a/src/H5Sselect.c b/src/H5Sselect.c index d67b8e6f7e0..ef9994ec1b2 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -1945,7 +1945,7 @@ H5S_select_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *en /* Loop over selection bounds and block, checking for overlap */ for (u = 0; u < space->extent.rank; u++) /* If selection bounds & block don't overlap, can leave now */ - if (!H5S_RANGE_OVERLAP(low[u], high[u], start[u], end[u])) + if (!H5_RANGE_OVERLAP(low[u], high[u], start[u], end[u])) HGOTO_DONE(false); } /* end if */ diff --git a/src/H5T.c b/src/H5T.c index 640050b8145..ba93eefd1e4 100644 --- a/src/H5T.c +++ b/src/H5T.c @@ -41,6 +41,7 @@ #include "H5Pprivate.h" /* Property lists */ #include "H5Tpkg.h" /* Datatypes */ #include "H5VLprivate.h" /* Virtual Object Layer */ +#include "H5VMprivate.h" /* Vectors and arrays */ /****************/ /* Local Macros */ @@ -6692,3 +6693,51 @@ H5T__get_path_table_npaths(void) FUNC_LEAVE_NOAPI(ret_value) } + +/*------------------------------------------------------------------------- + * Function: H5T_is_numeric_with_unusual_unused_bits + * + * Purpose: Detect if a datatype is a numeric datatype (int, float, or + * bitfield) with an unusual # of unused bits. This means + * that the precision (i.e. the # of bits used) is less than + * the size of the datatype, at power-of-two boundaries. + * + * Return: true/false on success, can't fail + * + *------------------------------------------------------------------------- + */ +bool +H5T_is_numeric_with_unusual_unused_bits(const H5T_t *dt) +{ + bool ret_value = false; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + assert(dt); + assert(dt->shared); + + /* Is the correct type? */ + if (H5T_INTEGER == dt->shared->type || H5T_FLOAT == dt->shared->type || + H5T_BITFIELD == dt->shared->type) { +#if LDBL_MANT_DIG == 106 + /* This currently won't work for the IBM long double type */ + if (H5T_FLOAT == dt->shared->type && dt->shared->size == 16 && + (dt->shared->u.atomic.prec == 64 || dt->shared->u.atomic.prec == 128)) + HGOTO_DONE(false); +#endif + + /* Has unused bits? */ + if (dt->shared->u.atomic.prec < (dt->shared->size * 8)) { + unsigned surround_bits = + 1U << (1 + H5VM_log2_gen((dt->shared->u.atomic.prec + dt->shared->u.atomic.offset) - 1)); + + /* Unused bits are unusually large? */ + if (dt->shared->size > 1 && ((dt->shared->size * 8) > surround_bits)) + HGOTO_DONE(true); + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5T_is_numeric_with_unusual_unused_bits() */ diff --git a/src/H5Tcommit.c b/src/H5Tcommit.c index 5c4b4be9553..2e23764fa7a 100644 --- a/src/H5Tcommit.c +++ b/src/H5Tcommit.c @@ -424,11 +424,13 @@ H5T__commit_anon(H5F_t *file, H5T_t *type, hid_t tcpl_id) herr_t H5T__commit(H5F_t *file, H5T_t *type, hid_t tcpl_id) { - H5O_loc_t temp_oloc; /* Temporary object header location */ - H5G_name_t temp_path; /* Temporary path */ - bool loc_init = false; /* Have temp_oloc and temp_path been initialized? */ - size_t dtype_size; /* Size of the datatype message */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_t *oh = NULL; /* Pointer to actual object header */ + H5O_loc_t temp_oloc; /* Temporary object header location */ + H5G_name_t temp_path; /* Temporary path */ + bool loc_init = false; /* Have temp_oloc and temp_path been initialized? */ + bool ohdr_created = false; /* Has the object header been created yet? */ + size_t dtype_size; /* Size of the datatype message */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -481,9 +483,23 @@ H5T__commit(H5F_t *file, H5T_t *type, hid_t tcpl_id) */ if (H5O_create(file, dtype_size, (size_t)1, tcpl_id, &temp_oloc) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to create datatype object header"); - if (H5O_msg_create(&temp_oloc, H5O_DTYPE_ID, H5O_MSG_FLAG_CONSTANT | H5O_MSG_FLAG_DONTSHARE, - H5O_UPDATE_TIME, type) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to update type header message"); + ohdr_created = true; + + /* Pin the object header */ + if (NULL == (oh = H5O_pin(&temp_oloc))) + HGOTO_ERROR(H5E_ATTR, H5E_CANTPIN, FAIL, "unable to pin object header"); + + /* Check for creating committed datatype with unusual datatype */ + if (H5T_is_numeric_with_unusual_unused_bits(type) && + !(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, + "creating committed datatype with unusual datatype, see documentation for " + "H5Pset_relax_file_integrity_checks for details."); + + /* Insert the datatype message */ + if (H5O_msg_append_oh(file, oh, H5O_DTYPE_ID, H5O_MSG_FLAG_CONSTANT | H5O_MSG_FLAG_DONTSHARE, + H5O_UPDATE_TIME, type) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to insert type header message"); /* Copy the new object header's location into the datatype, taking ownership of it */ if (H5O_loc_copy_shallow(&(type->oloc), &temp_oloc) < 0) @@ -510,23 +526,39 @@ H5T__commit(H5F_t *file, H5T_t *type, hid_t tcpl_id) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "cannot mark datatype in memory"); done: + if (oh && H5O_unpin(oh) < 0) + HDONE_ERROR(H5E_DATATYPE, H5E_CANTUNPIN, FAIL, "unable to unpin object header"); + if (ret_value < 0) { - if (loc_init) { - H5O_loc_free(&temp_oloc); - H5G_name_free(&temp_path); - } /* end if */ - if ((type->shared->state == H5T_STATE_TRANSIENT || type->shared->state == H5T_STATE_RDONLY) && - (type->sh_loc.type == H5O_SHARE_TYPE_COMMITTED)) { - if (H5O_dec_rc_by_loc(&(type->oloc)) < 0) + /* Close & delete the object header on failure */ + if (ohdr_created) { + H5O_loc_t *oloc_ptr; /* Pointer to object header location */ + + /* Point at correct object header location, depending on state when failure occurred */ + if (loc_init) + oloc_ptr = &temp_oloc; + else + oloc_ptr = &(type->oloc); + if (H5O_dec_rc_by_loc(oloc_ptr) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CANTDEC, FAIL, "unable to decrement refcount on newly created object"); - if (H5O_close(&(type->oloc), NULL) < 0) + if (H5O_close(oloc_ptr, NULL) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "unable to release object header"); - if (H5O_delete(file, type->sh_loc.u.loc.oh_addr) < 0) + if (H5O_delete(file, oloc_ptr->addr) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CANTDELETE, FAIL, "unable to delete object header"); + } + + /* Release the location info, if the datatype doesn't own it */ + if (loc_init) { + H5O_loc_free(&temp_oloc); + H5G_name_free(&temp_path); + } + + /* Reset the shared state for the datatype */ + if ((type->shared->state == H5T_STATE_TRANSIENT || type->shared->state == H5T_STATE_RDONLY) && + (type->sh_loc.type == H5O_SHARE_TYPE_COMMITTED)) type->sh_loc.type = H5O_SHARE_TYPE_UNSHARED; - } /* end if */ - } /* end if */ + } FUNC_LEAVE_NOAPI(ret_value) } /* H5T__commit() */ diff --git a/src/H5Tconv.c b/src/H5Tconv.c index 422e3110497..55b6d7d01da 100644 --- a/src/H5Tconv.c +++ b/src/H5Tconv.c @@ -19,18 +19,21 @@ /****************/ #include "H5Tmodule.h" /* This source code file is part of the H5T module */ +#define H5R_FRIEND /* Suppress error about including H5Rpkg */ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5CXprivate.h" /* API Contexts */ -#include "H5Dprivate.h" /* Datasets */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5FLprivate.h" /* Free Lists */ -#include "H5Iprivate.h" /* IDs */ -#include "H5MMprivate.h" /* Memory management */ -#include "H5Tpkg.h" /* Datatypes */ +#include "H5private.h" /* Generic Functions */ +#include "H5CXprivate.h" /* API Contexts */ +#include "H5Dprivate.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Pprivate.h" /* Property lists */ +#include "H5Rpkg.h" /* References */ +#include "H5Tpkg.h" /* Datatypes */ /****************/ /* Local Macros */ @@ -3445,26 +3448,33 @@ H5T__conv_vlen(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T bool write_to_file = false; /* Flag to indicate writing to file */ htri_t parent_is_vlen; /* Flag to indicate parent is vlen datatype */ size_t bg_seq_len = 0; /* The number of elements in the background sequence */ - H5T_t *tsrc_cpy = NULL; /*temporary copy of source base datatype */ - H5T_t *tdst_cpy = NULL; /*temporary copy of destination base datatype */ - hid_t tsrc_id = H5I_INVALID_HID; /*temporary type atom */ - hid_t tdst_id = H5I_INVALID_HID; /*temporary type atom */ - uint8_t *s = NULL; /*source buffer */ - uint8_t *d = NULL; /*destination buffer */ - uint8_t *b = NULL; /*background buffer */ - ssize_t s_stride, d_stride; /*src and dst strides */ - ssize_t b_stride; /*bkg stride */ - size_t safe; /*how many elements are safe to process in each pass */ - size_t src_base_size; /*source base size*/ - size_t dst_base_size; /*destination base size*/ - void *conv_buf = NULL; /*temporary conversion buffer */ - size_t conv_buf_size = 0; /*size of conversion buffer in bytes */ - void *tmp_buf = NULL; /*temporary background buffer */ - size_t tmp_buf_size = 0; /*size of temporary bkg buffer */ - bool nested = false; /*flag of nested VL case */ - bool need_ids = false; /*whether we need IDs for the datatypes */ - size_t elmtno; /*element number counter */ - herr_t ret_value = SUCCEED; /* Return value */ + H5T_t *tsrc_cpy = NULL; /* Temporary copy of source base datatype */ + H5T_t *tdst_cpy = NULL; /* Temporary copy of destination base datatype */ + hid_t tsrc_id = H5I_INVALID_HID; /* Temporary type atom */ + hid_t tdst_id = H5I_INVALID_HID; /* Temporary type atom */ + uint8_t *s = NULL; /* Source buffer */ + uint8_t *d = NULL; /* Destination buffer */ + uint8_t *b = NULL; /* Background buffer */ + ssize_t s_stride = 0; /* Src stride */ + ssize_t d_stride = 0; /* Dst stride */ + ssize_t b_stride; /* Bkg stride */ + size_t safe = 0; /* How many elements are safe to process in each pass */ + size_t src_base_size; /* Source base size*/ + size_t dst_base_size; /* Destination base size*/ + void *conv_buf = NULL; /* Temporary conversion buffer */ + size_t conv_buf_size = 0; /* Size of conversion buffer in bytes */ + void *tmp_buf = NULL; /* Temporary background buffer */ + size_t tmp_buf_size = 0; /* Size of temporary bkg buffer */ + bool nested = false; /* Flag of nested VL case */ + bool need_ids = false; /* Whether we need IDs for the datatypes */ + size_t elmtno = 0; /* Element number counter */ + size_t orig_d_stride = 0; /* Original destination stride (used for error handling) */ + size_t orig_nelmts = nelmts; /* Original # of elements to convert (used for error handling) */ + bool convert_forward = + true; /* Current direction of conversion (forward or backward, used for error handling) */ + bool conversions_made = + false; /* Flag to indicate conversions have been performed, used for error handling */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -3601,6 +3611,10 @@ H5T__conv_vlen(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T if (write_to_file && parent_is_vlen && bkg != NULL) nested = true; + /* Save info for unraveling on errors */ + orig_d_stride = (size_t)d_stride; + convert_forward = !(d_stride > s_stride); + /* The outer loop of the type conversion macro, controlling which */ /* direction the buffer is walked */ while (nelmts > 0) { @@ -3782,6 +3796,9 @@ H5T__conv_vlen(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T } /* end if */ } /* end else */ + /* Indicate that elements have been converted, in case of error */ + conversions_made = true; + /* Advance pointers */ s += s_stride; d += d_stride; @@ -3801,6 +3818,49 @@ H5T__conv_vlen(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T } /* end switch */ done: + /* Release converted elements on error */ + if (ret_value < 0 && conversions_made) { + size_t dest_count; + + /* Set up for first pass to destroy references */ + if (nelmts < orig_nelmts || (convert_forward && elmtno < safe)) { + dest_count = orig_nelmts - nelmts; + + /* Set pointer to correct location, based on direction chosen */ + if (convert_forward) { + d = (uint8_t *)buf; + dest_count += elmtno; /* Include partial iteration in first pass, for forward conversions */ + } + else + d = (uint8_t *)buf + (nelmts * orig_d_stride); + + /* Destroy vlen elements that have already been converted */ + while (dest_count > 0) { + H5T_vlen_reclaim_elmt(d, dst); /* Ignore errors at this point */ + d += orig_d_stride; + dest_count--; + } + } + + /* Do any remaining partial iteration, if converting backwards */ + if (!convert_forward && elmtno < safe) { + dest_count = elmtno; + + /* Set pointer to correct location */ + if (d_stride > 0) + d = (uint8_t *)buf + ((nelmts - safe) * orig_d_stride); + else + d = (uint8_t *)buf + ((nelmts - elmtno) * orig_d_stride); + + /* Destroy references that have already been converted */ + while (dest_count > 0) { + H5T_vlen_reclaim_elmt(d, dst); /* Ignore errors at this point */ + d += orig_d_stride; + dest_count--; + } + } + } + if (tsrc_id >= 0) { if (H5I_dec_ref(tsrc_id) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CANTDEC, FAIL, "can't decrement reference on temporary ID"); @@ -4033,16 +4093,23 @@ H5T__conv_ref(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T_conv_ctx_t H5_ATTR_UNUSED *conv_ctx, size_t nelmts, size_t buf_stride, size_t bkg_stride, void *buf, void *bkg) { - uint8_t *s = NULL; /* source buffer */ - uint8_t *d = NULL; /* destination buffer */ - uint8_t *b = NULL; /* background buffer */ - ssize_t s_stride, d_stride; /* src and dst strides */ - ssize_t b_stride; /* bkg stride */ - size_t safe; /* how many elements are safe to process in each pass */ - void *conv_buf = NULL; /* temporary conversion buffer */ - size_t conv_buf_size = 0; /* size of conversion buffer in bytes */ - size_t elmtno; /* element number counter */ - herr_t ret_value = SUCCEED; /* return value */ + uint8_t *s = NULL; /* source buffer */ + uint8_t *d = NULL; /* destination buffer */ + uint8_t *b = NULL; /* background buffer */ + ssize_t s_stride = 0; /* src stride */ + ssize_t d_stride = 0; /* dst stride */ + ssize_t b_stride; /* bkg stride */ + size_t safe = 0; /* how many elements are safe to process in each pass */ + void *conv_buf = NULL; /* temporary conversion buffer */ + size_t conv_buf_size = 0; /* size of conversion buffer in bytes */ + size_t elmtno = 0; /* element number counter */ + size_t orig_d_stride = 0; /* Original destination stride (used for error handling) */ + size_t orig_nelmts = nelmts; /* Original # of elements to convert (used for error handling) */ + bool convert_forward = + true; /* Current direction of conversion (forward or backward, used for error handling) */ + bool conversions_made = + false; /* Flag to indicate conversions have been performed, used for error handling */ + herr_t ret_value = SUCCEED; /* return value */ FUNC_ENTER_PACKAGE @@ -4103,6 +4170,10 @@ H5T__conv_ref(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, else b_stride = 0; + /* Save info for unraveling on errors */ + orig_d_stride = (size_t)d_stride; + convert_forward = !(d_stride > s_stride); + /* The outer loop of the type conversion macro, controlling which */ /* direction the buffer is walked */ while (nelmts > 0) { @@ -4202,6 +4273,9 @@ H5T__conv_ref(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, } /* end else */ } /* end else */ + /* Indicate that elements have been converted, in case of error */ + conversions_made = true; + /* Advance pointers */ s += s_stride; d += d_stride; @@ -4221,6 +4295,52 @@ H5T__conv_ref(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, } /* end switch */ done: + /* Release converted elements on error */ + if (ret_value < 0 && conversions_made) { + H5R_ref_priv_t ref_priv; + size_t dest_count; + + /* Set up for first pass to destroy references */ + if (nelmts < orig_nelmts || (convert_forward && elmtno < safe)) { + dest_count = orig_nelmts - nelmts; + + /* Set pointer to correct location, based on direction chosen */ + if (convert_forward) { + d = (uint8_t *)buf; + dest_count += elmtno; /* Include partial iteration in first pass, for forward conversions */ + } + else + d = (uint8_t *)buf + (nelmts * orig_d_stride); + + /* Destroy references that have already been converted */ + while (dest_count > 0) { + memcpy(&ref_priv, d, sizeof(H5R_ref_priv_t)); + H5R__destroy(&ref_priv); /* Ignore errors at this point */ + d += orig_d_stride; + dest_count--; + } + } + + /* Do any remaining partial iteration, if converting backwards */ + if (!convert_forward && elmtno < safe) { + dest_count = elmtno; + + /* Set pointer to correct location */ + if (d_stride > 0) + d = (uint8_t *)buf + ((nelmts - safe) * orig_d_stride); + else + d = (uint8_t *)buf + ((nelmts - elmtno) * orig_d_stride); + + /* Destroy references that have already been converted */ + while (dest_count > 0) { + memcpy(&ref_priv, d, sizeof(H5R_ref_priv_t)); + H5R__destroy(&ref_priv); /* Ignore errors at this point */ + d += orig_d_stride; + dest_count--; + } + } + } + /* Release the conversion buffer (always allocated, except on errors) */ if (conv_buf) conv_buf = H5FL_BLK_FREE(ref_seq, conv_buf); diff --git a/src/H5Tprivate.h b/src/H5Tprivate.h index 7d8f27615a0..46b2c92fa83 100644 --- a/src/H5Tprivate.h +++ b/src/H5Tprivate.h @@ -139,7 +139,7 @@ H5_DLL herr_t H5T_convert(H5T_path_t *tpath, const H5T_t *src_type, const H5T_t size_t buf_stride, size_t bkg_stride, void *buf, void *bkg); H5_DLL herr_t H5T_reclaim(const H5T_t *type, struct H5S_t *space, void *buf); H5_DLL herr_t H5T_reclaim_cb(void *elem, const H5T_t *dt, unsigned ndim, const hsize_t *point, void *op_data); -H5_DLL herr_t H5T_vlen_reclaim_elmt(void *elem, H5T_t *dt); +H5_DLL herr_t H5T_vlen_reclaim_elmt(void *elem, const H5T_t *dt); H5_DLL htri_t H5T_set_loc(H5T_t *dt, H5VL_object_t *file, H5T_loc_t loc); H5_DLL htri_t H5T_is_sensible(const H5T_t *dt); H5_DLL uint32_t H5T_hash(H5F_t *file, const H5T_t *dt); @@ -157,6 +157,7 @@ H5_DLL bool H5T_already_vol_managed(const H5T_t *dt); H5_DLL htri_t H5T_is_vl_storage(const H5T_t *dt); H5_DLL herr_t H5T_invoke_vol_optional(H5T_t *dt, H5VL_optional_args_t *args, hid_t dxpl_id, void **req, H5VL_object_t **vol_obj_ptr); +H5_DLL bool H5T_is_numeric_with_unusual_unused_bits(const H5T_t *dt); /* Reference specific functions */ H5_DLL H5R_type_t H5T_get_ref_type(const H5T_t *dt); diff --git a/src/H5Tvlen.c b/src/H5Tvlen.c index 6cf11c77234..971dc3c5d22 100644 --- a/src/H5Tvlen.c +++ b/src/H5Tvlen.c @@ -1053,7 +1053,7 @@ H5T__vlen_reclaim(void *elem, const H5T_t *dt, H5T_vlen_alloc_info_t *alloc_info *------------------------------------------------------------------------- */ herr_t -H5T_vlen_reclaim_elmt(void *elem, H5T_t *dt) +H5T_vlen_reclaim_elmt(void *elem, const H5T_t *dt) { H5T_vlen_alloc_info_t vl_alloc_info; /* VL allocation info */ herr_t ret_value = SUCCEED; /* return value */ diff --git a/src/H5Zscaleoffset.c b/src/H5Zscaleoffset.c index fba1c7d6c2e..048344b763c 100644 --- a/src/H5Zscaleoffset.c +++ b/src/H5Zscaleoffset.c @@ -1210,6 +1210,8 @@ H5Z__filter_scaleoffset(unsigned flags, size_t cd_nelmts, const unsigned cd_valu minbits_mask <<= i * 8; minbits |= minbits_mask; } + if (minbits >= p.size * 8) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "minimum number of bits exceeds size of type"); /* retrieval of minval takes into consideration situation where sizeof * unsigned long long (datatype of minval) may change from compression diff --git a/src/H5encode.h b/src/H5encode.h index 5be75d57f77..690aee104d7 100644 --- a/src/H5encode.h +++ b/src/H5encode.h @@ -212,7 +212,7 @@ n = 0; \ (p) += 8; \ for (_i = 0; _i < sizeof(int64_t); _i++) \ - n = (n << 8) | *(--p); \ + n = (int64_t)(((uint64_t)n << 8) | *(--p)); \ (p) += 8; \ } while (0) @@ -224,7 +224,7 @@ n = 0; \ (p) += 8; \ for (_i = 0; _i < sizeof(uint64_t); _i++) \ - n = (n << 8) | *(--p); \ + n = (uint64_t)(((uint64_t)n << 8) | *(--p)); \ (p) += 8; \ } while (0) diff --git a/src/H5private.h b/src/H5private.h index a3b37e5d834..a32421bf2ed 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -318,6 +318,15 @@ /* limit the middle value to be within a range (inclusive) */ #define RANGE(LO, X, HI) MAX(LO, MIN(X, HI)) +/* Macro for checking if two ranges overlap one another */ +/* + * Check for the inverse of whether the ranges are disjoint. If they are + * disjoint, then the low bound of one of the ranges must be greater than the + * high bound of the other. + */ +/* (Assumes that low & high bounds are _inclusive_) */ +#define H5_RANGE_OVERLAP(L1, H1, L2, H2) (!((L1) > (H2) || (L2) > (H1))) + /* absolute value */ #ifndef ABS #define ABS(a) (((a) >= 0) ? (a) : -(a)) @@ -336,9 +345,19 @@ #define H5_EXP2(n) (1 << (n)) /* Check if a read of size bytes starting at ptr would overflow past - * the last valid byte, pointed to by buffer_end. + * the last valid byte, pointed to by buffer_end. Note that 'size' + * is expected to be of type size_t. Providing values of other + * datatypes may cause warnings due to the comparison against + * PTRDIFF_MAX and comparison of < 0 after conversion to ptrdiff_t. + * For the time being, these can be suppressed with + * H5_GCC_CLANG_DIAG_OFF("type-limits")/H5_GCC_CLANG_DIAG_ON("type-limits") */ -#define H5_IS_BUFFER_OVERFLOW(ptr, size, buffer_end) (((ptr) + (size)-1) > (buffer_end)) +#define H5_IS_BUFFER_OVERFLOW(ptr, size, buffer_end) \ + (((ptr) > (buffer_end)) || /* Bad precondition */ \ + (((size_t)(size) <= PTRDIFF_MAX) && \ + ((ptrdiff_t)(size) < 0)) || /* Account for (likely unintentional) negative 'size' */ \ + ((size_t)(size) > \ + (size_t)((((const uint8_t *)buffer_end) - ((const uint8_t *)ptr)) + 1))) /* Typical overflow */ /* Variant of H5_IS_BUFFER_OVERFLOW, used with functions such as H5Tdecode() * that don't take a size parameter, where we need to skip the bounds checks. @@ -451,8 +470,7 @@ (X) >= (Y)) #define H5_addr_cmp(X,Y) (H5_addr_eq((X), (Y)) ? 0 : \ (H5_addr_lt((X), (Y)) ? -1 : 1)) -#define H5_addr_overlap(O1,L1,O2,L2) (((O1) < (O2) && ((O1) + (L1)) > (O2)) || \ - ((O1) >= (O2) && (O1) < ((O2) + (L2)))) +#define H5_addr_overlap(O1,L1,O2,L2) H5_RANGE_OVERLAP(O1, ((O1)+(L1)), O2, ((O2)+(L2))) /* clang-format on */ /* diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index a9b935e80a8..687e72830d0 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -163,6 +163,8 @@ set (HDF5_REFERENCE_TEST_FILES test_filters_le.h5 th5s.h5 tlayouto.h5 + tmisc38a.h5 + tmisc38b.h5 tmtimen.h5 tmtimeo.h5 tsizeslheap.h5 diff --git a/test/testfiles/tmisc38a.h5 b/test/testfiles/tmisc38a.h5 new file mode 100644 index 0000000000000000000000000000000000000000..cf516f0a2aba0f7300f1e4ad0d026ab87039a6c7 GIT binary patch literal 72368 zcmeHQe{fXA9pAf<18USV zoKB}=r&T+OBavy-R)xHih6-uvbdaO|QN{vVH5G9j(-|r0wK$ApER#yE``z7dZf|q% z-f?-xAM@Sh-oD-MXTQ7qxp#M;@4k2M9vr>0=ctS0ZuKI zJ2k4@qB>|%__F^HymtN* zz1Izvzuo^__pdwpXCNw)6{Q;ktXLesa>IBgkFY>c9$3W+T%>ZdT)>ZfKu!)vSV&+j zHfsW#l`N1A3i;~+?7?6#AnmMk*RDVuUWZVx*FlSr@>dXOQl>?6AYkNaifRFsFLeHG zK4bNw-mgu=f`xMf{9OWH&+UcTJ*z>ou$r$M`n;M&%~3amtjE`XUr%=2XWi=CaaEn$ zaTO2j+SSQP(T^*v1FllX)&9$%%-Gl-SI9?L9S8Zp&ZhA>LJG5~AvSfp*oRLR^*SpE z2Ecyecu5>gORSr~Hj8pVjiX4kD`#6)HLqW@Y3HRn;SX#tYh@Yz zW1Uw%%zv<;Nqqvtke~}O9rJY!AJ`$=?oEFuD?hzq{&Pc*st->lKryG*;^0&=K;k{o z7D@A#@tsqm22y>Rs8lD~UXvOo&(El)=OO21;i}Vd2GOM@>vO{Kc0s`Nl+_>Ncz=Z! z2fOua;y5$}cyoMNaN_|F$>I+$93Mxiz`^@H;B(0%y|&%N!I3j6KhFzL9#G{$$LZu{ zi$5UcinD^UUBts5;;cf|pG&_c&Qc!)5o3IAIBR_RJUnrwayDRPLy<6M=VPjuAO0(t75tTQ}-u^-gWs zc7+9Y$l>z?b6#aRZ{E7w4F{iknI88Hd~mt^9J07CoBNh|%xSeU;i2wi-=wK^juVU2 z@<)`vlVJ{ZP=>tMwlU{U*;~*2vqLX~9qO$LE$4nj8FFrX)2aNG8!ldQ{6wb5z&+*n zTdKX75cebURwtGT4|RPf7tC$nef#0oo!eUa?fZ+0ii*d5Bf);NtoyYGU$?4@*{Wpe z-R-+4lpbzNV8MP6IX z2T*w~{FAxg-?yT4&jSy#=62}*z>%t|1jqMD%eLs*`ul;x zVOxBIL>!jUPB}3^3=jjv05L!e5Cg=3KMeS`9$1~*dLSrt7`eV42z8@f4}`TrctAwM zvz(K*9@u_`Yz5L2WT=p!=6WE=R=yqxY1FHDKCeE=(Wkazy&&JX;8;*P@`?-IS2(Y@ z;E-D9Fmb5&aoFz|>me8iAP&ojr<@od28aP-fEXYKhyh~29|kZks8Q$V^S@5O~w#@j+m{%W*<~028aP-fEXYKh=DH@1JN(kY(58*@l9`gU8k({He9#I zQ-^t-GSmTGqt)BE9zP1beQ1_^-m>a#SQj3lQOEL1IiIgA8>{)qE2Q2@*egCNDJlIZ zbYI2piKU0f@A-0N3#xLsf74iK-a0TsUe=0H#!_>`f}#d5uYLp_Xk%vZYLOn*NleT|e+P7DwO!~iis z3=jjvz?d@No4)3MGc%71RAxx!v$yh zJLsXW-TZ4^(JnDS3=jjv05L!e5CbU&rlodiFFOOi>1)1zEj3$MbZF{pF0aK3wyv+C zZnXM3wSJwqzGmAbMKF9>o+xTZCu^F zwRyOOn;ku**;jA6YbE!lJ8-ot2Wj?IEzNCdtr>H%WB2LyEiIecvW;J}qk}a6>oyv# zjsA!2*xInK-?Z`0MzI;DAkDvF2QN3TU$b>p%bGTm`%LwKpSx!5`sS@`On9bxG02Ad z)@>dE<_CjWk27j$xohoO{v4WzLWX+LWWmaHTStJof3qI_V~mN{Vc~IU=K02l%V$bY zcwPnln(8ux*L{LtucV!a+5fNvW1J|a zXg@hTZTPyesUbFXyPAcr zw?Q+WI9?J5lYade)JV8PtI(}0pI7;baAxX`3e8y@d zD#xvxjyB}#2yZ9V_aIE1se6MHWmCkt>FDcRaHhXjg~I`%Kg3}f?UWM(!~iis3=jjv z05L!e_``s2I=Xrq8xY@M4zp}JI$m=81dLqQ(NLF%j@}!S6o6C&*(fCAr=_ERVd!2t zw2|Lqe$BwS>D;18(u1kDQ6HoK8x#)9=nrvNMmy!i05L!e5Cg;jF+dCu1O71Jn~wH* zJz9<{z*fuenKHahzP28H{`ZmskcywUj;{8g`!OYWO&sc3Y`#(H-mt20)47i-9DXGq zai+g+($PpQ<-`CnKnxHA!~iis42&rQzUgSb9xb(=KkLys=;+yhmQ0O_jt+Y~D<=(o zjBr!;zOQg!Q2^A(h{t@QysU+J>FBE!4%?wW>=(;ur<@od28aP-fEXYKhyh~29|nBW z(W73Eu1UK-T3+`tj&0%hF@%|R@ds$aFn_>LEotl0QE{P&=XKK1(mT0bA3ejP>7h3z z;9CX`^(M<@%4&l9&cE_@XX{N0X)%D+jG;>eP!&X zrOz(iD10}+t|I`> zrvK%#v%Rfx`$FNoYZpB4>`Nx1k;IXU%PubKa=N!|6s>UiJeCVwqNL~-*;t4 z$18RNAFADIZ?oqwOU9fod!V+b_Ql^i-5(40XHQkK)c8^(cHVH_7+7)8l4Bh=F&<}Z z*51c=UO%VncD8NV;J$^nGnY+@CGD<6PfIiqi!t$M$86>}O#Imn->;3-4g^ z$Z)baO#H>{7~?SU=h(biFY9F$tfID})`~=|h^3MQpAbockBcP1B_c_1T%;Jdj7hu$ z`;NrdTDE1kO1yL}@%q^GG3#^D=ctcOAG1CeeUAFr^fBvm5&LS)`q=a_>vPfPc!P7` zSx5-~sw5Cb_e;M+RdsLw}dUq|~LpAD=Dh55r+M?2QX>v#e) zs0p!jH1c~=VQ%tH`L3g_R_j?_>u66XoYy+qOA6<;j+Rh3oEfg0;d&dE(M~xrKnxHA z!~iis3=jjvfIkfQrlUuFK05t=T|sdlu>jj5e*gmD4ZSq9E6UfAq{GyQcc9QsFph{H14DJKSq0b+m{AO?s5Vt^R%hXLQ> zkx{SfUvfOsc|lU(Ydn&#j(*k9z4+{)p2addE%#7wyXo8+E;!R)lft2Y^oKYsqn&bM zfEXYKhyh}N7$63S0e=|qO-F~T#M*l?D`(|J#ie+k0rPh_qPqh8QdhW$MgIN==sc*K zz-Fg?&#X|`Lu}*9bz1>1eteGSaBU1MxuZTT=ljL{p{@7ND3ojR00B8Epd+uB;g>;X zNVu*aSL^yuIT=7&)n0{L^h>QW!){cqueVH64KiS$szu>W1+(LhsrHL*adD)o_p1DS zU3Q$`R=0Qrj|;c{9#e6FiNm-6+jSc_JRZ=F`AHX?>F*rHBS;zL!~iis3=jjv05L!e zj41=Y#Us_Z#Un+5T~U~z9*;m>KJf@B_6+ezdzqB-dBr37>gbZUVODJTes8iQ10vjq s{AvT|rlW6h!I?UGox))s=nrvNMmy!i05L!e5Cg;jF+dCu1O71Ze?AA+6#xJL literal 0 HcmV?d00001 diff --git a/test/testfiles/tmisc38b.h5 b/test/testfiles/tmisc38b.h5 new file mode 100644 index 0000000000000000000000000000000000000000..7f58280ac8d3c7577d86eb782d837d57c08fc151 GIT binary patch literal 1400 zcmeD5aB<`1lHy_j0S*oZ76t(@6Gr@pf(llM2#gPtPk=HQp>zk7Ucm%mFfxE31A_!q zTo7tLy1I}cS62q0N|^aD8mf)KfCa+hfC-G!BPs+uTpa^I9*%(eS^-L*fTlAjm!UMT zw74`eCq5;yB(bEjAQdJKOP>}n0U~Ke1~zc|g%F?=3Q2)5^O%9@8N?O<3cp|m8L9vx z7+8VoK;leHOkkxP&>X 0) { + CHECK(did, H5I_INVALID_HID, "H5Dcreate2"); + + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + } + else { + VERIFY(did, H5I_INVALID_HID, "H5Dcreate2"); + } + + gid = H5Gopen2(fid, "/", H5P_DEFAULT); + CHECK(gid, H5I_INVALID_HID, "H5Gopen2"); + + /* Create an attribute with the unusual datatype */ + H5E_BEGIN_TRY + { + aid = H5Acreate2(gid, MISC38C_ATTRNAME, tid, sid, H5P_DEFAULT, H5P_DEFAULT); + } + H5E_END_TRY + if (u > 0) { + CHECK(aid, H5I_INVALID_HID, "H5Acreate2"); + + ret = H5Aclose(aid); + CHECK(ret, FAIL, "H5Aclose"); + } + else { + VERIFY(aid, H5I_INVALID_HID, "H5Acreate2"); + } + + ret = H5Gclose(gid); + CHECK(ret, FAIL, "H5Gclose"); + + ret = H5Sclose(sid); + CHECK(ret, FAIL, "H5Sclose"); + + /* Create a committed datatype with the unusual datatype */ + H5E_BEGIN_TRY + { + ret = H5Tcommit2(fid, MISC38C_TYPENAME, tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + } + H5E_END_TRY + if (u > 0) { + CHECK(ret, FAIL, "H5Tcommit2"); + } + else { + VERIFY(ret, FAIL, "H5Tcommit2"); + } + + if (tid != H5I_INVALID_HID) { + ret = H5Tclose(tid); + CHECK(ret, FAIL, "H5Tclose"); + } + + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + } +} /* end test_misc38() */ + +/**************************************************************** +** +** test_misc39(): ** Test for issue where the type conversion path table cache ** would grow continuously when variable-length datatypes ** are involved due to file VOL object comparisons causing @@ -6271,7 +6511,7 @@ test_misc37(void) ** ****************************************************************/ static void -test_misc38(void) +test_misc39(void) { H5VL_object_t *file_vol_obj = NULL; const char *buf[] = {"attr_value"}; @@ -6309,7 +6549,7 @@ test_misc38(void) */ init_npaths = H5T__get_path_table_npaths(); - file_id = H5Fcreate(MISC38_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file_id = H5Fcreate(MISC39_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(file_id, H5I_INVALID_HID, "H5Fcreate"); /* Check if native VOL is being used */ @@ -6444,7 +6684,7 @@ test_misc38(void) CHECK_PTR(vlen_rbuf, "vlen varstr read buf allocation"); for (size_t i = 0; i < 10; i++) { - file_id = H5Fopen(MISC38_FILE, H5F_ACC_RDONLY, H5P_DEFAULT); + file_id = H5Fopen(MISC39_FILE, H5F_ACC_RDONLY, H5P_DEFAULT); CHECK(file_id, H5I_INVALID_HID, "H5Fopen"); /* Retrieve file's VOL object field for further use */ @@ -6546,7 +6786,7 @@ test_misc38(void) /**************************************************************** ** -** test_misc39(): Ensure H5Pset_est_link_info() handles large +** test_misc40(): Ensure H5Pset_est_link_info() handles large ** values ** ** H5Pset_est_link_info() values can be set to large values, @@ -6560,7 +6800,7 @@ test_misc38(void) ** ****************************************************************/ static void -test_misc39(void) +test_misc40(void) { hid_t fid = H5I_INVALID_HID; /* File ID */ hid_t gid = H5I_INVALID_HID; /* Group ID */ @@ -6581,7 +6821,7 @@ test_misc39(void) CHECK(ret, FAIL, "H5Pset_libver_bounds"); /* Create the file */ - fid = H5Fcreate(MISC39_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + fid = H5Fcreate(MISC40_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); CHECK(fid, H5I_INVALID_HID, "H5Fcreate"); /* Compose group creation property list */ @@ -6639,16 +6879,16 @@ test_misc39(void) ret = H5Pclose(gcpl); CHECK(ret, FAIL, "H5Pclose"); -} /* end test_misc39() */ +} /* end test_misc40() */ /**************************************************************** ** -** test_misc40(): Test that object creation properties are propagated +** test_misc41(): Test that object creation properties are propagated ** to intermediate groups. ** ****************************************************************/ static void -test_misc40(void) +test_misc41(void) { hid_t lcpl = H5I_INVALID_HID; hid_t gcpl = H5I_INVALID_HID; @@ -6675,7 +6915,7 @@ test_misc40(void) status = H5Pset_create_intermediate_group(lcpl, 1); CHECK(status, FAIL, "H5Pset_create_intermediate_group"); - fid = H5Fcreate(MISC40_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(MISC41_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fcreate"); /* @@ -6856,7 +7096,7 @@ test_misc40(void) status = H5Pclose(lcpl); CHECK(status, FAIL, "H5Pclose"); -} /* end test_misc40() */ +} /* end test_misc41() */ /**************************************************************** ** @@ -6930,9 +7170,10 @@ test_misc(void) test_misc35(); /* Test behavior of free-list & allocation statistics API calls */ test_misc36(); /* Exercise H5atclose and H5is_library_terminating */ test_misc37(); /* Test for seg fault failure at file close */ - test_misc38(); /* Test for type conversion path table issue */ - test_misc39(); /* Ensure H5Pset_est_link_info() handles large values */ - test_misc40(); /* Test object properties propagated to intermediate groups */ + test_misc38(); /* Test for seg fault when opening corrupted object header */ + test_misc39(); /* Test for type conversion path table issue */ + test_misc40(); /* Ensure H5Pset_est_link_info() handles large values */ + test_misc41(); /* Test object properties propagated to intermediate groups */ } /* test_misc() */ @@ -6988,9 +7229,10 @@ cleanup_misc(void) #ifndef H5_NO_DEPRECATED_SYMBOLS H5Fdelete(MISC31_FILE, H5P_DEFAULT); #endif /* H5_NO_DEPRECATED_SYMBOLS */ - H5Fdelete(MISC38_FILE, H5P_DEFAULT); + H5Fdelete(MISC38C_FILE, H5P_DEFAULT); H5Fdelete(MISC39_FILE, H5P_DEFAULT); H5Fdelete(MISC40_FILE, H5P_DEFAULT); + H5Fdelete(MISC41_FILE, H5P_DEFAULT); } H5E_END_TRY } /* end cleanup_misc() */ diff --git a/tools/lib/h5tools_str.c b/tools/lib/h5tools_str.c index 47893ec9390..44e9e681371 100644 --- a/tools/lib/h5tools_str.c +++ b/tools/lib/h5tools_str.c @@ -1207,54 +1207,58 @@ h5tools_str_sprint(h5tools_str_t *str, const h5tool_format_t *info, hid_t contai /* * Object references -- show the type and OID of the referenced object. */ - H5O_info2_t oi; - char *obj_tok_str = NULL; - H5TOOLS_DEBUG("H5T_REFERENCE:H5T_STD_REF_OBJ"); obj = H5Rdereference2(container, H5P_DEFAULT, H5R_OBJECT, vp); - H5Oget_info3(obj, &oi, H5O_INFO_BASIC); + if (obj >= 0) { + H5O_info2_t oi; + char *obj_tok_str = NULL; + + H5Oget_info3(obj, &oi, H5O_INFO_BASIC); + + /* Print object type and close object */ + switch (oi.type) { + case H5O_TYPE_GROUP: + h5tools_str_append(str, H5_TOOLS_GROUP); + break; + + case H5O_TYPE_DATASET: + h5tools_str_append(str, H5_TOOLS_DATASET); + break; + + case H5O_TYPE_NAMED_DATATYPE: + h5tools_str_append(str, H5_TOOLS_DATATYPE); + break; + + case H5O_TYPE_MAP: + h5tools_str_append(str, H5_TOOLS_MAP); + break; + + case H5O_TYPE_UNKNOWN: + case H5O_TYPE_NTYPES: + default: + h5tools_str_append(str, "%u-", (unsigned)oi.type); + break; + } - /* Print object type and close object */ - switch (oi.type) { - case H5O_TYPE_GROUP: - h5tools_str_append(str, H5_TOOLS_GROUP); - break; + /* Print OID */ + H5Otoken_to_str(obj, &oi.token, &obj_tok_str); - case H5O_TYPE_DATASET: - h5tools_str_append(str, H5_TOOLS_DATASET); - break; + H5Oclose(obj); - case H5O_TYPE_NAMED_DATATYPE: - h5tools_str_append(str, H5_TOOLS_DATATYPE); - break; + if (info->obj_hidefileno) + h5tools_str_append(str, info->obj_format, obj_tok_str); + else + h5tools_str_append(str, info->obj_format, oi.fileno, obj_tok_str); - case H5O_TYPE_MAP: - h5tools_str_append(str, H5_TOOLS_MAP); - break; + if (obj_tok_str) { + H5free_memory(obj_tok_str); + obj_tok_str = NULL; + } - case H5O_TYPE_UNKNOWN: - case H5O_TYPE_NTYPES: - default: - h5tools_str_append(str, "%u-", (unsigned)oi.type); - break; + h5tools_str_sprint_old_reference(str, container, H5R_OBJECT, vp); } - - /* Print OID */ - H5Otoken_to_str(obj, &oi.token, &obj_tok_str); - - H5Oclose(obj); - - if (info->obj_hidefileno) - h5tools_str_append(str, info->obj_format, obj_tok_str); else - h5tools_str_append(str, info->obj_format, oi.fileno, obj_tok_str); - - if (obj_tok_str) { - H5free_memory(obj_tok_str); - obj_tok_str = NULL; - } - - h5tools_str_sprint_old_reference(str, container, H5R_OBJECT, vp); + h5tools_str_append(str, ""); } /* end else if (H5Tequal(type, H5T_STD_REF_OBJ)) */ } break; diff --git a/tools/test/h5dump/CMakeTests.cmake b/tools/test/h5dump/CMakeTests.cmake index 3a863bdee72..2369f63c746 100644 --- a/tools/test/h5dump/CMakeTests.cmake +++ b/tools/test/h5dump/CMakeTests.cmake @@ -1285,7 +1285,7 @@ ADD_H5_COMP_TEST (tfletcher32 0 0 --enable-error-stack -H -p -d fletcher32 tfilters.h5) # nbit - ADD_H5_COMP_TEST (tnbit 0 10 --enable-error-stack -H -p -d nbit tfilters.h5) + ADD_H5_COMP_TEST (tnbit 0 1 --enable-error-stack -H -p -d nbit tfilters.h5) # scaleoffset ADD_H5_COMP_TEST (tscaleoffset 0 4 --enable-error-stack -H -p -d scaleoffset tfilters.h5) diff --git a/tools/test/h5dump/expected/tnbit.ddl b/tools/test/h5dump/expected/tnbit.ddl index 35c111fb5b3..3801c1bf655 100644 --- a/tools/test/h5dump/expected/tnbit.ddl +++ b/tools/test/h5dump/expected/tnbit.ddl @@ -1,10 +1,10 @@ HDF5 "tfilters.h5" { DATASET "nbit" { - DATATYPE 32-bit little-endian integer 3-bit precision + DATATYPE 32-bit little-endian integer 17-bit precision DATASPACE SIMPLE { ( 20, 10 ) / ( 20, 10 ) } STORAGE_LAYOUT { CHUNKED ( 10, 5 ) - SIZE XXXX (10.XXX:1 COMPRESSION) + SIZE XXXX (1.XXX:1 COMPRESSION) } FILTERS { COMPRESSION NBIT diff --git a/tools/test/h5dump/expected/treadintfilter.ddl b/tools/test/h5dump/expected/treadintfilter.ddl index fbad3f67369..7a670a2a25d 100644 --- a/tools/test/h5dump/expected/treadintfilter.ddl +++ b/tools/test/h5dump/expected/treadintfilter.ddl @@ -78,29 +78,29 @@ DATASET "fletcher32" { } } DATASET "nbit" { - DATATYPE 32-bit little-endian integer 3-bit precision + DATATYPE 32-bit little-endian integer 17-bit precision DATASPACE SIMPLE { ( 20, 10 ) / ( 20, 10 ) } DATA { - (0,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1, - (1,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3, - (2,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3, - (3,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1, - (4,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1, - (5,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3, - (6,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3, - (7,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1, - (8,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1, - (9,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3, - (10,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3, - (11,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1, - (12,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1, - (13,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3, - (14,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3, - (15,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1, - (16,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1, - (17,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3, - (18,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3, - (19,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1 + (0,0): 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + (1,0): 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + (2,0): 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + (3,0): 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + (4,0): 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + (5,0): 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + (6,0): 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + (7,0): 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + (8,0): 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + (9,0): 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, + (10,0): 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + (11,0): 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + (12,0): 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + (13,0): 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + (14,0): 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + (15,0): 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + (16,0): 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, + (17,0): 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + (18,0): 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, + (19,0): 190, 191, 192, 193, 194, 195, 196, 197, 198, 199 } } DATASET "scaleoffset" { diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index bf0bfd4a251..c7d5fe66c28 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -5615,7 +5615,7 @@ gent_filters(void) assert(ret >= 0); tid = H5Tcopy(H5T_NATIVE_INT); - H5Tset_precision(tid, H5Tget_size(tid) - 1); + H5Tset_precision(tid, (H5Tget_size(tid) * 4) + 1); ret = make_dset(fid, "nbit", sid, tid, dcpl, buf1); assert(ret >= 0); diff --git a/tools/test/h5dump/testfiles/tfilters.h5 b/tools/test/h5dump/testfiles/tfilters.h5 index 7c33e55aae13257bff664e1d36312dc5fcb08128..23d68a9b72b9bcf84799697846909d37086aae54 100644 GIT binary patch delta 1537 zcmX|BeN0nV6hH61w)B-cN-I(o+J_6!F_i+9;6QkynT%|cMckZW^q~!OWhNEpU>TE( z$wUQ}dUc3%QlnERQ)?bFG||1{S7&O{#HgqdyKkmstNYL{xVhU>xOxAabI%vXnVgaYX?<#>a0p3%Q*%FN0VzSo(HzCYA*h5GBT+~pFg$e>hM)-+fDsZT5+l$B6bCz@oFGl*RlM9xyD2;L z6P&py8392>5CjBeP(;cl0t+&U;aVs+!EwS%@H}XNBxwUsP0H)h6e$jvPAUMS zun|fj4JCroE%VRbh&Z{Mbldv3sVS-+OQcwcK zbus~J6v6~*AxZ;{TPes|1lmHSzz#wy9chCslKE&mY4!lT0VZiGQn}?TXb-g#PQXf- z!%vgqopOl|QyWM%0htk4&uIgK;UCHVK1+34S^$UI#5tj4^n&L(_cVJ9Deblq@n*mmdbrs_hk zB!v!lEiD<$X!T}%@zYoOu^EcTa|)l!b($<|O_uu|`}2NZl+G-=bj`m!|0%0}di(So zHNAZ{y?ykvYfpyKn;k;9>DP9j`b@mK;p_j*W@MT7=ty#9OMYteWhor)e)*>2E~D-r zL;c8dN(peOHgjQa@u%gn+HW@A`^VU?R_8CMKY!$`ep@v=ht`LYmKu`*B2)(z3oUlDRED#Q7d)m%Pl2G`cC{C ztWQ;3KJuWuw`3^YJXCyL5a8WaM35a7ruLcsd*^UkfluVSI0pZ1o#_f>VNH)6JFA-~ z6ff8(6_54S+f=Q~Yrq=8!&b#qosA<3?X!~SB_|t46#7F%p_^H8Z-bU1E8_UN@fAU| zjzjA>Rx3nfVO;7T#;lK`KtIN;<|vT0M%?=TtjJ0_zF~4rA@**~IDjv*E9ir(@C;oB zTfch38KID8BNXPi~dHRQMgapJ(|7bDGm+!JfA-Gh&k#4?f?a4RvV#F|&Yh>-o delta 1156 zcmX|Ae@s(X6n?LzrLV2F^hqeLyjL_**odv`oXxDG%l?2k6f$&dI^Z#?EnT6tUF^2% z+EHs%Ahz6@mcd3%w#o#x^MQl9m^Kn+im@XZ&V)H2)96O9%!EnUbno@o&Hd*)Ip25A zx#znN=NW;&7@lHAVA%WONx0$rFqfUB>^k~3(1@huL$-tg!+?dz5;0PNlyWLrs@`zj zwV6Q8@tfaHw{j^sa@iHFJoA#zKB8W8!!l~4q3^&*@tQfyBijpYpYN1yq+!X8`}Y()XMS`2GUy>TIF4B!5&LZq@QE+k zvf$R|jfyXHT4JtRPhAKao4{_F&OFn6T)#CMiNu4xs{(ySrpu1?h_S(vFs`fj!-*G!A)$XFz?gx9V3$MqBvS{^pqfcrDj%=o**^axM>)V~{SElL?W^atkiKn~N z`v%-UGF;Pkn9cSCi=OpPhL>2_-$+N*FUnd+=q)02I2esUlVj^3=we}U?QEyyPf(~D zCu9EEEFkRTkO2!Hrc8Funlu?@77({ekqsXvH)~Azr6Yyxi9J6}p+kV)v(-qR_-Gw% z>T{oEjP67p8Ak8c{Q4aAVz;HaAva8A)-9~lQ2!1-DO#Fea z_7Ea4r%ANZ9dkmd??5_8D}_#xYX^67!HSliER+Q_z6tO6zvReQWAt!``%=TThq^k1 zpaK&!G(K7eyc-Y#Pk;jgm=HJv9MX)oAz;s0L)!WJfsFI1RzN?MPO(9ZqDWCG2n+~; LpHhqWB)tAVhD@nf diff --git a/tools/test/h5dump/testh5dump.sh.in b/tools/test/h5dump/testh5dump.sh.in index daba93b3509..0964d7dda6d 100644 --- a/tools/test/h5dump/testh5dump.sh.in +++ b/tools/test/h5dump/testh5dump.sh.in @@ -1430,7 +1430,7 @@ TOOLTEST tshuffle.ddl --enable-error-stack -H -p -d shuffle tfilters.h5 # fletcher32 TOOLTESTC 0 tfletcher32.ddl --enable-error-stack -H -p -d fletcher32 tfilters.h5 # nbit -TOOLTESTC 10 tnbit.ddl --enable-error-stack -H -p -d nbit tfilters.h5 +TOOLTESTC 1 tnbit.ddl --enable-error-stack -H -p -d nbit tfilters.h5 # scaleoffset TOOLTESTC 4 tscaleoffset.ddl --enable-error-stack -H -p -d scaleoffset tfilters.h5 # all diff --git a/tools/test/h5stat/expected/h5stat_filters-F.ddl b/tools/test/h5stat/expected/h5stat_filters-F.ddl index d44445bb2e3..065d0c97477 100644 --- a/tools/test/h5stat/expected/h5stat_filters-F.ddl +++ b/tools/test/h5stat/expected/h5stat_filters-F.ddl @@ -4,12 +4,12 @@ File space information for file metadata (in bytes): Superblock extension: 0 User block: 0 Object headers: (total/unused) - Groups: 48/8 - Datasets(exclude compact data): 4136/1344 + Groups: 40/0 + Datasets(exclude compact data): 4128/1088 Datatypes: 80/0 Groups: B-tree/List: 1200 - Heap: 288 + Heap: 384 Attributes: B-tree/List: 0 Heap: 0 diff --git a/tools/test/h5stat/expected/h5stat_filters-UD.ddl b/tools/test/h5stat/expected/h5stat_filters-UD.ddl index 4efafd13c40..9f6335aaed6 100644 --- a/tools/test/h5stat/expected/h5stat_filters-UD.ddl +++ b/tools/test/h5stat/expected/h5stat_filters-UD.ddl @@ -1,5 +1,5 @@ Filename: h5stat_filters.h5 File space information for datasets' metadata (in bytes): - Object headers (total/unused): 4136/1344 + Object headers (total/unused): 4128/1088 Index for Chunked datasets: 31392 Heap: 72 diff --git a/tools/test/h5stat/expected/h5stat_filters-d.ddl b/tools/test/h5stat/expected/h5stat_filters-d.ddl index 6e6dd6140dd..eee7e1845d4 100644 --- a/tools/test/h5stat/expected/h5stat_filters-d.ddl +++ b/tools/test/h5stat/expected/h5stat_filters-d.ddl @@ -12,7 +12,7 @@ Dataset dimension information: # of datasets with dimension size 100 - 999: 1 Total # of datasets: 1 Dataset storage information: - Total raw data size: 8659 + Total raw data size: 9046 Total external raw data size: 400 Dataset layout information: Dataset layout counts[COMPACT]: 1 diff --git a/tools/test/h5stat/expected/h5stat_filters-dT.ddl b/tools/test/h5stat/expected/h5stat_filters-dT.ddl index b14ca9f9745..e513b3a5f28 100644 --- a/tools/test/h5stat/expected/h5stat_filters-dT.ddl +++ b/tools/test/h5stat/expected/h5stat_filters-dT.ddl @@ -12,7 +12,7 @@ Dataset dimension information: # of datasets with dimension size 100 - 999: 1 Total # of datasets: 1 Dataset storage information: - Total raw data size: 8659 + Total raw data size: 9046 Total external raw data size: 400 Dataset layout information: Dataset layout counts[COMPACT]: 1 diff --git a/tools/test/h5stat/expected/h5stat_filters.ddl b/tools/test/h5stat/expected/h5stat_filters.ddl index 9f9e146f08f..7383f0b31ff 100644 --- a/tools/test/h5stat/expected/h5stat_filters.ddl +++ b/tools/test/h5stat/expected/h5stat_filters.ddl @@ -12,12 +12,12 @@ File space information for file metadata (in bytes): Superblock extension: 0 User block: 0 Object headers: (total/unused) - Groups: 48/8 - Datasets(exclude compact data): 4136/1344 + Groups: 40/0 + Datasets(exclude compact data): 4128/1088 Datatypes: 80/0 Groups: B-tree/List: 1200 - Heap: 288 + Heap: 384 Attributes: B-tree/List: 0 Heap: 0 @@ -50,7 +50,7 @@ Dataset dimension information: # of datasets with dimension size 100 - 999: 1 Total # of datasets: 1 Dataset storage information: - Total raw data size: 8659 + Total raw data size: 9046 Total external raw data size: 400 Dataset layout information: Dataset layout counts[COMPACT]: 1 @@ -91,9 +91,9 @@ Free-space section bins: File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR File space page size: 4096 bytes Summary of file space information: - File metadata: 37312 bytes - Raw data: 8659 bytes + File metadata: 37392 bytes + Raw data: 9046 bytes Amount/Percent of tracked free space: 0 bytes/0.0% - Unaccounted space: 301 bytes -Total space: 46272 bytes + Unaccounted space: 258 bytes +Total space: 46696 bytes External raw data: 400 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_filters.h5 b/tools/test/h5stat/testfiles/h5stat_filters.h5 index 5b5f4bb7a689743ca67307a9c8b8ed6949737ae6..23d68a9b72b9bcf84799697846909d37086aae54 100644 GIT binary patch delta 3526 zcmb_ee^3ob$D*%MFsUAmi9~GzbCH@%OcxWAYl<^B^^ z$AXnnH?OODsI(LS1C>wqpv3tVCT)XV-)GyT5m~;$MV}%fZsX%5{mg*AWZ)NM!L}4U zfl1-Ed&p*FB>~R51VI+#Bdje?&{v!FfB2Y8a>>>R;2C19@9-5E>>SB5SPV;FgnAM* zi=@jGN8%#a9pJ6FXFI7yDa6hT=vkFA_v>=4M;Sxrd0c&haYUWdAImra$`Lun(M$r+ z!!wR3@VFq0ky6dvHAoI=fHj?Y-TEBr7sx3=)D#H_g(TNiOq{!nq`WBFS8cFzzL|r` zoXP4o7;uC(Xzjf3Tf26;uejRjslGGn0J4hr$N)ayFXv`E_%e5mfb1b zR-oMoV=_7fqpR!|7+#Ji(@$7X1c_+5+tOU;7TvYYwarg@>d4?CHwl&%2x$cxyb)&i zzlBDzC5zDrv-dNFECQC-69V-u@UNV4K_PrrMw^Xi3)TZezoV~u73si>b zVy8ipM}x$MnrN;_75Ew^2bvh<9ifpLD&wJ%8oWPnu}~o%WN3GS6YL;Tr_Tcm7-dez zGOB<9SWl-@AV%iFAYu_Y;TTlG-RdgYC~;2I52Mfu_d+VfD0Gd43y=s0U=5X&;50iU z?AVJMV2H}OM^2J~N|hl9GL*pzdR#0BU{E-*9eJr74-~>8hy%fCP_Tz?I|DWJc$*xh zn1Jg>0holRVHKpwvA}x`16V>1cmc}cE;&&)2P0(cM{S@$Ei46_0=Y2XX;F+Git-GM zQ|SeAvK#}ZZZ3c(VVa6sDW`$e%S(oK2|G|S{EWiNkmG`aZ1Cd-s%#%T2w6&kZ1zS} z;{o(2yb6y(wsW2m>4{MAG30~&z|*a=0jP>J!Hg4{F)grO5w zL%K88$qyL1a2d*kpTinCQ;C;_fME)^<)Jn3GE~ZUL854$qKdqW+~9-tGBuhMsYFb0 z7SsjjVIwS6SW0%QPNpgOP9wirCo|exYl?;y#s0gxr0d*~wOjx2@Rk^RWa5$*tH&Md zaYy#P*?F{M)1{G#oF^B+IV+5?i(M0W$n1<+=A=#edb0dFkP=l$c$W7xxwA%nXZ5zkHKqdraIv zKmUMy&y#`s|3O@y1?|ekl(XaRu}8E$Q!gJXFFBXlF`W6*UryPsEiitaT@;<`$#OiN z<+!r<$g(f)%i!)ie_>>8{)}Lm`S^6U1@Bve_f0;3;p%8cTcb4H`q>`8<&8+o_FwVPdLlaHF-#O4ANV5zJBq__T1uwYxJ|%wOsyp>X5~f ze{a*e|pgW_Rd3> zJ4~+mw&KL{j~mloQ^cz^atAv0%I=cmufO`AV3YB^gz}v%834v`k{}JrIG~Lv9Web-H+<) zchohrdA0XagC&=ZG+N&Rh-QstQDHenYEW~^JhkvtK2W|I0S~giuX;CYLcA^>`|npt zvq$GxpJJRT${GDW<5cJzrZO#>Jkc#%o288t+^y&wadG)TC8q56>gEy9jZlGUvj`^Bik`?Ef3V`afr%gVxcY*4DUgc=mFJ%ok?X@jD6NT1ZQ8v5b}4V6|y zn>MlzX_=b=AZ1%cGR4`_LqVi>gC?5rSrE~^n7)2Ye!O*q)`i(!&fALAZ@N6aX;$M) zJ%lz`t$V_1mWLMDokVQ|uDO{C1J8x4(0Y)J*%qTiT6wCl8Hmkl_07bu#_TrHPc^33 zsbGIs0CiJQo!=4dtv#ve{Z_Pm4?`R12}j%0%M30ZW)Vpmw?)s{>)#me=3X6h4JW@9 fO4jo#6f{L*PbQIpbSpmd0)yc_?K`pe0rUR=AMfKj delta 5250 zcmbtY3s6)?7Jcsx^Lc#1ar2MTlv#}D4xUz1={gi9aN>W;7bljv|^VxOp?d~@`kWh(f zIP|@@@4J1w`*rtuT)!y=Zb*%BB^sfp+~*~5l-U$r@o3@BA~)1>!Oiu88To}&T;0_hbnUKMCZvhkBTzBU%p`GiExlAbF9ktt-o)N)n z*IHJ$alLIL(+yf$X9>o7X7DnDP}9gCV3qLqCz+5T9tkcIQ$#@$uq!)njunhl4^IGz z)!3r7Ff8SDcjBLwdo-FR-1I)Oop1%tmyy9eu2c7_>3Hw`(H0 z+LtIt4{H>7I)XBfx+pJE>HH1vErpd1jVb04)irFT8GlaZoO_LT|aCEU9 zd>)EcKHN|YnZ|rO3&&2RrslHOq4{nH5{QPz13y@d>acA1)wh7E!gexE$ zCNj;9Im)ZT8Z3l()}KH51=(La?qhw5lF^K^mqVt81AiL7==^P6%`P!@)~PGhMh=;# z$(c?TjjpbIC#>BmNEg}S)=l!SHrI%;dT{IooO(3q)D4PJ?Wsef+6!fB{6UZFmB)L9 z=&vK{sqUG|-{X0e*fkLbfu4l-@B4z~9tno>)1g(xK)JWB0X?FB%;gOcDhDR(z+!j+ z8fqIXXyEE%7y{kd0TWs`*4LAAQlW`kZ7Mx8qn82&8a=eR89#V)1Uog>wI}DEioORqt%Id~~jBAH?yT~yE zYQ%KF&$%rhY8L>@XsB}laG42NfOJ3_U<4o)U1$YNQ z0DizLfa8E?0Ve>j0!{*U15N>+1N;%7UU&=WxcIT7;18~PJOJqn9(8zx!Tml|y->BHsJvudX=zmnt|}YfSSm3Y_YxL55m2j!a?Fhlf`FF?NjJrFrsVVmCj<2~ zMAl*>6yq9Xk}fI>!j8d|NtPKTp`0?w27?sZfKmW#3G>l+3HNpX2JOa`2FW`J?HSy@ zG72=frNlIxy*7TkrP0~@b6CUK+c|}Yxc9xSs5nv0 zn^4o78w>;-=Ss@nx9wTD=Bj0f#}O5uU2Uu=j;$z;+IeF4Rn3LBcK&7g&4eBGk%uR` zp5HP)Cb_-jQslcUR!JS2?$YYdZJoPx>8-iX6iR{kq{b?u!y=i>m+i_2|x+gz{%DZGLyf)Ii{L)$K95!S!i_ z>px#wb!$qOdup(7|M4#u$9VJ_&#FzfEw!mzYg50yv}}VuCp6l9Z$Vzkf%;R@mdl$& zovyJp?SIV|Pu$g7|1G~X@8fZcJ?)RX+n;R>284@ehQkS*lCjwK@*DR^inGJtwWB>E zUinyc(PiWZA^BovR#Zu`y|`$7(fW-QC1j~)ShDI^@bXQB#{!l%gn}sH<)2d@#1G7(hwVq5?uXoi8PX;(N_CQ>U-v zD`MQMI=~;|pC_)py$|quy7BI9SNg_;0!^5Z=p}V67SduhN`*UNk9uViSvt6fknk(& g$J`#?xBBZmVVAn|?p<^?+K>W5VY5-uOb?0w17Y_yEC2ui