Partial merge issue #642 develop branch PRs to Hdf5 1 10

MANIFEST

@@ -2752,6 +2752,8 @@
 ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_test.ddl
 ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl
 ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_zero.ddl
+./tools/test/h5repack/testfiles/h5repack_CVE-2018-17432.h5
+./tools/test/h5repack/testfiles/h5repack_CVE-2018-14460.h5
 ./tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl
 ./tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl
 ./tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl

c++/test/dsets.cpp

@@ -1169,7 +1169,7 @@ test_chunk_cache(const FileAccPropList &fapl)
         // Verify that chunk cache parameters are the same
         int    mdc_nelmts = 0;
         size_t nslots_1 = 0, nslots_4 = 0, nbytes_1 = 0, nbytes_4 = 0;
-        double w0_1 = 0.0F, w0_4 = 0.0F;
+        double w0_1 = 0.0, w0_4 = 0.0;
         fapl_def.getCache(mdc_nelmts, nslots_1, nbytes_1, w0_1);
         dapl.getChunkCache(nslots_4, nbytes_4, w0_4);
         verify_val(nslots_1, nslots_4, "DSetAccPropList::getChunkCache", __LINE__, __FILE__);
@@ -1187,7 +1187,7 @@ test_chunk_cache(const FileAccPropList &fapl)
         // Set new rdcc settings on fapl local
         size_t nslots_2 = nslots_1 * 2;
         size_t nbytes_2 = nbytes_1 * 2;
-        double w0_2     = w0_1 / (double)2.0F;
+        double w0_2     = w0_1 / 2.0;
         fapl_local.getCache(mdc_nelmts, nslots_2, nbytes_2, w0_2);
 
         // Create a new file using default fcpl and the passed-in fapl

hl/src/H5LT.c

@@ -822,7 +822,7 @@ H5LTopen_file_image(void *buf_ptr, size_t buf_size, unsigned flags)
     char     file_name[64];           /* Filename buffer */
     size_t   alloc_incr;              /* Buffer allocation increment */
     size_t   min_incr  = 65536;       /* Minimum buffer increment */
-    double   buf_prcnt = 0.1f;        /* Percentage of buffer size to set
+    double   buf_prcnt = 0.1;         /* Percentage of buffer size to set
                                           as increment */
     static long                 file_name_counter;
     H5FD_file_image_callbacks_t callbacks = {&image_malloc, &image_memcpy, &image_realloc, &image_free,

release_docs/RELEASE.txt

@@ -339,7 +339,18 @@ Bug Fixes since HDF5-1.10.7 release
 ===================================
     Library
     -------
-    - Fixed CVE-2018-17435
+    - Fixed CVE-2018-14460
+
+        The tool h5repack produced a segfault when the rank in dataspace
+        message was corrupted, causing invalid read while decoding the
+        dimension sizes.
+
+        The problem was fixed by ensuring that decoding the dimension sizes
+        and max values will not go beyong the end of the buffer.
+
+        (BMR - 2021/05/12, HDFFV-11223)
+
+    - Fixed CVE-2018-11206
 
         The tool h5dump produced a segfault when the size of a fill value
         message was corrupted and caused a buffer overflow.

src/H5C.c

@@ -459,8 +459,8 @@ H5C_create(size_t max_cache_size, size_t min_clean_size, int max_type_id,
     (cache_ptr->resize_ctl).max_increment       = H5C__DEF_AR_MAX_INCREMENT;
 
     (cache_ptr->resize_ctl).flash_incr_mode = H5C_flash_incr__off;
-    (cache_ptr->resize_ctl).flash_multiple  = 1.0f;
-    (cache_ptr->resize_ctl).flash_threshold = 0.25f;
+    (cache_ptr->resize_ctl).flash_multiple  = 1.0;
+    (cache_ptr->resize_ctl).flash_threshold = 0.25;
 
     (cache_ptr->resize_ctl).decr_mode              = H5C_decr__off;
     (cache_ptr->resize_ctl).upper_hr_threshold     = H5C__DEF_AR_UPPER_THRESHHOLD;
@@ -2718,7 +2718,7 @@ H5C_set_cache_auto_resize_config(H5C_t *cache_ptr, H5C_auto_size_ctl_t *config_p
             break;
 
         case H5C_incr__threshold:
-            if ((config_ptr->lower_hr_threshold <= (double)0.0f) || (config_ptr->increment <= (double)1.0f) ||
+            if ((config_ptr->lower_hr_threshold <= 0.0) || (config_ptr->increment <= 1.0) ||
                 ((config_ptr->apply_max_increment) && (config_ptr->max_increment <= 0)))
                 cache_ptr->size_increase_possible = FALSE;
             break;
@@ -2738,21 +2738,21 @@ H5C_set_cache_auto_resize_config(H5C_t *cache_ptr, H5C_auto_size_ctl_t *config_p
             break;
 
         case H5C_decr__threshold:
-            if ((config_ptr->upper_hr_threshold >= (double)1.0f) || (config_ptr->decrement >= (double)1.0f) ||
+            if ((config_ptr->upper_hr_threshold >= 1.0) || (config_ptr->decrement >= 1.0) ||
                 ((config_ptr->apply_max_decrement) && (config_ptr->max_decrement <= 0)))
                 cache_ptr->size_decrease_possible = FALSE;
             break;
 
         case H5C_decr__age_out:
-            if (((config_ptr->apply_empty_reserve) && (config_ptr->empty_reserve >= (double)1.0f)) ||
+            if (((config_ptr->apply_empty_reserve) && (config_ptr->empty_reserve >= 1.0)) ||
                 ((config_ptr->apply_max_decrement) && (config_ptr->max_decrement <= 0)))
                 cache_ptr->size_decrease_possible = FALSE;
             break;
 
         case H5C_decr__age_out_with_threshold:
-            if (((config_ptr->apply_empty_reserve) && (config_ptr->empty_reserve >= (double)1.0f)) ||
+            if (((config_ptr->apply_empty_reserve) && (config_ptr->empty_reserve >= 1.0)) ||
                 ((config_ptr->apply_max_decrement) && (config_ptr->max_decrement <= 0)) ||
-                (config_ptr->upper_hr_threshold >= (double)1.0f))
+                (config_ptr->upper_hr_threshold >= 1.0))
                 cache_ptr->size_decrease_possible = FALSE;
             break;
 
@@ -3730,8 +3730,7 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags)
                 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
                             "initial_size must be in the interval [min_size, max_size]")
 
-            if ((config_ptr->min_clean_fraction < (double)0.0f) ||
-                (config_ptr->min_clean_fraction > (double)1.0f))
+            if ((config_ptr->min_clean_fraction < 0.0) || (config_ptr->min_clean_fraction > 1.0))
                 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
                             "min_clean_fraction must be in the interval [0.0, 1.0]")
 
@@ -3747,12 +3746,11 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags)
                 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Invalid incr_mode")
 
             if (config_ptr->incr_mode == H5C_incr__threshold) {
-                if ((config_ptr->lower_hr_threshold < (double)0.0f) ||
-                    (config_ptr->lower_hr_threshold > (double)1.0f))
+                if ((config_ptr->lower_hr_threshold < 0.0) || (config_ptr->lower_hr_threshold > 1.0))
                     HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
                                 "lower_hr_threshold must be in the range [0.0, 1.0]")
 
-                if (config_ptr->increment < (double)1.0f)
+                if (config_ptr->increment < 1.0)
                     HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
                                 "increment must be greater than or equal to 1.0")
 
@@ -3767,12 +3765,10 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags)
                     break;
 
                 case H5C_flash_incr__add_space:
-                    if ((config_ptr->flash_multiple < (double)0.1f) ||
-                        (config_ptr->flash_multiple > (double)10.0f))
+                    if ((config_ptr->flash_multiple < 0.1) || (config_ptr->flash_multiple > 10.0))
                         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
                                     "flash_multiple must be in the range [0.1, 10.0]")
-                    if ((config_ptr->flash_threshold < (double)0.1f) ||
-                        (config_ptr->flash_threshold > (double)1.0f))
+                    if ((config_ptr->flash_threshold < 0.1) || (config_ptr->flash_threshold > 1.0))
                         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
                                     "flash_threshold must be in the range [0.1, 1.0]")
                     break;
@@ -3793,10 +3789,10 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags)
             }
 
             if (config_ptr->decr_mode == H5C_decr__threshold) {
-                if (config_ptr->upper_hr_threshold > (double)1.0f)
+                if (config_ptr->upper_hr_threshold > 1.0)
                     HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "upper_hr_threshold must be <= 1.0")
 
-                if ((config_ptr->decrement > (double)1.0f) || (config_ptr->decrement < (double)0.0f))
+                if ((config_ptr->decrement > 1.0) || (config_ptr->decrement < 0.0))
                     HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "decrement must be in the interval [0.0, 1.0]")
 
                 /* no need to check max_decrement as it is a size_t
@@ -3812,8 +3808,8 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags)
                 if (config_ptr->epochs_before_eviction > H5C__MAX_EPOCH_MARKERS)
                     HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "epochs_before_eviction too big")
 
-                if ((config_ptr->apply_empty_reserve) && ((config_ptr->empty_reserve > (double)1.0f) ||
-                                                          (config_ptr->empty_reserve < (double)0.0f)))
+                if ((config_ptr->apply_empty_reserve) &&
+                    ((config_ptr->empty_reserve > 1.0) || (config_ptr->empty_reserve < 0.0)))
                     HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
                                 "empty_reserve must be in the interval [0.0, 1.0]")
 
@@ -3823,8 +3819,7 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags)
             } /* H5C_decr__age_out || H5C_decr__age_out_with_threshold */
 
             if (config_ptr->decr_mode == H5C_decr__age_out_with_threshold) {
-                if ((config_ptr->upper_hr_threshold > (double)1.0f) ||
-                    (config_ptr->upper_hr_threshold < (double)0.0f))
+                if ((config_ptr->upper_hr_threshold > 1.0) || (config_ptr->upper_hr_threshold < 0.0))
                     HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
                                 "upper_hr_threshold must be in the interval [0.0, 1.0]")
             } /* H5C_decr__age_out_with_threshold */

src/H5Cdbg.c

@@ -429,11 +429,11 @@ H5C_stats(H5C_t *cache_ptr, const char *cache_name,
     int32_t aggregate_max_pins             = 0;
     double  hit_rate;
     double  prefetch_use_rate;
-    double  average_successful_search_depth                   = 0.0f;
-    double  average_failed_search_depth                       = 0.0f;
-    double  average_entries_skipped_per_calls_to_msic         = 0.0f;
-    double  average_dirty_pf_entries_skipped_per_call_to_msic = 0.0f;
-    double  average_entries_scanned_per_calls_to_msic         = 0.0f;
+    double  average_successful_search_depth                   = 0.0;
+    double  average_failed_search_depth                       = 0.0;
+    double  average_entries_skipped_per_calls_to_msic         = 0.0;
+    double  average_dirty_pf_entries_skipped_per_call_to_msic = 0.0;
+    double  average_entries_scanned_per_calls_to_msic         = 0.0;
 #endif                          /* H5C_COLLECT_CACHE_STATS */
     herr_t ret_value = SUCCEED; /* Return value */
 
@@ -492,9 +492,9 @@ H5C_stats(H5C_t *cache_ptr, const char *cache_name,
     }  /* end for */
 
     if ((total_hits > 0) || (total_misses > 0))
-        hit_rate = (double)100.0f * ((double)(total_hits)) / ((double)(total_hits + total_misses));
+        hit_rate = 100.0 * ((double)(total_hits)) / ((double)(total_hits + total_misses));
     else
-        hit_rate = 0.0f;
+        hit_rate = 0.0;
 
     if (cache_ptr->successful_ht_searches > 0)
         average_successful_search_depth = ((double)(cache_ptr->total_successful_ht_search_depth)) /
@@ -630,10 +630,9 @@ H5C_stats(H5C_t *cache_ptr, const char *cache_name,
               (long long)(cache_ptr->evictions[H5AC_PREFETCHED_ENTRY_ID]));
 
     if (cache_ptr->prefetches > 0)
-        prefetch_use_rate =
-            (double)100.0f * ((double)(cache_ptr->prefetch_hits)) / ((double)(cache_ptr->prefetches));
+        prefetch_use_rate = 100.0 * ((double)(cache_ptr->prefetch_hits)) / ((double)(cache_ptr->prefetches));
     else
-        prefetch_use_rate = 0.0f;
+        prefetch_use_rate = 0.0;
 
     HDfprintf(stdout, "%s  prefetched entry use rate          = %lf\n", cache_ptr->prefix, prefetch_use_rate);
 
@@ -658,10 +657,10 @@ H5C_stats(H5C_t *cache_ptr, const char *cache_name,
                       ((cache_ptr->class_table_ptr))[i]->name);
 
             if ((cache_ptr->hits[i] > 0) || (cache_ptr->misses[i] > 0))
-                hit_rate = (double)100.0f * ((double)(cache_ptr->hits[i])) /
+                hit_rate = 100.0 * ((double)(cache_ptr->hits[i])) /
                            ((double)(cache_ptr->hits[i] + cache_ptr->misses[i]));
             else
-                hit_rate = 0.0f;
+                hit_rate = 0.0;
 
             HDfprintf(stdout, "%s    hits / misses / hit_rate       = %ld / %ld / %f\n", cache_ptr->prefix,
                       (long)(cache_ptr->hits[i]), (long)(cache_ptr->misses[i]), hit_rate);

src/H5Cimage.c

@@ -980,15 +980,14 @@ H5C_image_stats(H5C_t *cache_ptr, hbool_t H5_ATTR_UNUSED print_header)
     } /* end for */
 
     if ((total_hits > 0) || (total_misses > 0))
-        hit_rate = (double)100.0f * ((double)(total_hits)) / ((double)(total_hits + total_misses));
+        hit_rate = 100.0 * ((double)(total_hits)) / ((double)(total_hits + total_misses));
     else
-        hit_rate = 0.0f;
+        hit_rate = 0.0;
 
     if (cache_ptr->prefetches > 0)
-        prefetch_use_rate =
-            (double)100.0f * ((double)(cache_ptr->prefetch_hits)) / ((double)(cache_ptr->prefetches));
+        prefetch_use_rate = 100.0 * ((double)(cache_ptr->prefetch_hits)) / ((double)(cache_ptr->prefetches));
     else
-        prefetch_use_rate = 0.0f;
+        prefetch_use_rate = 0.0;
 
     if (print_header) {
         HDfprintf(stdout, "\nhit     prefetches      prefetch              image  pf hit\n");

src/H5Clog.h

@@ -79,10 +79,10 @@ typedef struct H5C_log_class_t {
 
 /* Logging information */
 struct H5C_log_info_t {
-    hbool_t          enabled; /* Was the logging set up? */
-    hbool_t          logging; /* Are we currently logging? */
-    H5C_log_class_t *cls;     /* Callbacks for writing log messages */
-    void *           udata;   /* Log-specific data */
+    hbool_t                enabled; /* Was the logging set up? */
+    hbool_t                logging; /* Are we currently logging? */
+    const H5C_log_class_t *cls;     /* Callbacks for writing log messages */
+    void *                 udata;   /* Log-specific data */
 };
 
 /*****************************/

src/H5Cprivate.h

@@ -115,20 +115,20 @@
 #define H5C__CURR_CACHE_IMAGE_CTL_VER     1
 
 /* Default configuration settings */
-#define H5C__DEF_AR_UPPER_THRESHHOLD 0.9999f
-#define H5C__DEF_AR_LOWER_THRESHHOLD 0.9f
+#define H5C__DEF_AR_UPPER_THRESHHOLD 0.9999
+#define H5C__DEF_AR_LOWER_THRESHHOLD 0.9
 #define H5C__DEF_AR_MAX_SIZE         ((size_t)(16 * 1024 * 1024))
 #define H5C__DEF_AR_INIT_SIZE        ((size_t)(1 * 1024 * 1024))
 #define H5C__DEF_AR_MIN_SIZE         ((size_t)(1 * 1024 * 1024))
-#define H5C__DEF_AR_MIN_CLEAN_FRAC   0.5f
-#define H5C__DEF_AR_INCREMENT        2.0f
+#define H5C__DEF_AR_MIN_CLEAN_FRAC   0.5
+#define H5C__DEF_AR_INCREMENT        2.0
 #define H5C__DEF_AR_MAX_INCREMENT    ((size_t)(2 * 1024 * 1024))
-#define H5C__DEF_AR_FLASH_MULTIPLE   1.0f
-#define H5C__DEV_AR_FLASH_THRESHOLD  0.25f
-#define H5C__DEF_AR_DECREMENT        0.9f
+#define H5C__DEF_AR_FLASH_MULTIPLE   1.0
+#define H5C__DEV_AR_FLASH_THRESHOLD  0.25
+#define H5C__DEF_AR_DECREMENT        0.9
 #define H5C__DEF_AR_MAX_DECREMENT    ((size_t)(1 * 1024 * 1024))
 #define H5C__DEF_AR_EPCHS_B4_EVICT   3
-#define H5C__DEF_AR_EMPTY_RESERVE    0.05f
+#define H5C__DEF_AR_EMPTY_RESERVE    0.05
 #define H5C__MIN_AR_EPOCH_LENGTH     100
 #define H5C__DEF_AR_EPOCH_LENGTH     50000
 #define H5C__MAX_AR_EPOCH_LENGTH     1000000

src/H5Dchunk.c

@@ -708,10 +708,8 @@ H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims, const hsize
     } /* end for */
 
     /* Get the "down" sizes for each dimension */
-    if (H5VM_array_down(ndims, layout->chunks, layout->down_chunks) < 0)
-        HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't compute 'down' chunk size value")
-    if (H5VM_array_down(ndims, layout->max_chunks, layout->max_down_chunks) < 0)
-        HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't compute 'down' chunk size value")
+    H5VM_array_down(ndims, layout->chunks, layout->down_chunks);
+    H5VM_array_down(ndims, layout->max_chunks, layout->max_down_chunks);
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)

src/H5Dearray.c

@@ -1160,9 +1160,7 @@ H5D__earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *uda
 static herr_t
 H5D__earray_idx_resize(H5O_layout_chunk_t *layout)
 {
-    herr_t ret_value = SUCCEED; /* Return value */
-
-    FUNC_ENTER_STATIC
+    FUNC_ENTER_STATIC_NOERR
 
     /* Check args */
     HDassert(layout);
@@ -1182,22 +1180,18 @@ H5D__earray_idx_resize(H5O_layout_chunk_t *layout)
         H5VM_swizzle_coords(hsize_t, swizzled_chunks, layout->u.earray.unlim_dim);
 
         /* Get the swizzled "down" sizes for each dimension */
-        if (H5VM_array_down((layout->ndims - 1), swizzled_chunks, layout->u.earray.swizzled_down_chunks) < 0)
-            HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't compute swizzled 'down' chunk size value")
+        H5VM_array_down((layout->ndims - 1), swizzled_chunks, layout->u.earray.swizzled_down_chunks);
 
         /* Get the swizzled max number of chunks in each dimension */
         H5MM_memcpy(swizzled_max_chunks, layout->max_chunks,
                     (layout->ndims - 1) * sizeof(swizzled_max_chunks[0]));
         H5VM_swizzle_coords(hsize_t, swizzled_max_chunks, layout->u.earray.unlim_dim);
 
         /* Get the swizzled max "down" sizes for each dimension */
-        if (H5VM_array_down((layout->ndims - 1), swizzled_max_chunks,
-                            layout->u.earray.swizzled_max_down_chunks) < 0)
-            HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't compute swizzled 'down' chunk size value")
-    } /* end if */
+        H5VM_array_down((layout->ndims - 1), swizzled_max_chunks, layout->u.earray.swizzled_max_down_chunks);
+    }
 
-done:
-    FUNC_LEAVE_NOAPI(ret_value)
+    FUNC_LEAVE_NOAPI(SUCCEED)
 } /* end H5D__earray_idx_resize() */
 
 /*-------------------------------------------------------------------------

src/H5Dlayout.c

@@ -502,6 +502,7 @@ H5D__layout_oh_create(H5F_t *file, H5O_t *oh, H5D_t *dset, hid_t dapl_id)
         H5HL_t *   heap;                                /* Pointer to local heap for EFL file names */
         size_t     heap_size = H5HL_ALIGN(1);
         size_t     u;
+        size_t     name_offset;
 
         /* Determine size of heap needed to stored the file names */
         for (u = 0; u < efl->nused; ++u)
@@ -516,24 +517,22 @@ H5D__layout_oh_create(H5F_t *file, H5O_t *oh, H5D_t *dset, hid_t dapl_id)
             HGOTO_ERROR(H5E_DATASET, H5E_CANTPROTECT, FAIL, "unable to protect EFL file name heap")
 
         /* Insert "empty" name first */
-        if (UFAIL == H5HL_insert(file, heap, (size_t)1, "")) {
+        if (H5HL_insert(file, heap, (size_t)1, "", &name_offset) < 0) {
             H5HL_unprotect(heap);
             HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert file name into heap")
-        } /* end if */
+        }
 
         for (u = 0; u < efl->nused; ++u) {
-            size_t offset; /* Offset of file name in heap */
-
             /* Insert file name into heap */
-            if (UFAIL ==
-                (offset = H5HL_insert(file, heap, HDstrlen(efl->slot[u].name) + 1, efl->slot[u].name))) {
+            if (H5HL_insert(file, heap, HDstrlen(efl->slot[u].name) + 1, efl->slot[u].name, &name_offset) <
+                0) {
                 H5HL_unprotect(heap);
                 HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert file name into heap")
-            } /* end if */
+            }
 
             /* Store EFL file name offset */
-            efl->slot[u].name_offset = offset;
-        } /* end for */
+            efl->slot[u].name_offset = name_offset;
+        }
 
         /* Release the heap */
         if (H5HL_unprotect(heap) < 0)