Modifications for the following items in the punch list:

(A) #5: Add the "pb_expansion_threshold" field to the "H5F_vfd_swmr_config_t" structure and update H5Pset_vfd_swmr_config() and H5Pget_vfd_swmr_config() accordingly (B) #13 bullet 2: Comment H5F_vfd_swmr_config_t in H5Fpublic.h properly (copied from John's description in the RFC) (C) Change the field name "vfd_swmr_writer" to "writer" in "struct H5F_vfd_swmr_config_t" (as indicated on page 11 in the RFC) and all references to it
HDFGroup · Nov 14, 2019 · 227688c · 227688c
1 parent f2bb4e3
commit 227688c
Show file tree

Hide file tree

Showing 30 changed files with 115 additions and 35 deletions.
diff --git a/src/H5AC.c b/src/H5AC.c
@@ -427,7 +427,7 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, H5AC_cache_image_co
      * specified.
      */
     if ( ( H5F_VFD_SWMR_CONFIG(f) ) && 
-         ( !f->shared->vfd_swmr_config.vfd_swmr_writer ) ) {
+         ( !f->shared->vfd_swmr_config.writer ) ) {
 
         HDassert(!(H5F_INTENT(f) & H5F_ACC_RDWR));
         HDassert(f->shared->fs_page_size > 0);

diff --git a/src/H5FDvfd_swmr.c b/src/H5FDvfd_swmr.c
@@ -283,7 +283,7 @@ H5FD_vfd_swmr_open(const char *name, unsigned flags, hid_t fapl_id,
                     "can't get VFD SWMR config info")
 
     /* Ensure that this is the reader */
-    HDassert(!vfd_swmr_config->vfd_swmr_writer);
+    HDassert(!vfd_swmr_config->writer);
 
     /* Create the new driver struct */
     if(NULL == (file = H5FL_CALLOC(H5FD_vfd_swmr_t)))

diff --git a/src/H5Fint.c b/src/H5Fint.c
@@ -3677,7 +3677,7 @@ H5F__vfd_swmr_init(H5F_t *f, hbool_t file_create)
 
     if(H5F_INTENT(f) & H5F_ACC_RDWR) {
 
-        HDassert(f->shared->vfd_swmr_config.vfd_swmr_writer);
+        HDassert(f->shared->vfd_swmr_config.writer);
 
         vfd_swmr_writer_g = f->shared->vfd_swmr_writer = TRUE;
         tick_num_g = f->shared->tick_num = 1;
@@ -3731,7 +3731,7 @@ H5F__vfd_swmr_init(H5F_t *f, hbool_t file_create)
 
     } else { /* VFD SWMR reader  */
 
-        HDassert(!f->shared->vfd_swmr_config.vfd_swmr_writer);
+        HDassert(!f->shared->vfd_swmr_config.writer);
 
         vfd_swmr_writer_g = f->shared->vfd_swmr_writer = FALSE;
 

diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
@@ -522,6 +522,7 @@ typedef struct H5F_t H5F_t;
   /* hbool_t    vfd_swmr_writer         = */ FALSE,                                 \
   /* hbool_t    flush_raw_data          = */ FALSE,                                 \
   /* int32_t    md_pages_reserved       = */ 0,                                     \
+  /* int32_t    pb_expansion_threshold  = */ 0,                                     \
   /* char       md_file_path[]          = */ "",                                    \
   /* char       log_file_path[]         = */ ""                                     \
 }

diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h
@@ -220,13 +220,89 @@ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata);
 /* VFD SWMR configuration data used by H5Pset/get_vfd_swmr_config */
 #define H5F__CURR_VFD_SWMR_CONFIG_VERSION 1
 #define H5F__MAX_VFD_SWMR_FILE_NAME_LEN   1024
+#define H5F__MAX_PB_EXPANSION_THRESHOLD   100
+/*
+ *  struct H5F_vfd_swmr_config_t
+ *
+ *  Instances of H5F_vfd_swmr_config_t are used by VFD SWMR writers and readers 
+ *  to pass necessary configuration data to the HDF5 library on file open (or 
+ *  creation, in the case of writers).
+ *
+ *  The fields of the structure are discussed below:
+ *      version:
+ *          An integer field indicating the version of the H5F_vfd_swmr_config 
+ *          structure used.  This field must always be set to a known version 
+ *          number.  The most recent version of the structure will always be 
+ *          H5F__CURR_VFD_SWMR_CONFIG_VERSION.
+ *  
+ *      tick_len:
+ *          An integer field containing the length of a tick in tenths of 
+ *          a second.  If tick_len is zero, end of tick processing may only be 
+ *          triggered manually via the H5Fvfd_swmr_end_tick() function.
+ *
+ *      max_lag:
+ *          An integer field indicating the maximum expected lag (in ticks)
+ *          between the writer and the readers.  This value must be at least 3, 
+ *          with 10 being the recommended minimum value.
+ * 
+ *      writer:
+ *          A boolean flag indicating whether the file opened with this FAPL entry
+ *          will be opened R/W. (i.e. as a VFD SWMR writer)
+ *
+ *      flush_raw_data:
+ *          A boolean flag indicating whether raw data should be flushed 
+ *          as part of the end of tick processing.  If set to TRUE, raw 
+ *          data will be flushed and thus be consistent with the metadata file.
+ *          However, this will also greatly increase end of tick I/O, and will
+ *          likely break any real time guarantees unless a very large tick_len 
+ *          is selected.
+ *
+ *      md_pages_reserved:
+ *          An integer field indicating the number of pages reserved 
+ *          at the head of the metadata file.  This value must be greater than 
+ *          or equal to 1.  
+ *          When the metadata file is created, the specified number of pages is 
+ *          reserved at the head of the metadata file.  In the current
+ *          implementation, the size of the metadata file header plus the 
+ *          index is limited to this size.
+ *          Further, in the POSIX case, when readers check for an updated index, 
+ *          this check will start with a read of md_pages_reserved pages from
+ *          the head of the metadata file.
+ *         
+ *      pb_expansion_threshold:
+ *          An integer field indicating the threshold for the page buffer size.
+ *          During a tick, the page buffer must expand as necessary to retain copies 
+ *          of all modified metadata pages and multi-page metadata entries.  
+ *          If the page buffer size exceeds this thresold, an early end of tick 
+ *          will be triggered.
+ *          Note that this is not a limit on the maximum page buffer size, as the 
+ *          metadata cache is flushed as part of end of tick processing.
+ *          This threshold must be in the range [0, 100].  If the threshold is 0, 
+ *          the feature is disabled.  For all other values, the page buffer size is 
+ *          multiplied by this threshold.  If this value is exceeded, an early end 
+ *          of tick is triggered. 
+ *
+ *      md_file_path:
+ *          POSIX: this field contains the path of the metadata file.  
+ *          NFS: it contains the path and base name of the metadata file 
+ *          updater files.
+ *          Object store: it contains the base URL for the objects used
+ *          to store metadata file updater objects.
+ *
+ *      log_file_path:
+ *          This field contains the path to the log file.  If defined, this path should 
+ *          be unique to each process.  If this field contains the empty string, a log 
+ *          file will not be created.
+ *
+ */
 typedef struct H5F_vfd_swmr_config_t {
     int32_t     version;
     uint32_t    tick_len;
     uint32_t    max_lag;
-    hbool_t     vfd_swmr_writer;/****/
+    hbool_t     writer;
     hbool_t     flush_raw_data;
     int32_t     md_pages_reserved;
+    int32_t     pb_expansion_threshold;
     char        md_file_path[H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1];
     char        log_file_path[H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1];
 } H5F_vfd_swmr_config_t;

diff --git a/src/H5PB.c b/src/H5PB.c
@@ -566,7 +566,7 @@ H5PB_create(H5F_t *f, size_t size, unsigned page_buf_min_meta_perc,
     /* compute vfd_swmr_writer */
     if ( ( H5F_VFD_SWMR_CONFIG(f) ) && ( H5F_INTENT(f) & H5F_ACC_RDWR ) ) {
 
-        HDassert(f->shared->vfd_swmr_config.vfd_swmr_writer);
+        HDassert(f->shared->vfd_swmr_config.writer);
         vfd_swmr_writer = TRUE;
     }
 
@@ -650,7 +650,7 @@ H5PB_create(H5F_t *f, size_t size, unsigned page_buf_min_meta_perc,
          ( 0 == (H5F_INTENT(f) & H5F_ACC_RDWR) ) ) {
 
         HDassert(f->shared->lf);
-        HDassert(! f->shared->vfd_swmr_config.vfd_swmr_writer);
+        HDassert(! f->shared->vfd_swmr_config.writer);
 
         H5FD_vfd_swmr_set_pb_configured(f->shared->lf);
     }

diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c
@@ -3974,9 +3974,10 @@ H5P__facc_vfd_swmr_config_enc(const void *value, void **_pp, size_t *size)
         INT32ENCODE(*pp, (int32_t)config->version);
         INT32ENCODE(*pp, (int32_t)config->tick_len);
         INT32ENCODE(*pp, (int32_t)config->max_lag);
-        H5_ENCODE_UNSIGNED(*pp, config->vfd_swmr_writer);
+        H5_ENCODE_UNSIGNED(*pp, config->writer);
         H5_ENCODE_UNSIGNED(*pp, config->flush_raw_data);
         INT32ENCODE(*pp, (int32_t)config->md_pages_reserved);
+        INT32ENCODE(*pp, (int32_t)config->pb_expansion_threshold);
         HDmemcpy(*pp, (const uint8_t *)(config->md_file_path), (size_t)(H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1));
         *pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1;
         HDmemcpy(*pp, (const uint8_t *)(config->log_file_path), (size_t)(H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1));
@@ -3985,7 +3986,7 @@ H5P__facc_vfd_swmr_config_enc(const void *value, void **_pp, size_t *size)
     } /* end if */
 
     /* Compute encoded size */
-    *size += ( (4 * sizeof(int32_t)) + 
+    *size += ( (5 * sizeof(int32_t)) + 
                (2 * sizeof(unsigned)) + 
                (2 * (H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1)) );
 
@@ -4028,11 +4029,12 @@ H5P__facc_vfd_swmr_config_dec(const void **_pp, void *_value)
     INT32DECODE(*pp, config->tick_len);
     INT32DECODE(*pp, config->max_lag);
 
-    H5_DECODE_UNSIGNED(*pp, config->vfd_swmr_writer);
+    H5_DECODE_UNSIGNED(*pp, config->writer);
     H5_DECODE_UNSIGNED(*pp, config->flush_raw_data);
 
     /* int */
     INT32DECODE(*pp, config->md_pages_reserved);
+    INT32DECODE(*pp, config->pb_expansion_threshold);
 
     HDstrcpy(config->md_file_path, (const char *)(*pp));
     *pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1;
@@ -5106,6 +5108,10 @@ H5Pset_vfd_swmr_config(hid_t plist_id, H5F_vfd_swmr_config_t *config_ptr)
     if(config_ptr->md_pages_reserved < 1 )
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "md_pages_reserved must be at least 1")
 
+    /* This field must be in the range [0, 100] */
+    if(config_ptr->pb_expansion_threshold < 0 || config_ptr->pb_expansion_threshold > H5F__MAX_PB_EXPANSION_THRESHOLD)
+        HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "pb_expansion_threshold out of range")
+
     /* Must provide the path for the metadata file */
     name_len = HDstrlen(config_ptr->md_file_path);
     if(name_len == 0)
@@ -5118,7 +5124,7 @@ H5Pset_vfd_swmr_config(hid_t plist_id, H5F_vfd_swmr_config_t *config_ptr)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set metadata cache initial config")
 
     /* Hard-wired to use SWMR VFD */
-    if(!config_ptr->vfd_swmr_writer) {
+    if(!config_ptr->writer) {
         if(H5P_set_driver(plist, H5FD_VFD_SWMR, NULL) < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set VFD SWMR driver info")
     }

diff --git a/test/page_buffer.c b/test/page_buffer.c
@@ -113,7 +113,7 @@ swmr_fapl_augment(hid_t fapl, const char *filename, uint32_t max_lag)
       .version = H5F__CURR_VFD_SWMR_CONFIG_VERSION
     , .tick_len = 4
     , .max_lag = 5
-    , .vfd_swmr_writer = true
+    , .writer = true
     , .md_pages_reserved = 128
     };
     const char *dname;

diff --git a/test/testfiles/plist_files/def_fapl_32be b/test/testfiles/plist_files/def_fapl_32be
diff --git a/test/testfiles/plist_files/def_fapl_32le b/test/testfiles/plist_files/def_fapl_32le
diff --git a/test/testfiles/plist_files/def_fapl_64be b/test/testfiles/plist_files/def_fapl_64be
diff --git a/test/testfiles/plist_files/def_fapl_64le b/test/testfiles/plist_files/def_fapl_64le
diff --git a/test/testfiles/plist_files/fapl_32be b/test/testfiles/plist_files/fapl_32be
diff --git a/test/testfiles/plist_files/fapl_32le b/test/testfiles/plist_files/fapl_32le
diff --git a/test/testfiles/plist_files/fapl_64be b/test/testfiles/plist_files/fapl_64be
diff --git a/test/testfiles/plist_files/fapl_64le b/test/testfiles/plist_files/fapl_64le
diff --git a/test/testfiles/plist_files/lapl_32be b/test/testfiles/plist_files/lapl_32be
diff --git a/test/testfiles/plist_files/lapl_32le b/test/testfiles/plist_files/lapl_32le
diff --git a/test/testfiles/plist_files/lapl_64be b/test/testfiles/plist_files/lapl_64be
diff --git a/test/testfiles/plist_files/lapl_64le b/test/testfiles/plist_files/lapl_64le
diff --git a/test/vfd_swmr.c b/test/vfd_swmr.c
@@ -135,7 +135,7 @@ test_fapl(void)
 
     /* Set md_file_path */
     HDstrcpy(my_config->md_file_path, MD_FILENAME);
-    my_config->vfd_swmr_writer = TRUE;
+    my_config->writer = TRUE;
 
     /* Should succeed in setting the configuration info */
     if(H5Pset_vfd_swmr_config(fapl, my_config) < 0)
@@ -242,7 +242,7 @@ test_file_fapl(void)
     config1->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     config1->tick_len = 4; 
     config1->max_lag = 6;
-    config1->vfd_swmr_writer = TRUE;
+    config1->writer = TRUE;
     config1->md_pages_reserved = 2;
     HDstrcpy(config1->md_file_path, MD_FILENAME);
 
@@ -332,7 +332,7 @@ test_file_fapl(void)
     config2->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     config2->tick_len = 4; 
     config2->max_lag = 10;
-    config2->vfd_swmr_writer = TRUE;
+    config2->writer = TRUE;
     config2->md_pages_reserved = 2;
     HDstrcpy(config2->md_file_path, MD_FILENAME);
 
@@ -378,7 +378,7 @@ test_file_fapl(void)
     config2->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     config2->tick_len = 4; 
     config2->max_lag = 10;
-    config2->vfd_swmr_writer = FALSE;
+    config2->writer = FALSE;
     config2->md_pages_reserved = 2;
     HDstrcpy(config2->md_file_path, MD_FILENAME);
 
@@ -409,10 +409,10 @@ test_file_fapl(void)
         TEST_ERROR;
 
     /* Verify that the retrieved config is a writer */
-    if(file_config->vfd_swmr_writer == FALSE)
+    if(file_config->writer == FALSE)
         TEST_ERROR;
     /* Verify that the retrieved config is not the same as the initial configuration */
-    if(file_config->vfd_swmr_writer == config2->vfd_swmr_writer)
+    if(file_config->writer == config2->writer)
         TEST_ERROR;
 
     /* Closing */
@@ -524,7 +524,7 @@ test_file_end_tick(void)
     my_config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     my_config->tick_len = 3; 
     my_config->max_lag = 3;
-    my_config->vfd_swmr_writer = TRUE;
+    my_config->writer = TRUE;
     my_config->md_pages_reserved = 2;
     HDstrcpy(my_config->md_file_path, MD_FILENAME);
 
@@ -655,7 +655,7 @@ test_writer_create_open_flush(void)
     my_config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     my_config->tick_len = 1; 
     my_config->max_lag = 3;
-    my_config->vfd_swmr_writer = TRUE;
+    my_config->writer = TRUE;
     my_config->md_pages_reserved = 1;
     HDstrcpy(my_config->md_file_path, MD_FILENAME);
 
@@ -792,7 +792,7 @@ test_writer_md(void)
     my_config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     my_config->tick_len = 1; 
     my_config->max_lag = 3;
-    my_config->vfd_swmr_writer = TRUE;
+    my_config->writer = TRUE;
     my_config->md_pages_reserved = 256;
     HDstrcpy(my_config->md_file_path, MD_FILENAME);
 
@@ -1075,7 +1075,7 @@ test_reader_md_concur(void)
     config_writer->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     config_writer->tick_len = 1; 
     config_writer->max_lag = 3;
-    config_writer->vfd_swmr_writer = TRUE;
+    config_writer->writer = TRUE;
     config_writer->md_pages_reserved = 256;
     HDstrcpy(config_writer->md_file_path, MD_FILENAME);
 
@@ -1160,7 +1160,7 @@ test_reader_md_concur(void)
         config_reader->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
         config_reader->tick_len = 1;
         config_reader->max_lag = 3;
-        config_reader->vfd_swmr_writer = FALSE;
+        config_reader->writer = FALSE;
         config_reader->md_pages_reserved = 256;
         HDstrcpy(config_reader->md_file_path, MD_FILENAME);
 

diff --git a/test/vfd_swmr_addrem_writer.c b/test/vfd_swmr_addrem_writer.c
@@ -125,7 +125,7 @@ open_skeleton(const char *filename, unsigned verbose)
     config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     config->tick_len = 4;
     config->max_lag = 5;
-    config->vfd_swmr_writer = TRUE;
+    config->writer = TRUE;
     config->md_pages_reserved = 128;
     HDstrcpy(config->md_file_path, "./my_md_file");
 

diff --git a/test/vfd_swmr_generator.c b/test/vfd_swmr_generator.c
@@ -178,7 +178,7 @@ gen_skeleton(const char *filename, hbool_t verbose, hbool_t vfd_swmr_write,
         config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
         config->tick_len = 4;
         config->max_lag = 10;
-        config->vfd_swmr_writer = TRUE;
+        config->writer = TRUE;
         config->md_pages_reserved = 128; 
         HDstrcpy(config->md_file_path, "my_md_file");
 

diff --git a/test/vfd_swmr_reader.c b/test/vfd_swmr_reader.c
@@ -311,7 +311,7 @@ read_records(const char *filename, hbool_t verbose, FILE *verbose_file,
     config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     config->tick_len = 4;
     config->max_lag = 5;
-    config->vfd_swmr_writer = FALSE;
+    config->writer = FALSE;
     config->md_pages_reserved = 128;
     HDstrcpy(config->md_file_path, "./my_md_file");
 

diff --git a/test/vfd_swmr_remove_reader.c b/test/vfd_swmr_remove_reader.c
@@ -294,7 +294,7 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds,
     config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     config->tick_len = 4;
     config->max_lag = 5;
-    config->vfd_swmr_writer = FALSE;
+    config->writer = FALSE;
     config->md_pages_reserved = 128;
     HDstrcpy(config->md_file_path, "./my_md_file");
 

diff --git a/test/vfd_swmr_remove_writer.c b/test/vfd_swmr_remove_writer.c
@@ -125,7 +125,7 @@ open_skeleton(const char *filename, unsigned verbose, unsigned old)
     config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     config->tick_len = 4;
     config->max_lag = 5;
-    config->vfd_swmr_writer = TRUE;
+    config->writer = TRUE;
     config->md_pages_reserved = 128;
     HDstrcpy(config->md_file_path, "./my_md_file");
 

diff --git a/test/vfd_swmr_sparse_reader.c b/test/vfd_swmr_sparse_reader.c
@@ -221,7 +221,7 @@ read_records(const char *filename, unsigned verbose, unsigned long nrecords,
     config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     config->tick_len = 4;
     config->max_lag = 5;
-    config->vfd_swmr_writer = FALSE;
+    config->writer = FALSE;
     config->md_pages_reserved = 128;
     HDstrcpy(config->md_file_path, "./my_md_file");
 

diff --git a/test/vfd_swmr_sparse_writer.c b/test/vfd_swmr_sparse_writer.c
@@ -126,7 +126,7 @@ open_skeleton(const char *filename, unsigned verbose)
     config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     config->tick_len = 4;
     config->max_lag = 5;
-    config->vfd_swmr_writer = TRUE;
+    config->writer = TRUE;
     config->md_pages_reserved = 128;
     HDstrcpy(config->md_file_path, "./my_md_file");
 

diff --git a/test/vfd_swmr_writer.c b/test/vfd_swmr_writer.c
@@ -129,7 +129,7 @@ open_skeleton(const char *filename, hbool_t verbose, FILE *verbose_file,
     config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     config->tick_len = 4;
     config->max_lag = 5;
-    config->vfd_swmr_writer = TRUE;
+    config->writer = TRUE;
     config->md_pages_reserved = 128;
     HDstrcpy(config->md_file_path, "./my_md_file");
 

diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c
@@ -461,11 +461,8 @@ swmr_fapl_augment(hid_t fapl, const char *fname)
     config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
     config->tick_len = 4;
     config->max_lag = 5;
-    config->vfd_swmr_writer = FALSE;
+    config->writer = FALSE;
     config->md_pages_reserved = 128;
-#if 0
-    config->md_open_tries = 1;
-#endif
 
     if ((tname = strdup(fname)) == NULL) {
         HDfprintf(rawerrorstream, "temporary string allocation failed\n");