From 11486e26a23a9fe45199515e02c8062270cd8894 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Wed, 10 Mar 2021 11:07:21 -0600 Subject: [PATCH 1/4] Changing the signal handling to named pipe for better portablity. --- test/testvfdswmr.sh.in | 5 +- test/vfd_swmr_group_writer.c | 159 ++++++++++++++++++++++++++--------- 2 files changed, 122 insertions(+), 42 deletions(-) diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index a2e47b1fa6a..58c852ef671 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -616,11 +616,11 @@ fi if [ ${do_groups:-no} = yes ]; then echo launch vfd_swmr_group_writer catch_out_err_and_rc vfd_swmr_group_writer \ - ../vfd_swmr_group_writer -q -u 10 -n 10000 & + ../vfd_swmr_group_writer -q -c 1000 -n 10000 & pid_writer=$! catch_out_err_and_rc vfd_swmr_group_reader \ - ../vfd_swmr_group_reader -q -u 10 -n 10000 -W & + ../vfd_swmr_group_reader -q -c 1000 -n 10000 & pid_reader=$! # Wait for the reader to finish before signalling the @@ -628,7 +628,6 @@ if [ ${do_groups:-no} = yes ]; then # reader will find the shadow file when it opens # the .h5 file. wait $pid_reader - kill -USR1 $(cat vfd_swmr_group_writer.pid) wait $pid_writer # Collect exit code of the reader diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 2f355c2c1c2..112bcb411b9 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -12,13 +12,13 @@ */ #include +#include /* getopt(3) */ #define H5F_FRIEND /*suppress error about including H5Fpkg */ #include "hdf5.h" #include "H5Fpkg.h" -// #include "H5Iprivate.h" #include "H5HGprivate.h" #include "H5VLprivate.h" @@ -29,8 +29,8 @@ typedef struct { hid_t file, filetype, one_by_one_sid; char filename[PATH_MAX]; char progname[PATH_MAX]; - struct timespec update_interval; unsigned int asteps; + unsigned int csteps; unsigned int nsteps; bool wait_for_signal; bool use_vfd_swmr; @@ -42,12 +42,10 @@ typedef struct { , .filename = "" \ , .filetype = H5T_NATIVE_UINT32 \ , .asteps = 10 \ + , .csteps = 10 \ , .nsteps = 100 \ , .wait_for_signal = true \ - , .use_vfd_swmr = true \ - , .update_interval = (struct timespec){ \ - .tv_sec = 0 \ - , .tv_nsec = 1000000000UL / 30 /* 1/30 second */}} + , .use_vfd_swmr = true} static void state_init(state_t *, int, char **); @@ -56,7 +54,7 @@ static const hid_t badhid = H5I_INVALID_HID; static void usage(const char *progname) { - fprintf(stderr, "usage: %s [-S] [-W] [-a steps] [-b]\n" + fprintf(stderr, "usage: %s [-S] [-W] [-a steps] [-b] [-c]\n" " [-n iterations] [-u milliseconds]\n" "\n" "-S: do not use VFD SWMR\n" @@ -64,6 +62,7 @@ usage(const char *progname) " exiting\n" "-a steps: `steps` between adding attributes\n" "-b: write data in big-endian byte order\n" + "-c steps: `steps` between communication between the writer and reader\n" "-n iterations: how many times to expand each dataset\n" "-u ms: milliseconds interval between updates\n" " to %s.h5\n" @@ -80,13 +79,12 @@ state_init(state_t *s, int argc, char **argv) const hsize_t dims = 1; char tfile[PATH_MAX]; char *end; - unsigned long millis; *s = ALL_HID_INITIALIZER; esnprintf(tfile, sizeof(tfile), "%s", argv[0]); esnprintf(s->progname, sizeof(s->progname), "%s", HDbasename(tfile)); - while ((ch = getopt(argc, argv, "SWa:bn:qu:")) != -1) { + while ((ch = getopt(argc, argv, "SWa:bc:n:qu:")) != -1) { switch (ch) { case 'S': s->use_vfd_swmr = false; @@ -95,6 +93,7 @@ state_init(state_t *s, int argc, char **argv) s->wait_for_signal = false; break; case 'a': + case 'c': case 'n': errno = 0; tmp = strtoul(optarg, &end, 0); @@ -109,7 +108,9 @@ state_init(state_t *s, int argc, char **argv) if (ch == 'a') s->asteps = (unsigned)tmp; - else + else if (ch == 'c') + s->csteps = (unsigned)tmp; + else if (ch == 'n') s->nsteps = (unsigned)tmp; break; case 'b': @@ -118,21 +119,6 @@ state_init(state_t *s, int argc, char **argv) case 'q': verbosity = 0; break; - case 'u': - errno = 0; - millis = strtoul(optarg, &end, 0); - if (millis == ULONG_MAX && errno == ERANGE) { - err(EXIT_FAILURE, - "option -p argument \"%s\"", optarg); - } else if (*end != '\0') { - errx(EXIT_FAILURE, - "garbage after -p argument \"%s\"", optarg); - } - s->update_interval.tv_sec = (time_t)(millis / 1000UL); - s->update_interval.tv_nsec = - (long)((millis * 1000000UL) % 1000000000UL); - dbgf(1, "%lu milliseconds between updates\n", millis); - break; case '?': default: usage(s->progname); @@ -146,6 +132,12 @@ state_init(state_t *s, int argc, char **argv) if ((s->one_by_one_sid = H5Screate_simple(1, &dims, &dims)) < 0) errx(EXIT_FAILURE, "H5Screate_simple failed"); + if( s->csteps < 1 || s->csteps > s->nsteps) + errx(EXIT_FAILURE, "communication interval is out of bounds"); + + if( s->asteps < 1 || s->asteps > s->nsteps) + errx(EXIT_FAILURE, "attribute interval is out of bounds"); + if (argc > 0) errx(EXIT_FAILURE, "unexpected command-line arguments"); @@ -259,17 +251,30 @@ verify_group(state_t *s, unsigned int which) return result; } +/* Sleep for `tenths` tenths of a second */ +static void +decisleep(uint32_t tenths) +{ + uint64_t nsec = tenths * 100 * 1000 * 1000; + + H5_nanosleep(nsec); +} + int main(int argc, char **argv) { hid_t fapl, fcpl; - sigset_t oldsigs; herr_t ret; unsigned step; bool writer; state_t s; const char *personality; H5F_vfd_swmr_config_t config; + const char *fifo_writer_to_reader = "./fifo_group_writer_to_reader"; + const char *fifo_reader_to_writer = "./fifo_group_reader_to_writer"; + int fd_writer_to_reader, fd_reader_to_writer; + int notify = 0, verify = 0; + unsigned int i; state_init(&s, argc, argv); @@ -310,27 +315,87 @@ main(int argc, char **argv) if (s.file == badhid) errx(EXIT_FAILURE, writer ? "H5Fcreate" : "H5Fopen"); - block_signals(&oldsigs); + /* Writer creates two named pipes(FIFO) to coordinate two-way communication */ + if (writer) { + if (HDmkfifo(fifo_writer_to_reader, 0600) < 0) + errx(EXIT_FAILURE, "HDmkfifo"); + + if (HDmkfifo(fifo_reader_to_writer, 0600) < 0) + errx(EXIT_FAILURE, "HDmkfifo"); + } + + /* Both the writer and reader open the pipes */ + if ((fd_writer_to_reader = HDopen(fifo_writer_to_reader, O_RDWR)) < 0) + errx(EXIT_FAILURE, "fifo_writer_to_reader open failed"); + + if ((fd_reader_to_writer = HDopen(fifo_reader_to_writer, O_RDWR)) < 0) + errx(EXIT_FAILURE, "fifo_reader_to_writer open failed"); if (writer) { + /* Writer tells reader to start */ + if (HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) + err(EXIT_FAILURE, "write failed"); + for (step = 0; step < s.nsteps; step++) { - dbgf(2, "step %d\n", step); + dbgf(2, "writer: step %d\n", step); + write_group(&s, step); - nanosleep(&s.update_interval, NULL); + + /* At communication interval, notifies the reader and waits for its response */ + if (step % s.csteps == 0) { + /* Bump up the value and notify the reader */ + notify++; + if (HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) + err(EXIT_FAILURE, "write failed"); + + /* During the wait, writer makes repeated HDF5 API calls + * to trigger EOT at approximately the correct time */ + for(i = 0; i < config.max_lag + 1; i++) { + decisleep(config.tick_len); + H5Aexists(s.file, "nonexistent"); + } + + /* Receive the same value from the reader and verify it */ + verify++; + if (HDread(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) + err(EXIT_FAILURE, "read failed"); + + if (notify != verify) + errx(EXIT_FAILURE, "received message %d, expecting %d", notify, verify); + } } } else { - for (step = 0; step < s.nsteps;) { - dbgf(2, "step %d\n", step); - if (verify_group(&s, step)) - step++; - nanosleep(&s.update_interval, NULL); - } - } + /* Start to verify group creation after receiving the writer's notice */ + if (HDread(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) + err(EXIT_FAILURE, "read failed"); - if (s.use_vfd_swmr && s.wait_for_signal) - await_signal(s.file); + /* Both notify and verify are 0 now */ + if (notify != verify) + errx(EXIT_FAILURE, "received message %d, expecting %d", notify, verify); + + for (step = 0; step < s.nsteps; step++) { + dbgf(2, "reader: step %d\n", step); - restore_signals(&oldsigs); + while (!verify_group(&s, step)) + ; + + /* At communication interval, waits for the writer's notice and responds back */ + if (step % s.csteps == 0) { + verify++; + + /* Receive the notify that the writer bumped up the value */ + if (HDread(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) + err(EXIT_FAILURE, "read failed"); + + if (notify != verify) + errx(EXIT_FAILURE, "received message %d, expecting %d", notify, verify); + + /* Send back the same nofity value for acknowledgement */ + if (HDwrite(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) + err(EXIT_FAILURE, "write failed"); + } + } + } if (H5Pclose(fapl) < 0) errx(EXIT_FAILURE, "H5Pclose(fapl)"); @@ -341,5 +406,21 @@ main(int argc, char **argv) if (H5Fclose(s.file) < 0) errx(EXIT_FAILURE, "H5Fclose"); + /* Close the named pipes */ + if (HDclose(fd_writer_to_reader) < 0) + errx(EXIT_FAILURE, "HDclose"); + + if (HDclose(fd_reader_to_writer) < 0) + errx(EXIT_FAILURE, "HDclose"); + + /* Reader finishes last and deletes the named pipes */ + if(!writer) { + if(HDremove(fifo_writer_to_reader) != 0) + errx(EXIT_FAILURE, "fifo_writer_to_reader deletion failed"); + + if(HDremove(fifo_reader_to_writer) != 0) + errx(EXIT_FAILURE, "fifo_reader_to_writer deletion failed"); + } + return EXIT_SUCCESS; } From 923e15b749a8a587cae3d0b738efd5f85bae30d3 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Mon, 15 Mar 2021 10:49:43 -0500 Subject: [PATCH 2/4] Some minor changes. --- test/vfd_swmr_group_writer.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 112bcb411b9..a3b1833d5a3 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -12,7 +12,6 @@ */ #include -#include /* getopt(3) */ #define H5F_FRIEND /*suppress error about including H5Fpkg */ @@ -55,7 +54,7 @@ static void usage(const char *progname) { fprintf(stderr, "usage: %s [-S] [-W] [-a steps] [-b] [-c]\n" - " [-n iterations] [-u milliseconds]\n" + " [-n iterations]\n" "\n" "-S: do not use VFD SWMR\n" "-W: do not wait for a signal before\n" @@ -64,7 +63,6 @@ usage(const char *progname) "-b: write data in big-endian byte order\n" "-c steps: `steps` between communication between the writer and reader\n" "-n iterations: how many times to expand each dataset\n" - "-u ms: milliseconds interval between updates\n" " to %s.h5\n" "\n", progname, progname); @@ -84,7 +82,7 @@ state_init(state_t *s, int argc, char **argv) esnprintf(tfile, sizeof(tfile), "%s", argv[0]); esnprintf(s->progname, sizeof(s->progname), "%s", HDbasename(tfile)); - while ((ch = getopt(argc, argv, "SWa:bc:n:qu:")) != -1) { + while ((ch = getopt(argc, argv, "SWa:bc:n:q")) != -1) { switch (ch) { case 'S': s->use_vfd_swmr = false; @@ -315,8 +313,12 @@ main(int argc, char **argv) if (s.file == badhid) errx(EXIT_FAILURE, writer ? "H5Fcreate" : "H5Fopen"); - /* Writer creates two named pipes(FIFO) to coordinate two-way communication */ + /* Use two named pipes(FIFO) to coordinate the writer and reader for + * two-way communication so that the two sides can move forward together. + * One is for the writer to write to the reader. + * The other one is for the reader to signal the writer. */ if (writer) { + /* Writer creates two named pipes(FIFO) */ if (HDmkfifo(fifo_writer_to_reader, 0600) < 0) errx(EXIT_FAILURE, "HDmkfifo"); @@ -343,7 +345,7 @@ main(int argc, char **argv) /* At communication interval, notifies the reader and waits for its response */ if (step % s.csteps == 0) { - /* Bump up the value and notify the reader */ + /* Bump up the value of notify to notice the reader to start to read */ notify++; if (HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) err(EXIT_FAILURE, "write failed"); @@ -381,9 +383,10 @@ main(int argc, char **argv) /* At communication interval, waits for the writer's notice and responds back */ if (step % s.csteps == 0) { + /* The writer should have bumped up the value of notify. + * Do the same with verify and confirm it */ verify++; - /* Receive the notify that the writer bumped up the value */ if (HDread(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) err(EXIT_FAILURE, "read failed"); @@ -406,7 +409,7 @@ main(int argc, char **argv) if (H5Fclose(s.file) < 0) errx(EXIT_FAILURE, "H5Fclose"); - /* Close the named pipes */ + /* Both the writer and reader close the named pipes */ if (HDclose(fd_writer_to_reader) < 0) errx(EXIT_FAILURE, "HDclose"); From 20cbe6895872b162ca04ed371d211c8c5124e0db Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Mon, 15 Mar 2021 11:01:25 -0500 Subject: [PATCH 3/4] Adding more comments. --- test/vfd_swmr_group_writer.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index a3b1833d5a3..5e01c4eaf74 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -334,7 +334,7 @@ main(int argc, char **argv) errx(EXIT_FAILURE, "fifo_reader_to_writer open failed"); if (writer) { - /* Writer tells reader to start */ + /* Writer tells reader to start verification */ if (HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) err(EXIT_FAILURE, "write failed"); @@ -357,7 +357,8 @@ main(int argc, char **argv) H5Aexists(s.file, "nonexistent"); } - /* Receive the same value from the reader and verify it */ + /* Receive the same value from the reader and verify it before + * going to the next step */ verify++; if (HDread(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) err(EXIT_FAILURE, "read failed"); @@ -393,7 +394,8 @@ main(int argc, char **argv) if (notify != verify) errx(EXIT_FAILURE, "received message %d, expecting %d", notify, verify); - /* Send back the same nofity value for acknowledgement */ + /* Send back the same nofity value for acknowledgement to tell the writer + * move to the next step */ if (HDwrite(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) err(EXIT_FAILURE, "write failed"); } From 796018e5582d9c3d0d009cb58e4d4898723b0d32 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Tue, 16 Mar 2021 20:39:01 -0500 Subject: [PATCH 4/4] Re-arranged the communication between the writer and reader through named pipes to facilitate possible test improvements in the future. --- test/vfd_swmr_group_writer.c | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 5e01c4eaf74..3dd84ab9d21 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -62,10 +62,9 @@ usage(const char *progname) "-a steps: `steps` between adding attributes\n" "-b: write data in big-endian byte order\n" "-c steps: `steps` between communication between the writer and reader\n" - "-n iterations: how many times to expand each dataset\n" - " to %s.h5\n" + "-n ngroups: the number of groups\n" "\n", - progname, progname); + progname); exit(EXIT_FAILURE); } @@ -334,10 +333,6 @@ main(int argc, char **argv) errx(EXIT_FAILURE, "fifo_reader_to_writer open failed"); if (writer) { - /* Writer tells reader to start verification */ - if (HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) - err(EXIT_FAILURE, "write failed"); - for (step = 0; step < s.nsteps; step++) { dbgf(2, "writer: step %d\n", step); @@ -368,34 +363,29 @@ main(int argc, char **argv) } } } else { - /* Start to verify group creation after receiving the writer's notice */ - if (HDread(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) - err(EXIT_FAILURE, "read failed"); - - /* Both notify and verify are 0 now */ - if (notify != verify) - errx(EXIT_FAILURE, "received message %d, expecting %d", notify, verify); - for (step = 0; step < s.nsteps; step++) { dbgf(2, "reader: step %d\n", step); - while (!verify_group(&s, step)) - ; - - /* At communication interval, waits for the writer's notice and responds back */ + /* At communication interval, waits for the writer to finish creation before starting verification */ if (step % s.csteps == 0) { /* The writer should have bumped up the value of notify. * Do the same with verify and confirm it */ verify++; + /* Receive the notify that the writer bumped up the value */ if (HDread(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) err(EXIT_FAILURE, "read failed"); if (notify != verify) errx(EXIT_FAILURE, "received message %d, expecting %d", notify, verify); + } + while (!verify_group(&s, step)) + ; + + if (step % s.csteps == 0) { /* Send back the same nofity value for acknowledgement to tell the writer - * move to the next step */ + * move to the next step. */ if (HDwrite(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) err(EXIT_FAILURE, "write failed"); }