Feature #2911 tc_stat_set_hdr (#2916)

* Per #2911, no real changes for Stat-Analysis. Just changing order of variables for consistency. * Per #2911, add StatHdrColumns::apply_set_hdr_opts(...) function to be used by TC-Stat. * Per #2911, move ByColumn to the TCStatJob base class and add HdrName and HdrValue to support the -set_hdr job command. * Per #2911, update GSI tools to call the newly added StatHdrColumns::apply_set_hdr_opts(...) function. * Per #2911, update logic of Stat-Analysis for consistency to make use of common apply_set_hdr_opts() function. * Per #2911, add DataLine::set_item() function to support -set_hdr options. * Per #2911, just update contents of error message * Per #2911, add TCStatLine member functions for has() and get_offset(). * Per #2911, update tc_stat to support applying -set_hdr to TC-Stat filter jobs. * Per #2911, revise TC-Stat config files to exercise the -set_hdr job command option * Per #2911, update TC-Stat documentation to mention the -set_hdr job command option * Per #2911, add note * Per #2911, as recommended by SonarQube, make some of these member functions const.
dtcenter · Jun 18, 2024 · 95ad048 · 95ad048
1 parent 4179f55
commit 95ad048
Show file tree

Hide file tree

Showing 23 changed files with 577 additions and 374 deletions.
diff --git a/docs/Users_Guide/tc-stat.rst b/docs/Users_Guide/tc-stat.rst
@@ -400,6 +400,8 @@ The output generated from the TC-Stat tool contains statistics produced by the a
 
 This job command finds and filters TCST lines down to those meeting the criteria selected by the filter's options. The filtered TCST lines are written to a file specified by the **-dump_row** option. The TCST output from this job follows the TCST output description in :numref:`tc-dland` and :numref:`tc-pairs`.
 
+ The "-set_hdr" job command option can be used to override any of the output header strings (e.g. "-set_hdr DESC EVENT_EQUAL" sets the output DESC column to "EVENT_EQUAL").
+
 **Job: Summary**
 
 This job produces summary statistics for the column name specified by the **-column** option. The output of the summary job consists of three rows:
@@ -475,6 +477,18 @@ Users may also specify the **-out_alpha** option to define the alpha value for t
 
 Users may also specify the **-out_stat** option to write the contingency table counts and statistics (for the CTC and CTS output line types) to an output STAT file. Information about the RIRW timing information and filtering criteria are written to the STAT header columns while the contingency table counts and/or statistics are written to the CTC and/or CTS output columns.
 
+When using the "-out_stat" option to create a .stat output file and stratifying results using one or more "-by" job command options, those columns may be referenced in the "-set_hdr" option.
+
+.. code-block:: none
+
+  -job rirw -line_type TCMPR -by CYCLONE -out_stat ctc.stat -set_hdr DESC CYCLONE
+
+When using multiple "-by" options, use "CASE" to reference the full case information string.
+
+.. code-block:: none
+
+  -job rirw -line_type TCMPR -by CYCLONE,LEAD -out_stat ctc.stat -set_hdr DESC CASE
+
 **Job: PROBRIRW**
 
 The PROBRIRW job produces probabilistic contingency table counts and statistics defined by placing forecast probabilities and BEST track rapid intensification events into an Nx2 contingency table. Users may specify several job command options to configure the behavior of this job:

diff --git a/internal/test_unit/config/TCStatConfig_ALAL2010 b/internal/test_unit/config/TCStatConfig_ALAL2010
@@ -208,11 +208,11 @@ jobs = [
    "-job summary -column TRACK -by AMODEL      -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_vs_BCLP.tcst",
    "-job summary -column TRACK -by AMODEL,INIT -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_vs_BCLP_by_INIT.tcst",
    "-job summary -column WIND  -by AMODEL      -column_union true",
-   "-job filter  -amodel AHWI  -rirw_track BDECK -rirw_thresh >=30  -rirw_exact FALSE -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_ri.tcst",
-   "-job filter  -amodel AHWI  -rirw_track BDECK -rirw_thresh <=-30 -rirw_exact TRUE  -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_rw.tcst",
+   "-job filter  -amodel AHWI  -rirw_track BDECK -rirw_thresh >=30  -rirw_exact FALSE -set_hdr DESC RI -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_ri.tcst",
+   "-job filter  -amodel AHWI  -rirw_track BDECK -rirw_thresh <=-30 -rirw_exact TRUE  -set_hdr DESC RW -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_rw.tcst",
    "-job rirw                  -rirw_window 00   -rirw_thresh <=-15 -out_line_type CTC,CTS,MPR",
    "-job rirw                  -rirw_window 12   -rirw_thresh <=-15 -out_line_type CTC,CTS,MPR",
-   "-job rirw                  -rirw_window 12   -rirw_thresh <=-15 -out_line_type CTC,CTS -by amodel -out_stat ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_rirw.stat"
+   "-job rirw                  -rirw_window 12   -rirw_thresh <=-15 -out_line_type CTC,CTS -by amodel -set_hdr DESC AMODEL -out_stat ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_rirw.stat"
 ];
 
 //

diff --git a/internal/test_unit/config/TCStatConfig_PROBRIRW b/internal/test_unit/config/TCStatConfig_PROBRIRW
@@ -205,7 +205,7 @@ out_valid_mask = "";
 // Array of TCStat analysis jobs to be performed on the filtered data
 //
 jobs = [
-   "-job filter   -dump_row ${MET_TEST_OUTPUT}/tc_stat/PROBRIRW_filter_ee.tcst",
+   "-job filter   -set_hdr DESC EVENT_EQUAL -dump_row ${MET_TEST_OUTPUT}/tc_stat/PROBRIRW_filter_ee.tcst",
    "-job summary  -column TK_ERR -by AMODEL -probrirw_thresh 30 -column_thresh PROBRIRW_PROB >0 -dump_row ${MET_TEST_OUTPUT}/tc_stat/PROBRIRW_summary_tk_err.tcst",
    "-job probrirw -column_thresh RIRW_WINDOW ==24 -by AMODEL -probrirw_thresh 30 -probrirw_bdelta_thresh >=30 -out_line_type PCT,PSTD,PRC,PJC -dump_row ${MET_TEST_OUTPUT}/tc_stat/PROBRIRW_probrirw.tcst",
    "-job summary  -column TK_ERR -by AMODEL,LEAD -amodel GPMI,GPMN -event_equal TRUE",

diff --git a/src/basic/vx_util/data_line.cc b/src/basic/vx_util/data_line.cc
@@ -232,6 +232,33 @@ return;
 ////////////////////////////////////////////////////////////////////////
 
 
+void DataLine::set_item(int k, const ConcatString &item_cs)
+
+{
+
+if ( (k < 0) || (k >= N_items) )  {
+
+   ConcatString cs = (File ? File->filename() : "");
+
+   mlog << Error << "\nDataLine::set_item(int) -> "
+        << "range check error setting line number " << LineNumber
+        << ", item number " << k+1 << " of " << N_items
+        << " from file \"" << cs << "\"\n\n";
+
+   exit ( 1 );
+
+}
+
+Items[k] = item_cs;
+
+return;
+
+}
+
+
+////////////////////////////////////////////////////////////////////////
+
+
 const char * DataLine::get_item(int k) const
 
 {

diff --git a/src/basic/vx_util/data_line.h b/src/basic/vx_util/data_line.h
@@ -98,6 +98,12 @@ class DataLine {
 
       void dump(std::ostream &, int depth = 0) const;
 
+         //
+         //  set stuff
+         //
+
+      void set_item(int, const ConcatString &);
+
          //
          //  retrieve stuff
          //

diff --git a/src/libcode/vx_analysis_util/stat_job.cc b/src/libcode/vx_analysis_util/stat_job.cc
@@ -1174,13 +1174,13 @@ void STATAnalysisJob::parse_job_command(const char *jobstring) {
       else if(jc_array[i] == "-column_str_exc" ) {
          column_str_exc_map.clear();
       }
+      else if(jc_array[i] == "-by"             ) {
+         by_column.clear();
+      }
       else if(jc_array[i] == "-set_hdr"        ) {
          hdr_name.clear();
          hdr_value.clear();
       }
-      else if(jc_array[i] == "-by"             ) {
-         by_column.clear();
-      }
       else if(jc_array[i] == "-out_line_type"  ) {
          out_line_type.clear();
       }
@@ -1450,15 +1450,15 @@ void STATAnalysisJob::parse_job_command(const char *jobstring) {
          }
          i+=2;
       }
+      else if(jc_array[i] == "-by") {
+         by_column.add_css(to_upper(jc_array[i+1]));
+         i+=1;
+      }
       else if(jc_array[i] == "-set_hdr") {
          hdr_name.add_css(to_upper(jc_array[i+1]));
          hdr_value.add_css(jc_array[i+2]);
          i+=2;
       }
-      else if(jc_array[i] == "-by") {
-         by_column.add_css(to_upper(jc_array[i+1]));
-         i+=1;
-      }
       else if(jc_array[i] == "-dump_row") {
          set_dump_row(jc_array[i+1].c_str());
          i++;

diff --git a/src/libcode/vx_analysis_util/stat_job.h b/src/libcode/vx_analysis_util/stat_job.h
@@ -234,14 +234,17 @@ class STATAnalysisJob {
       std::map<ConcatString,StringArray> column_str_inc_map;
       std::map<ConcatString,StringArray> column_str_exc_map;
 
-      StringArray hdr_name;
-      StringArray hdr_value;
-
       //
       // Store the case information for the -by option
       //
       StringArray by_column;
 
+      //
+      // Options for -set_hdr output
+      //
+      StringArray hdr_name;
+      StringArray hdr_value;
+
       //
       // Variables used to the store the analysis job specification
       //

diff --git a/src/libcode/vx_stat_out/stat_hdr_columns.cc b/src/libcode/vx_stat_out/stat_hdr_columns.cc
@@ -15,6 +15,9 @@
 
 using namespace std;
 
+////////////////////////////////////////////////////////////////////////
+
+static const string case_str = "CASE";
 
 ////////////////////////////////////////////////////////////////////////
 //
@@ -337,6 +340,217 @@ void StatHdrColumns::set_alpha(const double a) {
 
 ////////////////////////////////////////////////////////////////////////
 
+void StatHdrColumns::apply_set_hdr_opts(
+        const StringArray &hdr_cols, const StringArray &hdr_vals) {
+   StringArray case_cols;
+   StringArray case_vals;
+
+   // Call other implementation without case information
+   apply_set_hdr_opts(hdr_cols, hdr_vals, case_cols, case_vals);
+
+   return;
+}
+
+////////////////////////////////////////////////////////////////////////
+//
+// Use the current -set_hdr options to populate the STAT header columns,
+// substituting in case-specific values, as needed.
+//
+////////////////////////////////////////////////////////////////////////
+
+void StatHdrColumns::apply_set_hdr_opts(
+        const StringArray &hdr_cols, const StringArray &hdr_vals,
+        const StringArray &case_cols, const StringArray &case_vals) {
+
+   // No updates needed
+   if(hdr_cols.n() == 0) return;
+
+   // Sanity check lengths
+   if(hdr_cols.n() != hdr_vals.n()) {
+      mlog << Error << "\nStatHdrColumns::apply_set_hdr_opts() -> "
+           << "the number of -set_hdr columns names (" << hdr_cols.n()
+           << " and values (" << hdr_vals.n() << " must match!\n\n";
+      exit(1);
+   }
+   if(case_cols.n()  != case_vals.n()) {
+      mlog << Error << "\nStatHdrColumns::apply_set_hdr_opts() -> "
+           << "the number of case columns names (" << case_cols.n()
+           << " and values (" << case_vals.n() << " must match!\n\n";
+      exit(1);
+   }
+
+   int index;
+   ConcatString cs;
+   SingleThresh st;
+
+   // MODEL
+   if(hdr_cols.has("MODEL", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_model(cs.c_str());
+   }
+
+   // DESC
+   if(hdr_cols.has("DESC", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_desc(cs.c_str());
+   }
+
+   // FCST_LEAD
+   if(hdr_cols.has("FCST_LEAD", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_fcst_lead_sec(timestring_to_sec(cs.c_str()));
+   }
+
+   // FCST_VALID_BEG, FCST_VALID_END
+   if(hdr_cols.has("FCST_VALID_BEG", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_fcst_valid_beg(timestring_to_unix(cs.c_str()));
+   }
+   if(hdr_cols.has("FCST_VALID_END", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_fcst_valid_end(timestring_to_unix(cs.c_str()));
+   }
+
+   // OBS_LEAD
+   if(hdr_cols.has("OBS_LEAD", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_obs_lead_sec(timestring_to_sec(cs.c_str()));
+   }
+
+   // OBS_VALID_BEG, OBS_VALID_END
+   if(hdr_cols.has("OBS_VALID_BEG", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_obs_valid_beg(timestring_to_unix(cs.c_str()));
+   }
+   if(hdr_cols.has("OBS_VALID_END", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_obs_valid_end(timestring_to_unix(cs.c_str()));
+   }
+
+   // FCST_VAR
+   if(hdr_cols.has("FCST_VAR", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_fcst_var(cs.c_str());
+   }
+
+   // FCST_UNITS
+   if(hdr_cols.has("FCST_UNITS", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_fcst_units(cs.c_str());
+   }
+
+   // FCST_LEV
+   if(hdr_cols.has("FCST_LEV", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_fcst_lev(cs.c_str());
+   }
+
+   // OBS_VAR
+   if(hdr_cols.has("OBS_VAR", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_obs_var(cs.c_str());
+   }
+
+   // OBS_UNITS
+   if(hdr_cols.has("OBS_UNITS", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_obs_units(cs.c_str());
+   }
+
+   // OBS_LEV
+   if(hdr_cols.has("OBS_LEV", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_obs_lev(cs.c_str());
+   }
+
+   // OBTYPE
+   if(hdr_cols.has("OBTYPE", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_obtype(cs.c_str());
+   }
+
+   // VX_MASK
+   if(hdr_cols.has("VX_MASK", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_mask(cs.c_str());
+   }
+
+   // INTERP_MTHD
+   if(hdr_cols.has("INTERP_MTHD", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_interp_mthd(cs.c_str());
+   }
+
+   // INTERP_PNTS
+   if(hdr_cols.has("INTERP_PNTS", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_interp_wdth(nint(sqrt(atof(cs.c_str()))));
+   }
+
+   // FCST_THRESH
+   if(hdr_cols.has("FCST_THRESH", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      st.set(cs.c_str());
+      set_fcst_thresh(st);
+   }
+
+   // OBS_THRESH
+   if(hdr_cols.has("OBS_THRESH", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      st.set(cs.c_str());
+      set_obs_thresh(st);
+   }
+
+   // COV_THRESH
+   if(hdr_cols.has("COV_THRESH", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      st.set(cs.c_str());
+      set_cov_thresh(st);
+   }
+
+   // ALPHA
+   if(hdr_cols.has("ALPHA", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_alpha(atof(cs.c_str()));
+   }
+
+   // LINE_TYPE
+   if(hdr_cols.has("LINE_TYPE", index)) {
+      cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals);
+      set_line_type(cs.c_str());
+   }
+
+   return;
+}
+
+////////////////////////////////////////////////////////////////////////
+
+ConcatString StatHdrColumns::get_set_hdr_str(const std::string &hdr_val,
+                const StringArray &case_cols, const StringArray &case_vals) const {
+   ConcatString cs;
+   int index;
+
+   // Check for the full CASE string
+   if(case_str.compare(hdr_val) == 0) {
+      cs = case_vals.serialize(":");
+   }
+   // Check for one of the case columns
+   else if(case_cols.has(hdr_val, index)) {
+      cs = case_vals[index];
+   }
+   // Otherwise, use the current header value
+   else {
+      cs = hdr_val;
+   }
+
+   // Sanity check for empty strings
+   if(cs.empty()) cs = na_str;
+
+   return cs;
+}
+
+////////////////////////////////////////////////////////////////////////
+
 ConcatString StatHdrColumns::get_fcst_thresh_str() const {
    ConcatString cs;
 

diff --git a/src/libcode/vx_stat_out/stat_hdr_columns.h b/src/libcode/vx_stat_out/stat_hdr_columns.h
@@ -151,6 +151,14 @@ class StatHdrColumns {
       void set_cov_thresh    (const ThreshArray);
       void set_alpha         (const double);
 
+      // Apply -set_hdr overrides
+      void apply_set_hdr_opts(const StringArray &, const StringArray &);
+      void apply_set_hdr_opts(const StringArray &, const StringArray &,
+                              const StringArray &, const StringArray &);
+
+      ConcatString get_set_hdr_str(const std::string &,
+                                   const StringArray &, const StringArray &) const;
+
       // Get functions
       ConcatString get_model             () const;
       ConcatString get_desc              () const;