From 95ad048b51080747e953e7f086a68b94bce622cf Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 18 Jun 2024 12:06:32 -0600 Subject: [PATCH] Feature #2911 tc_stat_set_hdr (#2916) * Per #2911, no real changes for Stat-Analysis. Just changing order of variables for consistency. * Per #2911, add StatHdrColumns::apply_set_hdr_opts(...) function to be used by TC-Stat. * Per #2911, move ByColumn to the TCStatJob base class and add HdrName and HdrValue to support the -set_hdr job command. * Per #2911, update GSI tools to call the newly added StatHdrColumns::apply_set_hdr_opts(...) function. * Per #2911, update logic of Stat-Analysis for consistency to make use of common apply_set_hdr_opts() function. * Per #2911, add DataLine::set_item() function to support -set_hdr options. * Per #2911, just update contents of error message * Per #2911, add TCStatLine member functions for has() and get_offset(). * Per #2911, update tc_stat to support applying -set_hdr to TC-Stat filter jobs. * Per #2911, revise TC-Stat config files to exercise the -set_hdr job command option * Per #2911, update TC-Stat documentation to mention the -set_hdr job command option * Per #2911, add note * Per #2911, as recommended by SonarQube, make some of these member functions const. --- docs/Users_Guide/tc-stat.rst | 14 ++ .../test_unit/config/TCStatConfig_ALAL2010 | 6 +- .../test_unit/config/TCStatConfig_PROBRIRW | 2 +- src/basic/vx_util/data_line.cc | 27 +++ src/basic/vx_util/data_line.h | 6 + src/libcode/vx_analysis_util/stat_job.cc | 14 +- src/libcode/vx_analysis_util/stat_job.h | 9 +- src/libcode/vx_stat_out/stat_hdr_columns.cc | 214 ++++++++++++++++++ src/libcode/vx_stat_out/stat_hdr_columns.h | 8 + src/libcode/vx_tc_util/tc_columns.cc | 19 +- src/libcode/vx_tc_util/tc_columns.h | 2 + src/libcode/vx_tc_util/tc_hdr_columns.cc | 86 +++++++ src/libcode/vx_tc_util/tc_hdr_columns.h | 3 + src/libcode/vx_tc_util/tc_stat_line.cc | 30 +++ src/libcode/vx_tc_util/tc_stat_line.h | 2 + .../core/stat_analysis/aggr_stat_line.cc | 153 +++---------- src/tools/core/stat_analysis/aggr_stat_line.h | 8 +- src/tools/other/gsi_tools/gsi_util.cc | 167 ++------------ src/tools/other/gsi_tools/gsi_util.h | 28 +-- src/tools/other/gsi_tools/gsid2mpr.cc | 27 ++- src/tools/tc_utils/tc_stat/tc_stat.cc | 1 + src/tools/tc_utils/tc_stat/tc_stat_job.cc | 103 +++++---- src/tools/tc_utils/tc_stat/tc_stat_job.h | 22 +- 23 files changed, 577 insertions(+), 374 deletions(-) diff --git a/docs/Users_Guide/tc-stat.rst b/docs/Users_Guide/tc-stat.rst index bdcbc35327..3cddda63f2 100644 --- a/docs/Users_Guide/tc-stat.rst +++ b/docs/Users_Guide/tc-stat.rst @@ -400,6 +400,8 @@ The output generated from the TC-Stat tool contains statistics produced by the a This job command finds and filters TCST lines down to those meeting the criteria selected by the filter's options. The filtered TCST lines are written to a file specified by the **-dump_row** option. The TCST output from this job follows the TCST output description in :numref:`tc-dland` and :numref:`tc-pairs`. + The "-set_hdr" job command option can be used to override any of the output header strings (e.g. "-set_hdr DESC EVENT_EQUAL" sets the output DESC column to "EVENT_EQUAL"). + **Job: Summary** This job produces summary statistics for the column name specified by the **-column** option. The output of the summary job consists of three rows: @@ -475,6 +477,18 @@ Users may also specify the **-out_alpha** option to define the alpha value for t Users may also specify the **-out_stat** option to write the contingency table counts and statistics (for the CTC and CTS output line types) to an output STAT file. Information about the RIRW timing information and filtering criteria are written to the STAT header columns while the contingency table counts and/or statistics are written to the CTC and/or CTS output columns. +When using the "-out_stat" option to create a .stat output file and stratifying results using one or more "-by" job command options, those columns may be referenced in the "-set_hdr" option. + +.. code-block:: none + + -job rirw -line_type TCMPR -by CYCLONE -out_stat ctc.stat -set_hdr DESC CYCLONE + +When using multiple "-by" options, use "CASE" to reference the full case information string. + +.. code-block:: none + + -job rirw -line_type TCMPR -by CYCLONE,LEAD -out_stat ctc.stat -set_hdr DESC CASE + **Job: PROBRIRW** The PROBRIRW job produces probabilistic contingency table counts and statistics defined by placing forecast probabilities and BEST track rapid intensification events into an Nx2 contingency table. Users may specify several job command options to configure the behavior of this job: diff --git a/internal/test_unit/config/TCStatConfig_ALAL2010 b/internal/test_unit/config/TCStatConfig_ALAL2010 index dd0de6084d..b46d0f72c3 100644 --- a/internal/test_unit/config/TCStatConfig_ALAL2010 +++ b/internal/test_unit/config/TCStatConfig_ALAL2010 @@ -208,11 +208,11 @@ jobs = [ "-job summary -column TRACK -by AMODEL -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_vs_BCLP.tcst", "-job summary -column TRACK -by AMODEL,INIT -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_vs_BCLP_by_INIT.tcst", "-job summary -column WIND -by AMODEL -column_union true", - "-job filter -amodel AHWI -rirw_track BDECK -rirw_thresh >=30 -rirw_exact FALSE -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_ri.tcst", - "-job filter -amodel AHWI -rirw_track BDECK -rirw_thresh <=-30 -rirw_exact TRUE -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_rw.tcst", + "-job filter -amodel AHWI -rirw_track BDECK -rirw_thresh >=30 -rirw_exact FALSE -set_hdr DESC RI -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_ri.tcst", + "-job filter -amodel AHWI -rirw_track BDECK -rirw_thresh <=-30 -rirw_exact TRUE -set_hdr DESC RW -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_rw.tcst", "-job rirw -rirw_window 00 -rirw_thresh <=-15 -out_line_type CTC,CTS,MPR", "-job rirw -rirw_window 12 -rirw_thresh <=-15 -out_line_type CTC,CTS,MPR", - "-job rirw -rirw_window 12 -rirw_thresh <=-15 -out_line_type CTC,CTS -by amodel -out_stat ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_rirw.stat" + "-job rirw -rirw_window 12 -rirw_thresh <=-15 -out_line_type CTC,CTS -by amodel -set_hdr DESC AMODEL -out_stat ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_rirw.stat" ]; // diff --git a/internal/test_unit/config/TCStatConfig_PROBRIRW b/internal/test_unit/config/TCStatConfig_PROBRIRW index f8645a1e1e..679b681e59 100644 --- a/internal/test_unit/config/TCStatConfig_PROBRIRW +++ b/internal/test_unit/config/TCStatConfig_PROBRIRW @@ -205,7 +205,7 @@ out_valid_mask = ""; // Array of TCStat analysis jobs to be performed on the filtered data // jobs = [ - "-job filter -dump_row ${MET_TEST_OUTPUT}/tc_stat/PROBRIRW_filter_ee.tcst", + "-job filter -set_hdr DESC EVENT_EQUAL -dump_row ${MET_TEST_OUTPUT}/tc_stat/PROBRIRW_filter_ee.tcst", "-job summary -column TK_ERR -by AMODEL -probrirw_thresh 30 -column_thresh PROBRIRW_PROB >0 -dump_row ${MET_TEST_OUTPUT}/tc_stat/PROBRIRW_summary_tk_err.tcst", "-job probrirw -column_thresh RIRW_WINDOW ==24 -by AMODEL -probrirw_thresh 30 -probrirw_bdelta_thresh >=30 -out_line_type PCT,PSTD,PRC,PJC -dump_row ${MET_TEST_OUTPUT}/tc_stat/PROBRIRW_probrirw.tcst", "-job summary -column TK_ERR -by AMODEL,LEAD -amodel GPMI,GPMN -event_equal TRUE", diff --git a/src/basic/vx_util/data_line.cc b/src/basic/vx_util/data_line.cc index 62b7e0c923..5c283e258d 100644 --- a/src/basic/vx_util/data_line.cc +++ b/src/basic/vx_util/data_line.cc @@ -232,6 +232,33 @@ return; //////////////////////////////////////////////////////////////////////// +void DataLine::set_item(int k, const ConcatString &item_cs) + +{ + +if ( (k < 0) || (k >= N_items) ) { + + ConcatString cs = (File ? File->filename() : ""); + + mlog << Error << "\nDataLine::set_item(int) -> " + << "range check error setting line number " << LineNumber + << ", item number " << k+1 << " of " << N_items + << " from file \"" << cs << "\"\n\n"; + + exit ( 1 ); + +} + +Items[k] = item_cs; + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + + const char * DataLine::get_item(int k) const { diff --git a/src/basic/vx_util/data_line.h b/src/basic/vx_util/data_line.h index bb9de49c8e..b3ef288b5e 100644 --- a/src/basic/vx_util/data_line.h +++ b/src/basic/vx_util/data_line.h @@ -98,6 +98,12 @@ class DataLine { void dump(std::ostream &, int depth = 0) const; + // + // set stuff + // + + void set_item(int, const ConcatString &); + // // retrieve stuff // diff --git a/src/libcode/vx_analysis_util/stat_job.cc b/src/libcode/vx_analysis_util/stat_job.cc index dda0a08bd9..ced4361068 100644 --- a/src/libcode/vx_analysis_util/stat_job.cc +++ b/src/libcode/vx_analysis_util/stat_job.cc @@ -1174,13 +1174,13 @@ void STATAnalysisJob::parse_job_command(const char *jobstring) { else if(jc_array[i] == "-column_str_exc" ) { column_str_exc_map.clear(); } + else if(jc_array[i] == "-by" ) { + by_column.clear(); + } else if(jc_array[i] == "-set_hdr" ) { hdr_name.clear(); hdr_value.clear(); } - else if(jc_array[i] == "-by" ) { - by_column.clear(); - } else if(jc_array[i] == "-out_line_type" ) { out_line_type.clear(); } @@ -1450,15 +1450,15 @@ void STATAnalysisJob::parse_job_command(const char *jobstring) { } i+=2; } + else if(jc_array[i] == "-by") { + by_column.add_css(to_upper(jc_array[i+1])); + i+=1; + } else if(jc_array[i] == "-set_hdr") { hdr_name.add_css(to_upper(jc_array[i+1])); hdr_value.add_css(jc_array[i+2]); i+=2; } - else if(jc_array[i] == "-by") { - by_column.add_css(to_upper(jc_array[i+1])); - i+=1; - } else if(jc_array[i] == "-dump_row") { set_dump_row(jc_array[i+1].c_str()); i++; diff --git a/src/libcode/vx_analysis_util/stat_job.h b/src/libcode/vx_analysis_util/stat_job.h index 208c6d4894..88b709adf7 100644 --- a/src/libcode/vx_analysis_util/stat_job.h +++ b/src/libcode/vx_analysis_util/stat_job.h @@ -234,14 +234,17 @@ class STATAnalysisJob { std::map column_str_inc_map; std::map column_str_exc_map; - StringArray hdr_name; - StringArray hdr_value; - // // Store the case information for the -by option // StringArray by_column; + // + // Options for -set_hdr output + // + StringArray hdr_name; + StringArray hdr_value; + // // Variables used to the store the analysis job specification // diff --git a/src/libcode/vx_stat_out/stat_hdr_columns.cc b/src/libcode/vx_stat_out/stat_hdr_columns.cc index 005499d8ed..3c3560b587 100644 --- a/src/libcode/vx_stat_out/stat_hdr_columns.cc +++ b/src/libcode/vx_stat_out/stat_hdr_columns.cc @@ -15,6 +15,9 @@ using namespace std; +//////////////////////////////////////////////////////////////////////// + +static const string case_str = "CASE"; //////////////////////////////////////////////////////////////////////// // @@ -337,6 +340,217 @@ void StatHdrColumns::set_alpha(const double a) { //////////////////////////////////////////////////////////////////////// +void StatHdrColumns::apply_set_hdr_opts( + const StringArray &hdr_cols, const StringArray &hdr_vals) { + StringArray case_cols; + StringArray case_vals; + + // Call other implementation without case information + apply_set_hdr_opts(hdr_cols, hdr_vals, case_cols, case_vals); + + return; +} + +//////////////////////////////////////////////////////////////////////// +// +// Use the current -set_hdr options to populate the STAT header columns, +// substituting in case-specific values, as needed. +// +//////////////////////////////////////////////////////////////////////// + +void StatHdrColumns::apply_set_hdr_opts( + const StringArray &hdr_cols, const StringArray &hdr_vals, + const StringArray &case_cols, const StringArray &case_vals) { + + // No updates needed + if(hdr_cols.n() == 0) return; + + // Sanity check lengths + if(hdr_cols.n() != hdr_vals.n()) { + mlog << Error << "\nStatHdrColumns::apply_set_hdr_opts() -> " + << "the number of -set_hdr columns names (" << hdr_cols.n() + << " and values (" << hdr_vals.n() << " must match!\n\n"; + exit(1); + } + if(case_cols.n() != case_vals.n()) { + mlog << Error << "\nStatHdrColumns::apply_set_hdr_opts() -> " + << "the number of case columns names (" << case_cols.n() + << " and values (" << case_vals.n() << " must match!\n\n"; + exit(1); + } + + int index; + ConcatString cs; + SingleThresh st; + + // MODEL + if(hdr_cols.has("MODEL", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_model(cs.c_str()); + } + + // DESC + if(hdr_cols.has("DESC", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_desc(cs.c_str()); + } + + // FCST_LEAD + if(hdr_cols.has("FCST_LEAD", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_fcst_lead_sec(timestring_to_sec(cs.c_str())); + } + + // FCST_VALID_BEG, FCST_VALID_END + if(hdr_cols.has("FCST_VALID_BEG", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_fcst_valid_beg(timestring_to_unix(cs.c_str())); + } + if(hdr_cols.has("FCST_VALID_END", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_fcst_valid_end(timestring_to_unix(cs.c_str())); + } + + // OBS_LEAD + if(hdr_cols.has("OBS_LEAD", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obs_lead_sec(timestring_to_sec(cs.c_str())); + } + + // OBS_VALID_BEG, OBS_VALID_END + if(hdr_cols.has("OBS_VALID_BEG", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obs_valid_beg(timestring_to_unix(cs.c_str())); + } + if(hdr_cols.has("OBS_VALID_END", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obs_valid_end(timestring_to_unix(cs.c_str())); + } + + // FCST_VAR + if(hdr_cols.has("FCST_VAR", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_fcst_var(cs.c_str()); + } + + // FCST_UNITS + if(hdr_cols.has("FCST_UNITS", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_fcst_units(cs.c_str()); + } + + // FCST_LEV + if(hdr_cols.has("FCST_LEV", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_fcst_lev(cs.c_str()); + } + + // OBS_VAR + if(hdr_cols.has("OBS_VAR", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obs_var(cs.c_str()); + } + + // OBS_UNITS + if(hdr_cols.has("OBS_UNITS", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obs_units(cs.c_str()); + } + + // OBS_LEV + if(hdr_cols.has("OBS_LEV", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obs_lev(cs.c_str()); + } + + // OBTYPE + if(hdr_cols.has("OBTYPE", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obtype(cs.c_str()); + } + + // VX_MASK + if(hdr_cols.has("VX_MASK", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_mask(cs.c_str()); + } + + // INTERP_MTHD + if(hdr_cols.has("INTERP_MTHD", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_interp_mthd(cs.c_str()); + } + + // INTERP_PNTS + if(hdr_cols.has("INTERP_PNTS", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_interp_wdth(nint(sqrt(atof(cs.c_str())))); + } + + // FCST_THRESH + if(hdr_cols.has("FCST_THRESH", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + st.set(cs.c_str()); + set_fcst_thresh(st); + } + + // OBS_THRESH + if(hdr_cols.has("OBS_THRESH", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + st.set(cs.c_str()); + set_obs_thresh(st); + } + + // COV_THRESH + if(hdr_cols.has("COV_THRESH", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + st.set(cs.c_str()); + set_cov_thresh(st); + } + + // ALPHA + if(hdr_cols.has("ALPHA", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_alpha(atof(cs.c_str())); + } + + // LINE_TYPE + if(hdr_cols.has("LINE_TYPE", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_line_type(cs.c_str()); + } + + return; +} + +//////////////////////////////////////////////////////////////////////// + +ConcatString StatHdrColumns::get_set_hdr_str(const std::string &hdr_val, + const StringArray &case_cols, const StringArray &case_vals) const { + ConcatString cs; + int index; + + // Check for the full CASE string + if(case_str.compare(hdr_val) == 0) { + cs = case_vals.serialize(":"); + } + // Check for one of the case columns + else if(case_cols.has(hdr_val, index)) { + cs = case_vals[index]; + } + // Otherwise, use the current header value + else { + cs = hdr_val; + } + + // Sanity check for empty strings + if(cs.empty()) cs = na_str; + + return cs; +} + +//////////////////////////////////////////////////////////////////////// + ConcatString StatHdrColumns::get_fcst_thresh_str() const { ConcatString cs; diff --git a/src/libcode/vx_stat_out/stat_hdr_columns.h b/src/libcode/vx_stat_out/stat_hdr_columns.h index 6a1c5da0fb..8fb9e4f564 100644 --- a/src/libcode/vx_stat_out/stat_hdr_columns.h +++ b/src/libcode/vx_stat_out/stat_hdr_columns.h @@ -151,6 +151,14 @@ class StatHdrColumns { void set_cov_thresh (const ThreshArray); void set_alpha (const double); + // Apply -set_hdr overrides + void apply_set_hdr_opts(const StringArray &, const StringArray &); + void apply_set_hdr_opts(const StringArray &, const StringArray &, + const StringArray &, const StringArray &); + + ConcatString get_set_hdr_str(const std::string &, + const StringArray &, const StringArray &) const; + // Get functions ConcatString get_model () const; ConcatString get_desc () const; diff --git a/src/libcode/vx_tc_util/tc_columns.cc b/src/libcode/vx_tc_util/tc_columns.cc index 6f7e7f2d6c..f91431718f 100644 --- a/src/libcode/vx_tc_util/tc_columns.cc +++ b/src/libcode/vx_tc_util/tc_columns.cc @@ -170,7 +170,8 @@ void write_prob_rirw_header_row(int hdr_flag, int n_thresh, AsciiTable &at, //////////////////////////////////////////////////////////////////////// void write_track_pair_info(TcHdrColumns &hdr, const TrackPairInfo &p, - AsciiTable &at, int &i_row) { + AsciiTable &at, int &i_row, + const StringArray &hdr_name, const StringArray &hdr_value) { // Loop through the TrackPairInfo points for(int i=0; iget_item("DESC", false)); } + // Apply -set_hdr options + if(hdr_name.n() > 0) hdr.apply_set_hdr_opts(hdr_name, hdr_value); + // Write the header columns write_tc_header_cols(hdr, at, i_row); @@ -221,6 +225,19 @@ void write_track_pair_info(TcHdrColumns &hdr, const TrackPairInfo &p, //////////////////////////////////////////////////////////////////////// +void write_track_pair_info(TcHdrColumns &hdr, const TrackPairInfo &p, + AsciiTable &at, int &i_row) { + + StringArray hdr_name; + StringArray hdr_value; + + write_track_pair_info(hdr, p, at, i_row, hdr_name, hdr_value); + + return; +} + +//////////////////////////////////////////////////////////////////////// + void write_prob_rirw_pair_info(TcHdrColumns &hdr, const ProbRIRWPairInfo &p, AsciiTable &at, int &i_row) { diff --git a/src/libcode/vx_tc_util/tc_columns.h b/src/libcode/vx_tc_util/tc_columns.h index 9a5aee0181..0d3a83e0a8 100644 --- a/src/libcode/vx_tc_util/tc_columns.h +++ b/src/libcode/vx_tc_util/tc_columns.h @@ -166,6 +166,8 @@ extern void write_prob_rirw_header_row(int, int, AsciiTable &, int, int); // Write out the data lines extern void write_track_pair_info (TcHdrColumns &, const TrackPairInfo &, AsciiTable &, int &); +extern void write_track_pair_info (TcHdrColumns &, const TrackPairInfo &, AsciiTable &, int &, + const StringArray &, const StringArray &); extern void write_prob_rirw_pair_info(TcHdrColumns &, const ProbRIRWPairInfo &, AsciiTable &, int &); // Write out the header entries diff --git a/src/libcode/vx_tc_util/tc_hdr_columns.cc b/src/libcode/vx_tc_util/tc_hdr_columns.cc index 3708417560..9cb4c1a77b 100644 --- a/src/libcode/vx_tc_util/tc_hdr_columns.cc +++ b/src/libcode/vx_tc_util/tc_hdr_columns.cc @@ -64,3 +64,89 @@ void TcHdrColumns::clear() { } //////////////////////////////////////////////////////////////////////// + +void TcHdrColumns::apply_set_hdr_opts( + const StringArray &hdr_cols, const StringArray &hdr_vals) { + + // No updates needed + if(hdr_cols.n() == 0) return; + + int index; + + // Sanity check lengths + if(hdr_cols.n() != hdr_vals.n()) { + mlog << Error << "\nTcHdrColumns::apply_set_hdr_opts() -> " + << "the number of -set_hdr columns names (" << hdr_cols.n() + << " and values (" << hdr_vals.n() << " must match!\n\n"; + exit(1); + } + + // AMODEL + if(hdr_cols.has("AMODEL", index)) { + set_adeck_model(hdr_vals[index]); + } + + // BMODEL + if(hdr_cols.has("BMODEL", index)) { + set_bdeck_model(hdr_vals[index]); + } + + // DESC + if(hdr_cols.has("DESC", index)) { + set_desc(hdr_vals[index]); + } + + // STORM_ID + if(hdr_cols.has("STORM_ID", index)) { + set_storm_id(hdr_vals[index]); + } + + // BASIN + if(hdr_cols.has("BASIN", index)) { + set_basin(hdr_vals[index]); + } + + // CYCLONE + if(hdr_cols.has("CYCLONE", index)) { + set_cyclone(hdr_vals[index]); + } + + // STORM_NAME + if(hdr_cols.has("STORM_NAME", index)) { + set_storm_name(hdr_vals[index]); + } + + // INIT + if(hdr_cols.has("INIT", index)) { + set_init(timestring_to_sec(hdr_vals[index].c_str())); + } + + // LEAD + if(hdr_cols.has("LEAD", index)) { + set_lead(timestring_to_sec(hdr_vals[index].c_str())); + } + + // VALID + if(hdr_cols.has("VALID", index)) { + set_valid(timestring_to_sec(hdr_vals[index].c_str())); + } + + // INIT_MASK + if(hdr_cols.has("INIT_MASK", index)) { + set_init_mask(hdr_vals[index]); + } + + // VALID_MASK + if(hdr_cols.has("VALID_MASK", index)) { + set_valid_mask(hdr_vals[index]); + } + + // LINE_TYPE + if(hdr_cols.has("LINE_TYPE", index)) { + set_line_type(hdr_vals[index]); + } + + return; +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_tc_util/tc_hdr_columns.h b/src/libcode/vx_tc_util/tc_hdr_columns.h index f632dfaf56..60ecf39260 100644 --- a/src/libcode/vx_tc_util/tc_hdr_columns.h +++ b/src/libcode/vx_tc_util/tc_hdr_columns.h @@ -70,6 +70,9 @@ class TcHdrColumns { void set_valid_mask (const ConcatString &); void set_line_type (const ConcatString &); + // Apply -set_hdr overrides + void apply_set_hdr_opts(const StringArray &, const StringArray &); + // Get functions ConcatString adeck_model () const; ConcatString bdeck_model () const; diff --git a/src/libcode/vx_tc_util/tc_stat_line.cc b/src/libcode/vx_tc_util/tc_stat_line.cc index a8556a7dd2..d46dc41c77 100644 --- a/src/libcode/vx_tc_util/tc_stat_line.cc +++ b/src/libcode/vx_tc_util/tc_stat_line.cc @@ -164,6 +164,36 @@ bool TCStatLine::is_header() const { //////////////////////////////////////////////////////////////////////// +bool TCStatLine::has(const char *col_str) const { + return !is_bad_data(get_offset(col_str)); +} + +//////////////////////////////////////////////////////////////////////// + +int TCStatLine::get_offset(const char *col_str) const { + int offset = bad_data_int; + + // + // Search for matching header column + // + offset = HdrLine->col_offset(col_str); + + // + // If not found, check extra header columns + // + if(is_bad_data(offset)) { + if(!get_file()->header().has(col_str, offset)) offset = bad_data_int; + } + + // + // Return the offset value + // + + return offset; +} + +//////////////////////////////////////////////////////////////////////// + ConcatString TCStatLine::get(const char *col_str, bool check_na) const { ConcatString cs = (string)get_item(col_str, check_na); diff --git a/src/libcode/vx_tc_util/tc_stat_line.h b/src/libcode/vx_tc_util/tc_stat_line.h index 276fad1638..e39dcf7f70 100644 --- a/src/libcode/vx_tc_util/tc_stat_line.h +++ b/src/libcode/vx_tc_util/tc_stat_line.h @@ -75,6 +75,8 @@ class TCStatLine : public DataLine { // Retrieve values of the header columns // + bool has (const char *) const; + int get_offset (const char *) const; ConcatString get (const char *, bool check_na = true) const; const char * get_item (const char *, bool check_na = true) const; const char * get_item (int, bool check_na = true) const; diff --git a/src/tools/core/stat_analysis/aggr_stat_line.cc b/src/tools/core/stat_analysis/aggr_stat_line.cc index c46c35c134..c40fb0eb64 100644 --- a/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -41,6 +41,7 @@ // to VL1L2, VAL1L2, and VCNT. // 019 02/21/24 Halley Gotway MET #2583 Add observation error // ECNT statistics. +// 020 06/14/24 Halley Gotway MET #2911 Call apply_set_hdr_opts(). // //////////////////////////////////////////////////////////////////////// @@ -62,7 +63,6 @@ using namespace std; - //////////////////////////////////////////////////////////////////////// static bool is_precip_var_name(const ConcatString &s); @@ -350,25 +350,17 @@ StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, const StringArray &hdr_cols, const StringArray &hdr_vals, const STATLineType lt) { - ConcatString css; - StringArray case_vals; ThreshArray ta; + ConcatString css; double out_alpha; - int index, wdth; + int wdth; StatHdrColumns shc; - // Split up the current case into values - case_vals = cur_case.split(":"); - // MODEL - shc.set_model( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "MODEL", model, false).c_str()); + shc.set_model(get_col_css(cur_case, "MODEL", model, false).c_str()); // DESC - shc.set_desc( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "DESC", desc, false).c_str()); + shc.set_desc(get_col_css(cur_case, "DESC", desc, false).c_str()); // FCST_LEAD css = write_css_hhmmss(fcst_lead); @@ -378,26 +370,11 @@ StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, << fcst_lead.n() << " unique FCST_LEAD values: " << css << "\n"; } - if(hdr_cols.has("FCST_LEAD", index)) { - shc.set_fcst_lead_sec(timestring_to_sec(hdr_vals[index].c_str())); - } - else { - shc.set_fcst_lead_sec(fcst_lead.max()); - } + shc.set_fcst_lead_sec(fcst_lead.max()); // FCST_VALID_BEG, FCST_VALID_END - if(hdr_cols.has("FCST_VALID_BEG", index)) { - shc.set_fcst_valid_beg(timestring_to_unix(hdr_vals[index].c_str())); - } - else { - shc.set_fcst_valid_beg(fcst_valid_beg); - } - if(hdr_cols.has("FCST_VALID_END", index)) { - shc.set_fcst_valid_end(timestring_to_unix(hdr_vals[index].c_str())); - } - else { - shc.set_fcst_valid_end(fcst_valid_end); - } + shc.set_fcst_valid_beg(fcst_valid_beg); + shc.set_fcst_valid_end(fcst_valid_end); // OBS_LEAD css = write_css_hhmmss(obs_lead); @@ -407,71 +384,38 @@ StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, << obs_lead.n() << " unique OBS_LEAD values: " << css << "\n"; } - if(hdr_cols.has("OBS_LEAD", index)) { - shc.set_obs_lead_sec(timestring_to_sec(hdr_vals[index].c_str())); - } - else { - shc.set_obs_lead_sec(obs_lead.max()); - } + shc.set_obs_lead_sec(obs_lead.max()); // OBS_VALID_BEG, OBS_VALID_END - if(hdr_cols.has("OBS_VALID_BEG", index)) { - shc.set_obs_valid_beg(timestring_to_unix(hdr_vals[index].c_str())); - } - else { - shc.set_obs_valid_beg(obs_valid_beg); - } - if(hdr_cols.has("OBS_VALID_END", index)) { - shc.set_obs_valid_end(timestring_to_unix(hdr_vals[index].c_str())); - } - else { - shc.set_obs_valid_end(obs_valid_end); - } + shc.set_obs_valid_beg(obs_valid_beg); + shc.set_obs_valid_end(obs_valid_end); // FCST_VAR - shc.set_fcst_var( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "FCST_VAR", fcst_var, false)); + shc.set_fcst_var(get_col_css(cur_case, "FCST_VAR", fcst_var, false)); // FCST_UNITS - shc.set_fcst_units( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "FCST_UNITS", fcst_units, false)); + shc.set_fcst_units(get_col_css(cur_case, "FCST_UNITS", fcst_units, false)); // FCST_LEV - shc.set_fcst_lev( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "FCST_LEV", fcst_lev, false).c_str()); + shc.set_fcst_lev(get_col_css(cur_case, "FCST_LEV", fcst_lev, false).c_str()); // OBS_VAR - shc.set_obs_var( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "OBS_VAR", obs_var, false)); + shc.set_obs_var(get_col_css(cur_case, "OBS_VAR", obs_var, false)); // OBS_UNITS - shc.set_obs_units( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "OBS_UNITS", obs_units, false)); + shc.set_obs_units(get_col_css(cur_case, "OBS_UNITS", obs_units, false)); // OBS_LEV - shc.set_obs_lev( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "OBS_LEV", obs_lev, false).c_str()); + shc.set_obs_lev(get_col_css(cur_case, "OBS_LEV", obs_lev, false).c_str()); // OBTYPE - shc.set_obtype( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "OBTYPE", obtype, false).c_str()); + shc.set_obtype(get_col_css(cur_case, "OBTYPE", obtype, false).c_str()); // VX_MASK - shc.set_mask( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "VX_MASK", vx_mask, false).c_str()); + shc.set_mask(get_col_css(cur_case, "VX_MASK", vx_mask, false).c_str()); // INTERP_MTHD - shc.set_interp_mthd( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "INTERP_MTHD", interp_mthd, true)); + shc.set_interp_mthd(get_col_css(cur_case, "INTERP_MTHD", interp_mthd, true)); // INTERP_PNTS css = write_css(interp_pnts); @@ -485,28 +429,21 @@ StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, else { wdth = nint(sqrt(interp_pnts[0])); } - - if(hdr_cols.has("INTERP_PNTS", index)) { - wdth = nint(sqrt(atof(hdr_vals[index].c_str()))); - } shc.set_interp_wdth(wdth); // FCST_THRESH ta.clear(); - ta.add_css(get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "FCST_THRESH", fcst_thresh, true).c_str()); + ta.add_css(get_col_css(cur_case, "FCST_THRESH", fcst_thresh, true).c_str()); shc.set_fcst_thresh(ta); // OBS_THRESH ta.clear(); - ta.add_css(get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "OBS_THRESH", obs_thresh, true).c_str()); + ta.add_css(get_col_css(cur_case, "OBS_THRESH", obs_thresh, true).c_str()); shc.set_obs_thresh(ta); // COV_THRESH ta.clear(); - ta.add_css(get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "COV_THRESH", cov_thresh, true).c_str()); + ta.add_css(get_col_css(cur_case, "COV_THRESH", cov_thresh, true).c_str()); shc.set_cov_thresh(ta); // ALPHA @@ -521,34 +458,27 @@ StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, else { out_alpha = alpha[0]; } - - if(hdr_cols.has("ALPHA", index)) { - out_alpha = atof(hdr_vals[index].c_str()); - } shc.set_alpha(out_alpha); // LINE_TYPE shc.set_line_type(statlinetype_to_string(lt)); + // Apply the -set_hdr options + StringArray case_vals = cur_case.split(":"); + shc.apply_set_hdr_opts(hdr_cols, hdr_vals, case_cols, case_vals); + return shc; } //////////////////////////////////////////////////////////////////////// -ConcatString StatHdrInfo::get_shc_str(const ConcatString &cur_case, - const StringArray &case_cols, - const StringArray &case_vals, - const StringArray &hdr_cols, - const StringArray &hdr_vals, +ConcatString StatHdrInfo::get_col_css(const ConcatString &cur_case, const char *col_name, const StringArray &col_vals, - bool warning) { - ConcatString css, shc_str; - int hdr_index, case_index; - + bool warning) const { // Build comma-separated list of column values - css = write_css(col_vals); + ConcatString css(write_css(col_vals)); // Check for multiple entries if(col_vals.n() > 1) { @@ -560,28 +490,7 @@ ConcatString StatHdrInfo::get_shc_str(const ConcatString &cur_case, else mlog << Debug(2) << msg; } - // Check the header options. - if(hdr_cols.has(col_name, hdr_index)) { - - // Check for the full CASE string. - if(case_str.compare(hdr_vals[hdr_index]) == 0) { - shc_str = cur_case; - } - // Check for one of the case columns. - else if(case_cols.has(hdr_vals[hdr_index], case_index)) { - shc_str = case_vals[case_index]; - } - // Otherwise, use the constant header string. - else { - shc_str = hdr_vals[hdr_index]; - } - } - // Otherwise, use the comma-separated list of values. - else { - shc_str = css; - } - - return shc_str; + return css; } //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/stat_analysis/aggr_stat_line.h b/src/tools/core/stat_analysis/aggr_stat_line.h index a934491a1f..e79c0155eb 100644 --- a/src/tools/core/stat_analysis/aggr_stat_line.h +++ b/src/tools/core/stat_analysis/aggr_stat_line.h @@ -80,14 +80,10 @@ struct StatHdrInfo { const StringArray &hdr_cols, const StringArray &hdr_vals, const STATLineType lt); - ConcatString get_shc_str(const ConcatString &cur_case, - const StringArray &case_cols, - const StringArray &case_vals, - const StringArray &hdr_cols, - const StringArray &hdr_vals, + ConcatString get_col_css(const ConcatString &cur_case, const char *col_name, const StringArray &col_vals, - bool warning); + bool warning) const; }; struct AggrSummaryInfo { diff --git a/src/tools/other/gsi_tools/gsi_util.cc b/src/tools/other/gsi_tools/gsi_util.cc index 5381ebe2d1..82b4000ae9 100644 --- a/src/tools/other/gsi_tools/gsi_util.cc +++ b/src/tools/other/gsi_tools/gsi_util.cc @@ -301,152 +301,35 @@ bool is_retr(const char *s) { void setup_header(StatHdrColumns &shc, const StringArray &name, const StringArray &value, const char *default_line_type) { - int index; - SingleThresh st; - - // MODEL - if(name.has("MODEL", index)) { - shc.set_model(value[index].c_str()); - } - else { - shc.set_model(default_model); - } - - // DESC - if(name.has("DESC", index)) { - shc.set_desc(value[index].c_str()); - } - else { - shc.set_desc(default_desc); - } - - // FCST_LEAD - if(name.has("FCST_LEAD", index)) { - shc.set_fcst_lead_sec(timestring_to_sec(value[index].c_str())); - } - else { - shc.set_fcst_lead_sec(default_lead); - } - - // FCST_VALID_BEG, FCST_VALID_END - if(name.has("FCST_VALID_BEG", index)) { - shc.set_fcst_valid_beg(timestring_to_unix(value[index].c_str())); - not_has_FCST_VALID_BEG = false; - } - if(name.has("FCST_VALID_END", index)) { - shc.set_fcst_valid_end(timestring_to_unix(value[index].c_str())); - not_has_FCST_VALID_END = false; - } - - // OBS_LEAD - if(name.has("OBS_LEAD", index)) { - shc.set_obs_lead_sec(timestring_to_sec(value[index].c_str())); - } - else { - shc.set_obs_lead_sec(default_lead); - } - - // OBS_VALID_BEG, OBS_VALID_END - if(name.has("OBS_VALID_BEG", index)) { - shc.set_obs_valid_beg(timestring_to_unix(value[index].c_str())); - not_has_OBS_VALID_BEG = false; - } - if(name.has("OBS_VALID_END", index)) { - shc.set_obs_valid_end(timestring_to_unix(value[index].c_str())); - not_has_OBS_VALID_END = false; - } - - // FCST_VAR - if(name.has("FCST_VAR", index)) { - shc.set_fcst_var(value[index]); - not_has_FCST_VAR = false; - } - - // FCST_LEV - if(name.has("FCST_LEV", index)) { - shc.set_fcst_lev(value[index].c_str()); - } - else { - shc.set_fcst_lev(default_lev); - } - - // OBS_VAR - if(name.has("OBS_VAR", index)) { - shc.set_obs_var(value[index]); - not_has_OBS_VAR = false; - } - - // OBS_LEV - if(name.has("OBS_LEV", index)) { - shc.set_obs_lev(value[index].c_str()); - } - else { - shc.set_obs_lev(default_lev); - } - - // OBTYPE - if(name.has("OBTYPE", index)) { - shc.set_obtype(value[index].c_str()); - not_has_OBTYPE = false; - } - else { - shc.set_obtype(default_obtype); - } - - // VX_MASK - if(name.has("VX_MASK", index)) { - shc.set_mask(value[index].c_str()); - } - else { - shc.set_mask(default_vx_mask); - } - - // INTERP_MTHD - if(name.has("INTERP_MTHD", index)) { - shc.set_interp_mthd(value[index]); - } - else { - shc.set_interp_mthd((string)default_interp_mthd); - } - - // INTERP_PNTS - if(name.has("INTERP_PNTS", index)) { - shc.set_interp_wdth(nint(sqrt(atof(value[index].c_str())))); - } - else { - shc.set_interp_wdth(default_interp_wdth); - } - - // FCST_THRESH - if(name.has("FCST_THRESH", index)) st.set(value[index].c_str()); - else st.set(default_thresh); + SingleThresh st(na_str); + + // Initialize the header + shc.set_model("GSI"); + shc.set_desc(na_str); + shc.set_fcst_lead_sec(0); + shc.set_fcst_valid_beg(0); + shc.set_fcst_valid_end(0); + shc.set_obs_lead_sec(0); + shc.set_obs_valid_beg(0); + shc.set_obs_valid_end(0); + shc.set_fcst_var(na_str); + shc.set_fcst_units(na_str); + shc.set_fcst_lev(na_str); + shc.set_obs_var(na_str); + shc.set_obs_units(na_str); + shc.set_obs_lev(na_str); + shc.set_obtype(na_str); + shc.set_mask(na_str); + shc.set_interp_mthd(na_str); + shc.set_interp_wdth(0); shc.set_fcst_thresh(st); - - // OBS_THRESH - if(name.has("OBS_THRESH", index)) st.set(value[index].c_str()); - else st.set(default_thresh); shc.set_obs_thresh(st); - - // COV_THRESH - if(name.has("COV_THRESH", index)) st.set(value[index].c_str()); - else st.set(default_thresh); shc.set_cov_thresh(st); + shc.set_alpha(bad_data_double); + shc.set_line_type(default_line_type); - // ALPHA - if(name.has("ALPHA", index)) { - shc.set_alpha(atof(value[index].c_str())); - } - else { - shc.set_alpha(default_alpha); - } - - // LINE_TYPE - if(name.has("LINE_TYPE", index)) { - shc.set_line_type(value[index].c_str()); - } - else { - shc.set_line_type(default_line_type); - } + // Apply the -set_hdr options + shc.apply_set_hdr_opts(name, value); return; } diff --git a/src/tools/other/gsi_tools/gsi_util.h b/src/tools/other/gsi_tools/gsi_util.h index b635884ed2..dfd4390247 100644 --- a/src/tools/other/gsi_tools/gsi_util.h +++ b/src/tools/other/gsi_tools/gsi_util.h @@ -18,21 +18,9 @@ //////////////////////////////////////////////////////////////////////// // Constants -static const char default_model[] = "GSI"; -static const char default_desc[] = "NA"; -static const int default_lead = 0; -static const char default_lev[] = "NA"; -static const char default_obtype[] = "NA"; -static const char default_vx_mask[] = "NA"; -static const char default_interp_mthd[] = "NA"; -static const int default_interp_wdth = 0; -static const char default_thresh[] = "NA"; -static const double default_alpha = bad_data_double; - -static const int bad_setup_qc = -999; -static const char key_sep[] = ":"; - -static const char conv_id_str[] = "conv"; +static const int bad_setup_qc = -999; +static const char key_sep[] = ":"; +static const char conv_id_str[] = "conv"; static const char *micro_id_str [] = { "amsua", "amsub", "mhs", @@ -83,16 +71,6 @@ struct RadData { //////////////////////////////////////////////////////////////////////// -static bool not_has_FCST_VALID_BEG = true; -static bool not_has_FCST_VALID_END = true; -static bool not_has_OBS_VALID_BEG = true; -static bool not_has_OBS_VALID_END = true; -static bool not_has_FCST_VAR = true; -static bool not_has_OBS_VAR = true; -static bool not_has_OBTYPE = true; - -//////////////////////////////////////////////////////////////////////// - ConvData parse_conv_data(const ConvRecord &r, const int i); RadData parse_rad_data (const RadRecord &r, const int i, const int chval, const int use); diff --git a/src/tools/other/gsi_tools/gsid2mpr.cc b/src/tools/other/gsi_tools/gsid2mpr.cc index 8915b73ce6..579d41bd19 100644 --- a/src/tools/other/gsi_tools/gsid2mpr.cc +++ b/src/tools/other/gsi_tools/gsid2mpr.cc @@ -47,7 +47,6 @@ using namespace std; - //////////////////////////////////////////////////////////////////////// static void process_conv(const char *conv_filename, const char *output_filename); @@ -365,13 +364,13 @@ void write_mpr_row_conv(AsciiTable &at, int row, const ConvData &d) { int col; // Update header for current data - if(not_has_FCST_VALID_BEG) shc.set_fcst_valid_beg(d.fcst_ut); - if(not_has_FCST_VALID_END) shc.set_fcst_valid_end(d.fcst_ut); - if(not_has_OBS_VALID_BEG) shc.set_obs_valid_beg(d.obs_ut); - if(not_has_OBS_VALID_END) shc.set_obs_valid_end(d.obs_ut); - if(not_has_FCST_VAR) shc.set_fcst_var(d.var); - if(not_has_OBS_VAR) shc.set_obs_var(d.var); - if(not_has_OBTYPE) shc.set_obtype(d.obtype.c_str()); + if(!hdr_name.has("FCST_VALID_BEG")) shc.set_fcst_valid_beg(d.fcst_ut); + if(!hdr_name.has("FCST_VALID_END")) shc.set_fcst_valid_end(d.fcst_ut); + if(!hdr_name.has("OBS_VALID_BEG")) shc.set_obs_valid_beg(d.obs_ut); + if(!hdr_name.has("OBS_VALID_END")) shc.set_obs_valid_end(d.obs_ut); + if(!hdr_name.has("FCST_VAR")) shc.set_fcst_var(d.var); + if(!hdr_name.has("OBS_VAR")) shc.set_obs_var(d.var); + if(!hdr_name.has("OBTYPE")) shc.set_obtype(d.obtype.c_str()); // Write header columns write_header_cols(shc, at, row); @@ -420,12 +419,12 @@ void write_mpr_row_rad(AsciiTable &at, int row, const RadData & d) { int col; // Update header for current data - if(not_has_FCST_VALID_BEG) shc.set_fcst_valid_beg(d.fcst_ut); - if(not_has_FCST_VALID_END) shc.set_fcst_valid_end(d.fcst_ut); - if(not_has_OBS_VALID_BEG) shc.set_obs_valid_beg(d.obs_ut); - if(not_has_OBS_VALID_END) shc.set_obs_valid_end(d.obs_ut); - if(not_has_FCST_VAR) shc.set_fcst_var(d.var); - if(not_has_OBS_VAR) shc.set_obs_var(d.var); + if(!hdr_name.has("FCST_VALID_BEG")) shc.set_fcst_valid_beg(d.fcst_ut); + if(!hdr_name.has("FCST_VALID_END")) shc.set_fcst_valid_end(d.fcst_ut); + if(!hdr_name.has("OBS_VALID_BEG")) shc.set_obs_valid_beg(d.obs_ut); + if(!hdr_name.has("OBS_VALID_END")) shc.set_obs_valid_end(d.obs_ut); + if(!hdr_name.has("FCST_VAR")) shc.set_fcst_var(d.var); + if(!hdr_name.has("OBS_VAR")) shc.set_obs_var(d.var); // Write header columns write_header_cols(shc, at, row); diff --git a/src/tools/tc_utils/tc_stat/tc_stat.cc b/src/tools/tc_utils/tc_stat/tc_stat.cc index 28025107aa..a5592c7b15 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat.cc @@ -22,6 +22,7 @@ // 004 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main // 005 09/28/22 Prestopnik MET #2227 Remove namespace std from header files // 006 10/06/22 Halley Gotway MET #392 Incorporate diagnostics +// 007 06/14/24 Halley Gotway MET #2911 Support -set_hdr job command option // //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/tc_utils/tc_stat/tc_stat_job.cc b/src/tools/tc_utils/tc_stat/tc_stat_job.cc index ec8172776c..2bfca419fd 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_job.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_job.cc @@ -184,6 +184,8 @@ void TCStatJob::init_from_scratch() { ValidMask.set_ignore_case(1); LineType.set_ignore_case(1); TrackWatchWarn.set_ignore_case(1); + ByColumn.set_ignore_case(1); + HdrName.set_ignore_case(1); clear(); @@ -230,6 +232,10 @@ void TCStatJob::clear() { EventEqualLead.clear(); EventEqualCases.clear(); + ByColumn.clear(); + HdrName.clear(); + HdrValue.clear(); + DumpFile.clear(); close_dump_file(); JobOut = (ofstream *) nullptr; @@ -315,6 +321,11 @@ void TCStatJob::assign(const TCStatJob & j) { InitDiagThreshMap = j.InitDiagThreshMap; PrintDiagWarning = j.PrintDiagWarning; + ByColumn = j.ByColumn; + + HdrName = j.HdrName; + HdrValue = j.HdrValue; + DumpFile = j.DumpFile; open_dump_file(); @@ -514,6 +525,15 @@ void TCStatJob::dump(ostream & out, int depth) const { out << prefix << "OutValidMask = " << (OutValidMaskName.nonempty() ? OutValidMaskName.text() : na_str) << "\n"; + out << prefix << "ByColumn ...\n"; + ByColumn.dump(out, depth + 1); + + out << prefix << "HdrName ...\n"; + HdrName.dump(out, depth + 1); + + out << prefix << "HdrValue ...\n"; + HdrValue.dump(out, depth + 1); + out << prefix << "DumpFile = " << (DumpFile.nonempty() ? DumpFile.text() : na_str) << "\n"; out << prefix << "StatFile = " << (StatFile.nonempty() ? StatFile.text() : na_str) << "\n"; @@ -1086,6 +1106,9 @@ StringArray TCStatJob::parse_job_command(const char *jobstring) { else if(c.compare("-event_equal_lead" ) == 0) { EventEqualLead.add_css_sec(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-out_init_mask" ) == 0) { set_out_init_mask(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-out_valid_mask" ) == 0) { set_out_valid_mask(a[i+1].c_str()); a.shift_down(i, 1); } + else if(c.compare("-by" ) == 0) { ByColumn.add_css(to_upper(a[i+1])); a.shift_down(i, 1); } + else if(c.compare("-set_hdr" ) == 0) { HdrName.add(to_upper(a[i+1])); + HdrValue.add(a[i+2]); a.shift_down(i, 2); } else if(c.compare("-dump_row" ) == 0) { DumpFile = a[i+1]; open_dump_file(); a.shift_down(i, 1); } else if(c.compare("-out_stat" ) == 0) { StatFile = a[i+1]; open_stat_file(); a.shift_down(i, 1); } else { b.add(a[i]); } @@ -1206,7 +1229,8 @@ void TCStatJob::close_stat_file() { //////////////////////////////////////////////////////////////////////// -void TCStatJob::dump_pair(const TrackPairInfo &pair, ofstream *out) { +void TCStatJob::dump_pair(const TrackPairInfo &pair, ofstream *out, + bool do_set_hdr) const { if(!out || pair.n_points() == 0) return; @@ -1258,7 +1282,8 @@ void TCStatJob::dump_pair(const TrackPairInfo &pair, ofstream *out) { // Write the TrackPairInfo object i_row = hdr_row; - write_track_pair_info(tchc, pair, out_at, i_row); + if(do_set_hdr) write_track_pair_info(tchc, pair, out_at, i_row, HdrName, HdrValue); + else write_track_pair_info(tchc, pair, out_at, i_row); // Write the AsciiTable to the file *out << out_at; @@ -1268,11 +1293,23 @@ void TCStatJob::dump_pair(const TrackPairInfo &pair, ofstream *out) { //////////////////////////////////////////////////////////////////////// -void TCStatJob::dump_line(const TCStatLine &line, ofstream *out) { +void TCStatJob::dump_line(const TCStatLine &line, ofstream *out, + bool do_set_hdr) const { if(!out) return; - *out << line; + // Apply -set_hdr options, if requested + if(do_set_hdr) { + TCStatLine line_set_hdr = line; + for(int i=0; i 0) s << "-dump_row " << DumpFile << " "; if(StatFile.length() > 0) @@ -1851,7 +1892,7 @@ void TCStatJobFilter::filter_tracks(TCPointCounts &n) { mlog << Debug(4) << "Processing track pair: " << pair.case_info() << "\n"; - if(DumpOut) dump_pair(pair, DumpOut); + if(DumpOut) dump_pair(pair, DumpOut, true); } } // end while } // end else @@ -1888,7 +1929,7 @@ void TCStatJobFilter::filter_lines(TCPointCounts &n) { // Check if this line should be kept if(!is_keeper_line(line, n)) continue; - if(DumpOut) dump_line(line, DumpOut); + if(DumpOut) dump_line(line, DumpOut, true); } // end while } // end else @@ -1955,7 +1996,6 @@ void TCStatJobSummary::init_from_scratch() { // Ignore case when performing comparisons ReqColumn.set_ignore_case(1); Column.set_ignore_case(1); - ByColumn.set_ignore_case(1); clear(); @@ -1972,7 +2012,6 @@ void TCStatJobSummary::clear() { ReqColumn.clear(); Column.clear(); - ByColumn.clear(); SummaryMap.clear(); // Set to default value @@ -1992,7 +2031,6 @@ void TCStatJobSummary::assign(const TCStatJobSummary & j) { ReqColumn = j.ReqColumn; Column = j.Column; ColumnUnion = j.ColumnUnion; - ByColumn = j.ByColumn; SummaryMap = j.SummaryMap; OutAlpha = j.OutAlpha; FSPThresh = j.FSPThresh; @@ -2026,7 +2064,6 @@ StringArray TCStatJobSummary::parse_job_command(const char *jobstring) { if(c.compare("-column" ) == 0) { ReqColumn.add_css(to_upper(a[i+1])); add_column(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-column_union") == 0) { ColumnUnion = string_to_bool(a[i+1].c_str()); a.shift_down(i, 1); } - else if(c.compare("-by" ) == 0) { ByColumn.add_css(to_upper(a[i+1])); a.shift_down(i, 1); } else if(c.compare("-out_alpha" ) == 0) { OutAlpha = atof(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-fsp_thresh" ) == 0) { FSPThresh.set(a[i+1].c_str()); a.shift_down(i, 1); } else { b.add(a[i]); } @@ -2090,8 +2127,6 @@ ConcatString TCStatJobSummary::serialize() const { s << "-column " << ReqColumn[i] << " "; if(ColumnUnion != default_column_union) s << "-column_union " << bool_to_string(ColumnUnion) << " "; - for(i=0; isecond.InitMask); sa.add_uniq(it->second.ValidMask); - + // Use the basin names instead if(sa.n() == 1 && sa[0] == na_str) { sa.clear(); @@ -3889,16 +3917,21 @@ void TCStatJobRIRW::do_stat_output(ostream &out) { // c = 0; + // Split the current map key, eliminating the job name in the first entry + StringArray ByValue = it->first.split(":"); + ByValue.shift_down(0, 1); + // // CTC output line // if(OutLineType.has(stat_ctc_str)) { shc.set_alpha(bad_data_double); shc.set_line_type(stat_ctc_str); + shc.apply_set_hdr_opts(HdrName, HdrValue, ByColumn, ByValue); write_header_cols(shc, stat_at, stat_row); write_ctc_cols(it->second.Info, stat_at, stat_row++, n_header_columns); } - + // // CTS output line // @@ -3910,12 +3943,12 @@ void TCStatJobRIRW::do_stat_output(ostream &out) { it->second.Info.allocate_n_alpha(1); it->second.Info.alpha[0] = OutAlpha; shc.set_alpha(OutAlpha); - + // // Compute the stats and confidence intervals for this // CTSInfo object // - + it->second.Info.compute_stats(); it->second.Info.compute_ci(); @@ -3923,14 +3956,15 @@ void TCStatJobRIRW::do_stat_output(ostream &out) { // Write the data line // shc.set_line_type(stat_cts_str); + shc.apply_set_hdr_opts(HdrName, HdrValue, ByColumn, ByValue); write_header_cols(shc, stat_at, stat_row); write_cts_cols(it->second.Info, 0, stat_at, stat_row++, n_header_columns); } } // end for it - + // Write the table out << stat_at << "\n" << flush; - + return; } @@ -3981,9 +4015,6 @@ void TCStatJobProbRIRW::init_from_scratch() { TCStatJob::init_from_scratch(); - // Ignore case when performing comparisons - ByColumn.set_ignore_case(1); - clear(); return; @@ -3997,7 +4028,6 @@ void TCStatJobProbRIRW::clear() { JobType = TCStatJobType::ProbRIRW; - ByColumn.clear(); ProbRIRWMap.clear(); // Set to default values @@ -4023,7 +4053,6 @@ void TCStatJobProbRIRW::assign(const TCStatJobProbRIRW & j) { ProbRIRWExact = j.ProbRIRWExact; ProbRIRWBDeltaThresh = j.ProbRIRWBDeltaThresh; ProbRIRWProbThresh = j.ProbRIRWProbThresh; - ByColumn = j.ByColumn; MaxNThresh = j.MaxNThresh; NDumpLines = j.NDumpLines; OutAlpha = j.OutAlpha; @@ -4060,8 +4089,7 @@ StringArray TCStatJobProbRIRW::parse_job_command(const char *jobstring) { } // Check job command options - if(c.compare("-by" ) == 0) { ByColumn.add_css(to_upper(a[i+1])); a.shift_down(i, 1); } - else if(c.compare("-out_alpha" ) == 0) { OutAlpha = atof(a[i+1].c_str()); a.shift_down(i, 1); } + if(c.compare("-out_alpha" ) == 0) { OutAlpha = atof(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-out_line_type" ) == 0) { OutLineType.add_css(to_upper(a[i+1])); a.shift_down(i, 1); } else if(c.compare("-probrirw_exact" ) == 0) { ProbRIRWExact = string_to_bool(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-probrirw_bdelta_thresh") == 0) { ProbRIRWBDeltaThresh.set(a[i+1].c_str()); a.shift_down(i, 1); } @@ -4157,9 +4185,6 @@ ConcatString TCStatJobProbRIRW::serialize() const { s << "-probrirw_prob_thresh " << prob_thresh_to_string(ProbRIRWProbThresh) << " "; // Add ProbRIRW job-specific options - for(i=0; i InitDiagThreshMap; StringArray PrintDiagWarning; + // Store the case information + StringArray ByColumn; + + // Options for -set_hdr output + StringArray HdrName; + StringArray HdrValue; + // Variables to the store the analysis job specification ConcatString DumpFile; // Dump TrackPairInfo used to a file std::ofstream *DumpOut; // Dump output file stream @@ -448,9 +457,6 @@ class TCStatJobSummary : public TCStatJob { StringArray Column; bool ColumnUnion; - // Store the case information - StringArray ByColumn; - // Confidence interval alpha value double OutAlpha; @@ -504,9 +510,6 @@ class TCStatJobRIRW : public TCStatJob { void do_cts_output (std::ostream &); void do_mpr_output (std::ostream &); void do_stat_output(std::ostream &); - - // Store the case information - StringArray ByColumn; // Confidence interval alpha value double OutAlpha; @@ -555,9 +558,6 @@ class TCStatJobProbRIRW : public TCStatJob { SingleThresh ProbRIRWBDeltaThresh; // Threshold the BEST track change ThreshArray ProbRIRWProbThresh; // Array of probabilities for PCT bins - // Store the case information - StringArray ByColumn; - // Maximum number of thresholds encountered int MaxNThresh; int NDumpLines;