From 01043c025105110f7b6244ba8ce988d3cdd3fd3d Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 19 Sep 2024 12:54:41 -0600 Subject: [PATCH] Feature #2924 parse_config PR 2 (#2975) * Per #2924, remove GenEnsProd config file comment about parsing desc separately from each obs.field entry because the obs dictionary does not exist in the GenEnsProd config file. * Per #2924, update list of needed config entry names * Per #2924, remove const from the parent() member function so that we can perform lookups for the parent. * Per #2924, update the signature for and logic of the utility functions that retrieve the climatology data. Rather than requiring all the climo_mean and climo_stdev dictionary entries to be defined at the same config file context level, parse each one individually. This enables the METplus wrappers to only partially override this dictionary and still rely on the default values provided in MET's default configuration files. * Per #2924, update all calls to the climatology utility functions based on the new function signature. Also update the tools to check the number of climo fields separately for the forecast and observation climos. * Per #2924, update the parsing logic for the climatology regrid dictionary. Use config.fcst.climo_mean.regrid first, config.fcst.regrid second, and config.climo_mean.regrid third. Notably, DO NOT use config.regrid. This is definitely the problem with having regrid specified at mutliple config file context levels. It makes the logic for which to use when very messy. * Per #2924, forgot to add an else to print an error * Per #2924, remove extraneous semicolon * Per #2924, move 'fcst.regrid' into 'fcst.climo_mean.regrid'. Defining the climatology regridding logic inside fcst is problematic because it applies to the forecast data as well and you end up with the verification grid being undefined. So the climo regridding logic must be defined in 'climo_mean.regrid' either within the 'fcst' and 'obs' dictionaries or at the top-level config context. * Per #2924, based on PR feedback from @georgemccabe, add the Upper_Left, Upper_Right, Lower_Right, and Lower_Left interpolation methods to the list of valid options for regridding, as already indicated in the MET User's Guide. * Per #2924, update the logic of parse_conf_regrid() to (hopefully) make it work the way @georgemccabe expects it to. It now uses pointers to both the primary and default dictionaries and parses each entry individually. * Per #2924, need to check for non-null pointer before using it * Per #2924, revise the climo_name dictionary lookup logic when parsing the regrid dictionary. * Per #2924, update logic for handling RegridInfo * Per #2924, remove the default regridding information from the 'Searching' log message to avoid confusion. * Per #2924, escape sequences, like \n, cannot be used inside R-string literals. * Per #2924, update the logic of check_climo_n_vx() * Per #2924, revise logic in read_climo_data_plane_array(). Check the number of climo fields provided. If there's 0, just return since no data has been requested. If there's 1, use it regardless of the number of input fields. If there's more than 1, just use the requested i_vx index value. * Per #2924, update Series-Analysis to set both i_fcst and i_obs when looping over the series entries. * Per #2924, no real change. Just whitespace. * Unrelated to #2924, superficial changes to formatting of method_name strings for consistency. * Per #2924, add a new series_analysis test that ERRORS OUT prior to this PR but works after the changes in this PR. --------- Co-authored-by: MET Tools Test Account --- .../config/SeriesAnalysisConfig_const_climo | 162 ++++++++++++++++++ .../test_unit/xml/unit_climatology_1.0deg.xml | 28 +++ src/basic/vx_cal/is_leap_year.cc | 4 +- src/basic/vx_config/config_util.cc | 24 +-- src/basic/vx_config/threshold.cc | 2 +- src/libcode/vx_data2d_nc_cf/nc_cf_file.cc | 2 +- src/libcode/vx_statistics/read_climo.cc | 10 +- .../core/series_analysis/series_analysis.cc | 13 +- 8 files changed, 222 insertions(+), 23 deletions(-) create mode 100644 internal/test_unit/config/SeriesAnalysisConfig_const_climo diff --git a/internal/test_unit/config/SeriesAnalysisConfig_const_climo b/internal/test_unit/config/SeriesAnalysisConfig_const_climo new file mode 100644 index 0000000000..df991f208e --- /dev/null +++ b/internal/test_unit/config/SeriesAnalysisConfig_const_climo @@ -0,0 +1,162 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Series-Analysis configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "GFS"; + +// +// Output description to be written +// +desc = "NA"; + +// +// Output observation type to be written +// +obtype = "GFSANL"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = { + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; +} + +//////////////////////////////////////////////////////////////////////////////// + +censor_thresh = []; +censor_val = []; +cat_thresh = []; +cnt_thresh = [ NA ]; +cnt_logic = UNION; + +// +// Forecast and observation fields to be verified +// +fcst = { + field = [ + { name = "TMP"; level = "P850"; valid_time = "20120409_12"; }, + { name = "TMP"; level = "P850"; valid_time = "20120410_00"; }, + { name = "TMP"; level = "P850"; valid_time = "20120410_12"; } + ]; +} +obs = { + field = [ + { name = "TMP"; level = "P850"; valid_time = "20120409_12"; }, + { name = "TMP"; level = "P850"; valid_time = "20120410_00"; }, + { name = "TMP"; level = "P850"; valid_time = "20120410_12"; } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology data +// +climo_mean = fcst; +climo_mean = { + + file_name = [ ${CLIMO_MEAN_FILE_LIST} ]; + + field = [ + { name = "TMP"; level = "P850"; valid_time = "19590409_00"; } + ]; + + regrid = { + method = BILIN; + width = 2; + vld_thresh = 0.5; + } + + time_interp_method = NEAREST; + day_interval = NA; + hour_interval = NA; +} + +climo_stdev = climo_mean; +climo_stdev = { + file_name = [ ${CLIMO_STDEV_FILE_LIST} ]; +} + +climo_cdf = { + cdf_bins = 1; + center_bins = FALSE; + direct_prob = FALSE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = "1"; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +mask = { + grid = ""; + poly = ""; +} + +// +// Number of grid points to be processed concurrently. Set smaller to use less +// memory but increase the number of passes through the data. If set <= 0, all +// grid points are processed concurrently. +// +block_size = 0; + +// +// Ratio of valid matched pairs to compute statistics for a grid point +// +vld_thresh = 0.5; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// +output_stats = { + fho = [ ]; + ctc = [ ]; + cts = [ ]; + mctc = [ ]; + mcts = [ ]; + cnt = [ "TOTAL", "RMSE", "ANOM_CORR" ]; + sl1l2 = [ ]; + sal1l2 = [ ]; + pct = [ ]; + pstd = [ ]; + pjc = [ ]; + prc = [ ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +hss_ec_value = NA; +rank_corr_flag = FALSE; +tmp_dir = "/tmp"; +version = "V12.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/internal/test_unit/xml/unit_climatology_1.0deg.xml b/internal/test_unit/xml/unit_climatology_1.0deg.xml index fcd6b59668..699026825e 100644 --- a/internal/test_unit/xml/unit_climatology_1.0deg.xml +++ b/internal/test_unit/xml/unit_climatology_1.0deg.xml @@ -186,6 +186,34 @@ + + &MET_BIN;/series_analysis + + CLIMO_MEAN_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590409" + + + CLIMO_STDEV_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590409" + + + + \ + -fcst &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F012.grib2 \ + &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F024.grib2 \ + &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F036.grib2 \ + -obs &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120409_1200_000.grb2 \ + &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120410_0000_000.grb2 \ + &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120410_1200_000.grb2 \ + -out &OUTPUT_DIR;/climatology_1.0deg/series_analysis_GFS_CLIMO_1.0DEG_CONST_CLIMO.nc \ + -config &CONFIG_DIR;/SeriesAnalysisConfig_const_climo \ + -v 3 + + + &OUTPUT_DIR;/climatology_1.0deg/series_analysis_GFS_CLIMO_1.0DEG_CONST_CLIMO.nc + + + &MET_BIN;/series_analysis diff --git a/src/basic/vx_cal/is_leap_year.cc b/src/basic/vx_cal/is_leap_year.cc index d37854d690..a383041475 100644 --- a/src/basic/vx_cal/is_leap_year.cc +++ b/src/basic/vx_cal/is_leap_year.cc @@ -102,7 +102,7 @@ void adjuste_day_for_month_year_units(int &day, int &month, int &year, double mo // Compute remaining days from the month fraction bool day_adjusted = false; const int day_offset = (int)(month_fraction * DAYS_PER_MONTH + 0.5); - const char *method_name = "adjuste_day() --> "; + const char *method_name = "adjuste_day_for_month_year_units() -> "; day += day_offset; if (day == 1 && abs(month_fraction-0.5) < DAY_EPSILON) { @@ -162,7 +162,7 @@ unixtime add_to_unixtime(unixtime base_unixtime, int sec_per_unit, unixtime ut; auto time_value_ut = (unixtime)time_value; double time_fraction = time_value - (double)time_value_ut; - const char *method_name = "add_to_unixtime() -->"; + const char *method_name = "add_to_unixtime() -> "; if (sec_per_unit == SEC_MONTH || sec_per_unit == SEC_YEAR) { if (time_value < 0) { diff --git a/src/basic/vx_config/config_util.cc b/src/basic/vx_config/config_util.cc index 244b2d9b2e..5cce67dfcb 100644 --- a/src/basic/vx_config/config_util.cc +++ b/src/basic/vx_config/config_util.cc @@ -2622,28 +2622,28 @@ void check_mask_names(const StringArray &sa) { /////////////////////////////////////////////////////////////////////////////// -void check_climo_n_vx(Dictionary *dict, const int n_vx) { - int n; +void check_climo_n_vx(Dictionary *dict, const int n_input) { + int n_climo; // Check for a valid number of climatology mean fields - n = parse_conf_n_vx(dict->lookup_array(conf_key_climo_mean_field, false)); - if(n != 0 && n != n_vx) { + n_climo = parse_conf_n_vx(dict->lookup_array(conf_key_climo_mean_field, false)); + if(n_climo != 0 && n_climo != 1 && n_climo != n_input) { mlog << Error << "\ncheck_climo_n_vx() -> " << "The number of climatology mean fields in \"" - << conf_key_climo_mean_field - << "\" must be zero or match the number (" << n_vx - << ") in \"" << conf_key_fcst_field << "\".\n\n"; + << conf_key_climo_mean_field << "\" (" << n_climo + << ") must be 0, 1, or match the number of input fields (" + << n_input << ").\n\n"; exit(1); } // Check for a valid number of climatology standard deviation fields - n = parse_conf_n_vx(dict->lookup_array(conf_key_climo_stdev_field, false)); - if(n != 0 && n != n_vx) { + n_climo = parse_conf_n_vx(dict->lookup_array(conf_key_climo_stdev_field, false)); + if(n_climo != 0 && n_climo != 1 && n_climo != n_input) { mlog << Error << "\ncheck_climo_n_vx() -> " << "The number of climatology standard deviation fields in \"" - << conf_key_climo_stdev_field - << "\" must be zero or match the number (" - << n_vx << ") in \"" << conf_key_fcst_field << "\".\n\n"; + << conf_key_climo_stdev_field << "\" (" << n_climo + << ") must be 0, 1, or match the number of input fields (" + << n_input << ").\n\n"; exit(1); } diff --git a/src/basic/vx_config/threshold.cc b/src/basic/vx_config/threshold.cc index cbf0a3cb7d..ef650ef2c0 100644 --- a/src/basic/vx_config/threshold.cc +++ b/src/basic/vx_config/threshold.cc @@ -114,7 +114,7 @@ if ( !match && mlog << Debug(2) << R"(Please replace the deprecated "SCP" and "CDP" )" << R"(threshold types with "SOCP" and "OCDP", respectively, in the ")" - << str << R"(" threshold string.\n)"; + << str << R"(" threshold string.)" << "\n"; print_climo_perc_thresh_log_message = false; diff --git a/src/libcode/vx_data2d_nc_cf/nc_cf_file.cc b/src/libcode/vx_data2d_nc_cf/nc_cf_file.cc index e985870169..41c8106f31 100644 --- a/src/libcode/vx_data2d_nc_cf/nc_cf_file.cc +++ b/src/libcode/vx_data2d_nc_cf/nc_cf_file.cc @@ -2299,7 +2299,7 @@ void NcCfFile::get_grid_mapping_polar_stereographic(const NcVar *grid_mapping_va { double x_coord_to_m_cf = 1.0; double y_coord_to_m_cf = 1.0; - static const string method_name = "NcCfFile::get_grid_mapping_polar_stereographic() --> "; + static const string method_name = "NcCfFile::get_grid_mapping_polar_stereographic() -> "; // Get projection attributes // proj_origin_lat: either 90.0 or -90.0, to decide the northern/southern hemisphere diff --git a/src/libcode/vx_statistics/read_climo.cc b/src/libcode/vx_statistics/read_climo.cc index 9559e4c4a2..8f8ddd8e9b 100644 --- a/src/libcode/vx_statistics/read_climo.cc +++ b/src/libcode/vx_statistics/read_climo.cc @@ -103,8 +103,14 @@ DataPlaneArray read_climo_data_plane_array(Dictionary *dict, cs << cs_erase << climo_name << "." << conf_key_field; Dictionary *field_dict = dict->lookup_array(cs.c_str(), false); - // Get the i-th array entry - Dictionary i_dict = parse_conf_i_vx_dict(field_dict, i_vx); + // Determine which climo array entry to use + int i_climo_field = bad_data_int; + if(field_dict->n_entries() == 0) return dpa; + else if(field_dict->n_entries() == 1) i_climo_field = 0; + else i_climo_field = i_vx; + + // Parse the climo dictionary + Dictionary i_dict = parse_conf_i_vx_dict(field_dict, i_climo_field); // Parse the "regrid" dictionary from the top-level // config file context (e.g. "config.climo_mean.regrid") diff --git a/src/tools/core/series_analysis/series_analysis.cc b/src/tools/core/series_analysis/series_analysis.cc index 51dd74364f..a817768c3a 100644 --- a/src/tools/core/series_analysis/series_analysis.cc +++ b/src/tools/core/series_analysis/series_analysis.cc @@ -363,7 +363,7 @@ void process_command_line(int argc, char **argv) { if(fcst_files.n() != n_series_pair) { mlog << Error << "\nprocess_command_line() -> " << R"(when using the "-paired" command line option, the )" - << "the file list length (" << fcst_files.n() + << "file list length (" << fcst_files.n() << ") and series length (" << n_series_pair << ") must match.\n\n"; usage(); @@ -861,11 +861,14 @@ void process_scores() { // Loop over the series variable for(int i_series=0; i_series 1 ? i_series : 0); + int i_obs = (conf_info.get_n_obs() > 1 ? i_series : 0); // Store the current VarInfo objects - fcst_info = conf_info.fcst_info[i_fcst]; + fcst_info = (conf_info.get_n_fcst() > 1 ? + conf_info.fcst_info[i_series] : + conf_info.fcst_info[0]); obs_info = (conf_info.get_n_obs() > 1 ? conf_info.obs_info[i_series] : conf_info.obs_info[0]); @@ -914,12 +917,12 @@ void process_scores() { ocmn_dp = read_climo_data_plane( conf_info.conf.lookup_dictionary(conf_key_obs), conf_key_climo_mean, - i_fcst, fcst_dp.valid(), grid, + i_obs, fcst_dp.valid(), grid, "observation climatology mean"); ocsd_dp = read_climo_data_plane( conf_info.conf.lookup_dictionary(conf_key_obs), conf_key_climo_stdev, - i_fcst, fcst_dp.valid(), grid, + i_obs, fcst_dp.valid(), grid, "observation climatology standard deviation"); bool fcmn_flag = !fcmn_dp.is_empty();