From e8a2cf457c89b8a06b5d98e172cf794f7b8667a7 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 16 Oct 2024 14:08:46 -0600 Subject: [PATCH] Feature #2893 output_obtype (#2995) * Per #2893, add obtype_as_obs_message_type_flag option for Point-Stat and Ensemble-Stat that applies when writing MPR, SEEPS_MPR, and ORANK line types. If true, write the input observation message type to the OBTYPE output column rather than the name of the group. * Per #2893, add option to default config files and fix compilation errors. * Per #2893, change the name from obtype_as_obs_message_type_flag to obtype_as_group_val_flag. * Per #2893, add docs ci-run-unit * Per #2893, add obtype_as_group_val_flag entry for all config files that already include the message_type_group_map entry. * Per #2893, modify existing PointStatConfig_WINDS config file to have it write MPR output with obtype_as_group_val_flag = TRUE. Note that I switched the message types from ADPSFC, SFCSHP, USERSF to just USERSF. The original reason for the 3 was to prove that the counts for ADPSFC + SFCSHP = USERSF. * Per #2893, update commit history for ensemble_stat and point_stat * Per #2893, revert back PointStatConfig_WINDS to minimize diffs * Per #2893, add a Point-Stat unit test to demonstrate setting the obtype_as_group_val_flag option. * Per #2893, need to reset the obtype after writing all MPR, SEEPS_MPR, and ORANK line types in Point-Stat * Per #2893, tweak Point-Stat configuration to also write HIRA ORANK and PCT outputs. --- data/config/EnsembleStatConfig_default | 2 + data/config/PointStatConfig_default | 2 + docs/Users_Guide/config_options.rst | 19 ++ docs/Users_Guide/ensemble-stat.rst | 1 + docs/Users_Guide/point-stat.rst | 1 + .../config/EnsembleStatConfig_python | 2 + .../EnsembleStatConfig_single_file_grib | 2 + .../config/EnsembleStatConfig_single_file_nc | 2 + .../config/PointStatConfig_LAND_TOPO_MASK | 2 + .../config/PointStatConfig_MPR_OBTYPE | 166 ++++++++++++++++++ .../test_unit/config/PointStatConfig_WINDS | 2 + .../test_unit/config/PointStatConfig_airnow | 2 + internal/test_unit/xml/unit_point_stat.xml | 20 +++ scripts/config/EnsembleStatConfig | 2 + src/basic/vx_config/config_constants.h | 1 + src/libcode/vx_stat_out/stat_columns.cc | 14 +- src/libcode/vx_stat_out/stat_columns.h | 7 +- src/libcode/vx_statistics/pair_base.cc | 10 +- src/libcode/vx_statistics/pair_base.h | 12 +- .../vx_statistics/pair_data_ensemble.cc | 5 +- src/libcode/vx_statistics/pair_data_point.cc | 27 +-- src/libcode/vx_statistics/pair_data_point.h | 17 +- src/tools/core/ensemble_stat/ensemble_stat.cc | 11 +- .../ensemble_stat/ensemble_stat_conf_info.cc | 5 + .../ensemble_stat/ensemble_stat_conf_info.h | 1 + src/tools/core/point_stat/point_stat.cc | 35 +++- .../core/point_stat/point_stat_conf_info.cc | 5 + .../core/point_stat/point_stat_conf_info.h | 1 + src/tools/other/gsi_tools/gsidens2orank.cc | 4 +- 29 files changed, 340 insertions(+), 40 deletions(-) create mode 100644 internal/test_unit/config/PointStatConfig_MPR_OBTYPE diff --git a/data/config/EnsembleStatConfig_default b/data/config/EnsembleStatConfig_default index b5b39eb803..c4b2463f7d 100644 --- a/data/config/EnsembleStatConfig_default +++ b/data/config/EnsembleStatConfig_default @@ -120,6 +120,8 @@ message_type_group_map = [ { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + // // Ensemble bin sizes // May be set separately in each "obs.field" entry diff --git a/data/config/PointStatConfig_default b/data/config/PointStatConfig_default index 0b34c7537d..95665fe017 100644 --- a/data/config/PointStatConfig_default +++ b/data/config/PointStatConfig_default @@ -115,6 +115,8 @@ message_type_group_map = [ { key = "WATERSF"; val = "SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + //////////////////////////////////////////////////////////////////////////////// // diff --git a/docs/Users_Guide/config_options.rst b/docs/Users_Guide/config_options.rst index 5ef04f530e..b85b2d233b 100644 --- a/docs/Users_Guide/config_options.rst +++ b/docs/Users_Guide/config_options.rst @@ -667,6 +667,25 @@ used. { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag +------------------------ + +The "obtype_as_group_val_flag" entry is a boolean that controls how the +OBTYPE header column is populated for message type groups defined in +"message_type_group_map". If set to TRUE and when writing matched pair +line types (MPR, SEEPS_MPR, and ORANK), write OBTYPE as the group map +*value*, i.e. the input message type for each individual observation. +If set to FALSE (default) and for all other line types, write OBTYPE +as the group map key, i.e. the name of the message type group. + +For example, if FALSE, write the OBTYPE column in the MPR line type +as the "ANYAIR" message type group name. If TRUE, write OBTYPE as "AIRCAR" +or "AIRCFT", based on the input message type of each point observation. + +.. code-block:: none + + obtyp_as_group_val_flag = FALSE; + message_type_map ---------------- diff --git a/docs/Users_Guide/ensemble-stat.rst b/docs/Users_Guide/ensemble-stat.rst index 66e0187814..1680164d70 100644 --- a/docs/Users_Guide/ensemble-stat.rst +++ b/docs/Users_Guide/ensemble-stat.rst @@ -181,6 +181,7 @@ ____________________ obs_summary = NONE; obs_perc_value = 50; message_type_group_map = [...]; + obtype_as_group_val_flag = FALSE; grid_weight_flag = NONE; point_weight_flag = NONE; output_prefix = ""; diff --git a/docs/Users_Guide/point-stat.rst b/docs/Users_Guide/point-stat.rst index addaa206e2..41e154ac8c 100644 --- a/docs/Users_Guide/point-stat.rst +++ b/docs/Users_Guide/point-stat.rst @@ -362,6 +362,7 @@ ________________________ obs_summary = NONE; obs_perc_value = 50; message_type_group_map = [...]; + obtype_as_group_val_flag = FALSE; point_weight_flag = NONE; tmp_dir = "/tmp"; output_prefix = ""; diff --git a/internal/test_unit/config/EnsembleStatConfig_python b/internal/test_unit/config/EnsembleStatConfig_python index e6be01ea44..1b32091705 100644 --- a/internal/test_unit/config/EnsembleStatConfig_python +++ b/internal/test_unit/config/EnsembleStatConfig_python @@ -119,6 +119,8 @@ message_type_group_map = [ { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + // // Ensemble bin sizes // May be set separately in each "obs.field" entry diff --git a/internal/test_unit/config/EnsembleStatConfig_single_file_grib b/internal/test_unit/config/EnsembleStatConfig_single_file_grib index 831b5dce36..ba1ec381a2 100644 --- a/internal/test_unit/config/EnsembleStatConfig_single_file_grib +++ b/internal/test_unit/config/EnsembleStatConfig_single_file_grib @@ -131,6 +131,8 @@ message_type_group_map = [ { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + // // Ensemble bin sizes // May be set separately in each "obs.field" entry diff --git a/internal/test_unit/config/EnsembleStatConfig_single_file_nc b/internal/test_unit/config/EnsembleStatConfig_single_file_nc index 99403c0059..f9d18a2129 100644 --- a/internal/test_unit/config/EnsembleStatConfig_single_file_nc +++ b/internal/test_unit/config/EnsembleStatConfig_single_file_nc @@ -137,6 +137,8 @@ message_type_group_map = [ { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + // // Ensemble bin sizes // May be set separately in each "obs.field" entry diff --git a/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK b/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK index 780081fb33..49cbf781dd 100644 --- a/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK +++ b/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK @@ -81,6 +81,8 @@ message_type_group_map = [ { key = "WATERSF"; val = "SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + //////////////////////////////////////////////////////////////////////////////// climo_mean = fcst; diff --git a/internal/test_unit/config/PointStatConfig_MPR_OBTYPE b/internal/test_unit/config/PointStatConfig_MPR_OBTYPE new file mode 100644 index 0000000000..6aa68e9842 --- /dev/null +++ b/internal/test_unit/config/PointStatConfig_MPR_OBTYPE @@ -0,0 +1,166 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Point-Stat configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +model = "FCST"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +//////////////////////////////////////////////////////////////////////////////// + +regrid = { + to_grid = NONE; + method = NEAREST; + width = 1; +} + +//////////////////////////////////////////////////////////////////////////////// + +obs_window = { + beg = ${BEG_DS}; + end = ${END_DS}; +} + +//////////////////////////////////////////////////////////////////////////////// + +mpr_column = []; +mpr_thresh = []; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; +eclv_points = 0.05; + +// +// Mapping of message type group name to comma-separated list of values. +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; }, + { key = "LANDSF"; val = "ADPSFC,MSONET"; }, + { key = "WATERSF"; val = "SFCSHP"; } +]; + +obtype_as_group_val_flag = TRUE; + +fcst = { + sid_inc = []; + sid_exc = []; + obs_quality_inc = []; + obs_quality_exc = []; + + message_type = [ "SURFACE" ]; + + field = [ + { name = "TMP"; level = "Z2"; } + ]; + +} +obs = fcst; + +//////////////////////////////////////////////////////////////////////////////// + +climo_mean = fcst; +climo_mean = { + file_name = [ ${CLIMO_FILE} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +mask = { + grid = [ "FULL" ]; + poly = []; + sid = []; + llpnt = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 200; + rng = "mt19937"; + seed = "1"; +} + +//////////////////////////////////////////////////////////////////////////////// + +interp = { + vld_thresh = 1.0; + + type = [ + { + method = NEAREST; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +hira = { + flag = TRUE; + width = [ 3 ]; + vld_thresh = 1.0; + cov_thresh = [ ==0.25 ]; + shape = SQUARE; + prob_cat_thresh = [ >273 ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +output_flag = { + fho = NONE; + ctc = NONE; + cts = NONE; + mctc = NONE; + mcts = NONE; + cnt = STAT; + sl1l2 = STAT; + sal1l2 = STAT; + vl1l2 = NONE; + val1l2 = NONE; + vcnt = NONE; + pct = STAT; + pstd = NONE; + pjc = NONE; + prc = NONE; + ecnt = STAT; + rps = NONE; + orank = STAT; + eclv = NONE; + mpr = STAT; + seeps = NONE; + seeps_mpr = NONE; +} + +//////////////////////////////////////////////////////////////////////////////// +// Threshold for SEEPS p1 (Probability of being dry) + +seeps_p1_thresh = NA; + +//////////////////////////////////////////////////////////////////////////////// + +duplicate_flag = NONE; +rank_corr_flag = TRUE; + +point_weight_flag = NONE; + +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V12.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/internal/test_unit/config/PointStatConfig_WINDS b/internal/test_unit/config/PointStatConfig_WINDS index ba385a0304..4dac7fe51e 100644 --- a/internal/test_unit/config/PointStatConfig_WINDS +++ b/internal/test_unit/config/PointStatConfig_WINDS @@ -52,6 +52,8 @@ message_type_group_map = [ { key = "USERSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + fcst = { sid_inc = []; sid_exc = []; diff --git a/internal/test_unit/config/PointStatConfig_airnow b/internal/test_unit/config/PointStatConfig_airnow index 5b5f97c915..4e7cfe79ba 100644 --- a/internal/test_unit/config/PointStatConfig_airnow +++ b/internal/test_unit/config/PointStatConfig_airnow @@ -92,6 +92,8 @@ message_type_group_map = [ { key = "WATERSF"; val = "SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + //////////////////////////////////////////////////////////////////////////////// // diff --git a/internal/test_unit/xml/unit_point_stat.xml b/internal/test_unit/xml/unit_point_stat.xml index 2c90567c80..8e798e03ec 100644 --- a/internal/test_unit/xml/unit_point_stat.xml +++ b/internal/test_unit/xml/unit_point_stat.xml @@ -72,6 +72,26 @@ + + &MET_BIN;/point_stat + + BEG_DS -300 + END_DS 300 + OUTPUT_PREFIX GRIB1_NAM_GDAS_MPR_OBTYPE + CONFIG_DIR &CONFIG_DIR; + CLIMO_FILE "&DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012_gNam.grib" + + \ + &DATA_DIR_MODEL;/grib1/nam/nam_2012040900_F012.grib \ + &OUTPUT_DIR;/pb2nc/gdas1.20120409.t12z.prepbufr.nc \ + &CONFIG_DIR;/PointStatConfig_MPR_OBTYPE \ + -outdir &OUTPUT_DIR;/point_stat -v 1 + + + &OUTPUT_DIR;/point_stat/point_stat_GRIB1_NAM_GDAS_MPR_OBTYPE_120000L_20120409_120000V.stat + + + &MET_BIN;/point_stat diff --git a/scripts/config/EnsembleStatConfig b/scripts/config/EnsembleStatConfig index 9071cf77e4..5353c92579 100644 --- a/scripts/config/EnsembleStatConfig +++ b/scripts/config/EnsembleStatConfig @@ -124,6 +124,8 @@ message_type_group_map = [ { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + // // Ensemble bin sizes // May be set separately in each "obs.field" entry diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index 12f4b2025b..9f1366a681 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -549,6 +549,7 @@ static const char conf_key_model[] = "model"; static const char conf_key_desc[] = "desc"; static const char conf_key_obtype[] = "obtype"; static const char conf_key_output_flag[] = "output_flag"; +static const char conf_key_obtype_as_group_val_flag[] = "obtype_as_group_val_flag"; static const char conf_key_obs_window[] = "obs_window"; static const char conf_key_beg[] = "beg"; static const char conf_key_end[] = "end"; diff --git a/src/libcode/vx_stat_out/stat_columns.cc b/src/libcode/vx_stat_out/stat_columns.cc index ac8530f6a5..c805d447fd 100644 --- a/src/libcode/vx_stat_out/stat_columns.cc +++ b/src/libcode/vx_stat_out/stat_columns.cc @@ -1548,6 +1548,7 @@ void write_mpr_row(StatHdrColumns &shc, const PairDataPoint *pd_ptr, STATOutputType out_type, AsciiTable &stat_at, int &stat_row, AsciiTable &txt_at, int &txt_row, + bool update_obtype, bool update_thresh) { // MPR line type @@ -1567,6 +1568,9 @@ void write_mpr_row(StatHdrColumns &shc, const PairDataPoint *pd_ptr, // Write a line for each matched pair for(int i=0; in_obs; i++) { + // MET #2893 write individual obs message type + if(update_obtype) shc.set_obtype(pd_ptr->typ_sa[i].c_str()); + // Set the observation valid time shc.set_obs_valid_beg(pd_ptr->vld_ta[i]); shc.set_obs_valid_end(pd_ptr->vld_ta[i]); @@ -1645,6 +1649,7 @@ void write_seeps_mpr_row(StatHdrColumns &shc, const PairDataPoint *pd_ptr, STATOutputType out_type, AsciiTable &stat_at, int &stat_row, AsciiTable &txt_at, int &txt_row, + bool update_obtype, bool update_thresh) { // SEEPS line type @@ -1668,6 +1673,9 @@ void write_seeps_mpr_row(StatHdrColumns &shc, const PairDataPoint *pd_ptr, if(!pd_ptr->seeps_mpr[i] || is_bad_data(pd_ptr->seeps_mpr[i]->score)) continue; + // MET #2893 write individual obs message type + if(update_obtype) shc.set_obtype(pd_ptr->typ_sa[i].c_str()); + // Set the observation valid time shc.set_obs_valid_beg(pd_ptr->vld_ta[i]); shc.set_obs_valid_end(pd_ptr->vld_ta[i]); @@ -1899,7 +1907,8 @@ void write_phist_row(StatHdrColumns &shc, const PairDataEnsemble *pd_ptr, void write_orank_row(StatHdrColumns &shc, const PairDataEnsemble *pd_ptr, STATOutputType out_type, AsciiTable &stat_at, int &stat_row, - AsciiTable &txt_at, int &txt_row) { + AsciiTable &txt_at, int &txt_row, + bool update_obtype) { // Observation Rank line type shc.set_line_type(stat_orank_str); @@ -1913,6 +1922,9 @@ void write_orank_row(StatHdrColumns &shc, const PairDataEnsemble *pd_ptr, // Write a line for each ensemble pair for(int i=0; in_obs; i++) { + // MET #2893 write individual obs message type + if(update_obtype) shc.set_obtype(pd_ptr->typ_sa[i].c_str()); + // Set the observation valid time shc.set_obs_valid_beg(pd_ptr->vld_ta[i]); shc.set_obs_valid_end(pd_ptr->vld_ta[i]); diff --git a/src/libcode/vx_stat_out/stat_columns.h b/src/libcode/vx_stat_out/stat_columns.h index 7eb21c6fa0..e8998e1fa4 100644 --- a/src/libcode/vx_stat_out/stat_columns.h +++ b/src/libcode/vx_stat_out/stat_columns.h @@ -106,13 +106,13 @@ extern void write_dmap_row (StatHdrColumns &, const DMAPInfo &, STATOutputType, AsciiTable &, int &, AsciiTable &, int &); extern void write_mpr_row (StatHdrColumns &, const PairDataPoint *, STATOutputType, AsciiTable &, int &, AsciiTable &, int &, - bool update_thresh = true); + bool update_obtype, bool update_thresh = true); extern void write_seeps_row (StatHdrColumns &, const SeepsAggScore *, STATOutputType, AsciiTable &, int &, AsciiTable &, int &, bool update_thresh = true); extern void write_seeps_mpr_row (StatHdrColumns &, const PairDataPoint *, STATOutputType, AsciiTable &, int &, AsciiTable &, int &, - bool update_thresh = true); + bool update_obtype, bool update_thresh = true); extern void write_isc_row (StatHdrColumns &, const ISCInfo &, STATOutputType, AsciiTable &, int &, AsciiTable &, int &); extern void write_ecnt_row (StatHdrColumns &, const ECNTInfo &, STATOutputType, @@ -124,7 +124,8 @@ extern void write_rhist_row (StatHdrColumns &, const PairDataEnsemble *, STATOut extern void write_phist_row (StatHdrColumns &, const PairDataEnsemble *, STATOutputType, AsciiTable &, int &, AsciiTable &, int &); extern void write_orank_row (StatHdrColumns &, const PairDataEnsemble *, STATOutputType, - AsciiTable &, int &, AsciiTable &, int &); + AsciiTable &, int &, AsciiTable &, int &, + bool update_obtype = false); extern void write_ssvar_row (StatHdrColumns &, const PairDataEnsemble *, double, STATOutputType, AsciiTable &, int &, AsciiTable &, int &); extern void write_relp_row (StatHdrColumns &, const PairDataEnsemble *, STATOutputType, diff --git a/src/libcode/vx_statistics/pair_base.cc b/src/libcode/vx_statistics/pair_base.cc index 89be360b65..a3b383f9fa 100644 --- a/src/libcode/vx_statistics/pair_base.cc +++ b/src/libcode/vx_statistics/pair_base.cc @@ -104,6 +104,7 @@ void PairBase::clear() { ocsd_na.clear(); ocdf_na.clear(); + typ_sa.clear(); sid_sa.clear(); lat_na.clear(); lon_na.clear(); @@ -155,6 +156,7 @@ void PairBase::erase() { ocsd_na.erase(); ocdf_na.erase(); + typ_sa.clear(); // no erase option sid_sa.clear(); // no erase option lat_na.erase(); lon_na.erase(); @@ -427,7 +429,7 @@ void PairBase::compute_climo_cdf() { //////////////////////////////////////////////////////////////////////// -bool PairBase::add_point_obs(const char *sid, +bool PairBase::add_point_obs(const char *typ, const char *sid, double lat, double lon, double x, double y, unixtime ut, double lvl, double elv, double o, const char *qc, @@ -476,6 +478,7 @@ bool PairBase::add_point_obs(const char *sid, } else { station_values_t val; + val.typ = string(typ); val.sid = string(sid); val.lat = lat; val.lon = lon; @@ -497,6 +500,7 @@ bool PairBase::add_point_obs(const char *sid, } if(obs_summary == ObsSummary::None) { + typ_sa.add(typ); sid_sa.add(sid); lat_na.add(lat); lon_na.add(lon); @@ -520,7 +524,7 @@ bool PairBase::add_point_obs(const char *sid, //////////////////////////////////////////////////////////////////////// -void PairBase::set_point_obs(int i_obs, const char *sid, +void PairBase::set_point_obs(int i_obs, const char *typ, const char *sid, double lat, double lon, double x, double y, unixtime ut, double lvl, double elv, double o, const char *qc, @@ -534,6 +538,7 @@ void PairBase::set_point_obs(int i_obs, const char *sid, exit(1); } + typ_sa.set(i_obs, typ); sid_sa.set(i_obs, sid); lat_na.set(i_obs, lat); lon_na.set(i_obs, lon); @@ -751,6 +756,7 @@ void PairBase::calc_obs_summary(){ // Store summarized value in the map svt.summary_val = ob.val; + typ_sa.add (svt.typ.c_str()); sid_sa.add (svt.sid.c_str()); lat_na.add (svt.lat); lon_na.add (svt.lon); diff --git a/src/libcode/vx_statistics/pair_base.h b/src/libcode/vx_statistics/pair_base.h index 03b186a750..a734d1af31 100644 --- a/src/libcode/vx_statistics/pair_base.h +++ b/src/libcode/vx_statistics/pair_base.h @@ -36,6 +36,7 @@ struct station_values_t { void clear(); + std::string typ; std::string sid; double lat; double lon; @@ -110,6 +111,7 @@ class PairBase { NumArray ocdf_na; // Observation climatology cumulative distribution function [n_obs] // Point Observation Information + StringArray typ_sa; // Message type [n_obs] StringArray sid_sa; // Station ID [n_obs] NumArray lat_na; // Latitude [n_obs] NumArray lon_na; // Longitude [n_obs] @@ -168,13 +170,15 @@ class PairBase { ob_val_t compute_median(std::string sng_key); ob_val_t compute_percentile(std::string sng_key, int perc); - bool add_point_obs(const char *, double, double, double, double, + bool add_point_obs(const char *, const char *, + double, double, double, double, unixtime, double, double, double, const char *, const ClimoPntInfo &, double); - void set_point_obs(int, const char *, double, double, double, double, - unixtime, double, double, double, const char *, - const ClimoPntInfo &, double); + void set_point_obs(int, const char *, const char *, + double, double, double, double, + unixtime, double, double, double, + const char *, const ClimoPntInfo &, double); void add_grid_obs(double, const ClimoPntInfo &, double); diff --git a/src/libcode/vx_statistics/pair_data_ensemble.cc b/src/libcode/vx_statistics/pair_data_ensemble.cc index f4ccbe279e..0d80abce1a 100644 --- a/src/libcode/vx_statistics/pair_data_ensemble.cc +++ b/src/libcode/vx_statistics/pair_data_ensemble.cc @@ -1305,8 +1305,9 @@ void VxPairDataEnsemble::add_point_obs(float *hdr_arr, int *hdr_typ_arr, // Add the observation value // Weight is from the nearest grid point int n = three_to_one(i_msg_typ, i_mask, i_interp); - if(!pd[n].add_point_obs(hdr_sid_str, hdr_lat, hdr_lon, - obs_x, obs_y, hdr_ut, obs_lvl, obs_hgt, + if(!pd[n].add_point_obs(hdr_typ_str, hdr_sid_str, + hdr_lat, hdr_lon, obs_x, obs_y, + hdr_ut, obs_lvl, obs_hgt, obs_v, obs_qty, cpi, default_weight)) { if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { diff --git a/src/libcode/vx_statistics/pair_data_point.cc b/src/libcode/vx_statistics/pair_data_point.cc index 5c8188e868..85595d69d3 100644 --- a/src/libcode/vx_statistics/pair_data_point.cc +++ b/src/libcode/vx_statistics/pair_data_point.cc @@ -156,7 +156,8 @@ void PairDataPoint::assign(const PairDataPoint &pd) { ClimoPntInfo cpi(pd.fcmn_na[i], pd.fcsd_na[i], pd.ocmn_na[i], pd.ocsd_na[i]); - if(add_point_pair(pd.sid_sa[i].c_str(), pd.lat_na[i], pd.lon_na[i], + if(add_point_pair(pd.typ_sa[i].c_str(), pd.sid_sa[i].c_str(), + pd.lat_na[i], pd.lon_na[i], pd.x_na[i], pd.y_na[i], pd.vld_ta[i], pd.lvl_na[i], pd.elv_na[i], pd.f_na[i], pd.o_na[i], pd.o_qc_sa[i].c_str(), @@ -176,13 +177,14 @@ void PairDataPoint::assign(const PairDataPoint &pd) { //////////////////////////////////////////////////////////////////////// -bool PairDataPoint::add_point_pair(const char *sid, double lat, double lon, +bool PairDataPoint::add_point_pair(const char *typ, const char *sid, + double lat, double lon, double x, double y, unixtime ut, double lvl, double elv, double f, double o, const char *qc, const ClimoPntInfo &cpi, double wgt) { - if(!add_point_obs(sid, lat, lon, x, y, ut, lvl, elv, o, qc, + if(!add_point_obs(typ, sid, lat, lon, x, y, ut, lvl, elv, o, qc, cpi, wgt)) return false; f_na.add(f); @@ -233,7 +235,8 @@ void PairDataPoint::set_seeps_score(SeepsScore *seeps, int index) { //////////////////////////////////////////////////////////////////////// -void PairDataPoint::set_point_pair(int i_obs, const char *sid, +void PairDataPoint::set_point_pair(int i_obs, + const char *typ, const char *sid, double lat, double lon, double x, double y, unixtime ut, double lvl, double elv, @@ -248,7 +251,7 @@ void PairDataPoint::set_point_pair(int i_obs, const char *sid, exit(1); } - set_point_obs(i_obs, sid, lat, lon, x, y, ut, lvl, elv, + set_point_obs(i_obs, typ, sid, lat, lon, x, y, ut, lvl, elv, o, qc, cpi, wgt); f_na.set(i_obs, f); @@ -380,8 +383,8 @@ PairDataPoint PairDataPoint::subset_pairs_cnt_thresh( // Handle point data if(is_point_vx()) { - if(out_pd.add_point_pair(sid_sa[i].c_str(), lat_na[i], - lon_na[i], x_na[i], y_na[i], + if(out_pd.add_point_pair(typ_sa[i].c_str(), sid_sa[i].c_str(), + lat_na[i], lon_na[i], x_na[i], y_na[i], vld_ta[i], lvl_na[i], elv_na[i], f_na[i], o_na[i], o_qc_sa[i].c_str(), cpi, wgt_na[i])) { @@ -621,7 +624,7 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, // Add the forecast, climatological, and observation data // Weight is from the nearest grid point int n = three_to_one(i_msg_typ, i_mask, i_interp); - if(!pd[n].add_point_pair(hdr_sid_str, + if(!pd[n].add_point_pair(hdr_typ_str, hdr_sid_str, hdr_lat, hdr_lon, obs_x, obs_y, hdr_ut, obs_lvl, obs_hgt, fcst_v, obs_v, obs_qty, cpi, default_weight)) { @@ -986,14 +989,14 @@ void subset_wind_pairs(const PairDataPoint &pd_u, const PairDataPoint &pd_v, // Handle point data if(pd_u.is_point_vx()) { - out_pd_u.add_point_pair(pd_u.sid_sa[i].c_str(), + out_pd_u.add_point_pair(pd_u.typ_sa[i].c_str(), pd_u.sid_sa[i].c_str(), pd_u.lat_na[i], pd_u.lon_na[i], pd_u.x_na[i], pd_u.y_na[i], pd_u.vld_ta[i], pd_u.lvl_na[i], pd_u.elv_na[i], pd_u.f_na[i], pd_u.o_na[i], pd_u.o_qc_sa[i].c_str(), u_cpi, pd_u.wgt_na[i]); - out_pd_v.add_point_pair(pd_v.sid_sa[i].c_str(), + out_pd_v.add_point_pair(pd_v.typ_sa[i].c_str(), pd_v.sid_sa[i].c_str(), pd_v.lat_na[i], pd_v.lon_na[i], pd_v.x_na[i], pd_v.y_na[i], pd_v.vld_ta[i], pd_v.lvl_na[i], pd_v.elv_na[i], @@ -1064,8 +1067,8 @@ PairDataPoint subset_climo_cdf_bin(const PairDataPoint &pd, // Handle point data if(pd.is_point_vx()) { - out_pd.add_point_pair(pd.sid_sa[i].c_str(), pd.lat_na[i], - pd.lon_na[i], pd.x_na[i], pd.y_na[i], + out_pd.add_point_pair(pd.typ_sa[i].c_str(), pd.sid_sa[i].c_str(), + pd.lat_na[i], pd.lon_na[i], pd.x_na[i], pd.y_na[i], pd.vld_ta[i], pd.lvl_na[i], pd.elv_na[i], pd.f_na[i], pd.o_na[i], pd.o_qc_sa[i].c_str(), cpi, pd.wgt_na[i]); diff --git a/src/libcode/vx_statistics/pair_data_point.h b/src/libcode/vx_statistics/pair_data_point.h index 529975beb3..5f43a5e185 100644 --- a/src/libcode/vx_statistics/pair_data_point.h +++ b/src/libcode/vx_statistics/pair_data_point.h @@ -56,17 +56,20 @@ class PairDataPoint : public PairBase { void extend(int); - bool add_point_pair(const char *, double, double, double, double, - unixtime, double, double, double, double, const char *, - const ClimoPntInfo &, double); + bool add_point_pair(const char *, const char *, + double, double, double, double, + unixtime, double, double, double, double, + const char *, const ClimoPntInfo &, double); + void load_seeps_climo(const ConcatString &seeps_climo_name); void set_seeps_thresh(const SingleThresh &p1_thresh); void set_seeps_score(SeepsScore *, int index=-1); - void set_point_pair(int, const char *, double, double, double, double, - unixtime, double, double, double, double, const char *, - const ClimoPntInfo &, double, - const SeepsScore *); + void set_point_pair(int, const char *, const char *, + double, double, double, double, + unixtime, double, double, double, double, + const char *, const ClimoPntInfo &, + double, const SeepsScore *); bool add_grid_pair(double, double, const ClimoPntInfo &, double); diff --git a/src/tools/core/ensemble_stat/ensemble_stat.cc b/src/tools/core/ensemble_stat/ensemble_stat.cc index 81c0195569..68cfe50cc1 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -77,6 +77,8 @@ // 044 06/17/24 Halley Gotway MET #2856 Reinitialize climo_cdf pointer // 045 07/05/24 Halley Gotway MET #2924 Support forecast climatology. // 046 10/08/24 Halley Gotway MET #2887 Compute weighted contingency tables. +// 047 10/14/24 Halley Gotway MET #2279 Add point_weight_flag option. +// 048 10/15/24 Halley Gotway MET #2893 Write individual pair OBTYPE. // //////////////////////////////////////////////////////////////////////// @@ -2237,10 +2239,17 @@ void write_txt_files(const EnsembleStatVxOpt &vx_opt, // Set the header column shc.set_obs_thresh(na_str); + // Store current obtype value + string cur_obtype = shc.get_obtype(); + write_orank_row(shc, &pd_all, vx_opt.output_flag[i_orank], stat_at, i_stat_row, - txt_at[i_orank], i_txt_row[i_orank]); + txt_at[i_orank], i_txt_row[i_orank], + conf_info.obtype_as_group_val_flag); + + // Reset the obtype column + shc.set_obtype(cur_obtype.c_str()); // Reset the observation valid time shc.set_obs_valid_beg(vx_opt.vx_pd.beg_ut); diff --git a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc index 912f577803..c218975d6b 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc +++ b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc @@ -70,6 +70,7 @@ void EnsembleStatConfInfo::clear() { vld_ens_thresh = bad_data_double; vld_data_thresh = bad_data_double; msg_typ_group_map.clear(); + obtype_as_group_val_flag = false; msg_typ_sfc.clear(); mask_area_map.clear(); mask_sid_map.clear(); @@ -176,6 +177,10 @@ void EnsembleStatConfInfo::process_config(GrdFileType etype, msg_typ_sfc.parse_css(default_msg_typ_group_surface); } + // Conf: obtype_as_group_val_flag + obtype_as_group_val_flag = + conf.lookup_bool(conf_key_obtype_as_group_val_flag); + // Conf: ens_member_ids ens_member_ids = parse_conf_ens_member_ids(&conf); diff --git a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h index 36e8ba2136..36d6cf5f95 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h +++ b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h @@ -221,6 +221,7 @@ class EnsembleStatConfInfo { // Message type groups that should be processed together std::map msg_typ_group_map; StringArray msg_typ_sfc; + bool obtype_as_group_val_flag; // Mapping of mask names to MaskPlanes std::map mask_area_map; diff --git a/src/tools/core/point_stat/point_stat.cc b/src/tools/core/point_stat/point_stat.cc index 0b88407abc..735f85107e 100644 --- a/src/tools/core/point_stat/point_stat.cc +++ b/src/tools/core/point_stat/point_stat.cc @@ -105,6 +105,9 @@ // 053 10/03/22 Prestopnik MET #2227 Remove using namespace netCDF from header files. // 054 04/29/24 Halley Gotway MET #2795 Move level mismatch warning. // 055 07/05/24 Halley Gotway MET #2924 Support forecast climatology. +// 056 10/08/24 Halley Gotway MET #2887 Compute weighted contingency tables. +// 057 10/14/24 Halley Gotway MET #2279 Add point_weight_flag option. +// 058 10/15/24 Halley Gotway MET #2893 Write individual pair OBTYPE. // //////////////////////////////////////////////////////////////////////// @@ -1109,7 +1112,11 @@ void process_scores() { write_mpr_row(shc, pd_ptr, conf_info.vx_opt[i_vx].output_flag[i_mpr], stat_at, i_stat_row, - txt_at[i_mpr], i_txt_row[i_mpr]); + txt_at[i_mpr], i_txt_row[i_mpr], + conf_info.obtype_as_group_val_flag); + + // Reset the obtype column + shc.set_obtype(conf_info.vx_opt[i_vx].msg_typ[i_msg_typ].c_str()); // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut); @@ -1121,7 +1128,11 @@ void process_scores() { write_seeps_mpr_row(shc, pd_ptr, conf_info.vx_opt[i_vx].output_flag[i_seeps_mpr], stat_at, i_stat_row, - txt_at[i_seeps_mpr], i_txt_row[i_seeps_mpr]); + txt_at[i_seeps_mpr], i_txt_row[i_seeps_mpr], + conf_info.obtype_as_group_val_flag); + + // Reset the obtype column + shc.set_obtype(conf_info.vx_opt[i_vx].msg_typ[i_msg_typ].c_str()); // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut); @@ -1911,7 +1922,8 @@ void do_hira_ens(int i_vx, const PairDataPoint *pd_ptr) { pd_ptr->ocmn_na[j], pd_ptr->ocsd_na[j]); // Store the observation value - hira_pd.add_point_obs(pd_ptr->sid_sa[j].c_str(), + hira_pd.add_point_obs( + pd_ptr->typ_sa[j].c_str(), pd_ptr->sid_sa[j].c_str(), pd_ptr->lat_na[j], pd_ptr->lon_na[j], pd_ptr->x_na[j], pd_ptr->y_na[j], pd_ptr->vld_ta[j], pd_ptr->lvl_na[j], pd_ptr->elv_na[j], @@ -1966,7 +1978,11 @@ void do_hira_ens(int i_vx, const PairDataPoint *pd_ptr) { write_orank_row(shc, &hira_pd, conf_info.vx_opt[i_vx].output_flag[i_orank], stat_at, i_stat_row, - txt_at[i_orank], i_txt_row[i_orank]); + txt_at[i_orank], i_txt_row[i_orank], + conf_info.obtype_as_group_val_flag); + + // Reset the obtype column + shc.set_obtype(pd_ptr->msg_typ.c_str()); // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut); @@ -2108,7 +2124,9 @@ void do_hira_prob(int i_vx, const PairDataPoint *pd_ptr) { } // Store the fractional coverage pair - hira_pd.add_point_pair(pd_ptr->sid_sa[k].c_str(), + hira_pd.add_point_pair( + pd_ptr->typ_sa[k].c_str(), + pd_ptr->sid_sa[k].c_str(), pd_ptr->lat_na[k], pd_ptr->lon_na[k], pd_ptr->x_na[k], pd_ptr->y_na[k], pd_ptr->vld_ta[k], pd_ptr->lvl_na[k], pd_ptr->elv_na[k], @@ -2156,7 +2174,12 @@ void do_hira_prob(int i_vx, const PairDataPoint *pd_ptr) { write_mpr_row(shc, &hira_pd, conf_info.vx_opt[i_vx].output_flag[i_mpr], stat_at, i_stat_row, - txt_at[i_mpr], i_txt_row[i_mpr], false); + txt_at[i_mpr], i_txt_row[i_mpr], + conf_info.obtype_as_group_val_flag, + false); + + // Reset the obtype column + shc.set_obtype(pd_ptr->msg_typ.c_str()); // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut); diff --git a/src/tools/core/point_stat/point_stat_conf_info.cc b/src/tools/core/point_stat/point_stat_conf_info.cc index f3c7e511e2..cfd5e10432 100644 --- a/src/tools/core/point_stat/point_stat_conf_info.cc +++ b/src/tools/core/point_stat/point_stat_conf_info.cc @@ -69,6 +69,7 @@ void PointStatConfInfo::clear() { topo_use_obs_thresh.clear(); topo_interp_fcst_thresh.clear(); msg_typ_group_map.clear(); + obtype_as_group_val_flag = false; mask_area_map.clear(); mask_sid_map.clear(); point_weight_flag = PointWeightType::None; @@ -178,6 +179,10 @@ void PointStatConfInfo::process_config(GrdFileType ftype) { // Conf: message_type_group_map msg_typ_group_map = parse_conf_message_type_group_map(&conf); + // Conf: obtype_as_group_val_flag + obtype_as_group_val_flag = + conf.lookup_bool(conf_key_obtype_as_group_val_flag); + // Conf: fcst.field and obs.field fdict = conf.lookup_array(conf_key_fcst_field); odict = conf.lookup_array(conf_key_obs_field); diff --git a/src/tools/core/point_stat/point_stat_conf_info.h b/src/tools/core/point_stat/point_stat_conf_info.h index e5adb0df19..9db3081dd7 100644 --- a/src/tools/core/point_stat/point_stat_conf_info.h +++ b/src/tools/core/point_stat/point_stat_conf_info.h @@ -235,6 +235,7 @@ class PointStatConfInfo { // Message type groups that should be processed together std::map msg_typ_group_map; + bool obtype_as_group_val_flag; // Mapping of mask names to DataPlanes std::map mask_area_map; diff --git a/src/tools/other/gsi_tools/gsidens2orank.cc b/src/tools/other/gsi_tools/gsidens2orank.cc index 31aa675774..d7d4b5bee4 100644 --- a/src/tools/other/gsi_tools/gsidens2orank.cc +++ b/src/tools/other/gsi_tools/gsidens2orank.cc @@ -269,7 +269,7 @@ void process_conv_data(ConvData &d, int i_mem) { // Store the current observation info ClimoPntInfo cpi(bad_data_double, bad_data_double, bad_data_double, bad_data_double); - ens_pd.add_point_obs(d.sid.c_str(), d.lat, d.lon, + ens_pd.add_point_obs(d.obtype.c_str(), d.sid.c_str(), d.lat, d.lon, bad_data_double, bad_data_double, d.obs_ut, d.prs, d.elv, d.obs, na_str, cpi, default_weight); @@ -429,7 +429,7 @@ void process_rad_data(RadData &d, int i_mem) { // Store the current observation info ClimoPntInfo cpi(bad_data_double, bad_data_double, bad_data_double, bad_data_double); - ens_pd.add_point_obs(na_str, d.lat, d.lon, + ens_pd.add_point_obs(na_str, na_str, d.lat, d.lon, bad_data_double, bad_data_double, d.obs_ut, bad_data_double, d.elv, d.obs, na_str, cpi, default_weight);