From 3d7c5508fc5faec33b4bf45e5691c3d3783f1f47 Mon Sep 17 00:00:00 2001 From: johnhg Date: Thu, 24 Feb 2022 12:01:43 -0700 Subject: [PATCH] Update develop-ref after #2061 (#2069) Co-authored-by: Julie Prestopnik Co-authored-by: johnhg Co-authored-by: Seth Linden Co-authored-by: John Halley Gotway Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: Howard Soh Co-authored-by: John Halley Gotway Co-authored-by: jprestop Co-authored-by: Howard Soh Co-authored-by: Seth Linden Co-authored-by: hsoh-u Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: John Halley Gotway Co-authored-by: MET Tools Test Account Co-authored-by: mo-mglover <78152252+mo-mglover@users.noreply.github.com> Co-authored-by: davidalbo Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> --- .github/jobs/set_job_controls.sh | 26 +- .github/pull_request_template.md | 1 + met/README | 2 +- met/data/config/ConfigConstants | 7 + met/data/config/GenEnsProdConfig_default | 1 + met/docs/Users_Guide/appendixA.rst | 5 +- met/docs/Users_Guide/gen-ens-prod.rst | 20 ++ met/src/basic/vx_config/config_constants.h | 6 + met/src/basic/vx_config/config_util.cc | 30 ++ met/src/basic/vx_config/config_util.h | 2 + met/src/basic/vx_util/Makefile.am | 1 + met/src/basic/vx_util/data_plane.cc | 62 +++- met/src/basic/vx_util/data_plane.h | 11 +- met/src/basic/vx_util/normalize.cc | 125 +++++++ met/src/basic/vx_util/normalize.h | 56 ++++ met/src/basic/vx_util/vx_util.h | 1 + .../libcode/vx_data2d_grib2/data2d_grib2.cc | 2 +- .../core/series_analysis/series_analysis.cc | 7 + .../tools/other/gen_ens_prod/gen_ens_prod.cc | 305 ++++++++++++++---- .../gen_ens_prod/gen_ens_prod_conf_info.cc | 6 +- .../gen_ens_prod/gen_ens_prod_conf_info.h | 3 +- met/src/tools/other/pb2nc/pb2nc.cc | 2 +- test/config/GenEnsProdConfig | 1 + test/config/GenEnsProdConfig_normalize | 152 +++++++++ test/config/GenEnsProdConfig_single_file_grib | 1 + test/config/GenEnsProdConfig_single_file_nc | 1 + test/xml/unit_gen_ens_prod.xml | 28 ++ 27 files changed, 769 insertions(+), 95 deletions(-) create mode 100644 met/src/basic/vx_util/normalize.cc create mode 100644 met/src/basic/vx_util/normalize.h create mode 100644 test/config/GenEnsProdConfig_normalize diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index e6605f959c..14f85ddb93 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -10,8 +10,8 @@ input_data_version=develop truth_data_version=develop if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then - - # only run diff logic if pull request INTO + + # only run diff logic if pull request INTO # branches not ending with -ref if [ "${GITHUB_BASE_REF: -4}" != "-ref" ]; then @@ -36,25 +36,37 @@ elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then if [ "$branch_name" == "develop" ] || [ "${branch_name:0:6}" == "main_v" ]; then - run_diff=true - truth_data_version=${branch_name} + run_diff=true + truth_data_version=${branch_name} fi + # check commit messages for skip or force keywords + if grep -q "ci-skip-all" <<< "$commit_msg"; then + + run_compile=false + run_push=false + run_unit_tests=false + run_diff=false + run_update_truth=false + + fi + # check commit messages for ci-skip or ci-run keywords if grep -q "ci-skip-compile" <<< "$commit_msg"; then - run_compile=false + run_compile=false fi if grep -q "ci-run-unit" <<< "$commit_msg"; then - run_diff=true + run_diff=true fi + fi - + fi # if updating truth or running diff, run unit tests diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index c2984bffdf..7d99be08c6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -23,6 +23,7 @@ If **yes**, describe the new output and/or changes to the existing output:
## Pull Request Checklist ## See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details. +- [ ] Review the source issue metadata (required labels, projects, and milestone). - [ ] Complete the PR definition above. - [ ] Ensure the PR title matches the feature or bugfix branch name. - [ ] Define the PR metadata, as permissions allow. diff --git a/met/README b/met/README index d05da60c0e..9ea770c2bf 100644 --- a/met/README +++ b/met/README @@ -22,7 +22,7 @@ This is the main directory for the Model Evaluation Tools source code release. For questions, please: - Refer to the MET User's Guide: https://met.readthedocs.io/en/latest - Refer to the MET website: http://dtcenter.org/community-code/model-evaluation-tools-met -- Send mail to met_help@ucar.edu. +- Create a post in the METplus GitHub Discussions forum: https://github.com/dtcenter/METplus/discussions Dependencies ------------ diff --git a/met/data/config/ConfigConstants b/met/data/config/ConfigConstants index 29b2590ada..8cbbeef1e9 100644 --- a/met/data/config/ConfigConstants +++ b/met/data/config/ConfigConstants @@ -170,3 +170,10 @@ BETA = 8; TOP = TRUE; BOTTOM = FALSE; +// Normalization types +NONE = 1; +CLIMO_ANOM = 2; +CLIMO_STD_ANOM = 3; +FCST_ANOM = 4; +FCST_STD_ANOM = 5; + diff --git a/met/data/config/GenEnsProdConfig_default b/met/data/config/GenEnsProdConfig_default index 09746b626e..7a3db915cd 100644 --- a/met/data/config/GenEnsProdConfig_default +++ b/met/data/config/GenEnsProdConfig_default @@ -38,6 +38,7 @@ regrid = { // censor_thresh = []; censor_val = []; +normalize = NONE; cat_thresh = []; nc_var_str = ""; diff --git a/met/docs/Users_Guide/appendixA.rst b/met/docs/Users_Guide/appendixA.rst index 7bf41e810b..9b47a535bd 100644 --- a/met/docs/Users_Guide/appendixA.rst +++ b/met/docs/Users_Guide/appendixA.rst @@ -1777,9 +1777,8 @@ Try the following 2 things: MET_BUFRLIB=/home/username/BUFRLIB_v10.2.3 -After doing that, please try recompiling MET. If it fails, -please send met_help@ucar.edu the following log files. -"make_install.log" as well as "config.log". +After doing that, please try recompiling MET. If it fails, please submit the following log files: "make_install.log" as well as "config.log" with a new post in the `METplus GitHub Discussions Forum `_. + Command line double quotes -------------------------- diff --git a/met/docs/Users_Guide/gen-ens-prod.rst b/met/docs/Users_Guide/gen-ens-prod.rst index cd6f963b4f..e697dd5e1d 100644 --- a/met/docs/Users_Guide/gen-ens-prod.rst +++ b/met/docs/Users_Guide/gen-ens-prod.rst @@ -177,6 +177,26 @@ Each value in the array will replace the text **MET_ENS_MEMBER_ID**. to read a control member. This value is only used when the **-ctrl** command line argument is used. The value should not be found in the **ens_member_ids** array. +_____________________ + +.. code-block:: none + + normalize = NONE; + +The **normalize** option defines if and how the input ensemble member data should be normalized. Options are provided to normalize relative to an external climatology, specified using the **climo_mean** and **climo_stdev** dictionaries, or relative to current ensemble forecast being processed. The anomaly is computed by subtracting the (climatological or ensemble) mean from each ensemble memeber. The standard anomaly is computed by dividing the anomaly by the (climatological or ensemble) standard deviation. Values for the **normalize** option are described below: + +• **NONE** (default) to skip the normalization step and process the raw ensemble member data. + +• **CLIMO_ANOM** to subtract the climatological mean field. + +• **CLIMO_STD_ANOM** to subtract the climatological mean field and divide by the climatological standard deviation. + +• **FCST_ANOM** to subtract the current ensemble mean field. + +• **FCST_STD_ANOM** to subtract the current ensemble mean field and divide by the current ensemble standard deviation. + +Note that the **normalize** option may be specified separately for each entry in the **ens.field** array. + _______________________ .. code-block:: none diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h index f3c5550f02..ae88c78d3e 100644 --- a/met/src/basic/vx_config/config_constants.h +++ b/met/src/basic/vx_config/config_constants.h @@ -752,6 +752,12 @@ static const char conf_key_dist_parm[] = "dist_parm"; static const char conf_key_inst_bias_scale[] = "inst_bias_scale"; static const char conf_key_inst_bias_offset[] = "inst_bias_offset"; +// +// Gen-Ens-Prod specific parameter key names +// + +static const char conf_key_normalize[] = "normalize"; + // Distribution options static const char conf_val_normal[] = "NORMAL"; static const char conf_val_exponential[] = "EXPONENTIAL"; diff --git a/met/src/basic/vx_config/config_util.cc b/met/src/basic/vx_config/config_util.cc index edeabb649a..e08ef7d19b 100644 --- a/met/src/basic/vx_config/config_util.cc +++ b/met/src/basic/vx_config/config_util.cc @@ -2981,3 +2981,33 @@ StringArray parse_conf_ens_member_ids(Dictionary *dict) { } /////////////////////////////////////////////////////////////////////////////// + +NormalizeType parse_conf_normalize(Dictionary *dict) { + NormalizeType t = NormalizeType_None; + int v; + + if(!dict) { + mlog << Error << "\nparse_conf_normalize() -> " + << "empty dictionary!\n\n"; + exit(1); + } + + // Get the integer flag value for the current entry + v = dict->lookup_int(conf_key_normalize); + + // Convert integer to enumerated NormalizeType + if(v == conf_const.lookup_int(normalizetype_none_str)) t = NormalizeType_None; + else if(v == conf_const.lookup_int(normalizetype_climo_anom_str)) t = NormalizeType_ClimoAnom; + else if(v == conf_const.lookup_int(normalizetype_climo_std_anom_str)) t = NormalizeType_ClimoStdAnom; + else if(v == conf_const.lookup_int(normalizetype_fcst_anom_str)) t = NormalizeType_FcstAnom; + else if(v == conf_const.lookup_int(normalizetype_fcst_std_anom_str)) t = NormalizeType_FcstStdAnom; + else { + mlog << Error << "\nparse_conf_normalize() -> " + << "Unexpected value of " << v << ".\n\n"; + exit(1); + } + + return(t); +} + +/////////////////////////////////////////////////////////////////////////////// diff --git a/met/src/basic/vx_config/config_util.h b/met/src/basic/vx_config/config_util.h index 5afcc974a1..d39d08b615 100644 --- a/met/src/basic/vx_config/config_util.h +++ b/met/src/basic/vx_config/config_util.h @@ -22,6 +22,7 @@ #include "config_file.h" #include "data_file_type.h" #include "config_gaussian.h" +#include "normalize.h" //////////////////////////////////////////////////////////////////////// @@ -78,6 +79,7 @@ extern map extern void parse_conf_range_int(Dictionary *dict, int &beg, int &end); extern void parse_conf_range_double(Dictionary *dict, double &beg, double &end); extern StringArray parse_conf_ens_member_ids(Dictionary *dict); +extern NormalizeType parse_conf_normalize(Dictionary *dict); extern void check_mask_names(const StringArray &); diff --git a/met/src/basic/vx_util/Makefile.am b/met/src/basic/vx_util/Makefile.am index ae466f0963..d0ab31bce7 100644 --- a/met/src/basic/vx_util/Makefile.am +++ b/met/src/basic/vx_util/Makefile.am @@ -37,6 +37,7 @@ libvx_util_a_SOURCES = ascii_table.cc ascii_table.h \ thresh_array.cc thresh_array.h \ make_path.cc make_path.h \ memory.cc memory.h \ + normalize.cc normalize.h \ ordinal.cc ordinal.h \ roman_numeral.cc roman_numeral.h \ string_fxns.cc string_fxns.h \ diff --git a/met/src/basic/vx_util/data_plane.cc b/met/src/basic/vx_util/data_plane.cc index 994c988029..618627dd63 100644 --- a/met/src/basic/vx_util/data_plane.cc +++ b/met/src/basic/vx_util/data_plane.cc @@ -406,23 +406,71 @@ void DataPlane::censor(const ThreshArray &censor_thresh, /////////////////////////////////////////////////////////////////////////////// +void DataPlane::anomaly(const DataPlane &mn) { + + // Check dimensions + if(Nxy != mn.Nxy) { + mlog << Error << "\nDataPlane::anomaly() -> " + << "the data dimensions do not match: (" + << Nx << ", " << Ny << ") != (" + << mn.Nx << ", " << mn.Ny << ")!\n\n"; + exit(1); + } -void DataPlane::replace_bad_data(const double value) + // Subtract the mean + for(int i=0; i " + << "the data dimensions do not match: (" + << Nx << ", " << Ny << ") != (" + << mn.Nx << ", " << mn.Ny << ") != (" + << sd.Nx << ", " << sd.Ny << ")!\n\n"; + exit(1); + } - if ( is_bad_data(Data[j]) ) Data[j] = value; + // Subtract the mean and divide by the standard deviation + for(int i=0; i +#include +#include +#include +#include + +#include "normalize.h" + +/////////////////////////////////////////////////////////////////////////////// + +ConcatString normalizetype_to_string(const NormalizeType type) { + ConcatString s; + + // Convert enumerated NormalizeType to string + switch(type) { + + case NormalizeType_None: + s = normalizetype_none_str; + break; + + case NormalizeType_ClimoAnom: + s = normalizetype_climo_anom_str; + break; + + case NormalizeType_ClimoStdAnom: + s = normalizetype_climo_std_anom_str; + break; + + case NormalizeType_FcstAnom: + s = normalizetype_fcst_anom_str; + break; + + case NormalizeType_FcstStdAnom: + s = normalizetype_fcst_std_anom_str; + break; + + default: + mlog << Error << "\nnormalizetype_to_string() -> " + << "Unexpected NormalizeType value of " << type << ".\n\n"; + exit(1); + } + + return(s); +} + +/////////////////////////////////////////////////////////////////////////////// + +void normalize_data(DataPlane &dp, const NormalizeType type, + const DataPlane *cmn_ptr, const DataPlane *csd_ptr, + const DataPlane *fmn_ptr, const DataPlane *fsd_ptr) { + + mlog << Debug(3) << "Normalizing input data using " + << normalizetype_to_string(type) << ".\n"; + + // Supported types + switch(type) { + + case NormalizeType_None: + break; + + case NormalizeType_ClimoAnom: + if(!cmn_ptr || dp.nxy() != cmn_ptr->nxy()) { + mlog << Error << "\nnormalize_data() -> " + << "the climatology mean is required for " + << normalizetype_to_string(type) << ".\n\n"; + exit(1); + } + dp.anomaly(*cmn_ptr); + break; + + case NormalizeType_ClimoStdAnom: + if(!cmn_ptr || dp.nxy() != cmn_ptr->nxy() || + !csd_ptr || dp.nxy() != csd_ptr->nxy()) { + mlog << Error << "\nnormalize_data() -> " + << "the climatology mean and standard deviation are required for " + << normalizetype_to_string(type) << ".\n\n"; + exit(1); + } + dp.standard_anomaly(*cmn_ptr, *csd_ptr); + break; + + case NormalizeType_FcstAnom: + if(!fmn_ptr || dp.nxy() != fmn_ptr->nxy()) { + mlog << Error << "\nnormalize_data() -> " + << "the forecast mean is required for " + << normalizetype_to_string(type) << ".\n\n"; + exit(1); + } + dp.anomaly(*fmn_ptr); + break; + + case NormalizeType_FcstStdAnom: + if(!fmn_ptr || dp.nxy() != fmn_ptr->nxy() || + !fsd_ptr || dp.nxy() != fsd_ptr->nxy()) { + mlog << Error << "\nnormalize_data() -> " + << "the forecast mean and standard deviation are required for " + << normalizetype_to_string(type) << ".\n\n"; + exit(1); + } + dp.standard_anomaly(*fmn_ptr, *fsd_ptr); + break; + + default: + mlog << Error << "\nnormalize_data() -> " + << "unexpected NormalizeType value (" + << type << ")\n\n"; + exit(1); + } // end switch + + return; +} + +/////////////////////////////////////////////////////////////////////////////// diff --git a/met/src/basic/vx_util/normalize.h b/met/src/basic/vx_util/normalize.h new file mode 100644 index 0000000000..90eff14999 --- /dev/null +++ b/met/src/basic/vx_util/normalize.h @@ -0,0 +1,56 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2022 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __NORMALIZE_H__ +#define __NORMALIZE_H__ + +/////////////////////////////////////////////////////////////////////////////// + +#include "concat_string.h" +#include "data_plane.h" + +/////////////////////////////////////////////////////////////////////////////// + +// +// Enumeration for normalization types +// + +enum NormalizeType { + NormalizeType_None, // No normalization + NormalizeType_ClimoAnom, // Subtract climo mean + NormalizeType_ClimoStdAnom, // Subtract climo mean and divide stdev + NormalizeType_FcstAnom, // Subtract fcst mean + NormalizeType_FcstStdAnom // Subtract fcst mean and divide stdev +}; + +/////////////////////////////////////////////////////////////////////////////// + +// +// String corresponding to the enumerated values above +// +static const char normalizetype_none_str[] = "NONE"; +static const char normalizetype_climo_anom_str[] = "CLIMO_ANOM"; +static const char normalizetype_climo_std_anom_str[] = "CLIMO_STD_ANOM"; +static const char normalizetype_fcst_anom_str[] = "FCST_ANOM"; +static const char normalizetype_fcst_std_anom_str[] = "FCST_STD_ANOM"; + +/////////////////////////////////////////////////////////////////////////////// + +extern ConcatString normalizetype_to_string(const NormalizeType); + +extern void normalize_data(DataPlane &, const NormalizeType, + const DataPlane *, const DataPlane *, + const DataPlane *, const DataPlane *); + +/////////////////////////////////////////////////////////////////////////////// + +#endif // __NORMALIZE_H__ + +/////////////////////////////////////////////////////////////////////////////// diff --git a/met/src/basic/vx_util/vx_util.h b/met/src/basic/vx_util/vx_util.h index dceeaa62c1..e818c06da1 100644 --- a/met/src/basic/vx_util/vx_util.h +++ b/met/src/basic/vx_util/vx_util.h @@ -44,6 +44,7 @@ #include "long_array.h" #include "make_path.h" #include "memory.h" +#include "normalize.h" #include "num_array.h" #include "ordinal.h" #include "roman_numeral.h" diff --git a/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc b/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc index 5646a2f13c..0061d2fd00 100644 --- a/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc +++ b/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc @@ -704,7 +704,7 @@ void MetGrib2DataFile::read_grib2_record_list() { mlog << Error << "\nMetGrib2DataFile::data_plane() -> " << "PDS template number (" << gfld->ipdtnum << ") is not supported. " - << "Please email met_help@ucar.edu.\n\n"; + << "Please create a new post with this information in the METplus GitHub Discussions forum at https://github.com/dtcenter/METplus/discussions\n\n"; exit(1); } diff --git a/met/src/tools/core/series_analysis/series_analysis.cc b/met/src/tools/core/series_analysis/series_analysis.cc index 72a897fa02..fc4c9d802b 100644 --- a/met/src/tools/core/series_analysis/series_analysis.cc +++ b/met/src/tools/core/series_analysis/series_analysis.cc @@ -662,6 +662,7 @@ void process_scores() { VarInfo *obs_info = (VarInfo *) 0; PairDataPoint *pd_ptr = (PairDataPoint *) 0; DataPlane fcst_dp, obs_dp; + const char *method_name = "process_scores() "; // Climatology mean and standard deviation DataPlane cmn_dp, csd_dp; @@ -763,6 +764,12 @@ void process_scores() { } // end for i_series + if(0 == pd_ptr) { + mlog << Debug(3) << method_name + << "PairDataPoint is not set. Skip computing statistics for each grid point in the block.\n"; + continue; + } + // Compute statistics for each grid point in the block for(i=0; i " - << "trouble reading file \"" << infile << "\"\n\n"; - exit(1); - } - - // Read the gridded data field - if((found = mtddf->data_plane(*info, dp))) { - - // Setup the verification grid, if necessary - if(nxy == 0) process_grid(mtddf->grid()); - - // Create the output file, if necessary - if(nc_out == (NcFile *) 0) setup_nc_file(); - - // Regrid, if requested and necessary - if(!(mtddf->grid() == grid)) { - mlog << Debug(1) - << "Regridding field \"" << info->magic_str() - << "\" to the verification grid.\n"; - dp = met_regrid(dp, mtddf->grid(), grid, info->regrid()); - } - - // Store the valid time, if not already set - if(ens_valid_ut == (unixtime) 0) { - ens_valid_ut = dp.valid(); - } - // Check to make sure that the valid time doesn't change - else if(ens_valid_ut != dp.valid()) { - mlog << Warning << "\nget_data_plane() -> " - << "The valid time has changed, " - << unix_to_yyyymmdd_hhmmss(ens_valid_ut) - << " != " << unix_to_yyyymmdd_hhmmss(dp.valid()) - << " in \"" << infile << "\"\n\n"; - } - - } // end if found - - // Deallocate the data file pointer, if necessary - if(mtddf) { delete mtddf; mtddf = (Met2dDataFile *) 0; } - - return(found); -} - -//////////////////////////////////////////////////////////////////////// - void process_ensemble() { int i_var, i_ens, n_ens_vld, n_ens_inputs; bool need_reset; - DataPlane ens_dp, ctrl_dp, cmn_dp, csd_dp; + DataPlane ens_dp, ctrl_dp; + DataPlane cmn_dp, csd_dp; + DataPlane emn_dp, esd_dp; unixtime max_init_ut = bad_data_ll; - VarInfo * var_info; - ConcatString ens_file; + VarInfo *var_info; + ConcatString cs, ens_file; // Loop through each of the ensemble fields to be processed vector::const_iterator var_it = conf_info.ens_input.begin(); @@ -334,15 +287,23 @@ void process_ensemble() { // Need to reinitialize counts and sums for each ensemble field need_reset = true; + // Print out the normalization flag + cs << cs_erase; + if((*var_it)->normalize != NormalizeType_None) { + cs << " with normalize = " + << normalizetype_to_string((*var_it)->normalize); + } + cs << "\n"; + var_info = (*var_it)->get_var_info(); mlog << Debug(2) << "\n" << sep_str << "\n\n" << "Processing ensemble field: " - << (*var_it)->raw_magic_str << "\n"; + << (*var_it)->raw_magic_str << cs; n_ens_inputs = (*var_it)->inputs_n(); for(i_ens=n_ens_vld=0; i_ens < n_ens_inputs; i_ens++) { - // get file and VarInfo to process + // Get file and VarInfo to process ens_file = (*var_it)->get_file(i_ens); var_info = (*var_it)->get_var_info(i_ens); @@ -382,9 +343,19 @@ void process_ensemble() { conf_info.conf.lookup_array(conf_key_climo_stdev_field, false), i_var, ens_valid_ut, grid); + // Compute the ensemble summary data, if needed + if((*var_it)->normalize == NormalizeType_FcstAnom || + (*var_it)->normalize == NormalizeType_FcstStdAnom ) { + get_ens_mean_stdev((*var_it), emn_dp, esd_dp); + } + else { + emn_dp.erase(); + esd_dp.erase(); + } + // Read ensemble control member data, if provided if(ctrl_file.nonempty()) { - VarInfo * ctrl_info = (*var_it)->get_ctrl(i_ens); + VarInfo *ctrl_info = (*var_it)->get_ctrl(i_ens); mlog << Debug(3) << "\n" << "Reading control field: " @@ -400,6 +371,12 @@ void process_ensemble() { exit(1); } + // Normalize, if requested + if((*var_it)->normalize != NormalizeType_None) { + normalize_data(ctrl_dp, (*var_it)->normalize, + &cmn_dp, &csd_dp, &emn_dp, &esd_dp); + } + // Apply current data to the running sums and counts track_counts(*var_it, ctrl_dp, true, cmn_dp, csd_dp); @@ -414,6 +391,12 @@ void process_ensemble() { } // end if need_reset + // Normalize, if requested + if((*var_it)->normalize != NormalizeType_None) { + normalize_data(ens_dp, (*var_it)->normalize, + &cmn_dp, &csd_dp, &emn_dp, &esd_dp); + } + // Apply current data to the running sums and counts track_counts(*var_it, ens_dp, false, cmn_dp, csd_dp); @@ -447,6 +430,188 @@ void process_ensemble() { //////////////////////////////////////////////////////////////////////// +void get_ens_mean_stdev(GenEnsProdVarInfo *ens_info, + DataPlane &emn_dp, DataPlane &esd_dp) { + int i_ens, nxy, j; + double ens; + NumArray emn_cnt_na, emn_sum_na; + NumArray esd_cnt_na, esd_sum_na, esd_ssq_na; + VarInfo *var_info; + ConcatString ens_file; + DataPlane ens_dp; + + // Check for null pointer + if(!ens_info) { + mlog << Error << "\nget_ens_mean_stdev() -> " + << "null pointer!\n\n"; + exit(1); + } + + mlog << Debug(2) + << "Computing the ensemble mean and standard deviation for " + << ens_info->raw_magic_str << ".\n"; + + // Loop over the ensemble inputs + for(i_ens=0; i_ens < ens_info->inputs_n(); i_ens++) { + + // Get file and VarInfo to process + ens_file = ens_info->get_file(i_ens); + var_info = ens_info->get_var_info(i_ens); + + // Skip bad data files + if(!ens_file_vld[ens_info->get_file_index(i_ens)]) continue; + + // Read data and track the valid data count + if(!get_data_plane(ens_file.c_str(), etype, + var_info, ens_dp)) { + mlog << Warning << "\nget_ens_mean_stdev() -> " + << "ensemble field \"" << var_info->magic_str() + << "\" not found in file \"" << ens_file << "\"\n\n"; + continue; + } + + // Initialize sums, if needed + if(emn_cnt_na.n() == 0) { + nxy = ens_dp.nx()*ens_dp.ny(); + emn_cnt_na.set_const(0.0, nxy); + emn_sum_na = emn_cnt_na; + esd_cnt_na = emn_cnt_na; + esd_sum_na = emn_cnt_na; + esd_ssq_na = emn_cnt_na; + } + + // Update the counts and sums + for(j=0; jget_ctrl(0); + + // Error out if missing + if(!get_data_plane(ctrl_file.c_str(), etype, + var_info, ens_dp)) { + mlog << Error << "\nget_ens_mean_stdev() -> " + << "control member ensemble field \"" + << var_info->magic_str() + << "\" not found in file \"" << ctrl_file << "\"\n\n"; + exit(1); + } + + // Update counts and sums + for(j=0; j " + << "trouble reading file \"" << infile << "\"\n\n"; + exit(1); + } + + // Read the gridded data field + if((found = mtddf->data_plane(*info, dp))) { + + // Setup the verification grid, if necessary + if(nxy == 0) process_grid(mtddf->grid()); + + // Create the output file, if necessary + if(nc_out == (NcFile *) 0) setup_nc_file(); + + // Regrid, if requested and necessary + if(!(mtddf->grid() == grid)) { + mlog << Debug(1) + << "Regridding field \"" << info->magic_str() + << "\" to the verification grid.\n"; + dp = met_regrid(dp, mtddf->grid(), grid, info->regrid()); + } + + // Store the valid time, if not already set + if(ens_valid_ut == (unixtime) 0) { + ens_valid_ut = dp.valid(); + } + // Check to make sure that the valid time doesn't change + else if(ens_valid_ut != dp.valid()) { + mlog << Warning << "\nget_data_plane() -> " + << "The valid time has changed, " + << unix_to_yyyymmdd_hhmmss(ens_valid_ut) + << " != " << unix_to_yyyymmdd_hhmmss(dp.valid()) + << " in \"" << infile << "\"\n\n"; + } + + } // end if found + + // Deallocate the data file pointer, if necessary + if(mtddf) { delete mtddf; mtddf = (Met2dDataFile *) 0; } + + return(found); +} + +//////////////////////////////////////////////////////////////////////// + void clear_counts() { int i, j; @@ -471,7 +636,7 @@ void clear_counts() { //////////////////////////////////////////////////////////////////////// -void track_counts(GenEnsProdVarInfo * ens_info, const DataPlane &ens_dp, bool is_ctrl, +void track_counts(GenEnsProdVarInfo *ens_info, const DataPlane &ens_dp, bool is_ctrl, const DataPlane &cmn_dp, const DataPlane &csd_dp) { int i, j, k; double ens, cmn, csd; @@ -582,7 +747,7 @@ void setup_nc_file() { //////////////////////////////////////////////////////////////////////// -void write_ens_nc(GenEnsProdVarInfo * ens_info, int n_ens_vld, +void write_ens_nc(GenEnsProdVarInfo *ens_info, int n_ens_vld, const DataPlane &ens_dp, const DataPlane &cmn_dp, const DataPlane &csd_dp) { @@ -839,13 +1004,13 @@ void write_ens_nc(GenEnsProdVarInfo * ens_info, int n_ens_vld, //////////////////////////////////////////////////////////////////////// -void write_ens_var_float(GenEnsProdVarInfo * ens_info, float *ens_data, const DataPlane &dp, +void write_ens_var_float(GenEnsProdVarInfo *ens_info, float *ens_data, const DataPlane &dp, const char *type_str, const char *long_name_str) { NcVar ens_var; ConcatString ens_var_name, var_str, name_str, cs; - // Append nc_var_str config file entry + // Append nc_var_str config file entry, if specified cs = ens_info->nc_var_str; if(cs.length() > 0) var_str << "_" << cs; @@ -879,7 +1044,7 @@ void write_ens_var_float(GenEnsProdVarInfo * ens_info, float *ens_data, const Da } // Add the variable attributes - add_var_att_local(ens_info->get_var_info(), &ens_var, false, dp, + add_var_att_local(ens_info, &ens_var, false, dp, name_str.c_str(), long_name_str); // Write the data @@ -895,13 +1060,13 @@ void write_ens_var_float(GenEnsProdVarInfo * ens_info, float *ens_data, const Da //////////////////////////////////////////////////////////////////////// -void write_ens_var_int(GenEnsProdVarInfo * ens_info, int *ens_data, const DataPlane &dp, +void write_ens_var_int(GenEnsProdVarInfo *ens_info, int *ens_data, const DataPlane &dp, const char *type_str, const char *long_name_str) { NcVar ens_var; ConcatString ens_var_name, var_str, name_str, cs; - // Append nc_var_str config file entry + // Append nc_var_str config file entry, if specified cs = ens_info->nc_var_str; if(cs.length() > 0) var_str << "_" << cs; @@ -926,7 +1091,7 @@ void write_ens_var_int(GenEnsProdVarInfo * ens_info, int *ens_data, const DataPl << type_str; // Add the variable attributes - add_var_att_local(ens_info->get_var_info(), &ens_var, true, dp, + add_var_att_local(ens_info, &ens_var, true, dp, name_str.c_str(), long_name_str); // Write the data @@ -942,7 +1107,7 @@ void write_ens_var_int(GenEnsProdVarInfo * ens_info, int *ens_data, const DataPl //////////////////////////////////////////////////////////////////////// -void write_ens_data_plane(GenEnsProdVarInfo * ens_info, const DataPlane &ens_dp, const DataPlane &dp, +void write_ens_data_plane(GenEnsProdVarInfo *ens_info, const DataPlane &ens_dp, const DataPlane &dp, const char *type_str, const char *long_name_str) { // Allocate memory for this data @@ -962,11 +1127,13 @@ void write_ens_data_plane(GenEnsProdVarInfo * ens_info, const DataPlane &ens_dp, //////////////////////////////////////////////////////////////////////// -void add_var_att_local(VarInfo *info, NcVar *nc_var, bool is_int, +void add_var_att_local(GenEnsProdVarInfo *ens_info, + NcVar *nc_var, bool is_int, const DataPlane &dp, const char *name_str, const char *long_name_str) { ConcatString att_str; + VarInfo *info = ens_info->get_var_info(); // Construct the long name att_str << cs_erase diff --git a/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.cc b/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.cc index 917e66c76a..7c90036af7 100644 --- a/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.cc +++ b/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.cc @@ -62,10 +62,9 @@ void GenEnsProdConfInfo::clear() { desc.clear(); for(; var_it != ens_input.end(); var_it++) { - if(*var_it) { delete *var_it; } - } + ens_input.clear(); cdf_info.clear(); nbrhd_prob.clear(); @@ -258,6 +257,9 @@ void GenEnsProdConfInfo::process_config(GrdFileType etype, StringArray * ens_fil // Keep track of the maximum number of thresholds if(ens_info->cat_ta.n() > max_n_cat) max_n_cat = ens_info->cat_ta.n(); + // Conf: normalize + ens_info->normalize = parse_conf_normalize(&i_edict); + // Conf: ensemble_flag ens_info->nc_info = parse_nc_info(&i_edict); diff --git a/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.h b/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.h index 657a908026..f92a49d153 100644 --- a/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.h +++ b/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.h @@ -122,7 +122,8 @@ inline int GenEnsProdConfInfo::get_compression_level() { return(conf.nc_compress class GenEnsProdVarInfo: public EnsVarInfo { public: - GenEnsProdNcOutInfo nc_info; // Ensemble product outputs + NormalizeType normalize; // Ensemble normalization logic + GenEnsProdNcOutInfo nc_info; // Ensemble product outputs }; #endif /* __GEN_ENS_PROD_CONF_INFO_H__ */ diff --git a/met/src/tools/other/pb2nc/pb2nc.cc b/met/src/tools/other/pb2nc/pb2nc.cc index f08fca1038..64cbf77e3f 100644 --- a/met/src/tools/other/pb2nc/pb2nc.cc +++ b/met/src/tools/other/pb2nc/pb2nc.cc @@ -1017,7 +1017,7 @@ void process_pbfile(int i_pb) { // Initialize prev_hdr_lat = prev_hdr_lon = prev_hdr_elv = bad_data_double; - cape_level = cape_count = cape_cnt_too_big = 0; + cape_level = cape_count = cape_cnt_too_big = cape_cnt_surface_msgs = 0; cape_cnt_no_levels = cape_cnt_missing_values = cape_cnt_zero_values = 0; mlcape_count = mlcape_cnt_too_big = 0; mlcape_cnt_missing_values = mlcape_cnt_zero_values = 0; diff --git a/test/config/GenEnsProdConfig b/test/config/GenEnsProdConfig index 3c4af297ef..9e0e07dedd 100644 --- a/test/config/GenEnsProdConfig +++ b/test/config/GenEnsProdConfig @@ -38,6 +38,7 @@ regrid = { // censor_thresh = []; censor_val = []; +normalize = NONE; cat_thresh = []; nc_var_str = ""; diff --git a/test/config/GenEnsProdConfig_normalize b/test/config/GenEnsProdConfig_normalize new file mode 100644 index 0000000000..d07328e973 --- /dev/null +++ b/test/config/GenEnsProdConfig_normalize @@ -0,0 +1,152 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Gen-Ens-Prod configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "WRF"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// May be set separately in each "field" entry +// +regrid = { + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// May be set separately in each "field" entry +// +censor_thresh = []; +censor_val = []; +normalize = NONE; +cat_thresh = []; +nc_var_str = ""; + +// +// Ensemble fields to be processed +// +ens = { + ens_thresh = 0.80; + vld_thresh = 1.0; + + name = "TMP"; + level = "Z2"; + + field = [ + { normalize = NONE; }, + { normalize = CLIMO_ANOM; nc_var_str = "CLIMO_ANOM"; }, + { normalize = CLIMO_STD_ANOM; nc_var_str = "CLIMO_STD_ANOM"; }, + { normalize = FCST_ANOM; nc_var_str = "FCST_ANOM"; }, + { normalize = FCST_STD_ANOM; nc_var_str = "FCST_STD_ANOM"; } + ]; +} + +// +// IDs for ensemble members +// Only set if processing a single file with all ensembles +// +ens_member_ids = []; +control_id = ""; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Neighborhood ensemble probabilities +// +nbrhd_prob = { + width = [ 5 ]; + shape = CIRCLE; + vld_thresh = 0.0; +} + +// +// NMEP smoothing methods +// +nmep_smooth = { + vld_thresh = 0.0; + shape = CIRCLE; + gaussian_dx = 81.27; + gaussian_radius = 120; + type = [ + { + method = GAUSSIAN; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology data +// +climo_mean = ens; +climo_mean = { + + file_name = [ "${CLIMO_MEAN_FILE}" ]; + + regrid = { + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; + } + + time_interp_method = DW_MEAN; + day_interval = 1; + hour_interval = 6; +} + +climo_stdev = climo_mean; +climo_stdev = { + file_name = [ "${CLIMO_STDEV_FILE}" ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Ensemble product output types +// May be set separately in each "ens.field" entry +// +ensemble_flag = { + latlon = TRUE; + mean = TRUE; + stdev = TRUE; + minus = FALSE; + plus = FALSE; + min = TRUE; + max = TRUE; + range = FALSE; + vld_count = FALSE; + frequency = FALSE; + nep = FALSE; + nmep = FALSE; + climo = TRUE; + climo_cdp = TRUE; +} + +//////////////////////////////////////////////////////////////////////////////// + +version = "V10.1.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GenEnsProdConfig_single_file_grib b/test/config/GenEnsProdConfig_single_file_grib index cc5edda640..c46677ab41 100644 --- a/test/config/GenEnsProdConfig_single_file_grib +++ b/test/config/GenEnsProdConfig_single_file_grib @@ -38,6 +38,7 @@ regrid = { // censor_thresh = []; censor_val = []; +normalize = NONE; cat_thresh = []; nc_var_str = ""; diff --git a/test/config/GenEnsProdConfig_single_file_nc b/test/config/GenEnsProdConfig_single_file_nc index 1049dc6a3a..d8e6e5aa6a 100644 --- a/test/config/GenEnsProdConfig_single_file_nc +++ b/test/config/GenEnsProdConfig_single_file_nc @@ -38,6 +38,7 @@ regrid = { // censor_thresh = []; censor_val = []; +normalize = NONE; cat_thresh = []; nc_var_str = ""; diff --git a/test/xml/unit_gen_ens_prod.xml b/test/xml/unit_gen_ens_prod.xml index 3c014a08f2..5aba6f154e 100644 --- a/test/xml/unit_gen_ens_prod.xml +++ b/test/xml/unit_gen_ens_prod.xml @@ -130,4 +130,32 @@ + + echo "&DATA_DIR_MODEL;/grib1/arw-fer-gep1/arw-fer-gep1_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-sch-gep2/arw-sch-gep2_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-tom-gep3/arw-tom-gep3_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/nmm-fer-gep4/nmm-fer-gep4_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-fer-gep5/arw-fer-gep5_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-sch-gep6/arw-sch-gep6_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-tom-gep7/arw-tom-gep7_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/nmm-fer-gep8/nmm-fer-gep8_2012040912_F024.grib" \ + > &OUTPUT_DIR;/gen_ens_prod/input_file_list; \ + &MET_BIN;/gen_ens_prod + + CLIMO_MEAN_FILE &DATA_DIR_CLIMO;/NCEP_1.0deg/cmean_1d.19790410 + CLIMO_STDEV_FILE &DATA_DIR_CLIMO;/NCEP_1.0deg/cstdv_1d.19790410 + + \ + -ens &OUTPUT_DIR;/gen_ens_prod/input_file_list \ + -ctrl &DATA_DIR_MODEL;/grib1/arw-tom-gep0/arw-tom-gep0_2012040912_F024.grib \ + -config &CONFIG_DIR;/GenEnsProdConfig_normalize \ + -out &OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_NORMALIZE.nc \ + -v 2 + + + &OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_NORMALIZE.nc + + + +