Update develop-ref after #2061 (#2069)

Co-authored-by: Julie Prestopnik <[email protected]> Co-authored-by: johnhg <[email protected]> Co-authored-by: Seth Linden <[email protected]> Co-authored-by: John Halley Gotway <[email protected]> Co-authored-by: j-opatz <[email protected]> Co-authored-by: Howard Soh <[email protected]> Co-authored-by: John Halley Gotway <[email protected]> Co-authored-by: jprestop <[email protected]> Co-authored-by: Howard Soh <[email protected]> Co-authored-by: Seth Linden <[email protected]> Co-authored-by: hsoh-u <[email protected]> Co-authored-by: George McCabe <[email protected]> Co-authored-by: John Halley Gotway <[email protected]> Co-authored-by: MET Tools Test Account <[email protected]> Co-authored-by: mo-mglover <[email protected]> Co-authored-by: davidalbo <[email protected]> Co-authored-by: lisagoodrich <[email protected]>
dtcenter · Feb 24, 2022 · 3d7c550 · 3d7c550
1 parent e9499f1
commit 3d7c550
Show file tree

Hide file tree

Showing 27 changed files with 769 additions and 95 deletions.
diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh
@@ -10,8 +10,8 @@ input_data_version=develop
 truth_data_version=develop
 
 if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
-
-  # only run diff logic if pull request INTO
+    
+  # only run diff logic if pull request INTO 
   # branches not ending with -ref
   if [ "${GITHUB_BASE_REF: -4}" != "-ref" ]; then
 
@@ -36,25 +36,37 @@ elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then
     if [ "$branch_name" == "develop" ] ||
        [ "${branch_name:0:6}" == "main_v" ]; then
 
-      run_diff=true
-      truth_data_version=${branch_name}
+       run_diff=true
+       truth_data_version=${branch_name}
 
     fi
 
+    # check commit messages for skip or force keywords                                                                         
+    if grep -q "ci-skip-all" <<< "$commit_msg"; then
+
+       run_compile=false
+       run_push=false
+       run_unit_tests=false
+       run_diff=false
+       run_update_truth=false
+
+    fi    
+
     # check commit messages for ci-skip or ci-run keywords
     if grep -q "ci-skip-compile" <<< "$commit_msg"; then
 
-      run_compile=false
+       run_compile=false
 
     fi
 
     if grep -q "ci-run-unit" <<< "$commit_msg"; then
 
-      run_diff=true
+       run_diff=true
 
     fi
+
   fi
-
+  
 fi
 
 # if updating truth or running diff, run unit tests

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -23,6 +23,7 @@ If **yes**, describe the new output and/or changes to the existing output:</br>
 
 ## Pull Request Checklist ##
 See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details.
+- [ ] Review the source issue metadata (required labels, projects, and milestone).
 - [ ] Complete the PR definition above.
 - [ ] Ensure the PR title matches the feature or bugfix branch name.
 - [ ] Define the PR metadata, as permissions allow.

diff --git a/met/README b/met/README
@@ -22,7 +22,7 @@ This is the main directory for the Model Evaluation Tools source code release.
 For questions, please:
 - Refer to the MET User's Guide: https://met.readthedocs.io/en/latest
 - Refer to the MET website: http://dtcenter.org/community-code/model-evaluation-tools-met
-- Send mail to [email protected].
+- Create a post in the METplus GitHub Discussions forum: https://github.com/dtcenter/METplus/discussions
 
 Dependencies
 ------------

diff --git a/met/data/config/ConfigConstants b/met/data/config/ConfigConstants
@@ -170,3 +170,10 @@ BETA           = 8;
 TOP            = TRUE;
 BOTTOM         = FALSE;
 
+// Normalization types
+NONE           = 1;
+CLIMO_ANOM     = 2;
+CLIMO_STD_ANOM = 3;
+FCST_ANOM      = 4;
+FCST_STD_ANOM  = 5;
+
diff --git a/met/data/config/GenEnsProdConfig_default b/met/data/config/GenEnsProdConfig_default
@@ -38,6 +38,7 @@ regrid = {
 //
 censor_thresh = [];
 censor_val    = [];
+normalize     = NONE;
 cat_thresh    = [];
 nc_var_str    = "";
 

diff --git a/met/docs/Users_Guide/appendixA.rst b/met/docs/Users_Guide/appendixA.rst
@@ -1777,9 +1777,8 @@ Try the following 2 things:
 		
 		  MET_BUFRLIB=/home/username/BUFRLIB_v10.2.3 
 
-After doing that, please try recompiling MET. If it fails,
-please send [email protected] the following log files.
-"make_install.log" as well as "config.log".
+After doing that, please try recompiling MET. If it fails, please submit the following log files: "make_install.log" as well as "config.log" with a new post in the `METplus GitHub Discussions Forum <https://github.com/dtcenter/METplus/discussions>`_.
+
 
 Command line double quotes
 --------------------------

diff --git a/met/docs/Users_Guide/gen-ens-prod.rst b/met/docs/Users_Guide/gen-ens-prod.rst
@@ -177,6 +177,26 @@ Each value in the array will replace the text **MET_ENS_MEMBER_ID**.
 to read a control member. This value is only used when the **-ctrl** command line argument is
 used. The value should not be found in the **ens_member_ids** array.
 
+_____________________
+
+.. code-block:: none
+
+  normalize = NONE;
+
+The **normalize** option defines if and how the input ensemble member data should be normalized. Options are provided to normalize relative to an external climatology, specified using the **climo_mean** and **climo_stdev** dictionaries, or relative to current ensemble forecast being processed. The anomaly is computed by subtracting the (climatological or ensemble) mean from each ensemble memeber. The standard anomaly is computed by dividing the anomaly by the (climatological or ensemble) standard deviation. Values for the **normalize** option are described below:
+
+• **NONE** (default) to skip the normalization step and process the raw ensemble member data.
+
+• **CLIMO_ANOM** to subtract the climatological mean field.
+
+• **CLIMO_STD_ANOM** to subtract the climatological mean field and divide by the climatological standard deviation.
+
+• **FCST_ANOM** to subtract the current ensemble mean field.
+
+• **FCST_STD_ANOM** to subtract the current ensemble mean field and divide by the current ensemble standard deviation.
+
+Note that the **normalize** option may be specified separately for each entry in the **ens.field** array.
+
 _______________________
 
 .. code-block:: none

diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h
@@ -752,6 +752,12 @@ static const char conf_key_dist_parm[]        = "dist_parm";
 static const char conf_key_inst_bias_scale[]  = "inst_bias_scale";
 static const char conf_key_inst_bias_offset[] = "inst_bias_offset";
 
+//
+// Gen-Ens-Prod specific parameter key names
+//
+
+static const char conf_key_normalize[]   = "normalize";
+
 // Distribution options
 static const char conf_val_normal[]      = "NORMAL";
 static const char conf_val_exponential[] = "EXPONENTIAL";

diff --git a/met/src/basic/vx_config/config_util.cc b/met/src/basic/vx_config/config_util.cc
@@ -2981,3 +2981,33 @@ StringArray parse_conf_ens_member_ids(Dictionary *dict) {
 }
 
 ///////////////////////////////////////////////////////////////////////////////
+
+NormalizeType parse_conf_normalize(Dictionary *dict) {
+   NormalizeType t = NormalizeType_None;
+   int v;
+
+   if(!dict) {
+      mlog << Error << "\nparse_conf_normalize() -> "
+           << "empty dictionary!\n\n";
+      exit(1);
+   }
+
+   // Get the integer flag value for the current entry
+   v = dict->lookup_int(conf_key_normalize);
+
+   // Convert integer to enumerated NormalizeType
+        if(v == conf_const.lookup_int(normalizetype_none_str))           t = NormalizeType_None;
+   else if(v == conf_const.lookup_int(normalizetype_climo_anom_str))     t = NormalizeType_ClimoAnom;
+   else if(v == conf_const.lookup_int(normalizetype_climo_std_anom_str)) t = NormalizeType_ClimoStdAnom;
+   else if(v == conf_const.lookup_int(normalizetype_fcst_anom_str))      t = NormalizeType_FcstAnom;
+   else if(v == conf_const.lookup_int(normalizetype_fcst_std_anom_str))  t = NormalizeType_FcstStdAnom;
+   else {
+      mlog << Error << "\nparse_conf_normalize() -> "
+           << "Unexpected value of " << v << ".\n\n";
+      exit(1);
+   }
+
+   return(t);
+}
+
+///////////////////////////////////////////////////////////////////////////////
diff --git a/met/src/basic/vx_config/config_util.h b/met/src/basic/vx_config/config_util.h
@@ -22,6 +22,7 @@
 #include "config_file.h"
 #include "data_file_type.h"
 #include "config_gaussian.h"
+#include "normalize.h"
 
 ////////////////////////////////////////////////////////////////////////
 
@@ -78,6 +79,7 @@ extern map<ConcatString,ThreshArray>
 extern void            parse_conf_range_int(Dictionary *dict, int &beg, int &end);
 extern void            parse_conf_range_double(Dictionary *dict, double &beg, double &end);
 extern StringArray     parse_conf_ens_member_ids(Dictionary *dict);
+extern NormalizeType   parse_conf_normalize(Dictionary *dict);
 
 extern void         check_mask_names(const StringArray &);
 

diff --git a/met/src/basic/vx_util/Makefile.am b/met/src/basic/vx_util/Makefile.am
@@ -37,6 +37,7 @@ libvx_util_a_SOURCES = ascii_table.cc ascii_table.h \
                thresh_array.cc thresh_array.h \
                make_path.cc make_path.h \
                memory.cc memory.h \
+               normalize.cc normalize.h \
                ordinal.cc ordinal.h \
                roman_numeral.cc roman_numeral.h \
                string_fxns.cc string_fxns.h \

diff --git a/met/src/basic/vx_util/data_plane.cc b/met/src/basic/vx_util/data_plane.cc
@@ -406,23 +406,71 @@ void DataPlane::censor(const ThreshArray &censor_thresh,
 
 ///////////////////////////////////////////////////////////////////////////////
 
+void DataPlane::anomaly(const DataPlane &mn) {
+
+   // Check dimensions
+   if(Nxy != mn.Nxy) {
+      mlog << Error << "\nDataPlane::anomaly() -> "
+           << "the data dimensions do not match: ("
+           << Nx << ", " << Ny << ") != ("
+           << mn.Nx << ", " << mn.Ny << ")!\n\n";
+      exit(1);
+   }
 
-void DataPlane::replace_bad_data(const double value)
+   // Subtract the mean
+   for(int i=0; i<Nxy; i++) {
+      if(is_bad_data(Data[i]) || is_bad_data(mn.Data[i])) {
+         Data[i] = bad_data_double;
+      }
+      else {
+         Data[i] -= mn.Data[i];
+      }
+   }
 
-{
+   return;
+}
 
-int j;
+///////////////////////////////////////////////////////////////////////////////
 
-for (j=0; j<Nxy; ++j)  {
+void DataPlane::standard_anomaly(const DataPlane &mn,
+                                 const DataPlane &sd) {
+
+   // Check dimensions
+   if(Nxy != mn.Nxy || Nxy != sd.Nxy) {
+      mlog << Error << "\nDataPlane::standard_anomaly() -> "
+           << "the data dimensions do not match: ("
+           << Nx << ", " << Ny << ") != ("
+           << mn.Nx << ", " << mn.Ny << ") != ("
+           << sd.Nx << ", " << sd.Ny << ")!\n\n";
+      exit(1);
+   }
 
-   if ( is_bad_data(Data[j]) )  Data[j] = value;
+   // Subtract the mean and divide by the standard deviation
+   for(int i=0; i<Nxy; i++) {
+      if(is_bad_data(Data[i])    ||
+         is_bad_data(mn.Data[i]) ||
+         is_bad_data(sd.Data[i]) ||
+         is_eq(sd.Data[i], 0.0)) {
+         Data[i] = bad_data_double;
+      }
+      else {
+         Data[i] = (Data[i] - mn.Data[i])/sd.Data[i];
+      }
+   }
 
+   return;
 }
 
-return;
+///////////////////////////////////////////////////////////////////////////////
 
-}
+void DataPlane::replace_bad_data(const double value) {
 
+   for(int i=0; i<Nxy; i++) {
+      if(is_bad_data(Data[i])) Data[i] = value;
+   }
+
+   return;
+}
 
 ///////////////////////////////////////////////////////////////////////////////
 

diff --git a/met/src/basic/vx_util/data_plane.h b/met/src/basic/vx_util/data_plane.h
@@ -43,7 +43,6 @@ class DataPlane {
 
       int Nx;
       int Ny;
-
       int Nxy;   //  Nx*Ny
 
       unixtime InitTime;      // Initialization time in unixtime
@@ -86,6 +85,8 @@ class DataPlane {
 
       int      nx() const;
       int      ny() const;
+      int      nxy() const;
+
       bool     is_empty() const;
       bool     is_all_bad_data() const;
       int      n_good_data() const;
@@ -108,6 +109,9 @@ class DataPlane {
       void convert  (const UserFunc_1Arg &);
       void censor   (const ThreshArray &, const NumArray &);
 
+      void anomaly         (const DataPlane &);
+      void standard_anomaly(const DataPlane &, const DataPlane &);
+
       void replace_bad_data(const double value);
 
       int  two_to_one(int x, int y, bool to_north=true) const;
@@ -131,8 +135,9 @@ class DataPlane {
 
 ////////////////////////////////////////////////////////////////////////
 
-inline int DataPlane::nx() const { return (Nx); }
-inline int DataPlane::ny() const { return (Ny); }
+inline int DataPlane::nx()  const { return (Nx);  }
+inline int DataPlane::ny()  const { return (Ny);  }
+inline int DataPlane::nxy() const { return (Nxy); }
 
 inline bool DataPlane::is_empty() const { return (Nxy == 0); }