Feature 1695 ensemble single file (gen_ens_prod) (#2001)

Co-authored-by: johnhg <[email protected]>
dtcenter · Jan 12, 2022 · dbc7d10 · dbc7d10
1 parent 551f92b
commit dbc7d10
Show file tree

Hide file tree

Showing 20 changed files with 819 additions and 141 deletions.
diff --git a/met/data/config/GenEnsProdConfig_default b/met/data/config/GenEnsProdConfig_default
@@ -57,6 +57,13 @@ ens = {
    ];
 }
 
+//
+// IDs for ensemble members
+// Only set if processing a single file with all ensembles
+//
+ens_member_ids = [];
+control_id = "";
+
 ////////////////////////////////////////////////////////////////////////////////
 
 //

diff --git a/met/docs/Users_Guide/gen-ens-prod.rst b/met/docs/Users_Guide/gen-ens-prod.rst
@@ -67,7 +67,7 @@ Required arguments gen_ens_prod
 Optional arguments for gen_ens_prod
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-4. The **-ctrl file** option specifies the input file for the ensemble control member. Data for this member is included in the computation of the ensemble mean, but excluded from the spread. The control file should not appear in the **-ens** list of ensemble member files.
+4. The **-ctrl file** option specifies the input file for the ensemble control member. Data for this member is included in the computation of the ensemble mean, but excluded from the spread. The control file should not appear in the **-ens** list of ensemble member files (unless processing a single file that contains all ensemble members).
 
 5. The **-log** file outputs log messages to the specified file.
 
@@ -134,6 +134,50 @@ For each dictionary entry in the **field** array, give the name and vertical or
 
 _______________________
 
+.. code-block:: none
+
+  ens_member_ids = [];
+  control_id = "";
+
+The **ens_member_ids** array is only used if reading a single file that contains all ensemble members.
+It should contain a list of string identifiers that are substituted into the **ens** dictionary fields
+to determine which data to read from the file.
+The length of the array determines how many ensemble members will be processed for a given field.
+Each value in the array will replace the text **MET_ENS_MEMBER_ID**.
+
+**NetCDF Example:**
+
+.. code-block:: none
+
+  ens = {
+    field = [
+      {
+        name  = "fcst";
+        level = "(MET_ENS_MEMBER_ID,0,*,*)";
+      }
+    ];
+  }
+
+**GRIB Example:**
+
+.. code-block:: none
+
+  ens = {
+    field = [
+      {
+        name     = "fcst";
+        level    = "L0";
+        GRIB_ens = "+MET_ENS_MEMBER_ID";
+      }
+    ];
+  }
+
+**control_id** is a string that is substituted in the same way as the **ens_member_ids** values
+to read a control member. This value is only used when the **-ctrl** command line argument is
+used. The value should not be found in the **ens_member_ids** array.
+
+_______________________
+
 .. code-block:: none
 
   nbrhd_prob = {

diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h
@@ -632,6 +632,8 @@ static const char conf_key_eclv_points[]       = "eclv_points";
 static const char conf_key_var_name_map[]      = "var_name_map";
 static const char conf_key_metadata_map[]      = "metadata_map";
 static const char conf_key_missing_thresh[]    = "missing_thresh";
+static const char conf_key_control_id[]        = "control_id";
+static const char conf_key_ens_member_ids[]    = "ens_member_ids";
 
 //
 // Entries to override file metadata 

diff --git a/met/src/basic/vx_config/config_file.cc b/met/src/basic/vx_config/config_file.cc
@@ -252,7 +252,7 @@ ConcatString MetConfig::get_tmp_dir()
    // Use the MET_TMP_DIR environment variable, if set.
    if(!get_env("MET_TMP_DIR", tmp_dir)) {
       const DictionaryEntry * _e = lookup(conf_key_tmp_dir);
-      if ( LastLookupStatus ) tmp_dir = *(_e->string_value());
+      if ( LastLookupStatus ) tmp_dir = _e->string_value();
       else                    tmp_dir = default_tmp_dir;
    }
 

diff --git a/met/src/basic/vx_config/config_util.cc b/met/src/basic/vx_config/config_util.cc
@@ -2965,3 +2965,19 @@ ConcatString wavelettype_to_string(WaveletType type) {
 }
 
 ///////////////////////////////////////////////////////////////////////////////
+
+StringArray parse_conf_ens_member_ids(Dictionary *dict) {
+   const char *method_name = "parse_conf_ens_member_ids() -> ";
+
+   StringArray sa = parse_conf_string_array(dict, conf_key_ens_member_ids, method_name);
+
+   if(sa.n() > 0) {
+      mlog << Debug(4) << method_name
+           << "Ensemble Member IDs \"" << conf_key_ens_member_ids << "\" list contains "
+           << sa.n() << " entries.\n";
+   }
+
+   return(sa);
+}
+
+///////////////////////////////////////////////////////////////////////////////
diff --git a/met/src/basic/vx_config/config_util.h b/met/src/basic/vx_config/config_util.h
@@ -73,6 +73,7 @@ extern map<ConcatString,ThreshArray>
                        parse_conf_filter_attr_map(Dictionary *dict);
 extern void            parse_conf_range_int(Dictionary *dict, int &beg, int &end);
 extern void            parse_conf_range_double(Dictionary *dict, double &beg, double &end);
+extern StringArray     parse_conf_ens_member_ids(Dictionary *dict);
 
 extern void         check_mask_names(const StringArray &);
 

diff --git a/met/src/basic/vx_config/dictionary.cc b/met/src/basic/vx_config/dictionary.cc
@@ -762,21 +762,30 @@ return ( Bval );
 ////////////////////////////////////////////////////////////////////////
 
 
-const ConcatString * DictionaryEntry::string_value() const
+const ConcatString DictionaryEntry::string_value() const
 
 {
 
-if ( Type != StringType )  {
+   if ( Type != StringType )  {
 
-   mlog << Error
-        << "\nDictionaryEntry::string_value() -> bad type\n\n";
+      mlog << Error
+           << "\nDictionaryEntry::string_value() -> bad type\n\n";
 
-   exit ( 1 );
+      exit ( 1 );
 
-}
+   }
+
+   ConcatString sub_text = ConcatString(*Text);
+   ConcatString cur_env_val;
+   if ( get_env(met_ens_member_id, cur_env_val) )  {
 
+      if(!cur_env_val.empty()) {
+         sub_text.replace(met_ens_member_id, cur_env_val.c_str(), false);
+      }
+
+   }
 
-return ( Text );
+   return ( sub_text );
 
 }
 
@@ -1816,7 +1825,7 @@ if ( !Entry || !is_correct_type )  {
 
 }
 
-return ( *(Entry->string_value()) );
+return ( Entry->string_value() );
 
 }
 
@@ -1874,7 +1883,7 @@ if ( !Entry || !is_correct_type )  {
 
 if ( Entry->type() == StringType )  {
 
-   array.add( *(Entry->string_value()) );
+   array.add( Entry->string_value() );
 
    return ( array );
 
@@ -1911,7 +1920,7 @@ if ( Dict->n_entries() > 0 )  {
 
 for (int i=0; i<Dict->n_entries(); i++)  {
 
-   array.add(*(*Dict)[i]->string_value());
+   array.add((*Dict)[i]->string_value());
 
 }
 

diff --git a/met/src/basic/vx_config/dictionary.h b/met/src/basic/vx_config/dictionary.h
@@ -40,6 +40,9 @@ class Dictionary;        //  forward reference
 
 ////////////////////////////////////////////////////////////////////////
 
+static const char met_ens_member_id [] = "MET_ENS_MEMBER_ID";
+
+////////////////////////////////////////////////////////////////////////
 
 class DictionaryEntry {
 
@@ -122,7 +125,7 @@ class DictionaryEntry {
 
       int     n_args () const;
 
-      const ConcatString * string_value () const;
+      const ConcatString string_value () const;
 
       Dictionary * dict_value () const;
 

diff --git a/met/src/basic/vx_log/file_fxns.h b/met/src/basic/vx_log/file_fxns.h
@@ -22,7 +22,6 @@
 #include <fcntl.h>
 
 #include "concat_string.h"
-#include "file_fxns.h"
 
 ////////////////////////////////////////////////////////////////////////
 

diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h
@@ -95,6 +95,7 @@ class EnsembleStatVxOpt {
       VxPairDataEnsemble vx_pd;          // Ensemble pair data
 
       ConcatString   var_str;            // nc_pairs_var_str string
+      ConcatString   control_id;         // Control ID
 
       int            beg_ds;             // Begin observation time window offset
       int            end_ds;             // End observation time window offset