feature 1695 ensemble single file (ensemble_stat)

met/data/config/EnsembleStatConfig_default

@@ -62,6 +62,13 @@ ens = {
    ];
 }
 
+//
+// IDs for ensemble members
+// Only set if processing a single file with all ensembles
+//
+ens_member_ids = [];
+control_id = "";
+
 ////////////////////////////////////////////////////////////////////////////////
 
 //

met/docs/Users_Guide/ensemble-stat.rst

@@ -97,7 +97,7 @@ Optional arguments for ensemble_stat
 
 6. To override the simple ensemble mean value of the input ensemble members for the ECNT, SSVAR, and ORANK line types, the **-ens_mean file** option specifies an ensemble mean model data file. This option replaces the **-ssvar_mean file** option from earlier versions of MET.
 
-7. The **-ctrl file** option specifies an ensemble control member data file. The control member is included in the computation of the ensemble mean but excluded from the spread. The control file should not appear in the list of ensemble member files.
+7. The **-ctrl file** option specifies an ensemble control member data file. The control member is included in the computation of the ensemble mean but excluded from the spread. The control file should not appear in the list of ensemble member files (unless processing a single file that contains all ensemble members).
 
 8. To filter point observations by time, use **-obs_valid_beg time** in YYYYMMDD[_HH[MMSS]] format to set the beginning of the matching observation time window.
 
@@ -188,6 +188,50 @@ For each **field** listed in the forecast field, give the name and vertical or a
 
 _______________________
 
+.. code-block:: none
+
+  ens_member_ids = [];
+  control_id = "";
+
+The **ens_member_ids** array is only used if reading a single file that contains all ensemble members.
+It should contain a list of string identifiers that are substituted into the **ens** and/or **fcst** dictionary fields
+to determine which data to read from the file.
+The length of the array determines how many ensemble members will be processed for a given field.
+Each value in the array will replace the text **MET_ENS_MEMBER_ID**.
+
+**NetCDF Example:**
+
+.. code-block:: none
+
+  ens = {
+    field = [
+      {
+        name  = "fcst";
+        level = "(MET_ENS_MEMBER_ID,0,*,*)";
+      }
+    ];
+  }
+
+**GRIB Example:**
+
+.. code-block:: none
+
+  ens = {
+    field = [
+      {
+        name     = "fcst";
+        level    = "L0";
+        GRIB_ens = "MET_ENS_MEMBER_ID";
+      }
+    ];
+  }
+
+**control_id** is a string that is substituted in the same way as the **ens_member_ids** values
+to read a control member. This value is only used when the **-ctrl** command line argument is
+used. The value should not be found in the **ens_member_ids** array.
+
+_______________________
+
 .. code-block:: none
 
   nbrhd_prob = {

met/docs/Users_Guide/gen-ens-prod.rst

@@ -167,7 +167,7 @@ Each value in the array will replace the text **MET_ENS_MEMBER_ID**.
       {
         name     = "fcst";
         level    = "L0";
-        GRIB_ens = "+MET_ENS_MEMBER_ID";
+        GRIB_ens = "MET_ENS_MEMBER_ID";
       }
     ];
   }

met/src/libcode/vx_data2d/var_info.cc

@@ -790,3 +790,126 @@ int parse_set_attr_flag(Dictionary &dict, const char *key) {
 }
 
 ///////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////
+//
+//  Code for class EnsVarInfo
+//
+////////////////////////////////////////////////////////////////////////
+
+EnsVarInfo::EnsVarInfo() {
+   ctrl_info = NULL;
+}
+
+////////////////////////////////////////////////////////////////////////
+
+EnsVarInfo::~EnsVarInfo() {
+   clear();
+}
+///////////////////////////////////////////////////////////////////////////////
+
+EnsVarInfo::EnsVarInfo(const EnsVarInfo &f) {
+
+   clear();
+
+   assign(f);
+}
+
+////////////////////////////////////////////////////////////////////////
+
+void EnsVarInfo::clear() {
+   vector<InputInfo>::const_iterator it;
+   for(it = inputs.begin(); it != inputs.end(); it++) {
+      if((*it).var_info) { delete (*it).var_info; }
+   }
+
+   if(ctrl_info) { delete ctrl_info; }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+void EnsVarInfo::assign(const EnsVarInfo &v) {
+
+   // Copy
+   inputs        = v.inputs;
+   ctrl_info     = v.ctrl_info;
+
+   nc_var_str    = v.nc_var_str;
+   cat_ta        = v.cat_ta;
+   raw_magic_str = v.raw_magic_str;
+
+   return;
+}
+
+////////////////////////////////////////////////////////////////////////
+
+void EnsVarInfo::add_input(InputInfo input) {
+   inputs.push_back(input);
+}
+
+////////////////////////////////////////////////////////////////////////
+
+int EnsVarInfo::inputs_n() {
+   return inputs.size();
+}
+
+////////////////////////////////////////////////////////////////////////
+
+void EnsVarInfo::set_ctrl(VarInfo * ctrl) {
+   ctrl_info = ctrl;
+}
+
+////////////////////////////////////////////////////////////////////////
+
+VarInfo * EnsVarInfo::get_ctrl(int index) {
+   if(ctrl_info) {
+      return ctrl_info;
+   }
+   return inputs[index].var_info;
+}
+
+////////////////////////////////////////////////////////////////////////
+
+VarInfo * EnsVarInfo::get_var_info(int index) {
+   if(inputs[index].var_info) {
+      return inputs[index].var_info;
+   }
+   return inputs[0].var_info;
+}
+
+////////////////////////////////////////////////////////////////////////
+
+ConcatString EnsVarInfo::get_file(int index) {
+   int file_index = inputs[index].file_index;
+   return (*inputs[index].file_list)[file_index];
+}
+
+////////////////////////////////////////////////////////////////////////
+
+int EnsVarInfo::get_file_index(int index) {
+   return inputs[index].file_index;
+}
+
+////////////////////////////////////////////////////////////////////////
+
+ConcatString raw_magic_str(Dictionary i_edict, GrdFileType file_type) {
+   ConcatString magic_str;
+
+   ConcatString name = i_edict.lookup_string("name");
+   ConcatString level = i_edict.lookup_string("level", false);
+
+   if(level.empty()) {
+      return name;
+   }
+
+   if(level[0] != '(') {
+      magic_str << name << "/" << level;
+   } else {
+      magic_str << name << level;
+   }
+
+   return magic_str;
+
+}
+
+////////////////////////////////////////////////////////////////////////

met/src/libcode/vx_data2d/var_info.h

@@ -251,6 +251,54 @@ inline unixtime     VarInfo::valid_attr()     const { return(SetAttrValid);    }
 inline int          VarInfo::lead_attr()      const { return(SetAttrLead);     }
 inline int          VarInfo::accum_attr()     const { return(SetAttrAccum);    }
 
+////////////////////////////////////////////////////////////////////////
+
+//
+// Struct to store input file and field information
+//
+
+struct InputInfo {
+    VarInfo * var_info;         // Variable information to read
+    int file_index;             // Index in file_list of file to read
+    StringArray * file_list;    // Array of files (unallocated)
+};
+
+////////////////////////////////////////////////////////////////////////
+
+//
+// Struct to store ensemble information
+//
+class EnsVarInfo {
+
+private:
+    vector<InputInfo> inputs;     // Vector of InputInfo
+    VarInfo * ctrl_info;          // Field info for control member
+public:
+    EnsVarInfo();
+    ~EnsVarInfo();
+    EnsVarInfo(const EnsVarInfo &);
+
+    void clear();
+    void assign(const EnsVarInfo &);
+
+    void add_input(InputInfo);
+    int inputs_n();
+
+    void set_ctrl(VarInfo *);
+    VarInfo * get_ctrl(int);
+
+    // Get VarInfo from first InputInfo if requested VarInfo is NULL
+    VarInfo * get_var_info(int index=0);
+    ConcatString get_file(int index=0);
+    int get_file_index(int index=0);
+
+    ConcatString nc_var_str;      // Ensemble variable name strings
+    ThreshArray cat_ta;           // Ensemble categorical thresholds
+    ConcatString raw_magic_str;   // Magic string w/o var substitution
+};
+
+ConcatString raw_magic_str(Dictionary i_edict, GrdFileType file_type);
+
 ///////////////////////////////////////////////////////////////////////////////
 
 #endif  // __VAR_INFO_H__

met/src/libcode/vx_statistics/pair_data_ensemble.cc

@@ -891,7 +891,7 @@ VxPairDataEnsemble & VxPairDataEnsemble::operator=(const VxPairDataEnsemble &vx_
 
 void VxPairDataEnsemble::init_from_scratch() {
 
-   fcst_info    = (VarInfo *) 0;
+   fcst_info    = (EnsVarInfo *) 0;
    climo_info   = (VarInfo *) 0;
    obs_info     = (VarInfo *) 0;
    pd           = (PairDataEnsemble ***) 0;
@@ -910,7 +910,7 @@ void VxPairDataEnsemble::init_from_scratch() {
 void VxPairDataEnsemble::clear() {
    int i, j, k;
 
-   if(fcst_info)  { delete fcst_info;  fcst_info  = (VarInfo *) 0; }
+   if(fcst_info)  { delete fcst_info;  fcst_info  = (EnsVarInfo *) 0; }
    if(climo_info) { delete climo_info; climo_info = (VarInfo *) 0; }
    if(obs_info)   { delete obs_info;   obs_info   = (VarInfo *) 0; }
 
@@ -993,15 +993,14 @@ void VxPairDataEnsemble::assign(const VxPairDataEnsemble &vx_pd) {
 
 ////////////////////////////////////////////////////////////////////////
 
-void VxPairDataEnsemble::set_fcst_info(VarInfo *info) {
+void VxPairDataEnsemble::set_fcst_info(EnsVarInfo *info) {
    VarInfoFactory f;
 
    // Deallocate, if necessary
-   if(fcst_info) { delete fcst_info; fcst_info = (VarInfo *) 0; }
+   if(fcst_info) { delete fcst_info; fcst_info = (EnsVarInfo *) 0; }
 
    // Perform a deep copy
-   fcst_info = f.new_var_info(info->file_type());
-   *fcst_info = *info;
+   fcst_info = new EnsVarInfo(*info);
 
    return;
 }
@@ -1458,7 +1457,7 @@ void VxPairDataEnsemble::add_point_obs(float *hdr_arr, int *hdr_typ_arr,
    // below the observation point.
    else {
       // Interpolate using the observation pressure level or height
-      to_lvl = (fcst_info->level().type() == LevelType_Pres ?
+      to_lvl = (fcst_info->get_var_info()->level().type() == LevelType_Pres ?
                 obs_lvl : obs_hgt);
       find_vert_lvl(climo_mn_dpa, to_lvl, cmn_lvl_blw, cmn_lvl_abv);
    }
@@ -1472,7 +1471,7 @@ void VxPairDataEnsemble::add_point_obs(float *hdr_arr, int *hdr_typ_arr,
    // levels above and below the observation point.
    else {
       // Interpolate using the observation pressure level or height
-      to_lvl = (fcst_info->level().type() == LevelType_Pres ?
+      to_lvl = (fcst_info->get_var_info()->level().type() == LevelType_Pres ?
                 obs_lvl : obs_hgt);
       find_vert_lvl(climo_sd_dpa, to_lvl, csd_lvl_blw, csd_lvl_abv);
    }
@@ -1486,7 +1485,7 @@ void VxPairDataEnsemble::add_point_obs(float *hdr_arr, int *hdr_typ_arr,
    }
 
    // Set flag for specific humidity
-   bool spfh_flag = fcst_info->is_specific_humidity() &&
+   bool spfh_flag = fcst_info->get_var_info()->is_specific_humidity() &&
                      obs_info->is_specific_humidity();
 
    // Store pointer to ObsErrorEntry
@@ -1561,7 +1560,7 @@ void VxPairDataEnsemble::add_point_obs(float *hdr_arr, int *hdr_typ_arr,
 
             // Compute the interpolated climatology values using the
             // observation pressure level or height
-            to_lvl = (fcst_info->level().type() == LevelType_Pres ?
+            to_lvl = (fcst_info->get_var_info()->level().type() == LevelType_Pres ?
                       obs_lvl : obs_hgt);
 
             // Compute the interpolated climatology mean
@@ -1570,7 +1569,7 @@ void VxPairDataEnsemble::add_point_obs(float *hdr_arr, int *hdr_typ_arr,
                        pd[0][0][k].interp_mthd, pd[0][0][k].interp_wdth,
                        pd[0][0][k].interp_shape, gr.wrap_lon(),
                        interp_thresh, spfh_flag,
-                       fcst_info->level().type(),
+                       fcst_info->get_var_info()->level().type(),
                        to_lvl, cmn_lvl_blw, cmn_lvl_abv);
 
             // Check for bad data
@@ -1597,7 +1596,7 @@ void VxPairDataEnsemble::add_point_obs(float *hdr_arr, int *hdr_typ_arr,
                         pd[0][0][k].interp_mthd, pd[0][0][k].interp_wdth,
                         pd[0][0][k].interp_shape, gr.wrap_lon(),
                         interp_thresh, spfh_flag,
-                        fcst_info->level().type(),
+                        fcst_info->get_var_info()->level().type(),
                         to_lvl, csd_lvl_blw, csd_lvl_abv);
 
             // Check for bad data
@@ -1630,7 +1629,7 @@ void VxPairDataEnsemble::add_ens(int member, bool mn, Grid &gr) {
    double to_lvl, fcst_v;
 
    // Set flag for specific humidity
-   bool spfh_flag = fcst_info->is_specific_humidity() &&
+   bool spfh_flag = fcst_info->get_var_info()->is_specific_humidity() &&
                      obs_info->is_specific_humidity();
 
    // Loop through all the PairDataEnsemble objects and interpolate
@@ -1642,7 +1641,7 @@ void VxPairDataEnsemble::add_ens(int member, bool mn, Grid &gr) {
             for(l=0; l<pd[i][j][k].n_obs; l++) {
 
                // Interpolate using the observation pressure level or height
-               to_lvl = (fcst_info->level().type() == LevelType_Pres ?
+               to_lvl = (fcst_info->get_var_info()->level().type() == LevelType_Pres ?
                          pd[i][j][k].lvl_na[l] : pd[i][j][k].elv_na[l]);
 
                // For a single forecast field
@@ -1668,7 +1667,7 @@ void VxPairDataEnsemble::add_ens(int member, bool mn, Grid &gr) {
                            pd[0][0][k].interp_shape,
                            gr.wrap_lon(),
                            interp_thresh, spfh_flag,
-                           fcst_info->level().type(),
+                           fcst_info->get_var_info()->level().type(),
                            to_lvl, f_lvl_blw, f_lvl_abv);
 
                // Store the ensemble mean