Feature #2651 SeriesAnalysis -aggr argument and new use case (#2701)

* exclude build directory from PyCharm project

* refactor to satisfy SonarQube complaints

* Per #2651, add support for setting -aggr argument with file path. Refactor functions to resolve SonarQube complaints. Enhance unit tests to test multiple init times and -aggr argument

* add new config variables to basic use case

* change settings to match MET unit test

* added new config variables to set -aggr argument in SeriesAnalysis wrapper to documentation

* Per #2651, add new use case to demonstrate using the -aggr argument to SeriesAnalysis

* turn on new use case to test it in GHA

* remove blank line

* turn off use case to prepare for PR

.github/parm/use_case_groups.json

@@ -9,6 +9,11 @@
     "index_list": "30-58",
     "run": false
   },
+  {
+    "category": "met_tool_wrapper",
+    "index_list": "65",
+    "run": false
+  },
   {
     "category": "air_quality_and_comp",
     "index_list": "0",

docs/Users_Guide/glossary.rst

@@ -12978,3 +12978,16 @@ METplus Configuration Glossary
      See: :term:`<TOOL-NAME>_CLIMO_STDEV_VAR<n>_OPTIONS`
 
      | *Used by:* SeriesAnalysis
+
+   SERIES_ANALYSIS_AGGR_INPUT_TEMPLATE
+     Template used to specify the file path to pass to SeriesAnalysis using the
+     -aggr command line argument. This file is the output NetCDF file from a
+     previous SeriesAnalysis run.
+
+     | *Used by:* SeriesAnalysis
+
+   SERIES_ANALYSIS_AGGR_INPUT_DIR
+     Directory containing SeriesAnalysis output to be read by SeriesAnalysis
+     using the -aggr command line argument.
+
+     | *Used by:* SeriesAnalysis

docs/Users_Guide/wrappers.rst

@@ -7941,6 +7941,7 @@ METplus Configuration
 | :term:`OBS_SERIES_ANALYSIS_INPUT_DIR`
 | :term:`BOTH_SERIES_ANALYSIS_INPUT_DIR`
 | :term:`SERIES_ANALYSIS_TC_STAT_INPUT_DIR`
+| :term:`SERIES_ANALYSIS_AGGR_INPUT_DIR`
 | :term:`SERIES_ANALYSIS_OUTPUT_DIR`
 | :term:`FCST_SERIES_ANALYSIS_INPUT_TEMPLATE`
 | :term:`OBS_SERIES_ANALYSIS_INPUT_TEMPLATE`
@@ -7949,6 +7950,7 @@ METplus Configuration
 | :term:`OBS_SERIES_ANALYSIS_INPUT_FILE_LIST`
 | :term:`BOTH_SERIES_ANALYSIS_INPUT_FILE_LIST`
 | :term:`SERIES_ANALYSIS_TC_STAT_INPUT_TEMPLATE`
+| :term:`SERIES_ANALYSIS_AGGR_INPUT_TEMPLATE`
 | :term:`SERIES_ANALYSIS_OUTPUT_TEMPLATE`
 | :term:`SERIES_ANALYSIS_CLIMO_MEAN_FILE_NAME`
 | :term:`SERIES_ANALYSIS_CLIMO_MEAN_VAR<n>_NAME`

docs/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis_aggr.py

@@ -0,0 +1,115 @@
+"""
+SeriesAnalysis: Aggregate Output Use Case
+=========================================
+
+met_tool_wrapper/SeriesAnalysis/SeriesAnalysis_aggr.conf
+
+"""
+##############################################################################
+# Scientific Objective
+# --------------------
+#
+# Read in output from a previous SeriesAnalysis run into SeriesAnalysis to
+# aggregate the results.
+
+##############################################################################
+# Datasets
+# --------
+#
+# | **Forecast:** GFS 6 hour precipitation accumulation
+# | **Observation:** STAGE4 6 hour precipitation accumulation
+#
+# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases
+# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information.
+# |
+
+##############################################################################
+# METplus Components
+# ------------------
+#
+# This use case utilizes the METplus SeriesAnalysis wrapper to search for
+# files that are valid at a given run time and generates a command to run
+# the MET tool series_analysis if all required files are found.
+
+##############################################################################
+# METplus Workflow
+# ----------------
+#
+# SeriesAnalysis is the only tool called in this example. It processes the following
+# run times:
+#
+# | **Init:** 2012-04-09_0Z
+# | **Forecast lead:** 30, 36, and 42 hour
+# |
+
+##############################################################################
+# METplus Configuration
+# ---------------------
+#
+# METplus first loads all of the configuration files found in parm/metplus_config,
+# then it loads any configuration files passed to METplus via the command line,
+# e.g. parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis_aggr.conf
+#
+# .. highlight:: bash
+# .. literalinclude:: ../../../../parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis_aggr.conf
+
+##############################################################################
+# MET Configuration
+# ---------------------
+#
+# METplus sets environment variables based on user settings in the METplus configuration file. 
+# See :ref:`How METplus controls MET config file settings<metplus-control-met>` for more details. 
+#
+# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!**
+#
+# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to:
+# :ref:`Overriding Unsupported MET config file settings<met-config-overrides>`
+#
+# .. note:: See the :ref:`SeriesAnalysis MET Configuration<series-analysis-met-conf>` section of the User's Guide for more information on the environment variables used in the file below:
+#
+# .. highlight:: bash
+# .. literalinclude:: ../../../../parm/met_config/SeriesAnalysisConfig_wrapped
+
+##############################################################################
+# Running METplus
+# ---------------
+#
+# Pass the use case configuration file to the run_metplus.py script
+# along with any user-specific system configuration files if desired::
+#
+#    run_metplus.py /path/to/METplus/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis_aggr.conf /path/to/user_system.conf
+#
+# See :ref:`running-metplus` for more information.
+
+
+##############################################################################
+# Expected Output
+# ---------------
+#
+# A successful run will output the following both to the screen and to the logfile::
+#
+#   INFO: METplus has successfully finished running.
+#
+# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated.
+# Output for this use case will be found in series_analysis (relative to **OUTPUT_BASE**)
+# and will contain the following file:
+#
+# * series_analysis_AGGR_CMD_LINE_APCP_06_2012040900_to_2012041000.nc
+
+##############################################################################
+# Keywords
+# --------
+#
+# .. note::
+#
+#   * SeriesAnalysisUseCase
+#   * DiagnosticsUseCase
+#   * RuntimeFreqUseCase
+#   * GRIBFileUseCase
+#
+#   Navigate to the :ref:`quick-search` page to discover other similar use cases.
+#
+#
+#
+# sphinx_gallery_thumbnail_path = '_static/met_tool_wrapper-SeriesAnalysis.png'
+#

internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py

@@ -18,11 +18,14 @@
 obs_fmt = (f'field = [{{ name="{obs_name}"; '
            f'level="{obs_level_no_quotes}"; }}];')
 time_fmt = '%Y%m%d%H'
-#run_times = ['2005080700', '2005080712']
-run_times = ['2005080700',]
+run_times = ['2005080700', '2005080712']
 stat_list = 'TOTAL,RMSE,FBAR,OBAR'
 stat_list_quotes = '", "'.join(stat_list.split(','))
 stat_list_fmt = f'output_stats = {{cnt = ["{stat_list_quotes}"];}}'
+aggr_dir = '/some/fake/path/for'
+aggr_rel = 'aggr_file_<INIT_TIME>.nc'
+aggr_template = 'aggr_file_{init?fmt=%Y%m%d%H}.nc'
+both_file_list = '/some/fake/path/for/both/file_list.txt'
 
 
 def get_input_dirs(config):
@@ -613,6 +616,10 @@ def test_series_analysis_missing_inputs(metplus_config, get_test_data_dir,
                                            'time_interp_method = NEAREST;'
                                            'match_month = TRUE;day_interval = 30;'
                                            'hour_interval = 12;}')}),
+        ({'SERIES_ANALYSIS_AGGR_INPUT_TEMPLATE': os.path.join(aggr_dir, aggr_template), },
+         {}),
+        ({'SERIES_ANALYSIS_AGGR_INPUT_DIR': aggr_dir, 'SERIES_ANALYSIS_AGGR_INPUT_TEMPLATE': aggr_template,},
+         {}),
     ]
 )
 @pytest.mark.wrapper_a
@@ -638,31 +645,34 @@ def test_series_analysis_single_field(metplus_config, config_overrides,
     config_file = wrapper.c_dict.get('CONFIG_FILE')
     out_dir = wrapper.c_dict.get('OUTPUT_DIR')
     prefix = 'series_analysis_files_'
-    suffix = '_init_20050807000000_valid_ALL_lead_ALL.txt'
+    suffix = '_init_<INIT_TIME>0000_valid_ALL_lead_ALL.txt'
     fcst_file = f'{prefix}fcst{suffix}'
     obs_file = f'{prefix}obs{suffix}'
-
+
+    extra_args = ' '
+    if 'SERIES_ANALYSIS_AGGR_INPUT_TEMPLATE' in config_overrides:
+        extra_args += f'-aggr {os.path.join(aggr_dir, aggr_rel)} '
+
     if is_both:
-        expected_cmds = [(f"{app_path} "
-                      f"-both {out_dir}/{fcst_file} "
-                      f"-out {out_dir}/2005080700 "
-                      f"-config {config_file} {verbosity}"),
-                     ]
+        file_args = f"-both {out_dir}/{fcst_file}"
     else:
-        expected_cmds = [(f"{app_path} "
-                      f"-fcst {out_dir}/{fcst_file} "
-                      f"-obs {out_dir}/{obs_file} "
-                      f"-out {out_dir}/2005080700 "
-                      f"-config {config_file} {verbosity}"),
-                     ]
+        file_args = f"-fcst {out_dir}/{fcst_file} -obs {out_dir}/{obs_file}"
 
-    all_cmds = wrapper.run_all_times()
+    expected_cmds = []
+    for run_time in run_times:
+        cmd = (f"{app_path} {file_args} -out {out_dir}/<INIT_TIME>{extra_args}"
+               f"-config {config_file} {verbosity}")
+        expected_cmds.append(cmd.replace('<INIT_TIME>', run_time))
 
+    all_cmds = wrapper.run_all_times()
     expected_len = len(expected_cmds)
+    compare_cmds = all_cmds
     if 'SERIES_ANALYSIS_GENERATE_PLOTS' in config_overrides:
-        expected_len += 8
+        expected_len += 8 * len(expected_cmds)
+        compare_cmds = all_cmds[0::9][0:len(expected_cmds)]
         if 'SERIES_ANALYSIS_GENERATE_ANIMATIONS' in config_overrides:
             expected_len += 4
+
     assert len(all_cmds) == expected_len
 
     special_values = {
@@ -672,7 +682,7 @@ def test_series_analysis_single_field(metplus_config, config_overrides,
     if 'METPLUS_OUTPUT_STATS_DICT' not in env_var_values:
         special_values['METPLUS_OUTPUT_STATS_DICT'] = stat_list_fmt
     # only compare first command since the rest are not series_analysis
-    compare_command_and_env_vars(all_cmds[0:1], expected_cmds, env_var_values,
+    compare_command_and_env_vars(compare_cmds, expected_cmds, env_var_values,
                                  wrapper, special_values)
 
 
@@ -1204,9 +1214,9 @@ def test_get_netcdf_min_max(tmp_path_factory,
 
     wrapper = series_analysis_wrapper(metplus_config)
 
-    min, max = wrapper._get_netcdf_min_max(filepath, variable_name)
-    assert min == expected_min
-    assert max == expected_max
+    min_val, max_val = wrapper._get_netcdf_min_max(filepath, variable_name)
+    assert min_val == expected_min
+    assert max_val == expected_max
 
 
 @pytest.mark.wrapper_a
@@ -1237,7 +1247,7 @@ def test_run_once_per_lead(metplus_config):
     assert wrapper.isOK
     assert actual is True
 
-    # lead_hours = None
+    # lead_hours None
     with mock.patch.object(saw, 'ti_get_hours_from_lead', return_value=None):
         actual = wrapper.run_once_per_lead(None)
     assert actual is True

internal/tests/use_cases/all_use_cases.txt

@@ -64,6 +64,7 @@ Category: met_tool_wrapper
 62::TCDiag:: met_tool_wrapper/TCDiag/TCDiag.conf
 63::WaveletStat:: met_tool_wrapper/WaveletStat/WaveletStat.conf
 64::MADIS2NC:: met_tool_wrapper/MADIS2NC/MADIS2NC.conf
+65::SeriesAnalysis_aggr:: met_tool_wrapper/SeriesAnalysis/SeriesAnalysis_aggr.conf::netcdf4_env
 
 Category: air_quality_and_comp
 0::EnsembleStat_fcstICAP_obsMODIS_aod::model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf