From 05d78b0b1a0c013e1c63d88750ae2b9828e8d93c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 22 Mar 2023 14:52:00 -0600 Subject: [PATCH 01/12] add support for adding (x) after a MET variable name, e.g. convert -> convert(x) --- metplus/util/met_config.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py index 91b4115b3e..172b0113fd 100644 --- a/metplus/util/met_config.py +++ b/metplus/util/met_config.py @@ -521,6 +521,8 @@ def set_met_config_string(config, c_dict, mp_config, met_config_name, Default value is False @param default (Optional) if set, use this value as default if config is not set + @param add_x if True, add (x) to variable name, e.g. convert(x) + Default value is False """ mp_config_name = config.get_mp_config_name(mp_config) conf_value = _get_config_or_default( @@ -544,7 +546,10 @@ def set_met_config_string(config, c_dict, mp_config, met_config_name, c_key = c_dict_key if c_dict_key else met_config_name.upper() if met_config_name: - conf_value = f'{met_config_name} = {conf_value};' + config_name = met_config_name + if kwargs.get('add_x'): + config_name = f'{config_name}(x)' + conf_value = f'{config_name} = {conf_value};' c_dict[c_key] = conf_value return True @@ -734,8 +739,8 @@ def _parse_item_info(item_info): def _parse_extra_args(extra): """! Check string for extra option keywords and set them to True in - dictionary if they are found. Supports 'remove_quotes', 'uppercase' - and 'allow_empty' + dictionary if they are found. Supports 'remove_quotes', 'uppercase', + 'allow_empty', 'to_grid', 'default', and 'add_x'. @param extra string to parse for keywords @returns dictionary with extra args set if found in string @@ -750,6 +755,7 @@ def _parse_extra_args(extra): 'allow_empty', 'to_grid', 'default', + 'add_x', ) for extra_option in VALID_EXTRAS: if extra_option in extra: From a8e8062668b912696081d6d11410ac4fb524f66c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 22 Mar 2023 14:52:48 -0600 Subject: [PATCH 02/12] update TCPairs wrapper to use new add_x functionality to set diag_convert_map.convert --- metplus/wrappers/tc_pairs_wrapper.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index 3e06230117..8df1859e4c 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -439,7 +439,7 @@ def _handle_diag_convert_map(self): dict_items = { 'diag_source': 'string', 'key': 'list', - 'convert': ('string', 'remove_quotes'), + 'convert': ('string', 'remove_quotes,add_x'), } return_code = add_met_config_dict_list(config=self.config, app_name=self.app_name, @@ -450,11 +450,6 @@ def _handle_diag_convert_map(self): self.isOK = False return - # change "convert =" to "convert(x) =" - value = self.env_var_dict.get('METPLUS_DIAG_CONVERT_MAP_LIST', '') - value = value.replace('convert =', 'convert(x) =') - self.env_var_dict['METPLUS_DIAG_CONVERT_MAP_LIST'] = value - def _handle_diag(self, c_dict): diag_indices = list( find_indices_in_config_section(r'TC_PAIRS_DIAG_TEMPLATE(\d+)$', From a1ea9b1940eb1aaea46059d4047d8bf98ccd811a Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 22 Mar 2023 14:53:39 -0600 Subject: [PATCH 03/12] per #2082, add support for setting convert, censor_thresh, and censor_val in the MET regrid dictionary for all wrappers that call handle_regrid --- metplus/wrappers/command_builder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index b25c0e2f77..98855f3048 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1307,6 +1307,9 @@ def handle_regrid(self, c_dict, set_to_grid=True): dict_items['width'] = 'int' dict_items['vld_thresh'] = 'float' dict_items['shape'] = ('string', 'uppercase,remove_quotes') + dict_items['convert'] = ('string', 'remove_quotes,add_x') + dict_items['censor_thresh'] = ('list', 'remove_quotes') + dict_items['censor_val'] = ('list', 'remove_quotes') self.add_met_config_dict('regrid', dict_items) def handle_description(self, is_list=False): From 7fff4a8f07447ec01d78c38ba6526b5dd8adfcf1 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 23 Mar 2023 10:43:36 -0600 Subject: [PATCH 04/12] fixed bug where unset _THRESH variable causes unlogged failure in wrapper init step --- metplus/util/met_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py index 172b0113fd..8ca5aa2eff 100644 --- a/metplus/util/met_config.py +++ b/metplus/util/met_config.py @@ -640,7 +640,8 @@ def set_met_config_thresh(config, c_dict, mp_config, met_config_name, out_value = str(conf_value) c_dict[c_key] = out_value - return True + + return True def set_met_config_bool(config, c_dict, mp_config, met_config_name, From afb8b7edf9f0ae428238ce9ce2a17c44c3b5e5f1 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 23 Mar 2023 10:44:16 -0600 Subject: [PATCH 05/12] fixed bug to allow string for conv_radius e.g. grid_res/20 -- previously only integers were allowed --- metplus/wrappers/mtd_wrapper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metplus/wrappers/mtd_wrapper.py b/metplus/wrappers/mtd_wrapper.py index b74663ddab..432f7f375f 100755 --- a/metplus/wrappers/mtd_wrapper.py +++ b/metplus/wrappers/mtd_wrapper.py @@ -137,10 +137,11 @@ def read_field_values(self, c_dict, read_type, write_type): ) self.add_met_config(name='conv_radius', - data_type='int', + data_type='string', env_var_name=f'METPLUS_{write_type}_CONV_RADIUS', metplus_configs=[f'{read_type}_MTD_CONV_RADIUS', - 'MTD_CONV_RADIUS']) + 'MTD_CONV_RADIUS'], + extra_args={'remove_quotes': True}) self.add_met_config(name='conv_thresh', data_type='thresh', From a9e0a8757b6b91b37095818ee05f06a60147d3ee Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 23 Mar 2023 10:45:11 -0600 Subject: [PATCH 06/12] per #2082, added documentation and unit tests for new config variables. Also added missing unit tests for MTD and GridDiag wrappers --- docs/Users_Guide/glossary.rst | 165 +++++++++++++++++ docs/Users_Guide/wrappers.rst | 87 +++++++++ .../test_ensemble_stat_wrapper.py | 17 +- .../gen_ens_prod/test_gen_ens_prod_wrapper.py | 124 +++++++------ .../wrappers/grid_diag/test_grid_diag.py | 167 +++++++++++++++++ .../grid_stat/test_grid_stat_wrapper.py | 18 +- .../wrappers/mode/test_mode_wrapper.py | 18 +- .../pytests/wrappers/mtd/test_mtd_wrapper.py | 174 ++++++++++++++++++ .../point_stat/test_point_stat_wrapper.py | 18 +- .../series_analysis/test_series_analysis.py | 35 ++++ .../EnsembleStat/EnsembleStat.conf | 3 + .../GenEnsProd/GenEnsProd.conf | 3 + .../met_tool_wrapper/GridDiag/GridDiag.conf | 3 + .../met_tool_wrapper/GridStat/GridStat.conf | 7 + .../use_cases/met_tool_wrapper/MODE/MODE.conf | 3 + parm/use_cases/met_tool_wrapper/MTD/MTD.conf | 8 + .../Point2Grid/Point2Grid.conf | 8 +- .../met_tool_wrapper/PointStat/PointStat.conf | 5 + .../SeriesAnalysis/SeriesAnalysis.conf | 3 + .../met_tool_wrapper/TCRMW/TCRMW.conf | 3 + 20 files changed, 810 insertions(+), 59 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 58a81365c1..0102a6be99 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -10243,3 +10243,168 @@ METplus Configuration Glossary Specify the value for 'seeps_p1_thresh' in the MET configuration file for PointStat. | *Used by:* PointStat + + ENSEMBLE_STAT_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for EnsembleStat. + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for EnsembleStat. + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for EnsembleStat. + + | *Used by:* EnsembleStat + + GEN_ENS_PROD_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for GenEnsProd. + + | *Used by:* GenEnsProd + + GEN_ENS_PROD_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for GenEnsProd. + + | *Used by:* GenEnsProd + + GEN_ENS_PROD_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for GenEnsProd. + + | *Used by:* GenEnsProd + + GRID_DIAG_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for GridDiag. + + | *Used by:* GridDiag + + GRID_DIAG_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for GridDiag. + + | *Used by:* GridDiag + + GRID_DIAG_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for GridDiag. + + | *Used by:* GridDiag + + GRID_STAT_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + MODE_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for MODE. + + | *Used by:* MODE + + MTD_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for MTD. + + | *Used by:* MTD + + MTD_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for MTD. + + | *Used by:* MTD + + MTD_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for MTD. + + | *Used by:* MTD + + POINT2GRID_REGRID_WIDTH + Specify the value for 'regrid.width' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_REGRID_VLD_THRESH + Specify the value for 'regrid.vld_thresh' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_REGRID_SHAPE + Specify the value for 'regrid.shape' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT_STAT_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + SERIES_ANALYSIS_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for SeriesAnalysis. + + | *Used by:* SeriesAnalysis + + SERIES_ANALYSIS_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for SeriesAnalysis. + + | *Used by:* SeriesAnalysis + + SERIES_ANALYSIS_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for SeriesAnalysis. + + | *Used by:* SeriesAnalysis + + TC_RMW_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for TCRMW. + + | *Used by:* TCRMW + + TC_RMW_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for TCRMW. + + | *Used by:* TCRMW + + TC_RMW_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for TCRMW. + + | *Used by:* TCRMW diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 7febc1d79b..f3e50dc412 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -204,6 +204,9 @@ METplus Configuration | :term:`ENSEMBLE_STAT_REGRID_WIDTH` | :term:`ENSEMBLE_STAT_REGRID_VLD_THRESH` | :term:`ENSEMBLE_STAT_REGRID_SHAPE` +| :term:`ENSEMBLE_STAT_REGRID_CONVERT` +| :term:`ENSEMBLE_STAT_REGRID_CENSOR_THRESH` +| :term:`ENSEMBLE_STAT_REGRID_CENSOR_VAL` | :term:`ENSEMBLE_STAT_CONFIG_FILE` | :term:`ENSEMBLE_STAT_MET_OBS_ERR_TABLE` | :term:`ENSEMBLE_STAT_N_MEMBERS` @@ -394,6 +397,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`ENSEMBLE_STAT_REGRID_TO_GRID` - regrid.to_grid + * - :term:`ENSEMBLE_STAT_REGRID_CONVERT` + - regrid.convert + * - :term:`ENSEMBLE_STAT_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`ENSEMBLE_STAT_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_CENSOR_THRESH}** @@ -1050,6 +1059,9 @@ METplus Configuration | :term:`GEN_ENS_PROD_REGRID_WIDTH` | :term:`GEN_ENS_PROD_REGRID_VLD_THRESH` | :term:`GEN_ENS_PROD_REGRID_SHAPE` +| :term:`GEN_ENS_PROD_REGRID_CONVERT` +| :term:`GEN_ENS_PROD_REGRID_CENSOR_THRESH` +| :term:`GEN_ENS_PROD_REGRID_CENSOR_VAL` | :term:`GEN_ENS_PROD_CENSOR_THRESH` | :term:`GEN_ENS_PROD_CENSOR_VAL` | :term:`GEN_ENS_PROD_CAT_THRESH` @@ -1174,6 +1186,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`GEN_ENS_PROD_REGRID_TO_GRID` - regrid.to_grid + * - :term:`GEN_ENS_PROD_REGRID_CONVERT` + - regrid.convert + * - :term:`GEN_ENS_PROD_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`GEN_ENS_PROD_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_CENSOR_THRESH}** @@ -2669,6 +2687,9 @@ METplus Configuration | :term:`GRID_DIAG_REGRID_VLD_THRESH` | :term:`GRID_DIAG_REGRID_SHAPE` | :term:`GRID_DIAG_REGRID_TO_GRID` +| :term:`GRID_DIAG_REGRID_CONVERT` +| :term:`GRID_DIAG_REGRID_CENSOR_THRESH` +| :term:`GRID_DIAG_REGRID_CENSOR_VAL` | :term:`GRID_DIAG_DESC` | :term:`GRID_DIAG_SKIP_IF_OUTPUT_EXISTS` | :term:`GRID_DIAG_RUNTIME_FREQ` @@ -2725,6 +2746,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`GRID_DIAG_REGRID_TO_GRID` - regrid.to_grid + * - :term:`GRID_DIAG_REGRID_CONVERT` + - regrid.convert + * - :term:`GRID_DIAG_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`GRID_DIAG_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_CENSOR_THRESH}** @@ -2826,6 +2853,9 @@ METplus Configuration | :term:`GRID_STAT_REGRID_WIDTH` | :term:`GRID_STAT_REGRID_VLD_THRESH` | :term:`GRID_STAT_REGRID_SHAPE` +| :term:`GRID_STAT_REGRID_CONVERT` +| :term:`GRID_STAT_REGRID_CENSOR_THRESH` +| :term:`GRID_STAT_REGRID_CENSOR_VAL` | :term:`GRID_STAT_CLIMO_CDF_BINS` | :term:`GRID_STAT_CLIMO_CDF_CENTER_BINS` | :term:`GRID_STAT_CLIMO_CDF_WRITE_BINS` @@ -3025,6 +3055,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`GRID_STAT_REGRID_TO_GRID` - regrid.to_grid + * - :term:`GRID_STAT_REGRID_CONVERT` + - regrid.convert + * - :term:`GRID_STAT_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`GRID_STAT_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_FCST_FIELD}** @@ -3944,6 +3980,9 @@ METplus Configuration | :term:`MODE_REGRID_WIDTH` | :term:`MODE_REGRID_VLD_THRESH` | :term:`MODE_REGRID_SHAPE` +| :term:`MODE_REGRID_CONVERT` +| :term:`MODE_REGRID_CENSOR_THRESH` +| :term:`MODE_REGRID_CENSOR_VAL` | :term:`MODE_CONFIG_FILE` | :term:`FCST_MODE_INPUT_DATATYPE` | :term:`OBS_MODE_INPUT_DATATYPE` @@ -4105,6 +4144,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`MODE_REGRID_TO_GRID` - regrid.to_grid + * - :term:`MODE_REGRID_CONVERT` + - regrid.convert + * - :term:`MODE_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`MODE_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_GRID_RES}** @@ -4639,6 +4684,9 @@ METplus Configuration | :term:`MTD_REGRID_WIDTH` | :term:`MTD_REGRID_VLD_THRESH` | :term:`MTD_REGRID_SHAPE` +| :term:`MTD_REGRID_CONVERT` +| :term:`MTD_REGRID_CENSOR_THRESH` +| :term:`MTD_REGRID_CENSOR_VAL` | :term:`MTD_MET_CONFIG_OVERRIDES` | :term:`FCST_MTD_IS_PROB` | :term:`FCST_MTD_PROB_IN_GRIB_PDS` @@ -4730,6 +4778,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`MTD_REGRID_TO_GRID` - regrid.to_grid + * - :term:`MTD_REGRID_CONVERT` + - regrid.convert + * - :term:`MTD_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`MTD_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_FCST_FILE_TYPE}** @@ -5684,6 +5738,12 @@ METplus Configuration | :term:`POINT2GRID_QC_FLAGS` | :term:`POINT2GRID_ADP` | :term:`POINT2GRID_REGRID_METHOD` +| :term:`POINT2GRID_REGRID_WIDTH` +| :term:`POINT2GRID_REGRID_VLD_THRESH` +| :term:`POINT2GRID_REGRID_SHAPE` +| :term:`POINT2GRID_REGRID_CONVERT` +| :term:`POINT2GRID_REGRID_CENSOR_THRESH` +| :term:`POINT2GRID_REGRID_CENSOR_VAL` | :term:`POINT2GRID_GAUSSIAN_DX` | :term:`POINT2GRID_GAUSSIAN_RADIUS` | :term:`POINT2GRID_PROB_CAT_THRESH` @@ -5728,6 +5788,9 @@ Configuration | :term:`POINT_STAT_REGRID_WIDTH` | :term:`POINT_STAT_REGRID_VLD_THRESH` | :term:`POINT_STAT_REGRID_SHAPE` +| :term:`POINT_STAT_REGRID_CONVERT` +| :term:`POINT_STAT_REGRID_CENSOR_THRESH` +| :term:`POINT_STAT_REGRID_CENSOR_VAL` | :term:`POINT_STAT_MASK_GRID` | :term:`POINT_STAT_MASK_POLY` | :term:`POINT_STAT_MASK_SID` @@ -5913,6 +5976,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`POINT_STAT_REGRID_TO_GRID` - regrid.to_grid + * - :term:`POINT_STAT_REGRID_CONVERT` + - regrid.convert + * - :term:`POINT_STAT_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`POINT_STAT_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_FCST_FIELD}** @@ -6400,6 +6469,9 @@ METplus Configuration | :term:`SERIES_ANALYSIS_REGRID_WIDTH` | :term:`SERIES_ANALYSIS_REGRID_VLD_THRESH` | :term:`SERIES_ANALYSIS_REGRID_SHAPE` +| :term:`SERIES_ANALYSIS_REGRID_CONVERT` +| :term:`SERIES_ANALYSIS_REGRID_CENSOR_THRESH` +| :term:`SERIES_ANALYSIS_REGRID_CENSOR_VAL` | :term:`SERIES_ANALYSIS_STAT_LIST` | :term:`SERIES_ANALYSIS_IS_PAIRED` | :term:`SERIES_ANALYSIS_CUSTOM_LOOP_LIST` @@ -6563,6 +6635,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`SERIES_ANALYSIS_REGRID_TO_GRID` - regrid.to_grid + * - :term:`SERIES_ANALYSIS_REGRID_CONVERT` + - regrid.convert + * - :term:`SERIES_ANALYSIS_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`SERIES_ANALYSIS_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_CAT_THRESH}** @@ -8674,6 +8752,9 @@ METplus Configuration | :term:`TC_RMW_REGRID_WIDTH` | :term:`TC_RMW_REGRID_VLD_THRESH` | :term:`TC_RMW_REGRID_SHAPE` +| :term:`TC_RMW_REGRID_CONVERT` +| :term:`TC_RMW_REGRID_CENSOR_THRESH` +| :term:`TC_RMW_REGRID_CENSOR_VAL` | :term:`TC_RMW_N_RANGE` | :term:`TC_RMW_N_AZIMUTH` | :term:`TC_RMW_MAX_RANGE_KM` @@ -8881,6 +8962,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.width * - :term:`TC_RMW_REGRID_VLD_THRESH` - regrid.vld_thresh + * - :term:`TC_RMW_REGRID_CONVERT` + - regrid.convert + * - :term:`TC_RMW_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`TC_RMW_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_N_RANGE}** diff --git a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py index aad6656e24..cb8dfc66d4 100644 --- a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py @@ -244,14 +244,29 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, }, {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + ({'ENSEMBLE_STAT_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'ENSEMBLE_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'ENSEMBLE_STAT_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + ({'ENSEMBLE_STAT_REGRID_TO_GRID': 'FCST', 'ENSEMBLE_STAT_REGRID_METHOD': 'NEAREST', 'ENSEMBLE_STAT_REGRID_WIDTH': '1', 'ENSEMBLE_STAT_REGRID_VLD_THRESH': '0.5', 'ENSEMBLE_STAT_REGRID_SHAPE': 'SQUARE', + 'ENSEMBLE_STAT_REGRID_CONVERT': '2*x', + 'ENSEMBLE_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', + 'ENSEMBLE_STAT_REGRID_CENSOR_VAL': '12000,5000', }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' - 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' ), 'REGRID_TO_GRID': 'FCST'}), diff --git a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py index 2bcba73a17..572130433f 100644 --- a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -87,74 +87,90 @@ def handle_input_dir(config): }, {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), # 8 + ({'GEN_ENS_PROD_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + # 9 + ({'GEN_ENS_PROD_REGRID_CENSOR_THRESH': '>12000,<5000', }, + { + 'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + # 10 + ({'GEN_ENS_PROD_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + # 11 ({'GEN_ENS_PROD_REGRID_TO_GRID': 'FCST', 'GEN_ENS_PROD_REGRID_METHOD': 'NEAREST', 'GEN_ENS_PROD_REGRID_WIDTH': '1', 'GEN_ENS_PROD_REGRID_VLD_THRESH': '0.5', 'GEN_ENS_PROD_REGRID_SHAPE': 'SQUARE', + 'GEN_ENS_PROD_REGRID_CONVERT': '2*x', + 'GEN_ENS_PROD_REGRID_CENSOR_THRESH': '>12000,<5000', + 'GEN_ENS_PROD_REGRID_CENSOR_VAL': '12000,5000', }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' - 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' )}), - # 9 + # 12 ({'GEN_ENS_PROD_CLIMO_MEAN_INPUT_TEMPLATE': '/some/path/climo/filename.nc', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {file_name = ["/some/path/climo/filename.nc"];}', }), - # 10 + # 13 ({'GEN_ENS_PROD_CLIMO_STDEV_INPUT_TEMPLATE': '/some/path/climo/stdfile.nc', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {file_name = ["/some/path/climo/stdfile.nc"];}', }), - # 11 ensemble_flag + # 14 ensemble_flag ({'GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {latlon = FALSE;}'}), - # 12 + # 15 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {mean = FALSE;}'}), - # 13 + # 16 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {stdev = FALSE;}'}), - # 14 + # 17 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {minus = FALSE;}'}), - # 15 + # 18 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {plus = FALSE;}'}), - # 16 + # 19 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_MIN': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {min = FALSE;}'}), - # 17 + # 20 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_MAX': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {max = FALSE;}'}), - # 18 + # 21 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {range = FALSE;}'}), - # 19 + # 22 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT': 'FALSE', }, { 'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {vld_count = FALSE;}'}), - # 20 + # 23 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY': 'FALSE', }, { 'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {frequency = FALSE;}'}), - # 21 + # 24 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_NEP': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {nep = FALSE;}'}), - # 22 + # 25 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {nmep = FALSE;}'}), - # 23 + # 26 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {climo = FALSE;}'}), - # 24 + # 27 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDP': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {climo_cdp = FALSE;}'}), - # 25 + # 28 ({ 'GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON': 'FALSE', 'GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN': 'FALSE', @@ -180,40 +196,40 @@ def handle_input_dir(config): 'frequency = FALSE;nep = FALSE;' 'nmep = FALSE;climo = FALSE;' 'climo_cdp = FALSE;}')}), - # 26 + # 29 ({'GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', }, {'METPLUS_CLIMO_MEAN_DICT': ('climo_mean = {file_name = ' '["/some/climo_mean/file.txt"];}'),}), - # 27 + # 30 ({'GEN_ENS_PROD_CLIMO_MEAN_FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {field = [{name="CLM_NAME"; level="(0,0,*,*)";}];}'}), - # 28 + # 31 ({'GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD': 'NEAREST', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {regrid = {method = NEAREST;}}'}), - # 29 + # 32 ({'GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH': '1', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {regrid = {width = 1;}}'}), - # 30 + # 33 ({'GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH': '0.5', }, { 'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {regrid = {vld_thresh = 0.5;}}'}), - # 31 + # 34 ({'GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE': 'SQUARE', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {regrid = {shape = SQUARE;}}'}), - # 32 + # 35 ({'GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD': 'NEAREST', }, { 'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {time_interp_method = NEAREST;}'}), - # 33 + # 36 ({'GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH': 'True', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {match_month = TRUE;}'}), - # 34 + # 37 ({'GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL': '30', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {day_interval = 30;}'}), - # 35 + # 38 ({'GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL': '12', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {hour_interval = 12;}'}), - # 36 + # 39 ({ 'GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', 'GEN_ENS_PROD_CLIMO_MEAN_FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', @@ -234,42 +250,42 @@ def handle_input_dir(config): 'time_interp_method = NEAREST;' 'match_month = TRUE;day_interval = 30;' 'hour_interval = 12;}')}), - # 37 climo stdev + # 40 climo stdev ({'GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME': '/some/climo_stdev/file.txt', }, {'METPLUS_CLIMO_STDEV_DICT': ('climo_stdev = {file_name = ' '["/some/climo_stdev/file.txt"];}')}), - # 38 + # 41 ({'GEN_ENS_PROD_CLIMO_STDEV_FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {field = [{name="CLM_NAME"; level="(0,0,*,*)";}];}'}), - # 39 + # 42 ({'GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD': 'NEAREST', }, { 'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {regrid = {method = NEAREST;}}'}), - # 40 + # 43 ({'GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH': '1', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {regrid = {width = 1;}}'}), - # 41 + # 44 ({'GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH': '0.5', }, { 'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {regrid = {vld_thresh = 0.5;}}'}), - # 42 + # 45 ({'GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE': 'SQUARE', }, { 'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {regrid = {shape = SQUARE;}}'}), - # 43 + # 46 ({'GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD': 'NEAREST', }, { 'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {time_interp_method = NEAREST;}'}), - # 44 + # 47 ({'GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH': 'True', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {match_month = TRUE;}'}), - # 45 + # 48 ({'GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL': '30', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {day_interval = 30;}'}), - # 46 + # 49 ({'GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL': '12', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {hour_interval = 12;}'}), - # 47 + # 50 ({ 'GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME': '/some/climo_stdev/file.txt', 'GEN_ENS_PROD_CLIMO_STDEV_FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', @@ -290,16 +306,16 @@ def handle_input_dir(config): 'time_interp_method = NEAREST;' 'match_month = TRUE;day_interval = 30;' 'hour_interval = 12;}')}), - # 48 + # 51 ({'GEN_ENS_PROD_NBRHD_PROB_WIDTH': '5', }, {'METPLUS_NBRHD_PROB_DICT': 'nbrhd_prob = {width = [5];}'}), - # 49 + # 52 ({'GEN_ENS_PROD_NBRHD_PROB_SHAPE': 'circle', }, {'METPLUS_NBRHD_PROB_DICT': 'nbrhd_prob = {shape = CIRCLE;}'}), - # 50 + # 53 ({'GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH': '0.0', }, {'METPLUS_NBRHD_PROB_DICT': 'nbrhd_prob = {vld_thresh = 0.0;}'}), - # 51 + # 54 ({ 'GEN_ENS_PROD_NBRHD_PROB_WIDTH': '5', 'GEN_ENS_PROD_NBRHD_PROB_SHAPE': 'CIRCLE', @@ -311,26 +327,26 @@ def handle_input_dir(config): 'vld_thresh = 0.0;}' ) }), - # 52 + # 55 ({'GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH': '0.0', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {vld_thresh = 0.0;}'}), - # 53 + # 56 ({'GEN_ENS_PROD_NMEP_SMOOTH_SHAPE': 'circle', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {shape = CIRCLE;}'}), - # 54 + # 57 ({'GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX': '81.27', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {gaussian_dx = 81.27;}'}), - # 55 + # 58 ({'GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS': '120', }, { 'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {gaussian_radius = 120;}'}), - # 56 + # 59 ({'GEN_ENS_PROD_NMEP_SMOOTH_TYPE_METHOD': 'GAUSSIAN', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {type = [{method = GAUSSIAN;}];}'}), - # 57 + # 60 ({'GEN_ENS_PROD_NMEP_SMOOTH_TYPE_WIDTH': '1', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {type = [{width = 1;}];}'}), - # 58 + # 61 ({ 'GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH': '0.0', 'GEN_ENS_PROD_NMEP_SMOOTH_SHAPE': 'circle', @@ -346,13 +362,13 @@ def handle_input_dir(config): 'type = [{method = GAUSSIAN;width = 1;}];}' ) }), - # 59 + # 62 ({'GEN_ENS_PROD_ENS_MEMBER_IDS': '1,2,3,4', }, {'METPLUS_ENS_MEMBER_IDS': 'ens_member_ids = ["1", "2", "3", "4"];'}), - # 60 + # 63 ({'GEN_ENS_PROD_CONTROL_ID': '0', }, {'METPLUS_CONTROL_ID': 'control_id = "0";'}), - # 61 + # 64 ({'GEN_ENS_PROD_NORMALIZE': 'CLIMO_STD_ANOM', }, {'METPLUS_NORMALIZE': 'normalize = CLIMO_STD_ANOM;'}), diff --git a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py index 72e3a878d7..40c6e2eee2 100644 --- a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py +++ b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py @@ -9,6 +9,54 @@ from metplus.wrappers.grid_diag_wrapper import GridDiagWrapper +time_fmt = '%Y%m%d%H' +run_times = ['2016092900', '2016092906'] +data_dir = '/some/path/data' + +data_name_1 = 'APCP' +data_level = 'L0' +data_options_1 = 'n_bins = 55; range = [0, 55];' +data_name_2 = 'PWAT' +data_options_2 = 'n_bins = 35; range = [35, 70];' +data_fmt = ('data = {field = [ ' + f'{{ name="{data_name_1}"; level="{data_level}"; {data_options_1} }},' + f'{{ name="{data_name_2}"; level="{data_level}"; {data_options_2} }}' + ' ];}') + + +def set_minimum_config_settings(config): + # set config variables to prevent command from running and bypass check + # if input files actually exist + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + + # set process and time config variables + config.set('config', 'PROCESS_LIST', 'GridDiag') + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', time_fmt) + config.set('config', 'INIT_BEG', run_times[0]) + config.set('config', 'INIT_END', run_times[-1]) + config.set('config', 'INIT_INCREMENT', '6H') + config.set('config', 'LEAD_SEQ', '141, 144, 147') + config.set('config', 'GRID_DIAG_RUNTIME_FREQ', + 'RUN_ONCE_PER_INIT_OR_VALID') + config.set('config', 'GRID_DIAG_CONFIG_FILE', + '{PARM_BASE}/met_config/GridDiagConfig_wrapped') + config.set('config', 'GRID_DIAG_INPUT_DIR', data_dir) + config.set('config', 'GRID_DIAG_INPUT_TEMPLATE', + ('gfs.subset.t00z.pgrb2.0p25.f{lead?fmt=%H}, ' + 'gfs.subset.t00z.pgrb2.0p25.f{lead?fmt=%H}')) + config.set('config', 'GRID_DIAG_OUTPUT_DIR', + '{OUTPUT_BASE}/grid_diag/output') + config.set('config', 'GRID_DIAG_OUTPUT_TEMPLATE', + 'grid_diag.{valid?fmt=%Y%m%d%H}.nc') + config.set('config', 'BOTH_VAR1_NAME', data_name_1) + config.set('config', 'BOTH_VAR1_LEVELS', data_level) + config.set('config', 'BOTH_VAR1_OPTIONS', data_options_1) + config.set('config', 'BOTH_VAR2_NAME', data_name_2) + config.set('config', 'BOTH_VAR2_LEVELS', data_level) + config.set('config', 'BOTH_VAR2_OPTIONS', data_options_2) + @pytest.mark.parametrize( 'time_info, expected_subset', [ @@ -188,3 +236,122 @@ def test_get_config_file(metplus_config): config.set('config', 'GRID_DIAG_CONFIG_FILE', fake_config_name) wrapper = GridDiagWrapper(config) assert wrapper.c_dict['CONFIG_FILE'] == fake_config_name + + +@pytest.mark.parametrize( + 'config_overrides, env_var_values', [ + ({'GRID_DIAG_DESC': 'my_desc'}, + {'METPLUS_DESC': 'desc = "my_desc";'}), + + ({'GRID_DIAG_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_CENSOR_THRESH': 'censor_thresh = [>12000, <5000];'}), + + ({'GRID_DIAG_CENSOR_VAL': '12000, 5000', }, + {'METPLUS_CENSOR_VAL': 'censor_val = [12000, 5000];'}), + + ({'GRID_DIAG_MASK_GRID': 'FULL', }, + {'METPLUS_MASK_DICT': 'mask = {grid = "FULL";}'}), + + ({'GRID_DIAG_MASK_POLY': 'MET_BASE/poly/EAST.poly', }, + {'METPLUS_MASK_DICT': 'mask = {poly = "MET_BASE/poly/EAST.poly";}'}), + + ({'GRID_DIAG_MASK_GRID': 'FULL', + 'GRID_DIAG_MASK_POLY': 'MET_BASE/poly/EAST.poly',}, + {'METPLUS_MASK_DICT': ('mask = {grid = "FULL";' + 'poly = "MET_BASE/poly/EAST.poly";}')}), + + ({'GRID_DIAG_REGRID_TO_GRID': 'FCST',}, + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + + ({'GRID_DIAG_REGRID_METHOD': 'NEAREST',}, + {'METPLUS_REGRID_DICT': 'regrid = {method = NEAREST;}'}), + + ({'GRID_DIAG_REGRID_WIDTH': '1',}, + {'METPLUS_REGRID_DICT': 'regrid = {width = 1;}'}), + + ({'GRID_DIAG_REGRID_VLD_THRESH': '0.5',}, + {'METPLUS_REGRID_DICT': 'regrid = {vld_thresh = 0.5;}'}), + + ({'GRID_DIAG_REGRID_SHAPE': 'SQUARE',}, + {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + + ({'GRID_DIAG_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'GRID_DIAG_REGRID_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'GRID_DIAG_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + + ({'GRID_DIAG_REGRID_TO_GRID': 'FCST', + 'GRID_DIAG_REGRID_METHOD': 'NEAREST', + 'GRID_DIAG_REGRID_WIDTH': '1', + 'GRID_DIAG_REGRID_VLD_THRESH': '0.5', + 'GRID_DIAG_REGRID_SHAPE': 'SQUARE', + 'GRID_DIAG_REGRID_CONVERT': '2*x', + 'GRID_DIAG_REGRID_CENSOR_THRESH': '>12000,<5000', + 'GRID_DIAG_REGRID_CENSOR_VAL': '12000,5000', + }, + {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' + )}), + ] +) +@pytest.mark.wrapper +def test_grid_diag(metplus_config, config_overrides, env_var_values): + config = metplus_config + set_minimum_config_settings(config) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = GridDiagWrapper(config) + assert wrapper.isOK + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR') + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + + expected_cmds = [ + (f"{app_path} -data {file_list_dir}/grid_diag_files_input0" + "_init_20160929000000_valid_ALL_lead_ALL.txt " + f"-data {file_list_dir}/grid_diag_files_input1" + "_init_20160929000000_valid_ALL_lead_ALL.txt " + f"-config {config_file} -out {out_dir}/grid_diag.all.nc {verbosity}"), + (f"{app_path} -data {file_list_dir}/grid_diag_files_input0" + "_init_20160929060000_valid_ALL_lead_ALL.txt " + f"-data {file_list_dir}/grid_diag_files_input1" + "_init_20160929060000_valid_ALL_lead_ALL.txt " + f"-config {config_file} -out {out_dir}/grid_diag.all.nc {verbosity}"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + assert len(all_cmds) == len(expected_cmds) + + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd + + # check that environment variables were set properly + # including deprecated env vars (not in wrapper env var keys) + for env_var_key in env_var_keys: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + actual_value = match.split('=', 1)[1] + if env_var_key == 'METPLUS_DATA_DICT': + assert actual_value == data_fmt + else: + assert env_var_values.get(env_var_key, '') == actual_value diff --git a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py index 280103c68e..03e578e5a5 100644 --- a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py @@ -193,14 +193,30 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, }, {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + ({'GRID_STAT_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'GRID_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', }, + { + 'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'GRID_STAT_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + ({'GRID_STAT_REGRID_TO_GRID': 'FCST', 'GRID_STAT_REGRID_METHOD': 'NEAREST', 'GRID_STAT_REGRID_WIDTH': '1', 'GRID_STAT_REGRID_VLD_THRESH': '0.5', 'GRID_STAT_REGRID_SHAPE': 'SQUARE', + 'GRID_STAT_REGRID_CONVERT': '2*x', + 'GRID_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', + 'GRID_STAT_REGRID_CENSOR_VAL': '12000,5000', }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' - 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' ), 'REGRID_TO_GRID': 'FCST'}), diff --git a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py index fb9759face..07ff4bb110 100644 --- a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py +++ b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py @@ -93,14 +93,30 @@ def set_minimum_config_settings(config): }, {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + ({'MODE_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'MODE_REGRID_CENSOR_THRESH': '>12000,<5000', }, + { + 'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'MODE_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + ({'MODE_REGRID_TO_GRID': 'FCST', 'MODE_REGRID_METHOD': 'NEAREST', 'MODE_REGRID_WIDTH': '1', 'MODE_REGRID_VLD_THRESH': '0.5', 'MODE_REGRID_SHAPE': 'SQUARE', + 'MODE_REGRID_CONVERT': '2*x', + 'MODE_REGRID_CENSOR_THRESH': '>12000,<5000', + 'MODE_REGRID_CENSOR_VAL': '12000,5000', }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' - 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' ), 'REGRID_TO_GRID': 'FCST'}), diff --git a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py index 420e6d40fc..2101ccfefc 100644 --- a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py +++ b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py @@ -7,6 +7,19 @@ from metplus.wrappers.mtd_wrapper import MTDWrapper +fcst_dir = '/some/path/fcst' +obs_dir = '/some/path/obs' +fcst_name = 'APCP' +fcst_level = 'A03' +fcst_thresh = 'gt12.7' +obs_name = 'APCP_03' +obs_level_no_quotes = '(*,*)' +obs_level = f'"{obs_level_no_quotes}"' +obs_thresh = 'gt12.7' +fcst_fmt = f'field = {{ name="{fcst_name}"; level="{fcst_level}"; cat_thresh=[ gt12.7 ]; }};' +obs_fmt = (f'field = {{ name="{obs_name}"; ' + f'level="{obs_level_no_quotes}"; cat_thresh=[ gt12.7 ]; }};') + def get_test_data_dir(config, subdir): return os.path.join(config.getdir('METPLUS_BASE'), @@ -32,6 +45,167 @@ def mtd_wrapper(metplus_config, lead_seq=None): return MTDWrapper(config) +def set_minimum_config_settings(config): + # set config variables to prevent command from running and bypass check + # if input files actually exist + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + + # set process and time config variables + config.set('config', 'PROCESS_LIST', 'MTD') + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H') + config.set('config', 'INIT_BEG', '2005080700') + config.set('config', 'INIT_END', '2005080700') + config.set('config', 'INIT_INCREMENT', '12H') + config.set('config', 'LEAD_SEQ', '6H, 9H, 12H') + config.set('config', 'MTD_CONFIG_FILE', + '{PARM_BASE}/met_config/MTDConfig_wrapped') + config.set('config', 'FCST_MTD_INPUT_DIR', fcst_dir) + config.set('config', 'OBS_MTD_INPUT_DIR', obs_dir) + config.set('config', 'FCST_MTD_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}') + config.set('config', 'OBS_MTD_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d%H}/obs_file') + config.set('config', 'MTD_OUTPUT_DIR', + '{OUTPUT_BASE}/MTD/output') + config.set('config', 'MTD_OUTPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}') + + config.set('config', 'FCST_VAR1_NAME', fcst_name) + config.set('config', 'FCST_VAR1_LEVELS', fcst_level) + config.set('config', 'FCST_VAR1_THRESH', fcst_thresh) + config.set('config', 'OBS_VAR1_NAME', obs_name) + config.set('config', 'OBS_VAR1_LEVELS', obs_level) + config.set('config', 'OBS_VAR1_THRESH', obs_thresh) + + +@pytest.mark.parametrize( + 'config_overrides, env_var_values', [ + ({'MODEL': 'my_model'}, + {'METPLUS_MODEL': 'model = "my_model";'}), + + ({'MTD_DESC': 'my_desc'}, + {'METPLUS_DESC': 'desc = "my_desc";'}), + + ({'DESC': 'my_desc'}, + {'METPLUS_DESC': 'desc = "my_desc";'}), + + ({'OBTYPE': 'my_obtype'}, + {'METPLUS_OBTYPE': 'obtype = "my_obtype";'}), + + ({'MTD_REGRID_TO_GRID': 'FCST',}, + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + + ({'MTD_REGRID_METHOD': 'NEAREST',}, + {'METPLUS_REGRID_DICT': 'regrid = {method = NEAREST;}'}), + + ({'MTD_REGRID_WIDTH': '1',}, + {'METPLUS_REGRID_DICT': 'regrid = {width = 1;}'}), + + ({'MTD_REGRID_VLD_THRESH': '0.5',}, + {'METPLUS_REGRID_DICT': 'regrid = {vld_thresh = 0.5;}'}), + + ({'MTD_REGRID_SHAPE': 'SQUARE',}, + {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + + ({'MTD_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'MTD_REGRID_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'MTD_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + + ({'MTD_REGRID_TO_GRID': 'FCST', + 'MTD_REGRID_METHOD': 'NEAREST', + 'MTD_REGRID_WIDTH': '1', + 'MTD_REGRID_VLD_THRESH': '0.5', + 'MTD_REGRID_SHAPE': 'SQUARE', + 'MTD_REGRID_CONVERT': '2*x', + 'MTD_REGRID_CENSOR_THRESH': '>12000,<5000', + 'MTD_REGRID_CENSOR_VAL': '12000,5000', + }, + {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' + )}), + + ({'FCST_MTD_CONV_RADIUS': '40.0/grid_res'}, + {'METPLUS_FCST_CONV_RADIUS': 'conv_radius = 40.0/grid_res;'}), + + ({'OBS_MTD_CONV_RADIUS': '40.0/grid_res'}, + {'METPLUS_OBS_CONV_RADIUS': 'conv_radius = 40.0/grid_res;'}), + + ({'FCST_MTD_CONV_THRESH': '>=10.0'}, + {'METPLUS_FCST_CONV_THRESH': 'conv_thresh = >=10.0;'}), + + ({'OBS_MTD_CONV_THRESH': '>=10.0'}, + {'METPLUS_OBS_CONV_THRESH': 'conv_thresh = >=10.0;'}), + + ({'MTD_MIN_VOLUME': '1000'}, + {'METPLUS_MIN_VOLUME': 'min_volume = 1000;'}), + + ({'MTD_OUTPUT_PREFIX': 'my_output_prefix'}, + {'METPLUS_OUTPUT_PREFIX': 'output_prefix = "my_output_prefix";'}), + ] +) +@pytest.mark.wrapper +def test_mode_single_field(metplus_config, config_overrides, env_var_values): + config = metplus_config + + # set config variables needed to run + set_minimum_config_settings(config) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = MTDWrapper(config) + assert wrapper.isOK + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR') + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + expected_cmds = [(f"{app_path} {verbosity} " + f"-fcst {file_list_dir}/" + f"20050807060000_mtd_fcst_{fcst_name}.txt " + f"-obs {file_list_dir}/" + f"20050807060000_mtd_obs_{obs_name}.txt " + f"-config {config_file} " + f"-outdir {out_dir}/2005080706"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd + + # check that environment variables were set properly + # including deprecated env vars (not in wrapper env var keys) + for env_var_key in env_var_keys: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + value = match.split('=', 1)[1] + if env_var_key == 'METPLUS_FCST_FIELD': + assert value == fcst_fmt + elif env_var_key == 'METPLUS_OBS_FIELD': + assert value == obs_fmt + else: + assert env_var_values.get(env_var_key, '') == value + + @pytest.mark.wrapper def test_mtd_by_init_all_found(metplus_config): mw = mtd_wrapper(metplus_config, '1,2,3') diff --git a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py index 2e7ef0ef16..28abc6c04e 100755 --- a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py @@ -87,14 +87,30 @@ def test_met_dictionary_in_var_options(metplus_config): }, {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + ({'POINT_STAT_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'POINT_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', }, + { + 'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'POINT_STAT_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + ({'POINT_STAT_REGRID_TO_GRID': 'FCST', 'POINT_STAT_REGRID_METHOD': 'NEAREST', 'POINT_STAT_REGRID_WIDTH': '1', 'POINT_STAT_REGRID_VLD_THRESH': '0.5', 'POINT_STAT_REGRID_SHAPE': 'SQUARE', + 'POINT_STAT_REGRID_CONVERT': '2*x', + 'POINT_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', + 'POINT_STAT_REGRID_CENSOR_VAL': '12000,5000', }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' - 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' ), 'REGRID_TO_GRID': 'FCST'}), diff --git a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py index 08a83e771c..e96abbef6a 100644 --- a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py +++ b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py @@ -105,6 +105,41 @@ def set_minimum_config_settings(config): @pytest.mark.parametrize( 'config_overrides, env_var_values', [ + ({'SERIES_ANALYSIS_REGRID_METHOD': 'NEAREST',}, + {'METPLUS_REGRID_DICT': 'regrid = {method = NEAREST;}'}), + + ({'SERIES_ANALYSIS_REGRID_WIDTH': '1',}, + {'METPLUS_REGRID_DICT': 'regrid = {width = 1;}'}), + + ({'SERIES_ANALYSIS_REGRID_VLD_THRESH': '0.5',}, + {'METPLUS_REGRID_DICT': 'regrid = {vld_thresh = 0.5;}'}), + + ({'SERIES_ANALYSIS_REGRID_SHAPE': 'SQUARE',}, + {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + + ({'SERIES_ANALYSIS_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'SERIES_ANALYSIS_REGRID_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'SERIES_ANALYSIS_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + + ({'SERIES_ANALYSIS_REGRID_METHOD': 'NEAREST', + 'SERIES_ANALYSIS_REGRID_WIDTH': '1', + 'SERIES_ANALYSIS_REGRID_VLD_THRESH': '0.5', + 'SERIES_ANALYSIS_REGRID_SHAPE': 'SQUARE', + 'SERIES_ANALYSIS_REGRID_CONVERT': '2*x', + 'SERIES_ANALYSIS_REGRID_CENSOR_THRESH': '>12000,<5000', + 'SERIES_ANALYSIS_REGRID_CENSOR_VAL': '12000,5000', + }, + {'METPLUS_REGRID_DICT': ('regrid = {method = NEAREST;' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' + )}), # climo_mean ({'SERIES_ANALYSIS_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', }, {'METPLUS_CLIMO_MEAN_DICT': ('climo_mean = {file_name = ' diff --git a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf index 03aaee0657..da1c9d5e6d 100644 --- a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf +++ b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf @@ -113,6 +113,9 @@ ENSEMBLE_STAT_REGRID_METHOD = NEAREST ENSEMBLE_STAT_REGRID_WIDTH = 1 ENSEMBLE_STAT_REGRID_VLD_THRESH = 0.5 ENSEMBLE_STAT_REGRID_SHAPE = SQUARE +#ENSEMBLE_STAT_REGRID_CONVERT = +#ENSEMBLE_STAT_REGRID_CENSOR_THRESH = +#ENSEMBLE_STAT_REGRID_CENSOR_VAL = ENSEMBLE_STAT_CENSOR_THRESH = ENSEMBLE_STAT_CENSOR_VAL = diff --git a/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf b/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf index 696abe4008..3c4a300a40 100644 --- a/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf +++ b/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf @@ -102,6 +102,9 @@ GEN_ENS_PROD_N_MEMBERS = 6 #GEN_ENS_PROD_REGRID_WIDTH = 1 #GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 #GEN_ENS_PROD_REGRID_SHAPE = SQUARE +#GEN_ENS_PROD_REGRID_CONVERT = +#GEN_ENS_PROD_REGRID_CENSOR_THRESH = +#GEN_ENS_PROD_REGRID_CENSOR_VAL = #GEN_ENS_PROD_CENSOR_THRESH = #GEN_ENS_PROD_CENSOR_VAL = diff --git a/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf b/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf index c35fe3994a..4104ee6016 100644 --- a/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf +++ b/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf @@ -78,5 +78,8 @@ GRID_DIAG_CONFIG_FILE = {PARM_BASE}/met_config/GridDiagConfig_wrapped #GRID_DIAG_REGRID_WIDTH = 1 #GRID_DIAG_REGRID_VLD_THRESH = 0.5 #GRID_DIAG_REGRID_SHAPE = SQUARE +#GRID_DIAG_REGRID_CONVERT = +#GRID_DIAG_REGRID_CENSOR_THRESH = +#GRID_DIAG_REGRID_CENSOR_VAL = GRID_DIAG_MASK_POLY = MET_BASE/poly/SAO.poly diff --git a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf index ce378efb9b..94f563d660 100644 --- a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf +++ b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf @@ -90,6 +90,13 @@ GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped #OBS_GRID_STAT_FILE_TYPE = GRID_STAT_REGRID_TO_GRID = NONE +#GRID_STAT_REGRID_METHOD = +#GRID_STAT_REGRID_WIDTH = +#GRID_STAT_REGRID_VLD_THRESH = +#GRID_STAT_REGRID_SHAPE = +#GRID_STAT_REGRID_CONVERT = +#GRID_STAT_REGRID_CENSOR_THRESH = +#GRID_STAT_REGRID_CENSOR_VAL = #GRID_STAT_INTERP_FIELD = #GRID_STAT_INTERP_VLD_THRESH = diff --git a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf index 30ed8a2f16..07b392a36c 100644 --- a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf +++ b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf @@ -115,6 +115,9 @@ MODE_REGRID_TO_GRID = NONE #MODE_REGRID_WIDTH = #MODE_REGRID_VLD_THRESH = #MODE_REGRID_SHAPE = +#MODE_REGRID_CONVERT = +#MODE_REGRID_CENSOR_THRESH = +#MODE_REGRID_CENSOR_VAL = MODE_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_NAME}_{CURRENT_OBS_LEVEL} diff --git a/parm/use_cases/met_tool_wrapper/MTD/MTD.conf b/parm/use_cases/met_tool_wrapper/MTD/MTD.conf index 54ef200032..6f615537e0 100644 --- a/parm/use_cases/met_tool_wrapper/MTD/MTD.conf +++ b/parm/use_cases/met_tool_wrapper/MTD/MTD.conf @@ -87,6 +87,14 @@ OBTYPE = MC_PCP MTD_CONFIG_FILE = {PARM_BASE}/met_config/MTDConfig_wrapped MTD_REGRID_TO_GRID = OBS +#MTD_REGRID_METHOD = +#MTD_REGRID_WIDTH = +#MTD_REGRID_VLD_THRESH = +#MTD_REGRID_SHAPE = +#MTD_REGRID_CONVERT = +#MTD_REGRID_CENSOR_THRESH = +#MTD_REGRID_CENSOR_VAL = + MTD_MIN_VOLUME = 2000 diff --git a/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf b/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf index 50646d631a..ff4730d495 100644 --- a/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf +++ b/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf @@ -57,6 +57,13 @@ POINT2GRID_FILE_WINDOW_END = 0 # LOG_POINT2GRID_VERBOSITY = 1 POINT2GRID_REGRID_TO_GRID = G212 +POINT2GRID_REGRID_METHOD = MAX +#POINT2GRID_REGRID_WIDTH = +#POINT2GRID_REGRID_VLD_THRESH = +#POINT2GRID_REGRID_SHAPE = +#POINT2GRID_REGRID_CONVERT = +#POINT2GRID_REGRID_CENSOR_THRESH = +#POINT2GRID_REGRID_CENSOR_VAL = POINT2GRID_INPUT_FIELD =TMP POINT2GRID_INPUT_LEVEL = @@ -65,7 +72,6 @@ POINT2GRID_QC_FLAGS = 0 POINT2GRID_ADP = -POINT2GRID_REGRID_METHOD = MAX POINT2GRID_GAUSSIAN_DX = 81.271 POINT2GRID_GAUSSIAN_RADIUS = 120 diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf index be514f6a9f..3c6de6eb1f 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf @@ -151,6 +151,11 @@ OBTYPE = POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 +#POINT_STAT_REGRID_VLD_THRESH = +#POINT_STAT_REGRID_SHAPE = +#POINT_STAT_REGRID_CONVERT = +#POINT_STAT_REGRID_CENSOR_THRESH = +#POINT_STAT_REGRID_CENSOR_VAL = POINT_STAT_OUTPUT_PREFIX = diff --git a/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf b/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf index bcaa622247..7b5fdb8edf 100644 --- a/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf +++ b/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf @@ -108,6 +108,9 @@ SERIES_ANALYSIS_IS_PAIRED = False #SERIES_ANALYSIS_REGRID_WIDTH = #SERIES_ANALYSIS_REGRID_VLD_THRESH = #SERIES_ANALYSIS_REGRID_SHAPE = +#SERIES_ANALYSIS_REGRID_CONVERT = +#SERIES_ANALYSIS_REGRID_CENSOR_THRESH = +#SERIES_ANALYSIS_REGRID_CENSOR_VAL = #SERIES_ANALYSIS_CLIMO_MEAN_FILE_NAME = #SERIES_ANALYSIS_CLIMO_MEAN_FIELD = diff --git a/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf b/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf index 6d341db0a6..91c4791912 100644 --- a/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf +++ b/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf @@ -76,6 +76,9 @@ MODEL = fv3 #TC_RMW_REGRID_WIDTH = 1 #TC_RMW_REGRID_VLD_THRESH = 0.5 #TC_RMW_REGRID_SHAPE = SQUARE +#TC_RMW_REGRID_CONVERT = +#TC_RMW_REGRID_CENSOR_THRESH = +#TC_RMW_REGRID_CENSOR_VAL = TC_RMW_STORM_ID = AL142016 TC_RMW_BASIN = AL From ee3df42cd189870a3085e3323fa57daaaa81fdc3 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 23 Mar 2023 10:48:46 -0600 Subject: [PATCH 07/12] turn on use case group that is producing differences and add new settings for regrid.censor_thresh/val to test that no differences occur, ci-run-diff --- .github/parm/use_case_groups.json | 2 +- .../GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 056013dd1b..4e5b9992e8 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -32,7 +32,7 @@ { "category": "short_range", "index_list": "2-6", - "run": false + "run": true }, { "category": "short_range", diff --git a/parm/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf b/parm/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf index df6ead3e55..edcfa743fd 100644 --- a/parm/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf +++ b/parm/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf @@ -123,6 +123,8 @@ GEN_ENS_PROD_REGRID_TO_GRID = G211 GEN_ENS_PROD_REGRID_METHOD = MAX GEN_ENS_PROD_REGRID_WIDTH = 27 GEN_ENS_PROD_REGRID_VLD_THRESH = 0.0 +GEN_ENS_PROD_REGRID_CENSOR_THRESH = ==-9999 +GEN_ENS_PROD_REGRID_CENSOR_VAL = 0.0 GEN_ENS_PROD_CENSOR_THRESH = ==-9999 GEN_ENS_PROD_CENSOR_VAL = 0.0 From 0658c1ea9ff77233bda72a2f8357001cfdac9739 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 23 Mar 2023 11:21:35 -0600 Subject: [PATCH 08/12] add logic to use main_vX.Y truth data for branches that include main_vX.Y in the name, ci-run-diff --- .github/jobs/setup_and_run_diff.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/jobs/setup_and_run_diff.py b/.github/jobs/setup_and_run_diff.py index 671f00af52..12d033b4a2 100755 --- a/.github/jobs/setup_and_run_diff.py +++ b/.github/jobs/setup_and_run_diff.py @@ -4,6 +4,7 @@ import sys import subprocess import shlex +import re from docker_utils import VERSION_EXT, get_branch_name @@ -32,8 +33,9 @@ output_data_branch = os.environ.get('GITHUB_BASE_REF') else: branch_name = get_branch_name() - if branch_name.startswith('main_v'): - output_data_branch = branch_name + match = re.match(r'.*(main_v\d+\.\d+).*', branch_name) + if match: + output_data_branch = match.group(1) else: output_data_branch = 'develop' From e50f0abc253c003f41e107038b1bc1570c5f8e01 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 23 Mar 2023 11:59:08 -0600 Subject: [PATCH 09/12] clean up tests --- .../pytests/wrappers/grid_stat/test_grid_stat_wrapper.py | 3 +-- .../wrappers/series_analysis/test_series_analysis.py | 8 ++++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py index 03e578e5a5..cb8c9abc4e 100644 --- a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py @@ -197,8 +197,7 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), ({'GRID_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', }, - { - 'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), ({'GRID_STAT_REGRID_CENSOR_VAL': '12000,5000', }, {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), diff --git a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py index e96abbef6a..e5815a5b4c 100644 --- a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py +++ b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py @@ -105,6 +105,9 @@ def set_minimum_config_settings(config): @pytest.mark.parametrize( 'config_overrides, env_var_values', [ + ({'SERIES_ANALYSIS_REGRID_TO_GRID': 'FCST', }, + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + ({'SERIES_ANALYSIS_REGRID_METHOD': 'NEAREST',}, {'METPLUS_REGRID_DICT': 'regrid = {method = NEAREST;}'}), @@ -126,7 +129,8 @@ def set_minimum_config_settings(config): ({'SERIES_ANALYSIS_REGRID_CENSOR_VAL': '12000,5000', }, {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), - ({'SERIES_ANALYSIS_REGRID_METHOD': 'NEAREST', + ({'SERIES_ANALYSIS_REGRID_TO_GRID': 'FCST', + 'SERIES_ANALYSIS_REGRID_METHOD': 'NEAREST', 'SERIES_ANALYSIS_REGRID_WIDTH': '1', 'SERIES_ANALYSIS_REGRID_VLD_THRESH': '0.5', 'SERIES_ANALYSIS_REGRID_SHAPE': 'SQUARE', @@ -134,7 +138,7 @@ def set_minimum_config_settings(config): 'SERIES_ANALYSIS_REGRID_CENSOR_THRESH': '>12000,<5000', 'SERIES_ANALYSIS_REGRID_CENSOR_VAL': '12000,5000', }, - {'METPLUS_REGRID_DICT': ('regrid = {method = NEAREST;' + {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' 'width = 1;vld_thresh = 0.5;shape = SQUARE;' 'convert(x) = 2*x;' 'censor_thresh = [>12000, <5000];' From c9b5f7926cdc5129374c0954ed6beb929b9d45b2 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 23 Mar 2023 12:00:00 -0600 Subject: [PATCH 10/12] per #2082, call handle_regrid function instead of handling regrid dict to get changes that add convert, censor_thresh, censor_val --- metplus/wrappers/gen_ens_prod_wrapper.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/metplus/wrappers/gen_ens_prod_wrapper.py b/metplus/wrappers/gen_ens_prod_wrapper.py index 21e07305b4..74c87ffcc9 100755 --- a/metplus/wrappers/gen_ens_prod_wrapper.py +++ b/metplus/wrappers/gen_ens_prod_wrapper.py @@ -125,13 +125,7 @@ def create_c_dict(self): metplus_configs=['DESC', 'GEN_ENS_PROD_DESC'], ) - self.add_met_config_dict('regrid', { - 'to_grid': ('string', 'to_grid'), - 'method': ('string', 'uppercase,remove_quotes'), - 'width': 'int', - 'vld_thresh': 'float', - 'shape': ('string', 'uppercase,remove_quotes'), - }) + self.handle_regrid(c_dict) self.add_met_config(name='censor_thresh', data_type='list', From 92f42ab8aa0e850949411bcbfc6711329e219de6 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 23 Mar 2023 12:30:40 -0600 Subject: [PATCH 11/12] minor change to trigger run, ci-run-diff, ci-skip-unit-tests --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 4e5b9992e8..5e4c18fe99 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -32,7 +32,7 @@ { "category": "short_range", "index_list": "2-6", - "run": true + "run": true }, { "category": "short_range", From 0e6e96b729a7c49743665edbd99e5f0a3a273781 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 23 Mar 2023 12:49:17 -0600 Subject: [PATCH 12/12] turn off use case group --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 5e4c18fe99..056013dd1b 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -32,7 +32,7 @@ { "category": "short_range", "index_list": "2-6", - "run": true + "run": false }, { "category": "short_range",