From 44fc9f5a240446ade0679bf64db0b367b0e6b4e5 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 7 Nov 2022 16:55:57 -0700 Subject: [PATCH 01/20] Per #1898, add support for setting optional -diag argument to tc_pairs, refactored logic to set a/b/edeck arguments to remove unnecessary class variables --- metplus/wrappers/tc_pairs_wrapper.py | 112 +++++++++++++++------------ 1 file changed, 61 insertions(+), 51 deletions(-) diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index c86cc01ab8..b31ab0c789 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -81,9 +81,6 @@ def __init__(self, config, instance=None): self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), self.app_name) super().__init__(config, instance=instance) - self.adeck = [] - self.bdeck = [] - self.edeck = [] def create_c_dict(self): """! Create a dictionary containing all the values set in the @@ -191,12 +188,12 @@ def create_c_dict(self): ) c_dict['VALID_BEG'] = self.get_wrapper_or_generic_config('VALID_BEG') c_dict['VALID_END'] = self.get_wrapper_or_generic_config('VALID_END') - c_dict['ADECK_DIR'] = \ - self.config.getdir('TC_PAIRS_ADECK_INPUT_DIR', '') - c_dict['BDECK_DIR'] = \ - self.config.getdir('TC_PAIRS_BDECK_INPUT_DIR', '') - c_dict['EDECK_DIR'] = \ - self.config.getdir('TC_PAIRS_EDECK_INPUT_DIR', '') + c_dict['ADECK_DIR'] = self.config.getdir('TC_PAIRS_ADECK_INPUT_DIR', + '') + c_dict['BDECK_DIR'] = self.config.getdir('TC_PAIRS_BDECK_INPUT_DIR', + '') + c_dict['EDECK_DIR'] = self.config.getdir('TC_PAIRS_EDECK_INPUT_DIR', + '') c_dict['OUTPUT_DIR'] = self.config.getdir('TC_PAIRS_OUTPUT_DIR', '') if not c_dict['OUTPUT_DIR']: self.log_error('TC_PAIRS_OUTPUT_DIR must be set') @@ -225,22 +222,28 @@ def create_c_dict(self): 'TC_PAIRS_DLAND_FILE') c_dict['ADECK_TEMPLATE'] = ( - self.config.getraw('filename_templates', + self.config.getraw('config', 'TC_PAIRS_ADECK_TEMPLATE', '') ) c_dict['BDECK_TEMPLATE'] = ( - self.config.getraw('filename_templates', + self.config.getraw('config', 'TC_PAIRS_BDECK_TEMPLATE') ) c_dict['EDECK_TEMPLATE'] = ( - self.config.getraw('filename_templates', + self.config.getraw('config', 'TC_PAIRS_EDECK_TEMPLATE', '') ) + # read optional -diag argument variables + c_dict['DIAG_TEMPLATE'] = ( + self.config.getraw('config', 'TC_PAIRS_DIAG_INPUT_TEMPLATE') + ) + c_dict['DIAG_DIR'] = self.config.getdir('TC_PAIRS_DIAG_INPUT_DIR', '') + # handle output template output_template = ( self.config.getraw('config', 'TC_PAIRS_OUTPUT_TEMPLATE') @@ -578,6 +581,7 @@ def process_data(self, basin, cyclone, time_info): # find corresponding adeck or edeck files for bdeck_file in bdeck_files: + self.clear() self.logger.debug(f'Found BDECK: {bdeck_file}') # get current basin/cyclone that corresponds to bdeck @@ -617,9 +621,11 @@ def process_data(self, basin, cyclone, time_info): bdeck_list = self.reformat_files(bdeck_list, 'B', time_info) edeck_list = self.reformat_files(edeck_list, 'E', time_info) - self.adeck = adeck_list - self.bdeck = bdeck_list - self.edeck = edeck_list + self.args.append(f"-bdeck {' '.join(bdeck_list)}") + if adeck_list: + self.args.append(f"-adeck {' '.join(adeck_list)}") + if edeck_list: + self.args.append(f"-edeck {' '.join(edeck_list)}") # change wildcard basin/cyclone to 'all' for output filename if current_basin == self.WILDCARDS['basin']: @@ -630,6 +636,11 @@ def process_data(self, basin, cyclone, time_info): time_storm_info = time_info.copy() time_storm_info['basin'] = current_basin time_storm_info['cyclone'] = current_cyclone + + # find -diag file if requested + if not self._handle_diag_file(time_storm_info): + return [] + if not self.find_and_check_output_file(time_info=time_storm_info, check_extension='.tcst'): return [] @@ -811,39 +822,15 @@ def get_command(self): output file, as there is a default. Build command to run from arguments """ - if self.app_path is None: - self.log_error("No app path specified. You must use a subclass") - return None - - if not self.adeck and not self.edeck: - self.log_error('Neither ADECK nor EDECK files set') - return None - - if not self.bdeck: - self.log_error('BDECK file not set') - return None - - config_file = self.c_dict['CONFIG_FILE'] - if not config_file: - self.log_error('Config file not set') - return None - output_path = self.get_output_path() if not output_path: self.log_error('Output path not set') return None - cmd = '{} -v {}'.format(self.app_path, self.c_dict['VERBOSITY']) - cmd += ' -bdeck {}'.format(' '.join(self.bdeck)) - - if self.adeck: - cmd += ' -adeck {}'.format(' '.join(self.adeck)) - - if self.edeck: - cmd += ' -edeck {}'.format(' '.join(self.edeck)) - - cmd += ' -config {} -out {}'.format(config_file, output_path) - + cmd = (f"{self.app_path} -v {self.c_dict['VERBOSITY']}" + f" {' '.join(self.args)}" + f" -config {self.c_dict['CONFIG_FILE']}" + f" -out {output_path}") return cmd def read_modify_write_file(self, in_csvfile, storm_month, missing_values, @@ -917,20 +904,23 @@ def _read_all_files(self, input_dict): self.c_dict['BASIN'] = self.c_dict.get('BASIN_LIST', '') # Set up the environment variable to be used in the tc_pairs Config - self.bdeck = [self.c_dict['BDECK_DIR']] + self.args.append(f"-bdeck {' '.join(self.c_dict['BDECK_DIR'])}") + #self.bdeck = [self.c_dict['BDECK_DIR']] - adeck_dir = self.c_dict['ADECK_DIR'] - edeck_dir = self.c_dict['EDECK_DIR'] + if self.c_dict['ADECK_DIR']: + self.args.append(f"-adeck {' '.join(self.c_dict['ADECK_DIR'])}") - if adeck_dir: - self.adeck = [adeck_dir] - - if edeck_dir: - self.edeck = [edeck_dir] + if self.c_dict['EDECK_DIR']: + self.args.append(f"-edeck {' '.join(self.c_dict['EDECK_DIR'])}") # get output filename from template time_info = ti_calculate(input_dict) time_storm_info = self._add_storm_info_to_dict(time_info) + + # handle -diag file if requested + if not self._handle_diag_file(time_storm_info): + return [] + if not self.find_and_check_output_file(time_info=time_storm_info, check_extension='.tcst'): return [] @@ -940,6 +930,26 @@ def _read_all_files(self, input_dict): self.build() return self.all_commands + def _handle_diag_file(self, time_info): + """! Get optional -diag argument file path if requested. + + @param time_info dictionary containing values to substitute into + filename template + @returns True if file was found successfully or no file was requested. + False if the file does not exist. + """ + if not self.c_dict['DIAG_TEMPLATE']: + return True + + filepath = os.path.join(self.c_dict['DIAG_DIR'], + self.c_dict['DIAG_TEMPLATE']) + filepath = do_string_sub(filepath, **time_info) + if not os.path.exists(filepath): + self.log_error(f'Diag file does not exist: {filepath}') + return False + + self.args.append(f'-diag {filepath}') + return True def _add_storm_info_to_dict(self, time_info): """! Read from self.c_dict and add storm information to dictionary. From b89885a2621bb18adfbddaa87d38b441a2069ce3 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 8 Nov 2022 11:57:13 -0700 Subject: [PATCH 02/20] added new fake data files to test that correct commands are built for multiple calls to tc_pairs --- internal/tests/data/tc_pairs/adeck/amlq2014123118.gfso.0106 | 0 internal/tests/data/tc_pairs/bdeck/bmlq2014123118.gfso.0106 | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 internal/tests/data/tc_pairs/adeck/amlq2014123118.gfso.0106 create mode 100644 internal/tests/data/tc_pairs/bdeck/bmlq2014123118.gfso.0106 diff --git a/internal/tests/data/tc_pairs/adeck/amlq2014123118.gfso.0106 b/internal/tests/data/tc_pairs/adeck/amlq2014123118.gfso.0106 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/tests/data/tc_pairs/bdeck/bmlq2014123118.gfso.0106 b/internal/tests/data/tc_pairs/bdeck/bmlq2014123118.gfso.0106 new file mode 100644 index 0000000000..e69de29bb2 From 34216c5de9a4eed73299c0d74c782727f6984510 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 8 Nov 2022 11:58:17 -0700 Subject: [PATCH 03/20] cleaned up tests and added check to ensure command is generated correctly when running once processing a directory --- .../wrappers/tc_pairs/test_tc_pairs_wrapper.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index b7ad438f2a..65a4e5f9d0 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -265,9 +265,6 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, config.set('config', 'TC_PAIRS_EDECK_TEMPLATE', 'storm_id_e{basin?fmt=%s}{cyclone?fmt=%s}{date?fmt=%Y}.dat') - # LOOP_ORDER processes runs once, times runs once per time - config.set('config', 'LOOP_ORDER', 'processes') - # set config variable overrides for key, value in config_overrides.items(): config.set('config', key, value) @@ -481,9 +478,6 @@ def test_tc_pairs_loop_order_processes(metplus_config, loop_by, config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir) - # LOOP_ORDER processes runs once, times runs once per time - config.set('config', 'LOOP_ORDER', 'processes') - # set config variable overrides for key, value in config_overrides.items(): config.set('config', key, value) @@ -584,10 +578,6 @@ def test_tc_pairs_read_all_files(metplus_config, loop_by, config_overrides, config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir) - - # LOOP_ORDER processes runs once, times runs once per time - config.set('config', 'LOOP_ORDER', 'processes') - config.set('config', 'TC_PAIRS_READ_ALL_FILES', True) config.set('config', 'TC_PAIRS_OUTPUT_TEMPLATE', '') @@ -626,6 +616,8 @@ def test_tc_pairs_read_all_files(metplus_config, loop_by, config_overrides, assert len(all_cmds) == len(expected_cmds) for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd # check that environment variables were set properly for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: match = next((item for item in env_vars if From a8c4f02d63e113f533de1620ea2e6b187db62ab6 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 8 Nov 2022 11:58:55 -0700 Subject: [PATCH 04/20] expanded unit tests to check that -diag arguments are set properly and that multiple commands are generated properly --- .../tc_pairs/test_tc_pairs_wrapper.py | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index 65a4e5f9d0..b89d6c9f5e 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -461,11 +461,35 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, # 41 match_points ('VALID', {'TC_PAIRS_MATCH_POINTS': 'False', }, {'METPLUS_MATCH_POINTS': 'match_points = FALSE;'}), + # 42 -diag argument + ('VALID', { + 'TC_PAIRS_DIAG_TEMPLATE1': '/some/path/{valid?fmt=%Y%m%d%H}.dat', + 'TC_PAIRS_DIAG_SOURCE1': 'TCDIAG', + }, + {'DIAG_ARG': '-diag TCDIAG /some/path/2014121318.dat'}), + # 42 -diag argument with models + ('VALID', { + 'TC_PAIRS_DIAG_TEMPLATE1': '/some/path/{valid?fmt=%Y%m%d%H}.dat', + 'TC_PAIRS_DIAG_SOURCE1': 'TCDIAG', + 'TC_PAIRS_DIAG_MODELS1': 'OFCL, SHIP', + }, + {'DIAG_ARG': '-diag TCDIAG /some/path/2014121318.dat model=OFCL,SHIP'}), + # 43 2 -diag arguments + ('VALID', { + 'TC_PAIRS_DIAG_TEMPLATE1': '/some/path/{valid?fmt=%Y%m%d%H}.dat', + 'TC_PAIRS_DIAG_SOURCE1': 'TCDIAG', + 'TC_PAIRS_DIAG_TEMPLATE2': '/some/path/rt_{valid?fmt=%Y%m%d%H}.dat', + 'TC_PAIRS_DIAG_SOURCE2': 'LSDIAG_RT', + 'TC_PAIRS_DIAG_MODELS2': 'OFCL, SHIP', + }, + {'DIAG_ARG': ('-diag TCDIAG /some/path/2014121318.dat ' + '-diag LSDIAG_RT /some/path/rt_2014121318.dat ' + 'model=OFCL,SHIP')}), ] ) @pytest.mark.wrapper -def test_tc_pairs_loop_order_processes(metplus_config, loop_by, - config_overrides, env_var_values): +def test_tc_pairs_run(metplus_config, loop_by, config_overrides, + env_var_values): config = metplus_config remove_beg = remove_end = remove_match_points = False @@ -507,13 +531,27 @@ def test_tc_pairs_loop_order_processes(metplus_config, loop_by, verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" config_file = wrapper.c_dict.get('CONFIG_FILE') out_dir = wrapper.c_dict.get('OUTPUT_DIR') + diag_arg = '' + if 'DIAG_ARG' in env_var_values: + diag_arg = f" {env_var_values['DIAG_ARG']}" + expected_cmds = [(f"{app_path} {verbosity} " f"-bdeck {bdeck_dir}/bmlq2014123118.gfso.0104 " - f"-adeck {adeck_dir}/amlq2014123118.gfso.0104 " - f"-config {config_file} " + f"-adeck {adeck_dir}/amlq2014123118.gfso.0104{diag_arg}" + f" -config {config_file} " f"-out {out_dir}/mlq2014121318.gfso.0104"), ] + # add 2nd command for cyclone 106 unless specific cyclones are requested + if 'TC_PAIRS_CYCLONE' not in config_overrides: + expected_cmds.append( + (f"{app_path} {verbosity} " + f"-bdeck {bdeck_dir}/bmlq2014123118.gfso.0106 " + f"-adeck {adeck_dir}/amlq2014123118.gfso.0106{diag_arg}" + f" -config {config_file} " + f"-out {out_dir}/mlq2014121318.gfso.0106") + ) + all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") assert len(all_cmds) == len(expected_cmds) From b28491399aef16e1d63cf50751ef3f01e73b6db4 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 8 Nov 2022 11:59:26 -0700 Subject: [PATCH 05/20] per #1898, added support for setting -diag arguments and rearranged functions so they are in a more logical order --- metplus/wrappers/tc_pairs_wrapper.py | 190 ++++++++++++++++----------- 1 file changed, 114 insertions(+), 76 deletions(-) diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index b31ab0c789..c2b9b3aca2 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -23,7 +23,7 @@ from ..util import getlist, get_lead_sequence, skip_time, mkdir_p from ..util import ti_calculate from ..util import do_string_sub -from ..util import get_tags +from ..util import get_tags, find_indices_in_config_section from ..util.met_config import add_met_config_dict_list from ..util import time_generator, log_runtime_banner, add_to_time_input from . import CommandBuilder @@ -92,6 +92,8 @@ def create_c_dict(self): c_dict['VERBOSITY'] = self.config.getstr('config', 'LOG_TC_PAIRS_VERBOSITY', c_dict['VERBOSITY']) + c_dict['ALLOW_MULTIPLE_FILES'] = True + c_dict['MISSING_VAL_TO_REPLACE'] = ( self.config.getstr('config', 'TC_PAIRS_MISSING_VAL_TO_REPLACE', '-99') @@ -164,7 +166,7 @@ def create_c_dict(self): data_type='list', metplus_configs=['TC_PAIRS_STORM_NAME']) - self.handle_consensus() + self._handle_consensus() self.add_met_config(name='check_dup', data_type='bool') @@ -239,10 +241,7 @@ def create_c_dict(self): ) # read optional -diag argument variables - c_dict['DIAG_TEMPLATE'] = ( - self.config.getraw('config', 'TC_PAIRS_DIAG_INPUT_TEMPLATE') - ) - c_dict['DIAG_DIR'] = self.config.getdir('TC_PAIRS_DIAG_INPUT_DIR', '') + self._handle_diag(c_dict) # handle output template output_template = ( @@ -294,60 +293,6 @@ def create_c_dict(self): True) return c_dict - def _read_storm_info(self, c_dict): - """! Read config variables that specify the storms to process. Report - an error if attempting to filter by storm_id if also specifying - basin or cyclone. Sets c_dict depending on what is set: STORM_ID_LIST - if filtering by storm_id, or CYCLONE_LIST and BASIN_LIST otherwise - - @param c_dict dictionary to populate with values from config - @returns None - """ - storm_id_list = getlist( - self.config.getraw('config', 'TC_PAIRS_STORM_ID', '') - ) - cyclone_list = getlist( - self.config.getraw('config', 'TC_PAIRS_CYCLONE', '') - ) - basin_list = getlist( - self.config.getraw('config', 'TC_PAIRS_BASIN', '') - ) - - if storm_id_list: - # if using storm id and any other filter is set, report an error - if basin_list: - self.log_error('Cannot filter by both BASIN and STORM_ID') - - if cyclone_list: - self.log_error('Cannot filter by both CYCLONE and STORM_ID') - - c_dict['STORM_ID_LIST'] = storm_id_list - return - - # if storm_id is not used, set cyclone and basin lists if they are set - if cyclone_list: - c_dict['CYCLONE_LIST'] = cyclone_list - - if basin_list: - c_dict['BASIN_LIST'] = basin_list - - def handle_consensus(self): - dict_items = { - 'name': 'string', - 'members': 'list', - 'required': ('list', 'remove_quotes'), - 'min_req': 'int', - } - return_code = add_met_config_dict_list(config=self.config, - app_name=self.app_name, - output_dict=self.env_var_dict, - dict_name='consensus', - dict_items=dict_items) - if not return_code: - self.isOK = False - - return return_code - def run_all_times(self): """! Build up the command to invoke the MET tool tc_pairs. """ @@ -416,6 +361,95 @@ def run_at_time_loop_string(self, time_info): return self._loop_basin_and_cyclone(time_info) + def _read_storm_info(self, c_dict): + """! Read config variables that specify the storms to process. Report + an error if attempting to filter by storm_id if also specifying + basin or cyclone. Sets c_dict depending on what is set: STORM_ID_LIST + if filtering by storm_id, or CYCLONE_LIST and BASIN_LIST otherwise + + @param c_dict dictionary to populate with values from config + @returns None + """ + storm_id_list = getlist( + self.config.getraw('config', 'TC_PAIRS_STORM_ID', '') + ) + cyclone_list = getlist( + self.config.getraw('config', 'TC_PAIRS_CYCLONE', '') + ) + basin_list = getlist( + self.config.getraw('config', 'TC_PAIRS_BASIN', '') + ) + + if storm_id_list: + # if using storm id and any other filter is set, report an error + if basin_list: + self.log_error('Cannot filter by both BASIN and STORM_ID') + + if cyclone_list: + self.log_error('Cannot filter by both CYCLONE and STORM_ID') + + c_dict['STORM_ID_LIST'] = storm_id_list + return + + # if storm_id is not used, set cyclone and basin lists if they are set + if cyclone_list: + c_dict['CYCLONE_LIST'] = cyclone_list + + if basin_list: + c_dict['BASIN_LIST'] = basin_list + + def _handle_consensus(self): + dict_items = { + 'name': 'string', + 'members': 'list', + 'required': ('list', 'remove_quotes'), + 'min_req': 'int', + } + return_code = add_met_config_dict_list(config=self.config, + app_name=self.app_name, + output_dict=self.env_var_dict, + dict_name='consensus', + dict_items=dict_items) + if not return_code: + self.isOK = False + + return return_code + + def _handle_diag(self, c_dict): + diag_indices = list( + find_indices_in_config_section(r'TC_PAIRS_DIAG_TEMPLATE(\d+)$', + self.config, + index_index=1).keys() + ) + if not diag_indices: + return + + diag_info_list = [] + for idx in diag_indices: + template = ( + self.config.getraw('config', f'TC_PAIRS_DIAG_TEMPLATE{idx}') + ) + diag_dir = ( + self.config.getdir(f'TC_PAIRS_DIAG_DIR{idx}', '') + ) + if diag_dir: + template = os.path.join(diag_dir, template) + + source = ( + self.config.getraw('config', f'TC_PAIRS_DIAG_SOURCE{idx}') + ) + models = getlist( + self.config.getraw('config', f'TC_PAIRS_DIAG_MODELS{idx}') + ) + diag_info = { + 'template': template, + 'source': source, + 'models': models, + } + diag_info_list.append(diag_info) + + c_dict['DIAG_INFO_LIST'] = diag_info_list + def _loop_storm_ids(self, time_info): for storm_id in self.c_dict['STORM_ID_LIST']: @@ -638,7 +672,7 @@ def process_data(self, basin, cyclone, time_info): time_storm_info['cyclone'] = current_cyclone # find -diag file if requested - if not self._handle_diag_file(time_storm_info): + if not self._get_diag_file(time_storm_info): return [] if not self.find_and_check_output_file(time_info=time_storm_info, @@ -904,21 +938,20 @@ def _read_all_files(self, input_dict): self.c_dict['BASIN'] = self.c_dict.get('BASIN_LIST', '') # Set up the environment variable to be used in the tc_pairs Config - self.args.append(f"-bdeck {' '.join(self.c_dict['BDECK_DIR'])}") - #self.bdeck = [self.c_dict['BDECK_DIR']] + self.args.append(f"-bdeck {self.c_dict['BDECK_DIR']}") if self.c_dict['ADECK_DIR']: - self.args.append(f"-adeck {' '.join(self.c_dict['ADECK_DIR'])}") + self.args.append(f"-adeck {self.c_dict['ADECK_DIR']}") if self.c_dict['EDECK_DIR']: - self.args.append(f"-edeck {' '.join(self.c_dict['EDECK_DIR'])}") + self.args.append(f"-edeck {self.c_dict['EDECK_DIR']}") # get output filename from template time_info = ti_calculate(input_dict) time_storm_info = self._add_storm_info_to_dict(time_info) # handle -diag file if requested - if not self._handle_diag_file(time_storm_info): + if not self._get_diag_file(time_storm_info): return [] if not self.find_and_check_output_file(time_info=time_storm_info, @@ -930,7 +963,7 @@ def _read_all_files(self, input_dict): self.build() return self.all_commands - def _handle_diag_file(self, time_info): + def _get_diag_file(self, time_info): """! Get optional -diag argument file path if requested. @param time_info dictionary containing values to substitute into @@ -938,17 +971,22 @@ def _handle_diag_file(self, time_info): @returns True if file was found successfully or no file was requested. False if the file does not exist. """ - if not self.c_dict['DIAG_TEMPLATE']: + if not self.c_dict.get('DIAG_INFO_LIST'): return True + self.log_error(self.c_dict.get('DIAG_INFO_LIST')) + for diag_info in self.c_dict.get('DIAG_INFO_LIST'): + self.c_dict['DIAG_INPUT_TEMPLATE'] = diag_info['template'] + filepaths = self.find_data(time_info, data_type='DIAG', + return_list=True) + if not filepaths: + self.log_error('Could not get -diag files') + return False + + arg = f"-diag {diag_info['source']} {' '.join(filepaths)}" + if diag_info['models']: + arg = f"{arg} model={','.join(diag_info['models'])}" + self.args.append(arg) - filepath = os.path.join(self.c_dict['DIAG_DIR'], - self.c_dict['DIAG_TEMPLATE']) - filepath = do_string_sub(filepath, **time_info) - if not os.path.exists(filepath): - self.log_error(f'Diag file does not exist: {filepath}') - return False - - self.args.append(f'-diag {filepath}') return True def _add_storm_info_to_dict(self, time_info): From 748a0dcff486869fbda9ec4a33275879661c7a51 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 8 Nov 2022 13:10:45 -0700 Subject: [PATCH 06/20] added a fix to be able to import when running the doc_util script directly --- metplus/util/doc_util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index d32e42f778..2a82fa8dd5 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -3,7 +3,11 @@ import sys import os -from . import LOWER_TO_WRAPPER_NAME +try: + from . import LOWER_TO_WRAPPER_NAME +except ImportError: + sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) + from constants import LOWER_TO_WRAPPER_NAME def get_wrapper_name(process_name): From eab66e83296774e0ada7b9debdaeec0e2f9051fd Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 8 Nov 2022 16:33:34 -0700 Subject: [PATCH 07/20] per #1898, add support for setting diag_name and diag_convert_map in TCPairs wrapped config file --- docs/Users_Guide/glossary.rst | 28 ++++++++++++++--- docs/Users_Guide/wrappers.rst | 31 +++++++++++++++++++ .../tc_pairs/test_tc_pairs_wrapper.py | 24 ++++++++++++-- metplus/wrappers/tc_pairs_wrapper.py | 26 +++++++++++++++- parm/met_config/TCPairsConfig_wrapped | 7 +++++ .../TCPairs/TCPairs_extra_tropical.conf | 7 ++++- .../TCPairs/TCPairs_tropical.conf | 5 +++ 7 files changed, 120 insertions(+), 8 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 398383bd2a..21b13a018c 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -7569,22 +7569,22 @@ METplus Configuration Glossary | *Used by:* PB2NC TC_PAIRS_CONSENSUS_NAME - Specify the value for nth 'consensus.name' in the MET configuration file for TCPairs. + Specify the value for the nth 'consensus.name' in the MET configuration file for TCPairs. | *Used by:* TCPairs TC_PAIRS_CONSENSUS_MEMBERS - Specify the value for nth 'consensus.members' in the MET configuration file for TCPairs. + Specify the value for the nth 'consensus.members' in the MET configuration file for TCPairs. | *Used by:* TCPairs TC_PAIRS_CONSENSUS_REQUIRED - Specify the value for nth 'consensus.required' in the MET configuration file for TCPairs. + Specify the value for the nth 'consensus.required' in the MET configuration file for TCPairs. | *Used by:* TCPairs TC_PAIRS_CONSENSUS_MIN_REQ - Specify the value for nth 'consensus.min_req' in the MET configuration file for TCPairs. + Specify the value for the nth 'consensus.min_req' in the MET configuration file for TCPairs. | *Used by:* TCPairs @@ -10093,3 +10093,23 @@ METplus Configuration Glossary See :term:`STAT_ANALYSIS_FCST_VALID_END`. | *Used by:* StatAnalysis + + TC_PAIRS_DIAG_NAME + Specify the value for 'diag_name' in the MET configuration file for TCPairs. + + | *Used by:* TCPairs + + TC_PAIRS_DIAG_CONVERT_MAP_SOURCE + Specify the value for the nth 'diag_convert_map.source' in the MET configuration file for TCPairs. + + | *Used by:* TCPairs + + TC_PAIRS_DIAG_CONVERT_MAP_KEY + Specify the value for the nth 'diag_convert_map.key' in the MET configuration file for TCPairs. + + | *Used by:* TCPairs + + TC_PAIRS_DIAG_CONVERT_MAP_CONVERT + Specify the value for the nth 'diag_convert_map.convert' in the MET configuration file for TCPairs. + + | *Used by:* TCPairs diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 2c35d8d20e..6d216db8b1 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -8210,6 +8210,10 @@ METplus Configuration | :term:`TC_PAIRS_CHECK_DUP` | :term:`TC_PAIRS_INTERP12` | :term:`TC_PAIRS_MATCH_POINTS` +| :term:`TC_PAIRS_DIAG_NAME` +| :term:`TC_PAIRS_DIAG_CONVERT_MAP_SOURCE` +| :term:`TC_PAIRS_DIAG_CONVERT_MAP_KEY` +| :term:`TC_PAIRS_DIAG_CONVERT_MAP_CONVERT` | .. warning:: **DEPRECATED:** @@ -8492,6 +8496,33 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`TC_PAIRS_INTERP12` - interp12 +**${METPLUS_DIAG_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_PAIRS_DIAG_NAME` + - diag_name + +**${METPLUS_DIAG_CONVERT_MAP_LIST}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_PAIRS_DIAG_CONVERT_MAP_SOURCE` + - diag_convert_map.source + * - :term:`TC_PAIRS_DIAG_CONVERT_MAP_KEY` + - diag_convert_map.key + * - :term:`TC_PAIRS_DIAG_CONVERT_MAP_CONVERT` + - diag_convert_map.convert + + .. _tcrmw_wrapper: TCRMW diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index b89d6c9f5e..b0316f6650 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -467,14 +467,14 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, 'TC_PAIRS_DIAG_SOURCE1': 'TCDIAG', }, {'DIAG_ARG': '-diag TCDIAG /some/path/2014121318.dat'}), - # 42 -diag argument with models + # 43 -diag argument with models ('VALID', { 'TC_PAIRS_DIAG_TEMPLATE1': '/some/path/{valid?fmt=%Y%m%d%H}.dat', 'TC_PAIRS_DIAG_SOURCE1': 'TCDIAG', 'TC_PAIRS_DIAG_MODELS1': 'OFCL, SHIP', }, {'DIAG_ARG': '-diag TCDIAG /some/path/2014121318.dat model=OFCL,SHIP'}), - # 43 2 -diag arguments + # 44 2 -diag arguments ('VALID', { 'TC_PAIRS_DIAG_TEMPLATE1': '/some/path/{valid?fmt=%Y%m%d%H}.dat', 'TC_PAIRS_DIAG_SOURCE1': 'TCDIAG', @@ -485,6 +485,26 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, {'DIAG_ARG': ('-diag TCDIAG /some/path/2014121318.dat ' '-diag LSDIAG_RT /some/path/rt_2014121318.dat ' 'model=OFCL,SHIP')}), + # 45 + ('VALID', {'TC_PAIRS_DIAG_NAME': 'TC_DIAG, TC_DIAG2', }, + {'METPLUS_DIAG_NAME': 'diag_name = ["TC_DIAG", "TC_DIAG2"];'}), + # 46 + ('VALID', {'TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE': 'TCDIAG', }, + {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{source = "TCDIAG";}];'}), + # 47 + ('VALID', {'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', }, + {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{key = ["(10C)", "(10KT)", "(10M/S)"];}];'}), + # 48 + ('VALID', {'TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT': 'x/10', }, + {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{convert(x) = x/10;}];'}), + # 49 + ('VALID', { + 'TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE': 'TCDIAG', + 'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', + 'TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT': 'x/10', + }, + {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{source = "TCDIAG";key = ["(10C)", "(10KT)", "(10M/S)"];convert(x) = x/10;}];'}), + ] ) @pytest.mark.wrapper diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index c2b9b3aca2..9c52127ba8 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -63,6 +63,8 @@ class TCPairsWrapper(CommandBuilder): 'METPLUS_CHECK_DUP', 'METPLUS_INTERP12', 'METPLUS_MATCH_POINTS', + 'METPLUS_DIAG_NAME', + 'METPLUS_DIAG_CONVERT_MAP_LIST', ] WILDCARDS = { @@ -178,6 +180,10 @@ def create_c_dict(self): self.add_met_config(name='match_points', data_type='bool') + self.add_met_config(name='diag_name', data_type='list') + + self._handle_diag_convert_map() + # if unset, set match_points to TRUE to match old default in wrapped if not self.env_var_dict.get('METPLUS_MATCH_POINTS'): self.env_var_dict['METPLUS_MATCH_POINTS'] = 'match_points = TRUE;' @@ -413,7 +419,25 @@ def _handle_consensus(self): if not return_code: self.isOK = False - return return_code + def _handle_diag_convert_map(self): + dict_items = { + 'source': 'string', + 'key': 'list', + 'convert': ('string', 'remove_quotes'), + } + return_code = add_met_config_dict_list(config=self.config, + app_name=self.app_name, + output_dict=self.env_var_dict, + dict_name='diag_convert_map', + dict_items=dict_items) + if not return_code: + self.isOK = False + return + + # change "convert =" to "convert(x) =" + value = self.env_var_dict.get('METPLUS_DIAG_CONVERT_MAP_LIST', '') + value = value.replace('convert =', 'convert(x) =') + self.env_var_dict['METPLUS_DIAG_CONVERT_MAP_LIST'] = value def _handle_diag(self, c_dict): diag_indices = list( diff --git a/parm/met_config/TCPairsConfig_wrapped b/parm/met_config/TCPairsConfig_wrapped index 34f85e1a2f..4ee11dff9b 100644 --- a/parm/met_config/TCPairsConfig_wrapped +++ b/parm/met_config/TCPairsConfig_wrapped @@ -143,6 +143,13 @@ watch_warn = { time_offset = -14400; } + +//diag_name = +${METPLUS_DIAG_NAME} + +//diag_convert_map = { +${METPLUS_DIAG_CONVERT_MAP_LIST} + // // Indicate a version number for the contents of this configuration file. // The value should generally not be modified. diff --git a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf index 092a7f985c..92c98e5525 100644 --- a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf +++ b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf @@ -105,4 +105,9 @@ TC_PAIRS_MISSING_VAL = -9999 #TC_PAIRS_INTERP12 = -#TC_PAIRS_MATCH_POINTS = \ No newline at end of file +#TC_PAIRS_MATCH_POINTS = + +#TC_PAIRS_DIAG_NAME = +#TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE = +#TC_PAIRS_DIAG_CONVERT_MAP1_KEY = +#TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT = diff --git a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf index f7b103720a..ca20066421 100644 --- a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf +++ b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf @@ -100,3 +100,8 @@ TC_PAIRS_DLAND_FILE = MET_BASE/tc_data/dland_global_tenth_degree.nc #TC_PAIRS_INTERP12 = #TC_PAIRS_MATCH_POINTS = + +#TC_PAIRS_DIAG_NAME = +#TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE = +#TC_PAIRS_DIAG_CONVERT_MAP1_KEY = +#TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT = \ No newline at end of file From 756aab76753ea4022ddb86bef585acdd0a09653b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 9 Nov 2022 10:28:45 -0700 Subject: [PATCH 08/20] fixed bug in formatting list of dictionaries in wrapped MET config files, fixed incorrect unit tests, added unit tests to ensure multiple diag_convert_map list items are formatted properly --- .../tc_pairs/test_tc_pairs_wrapper.py | 21 +++++++++++++++++-- metplus/util/met_config.py | 2 +- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index b0316f6650..7c56e9656c 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -356,7 +356,7 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, {'METPLUS_CONSENSUS_LIST': ( 'consensus = [' '{name = "name1";members = ["member1a", "member1b"];' - 'required = [true, false];min_req = 1;}' + 'required = [true, false];min_req = 1;},' '{name = "name2";members = ["member2a", "member2b"];' 'required = [false, true];min_req = 2;}];' )}), @@ -439,7 +439,7 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, {'METPLUS_CONSENSUS_LIST': ( 'consensus = [' '{name = "name1";members = ["member1a", "member1b"];' - 'required = [true, false];min_req = 1;}' + 'required = [true, false];min_req = 1;},' '{name = "name2";members = ["member2a", "member2b"];' 'required = [false, true];min_req = 2;}];' )}), @@ -504,6 +504,23 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, 'TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT': 'x/10', }, {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{source = "TCDIAG";key = ["(10C)", "(10KT)", "(10M/S)"];convert(x) = x/10;}];'}), + # 50 + ('VALID', { + 'TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE': 'TCDIAG', + 'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', + 'TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT': 'x/10', + 'TC_PAIRS_DIAG_CONVERT_MAP2_SOURCE': 'LSDIAG_RT', + 'TC_PAIRS_DIAG_CONVERT_MAP2_KEY': 'LAT,LON,CSST,RSST,DSST,DSTA', + 'TC_PAIRS_DIAG_CONVERT_MAP2_CONVERT': 'x/100', + }, + {'METPLUS_DIAG_CONVERT_MAP_LIST': ('diag_convert_map = [{source = ' + '"TCDIAG";key = ["(10C)", ' + '"(10KT)", "(10M/S)"];' + 'convert(x) = x/10;},' + '{source = "LSDIAG_RT";key = [' + '"LAT", "LON", "CSST", "RSST", ' + '"DSST", "DSTA"];' + 'convert(x) = x/100;}];')}), ] ) diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py index ddb4ef71ca..194fd763cb 100644 --- a/metplus/util/met_config.py +++ b/metplus/util/met_config.py @@ -383,7 +383,7 @@ def format_met_config(data_type, c_dict, name, keys=None): # add square braces if list if 'list' in data_type: - output = f"[{output}];" + output = f"[{','.join(values)}];" # if name is not empty, add variable name and equals sign if name: From 2ea9ee344fbdc5700abc1efa087c6d92f9ad1fa0 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 9 Nov 2022 11:30:55 -0700 Subject: [PATCH 09/20] fixed bug introduced by last commit regarding a list of dictionaries --- metplus/util/met_config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py index 194fd763cb..06915d1c6a 100644 --- a/metplus/util/met_config.py +++ b/metplus/util/met_config.py @@ -376,14 +376,19 @@ def format_met_config(data_type, c_dict, name, keys=None): if not values: return '' - output = ''.join(values) + # separate items with commas if data type is list (not dictlist) + if data_type == 'list': + output = ','.join(values) + else: + output = ''.join(values) + # add curly braces if dictionary if 'dict' in data_type: output = f"{{{output}}}" # add square braces if list if 'list' in data_type: - output = f"[{','.join(values)}];" + output = f"[{output}];" # if name is not empty, add variable name and equals sign if name: From c377e30e3c374e44d7257871acfcc12129e87670 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 9 Nov 2022 11:31:15 -0700 Subject: [PATCH 10/20] added comment to note what is supported for nmep_smooth.type --- metplus/wrappers/gen_ens_prod_wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metplus/wrappers/gen_ens_prod_wrapper.py b/metplus/wrappers/gen_ens_prod_wrapper.py index ae137a1fc2..21e07305b4 100755 --- a/metplus/wrappers/gen_ens_prod_wrapper.py +++ b/metplus/wrappers/gen_ens_prod_wrapper.py @@ -195,6 +195,8 @@ def create_c_dict(self): 'vld_thresh': 'float', }) + # note: nmep_smooth.type is a list of dictionaries, + # but only setting 1 dictionary in the list is supported self.add_met_config_dict('nmep_smooth', { 'vld_thresh': 'float', 'shape': ('string', 'uppercase,remove_quotes'), From 29aed7cc30f44577819696fbc7479b277eeb2033 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 9 Nov 2022 11:55:55 -0700 Subject: [PATCH 11/20] added comments to describe each new unit test and number to easily track down which test fails --- .../wrappers/tc_pairs/test_tc_pairs_wrapper.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index 7c56e9656c..5b1492917b 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -485,26 +485,26 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, {'DIAG_ARG': ('-diag TCDIAG /some/path/2014121318.dat ' '-diag LSDIAG_RT /some/path/rt_2014121318.dat ' 'model=OFCL,SHIP')}), - # 45 + # 45 diag_name ('VALID', {'TC_PAIRS_DIAG_NAME': 'TC_DIAG, TC_DIAG2', }, {'METPLUS_DIAG_NAME': 'diag_name = ["TC_DIAG", "TC_DIAG2"];'}), - # 46 + # 46 diag_convert_map source ('VALID', {'TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE': 'TCDIAG', }, {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{source = "TCDIAG";}];'}), - # 47 + # 47 diag_convert_map key ('VALID', {'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', }, {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{key = ["(10C)", "(10KT)", "(10M/S)"];}];'}), - # 48 + # 48 diag_convert_map convert ('VALID', {'TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT': 'x/10', }, {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{convert(x) = x/10;}];'}), - # 49 + # 49 diag_convert_map 1 dictionary in list ('VALID', { 'TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE': 'TCDIAG', 'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', 'TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT': 'x/10', }, {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{source = "TCDIAG";key = ["(10C)", "(10KT)", "(10M/S)"];convert(x) = x/10;}];'}), - # 50 + # 50 diag_convert_map 2 dictionaries in list ('VALID', { 'TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE': 'TCDIAG', 'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', From b8c32405fa7ce13c595b33a7fced96f7e67345fa Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 9 Nov 2022 11:56:55 -0700 Subject: [PATCH 12/20] removed tests that shouldn't occur because all 3 items of diag_convert_map must be set or MET will error --- .../wrappers/tc_pairs/test_tc_pairs_wrapper.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index 5b1492917b..54fa02e22a 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -488,23 +488,14 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, # 45 diag_name ('VALID', {'TC_PAIRS_DIAG_NAME': 'TC_DIAG, TC_DIAG2', }, {'METPLUS_DIAG_NAME': 'diag_name = ["TC_DIAG", "TC_DIAG2"];'}), - # 46 diag_convert_map source - ('VALID', {'TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE': 'TCDIAG', }, - {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{source = "TCDIAG";}];'}), - # 47 diag_convert_map key - ('VALID', {'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', }, - {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{key = ["(10C)", "(10KT)", "(10M/S)"];}];'}), - # 48 diag_convert_map convert - ('VALID', {'TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT': 'x/10', }, - {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{convert(x) = x/10;}];'}), - # 49 diag_convert_map 1 dictionary in list + # 46 diag_convert_map 1 dictionary in list ('VALID', { 'TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE': 'TCDIAG', 'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', 'TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT': 'x/10', }, {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{source = "TCDIAG";key = ["(10C)", "(10KT)", "(10M/S)"];convert(x) = x/10;}];'}), - # 50 diag_convert_map 2 dictionaries in list + # 47 diag_convert_map 2 dictionaries in list ('VALID', { 'TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE': 'TCDIAG', 'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', From d97891693a05cd29069b1045975d45b410651932 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 9 Nov 2022 14:19:45 -0700 Subject: [PATCH 13/20] added support for handling desc in MET config that is a list like in TCStat --- metplus/wrappers/command_builder.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 6af2d49a66..eb60fb8617 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1300,7 +1300,7 @@ def handle_regrid(self, c_dict, set_to_grid=True): dict_items['shape'] = ('string', 'uppercase,remove_quotes') self.add_met_config_dict('regrid', dict_items) - def handle_description(self): + def handle_description(self, is_list=False): """! Get description from config. If _DESC is set, use that value. If not, check for DESC and use that if it is set. If set, set the METPLUS_DESC env_var_dict key to "desc = ;" @@ -1308,21 +1308,20 @@ def handle_description(self): """ # check if _DESC is set app_name_upper = self.app_name.upper() - conf_value = self.config.getstr('config', - f'{app_name_upper}_DESC', - '') + conf_value = self.config.getraw('config', f'{app_name_upper}_DESC') # if not, check if DESC is set if not conf_value: - conf_value = self.config.getstr('config', - 'DESC', - '') + conf_value = self.config.getraw('config', 'DESC') # if the value is set, set the DESC c_dict if conf_value: - self.env_var_dict['METPLUS_DESC'] = ( - f'desc = "{remove_quotes(conf_value)}";' - ) + if is_list: + value = '", "'.join(getlist(conf_value)) + value = f'["{value}"]' + else: + value = f'"{remove_quotes(conf_value)}"' + self.env_var_dict['METPLUS_DESC'] = f'desc = {value};' def get_output_prefix(self, time_info=None, set_env_vars=True): """! Read {APP_NAME}_OUTPUT_PREFIX from config. If time_info is set From 2adc5633e0556989ebf1706ce86603de9c9032c0 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 9 Nov 2022 14:23:36 -0700 Subject: [PATCH 14/20] Per #1898, added support for diag_thresh_name, diag_thresh_val, init_diag_thresh_name, and init_diag_thresh_val. Also added support for line_type, event_equal, event_equal_lead, out_init_mask, and valid_init_mask in case those are needed later. Fixed bug where variables ending with 'thresh_val' were not properly formatted without quotation marks. Added unit tests to ensure all env vars are set properly which uncovered some bugs in the logic --- docs/Users_Guide/glossary.rst | 45 ++++ docs/Users_Guide/wrappers.rst | 109 ++++++++ .../wrappers/tc_stat/test_tc_stat_wrapper.py | 232 +++++++++++++++++- metplus/wrappers/tc_stat_wrapper.py | 98 +++++--- parm/met_config/TCStatConfig_wrapped | 34 +++ .../met_tool_wrapper/TCStat/TCStat.conf | 12 + 6 files changed, 488 insertions(+), 42 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 21b13a018c..96509d1e39 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -10113,3 +10113,48 @@ METplus Configuration Glossary Specify the value for the nth 'diag_convert_map.convert' in the MET configuration file for TCPairs. | *Used by:* TCPairs + + TC_STAT_DIAG_THRESH_NAME + Specify the value for 'diag_thresh_name' in the MET configuration file for TCStat. + + | *Used by:* TCStat + + TC_STAT_DIAG_THRESH_VAL + Specify the value for 'diag_thresh_val' in the MET configuration file for TCStat. + + | *Used by:* TCStat + + TC_STAT_INIT_DIAG_THRESH_NAME + Specify the value for 'init_diag_thresh_name' in the MET configuration file for TCStat. + + | *Used by:* TCStat + + TC_STAT_INIT_DIAG_THRESH_VAL + Specify the value for 'init_diag_thresh_val' in the MET configuration file for TCStat. + + | *Used by:* TCStat + + TC_STAT_LINE_TYPE + Specify the value for 'line_type' in the MET configuration file for TCStat. + + | *Used by:* TCStat + + TC_STAT_EVENT_EQUAL + Specify the value for 'event_equal' in the MET configuration file for TCStat. + + | *Used by:* TCStat + + TC_STAT_EVENT_EQUAL_LEAD + Specify the value for 'event_equal_lead' in the MET configuration file for TCStat. + + | *Used by:* TCStat + + TC_STAT_OUT_INIT_MASK + Specify the value for 'out_init_mask' in the MET configuration file for TCStat. + + | *Used by:* TCStat + + TC_STAT_OUT_VALID_MASK + Specify the value for 'out_valid_mask' in the MET configuration file for TCStat. + + | *Used by:* TCStat diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 6d216db8b1..3da7b74695 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -8883,6 +8883,15 @@ METplus Configuration | :term:`TC_STAT_COLUMN_STR_EXC_VAL` | :term:`TC_STAT_INIT_STR_EXC_NAME` | :term:`TC_STAT_INIT_STR_EXC_VAL` +| :term:`TC_STAT_DIAG_THRESH_NAME` +| :term:`TC_STAT_DIAG_THRESH_VAL` +| :term:`TC_STAT_INIT_DIAG_THRESH_NAME` +| :term:`TC_STAT_INIT_DIAG_THRESH_VAL` +| :term:`TC_STAT_LINE_TYPE` +| :term:`TC_STAT_EVENT_EQUAL` +| :term:`TC_STAT_EVENT_EQUAL_LEAD` +| :term:`TC_STAT_OUT_INIT_MASK` +| :term:`TC_STAT_OUT_VALID_MASK` | .. warning:: **DEPRECATED:** @@ -9144,6 +9153,17 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`TC_STAT_VALID_MASK` - valid_mask +**${METPLUS_LINE_TYPE}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_STAT_LINE_TYPE` + - line_type + **${METPLUS_TRACK_WATCH_WARN}** .. list-table:: @@ -9287,6 +9307,50 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`TC_STAT_INIT_STR_EXC_VAL` - init_str_exc_val +**${METPLUS_DIAG_THRESH_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_STAT_DIAG_THRESH_NAME` + - diag_thresh_name + +**${METPLUS_DIAG_THRESH_VAL}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_STAT_DIAG_THRESH_VAL` + - diag_thresh_val + +**${METPLUS_INIT_DIAG_THRESH_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_STAT_INIT_DIAG_THRESH_NAME` + - init_diag_thresh_name + +**${METPLUS_INIT_DIAG_THRESH_VAL}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_STAT_INIT_DIAG_THRESH_VAL` + - init_diag_thresh_val + **${METPLUS_WATER_ONLY}** .. list-table:: @@ -9364,6 +9428,51 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`TC_STAT_MET_CONFIG_OVERRIDES` - n/a +**${METPLUS_EVENT_EQUAL}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_STAT_EVENT_EQUAL` + - event_equal + +**${METPLUS_EVENT_EQUAL_LEAD}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_STAT_EVENT_EQUAL_LEAD` + - event_equal_lead + +**${METPLUS_OUT_INIT_MASK}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_STAT_OUT_INIT_MASK` + - out_init_mask + +**${METPLUS_OUT_VALID_MASK}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_STAT_OUT_VALID_MASK` + - out_valid_mask + + .. _user_script_wrapper: UserScript diff --git a/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py index ac7001898a..ab39253dad 100644 --- a/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py @@ -9,18 +9,30 @@ from metplus.wrappers.tc_stat_wrapper import TCStatWrapper from metplus.util import ti_calculate +loop_by = 'INIT' +run_times = ['20150301', '20150301'] +config_init_beg = '20170705' +config_init_end = '20170901' def get_config(metplus_config): # extra_configs = [] # extra_configs.append(os.path.join(os.path.dirname(__file__), # 'tc_stat_conf.conf')) config = metplus_config + + # set config variables to prevent command from running and bypass check + # if input files actually exist + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + config.set('config', 'PROCESS_LIST', 'TCStat') - config.set('config', 'INIT_BEG', '20150301') - config.set('config', 'INIT_END', '20150304') - config.set('config', 'INIT_INCREMENT', '21600') - config.set('config', 'TC_STAT_INIT_BEG', '20170705') - config.set('config', 'TC_STAT_INIT_END', '20170901') + config.set('config', 'LOOP_BY', loop_by) + config.set('config', 'INIT_TIME_FMT', '%Y%m%d') + config.set('config', 'INIT_BEG', run_times[0]) + config.set('config', 'INIT_END', run_times[-1]) + config.set('config', 'INIT_INCREMENT', '12H') + config.set('config', 'TC_STAT_INIT_BEG', config_init_beg) + config.set('config', 'TC_STAT_INIT_END', config_init_end) config.set('config', 'TC_STAT_INIT_HOUR', '00') config.set('config', 'TC_STAT_JOB_ARGS', ("-job summary -line_type TCMPR -column " @@ -43,6 +55,212 @@ def tc_stat_wrapper(metplus_config): config = get_config(metplus_config) return TCStatWrapper(config) +@pytest.mark.parametrize( + 'config_overrides, env_var_values', [ + # 0: no config overrides that set env vars + ({}, {}), + # 1 amodel + ({'TC_STAT_AMODEL': 'AMODEL1,AMODEL2'}, + {'METPLUS_AMODEL': 'amodel = ["AMODEL1", "AMODEL2"];'}), + # 2 bmodel + ({'TC_STAT_BMODEL': 'BMODEL1,BMODEL2'}, + {'METPLUS_BMODEL': 'bmodel = ["BMODEL1", "BMODEL2"];'}), + # 3 desc + ({'TC_STAT_DESC': 'DESC1,DESC2'}, + {'METPLUS_DESC': 'desc = ["DESC1", "DESC2"];'}), + # 4 storm_id + ({'TC_STAT_STORM_ID': 'AL092011, EP082022'}, + {'METPLUS_STORM_ID': 'storm_id = ["AL092011", "EP082022"];'}), + # 5 basin + ({'TC_STAT_BASIN': 'AL,EP'}, + {'METPLUS_BASIN': 'basin = ["AL", "EP"];'}), + # 6 cyclone + ({'TC_STAT_CYCLONE': '01,02,03'}, + {'METPLUS_CYCLONE': 'cyclone = ["01", "02", "03"];'}), + # 7 storm_name + ({'TC_STAT_STORM_NAME': 'KATRINA, SANDY'}, + {'METPLUS_STORM_NAME': 'storm_name = ["KATRINA", "SANDY"];'}), + # 8 init_beg + ({'TC_STAT_INIT_BEG': '19870201'}, + {'METPLUS_INIT_BEG': 'init_beg = "19870201";'}), + # 9 init_end + ({'TC_STAT_INIT_END': '20221109'}, + {'METPLUS_INIT_END': 'init_end = "20221109";'}), + # 10 init_inc + ({'TC_STAT_INIT_INC': '19870201_06, 20001231_23'}, + {'METPLUS_INIT_INC': 'init_inc = ["19870201_06", "20001231_23"];'}), + # 11 init_exc + ({'TC_STAT_INIT_EXC': '19870201_12, 20001231_09'}, + {'METPLUS_INIT_EXC': 'init_exc = ["19870201_12", "20001231_09"];'}), + # 12 valid_beg + ({'TC_STAT_VALID_BEG': '19870201'}, + {'METPLUS_VALID_BEG': 'valid_beg = "19870201";'}), + # 13 valid_end + ({'TC_STAT_VALID_END': '20221109'}, + {'METPLUS_VALID_END': 'valid_end = "20221109";'}), + # 14 valid_inc + ({'TC_STAT_VALID_INC': '19870201_06, 20001231_23'}, + {'METPLUS_VALID_INC': 'valid_inc = ["19870201_06", "20001231_23"];'}), + # 15 valid_exc + ({'TC_STAT_VALID_EXC': '19870201_12, 20001231_09'}, + {'METPLUS_VALID_EXC': 'valid_exc = ["19870201_12", "20001231_09"];'}), + # 16 init_hour + ({'TC_STAT_INIT_HOUR': '00,06,12,18'}, + {'METPLUS_INIT_HOUR': 'init_hour = ["00", "06", "12", "18"];'}), + # 17 valid_hour + ({'TC_STAT_VALID_HOUR': '00,06,12,18'}, + {'METPLUS_VALID_HOUR': 'valid_hour = ["00", "06", "12", "18"];'}), + # 18 lead + ({'TC_STAT_LEAD': '00,062359'}, + {'METPLUS_LEAD': 'lead = ["00", "062359"];'}), + # 19 lead_req + ({'TC_STAT_LEAD_REQ': '00,062359'}, + {'METPLUS_LEAD_REQ': 'lead_req = ["00", "062359"];'}), + # 20 init_mask + ({'TC_STAT_INIT_MASK': 'MET_BASE/poly/EAST.poly'}, + {'METPLUS_INIT_MASK': 'init_mask = ["MET_BASE/poly/EAST.poly"];'}), + # 21 valid_mask + ({'TC_STAT_VALID_MASK': 'MET_BASE/poly/EAST.poly'}, + {'METPLUS_VALID_MASK': 'valid_mask = ["MET_BASE/poly/EAST.poly"];'}), + # 22 line_type + ({'TC_STAT_LINE_TYPE': 'TCMPR'}, + {'METPLUS_LINE_TYPE': 'line_type = ["TCMPR"];'}), + # 23 track_watch_warn + ({'TC_STAT_TRACK_WATCH_WARN': 'HUWATCH, HUWARN'}, + {'METPLUS_TRACK_WATCH_WARN': 'track_watch_warn = ["HUWATCH", "HUWARN"];'}), + # 24 column_thresh_name + ({'TC_STAT_COLUMN_THRESH_NAME': 'ADLAND, BDLAND'}, + {'METPLUS_COLUMN_THRESH_NAME': 'column_thresh_name = ["ADLAND", "BDLAND"];'}), + # 25 column_thresh_val + ({'TC_STAT_COLUMN_THRESH_VAL': '>200, >200'}, + {'METPLUS_COLUMN_THRESH_VAL': 'column_thresh_val = [>200, >200];'}), + # 26 column_str_name + ({'TC_STAT_COLUMN_STR_NAME': 'LEVEL, LEVEL'}, + {'METPLUS_COLUMN_STR_NAME': 'column_str_name = ["LEVEL", "LEVEL"];'}), + # 27 column_str_val + ({'TC_STAT_COLUMN_STR_VAL': 'HU, TS'}, + {'METPLUS_COLUMN_STR_VAL': 'column_str_val = ["HU", "TS"];'}), + # 28 column_str_exc_name + ({'TC_STAT_COLUMN_STR_EXC_NAME': 'LEVEL, LEVEL'}, + {'METPLUS_COLUMN_STR_EXC_NAME': 'column_str_exc_name = ["LEVEL", "LEVEL"];'}), + # 29 column_str_exc_val + ({'TC_STAT_COLUMN_STR_EXC_VAL': 'HU, TS'}, + {'METPLUS_COLUMN_STR_EXC_VAL': 'column_str_exc_val = ["HU", "TS"];'}), + # 30 init_thresh_name + ({'TC_STAT_INIT_THRESH_NAME': 'ADLAND, BDLAND'}, + {'METPLUS_INIT_THRESH_NAME': 'init_thresh_name = ["ADLAND", "BDLAND"];'}), + # 31 init_thresh_val + ({'TC_STAT_INIT_THRESH_VAL': '>200, >200'}, + {'METPLUS_INIT_THRESH_VAL': 'init_thresh_val = [>200, >200];'}), + # 32 init_str_name + ({'TC_STAT_INIT_STR_NAME': 'LEVEL, LEVEL'}, + {'METPLUS_INIT_STR_NAME': 'init_str_name = ["LEVEL", "LEVEL"];'}), + # 33 init_str_val + ({'TC_STAT_INIT_STR_VAL': 'HU, TS'}, + {'METPLUS_INIT_STR_VAL': 'init_str_val = ["HU", "TS"];'}), + # 34 init_str_exc_name + ({'TC_STAT_INIT_STR_EXC_NAME': 'LEVEL, LEVEL'}, + {'METPLUS_INIT_STR_EXC_NAME': 'init_str_exc_name = ["LEVEL", "LEVEL"];'}), + # 35 init_str_exc_val + ({'TC_STAT_INIT_STR_EXC_VAL': 'HU, TS'}, + {'METPLUS_INIT_STR_EXC_VAL': 'init_str_exc_val = ["HU", "TS"];'}), + # 36 diag_thresh_name + ({'TC_STAT_DIAG_THRESH_NAME': 'ADLAND, BDLAND'}, + {'METPLUS_DIAG_THRESH_NAME': 'diag_thresh_name = ["ADLAND", "BDLAND"];'}), + # 37 diag_thresh_val + ({'TC_STAT_DIAG_THRESH_VAL': '>200, >200'}, + {'METPLUS_DIAG_THRESH_VAL': 'diag_thresh_val = [>200, >200];'}), + # 38 water_only + ({'TC_STAT_WATER_ONLY': 'true'}, + {'METPLUS_WATER_ONLY': 'water_only = TRUE;'}), + # 39 landfall + ({'TC_STAT_LANDFALL': 'true'}, + {'METPLUS_LANDFALL': 'landfall = TRUE;'}), + # 40 landfall_beg + ({'TC_STAT_LANDFALL_BEG': '-24'}, + {'METPLUS_LANDFALL_BEG': 'landfall_beg = "-24";'}), + # 41 landfall_end + ({'TC_STAT_LANDFALL_END': '00'}, + {'METPLUS_LANDFALL_END': 'landfall_end = "00";'}), + # 42 match_points + ({'TC_STAT_MATCH_POINTS': 'true'}, + {'METPLUS_MATCH_POINTS': 'match_points = TRUE;'}), + # 43 event_equal + ({'TC_STAT_EVENT_EQUAL': 'true'}, + {'METPLUS_EVENT_EQUAL': 'event_equal = TRUE;'}), + # 44 event_equal_lead + ({'TC_STAT_EVENT_EQUAL_LEAD': '06,12'}, + {'METPLUS_EVENT_EQUAL_LEAD': 'event_equal_lead = ["06", "12"];'}), + # 45 out_init_mask + ({'TC_STAT_OUT_INIT_MASK': 'MET_BASE/poly/EAST.poly', }, + {'METPLUS_OUT_INIT_MASK': 'out_init_mask = "MET_BASE/poly/EAST.poly";'}), + # 46 out_valid_mask + ({'TC_STAT_OUT_VALID_MASK': 'MET_BASE/poly/EAST.poly', }, + {'METPLUS_OUT_VALID_MASK': 'out_valid_mask = "MET_BASE/poly/EAST.poly";'}), + + ] +) +@pytest.mark.wrapper +def test_tc_stat_run(metplus_config, config_overrides, env_var_values): + config = get_config(metplus_config) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + if f'METPLUS_{loop_by}_BEG' not in env_var_values: + env_var_values[f'METPLUS_{loop_by}_BEG'] = ( + f'{loop_by.lower()}_beg = "{config_init_beg}";' + ) + + if f'METPLUS_{loop_by}_END' not in env_var_values: + env_var_values[f'METPLUS_{loop_by}_END'] = ( + f'{loop_by.lower()}_end = "{config_init_end}";' + ) + + if 'METPLUS_INIT_HOUR' not in env_var_values: + env_var_values['METPLUS_INIT_HOUR'] = f'init_hour = ["00"];' + + wrapper = TCStatWrapper(config) + assert wrapper.isOK + + if 'METPLUS_JOBS' not in env_var_values: + jobs = f"jobs = {wrapper.c_dict.get('JOBS')};" + env_var_values['METPLUS_JOBS'] = jobs + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + config_file = wrapper.c_dict.get('CONFIG_FILE') + lookin_dir = wrapper.c_dict.get('LOOKIN_DIR') + out_temp = wrapper.c_dict.get('OUTPUT_TEMPLATE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + out_arg = f' -out {out_dir}/{out_temp}' if out_temp else '' + + expected_cmds = [ + (f"{app_path} {verbosity} -lookin {lookin_dir} " + f"-config {config_file}{out_arg}"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + assert len(all_cmds) == len(expected_cmds) + + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd + + # check that environment variables were set properly + for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS + missing_env: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + print(f'Checking env var: {env_var_key}') + actual_value = match.split('=', 1)[1] + assert env_var_values.get(env_var_key, '') == actual_value + @pytest.mark.parametrize( 'overrides, c_dict', [ @@ -246,7 +464,9 @@ def test_handle_jobs_create_parent_dir(metplus_config, jobs, init_dt, else: time_info = None - wrapper = tc_stat_wrapper(metplus_config) + config = get_config(metplus_config) + config.set('config', 'DO_NOT_RUN_EXE', False) + wrapper = TCStatWrapper(config) # create directory path relative to OUTPUT_BASE to test that function # creates parent directories properly diff --git a/metplus/wrappers/tc_stat_wrapper.py b/metplus/wrappers/tc_stat_wrapper.py index 9782f84c8d..24a0066672 100755 --- a/metplus/wrappers/tc_stat_wrapper.py +++ b/metplus/wrappers/tc_stat_wrapper.py @@ -56,6 +56,7 @@ class TCStatWrapper(CommandBuilder): 'METPLUS_LEAD_REQ', 'METPLUS_INIT_MASK', 'METPLUS_VALID_MASK', + 'METPLUS_LINE_TYPE', 'METPLUS_TRACK_WATCH_WARN', 'METPLUS_COLUMN_THRESH_NAME', 'METPLUS_COLUMN_THRESH_VAL', @@ -75,6 +76,14 @@ class TCStatWrapper(CommandBuilder): 'METPLUS_COLUMN_STR_EXC_VAL', 'METPLUS_INIT_STR_EXC_NAME', 'METPLUS_INIT_STR_EXC_VAL', + 'METPLUS_DIAG_THRESH_NAME', + 'METPLUS_DIAG_THRESH_VAL', + 'METPLUS_INIT_DIAG_THRESH_NAME', + 'METPLUS_INIT_DIAG_THRESH_VAL', + 'METPLUS_EVENT_EQUAL', + 'METPLUS_EVENT_EQUAL_LEAD', + 'METPLUS_OUT_INIT_MASK', + 'METPLUS_OUT_VALID_MASK', ] def __init__(self, config, instance=None): @@ -141,32 +150,44 @@ def set_met_config_for_environment_variables(self): variables to be read by the MET config file. @param c_dict dictionary to add key/value pairs """ - self.handle_description() - - for config_list in ['amodel', - 'bmodel', - 'storm_id', - 'basin', - 'cyclone', - 'storm_name', - 'init_hour', - 'lead_req', - 'init_mask', - 'valid_mask', - 'valid_hour', - 'lead', - 'track_watch_warn', - 'column_thresh_name', - 'column_thresh_val', - 'column_str_name', - 'column_str_val', - 'init_thresh_name', - 'init_thresh_val', - 'init_str_name', - 'init_str_val', - ]: + self.handle_description(is_list=True) + + for config_list in [ + 'amodel', + 'bmodel', + 'storm_id', + 'basin', + 'cyclone', + 'storm_name', + 'init_hour', + 'lead_req', + 'init_mask', + 'valid_mask', + 'line_type', + 'valid_hour', + 'lead', + 'track_watch_warn', + 'column_thresh_name', + 'column_thresh_val', + 'column_str_name', + 'column_str_val', + 'init_thresh_name', + 'init_thresh_val', + 'init_str_name', + 'init_str_val', + 'diag_thresh_name', + 'diag_thresh_val', + 'init_diag_thresh_name', + 'init_diag_thresh_val', + 'event_equal_lead', + ]: + extra_args = {} + # remove quotation marks from *_thresh_val lists + if 'thresh_val' in config_list: + extra_args['remove_quotes'] = True self.add_met_config(name=config_list, - data_type='list') + data_type='list', + extra_args=extra_args) for iv_list in ['INIT', 'VALID']: self.add_met_config(name=f'{iv_list.lower()}_inc', @@ -178,22 +199,27 @@ def set_met_config_for_environment_variables(self): metplus_configs=[f'TC_STAT_{iv_list}_EXC', f'TC_STAT_{iv_list}_EXCLUDE']) - for config_str in ['INIT_BEG', - 'INIT_END', - 'VALID_BEG', - 'VALID_END', - 'LANDFALL_BEG', - 'LANDFALL_END', - ]: + for config_str in [ + 'INIT_BEG', + 'INIT_END', + 'VALID_BEG', + 'VALID_END', + 'LANDFALL_BEG', + 'LANDFALL_END', + 'OUT_INIT_MASK', + 'OUT_VALID_MASK', + ]: self.add_met_config(name=config_str.lower(), data_type='string', metplus_configs=[f'TC_STAT_{config_str}', config_str]) - for config_bool in ['water_only', - 'landfall', - 'match_points', - ]: + for config_bool in [ + 'water_only', + 'landfall', + 'match_points', + 'event_equal', + ]: self.add_met_config(name=config_bool, data_type='bool') diff --git a/parm/met_config/TCStatConfig_wrapped b/parm/met_config/TCStatConfig_wrapped index cef47ecc53..5b365544c1 100644 --- a/parm/met_config/TCStatConfig_wrapped +++ b/parm/met_config/TCStatConfig_wrapped @@ -86,6 +86,14 @@ ${METPLUS_LEAD_REQ} ${METPLUS_INIT_MASK} ${METPLUS_VALID_MASK} + +// +// Stratify by the LINE_TYPE column. +// +//line_type = +${METPLUS_LINE_TYPE} + + // // Stratify by checking the watch/warning status for each track point // common to both the ADECK and BDECK tracks. If the watch/warning status @@ -135,6 +143,19 @@ ${METPLUS_INIT_STR_EXC_NAME} //init_str_exc_val = ${METPLUS_INIT_STR_EXC_VAL} +//diag_thresh_name = +${METPLUS_DIAG_THRESH_NAME} + +//diag_thresh_val = +${METPLUS_DIAG_THRESH_VAL} + +//init_diag_thresh_name = +${METPLUS_INIT_DIAG_THRESH_NAME} + +//init_diag_thresh_val = +${METPLUS_INIT_DIAG_THRESH_VAL} + + // // Stratify by the ADECK and BDECK distances to land. // @@ -156,6 +177,19 @@ ${METPLUS_LANDFALL_END} // ${METPLUS_MATCH_POINTS} +//event_equal = +${METPLUS_EVENT_EQUAL} + +//event_equal_lead = +${METPLUS_EVENT_EQUAL_LEAD} + +//out_init_mask = +${METPLUS_OUT_INIT_MASK} + +//out_valid_mask = +${METPLUS_OUT_VALID_MASK} + + // // Array of TCStat analysis jobs to be performed on the filtered data // diff --git a/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf b/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf index cb6eb51d8c..ec1705915b 100644 --- a/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf +++ b/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf @@ -75,6 +75,8 @@ TC_STAT_LEAD_REQ = TC_STAT_INIT_MASK = TC_STAT_VALID_MASK = +#TC_STAT_LINE_TYPE = + TC_STAT_LEAD = TC_STAT_TRACK_WATCH_WARN = @@ -105,3 +107,13 @@ TC_STAT_MATCH_POINTS = false #TC_STAT_INIT_STR_EXC_NAME = #TC_STAT_INIT_STR_EXC_VAL = + +#TC_STAT_DIAG_THRESH_NAME = +#TC_STAT_DIAG_THRESH_VAL = +#TC_STAT_INIT_DIAG_THRESH_NAME = +#TC_STAT_INIT_DIAG_THRESH_VAL = + +#TC_STAT_EVENT_EQUAL = +#TC_STAT_EVENT_EQUAL_LEAD = +#TC_STAT_OUT_INIT_MASK = +#TC_STAT_OUT_VALID_MASK = From 42f115311fa665102314559701a2f3417b04a20e Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 9 Nov 2022 14:24:17 -0700 Subject: [PATCH 15/20] fixed bug where init_inc, init_exc, valid_inc, and valid_exc were not set from the METplus config --- metplus/wrappers/tc_stat_wrapper.py | 8 ++++---- parm/met_config/TCStatConfig_wrapped | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/metplus/wrappers/tc_stat_wrapper.py b/metplus/wrappers/tc_stat_wrapper.py index 24a0066672..ce09ba9400 100755 --- a/metplus/wrappers/tc_stat_wrapper.py +++ b/metplus/wrappers/tc_stat_wrapper.py @@ -44,12 +44,12 @@ class TCStatWrapper(CommandBuilder): 'METPLUS_STORM_NAME', 'METPLUS_INIT_BEG', 'METPLUS_INIT_END', - 'METPLUS_INIT_INCLUDE', - 'METPLUS_INIT_EXCLUDE', + 'METPLUS_INIT_INC', + 'METPLUS_INIT_EXC', 'METPLUS_VALID_BEG', 'METPLUS_VALID_END', - 'METPLUS_VALID_INCLUDE', - 'METPLUS_VALID_EXCLUDE', + 'METPLUS_VALID_INC', + 'METPLUS_VALID_EXC', 'METPLUS_INIT_HOUR', 'METPLUS_VALID_HOUR', 'METPLUS_LEAD', diff --git a/parm/met_config/TCStatConfig_wrapped b/parm/met_config/TCStatConfig_wrapped index 5b365544c1..916a15fed8 100644 --- a/parm/met_config/TCStatConfig_wrapped +++ b/parm/met_config/TCStatConfig_wrapped @@ -57,16 +57,16 @@ ${METPLUS_STORM_NAME} // ${METPLUS_INIT_BEG} ${METPLUS_INIT_END} -${METPLUS_INIT_INCLUDE} -${METPLUS_INIT_EXCLUDE} +${METPLUS_INIT_INC} +${METPLUS_INIT_EXC} // // Stratify by the VALID times. // ${METPLUS_VALID_BEG} ${METPLUS_VALID_END} -${METPLUS_VALID_INCLUDE} -${METPLUS_VALID_EXCLUDE} +${METPLUS_VALID_INC} +${METPLUS_VALID_EXC} // // Stratify by the initialization and valid hours and lead time. From f74152ceb79e91fdc50ad3a114983b172b33d216 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 9 Nov 2022 15:09:33 -0700 Subject: [PATCH 16/20] make tests that check env vars consistent and ensure they check all of the variables that should be checked -- fixed StatAnalysis test that was not working properly because of this --- .../ascii2nc/test_ascii2nc_wrapper.py | 8 +++-- .../test_ensemble_stat_wrapper.py | 7 +++-- .../gen_ens_prod/test_gen_ens_prod_wrapper.py | 7 +++-- .../grid_stat/test_grid_stat_wrapper.py | 7 +++-- .../wrappers/ioda2nc/test_ioda2nc_wrapper.py | 7 +++-- .../wrappers/mode/test_mode_wrapper.py | 31 +++++++++---------- .../wrappers/pb2nc/test_pb2nc_wrapper.py | 7 +++-- .../test_plot_point_obs_wrapper.py | 7 +++-- .../point_stat/test_point_stat_wrapper.py | 7 +++-- .../series_analysis/test_series_analysis.py | 6 +++- .../stat_analysis/test_stat_analysis.py | 26 +++++++++++----- .../wrappers/tc_gen/test_tc_gen_wrapper.py | 7 +++-- .../tc_pairs/test_tc_pairs_wrapper.py | 13 ++++++-- .../wrappers/tc_stat/test_tc_stat_wrapper.py | 3 +- 14 files changed, 89 insertions(+), 54 deletions(-) diff --git a/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py b/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py index 0af69609ad..0202f14b8d 100644 --- a/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py @@ -186,11 +186,13 @@ def test_ascii2nc_wrapper(metplus_config, config_overrides, assert all_commands[0][0] == expected_cmd env_vars = all_commands[0][1] + + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) - env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + - [name for name in env_var_values - if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) diff --git a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py index 2045453a5c..313beb3a86 100644 --- a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py @@ -635,15 +635,16 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides, print(f"ALL COMMANDS: {all_cmds}") assert len(all_cmds) == len(expected_cmds) + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert(cmd == expected_cmd) # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) - env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + - [name for name in env_var_values - if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) diff --git a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py index 49990fc528..2bcba73a17 100644 --- a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -412,15 +412,16 @@ def test_gen_ens_prod_single_field(metplus_config, config_overrides, print(f"ALL COMMANDS: {all_cmds}") assert len(all_cmds) == len(expected_cmds) + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert(cmd == expected_cmd) # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) - env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + - [name for name in env_var_values - if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) diff --git a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py index 9a0caed0b7..f8337d8650 100644 --- a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py @@ -723,15 +723,16 @@ def test_grid_stat_single_field(metplus_config, config_overrides, all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) - env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + - [name for name in env_var_values - if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) diff --git a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py index 99a98db0fc..461b9d1080 100644 --- a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py @@ -218,15 +218,16 @@ def test_ioda2nc_wrapper(metplus_config, config_overrides, print(f"ALL COMMANDS: {all_cmds}") assert len(all_cmds) == len(expected_cmds) + missing_env = [item for item in expected_env_vars + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) - env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + - [name for name in env_var_values - if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) diff --git a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py index 1ca480b49d..fb9759face 100644 --- a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py +++ b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py @@ -59,7 +59,7 @@ def set_minimum_config_settings(config): @pytest.mark.parametrize( - 'config_overrides, expected_output', [ + 'config_overrides, env_var_values', [ ({'MODEL': 'my_model'}, {'METPLUS_MODEL': 'model = "my_model";'}), @@ -316,8 +316,7 @@ def set_minimum_config_settings(config): ] ) @pytest.mark.wrapper_a -def test_mode_single_field(metplus_config, config_overrides, - expected_output): +def test_mode_single_field(metplus_config, config_overrides, env_var_values): config = metplus_config # set config variables needed to run @@ -344,7 +343,6 @@ def test_mode_single_field(metplus_config, config_overrides, f"{config_file} -outdir {out_dir}/2005080800"), ] - all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") @@ -366,19 +364,20 @@ def test_mode_single_field(metplus_config, config_overrides, if met_name in met_lists: default_val = f'[{default_val}]' - expected_output[f'METPLUS_{name}'] = ( + env_var_values[f'METPLUS_{name}'] = ( f'{met_name} = {default_val};' ) + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) - env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + - [name for name in expected_output - if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) @@ -389,18 +388,18 @@ def test_mode_single_field(metplus_config, config_overrides, elif env_var_key == 'METPLUS_OBS_FIELD': assert value == obs_fmt else: - assert expected_output.get(env_var_key, '') == value + assert env_var_values.get(env_var_key, '') == value @pytest.mark.parametrize( - 'config_overrides, expected_output', [ + 'config_overrides, env_var_values', [ ({'MODE_MULTIVAR_LOGIC': '#1 && #2 && #3', }, {'METPLUS_MULTIVAR_LOGIC': 'multivar_logic = "#1 && #2 && #3";'}), ] ) @pytest.mark.wrapper_a def test_mode_multi_variate(metplus_config, config_overrides, - expected_output): + env_var_values): config = metplus_config # set config variables needed to run @@ -440,15 +439,16 @@ def test_mode_multi_variate(metplus_config, config_overrides, all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) - env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + - [name for name in expected_output - if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) @@ -459,7 +459,7 @@ def test_mode_multi_variate(metplus_config, config_overrides, elif env_var_key == 'METPLUS_OBS_FIELD': assert value == obs_multi_fmt else: - assert expected_output.get(env_var_key, '') == value + assert env_var_values.get(env_var_key, '') == value @pytest.mark.parametrize( @@ -524,7 +524,6 @@ def test_config_synonyms(metplus_config, config_name, env_var_name, wrapper = MODEWrapper(config) assert wrapper.isOK - expected_output = f'{met_name} = {out_value};' assert wrapper.env_var_dict[env_var_name] == expected_output diff --git a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py index 8096cc1e4c..02e35d22d4 100644 --- a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py @@ -319,15 +319,16 @@ def test_pb2nc_all_fields(metplus_config, config_overrides, all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) - env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + - [name for name in env_var_values - if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) diff --git a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py index 172b4a93d6..1e416cc6c6 100644 --- a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py +++ b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py @@ -267,15 +267,16 @@ def test_plot_point_obs(metplus_config, config_overrides, env_var_values): assert len(all_cmds) == len(expected_cmds) assert not wrapper.errors + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) - env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + - [name for name in env_var_values - if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) diff --git a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py index 2c574d2476..f06035ab77 100755 --- a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py @@ -557,15 +557,16 @@ def test_point_stat_all_fields(metplus_config, config_overrides, fcst_fmt = f"field = [{','.join(fcst_fmts)}];" obs_fmt = f"field = [{','.join(obs_fmts)}];" + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) - env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + - [name for name in env_var_values - if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) diff --git a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py index 228e89114c..375f6c67d9 100644 --- a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py +++ b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py @@ -344,12 +344,16 @@ def test_series_analysis_single_field(metplus_config, config_overrides, all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert match is not None diff --git a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py index 76c277fd93..1ef38667f1 100644 --- a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py +++ b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py @@ -16,6 +16,7 @@ pp = pprint.PrettyPrinter() +JOB_ARGS = '-job filter' def stat_analysis_wrapper(metplus_config): """! Returns a default StatAnalysisWrapper with /path/to entries in the @@ -103,7 +104,7 @@ def set_minimum_config_settings(config): config.set('config', 'GROUP_LIST_ITEMS', 'DESC_LIST') config.set('config', 'LOOP_LIST_ITEMS', 'MODEL_LIST') config.set('config', 'MODEL_LIST', 'MODEL_A') - config.set('config', 'STAT_ANALYSIS_JOB1', '-job filter') + config.set('config', 'STAT_ANALYSIS_JOB1', JOB_ARGS) config.set('config', 'MODEL1', 'MODEL_A') config.set('config', 'MODEL1_STAT_ANALYSIS_LOOKIN_DIR', '{METPLUS_BASE}/internal/tests/data/stat_data') @@ -168,9 +169,9 @@ def set_minimum_config_settings(config): 'METPLUS_OBS_INIT_END': 'obs_init_end = "20221015_12";'}), # 16 - custom in MODEL1 ({'STAT_ANALYSIS_CUSTOM_LOOP_LIST': 'CUSTOM_MODEL', - 'STAT_ANALYSIS_MODEL1': '{custom}', - 'STAT_ANALYSIS_MODEL_LIST': '{custom}'}, - {'METPLUS_MODEL': 'model = "CUSTOM_MODEL";'}), + 'MODEL1': '{custom}', + 'MODEL_LIST': '{custom}'}, + {'METPLUS_MODEL': 'model = ["CUSTOM_MODEL"];'}), ] ) @pytest.mark.wrapper_d @@ -182,10 +183,19 @@ def test_valid_init_env_vars(metplus_config, config_overrides, for key, value in config_overrides.items(): config.set('config', key, value) + if 'METPLUS_MODEL' not in expected_env_vars: + expected_env_vars['METPLUS_MODEL'] = 'model = ["MODEL_A"];' + + if 'METPLUS_JOBS' not in expected_env_vars: + expected_env_vars['METPLUS_JOBS'] = f'jobs = ["{JOB_ARGS}"];' + wrapper = StatAnalysisWrapper(config) assert wrapper.isOK - runtime_settings_dict_list = wrapper._get_all_runtime_settings({}) + time_input = { + 'custom': config_overrides.get('STAT_ANALYSIS_CUSTOM_LOOP_LIST', '') + } + runtime_settings_dict_list = wrapper._get_all_runtime_settings(time_input) assert runtime_settings_dict_list first_runtime_only = [runtime_settings_dict_list[0]] @@ -195,8 +205,10 @@ def test_valid_init_env_vars(metplus_config, config_overrides, print(f"ALL COMMANDS: {all_cmds}") _, actual_env_vars = all_cmds[0] - env_var_keys = [item for item in wrapper.WRAPPER_ENV_VAR_KEYS - if 'BEG' in item or 'END' in item] + missing_env = [item for item in expected_env_vars + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for env_var_key in env_var_keys: match = next((item for item in actual_env_vars if item.startswith(env_var_key)), None) diff --git a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py index b3cb191fa0..84d9c90838 100644 --- a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py @@ -385,15 +385,16 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values): print(f"ALL COMMANDS: {all_cmds}") assert len(all_cmds) == len(expected_cmds) + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) - env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + - [name for name in env_var_values - if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index 54fa02e22a..c632947fce 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -584,12 +584,17 @@ def test_tc_pairs_run(metplus_config, loop_by, config_overrides, print(f"ALL COMMANDS: {all_cmds}") assert len(all_cmds) == len(expected_cmds) + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS + and item != 'DIAG_ARG'] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert match is not None @@ -681,11 +686,15 @@ def test_tc_pairs_read_all_files(metplus_config, loop_by, config_overrides, print(f"ALL COMMANDS: {all_cmds}") assert len(all_cmds) == len(expected_cmds) + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert match is not None diff --git a/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py index ab39253dad..40b66b6c1d 100644 --- a/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py @@ -247,13 +247,14 @@ def test_tc_stat_run(metplus_config, config_overrides, env_var_values): missing_env = [item for item in env_var_values if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS + missing_env: + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert match is not None From 729961dee74580e9767486b3a71f363d29cc29d0 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 9 Nov 2022 15:24:35 -0700 Subject: [PATCH 17/20] per #1926, add support for setting mask dictionary in SeriesAnalysis --- docs/Users_Guide/glossary.rst | 10 ++++++++++ docs/Users_Guide/wrappers.rst | 15 +++++++++++++++ .../series_analysis/test_series_analysis.py | 11 +++++++++++ metplus/wrappers/series_analysis_wrapper.py | 3 +++ parm/met_config/SeriesAnalysisConfig_wrapped | 6 ++---- .../SeriesAnalysis/SeriesAnalysis.conf | 3 +++ 6 files changed, 44 insertions(+), 4 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 96509d1e39..d8944f2000 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -10158,3 +10158,13 @@ METplus Configuration Glossary Specify the value for 'out_valid_mask' in the MET configuration file for TCStat. | *Used by:* TCStat + + SERIES_ANALYSIS_MASK_GRID + Specify the value for 'mask.grid' in the MET configuration file for SeriesAnalysis. + + | *Used by:* SeriesAnalysis + + SERIES_ANALYSIS_MASK_POLY + Specify the value for 'mask.poly' in the MET configuration file for SeriesAnalysis. + + | *Used by:* SeriesAnalysis diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 3da7b74695..a4d088dbef 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -6387,6 +6387,8 @@ METplus Configuration | :term:`OBS_SERIES_ANALYSIS_CAT_THRESH` | :term:`FCST_SERIES_ANALYSIS_IS_PROB` | :term:`FCST_SERIES_ANALYSIS_PROB_IN_GRIB_PDS` +| :term:`SERIES_ANALYSIS_MASK_GRID` +| :term:`SERIES_ANALYSIS_MASK_POLY` | .. warning:: **DEPRECATED:** @@ -6618,6 +6620,19 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`SERIES_ANALYSIS_CLIMO_CDF_DIRECT_PROB` - climo_cdf.direct_prob +**${METPLUS_MASK_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`SERIES_ANALYSIS_MASK_GRID` + - mask.grid + * - :term:`SERIES_ANALYSIS_MASK_POLY` + - mask.poly + **${METPLUS_BLOCK_SIZE}** .. list-table:: diff --git a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py index 375f6c67d9..08a83e771c 100644 --- a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py +++ b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py @@ -306,6 +306,17 @@ def set_minimum_config_settings(config): 'SERIES_ANALYSIS_CLIMO_CDF_DIRECT_PROB': 'False', }, {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {cdf_bins = 1.0;center_bins = TRUE;direct_prob = FALSE;}'}), + ({'SERIES_ANALYSIS_MASK_GRID': 'FULL', }, + {'METPLUS_MASK_DICT': 'mask = {grid = "FULL";}'}), + + ({'SERIES_ANALYSIS_MASK_POLY': 'MET_BASE/poly/EAST.poly', }, + {'METPLUS_MASK_DICT': 'mask = {poly = "MET_BASE/poly/EAST.poly";}'}), + + ({ + 'SERIES_ANALYSIS_MASK_GRID': 'FULL', + 'SERIES_ANALYSIS_MASK_POLY': 'MET_BASE/poly/EAST.poly', + }, + {'METPLUS_MASK_DICT': 'mask = {grid = "FULL";poly = "MET_BASE/poly/EAST.poly";}'}), ] ) diff --git a/metplus/wrappers/series_analysis_wrapper.py b/metplus/wrappers/series_analysis_wrapper.py index ca3897cdf8..f592737c20 100755 --- a/metplus/wrappers/series_analysis_wrapper.py +++ b/metplus/wrappers/series_analysis_wrapper.py @@ -57,6 +57,7 @@ class SeriesAnalysisWrapper(RuntimeFreqWrapper): 'METPLUS_FCST_CAT_THRESH', 'METPLUS_OBS_CAT_THRESH', 'METPLUS_CLIMO_CDF_DICT', + 'METPLUS_MASK_DICT', ] # handle deprecated env vars used pre v4.0.0 @@ -175,6 +176,8 @@ def create_c_dict(self): metplus_configs=['SERIES_ANALYSIS_CTS_LIST', 'SERIES_ANALYSIS_CTS']) + self.handle_mask(single_value=True) + c_dict['PAIRED'] = self.config.getbool('config', 'SERIES_ANALYSIS_IS_PAIRED', False) diff --git a/parm/met_config/SeriesAnalysisConfig_wrapped b/parm/met_config/SeriesAnalysisConfig_wrapped index 1dea82e844..c0c1175d24 100644 --- a/parm/met_config/SeriesAnalysisConfig_wrapped +++ b/parm/met_config/SeriesAnalysisConfig_wrapped @@ -91,10 +91,8 @@ boot = { // // Verification masking regions // -mask = { - grid = ""; - poly = ""; -} +//mask = { +${METPLUS_MASK_DICT} // // Number of grid points to be processed concurrently. Set smaller to use diff --git a/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf b/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf index 5d361be2be..bcaa622247 100644 --- a/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf +++ b/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf @@ -156,6 +156,9 @@ SERIES_ANALYSIS_OUTPUT_STATS_CNT = TOTAL, RMSE, FBAR, OBAR #SERIES_ANALYSIS_OUTPUT_STATS_PJC = #SERIES_ANALYSIS_OUTPUT_STATS_PRC = +#SERIES_ANALYSIS_MASK_GRID = +#SERIES_ANALYSIS_MASK_POLY = + ### # SeriesAnalysis Plotting From 86dbaebc0b0a15dd2983fcf25640e638d63bb44c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 9 Nov 2022 15:39:56 -0700 Subject: [PATCH 18/20] fixed typo that broke unit tests --- internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py index 461b9d1080..70b4936428 100644 --- a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py @@ -218,7 +218,7 @@ def test_ioda2nc_wrapper(metplus_config, config_overrides, print(f"ALL COMMANDS: {all_cmds}") assert len(all_cmds) == len(expected_cmds) - missing_env = [item for item in expected_env_vars + missing_env = [item for item in env_var_values if item not in wrapper.WRAPPER_ENV_VAR_KEYS] env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env From 66092a1c276599620aa87cd5fe921a1decf53bb3 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 16 Nov 2022 12:14:34 -0700 Subject: [PATCH 19/20] per #1898, made modifications to TCPairs settings based on changes in dtcenter/MET#2347 --- docs/Users_Guide/glossary.rst | 28 ++++++-- docs/Users_Guide/wrappers.rst | 26 ++++++-- .../tc_pairs/test_tc_pairs_wrapper.py | 66 ++++++++++++++----- metplus/wrappers/tc_pairs_wrapper.py | 22 ++++++- parm/met_config/TCPairsConfig_wrapped | 4 +- .../TCPairs/TCPairs_extra_tropical.conf | 9 ++- .../TCPairs/TCPairs_tropical.conf | 11 +++- 7 files changed, 127 insertions(+), 39 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index d8944f2000..3dd5e354dc 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -10094,13 +10094,33 @@ METplus Configuration Glossary | *Used by:* StatAnalysis - TC_PAIRS_DIAG_NAME - Specify the value for 'diag_name' in the MET configuration file for TCPairs. + TC_PAIRS_DIAG_INFO_MAP_DIAG_SOURCE + Specify the value for the nth 'diag_info_map.diag_source' in the MET configuration file for TCPairs. | *Used by:* TCPairs - TC_PAIRS_DIAG_CONVERT_MAP_SOURCE - Specify the value for the nth 'diag_convert_map.source' in the MET configuration file for TCPairs. + TC_PAIRS_DIAG_INFO_MAP_TRACK_SOURCE + Specify the value for the nth 'diag_info_map.track_source' in the MET configuration file for TCPairs. + + | *Used by:* TCPairs + + TC_PAIRS_DIAG_INFO_MAP_FIELD_SOURCE + Specify the value for the nth 'diag_info_map.field_source' in the MET configuration file for TCPairs. + + | *Used by:* TCPairs + + TC_PAIRS_DIAG_INFO_MAP_MATCH_TO_TRACK + Specify the value for the nth 'diag_info_map.match_to_track' in the MET configuration file for TCPairs. + + | *Used by:* TCPairs + + TC_PAIRS_DIAG_INFO_MAP_DIAG_NAME + Specify the value for the nth 'diag_info_map.diag_name' in the MET configuration file for TCPairs. + + | *Used by:* TCPairs + + TC_PAIRS_DIAG_CONVERT_MAP_DIAG_SOURCE + Specify the value for the nth 'diag_convert_map.diag_source' in the MET configuration file for TCPairs. | *Used by:* TCPairs diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index a4d088dbef..cff6d032f6 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -8225,8 +8225,12 @@ METplus Configuration | :term:`TC_PAIRS_CHECK_DUP` | :term:`TC_PAIRS_INTERP12` | :term:`TC_PAIRS_MATCH_POINTS` -| :term:`TC_PAIRS_DIAG_NAME` -| :term:`TC_PAIRS_DIAG_CONVERT_MAP_SOURCE` +| :term:`TC_PAIRS_DIAG_INFO_MAP_DIAG_SOURCE` +| :term:`TC_PAIRS_DIAG_INFO_MAP_TRACK_SOURCE` +| :term:`TC_PAIRS_DIAG_INFO_MAP_FIELD_SOURCE` +| :term:`TC_PAIRS_DIAG_INFO_MAP_MATCH_TO_TRACK` +| :term:`TC_PAIRS_DIAG_INFO_MAP_DIAG_NAME` +| :term:`TC_PAIRS_DIAG_CONVERT_MAP_DIAG_SOURCE` | :term:`TC_PAIRS_DIAG_CONVERT_MAP_KEY` | :term:`TC_PAIRS_DIAG_CONVERT_MAP_CONVERT` | @@ -8511,7 +8515,7 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`TC_PAIRS_INTERP12` - interp12 -**${METPLUS_DIAG_NAME}** +**${METPLUS_DIAG_INFO_MAP_LIST}** .. list-table:: :widths: 5 5 @@ -8519,8 +8523,16 @@ see :ref:`How METplus controls MET config file settings`. * - METplus Config(s) - MET Config File - * - :term:`TC_PAIRS_DIAG_NAME` - - diag_name + * - :term:`TC_PAIRS_DIAG_INFO_MAP_DIAG_SOURCE` + - diag_info_map.diag_source + * - :term:`TC_PAIRS_DIAG_INFO_MAP_TRACK_SOURCE` + - diag_info_map.track_source + * - :term:`TC_PAIRS_DIAG_INFO_MAP_FIELD_SOURCE` + - diag_info_map.field_source + * - :term:`TC_PAIRS_DIAG_INFO_MAP_MATCH_TO_TRACK` + - diag_info_map.match_to_track + * - :term:`TC_PAIRS_DIAG_INFO_MAP_DIAG_NAME` + - diag_info_map.diag_name **${METPLUS_DIAG_CONVERT_MAP_LIST}** @@ -8530,8 +8542,8 @@ see :ref:`How METplus controls MET config file settings`. * - METplus Config(s) - MET Config File - * - :term:`TC_PAIRS_DIAG_CONVERT_MAP_SOURCE` - - diag_convert_map.source + * - :term:`TC_PAIRS_DIAG_CONVERT_MAP_DIAG_SOURCE` + - diag_convert_map.diag_source * - :term:`TC_PAIRS_DIAG_CONVERT_MAP_KEY` - diag_convert_map.key * - :term:`TC_PAIRS_DIAG_CONVERT_MAP_CONVERT` diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index c632947fce..f27b6c2b71 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -485,34 +485,64 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, {'DIAG_ARG': ('-diag TCDIAG /some/path/2014121318.dat ' '-diag LSDIAG_RT /some/path/rt_2014121318.dat ' 'model=OFCL,SHIP')}), - # 45 diag_name - ('VALID', {'TC_PAIRS_DIAG_NAME': 'TC_DIAG, TC_DIAG2', }, - {'METPLUS_DIAG_NAME': 'diag_name = ["TC_DIAG", "TC_DIAG2"];'}), - # 46 diag_convert_map 1 dictionary in list + # 45 diag_convert_map 1 dictionary in list ('VALID', { - 'TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE': 'TCDIAG', + 'TC_PAIRS_DIAG_CONVERT_MAP1_DIAG_SOURCE': 'CIRA_DIAG', 'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', 'TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT': 'x/10', }, - {'METPLUS_DIAG_CONVERT_MAP_LIST': 'diag_convert_map = [{source = "TCDIAG";key = ["(10C)", "(10KT)", "(10M/S)"];convert(x) = x/10;}];'}), - # 47 diag_convert_map 2 dictionaries in list + {'METPLUS_DIAG_CONVERT_MAP_LIST': ( + 'diag_convert_map = [{diag_source = "CIRA_DIAG";' + 'key = ["(10C)", "(10KT)", "(10M/S)"];convert(x) = x/10;}];' + )}), + # 46 diag_convert_map 2 dictionaries in list ('VALID', { - 'TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE': 'TCDIAG', + 'TC_PAIRS_DIAG_CONVERT_MAP1_DIAG_SOURCE': 'CIRA_DIAG', 'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', 'TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT': 'x/10', - 'TC_PAIRS_DIAG_CONVERT_MAP2_SOURCE': 'LSDIAG_RT', + 'TC_PAIRS_DIAG_CONVERT_MAP2_DIAG_SOURCE': 'SHIPS_DIAG', 'TC_PAIRS_DIAG_CONVERT_MAP2_KEY': 'LAT,LON,CSST,RSST,DSST,DSTA', 'TC_PAIRS_DIAG_CONVERT_MAP2_CONVERT': 'x/100', }, - {'METPLUS_DIAG_CONVERT_MAP_LIST': ('diag_convert_map = [{source = ' - '"TCDIAG";key = ["(10C)", ' - '"(10KT)", "(10M/S)"];' - 'convert(x) = x/10;},' - '{source = "LSDIAG_RT";key = [' - '"LAT", "LON", "CSST", "RSST", ' - '"DSST", "DSTA"];' - 'convert(x) = x/100;}];')}), - + {'METPLUS_DIAG_CONVERT_MAP_LIST': ( + 'diag_convert_map = [{diag_source = "CIRA_DIAG";' + 'key = ["(10C)", "(10KT)", "(10M/S)"];convert(x) = x/10;},' + '{diag_source = "SHIPS_DIAG";key = ["LAT", "LON", "CSST", ' + '"RSST", "DSST", "DSTA"];convert(x) = x/100;}];' + )}), + # 47 diag_info_map 1 dictionary in list + ('VALID', { + 'TC_PAIRS_DIAG_INFO_MAP1_DIAG_SOURCE': 'CIRA_DIAG_RT', + 'TC_PAIRS_DIAG_INFO_MAP1_TRACK_SOURCE': 'GFS', + 'TC_PAIRS_DIAG_INFO_MAP1_FIELD_SOURCE': 'GFS_0p50', + 'TC_PAIRS_DIAG_INFO_MAP1_MATCH_TO_TRACK': 'GFS', + 'TC_PAIRS_DIAG_INFO_MAP1_DIAG_NAME': 'MY_NAME', + }, + {'METPLUS_DIAG_INFO_MAP_LIST': ( + 'diag_info_map = [{diag_source = "CIRA_DIAG_RT";' + 'track_source = "GFS";field_source = "GFS_0p50";' + 'match_to_track = ["GFS"];diag_name = ["MY_NAME"];}];')}), + # 48 diag_info_map 2 dictionaries in list + ('VALID', { + 'TC_PAIRS_DIAG_INFO_MAP1_DIAG_SOURCE': 'CIRA_DIAG_RT', + 'TC_PAIRS_DIAG_INFO_MAP1_TRACK_SOURCE': 'GFS', + 'TC_PAIRS_DIAG_INFO_MAP1_FIELD_SOURCE': 'GFS_0p50', + 'TC_PAIRS_DIAG_INFO_MAP1_MATCH_TO_TRACK': 'GFS', + 'TC_PAIRS_DIAG_INFO_MAP1_DIAG_NAME': 'MY_NAME', + 'TC_PAIRS_DIAG_INFO_MAP2_DIAG_SOURCE': 'SHIPS_DIAG_RT', + 'TC_PAIRS_DIAG_INFO_MAP2_TRACK_SOURCE': 'OFCL', + 'TC_PAIRS_DIAG_INFO_MAP2_FIELD_SOURCE': 'GFS_0p50', + 'TC_PAIRS_DIAG_INFO_MAP2_MATCH_TO_TRACK': 'OFCL', + 'TC_PAIRS_DIAG_INFO_MAP2_DIAG_NAME': 'MY_NAME2', + }, + {'METPLUS_DIAG_INFO_MAP_LIST': ( + 'diag_info_map = [{diag_source = "CIRA_DIAG_RT";' + 'track_source = "GFS";field_source = "GFS_0p50";' + 'match_to_track = ["GFS"];diag_name = ["MY_NAME"];},' + '{diag_source = "SHIPS_DIAG_RT";track_source = "OFCL";' + 'field_source = "GFS_0p50";match_to_track = ["OFCL"]' + ';diag_name = ["MY_NAME2"];}];' + )}), ] ) @pytest.mark.wrapper diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index 9c52127ba8..d85cf1fb1a 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -63,7 +63,7 @@ class TCPairsWrapper(CommandBuilder): 'METPLUS_CHECK_DUP', 'METPLUS_INTERP12', 'METPLUS_MATCH_POINTS', - 'METPLUS_DIAG_NAME', + 'METPLUS_DIAG_INFO_MAP_LIST', 'METPLUS_DIAG_CONVERT_MAP_LIST', ] @@ -180,7 +180,7 @@ def create_c_dict(self): self.add_met_config(name='match_points', data_type='bool') - self.add_met_config(name='diag_name', data_type='list') + self._handle_diag_info_map() self._handle_diag_convert_map() @@ -419,9 +419,25 @@ def _handle_consensus(self): if not return_code: self.isOK = False + def _handle_diag_info_map(self): + dict_items = { + 'diag_source': 'string', + 'track_source': 'string', + 'field_source': 'string', + 'match_to_track': 'list', + 'diag_name': 'list', + } + return_code = add_met_config_dict_list(config=self.config, + app_name=self.app_name, + output_dict=self.env_var_dict, + dict_name='diag_info_map', + dict_items=dict_items) + if not return_code: + self.isOK = False + def _handle_diag_convert_map(self): dict_items = { - 'source': 'string', + 'diag_source': 'string', 'key': 'list', 'convert': ('string', 'remove_quotes'), } diff --git a/parm/met_config/TCPairsConfig_wrapped b/parm/met_config/TCPairsConfig_wrapped index 4ee11dff9b..0a538a2007 100644 --- a/parm/met_config/TCPairsConfig_wrapped +++ b/parm/met_config/TCPairsConfig_wrapped @@ -144,8 +144,8 @@ watch_warn = { } -//diag_name = -${METPLUS_DIAG_NAME} +//diag_info_map = { +${METPLUS_DIAG_INFO_MAP_LIST} //diag_convert_map = { ${METPLUS_DIAG_CONVERT_MAP_LIST} diff --git a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf index 92c98e5525..3b63db308e 100644 --- a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf +++ b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf @@ -107,7 +107,12 @@ TC_PAIRS_MISSING_VAL = -9999 #TC_PAIRS_MATCH_POINTS = -#TC_PAIRS_DIAG_NAME = -#TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE = +#TC_PAIRS_DIAG_INFO_MAP1_DIAG_SOURCE = +#TC_PAIRS_DIAG_INFO_MAP1_TRACK_SOURCE = +#TC_PAIRS_DIAG_INFO_MAP1_FIELD_SOURCE = +#TC_PAIRS_DIAG_INFO_MAP1_MATCH_TO_TRACK = +#TC_PAIRS_DIAG_INFO_MAP1_DIAG_NAME = + +#TC_PAIRS_DIAG_CONVERT_MAP1_DIAG_SOURCE = #TC_PAIRS_DIAG_CONVERT_MAP1_KEY = #TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT = diff --git a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf index ca20066421..beca82e6d7 100644 --- a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf +++ b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf @@ -101,7 +101,12 @@ TC_PAIRS_DLAND_FILE = MET_BASE/tc_data/dland_global_tenth_degree.nc #TC_PAIRS_MATCH_POINTS = -#TC_PAIRS_DIAG_NAME = -#TC_PAIRS_DIAG_CONVERT_MAP1_SOURCE = +#TC_PAIRS_DIAG_INFO_MAP1_DIAG_SOURCE = +#TC_PAIRS_DIAG_INFO_MAP1_TRACK_SOURCE = +#TC_PAIRS_DIAG_INFO_MAP1_FIELD_SOURCE = +#TC_PAIRS_DIAG_INFO_MAP1_MATCH_TO_TRACK = +#TC_PAIRS_DIAG_INFO_MAP1_DIAG_NAME = + +#TC_PAIRS_DIAG_CONVERT_MAP1_DIAG_SOURCE = #TC_PAIRS_DIAG_CONVERT_MAP1_KEY = -#TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT = \ No newline at end of file +#TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT = From aa91458bfc3b203fba057a966c69e87dc35bf48b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 16 Nov 2022 15:38:12 -0700 Subject: [PATCH 20/20] Per #1898, removed setting of models in -diag arguments because they are now set in diag_info_map.match_to_track. Added documentation for TC_PAIRS_DIAG_SOURCE, TC_PAIRS_DIAG_DIR, and TC_PAIRS_DIAG_TEMPLATE that was missing --- docs/Users_Guide/glossary.rst | 15 +++++++++++++ docs/Users_Guide/wrappers.rst | 3 +++ .../tc_pairs/test_tc_pairs_wrapper.py | 21 ++++++------------- metplus/wrappers/tc_pairs_wrapper.py | 6 ------ .../TCPairs/TCPairs_extra_tropical.conf | 4 ++++ .../TCPairs/TCPairs_tropical.conf | 4 ++++ 6 files changed, 32 insertions(+), 21 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 3dd5e354dc..7a270558c4 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -10134,6 +10134,21 @@ METplus Configuration Glossary | *Used by:* TCPairs + TC_PAIRS_DIAG_DIR + Specify the (optional) directory for the nth -diag argument for TCPairs. + + | *Used by:* TCPairs + + TC_PAIRS_DIAG_TEMPLATE + Specify the (optional) template for the nth -diag argument for TCPairs. + + | *Used by:* TCPairs + + TC_PAIRS_DIAG_SOURCE + Specify the (optional) source string for the nth -diag argument for TCPairs. + + | *Used by:* TCPairs + TC_STAT_DIAG_THRESH_NAME Specify the value for 'diag_thresh_name' in the MET configuration file for TCStat. diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index cff6d032f6..6f86e71335 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -8233,6 +8233,9 @@ METplus Configuration | :term:`TC_PAIRS_DIAG_CONVERT_MAP_DIAG_SOURCE` | :term:`TC_PAIRS_DIAG_CONVERT_MAP_KEY` | :term:`TC_PAIRS_DIAG_CONVERT_MAP_CONVERT` +| :term:`TC_PAIRS_DIAG_SOURCE\` +| :term:`TC_PAIRS_DIAG_TEMPLATE\` +| :term:`TC_PAIRS_DIAG_DIR\` | .. warning:: **DEPRECATED:** diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index f27b6c2b71..958ae8aacb 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -467,25 +467,16 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, 'TC_PAIRS_DIAG_SOURCE1': 'TCDIAG', }, {'DIAG_ARG': '-diag TCDIAG /some/path/2014121318.dat'}), - # 43 -diag argument with models - ('VALID', { - 'TC_PAIRS_DIAG_TEMPLATE1': '/some/path/{valid?fmt=%Y%m%d%H}.dat', - 'TC_PAIRS_DIAG_SOURCE1': 'TCDIAG', - 'TC_PAIRS_DIAG_MODELS1': 'OFCL, SHIP', - }, - {'DIAG_ARG': '-diag TCDIAG /some/path/2014121318.dat model=OFCL,SHIP'}), - # 44 2 -diag arguments + # 43 2 -diag arguments ('VALID', { 'TC_PAIRS_DIAG_TEMPLATE1': '/some/path/{valid?fmt=%Y%m%d%H}.dat', 'TC_PAIRS_DIAG_SOURCE1': 'TCDIAG', 'TC_PAIRS_DIAG_TEMPLATE2': '/some/path/rt_{valid?fmt=%Y%m%d%H}.dat', 'TC_PAIRS_DIAG_SOURCE2': 'LSDIAG_RT', - 'TC_PAIRS_DIAG_MODELS2': 'OFCL, SHIP', }, {'DIAG_ARG': ('-diag TCDIAG /some/path/2014121318.dat ' - '-diag LSDIAG_RT /some/path/rt_2014121318.dat ' - 'model=OFCL,SHIP')}), - # 45 diag_convert_map 1 dictionary in list + '-diag LSDIAG_RT /some/path/rt_2014121318.dat')}), + # 44 diag_convert_map 1 dictionary in list ('VALID', { 'TC_PAIRS_DIAG_CONVERT_MAP1_DIAG_SOURCE': 'CIRA_DIAG', 'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', @@ -495,7 +486,7 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, 'diag_convert_map = [{diag_source = "CIRA_DIAG";' 'key = ["(10C)", "(10KT)", "(10M/S)"];convert(x) = x/10;}];' )}), - # 46 diag_convert_map 2 dictionaries in list + # 45 diag_convert_map 2 dictionaries in list ('VALID', { 'TC_PAIRS_DIAG_CONVERT_MAP1_DIAG_SOURCE': 'CIRA_DIAG', 'TC_PAIRS_DIAG_CONVERT_MAP1_KEY': '(10C),(10KT),(10M/S)', @@ -510,7 +501,7 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, '{diag_source = "SHIPS_DIAG";key = ["LAT", "LON", "CSST", ' '"RSST", "DSST", "DSTA"];convert(x) = x/100;}];' )}), - # 47 diag_info_map 1 dictionary in list + # 46 diag_info_map 1 dictionary in list ('VALID', { 'TC_PAIRS_DIAG_INFO_MAP1_DIAG_SOURCE': 'CIRA_DIAG_RT', 'TC_PAIRS_DIAG_INFO_MAP1_TRACK_SOURCE': 'GFS', @@ -522,7 +513,7 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, 'diag_info_map = [{diag_source = "CIRA_DIAG_RT";' 'track_source = "GFS";field_source = "GFS_0p50";' 'match_to_track = ["GFS"];diag_name = ["MY_NAME"];}];')}), - # 48 diag_info_map 2 dictionaries in list + # 47 diag_info_map 2 dictionaries in list ('VALID', { 'TC_PAIRS_DIAG_INFO_MAP1_DIAG_SOURCE': 'CIRA_DIAG_RT', 'TC_PAIRS_DIAG_INFO_MAP1_TRACK_SOURCE': 'GFS', diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index d85cf1fb1a..3e06230117 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -478,13 +478,9 @@ def _handle_diag(self, c_dict): source = ( self.config.getraw('config', f'TC_PAIRS_DIAG_SOURCE{idx}') ) - models = getlist( - self.config.getraw('config', f'TC_PAIRS_DIAG_MODELS{idx}') - ) diag_info = { 'template': template, 'source': source, - 'models': models, } diag_info_list.append(diag_info) @@ -1023,8 +1019,6 @@ def _get_diag_file(self, time_info): return False arg = f"-diag {diag_info['source']} {' '.join(filepaths)}" - if diag_info['models']: - arg = f"{arg} model={','.join(diag_info['models'])}" self.args.append(arg) return True diff --git a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf index 3b63db308e..15f2c69811 100644 --- a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf +++ b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf @@ -47,6 +47,10 @@ TC_PAIRS_BDECK_TEMPLATE = {date?fmt=%Y%m}/b{basin?fmt=%s}q{date?fmt=%Y%m}*.gfso. TC_PAIRS_REFORMAT_DIR = {OUTPUT_BASE}/track_data_atcf +#TC_PAIRS_DIAG_DIR1 = +#TC_PAIRS_DIAG_TEMPLATE1 = +#TC_PAIRS_DIAG_SOURCE1 = + TC_PAIRS_OUTPUT_DIR = {OUTPUT_BASE}/tc_pairs TC_PAIRS_OUTPUT_TEMPLATE = {date?fmt=%Y%m}/{basin?fmt=%s}q{date?fmt=%Y%m%d%H}.gfso.{cyclone?fmt=%s} diff --git a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf index beca82e6d7..26f1c27238 100644 --- a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf +++ b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf @@ -50,6 +50,10 @@ TC_PAIRS_BDECK_TEMPLATE = b{basin?fmt=%s}{cyclone?fmt=%s}{date?fmt=%Y}.dat TC_PAIRS_EDECK_INPUT_DIR = TC_PAIRS_EDECK_TEMPLATE = +#TC_PAIRS_DIAG_DIR1 = +#TC_PAIRS_DIAG_TEMPLATE1 = +#TC_PAIRS_DIAG_SOURCE1 = + TC_PAIRS_OUTPUT_DIR = {OUTPUT_BASE}/tc_pairs TC_PAIRS_OUTPUT_TEMPLATE = tc_pairs_{basin?fmt=%s}{date?fmt=%Y%m%d%H}.dat