per #2578, add MISSING before file path that is not found if input_th…

…resh is set and less than 1.0, added unit test to ensure correct behavior occurs

internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py

@@ -17,12 +17,7 @@ def get_test_data_dir(config, subdir=None):
     return top_dir
 
 
-def pcp_combine_wrapper(metplus_config, d_type):
-    """! Returns a default PCPCombineWrapper with /path/to entries in the
-         metplus_system.conf and metplus_runtime.conf configuration
-         files.  Subsequent tests can customize the final METplus configuration
-         to over-ride these /path/to values."""
-    config = metplus_config
+def set_minimum_config_settings(config, d_type):
     config.set('config', 'FCST_PCP_COMBINE_INPUT_ACCUMS', '6')
     config.set('config', 'FCST_PCP_COMBINE_INPUT_NAMES', 'P06M_NONE')
     config.set('config', 'FCST_PCP_COMBINE_INPUT_LEVELS', '"(*,*)"')
@@ -56,6 +51,13 @@ def pcp_combine_wrapper(metplus_config, d_type):
     elif d_type == "OBS":
         config.set('config', 'OBS_PCP_COMBINE_RUN', True)
 
+def pcp_combine_wrapper(metplus_config, d_type):
+    """! Returns a default PCPCombineWrapper with /path/to entries in the
+         metplus_system.conf and metplus_runtime.conf configuration
+         files.  Subsequent tests can customize the final METplus configuration
+         to over-ride these /path/to values."""
+    config = metplus_config
+    set_minimum_config_settings(config, d_type)
     return PCPCombineWrapper(config)
 
 
@@ -861,3 +863,45 @@ def test_subtract_method_zero_accum(metplus_config):
         for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds):
             # ensure commands are generated as expected
             assert cmd == expected_cmd
+
+
+@pytest.mark.parametrize(
+    'thresh, success', [
+        (None, False),
+        (0.6, True),
+        (1.0, False),
+    ]
+)
+@pytest.mark.wrapper
+def test_add_method_missing_input(metplus_config, thresh, success):
+    data_src = "OBS"
+    lookback = 6 * 3600
+    task_info = {
+        'valid': datetime.strptime("2016090415", '%Y%m%d%H')
+    }
+    time_info = ti_calculate(task_info)
+
+    config = metplus_config
+    set_minimum_config_settings(config, data_src)
+    if thresh is not None:
+        config.set('config', f'{data_src}_PCP_COMBINE_INPUT_THRESH', thresh)
+    wrapper = PCPCombineWrapper(config)
+
+    input_dir = get_test_data_dir(wrapper.config, subdir='accum')
+    files_found = wrapper.setup_add_method(time_info, lookback, data_src)
+    if not success:
+        assert not files_found
+        return
+
+    assert files_found
+
+    in_files = [item[0] for item in files_found]
+    print(f"Input files: {in_files}")
+    assert (len(in_files) == 6 and
+            f"{input_dir}/20160904/file.2016090415.01h" in in_files and
+            f"{input_dir}/20160904/file.2016090414.01h" in in_files and
+            f"{input_dir}/20160904/file.2016090413.01h" in in_files and
+            f"{input_dir}/20160904/file.2016090412.01h" in in_files and
+            f"MISSING{input_dir}/20160904/file.2016090411.01h" in in_files and
+            f"MISSING{input_dir}/20160904/file.2016090410.01h" in in_files
+            )

metplus/wrappers/pcp_combine_wrapper.py

@@ -171,16 +171,38 @@ def _set_fcst_or_obs_dict_items(self, d_type, c_dict):
             met_tool=self.app_name
         )
 
-        c_dict[f'{d_type}_INPUT_THRESH'] = (
-            self.config.getfloat('config', f'{d_type}_INPUT_THRESH')
-        )
-
+        self._set_input_thresh(c_dict, d_type)
         self._error_check_config(c_dict, d_type)
 
         # skip RuntimeFreq input file logic - remove once integrated
         c_dict['FIND_FILES'] = False
         return c_dict
 
+    def _set_input_thresh(self, c_dict, d_type):
+        """!Read input_thresh value from METplusConfig and set c_dict. Report
+        an error if value is not between 0 and 1. Set {d_type}_FILL_MISSING to
+        True if input_thresh is less than 1, meaning missing input is allowed.
+
+        @param c_dict dictionary to set values
+        @param d_type data type, either 'FCST' or 'OBS'
+        """
+        input_thresh = (
+            self.config.getfloat('config', f'{d_type}_PCP_COMBINE_INPUT_THRESH')
+        )
+        if input_thresh == float(MISSING_DATA_VALUE):
+            return
+
+        if input_thresh < 0 or input_thresh > 1:
+            self.log_error(f'{d_type}_PCP_COMBINE_INPUT_THRESH must be 0-1')
+            return
+
+        c_dict[f'{d_type}_INPUT_THRESH'] = input_thresh
+
+        # if missing input is allowed, add MISSING to path if file is not found
+        # subtract method does not support missing inputs
+        if input_thresh < 1 and c_dict[f'{d_type}_RUN_METHOD'] != "SUBTRACT":
+            c_dict[f'{d_type}_FILL_MISSING'] = True
+
     def _error_check_config(self, c_dict, d_type):
         """!Check c_dict values and log errors if anything is not set properly.
 
@@ -932,8 +954,8 @@ def _handle_name_argument(self, output_name, data_src):
         self.args.append(name_format)
 
     def _handle_input_thresh_argument(self, data_src):
-        input_thresh = self.c_dict[f'{data_src}_INPUT_THRESH']
-        if input_thresh == float(MISSING_DATA_VALUE):
+        input_thresh = self.c_dict.get(f'{data_src}_INPUT_THRESH')
+        if not input_thresh:
             return
 
         self.args.append(f'-input_thresh {input_thresh}')