diff --git a/.coveragerc b/.coveragerc index 4c62180950..04ab2270b8 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,3 +3,4 @@ relative_files = True source = metplus omit = metplus/wrappers/cyclone_plotter_wrapper.py + metplus/produtil/** \ No newline at end of file diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 05cc0b6196..8d12b81db2 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -276,6 +276,12 @@ "disabled": true, "run": false }, + { + "category": "short_range", + "index_list": "15", + "disabled": true, + "run":false + }, { "category": "space_weather", "index_list": "0-1", diff --git a/.github/update_truth_change_log.txt b/.github/update_truth_change_log.txt index fd6a31913f..1413e26b3e 100644 --- a/.github/update_truth_change_log.txt +++ b/.github/update_truth_change_log.txt @@ -7,3 +7,4 @@ [20240905_16:45:56 develop] dtcenter/MET#2959 - Correct bugs in SEDI and BAGSS contingency table statistics found in the CTS line type [20240910_19:03:38 develop] dtcenter/MET#2967 - Fixes integer overflow bug in the computation of the CTS and NBRCTS ORSS, ORSS_NCL, and ORSS_NCU columns. [20240926_16:50:38 develop] #2701 - #2701 added a new use case (SeriesAnalysis_aggr) that produces new output +[20241014_16:07:34 develop] dtcenter/MET#2988 - dtcenter/MET#2988 -- see issue #2719 for details diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index 10fe4c9a7a..56f8815dbe 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -169,6 +169,7 @@ Use Cases that Exceed GitHub Actions Disk Space ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - *model_applications/s2s_stratosphere/UserScript_fcstGFS_obsERA_StratospherePolar* - *model_applications/s2s_stratosphere/UserScript_fcstGFS_obsERA_StratosphereQBO* +- *model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning* .. _use_case_documentation: diff --git a/docs/Contributors_Guide/documentation.rst b/docs/Contributors_Guide/documentation.rst index a340774bd9..89373f4804 100644 --- a/docs/Contributors_Guide/documentation.rst +++ b/docs/Contributors_Guide/documentation.rst @@ -218,7 +218,7 @@ for more information. **What to bold:** - * Variables (e.g. **MET_INSTALL_DIR, INPUT_BASE, METCALCPY_HOME**, etc.) + * Environment Variables (e.g. **MET_INSTALL_DIR, INPUT_BASE, METCALCPY_HOME**, etc.) * Filenames (**line_defaults.yaml, contour_defaults.yaml, defaults.conf**, etc.) Italics @@ -395,6 +395,20 @@ resulting in the following displayed text: See `Overleaf's Guide - Matrices `_ for more information. +Literal Include +--------------- + +This feature is used often in the Release Guide, but there are other places it could be useful. +Using a literal include ensures only having to update the documentation in one place. +It is important to use a relative path (i.e. "../../") to get to the correct +directory of the file being referenced in the literal include. This will keep the +file linking to the correct version and branch. + +As an example, refer to the METplus User's Guide. See this +`line `_ +in the PointStat Wrapper Use Case documentation, which uses RST's :code:`.. literalinclude::` +and renders the text of the referenced file in this +`section `_. Literal Blocks -------------- @@ -425,6 +439,54 @@ and `literal blocks `_ for more information. +Substitution References +----------------------- + +Substitution references are replacements in the text as defined in the :code:`rst_epilog` +in the **docs/conf.py** file. For example, instead of hard coding the MET version in +the METplus documentation, a variable :code:`met_version` can be set to a value +(e.g. 12.0.0) in the **docs/conf.py** file in the following way: + +.. code-block:: ini + + met_version = 12.0.0 + +Then, this variable is used in the :code:`rst_epilog` to set up the substitution, +along with other variables. For example, in the **docs/conf.py** +file the following is defined: + +.. code-block:: ini + + rst_epilog = f""" + .. |copyright| replace:: {copyright} + .. |author_list| replace:: {author_list} + .. |release_date| replace:: {release_date} + .. |release_year| replace:: {release_year} + .. |release_info| replace:: {release_info} + .. |python_version| replace:: {python_version} + .. |met_version| replace:: {met_version} + """ + +The value 12.0.0 would then be replaced in the documentation wherever +:code:`|met_version|` is used in .rst files. For example: + +.. code-block:: ini + + MET version |met_version| or above + +There are some +`reserved substitution variables `_ +in Sphinx. These variables are +:code:`|release|`, :code:`|version|`, :code:`|today|`, and +:code:`|translation progress|`. Reserved substitution variables do not +need to be added to :code:`rst_epilog`, but other variables do. + +See +`Substitutions `_ +and +`rst_epilog `_ +for more information. + Line Blocks ----------- @@ -1413,8 +1475,7 @@ Read the Docs supports multiple versions for each repository. For the METplus components, the "latest" version will point to the latest official (stable) release. The "develop" or "development" version will point to the most up to date development code. There may also be other previous versions of the -software available in the version selector menu, which is accessible by -clicking in the bottom left corner of the documentation pages. +software available in the version selector menu. Automation rules allow project maintainers to automate actions on new branches and tags on repositories. For the METplus components, documentation is diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 6bc690bf76..2dcd640c4f 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -290,7 +290,12 @@ METplus Configuration Glossary | *Used by:* Point2Grid POINT2GRID_QC_FLAGS - Specify the qc flags name that is read by Point2Grid. + .. warning:: **DEPRECATED:** Please use :term:`POINT2GRID_GOES_QC_FLAGS`. + + | *Used by:* Point2Grid + + POINT2GRID_GOES_QC_FLAGS + Sets the value for the -goes_qc command line argument for Point2Grid. | *Used by:* Point2Grid @@ -11624,6 +11629,46 @@ METplus Configuration Glossary | *Used by:* PointStat + POINT2GRID_VALID_TIME + Specify the value for 'valid_time' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_OBS_WINDOW_BEG + Specify the value for 'obs_window.beg' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_OBS_WINDOW_END + Specify the value for 'obs_window.end' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_MESSAGE_TYPE + Specify the value for 'message_type' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_VAR_NAME_MAP_KEY + Specify the value for the nth 'var_name_map.key' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_VAR_NAME_MAP_VAL + Specify the value for the nth 'var_name_map.val' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_OBS_QUALITY_INC + Specify the value for 'obs_quality_inc' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_OBS_QUALITY_EXC + Specify the value for 'obs_quality_exc' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + MADIS2NC_CUSTOM_LOOP_LIST Sets custom string loop list for a specific wrapper. See :term:`CUSTOM_LOOP_LIST`. @@ -11812,105 +11857,39 @@ METplus Configuration Glossary | *Used by:* MADIS2NC - MADIS2NC_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for MADIS2NC. - - | *Used by:* MADIS2NC - - ASCII2NC_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for ASCII2NC. - - | *Used by:* ASCII2NC - - ENSEMBLE_STAT_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for EnsembleStat. - - | *Used by:* EnsembleStat - - GEN_ENS_PROD_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for GenEnsProd. - - | *Used by:* GenEnsProd - - GRID_DIAG_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for GridDiag. - - | *Used by:* GridDiag - GRID_STAT_TIME_OFFSET_WARNING Specify the value for 'time_offset_warning' in the MET configuration file for GridStat. | *Used by:* GridStat - IODA2NC_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for IODA2NC. - - | *Used by:* IODA2NC - MODE_TIME_OFFSET_WARNING Specify the value for 'time_offset_warning' in the MET configuration file for MODE. | *Used by:* MODE - MTD_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for MTD. - - | *Used by:* MTD - - PB2NC_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for PB2NC. - - | *Used by:* PB2NC - - PLOT_POINT_OBS_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for PlotPointObs. - - | *Used by:* PlotPointObs - - POINT_STAT_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for PointStat. - - | *Used by:* PointStat - - STAT_ANALYSIS_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for StatAnalysis. - - | *Used by:* StatAnalysis - SERIES_ANALYSIS_TIME_OFFSET_WARNING Specify the value for 'time_offset_warning' in the MET configuration file for SeriesAnalysis. | *Used by:* SeriesAnalysis - TC_DIAG_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for TCDiag. - - | *Used by:* TCDiag - - TC_GEN_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for TCGen. - - | *Used by:* TCGen - - TC_PAIRS_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for TCPairs. - - | *Used by:* TCPairs - - TC_RMW_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for TCRMW. + WAVELET_STAT_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for WaveletStat. - | *Used by:* TCRMW + | *Used by:* WaveletStat - TC_STAT_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for TCStat. + POINT2GRID_MET_CONFIG_OVERRIDES + Override any variables in the MET configuration file that are not + supported by the wrapper. This should be set to the full variable name + and value that you want to override, including the equal sign and the + ending semi-colon. The value is directly appended to the end of the + wrapped MET config file. - | *Used by:* TCStat + Example: + POINT2GRID_MET_CONFIG_OVERRIDES = desc = "override_desc"; model = "override_model"; - WAVELET_STAT_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for WaveletStat. + See :ref:`Overriding Unsupported MET config file settings` for more information - | *Used by:* WaveletStat + | *Used by:* Point2Grid ASCII2NC_VALID_BEG Specify the value for the command line argument '-valid_beg' for ASCII2NC. diff --git a/docs/Users_Guide/quicksearch.rst b/docs/Users_Guide/quicksearch.rst index 30aaca532e..c51d262d3d 100644 --- a/docs/Users_Guide/quicksearch.rst +++ b/docs/Users_Guide/quicksearch.rst @@ -156,6 +156,7 @@ Use Cases by METplus Feature: | `METplotpy <../search.html?q=METplotpyUseCase&check_keywords=yes&area=default>`_ | `MET_PYTHON_EXE Environment Variable <../search.html?q=MET_PYTHON_EXEUseCase&check_keywords=yes&area=default>`_ | `Multiple Conf File Use <../search.html?q=MultiConfUseCase&check_keywords=yes&area=default>`_ + | `MultivariateMODE <../search.html?q=MvMODEUseCase&check_keywords=yes&area=default>`_ | `Observation Time Summary <../search.html?q=ObsTimeSummaryUseCase&check_keywords=yes&area=default>`_ | `Observation Uncertainty <../search.html?q=ObsUncertaintyUseCase&check_keywords=yes&area=default>`_ | `Python Embedding Ingest <../search.html?q=PyEmbedIngestToolUseCase&check_keywords=yes&area=default>`_ @@ -187,6 +188,7 @@ Use Cases by METplus Feature: | **METplotpy**: *METplotpyUseCase* | **MET_PYTHON_EXE Environment Variable**: *MET_PYTHON_EXEUseCase* | **Multiple Conf File Use**: *MultiConfUseCase* + | **MultivariateMODE**: *MvMODEUseCase* | **Observation Time Summary**: *ObsTimeSummaryUseCase* | **Observation Uncertainty**: *ObsUncertaintyUseCase* | **Python Embedding Ingest**: *PyEmbedIngestToolUseCase* diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index c4da8506ad..aa2794377c 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -64,7 +64,6 @@ METplus Configuration | :term:`ASCII2NC_INC_VALID_TIMES` | :term:`ASCII2NC_SKIP_INIT_TIMES` | :term:`ASCII2NC_INC_INIT_TIMES` -| :term:`ASCII2NC_TIME_OFFSET_WARNING` | .. _ascii2nc-met-conf: @@ -124,18 +123,6 @@ ${METPLUS_TIME_SUMMARY_DICT} * - :term:`ASCII2NC_TIME_SUMMARY_VALID_THRESH` - time_summary.vld_thresh -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`ASCII2NC_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -391,7 +378,6 @@ METplus Configuration | :term:`ENSEMBLE_STAT_INC_VALID_TIMES` | :term:`ENSEMBLE_STAT_SKIP_INIT_TIMES` | :term:`ENSEMBLE_STAT_INC_INIT_TIMES` -| :term:`ENSEMBLE_STAT_TIME_OFFSET_WARNING` | .. warning:: **DEPRECATED:** @@ -1117,18 +1103,6 @@ ${METPLUS_OUTPUT_PREFIX} * - :term:`ENSEMBLE_STAT_OUTPUT_PREFIX` - output_prefix -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`ENSEMBLE_STAT_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -1352,7 +1326,6 @@ METplus Configuration | :term:`GEN_ENS_PROD_INC_VALID_TIMES` | :term:`GEN_ENS_PROD_SKIP_INIT_TIMES` | :term:`GEN_ENS_PROD_INC_INIT_TIMES` -| :term:`GEN_ENS_PROD_TIME_OFFSET_WARNING` .. _gen-ens-prod-met-conf: @@ -1706,18 +1679,6 @@ ${METPLUS_ENSEMBLE_FLAG_DICT} * - :term:`GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDP` - ensemble_flag.climo_cdp -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`GEN_ENS_PROD_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -3067,7 +3028,6 @@ METplus Configuration | :term:`GRID_DIAG_INC_VALID_TIMES` | :term:`GRID_DIAG_SKIP_INIT_TIMES` | :term:`GRID_DIAG_INC_INIT_TIMES` -| :term:`GRID_DIAG_TIME_OFFSET_WARNING` | .. _grid-diag-met-conf: @@ -3192,18 +3152,6 @@ ${METPLUS_MASK_DICT} .. note:: Since the default value in the MET config file for 'grid' is grid = [ "FULL" ];, setting GRID_DIAG_MASK_GRID to an empty string will result in a value of grid = []; in the MET config file. -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`GRID_DIAG_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -4195,7 +4143,6 @@ METplus Configuration | :term:`IODA2NC_INC_VALID_TIMES` | :term:`IODA2NC_SKIP_INIT_TIMES` | :term:`IODA2NC_INC_INIT_TIMES` -| :term:`IODA2NC_TIME_OFFSET_WARNING` .. _ioda2nc-met-conf: @@ -4418,18 +4365,6 @@ ${METPLUS_TIME_SUMMARY_DICT} * - :term:`IODA2NC_TIME_SUMMARY_VLD_THRESH` - time_summary.vld_thresh -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`IODA2NC_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -4490,7 +4425,6 @@ METplus Configuration | :term:`MADIS2NC_INC_VALID_TIMES` | :term:`MADIS2NC_SKIP_INIT_TIMES` | :term:`MADIS2NC_INC_INIT_TIMES` -| :term:`MADIS2NC_TIME_OFFSET_WARNING` | .. _madis2nc-met-conf: @@ -4550,18 +4484,6 @@ ${METPLUS_TIME_SUMMARY_DICT} * - :term:`MADIS2NC_TIME_SUMMARY_VLD_THRESH` - time_summary.vld_thresh -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`MADIS2NC_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -5690,7 +5612,6 @@ METplus Configuration | :term:`MTD_INC_VALID_TIMES` | :term:`MTD_SKIP_INIT_TIMES` | :term:`MTD_INC_INIT_TIMES` -| :term:`MTD_TIME_OFFSET_WARNING` | .. warning:: **DEPRECATED:** @@ -5923,18 +5844,6 @@ ${METPLUS_OUTPUT_PREFIX} * - :term:`MTD_OUTPUT_PREFIX` - output_prefix -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`MTD_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -5995,7 +5904,6 @@ METplus Configuration | :term:`PB2NC_INC_VALID_TIMES` | :term:`PB2NC_SKIP_INIT_TIMES` | :term:`PB2NC_INC_INIT_TIMES` -| :term:`PB2NC_TIME_OFFSET_WARNING` .. warning:: **DEPRECATED:** @@ -6200,18 +6108,6 @@ ${METPLUS_TIME_SUMMARY_DICT} * - :term:`PB2NC_TIME_SUMMARY_VALID_THRESH` - time_summary.vld_thresh -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`PB2NC_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -6484,7 +6380,6 @@ Configuration | :term:`PLOT_POINT_OBS_INC_VALID_TIMES` | :term:`PLOT_POINT_OBS_SKIP_INIT_TIMES` | :term:`PLOT_POINT_OBS_INC_INIT_TIMES` -| :term:`PLOT_POINT_OBS_TIME_OFFSET_WARNING` .. _plot-point-obs-met-conf: @@ -6814,18 +6709,6 @@ ${METPLUS_POINT_DATA} * - :term:`PLOT_POINT_OBS_POINT_DATA` - point_data -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`PLOT_POINT_OBS_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -6862,7 +6745,7 @@ METplus Configuration | :term:`POINT2GRID_REGRID_TO_GRID` | :term:`POINT2GRID_INPUT_FIELD` | :term:`POINT2GRID_INPUT_LEVEL` -| :term:`POINT2GRID_QC_FLAGS` +| :term:`POINT2GRID_GOES_QC_FLAGS` | :term:`POINT2GRID_ADP` | :term:`POINT2GRID_REGRID_METHOD` | :term:`POINT2GRID_GAUSSIAN_DX` @@ -6875,8 +6758,134 @@ METplus Configuration | :term:`POINT2GRID_INC_VALID_TIMES` | :term:`POINT2GRID_SKIP_INIT_TIMES` | :term:`POINT2GRID_INC_INIT_TIMES` +| :term:`POINT2GRID_VALID_TIME` +| :term:`POINT2GRID_OBS_WINDOW_BEG` +| :term:`POINT2GRID_OBS_WINDOW_END` +| :term:`POINT2GRID_MESSAGE_TYPE` +| :term:`POINT2GRID_VAR_NAME_MAP_KEY` +| :term:`POINT2GRID_VAR_NAME_MAP_VAL` +| :term:`POINT2GRID_OBS_QUALITY_INC` +| :term:`POINT2GRID_OBS_QUALITY_EXC` +| :term:`POINT2GRID_MET_CONFIG_OVERRIDES` | +.. warning:: **DEPRECATED:** + + | :term:`POINT2GRID_QC_FLAGS` + +.. _point2grid-met-conf: + +MET Configuration +----------------- + +Below is the wrapped MET configuration file used for this wrapper. +Environment variables are used to control entries in this configuration file. +The default value for each environment variable is obtained from +(except where noted below): + +`MET_INSTALL_DIR/share/met/config/Point2GridConfig_default `_ + +Below the file contents are descriptions of each environment variable +referenced in this file and the corresponding METplus configuration item used +to set the value of the environment variable. For detailed examples showing +how METplus sets the values of these environment variables, +see :ref:`How METplus controls MET config file settings`. + +.. dropdown:: Click to view parm/met_config/Point2GridConfig_wrapped + + .. literalinclude:: ../../parm/met_config/Point2GridConfig_wrapped + +Environment variables in wrapped MET config +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +${METPLUS_VALID_TIME} +^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT2GRID_VALID_TIME` + - valid_time + +${METPLUS_OBS_WINDOW_DICT} +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT2GRID_OBS_WINDOW_BEG` + - obs_window.beg + * - :term:`POINT2GRID_OBS_WINDOW_END` + - obs_window.end + +${METPLUS_MESSAGE_TYPE} +^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT2GRID_MESSAGE_TYPE` + - message_type + +${METPLUS_VAR_NAME_MAP_LIST} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT2GRID_VAR_NAME_MAP_KEY` + - var_name_map.key + * - :term:`POINT2GRID_VAR_NAME_MAP_VAL` + - var_name_map.val + +${METPLUS_OBS_QUALITY_INC} +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT2GRID_OBS_QUALITY_INC` + - obs_quality_inc + +${METPLUS_OBS_QUALITY_EXC} +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT2GRID_OBS_QUALITY_EXC` + - obs_quality_exc + +${METPLUS_MET_CONFIG_OVERRIDES} +""""""""""""""""""""""""""""""" + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT2GRID_MET_CONFIG_OVERRIDES` + - n/a + .. _point_stat_wrapper: PointStat @@ -7088,7 +7097,6 @@ Configuration | :term:`POINT_STAT_UGRID_MAX_DISTANCE_KM` | :term:`POINT_STAT_UGRID_COORDINATES_FILE` | :term:`POINT_STAT_UGRID_CONFIG_FILE` -| :term:`POINT_STAT_TIME_OFFSET_WARNING` | .. warning:: **DEPRECATED:** @@ -7779,18 +7787,6 @@ ${METPLUS_OUTPUT_PREFIX} * - :term:`POINT_STAT_OUTPUT_PREFIX` - output_prefix -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`POINT_STAT_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -8852,7 +8848,6 @@ The following values are optional in the METplus configuration file: | :term:`STAT_ANALYSIS_INC_VALID_TIMES` | :term:`STAT_ANALYSIS_SKIP_INIT_TIMES` | :term:`STAT_ANALYSIS_INC_INIT_TIMES` -| :term:`STAT_ANALYSIS_TIME_OFFSET_WARNING` .. warning:: **DEPRECATED:** @@ -9315,18 +9310,6 @@ ${METPLUS_HSS_EC_VALUE} * - :term:`STAT_ANALYSIS_HSS_EC_VALUE` - hss_ec_value -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`STAT_ANALYSIS_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -9407,7 +9390,6 @@ METplus Configuration | :term:`TC_DIAG_INC_VALID_TIMES` | :term:`TC_DIAG_SKIP_INIT_TIMES` | :term:`TC_DIAG_INC_INIT_TIMES` -| :term:`TC_DIAG_TIME_OFFSET_WARNING` | .. _tc-diag-met-conf: @@ -9835,18 +9817,6 @@ ${METPLUS_OUTPUT_PREFIX} * - :term:`TC_DIAG_OUTPUT_BASE_FORMAT` - output_base_format -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`TC_DIAG_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -9952,7 +9922,6 @@ METplus Configuration | :term:`TC_GEN_DLAND_FILE` | :term:`TC_GEN_BASIN_FILE` | :term:`TC_GEN_NC_PAIRS_GRID` -| :term:`TC_GEN_TIME_OFFSET_WARNING` .. warning:: **DEPRECATED:** @@ -10519,18 +10488,6 @@ ${METPLUS_NC_PAIRS_GRID} * - :term:`TC_GEN_NC_PAIRS_GRID` - nc_pairs_grid -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`TC_GEN_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -10626,7 +10583,6 @@ METplus Configuration | :term:`TC_PAIRS_INC_VALID_TIMES` | :term:`TC_PAIRS_SKIP_INIT_TIMES` | :term:`TC_PAIRS_INC_INIT_TIMES` -| :term:`TC_PAIRS_TIME_OFFSET_WARNING` | .. warning:: **DEPRECATED:** @@ -10965,18 +10921,6 @@ ${METPLUS_DIAG_CONVERT_MAP_LIST} * - :term:`TC_PAIRS_DIAG_CONVERT_MAP_CONVERT` - diag_convert_map.convert -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`TC_PAIRS_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -11042,7 +10986,6 @@ METplus Configuration | :term:`TC_RMW_INC_VALID_TIMES` | :term:`TC_RMW_SKIP_INIT_TIMES` | :term:`TC_RMW_INC_INIT_TIMES` -| :term:`TC_RMW_TIME_OFFSET_WARNING` | .. _tc-rmw-met-conf: @@ -11304,18 +11247,6 @@ ${METPLUS_RMW_SCALE} * - :term:`TC_RMW_SCALE` - rmw_scale -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`TC_RMW_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -11400,7 +11331,6 @@ METplus Configuration | :term:`TC_STAT_INC_VALID_TIMES` | :term:`TC_STAT_SKIP_INIT_TIMES` | :term:`TC_STAT_INC_INIT_TIMES` -| :term:`TC_STAT_TIME_OFFSET_WARNING` | .. warning:: **DEPRECATED:** @@ -12024,18 +11954,6 @@ ${METPLUS_JOBS} * - :term:`TC_STAT_JOBS_LIST` - jobs -${METPLUS_TIME_OFFSET_WARNING} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`TC_STAT_TIME_OFFSET_WARNING` - - time_offset_warning - ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" diff --git a/docs/_static/short_range-MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.png b/docs/_static/short_range-MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.png new file mode 100644 index 0000000000..34cffb6993 Binary files /dev/null and b/docs/_static/short_range-MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.png differ diff --git a/docs/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.py b/docs/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.py new file mode 100644 index 0000000000..77209d9cfe --- /dev/null +++ b/docs/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.py @@ -0,0 +1,179 @@ +""" +MODEMultivar: Create objects of brightness temps and radar reflectivity +======================================================================= + +model_applications/ +short_range/ +MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf + +""" +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + +############################################################################## +# Scientific Objective +# -------------------- +# +# This use case identifies convective objects, which are defined by +# the intersection of: 1) satellite infrared brightness temperature < 235 K and +# 2) radar reflectivity > 40 dBZ. +# Satellite brightness temperatures are used in conjunction with radar reflectivity +# to capture both the cloud top (satellite) and in-cloud (radar) characteristics. +# Convective objects are also defined as lightning thresholds exceeding the 10th percentile. +# A percentile threshold is used for lightning data as RRFS lightning has units +# which are “non-dimensional” and therefore cannot be directly compared to the +# Geostationary Lightning Mapper. + +############################################################################## +# Version Added +# ------------- +# +# METplus Version 6.0 + +############################################################################## +# Datasets +# -------- +# +# **Forecast:** Rapid Refresh Forecast System (RRFS) 3km resolution, +# channel 13 brightness temperature, +# composite reflectivity, and lightning strike density +# +# **Observation:** Geostationary Operational Environmental Satellites (GOES) 3km resolution, +# channel 13 brightness temperature; +# Multi-radar Multi-sensor (MRMS) 3km resolution, composite reflectivity; +# GOES Global Lightning Mapper (GLM) 3km resolution, flash_extent_density +# +# **Climatology:** None +# +# **Location:** All of the input data required for this use case can be found in a sample data +# tarball. Each use case category will have one or more sample data tarballs. It is only +# necessary to download the tarball with the use case’s dataset and not the entire +# collection of sample data. Click here to access the METplus releases page and download +# sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will set the value of +# INPUT_BASE. See :ref:`running-metplus` for more information. + +############################################################################## +# METplus Components +# ------------------ +# +# The only tool this use case calls is MODE, which will identify super-objects +# by intersection of the multiple variable fields. +# + +############################################################################## +# METplus Workflow +# ---------------- +# +# | **Beginning Time (INIT_BEG):** 2024-01-09 05:00 UTC +# | **End Time (INIT_END):** 2024-01-09 05:00 UTC +# | **Increment between beginning and end times (VALID_INCREMENT):** 1 Hour +# | **Sequence of forecast leads to process (LEAD_SEQ):** 9,10 +# +# This use case runs twice, once for each forecast lead time provided. It +# creates objects valid at 14UTC and 15UTC from 09 January 2024 are compared to +# the 9h and 10h forecasts initialized at 05UTC on 9 January 2024. +# Convective objects are identified with thresholds of +# satellite brightness temperature < 235 K and radar reflectivity > 40 dBZ, +# or lightning > 10th percentile. +# In this use case, MODE super-object intensity statistics are output for both +# radar reflectivity and lightning. Using the MODE_MULTIVAR_INTENSITY_FLAG, +# users can control for which variables super object intensity statistics will be output. +# If all are set to False, then no intensity information will be output +# and only statistics relative to the super-object geometry will be available. + + + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads all of the configuration files found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line, i.e. +# parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf + +############################################################################## +# MET Configuration +# --------------------- +# +# METplus sets environment variables based on user settings in the METplus configuration file. +# See :ref:`How METplus controls MET config file settings` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. dropdown:: MODEConfig_wrapped +# +# .. literalinclude:: ../../../../parm/met_config/MODEConfig_wrapped + +############################################################################## +# Python Embedding +# ---------------- +# +# This use case does not use any Python Embedding. + +############################################################################## +# Python Scripting +# ---------------- +# +# This use case does not use any Python Scripting. + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in +# {OUTPUT_BASE}/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning/f??, +# where the '??' characters will reflect the two forecast leads (09 and 10). +# Each of these directories will contain the following files with their appropriate +# verification times:: +# +# * mode_Fcst_LTNG_entireatmosphere_all_all_Obs_flash_extent_density_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_cts.txt +# * mode_Fcst_LTNG_entireatmosphere_all_all_Obs_flash_extent_density_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_obj.nc +# * mode_Fcst_LTNG_entireatmosphere_all_all_Obs_flash_extent_density_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_obj.txt +# * mode_Fcst_LTNG_entireatmosphere_all_all_Obs_flash_extent_density_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A.ps +# * mode_Fcst_REFC_entireatmosphere_consideredasinglelayer_all_all_Obs_MergedReflectivityQCComposite_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_cts.txt +# * mode_Fcst_REFC_entireatmosphere_consideredasinglelayer_all_all_Obs_MergedReflectivityQCComposite_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_obj.nc +# * mode_Fcst_REFC_entireatmosphere_consideredasinglelayer_all_all_Obs_MergedReflectivityQCComposite_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_obj.txt +# * mode_Fcst_REFC_entireatmosphere_consideredasinglelayer_all_all_Obs_MergedReflectivityQCComposite_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A.ps + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * MODEToolUseCase +# * ShortRangeAppUseCase +# * NetCDFFileUseCase +# * MvMODEUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# +# sphinx_gallery_thumbnail_path = '_static/short_range-MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.png' +# diff --git a/internal/scripts/docker/Dockerfile b/internal/scripts/docker/Dockerfile index e9f9c3ecbb..2df83a89c5 100644 --- a/internal/scripts/docker/Dockerfile +++ b/internal/scripts/docker/Dockerfile @@ -58,5 +58,5 @@ RUN if [ ${OBTAIN_SOURCE_CODE} != "none" ]; then \ sed -i 's|MET_INSTALL_DIR = /path/to|MET_INSTALL_DIR = /usr/local|g' parm/metplus_config/*.conf; \ sed -i 's|OUTPUT_BASE = /path/to|OUTPUT_BASE = /data/output|g' parm/metplus_config/*.conf; \ sed -i 's|INPUT_BASE = /path/to|INPUT_BASE = /data/input/METplus_Data|g' parm/metplus_config/*.conf; \ - python3 setup.py install; \ + pip install .; \ fi diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index 1504a3f161..ab3a8ae341 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -3,8 +3,8 @@ sonar.projectKey=METplus sonar.projectName=METplus sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME -sonar.sources=docs,internal,manage_externals,metplus,parm,produtil,ush -sonar.coverage.exclusions=internal/tests/**,parm/**,internal/scripts/**,manage_externals/**,docs/**,produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py +sonar.sources=docs,internal,manage_externals,metplus,parm,ush +sonar.coverage.exclusions=internal/tests/**,parm/**,metplus/parm/**,internal/scripts/**,manage_externals/**,docs/**,metplus/produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py sonar.python.coverage.reportPaths=coverage.xml sonar.sourceEncoding=UTF-8 diff --git a/internal/tests/pytests/util/run_util/test_run_util.py b/internal/tests/pytests/util/run_util/test_run_util.py index 6815e0b9d2..9b95e891e5 100644 --- a/internal/tests/pytests/util/run_util/test_run_util.py +++ b/internal/tests/pytests/util/run_util/test_run_util.py @@ -4,7 +4,6 @@ import os import re -import produtil import metplus.util.run_util as ru import metplus.util.wrapper_init as wi from metplus.wrappers.ensemble_stat_wrapper import EnsembleStatWrapper diff --git a/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py b/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py index dfffe82f22..79ea282ac9 100644 --- a/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py +++ b/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py @@ -354,7 +354,7 @@ def test_filename_does_not_match_template(): template = "{init?fmt=%Y%m%d%H}_dog_A{lead?fmt=%HH}h" filepath = "1987020103_cat_A03h" out = parse_template(template, filepath) - assert out == None + assert out is None @pytest.mark.util @@ -362,7 +362,7 @@ def test_filename_does_not_match_template_end(): template = "{init?fmt=%Y%m%d%H}_dog_A{lead?fmt=%HH}h" filepath = "1987020103_dog_A03d" out = parse_template(template, filepath) - assert out == None + assert out is None @pytest.mark.util @@ -379,7 +379,7 @@ def test_get_tags(): # value is the formatted output string, like 01 # ttype is the unit to check, i.e. 'H', 'M', 'S', 'd', 's' @pytest.mark.parametrize( - 'format, key, value, ttype', [ + 'fmt, key, value, ttype', [ ('H', 1, '01', 'H'), ('1H', 1, '1', 'H'), ('2H', 1, '01', 'H'), @@ -418,15 +418,15 @@ def test_get_tags(): ] ) @pytest.mark.util -def test_format_one_time_item(format, key ,value, ttype): - assert format_one_time_item(format, key, ttype) == value +def test_format_one_time_item(fmt, key ,value, ttype): + assert format_one_time_item(fmt, key, ttype) == value # format is the time format to use, like, %M or %H%M # seconds is the integer number of seconds of the offset to use, i.e. 3601 # value is the formatted output string, like 010001 @pytest.mark.parametrize( - 'format, seconds, value', [ + 'fmt, seconds, value', [ ('%H', 1, '00'), ('%M', 1, '00'), ('%S', 1, '01'), @@ -448,9 +448,9 @@ def test_format_one_time_item(format, key ,value, ttype): ] ) @pytest.mark.util -def test_format_hms(format, seconds, value): +def test_format_hms(fmt, seconds, value): # format should be something like %M or %H%M - assert format_hms(format, seconds == value) + assert format_hms(fmt, seconds == value) @pytest.mark.util @@ -539,7 +539,6 @@ def test_populate_match_dict(template, filepath, expected_match_dict, expected_v elif match_dict is None: # if expected is not None, fail if actual is None assert False - return num_keys = len(match_dict.keys()) expected_num_keys = len(expected_match_dict.keys()) @@ -562,8 +561,6 @@ def test_populate_match_dict(template, filepath, expected_match_dict, expected_v print(f"Incorrect valid shift. Actual {valid_shift}, Expected: {expected_valid_shift}") assert False - assert True - except TypeError: assert expected_match_dict is None and expected_valid_shift is None @@ -591,8 +588,6 @@ def test_get_fmt_info(fmt, filepath, identifier, expected_fmt_len, expected_matc print(f"Expected Dictionary: {expected_match_dict}") assert False - assert True - @pytest.mark.parametrize( 'templ, expected_filename', [ diff --git a/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py b/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py index 4209c2e047..0be6a1b369 100644 --- a/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py @@ -183,25 +183,6 @@ def test_ascii2nc_missing_inputs(metplus_config, get_test_data_dir, 'type = ["min", "max", "range", "mean", "stdev", "median", "p80"];' 'vld_freq = 0;vld_thresh = 0.5;}')}), - ({'TIME_OFFSET_WARNING': '3'}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;', - 'METPLUS_TIME_SUMMARY_DICT': - ('time_summary = {flag = FALSE;raw_data = FALSE;beg = "000000";' - 'end = "235959";step = 300;width = 600;' - 'grib_code = [11, 204, 211];obs_var = [];' - 'type = ["min", "max", "range", "mean", "stdev", "median", "p80"];' - 'vld_freq = 0;vld_thresh = 0.0;}') - }), - - ({'TIME_OFFSET_WARNING': '3', 'ASCII2NC_TIME_OFFSET_WARNING': '4'}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;', - 'METPLUS_TIME_SUMMARY_DICT': - ('time_summary = {flag = FALSE;raw_data = FALSE;beg = "000000";' - 'end = "235959";step = 300;width = 600;' - 'grib_code = [11, 204, 211];obs_var = [];' - 'type = ["min", "max", "range", "mean", "stdev", "median", "p80"];' - 'vld_freq = 0;vld_thresh = 0.0;}') - }), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py index b905a79705..b17171495d 100644 --- a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py @@ -637,13 +637,6 @@ def test_ensemble_stat_field_info(metplus_config, config_overrides, 'OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE': obs_point_template}, {}), - ({'ENSEMBLE_STAT_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - ({'TIME_OFFSET_WARNING': 2, 'ENSEMBLE_STAT_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), - # fcst climo_mean ({'ENSEMBLE_STAT_FCST_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', }, {'METPLUS_FCST_CLIMO_MEAN_DICT': 'climo_mean = {file_name = ["/some/climo_mean/file.txt"];}'}), diff --git a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py index 43f22587bf..8c7cf09863 100644 --- a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -446,15 +446,6 @@ def test_gen_ens_prod_missing_inputs(metplus_config, get_test_data_dir, allow_mi # 68 ({'GEN_ENS_PROD_NORMALIZE': 'CLIMO_STD_ANOM', }, {'METPLUS_NORMALIZE': 'normalize = CLIMO_STD_ANOM;'}), - # 69 - ({'GEN_ENS_PROD_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - # 70 - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - # 71 - ({'TIME_OFFSET_WARNING': 2, 'GEN_ENS_PROD_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), # ens climo_mean (quietly supported) ({'GEN_ENS_PROD_ENS_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', }, {'METPLUS_ENS_CLIMO_MEAN_DICT': 'climo_mean = {file_name = ["/some/climo_mean/file.txt"];}'}), diff --git a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py index 7179109ddc..d190d37f06 100644 --- a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py +++ b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py @@ -345,12 +345,6 @@ def test_get_config_file(metplus_config): 'censor_val = [12000, 5000];}' )}), - ({'GRID_DIAG_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - ({'TIME_OFFSET_WARNING': 2, 'GRID_DIAG_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py index cc49602b90..c6e89fbd25 100644 --- a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py @@ -230,15 +230,7 @@ def test_ioda2nc_missing_inputs(metplus_config, get_test_data_dir, missing, 'IODA2NC_NMSG': '10', }, {}, ' -iodafile *INPUT_DIR*/other/file.nc -valid_beg 20200309_12 -valid_end 20200310_12 -nmsg 10'), - # 40 - ({'IODA2NC_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}, ''), - # 41 - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}, ''), - # 42 - ({'TIME_OFFSET_WARNING': 2, 'IODA2NC_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}, ''), + ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/madis2nc/test_madis2nc_wrapper.py b/internal/tests/pytests/wrappers/madis2nc/test_madis2nc_wrapper.py index 59c2346968..86a7e14a57 100644 --- a/internal/tests/pytests/wrappers/madis2nc/test_madis2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/madis2nc/test_madis2nc_wrapper.py @@ -139,8 +139,6 @@ def test_madis2nc_missing_inputs(metplus_config, get_test_data_dir, 'MADIS2NC_MASK_GRID': 'mask_grid', 'MADIS2NC_MASK_POLY': '/some/path/to/mask/poly', 'MADIS2NC_MASK_SID': 'mask_sid,/some/path/to/mask/sid'}, {}), - ({'MADIS2NC_TIME_OFFSET_WARNING': '4', }, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) diff --git a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py index 97c9b6bd6c..6506afbff5 100644 --- a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py +++ b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py @@ -200,12 +200,6 @@ def test_mtd_missing_inputs(metplus_config, get_test_data_dir, ({'MTD_OUTPUT_PREFIX': 'my_output_prefix'}, {'METPLUS_OUTPUT_PREFIX': 'output_prefix = "my_output_prefix";'}), - ({'MTD_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - ({'TIME_OFFSET_WARNING': 2, 'MTD_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py index 958806cb2d..09f1807253 100644 --- a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py @@ -252,13 +252,6 @@ def test_find_input_files(metplus_config, offsets, offset_to_find): ({'PB2NC_VALID_BEGIN': valid_beg}, {}), ({'PB2NC_VALID_END': valid_end}, {}), ({'PB2NC_VALID_BEGIN': valid_beg, 'PB2NC_VALID_END': valid_end}, {}), - - ({'PB2NC_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - ({'TIME_OFFSET_WARNING': 2, 'PB2NC_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), # 1 extra file ({'PB2NC_INPUT_TEMPLATE': ('ndas.t{da_init?fmt=%H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}.nr,' 'another_file.nr')}, {}), diff --git a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py index 337d2d55ed..e69a324418 100644 --- a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py +++ b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py @@ -248,12 +248,6 @@ def test_plot_point_obs_missing_inputs(metplus_config, get_test_data_dir, ({'PLOT_POINT_OBS_POINT_DATA': point_data_input, }, {'METPLUS_POINT_DATA': f'point_data = {point_data_format}'}), - ({'PLOT_POINT_OBS_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - ({'TIME_OFFSET_WARNING': 2, 'PLOT_POINT_OBS_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper_c diff --git a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py index 87a980f41e..c9a869ff28 100644 --- a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py +++ b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py @@ -79,25 +79,56 @@ def test_point2grid_missing_inputs(metplus_config, get_test_data_dir, @pytest.mark.parametrize( - 'config_overrides, optional_args', [ - ({}, {}), - ({'POINT2GRID_REGRID_METHOD': 'UW_MEAN'}, ['-method UW_MEAN']), + 'config_overrides, env_var_values, optional_args', [ + ({}, {}, []), + ({'POINT2GRID_REGRID_METHOD': 'UW_MEAN'}, {}, ['-method UW_MEAN']), ({'POINT2GRID_REGRID_METHOD': 'UW_MEAN', - 'POINT2GRID_GAUSSIAN_DX': '2',}, + 'POINT2GRID_GAUSSIAN_DX': '2',}, {}, ['-method UW_MEAN', '-gaussian_dx 2']), - ({'POINT2GRID_GAUSSIAN_RADIUS': '81.231'}, + ({'POINT2GRID_GAUSSIAN_RADIUS': '81.231'}, {}, ['-gaussian_radius 81.231']), - ({'POINT2GRID_PROB_CAT_THRESH': '1'}, ['-prob_cat_thresh 1']), - ({'POINT2GRID_VLD_THRESH': '0.5'}, ['-vld_thresh 0.5']), - ({'POINT2GRID_QC_FLAGS': '0,1'}, ['-qc 0,1']), - ({'POINT2GRID_ADP': '{valid?fmt=%Y%m}.nc'}, ['-adp 201706.nc']), - ({'POINT2GRID_REGRID_TO_GRID': 'G212'}, []), - ({'POINT2GRID_REGRID_TO_GRID': 'lambert 614 428 12.190 -133.459 -95.0 12.19058 6367.47 25.0 N'}, []), - ({'POINT2GRID_INPUT_LEVEL': '(*,*)'}, []), + ({'POINT2GRID_PROB_CAT_THRESH': '1'}, {}, ['-prob_cat_thresh 1']), + ({'POINT2GRID_VLD_THRESH': '0.5'}, {}, ['-vld_thresh 0.5']), + ({'POINT2GRID_ADP': '{valid?fmt=%Y%m}.nc'}, {}, ['-adp 201706.nc']), + ({'POINT2GRID_REGRID_TO_GRID': 'G212'}, {}, []), + ({'POINT2GRID_REGRID_TO_GRID': 'lambert 614 428 12.190 -133.459 -95.0 12.19058 6367.47 25.0 N'}, {}, []), + ({'POINT2GRID_INPUT_LEVEL': '(*,*)'}, {}, []), + ({'POINT2GRID_VALID_TIME': '20240509_120800', }, + {'METPLUS_VALID_TIME': 'valid_time = "20240509_120800";'}, []), + + ({'POINT2GRID_OBS_WINDOW_BEG': '-5400', }, + {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {beg = -5400;}'}, []), + + ({'POINT2GRID_OBS_WINDOW_END': '3600', }, + {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {end = 3600;}'}, []), + + ({'POINT2GRID_OBS_WINDOW_BEG': '-3600', 'POINT2GRID_OBS_WINDOW_END': '5400'}, + {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {beg = -3600;end = 5400;}'}, []), + ({'POINT2GRID_MESSAGE_TYPE': 'ADPSFC, ADPUPA'}, + {'METPLUS_MESSAGE_TYPE': 'message_type = ["ADPSFC", "ADPUPA"];'}, []), + + ({'POINT2GRID_VAR_NAME_MAP1_KEY': '3', 'POINT2GRID_VAR_NAME_MAP1_VAL': 'MAGIC'}, + {'METPLUS_VAR_NAME_MAP_LIST': 'var_name_map = [{key = "3";val = "MAGIC";}];'}, []), + + ({'POINT2GRID_VAR_NAME_MAP1_KEY': '13', 'POINT2GRID_VAR_NAME_MAP1_VAL': 'LUCKY', + 'POINT2GRID_VAR_NAME_MAP2_KEY': '3', 'POINT2GRID_VAR_NAME_MAP2_VAL': 'MAGIC' + }, + {'METPLUS_VAR_NAME_MAP_LIST': 'var_name_map = [{key = "13";val = "LUCKY";},{key = "3";val = "MAGIC";}];'}, []), + + ({'POINT2GRID_OBS_QUALITY_INC': '0, 1, 2', }, + {'METPLUS_OBS_QUALITY_INC': 'obs_quality_inc = ["0", "1", "2"];'}, []), + + ({'POINT2GRID_OBS_QUALITY_EXC': '3,4, 5', }, + {'METPLUS_OBS_QUALITY_EXC': 'obs_quality_exc = ["3", "4", "5"];'}, []), + ({'POINT2GRID_GOES_QC_FLAGS': '0,1'}, {}, ['-goes_qc 0,1']), + ({'POINT2GRID_QC_FLAGS': '0,1'}, {}, ['-goes_qc 0,1']), + ({'POINT2GRID_GOES_QC_FLAGS': '0,1', 'POINT2GRID_QC_FLAGS': '2,3'}, {}, ['-goes_qc 0,1']), + ] ) @pytest.mark.wrapper -def test_point2grid_run(metplus_config, config_overrides, optional_args): +def test_point2grid_run(metplus_config, config_overrides, optional_args, + env_var_values): config = metplus_config set_minimum_config_settings(config) @@ -143,13 +174,19 @@ def test_point2grid_run(metplus_config, config_overrides, optional_args): else: level = '' + config_file = wrapper.c_dict.get('CONFIG_FILE') extra_args = " ".join(optional_args) + " " if optional_args else "" + + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + expected_cmds = [] - for idx in range(0, 3): + for idx in range(0, len(input_files)): expected_cmds.append( f'{app_path} {input_files[idx]} {grids[idx]} {output_files[idx]}' f' -field \'name="{input_name}"; level="{level}";\'' - f' {extra_args}{verbosity}' + f' -config {config_file} {extra_args}{verbosity}' ) all_cmds = wrapper.run_all_times() @@ -158,3 +195,30 @@ def test_point2grid_run(metplus_config, config_overrides, optional_args): for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd + + # check that environment variables were set properly + # including deprecated env vars (not in wrapper env var keys) + for env_var_key in env_var_keys: + print(f"ENV VAR: {env_var_key}") + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + value = match.split('=', 1)[1] + assert env_var_values.get(env_var_key, '') == value + + +@pytest.mark.wrapper +def test_get_config_file(metplus_config): + fake_config_name = '/my/config/file' + + config = metplus_config + default_config_file = os.path.join(config.getdir('PARM_BASE'), + 'met_config', + 'Point2GridConfig_wrapped') + + wrapper = Point2GridWrapper(config) + assert wrapper.c_dict['CONFIG_FILE'] == default_config_file + + config.set('config', 'POINT2GRID_CONFIG_FILE', fake_config_name) + wrapper = Point2GridWrapper(config) + assert wrapper.c_dict['CONFIG_FILE'] == fake_config_name diff --git a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py index bc59a795ad..3660a1b37a 100755 --- a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py @@ -671,13 +671,6 @@ def test_met_dictionary_in_var_options(metplus_config): ({'OBS_POINT_STAT_INPUT_TEMPLATE': '{valid?fmt=%Y%m%d%H}/obs_file,{valid?fmt=%Y%m%d%H}/obs_file2', }, {}), ({'OBS_POINT_STAT_INPUT_TEMPLATE': '{valid?fmt=%Y%m%d%H}/obs_file,{valid?fmt=%Y%m%d%H}/obs_file2,{valid?fmt=%Y%m%d%H}/obs_file3', }, {}), - ({'POINT_STAT_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - ({'TIME_OFFSET_WARNING': 2, 'POINT_STAT_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), - # fcst climo_mean ({'POINT_STAT_FCST_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', }, {'METPLUS_FCST_CLIMO_MEAN_DICT': 'climo_mean = {file_name = ["/some/climo_mean/file.txt"];}'}), diff --git a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py index 6d0f619ec7..9dc06f9f88 100644 --- a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py +++ b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py @@ -173,15 +173,7 @@ def set_minimum_config_settings(config): 'MODEL1': '{custom}', 'MODEL_LIST': '{custom}'}, {'METPLUS_MODEL': 'model = ["CUSTOM_MODEL"];'}), - # 17 time_offset_warning app - ({'STAT_ANALYSIS_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - # 18 time_offset_warning generic - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - # 19 time_offset_warning both - ({'TIME_OFFSET_WARNING': 2, 'STAT_ANALYSIS_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), + ] ) @pytest.mark.wrapper_d diff --git a/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py b/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py index 88b9ef1eb5..3f0c5ed75e 100644 --- a/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py @@ -239,12 +239,6 @@ def test_tc_diag_missing_inputs(metplus_config, get_test_data_dir, ({'TC_DIAG_ONE_TIME_PER_FILE_FLAG': 'false', }, {'METPLUS_ONE_TIME_PER_FILE_FLAG': 'one_time_per_file_flag = FALSE;'}), - ({'TC_DIAG_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - ({'TIME_OFFSET_WARNING': 2, 'TC_DIAG_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py index c9beca4fde..d1d5b24c23 100644 --- a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py @@ -339,12 +339,6 @@ def test_tc_gen_missing_inputs(metplus_config, get_test_data_dir, allow_missing, }, {'METPLUS_OPS_HIT_WINDOW_DICT': 'ops_hit_window = {beg = 1;end = 47;}'}), - ({'TC_GEN_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - ({'TIME_OFFSET_WARNING': 2, 'TC_GEN_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper_a diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index 70adbd62d5..7f9d560eed 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -607,12 +607,6 @@ def test_tc_pairs_storm_id_lists(metplus_config, get_test_data_dir, config_overr }, {'DIAG_ARG': '-diag TCDIAG /bmlq2014123118.gfso.0104',}), - ('VALID', {'TC_PAIRS_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - ('VALID', {'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - ('VALID', {'TIME_OFFSET_WARNING': 2, 'TC_PAIRS_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py index 0f52648e6b..8f07f3154b 100644 --- a/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py @@ -242,12 +242,6 @@ def test_tc_stat_handle_jobs(metplus_config, config_overrides, expected_dirs, ({'TC_STAT_OUTPUT_TEMPLATE': 'tc_stat.out.nc', }, {}), - ({'TC_STAT_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - ({'TIME_OFFSET_WARNING': 2, 'TC_STAT_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py b/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py index 7a4d3ea6e9..6721527a64 100644 --- a/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py +++ b/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py @@ -122,12 +122,6 @@ def set_minimum_config_settings(config): ({'TC_RMW_RMW_SCALE': '15', }, {'METPLUS_RMW_SCALE': 'rmw_scale = 15.0;'}), - ({'TC_RMW_TIME_OFFSET_WARNING': 3}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), - ({'TIME_OFFSET_WARNING': 2}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), - ({'TIME_OFFSET_WARNING': 2, 'TC_RMW_TIME_OFFSET_WARNING': 4}, - {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper diff --git a/internal/tests/use_cases/all_use_cases.txt b/internal/tests/use_cases/all_use_cases.txt index 1522963abe..ef58c7ff60 100644 --- a/internal/tests/use_cases/all_use_cases.txt +++ b/internal/tests/use_cases/all_use_cases.txt @@ -184,7 +184,7 @@ Category: short_range 12::UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection::model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf:: metplotpy_env 13::MODEMultivar_fcstHRRR_obsMRMS_HRRRanl::model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf 14::UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot::model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot.conf:: metdataio, metcalcpy, metplotpy,mp_analysis_env - +15::MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning::model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf Category: space_weather 0::GridStat_fcstGloTEC_obsGloTEC_vx7:: model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf diff --git a/metplus/parm b/metplus/parm new file mode 120000 index 0000000000..302dacab35 --- /dev/null +++ b/metplus/parm @@ -0,0 +1 @@ +../parm \ No newline at end of file diff --git a/produtil/README_produtil.md b/metplus/produtil/README_produtil.md similarity index 100% rename from produtil/README_produtil.md rename to metplus/produtil/README_produtil.md diff --git a/produtil/__init__.py b/metplus/produtil/__init__.py similarity index 100% rename from produtil/__init__.py rename to metplus/produtil/__init__.py diff --git a/produtil/batchsystem.py b/metplus/produtil/batchsystem.py similarity index 100% rename from produtil/batchsystem.py rename to metplus/produtil/batchsystem.py diff --git a/produtil/cluster.py b/metplus/produtil/cluster.py similarity index 100% rename from produtil/cluster.py rename to metplus/produtil/cluster.py diff --git a/produtil/config.py b/metplus/produtil/config.py similarity index 99% rename from produtil/config.py rename to metplus/produtil/config.py index f020838a10..ffffa40621 100644 --- a/produtil/config.py +++ b/metplus/produtil/config.py @@ -11,19 +11,18 @@ # decides what symbols are imported by "from produtil.config import *" __all__=['from_file','confwalker','ProdConfig','ENVIRONMENT','ProdTask'] -import collections,re,string,os,logging,threading +import collections,re,os,logging,threading import os.path,sys import datetime -import produtil.fileop, produtil.datastore -import produtil.numerics, produtil.log +import metplus.produtil.fileop as fileop import configparser from configparser import ConfigParser from io import StringIO -from produtil.datastore import Datastore,Task +from metplus.produtil.datastore import Datastore,Task -from produtil.numerics import to_datetime +from metplus.produtil.numerics import to_datetime, to_datetime_rel, fcst_hr_min from string import Formatter from configparser import NoOptionError,NoSectionError @@ -379,7 +378,7 @@ def get_value(self,key,args,kwargs): v=ap6.strftime(ANL_P6_KEYS[key]) elif '__ftime' in kwargs and '__atime' in kwargs and \ key in TIME_DIFF_KEYS: - (ihours,iminutes)=produtil.numerics.fcst_hr_min( + (ihours,iminutes)=fcst_hr_min( kwargs['__ftime'],kwargs['__atime']) if key=='fahr': v=int(ihours) @@ -702,7 +701,7 @@ def from_args(self,args=None,allow_files=True,allow_options=True, elif not os.path.isfile(path): logger.error(path+': conf file is not a regular file.') sys.exit(2) - elif not produtil.fileop.isnonempty(path) and verbose: + elif not fileop.isnonempty(path) and verbose: logger.warning( path+': conf file is empty. Will continue anyway.') if verbose: logger.info('Conf input: '+repr(path)) @@ -823,7 +822,7 @@ def getdatastore(self): with self: if self._datastore is None: dsfile=self.getstr('config','datastore') - self._datastore=produtil.datastore.Datastore(dsfile, + self._datastore=Datastore(dsfile, logger=self.log('datastore')) return self._datastore @@ -1001,7 +1000,7 @@ def makedirs(self,*args): with self: dirs=[self.getstr('dir',arg) for arg in args] for makeme in dirs: - produtil.fileop.makedirs(makeme) + fileop.makedirs(makeme) def keys(self,sec): """!get options in a section @@ -1107,13 +1106,13 @@ def timestrinterp(self,sec,string,ftime=None,atime=None,**kwargs): @param atime the analysis time or None @param kwargs more variables for string expansion""" if atime is not None: - atime=produtil.numerics.to_datetime(atime) + atime=to_datetime(atime) else: atime=self.cycle if ftime is None: ftime=atime else: - ftime=produtil.numerics.to_datetime_rel(ftime,atime) + ftime=to_datetime_rel(ftime,atime) with self: return self._time_formatter.format(string,__section=sec, __key='__string__',__depth=0,__conf=self._conf,ENV=ENVIRONMENT, @@ -1343,7 +1342,7 @@ def getbool(self,sec,opt,default=None,badtypeok=False,morevars=None,taskvars=Non ######################################################################## -class ProdTask(produtil.datastore.Task): +class ProdTask(Task): """!A subclass of produtil.datastore.Task that provides a variety of convenience functions related to unix conf files and logging.""" diff --git a/produtil/datastore.py b/metplus/produtil/datastore.py similarity index 98% rename from produtil/datastore.py rename to metplus/produtil/datastore.py index cb2a665956..42268fe3f4 100644 --- a/produtil/datastore.py +++ b/metplus/produtil/datastore.py @@ -6,9 +6,10 @@ Datum, which is the base class of anything that can be stored in the Datastore.""" -import sqlite3, threading, collections, re, contextlib, time, random,\ - traceback, datetime, logging, os, time -import produtil.fileop, produtil.locking, produtil.sigsafety, produtil.log +import sqlite3, threading, collections, re, contextlib, datetime, logging, os, time +import metplus.produtil.fileop as fileop +from metplus.produtil.locking import LockFile +from metplus.produtil.log import jlogger ##@var __all__ # Symbols exported by "from produtil.datastore import *" @@ -175,7 +176,7 @@ def __init__(self,filename,logger=None,locking=True): lockfile=filename+'.lock' if logger is not None: logger.debug('Lockfile is %s for database %s'%(lockfile,filename)) - self._file_lock=produtil.locking.LockFile( + self._file_lock=LockFile( lockfile,logger=logger,max_tries=300,sleep_time=0.1,first_warn=50) self._transtack=collections.defaultdict(list) with self.transaction(): @@ -884,7 +885,7 @@ def undeliver(self,delete=True,logger=None): @param logger a logging.Logger for log messages""" loc=self.location if loc and delete: - produtil.fileop.remove_file(filename=loc,logger=logger,info=True) + fileop.remove_file(filename=loc,logger=logger,info=True) self.available=False def deliver(self,location=None,frominfo=None,keep=True,logger=None, copier=None): @@ -917,7 +918,7 @@ def deliver(self,location=None,frominfo=None,keep=True,logger=None, raise UnknownLocation( '%s: no location known when delivering product. Specify a ' 'location to deliver().'%(self.did)) - produtil.fileop.deliver_file(frominfo,loc,keep=keep,logger=logger, + fileop.deliver_file(frominfo,loc,keep=keep,logger=logger, copier=copier) if setloc: self.set_loc_avail(loc,True) @@ -966,7 +967,7 @@ def check(self,frominfo=None,minsize=None,minage=None,logger=None): minsize=int(self.get('minsize',0)) if minage is None: minage=int(self.get('minage',20)) - if not produtil.fileop.check_file(loc,min_size=minsize, + if not fileop.check_file(loc,min_size=minsize, min_mtime_age=minage): if self.available: self.available=False @@ -1093,7 +1094,7 @@ def jlogfile(self): intended to receive only major errors, and per-job start and completion information. This is equivalent to simply accessing produtil.log.jlogger.""" - return produtil.log.jlogger + return jlogger def postmsg(self,message,*args,**kwargs): """!same as produtil.log.jlogger.info() @@ -1103,7 +1104,7 @@ def postmsg(self,message,*args,**kwargs): @param message the message @param args positional arguments for string replacement @param kwargs keyword arguments for string replacement.""" - produtil.log.jlogger.info(message,*args,**kwargs) + jlogger.info(message,*args,**kwargs) def setstate(self,val): """!Sets the state of this job. diff --git a/produtil/dbnalert.py b/metplus/produtil/dbnalert.py similarity index 98% rename from produtil/dbnalert.py rename to metplus/produtil/dbnalert.py index de8343146e..62f9830a2a 100644 --- a/produtil/dbnalert.py +++ b/metplus/produtil/dbnalert.py @@ -6,10 +6,9 @@ __all__=["DBNAlert"] import logging, os -import produtil.run -from produtil.prog import Runner -from produtil.run import checkrun, batchexe, alias, run +from metplus.produtil.prog import Runner +from metplus.produtil.run import batchexe, alias, run # Globals: diff --git a/produtil/externals b/metplus/produtil/externals similarity index 100% rename from produtil/externals rename to metplus/produtil/externals diff --git a/produtil/fileop.py b/metplus/produtil/fileop.py similarity index 99% rename from produtil/fileop.py rename to metplus/produtil/fileop.py index 348daa6006..6853f25a95 100644 --- a/produtil/fileop.py +++ b/metplus/produtil/fileop.py @@ -18,7 +18,7 @@ 'netcdfver','touch'] import os,tempfile,filecmp,stat,shutil,errno,random,time,fcntl,math,logging -import produtil.cluster, produtil.pipeline +import metplus.produtil.cluster as cluster module_logger=logging.getLogger('produtil.fileop') @@ -403,9 +403,9 @@ def deliver_file(infile,outfile,keep=True,verify=False,blocksize=1048576, the temp_file_object is an object that can be used to write to the file. The copier should NOT close the temp_file_object. """ if preserve_group is None: - preserve_group = not produtil.cluster.group_quotas() + preserve_group = not cluster.group_quotas() if copy_acl is None: - copy_acl = produtil.cluster.use_acl_for_rstdata() + copy_acl = cluster.use_acl_for_rstdata() if copier is not None: # Cannot simply do a "move" if we are using an external # function to copy. diff --git a/produtil/locking.py b/metplus/produtil/locking.py similarity index 97% rename from produtil/locking.py rename to metplus/produtil/locking.py index 1cf90fd277..0cd4a7e617 100644 --- a/produtil/locking.py +++ b/metplus/produtil/locking.py @@ -14,9 +14,9 @@ ... the file is now unlocked ... @endcode""" -import fcntl, time, errno, os.path -import produtil.retry as retry -import produtil.fileop +import fcntl, errno, os.path +import metplus.produtil.retry as retry +import metplus.produtil.fileop as fileop ##@var __all__ # Symbols exported by "from produtil.locking import *" @@ -110,7 +110,7 @@ def acquire_impl(self): 'the process was exiting.') thedir=os.path.dirname(self._filename) if thedir: - produtil.fileop.makedirs(thedir) + fileop.makedirs(thedir) if self._fd is None: self._fd=open(self._filename,'wb') try: diff --git a/produtil/log.py b/metplus/produtil/log.py similarity index 99% rename from produtil/log.py rename to metplus/produtil/log.py index 329cab79f2..0d6487f382 100644 --- a/produtil/log.py +++ b/metplus/produtil/log.py @@ -12,7 +12,7 @@ 'MasterLogHandler','JLogHandler','set_jlogfile' ] import logging, os, sys, traceback, threading -import produtil.batchsystem +import metplus.produtil.batchsystem as batchsystem ##@var logthread # string for log messages to indicate thread number/name @@ -379,7 +379,7 @@ def configureLogging(jlogfile=None, # Configure log formatting: jobstr=os.environ.get('job',None) if jobstr is None: - jobstr=produtil.batchsystem.jobname() + jobstr=batchsystem.jobname() jobstr=str(jobstr).replace('(','_').replace(')','_').replace('%','_') # Format for jlogfile domain logging to jlogfile: jformat=JLogFormatter( diff --git a/produtil/mpi_impl/__init__.py b/metplus/produtil/mpi_impl/__init__.py similarity index 89% rename from produtil/mpi_impl/__init__.py rename to metplus/produtil/mpi_impl/__init__.py index b986e7d76c..10923ff839 100644 --- a/produtil/mpi_impl/__init__.py +++ b/metplus/produtil/mpi_impl/__init__.py @@ -125,7 +125,8 @@ # __init__.py on how to modify it to achieve these steps. import logging -import produtil.fileop +import metplus.produtil.fileop +import metplus.produtil.prog ##@var __all__ # An empty list that indicates no symbols are exported by "from @@ -220,8 +221,8 @@ def register_implementations(logger=None): # no_implementation=None is used to detect if # register_implementations was called. global no_implementation - import produtil.mpi_impl.no_mpi - no_implementation=produtil.mpi_impl.no_mpi.Implementation.detect() + from metplus.produtil.mpi_impl.no_mpi import Implementation + no_implementation=Implementation.detect() # Now add each implementation. We need to wrap each around a # try...except so that NCEP Central Operations can delete the @@ -230,64 +231,64 @@ def register_implementations(logger=None): try: # If we have srun, and we're in a pack group... - import produtil.mpi_impl.srun_pack_groups - add_implementation(produtil.mpi_impl.srun_pack_groups.Implementation) + import metplus.produtil.mpi_impl.srun_pack_groups + add_implementation(metplus.produtil.mpi_impl.srun_pack_groups.Implementation) except ImportError: pass try: # This must be after the pack group case. # If we have srun and SLURM resources... - import produtil.mpi_impl.srun - add_implementation(produtil.mpi_impl.srun.Implementation) + import metplus.produtil.mpi_impl.srun + add_implementation(metplus.produtil.mpi_impl.srun.Implementation) except ImportError: pass try: - import produtil.mpi_impl.inside_aprun - add_implementation(produtil.mpi_impl.inside_aprun.Implementation) + import metplus.produtil.mpi_impl.inside_aprun + add_implementation(metplus.produtil.mpi_impl.inside_aprun.Implementation) except ImportError: pass try: - import produtil.mpi_impl.pbs_cray_intel - add_implementation(produtil.mpi_impl.pbs_cray_intel.Implementation) + import metplus.produtil.mpi_impl.pbs_cray_intel + add_implementation(metplus.produtil.mpi_impl.pbs_cray_intel.Implementation) except ImportError: pass try: - import produtil.mpi_impl.lsf_cray_intel - add_implementation(produtil.mpi_impl.lsf_cray_intel.Implementation) + import metplus.produtil.mpi_impl.lsf_cray_intel + add_implementation(metplus.produtil.mpi_impl.lsf_cray_intel.Implementation) except ImportError: pass try: - import produtil.mpi_impl.impi - add_implementation(produtil.mpi_impl.impi.Implementation) + import metplus.produtil.mpi_impl.impi + add_implementation(metplus.produtil.mpi_impl.impi.Implementation) except ImportError: pass try: - import produtil.mpi_impl.mpirun_lsf - add_implementation(produtil.mpi_impl.mpirun_lsf.Implementation) + import metplus.produtil.mpi_impl.mpirun_lsf + add_implementation(metplus.produtil.mpi_impl.mpirun_lsf.Implementation) except ImportError: pass try: - import produtil.mpi_impl.mpiexec_mpt - add_implementation(produtil.mpi_impl.mpiexec_mpt.Implementation) + import metplus.produtil.mpi_impl.mpiexec_mpt + add_implementation(metplus.produtil.mpi_impl.mpiexec_mpt.Implementation) except ImportError: pass try: - import produtil.mpi_impl.mpiexec - add_implementation(produtil.mpi_impl.mpiexec.Implementation) + import metplus.produtil.mpi_impl.mpiexec + add_implementation(metplus.produtil.mpi_impl.mpiexec.Implementation) except ImportError: pass try: - import produtil.mpi_impl.srun_shell - add_implementation(produtil.mpi_impl.srun_shell.Implementation) + import metplus.produtil.mpi_impl.srun_shell + add_implementation(metplus.produtil.mpi_impl.srun_shell.Implementation) except ImportError: pass @@ -315,7 +316,7 @@ def get_mpi(mpi_name=NO_NAME,force=False,logger=None,**kwargs): @raise NotImplementedError if the MPI implementation is unknown, or if the implementation is unavailble on this machine, and force=False""" - + import metplus.produtil.mpi_impl.no_mpi as no_mpi if logger is None: logger=logging.getLogger('mpi_impl') @@ -351,8 +352,8 @@ def get_mpi(mpi_name=NO_NAME,force=False,logger=None,**kwargs): result=detect( force=force,logger=logger,**kwargs) except (Exception, - produtil.fileop.FileOpError, - produtil.prog.ExitStatusException): + metplus.produtil.fileop.FileOpError, + metplus.produtil.prog.ExitStatusException): # Ignore exceptions related to an inability to detect the # MPI implementation. We assume the issue has already # been logged, and we move on to the next implementation's diff --git a/produtil/mpi_impl/mpi_impl_base.py b/metplus/produtil/mpi_impl/mpi_impl_base.py similarity index 98% rename from produtil/mpi_impl/mpi_impl_base.py rename to metplus/produtil/mpi_impl/mpi_impl_base.py index 572bfc63db..6f1bb4411b 100644 --- a/produtil/mpi_impl/mpi_impl_base.py +++ b/metplus/produtil/mpi_impl/mpi_impl_base.py @@ -12,9 +12,8 @@ import tempfile,stat,os, logging, io, re -import produtil.prog -import produtil.pipeline -from produtil.prog import shbackslash +from metplus.produtil.pipeline import Pipeline +from metplus.produtil.prog import shbackslash, Runner, ImmutableRunner module_logger=logging.getLogger('produtil.mpi_impl') @@ -139,8 +138,8 @@ def find_mpiserial(self,mpiserial_path,force): def runsync(self,logger=None): """!Runs the "sync" command as an exe().""" if logger is None: logger=self.logger - sync=produtil.prog.Runner(['/bin/sync']) - produtil.pipeline.Pipeline(sync,capture=True,logger=logger) + sync=Runner(['/bin/sync']) + Pipeline(sync,capture=True,logger=logger) def openmp(self,arg,threads): """!Does nothing. This implementation does not support OpenMP. @@ -168,7 +167,7 @@ def make_bigexe(self,exe,**kwargs): @returns an empty list @param exe The executable to run on compute nodes. @param kwargs Ignored.""" - return produtil.prog.ImmutableRunner([str(exe)],**kwargs) + return ImmutableRunner([str(exe)],**kwargs) class CMDFGen(object): diff --git a/produtil/mpi_impl/no_mpi.py b/metplus/produtil/mpi_impl/no_mpi.py similarity index 88% rename from produtil/mpi_impl/no_mpi.py rename to metplus/produtil/mpi_impl/no_mpi.py index 71dfc1bfd9..bb20f9a2b3 100644 --- a/produtil/mpi_impl/no_mpi.py +++ b/metplus/produtil/mpi_impl/no_mpi.py @@ -6,9 +6,8 @@ # produtil.run.mpiserial functions, providing the implementation # needed to run when MPI is unavailable. -import os, logging -import produtil.prog,produtil.pipeline -from .mpi_impl_base import MPIDisabled,ImplementationBase +import logging +from .mpi_impl_base import ImplementationBase module_logger=logging.getLogger('lsf_cray_intel') class Implementation(ImplementationBase): diff --git a/produtil/mpi_impl/srun.py b/metplus/produtil/mpi_impl/srun.py similarity index 90% rename from produtil/mpi_impl/srun.py rename to metplus/produtil/mpi_impl/srun.py index c3574ab1ee..d4518ed0fa 100644 --- a/produtil/mpi_impl/srun.py +++ b/metplus/produtil/mpi_impl/srun.py @@ -6,12 +6,14 @@ # commands. import os, logging, re -import produtil.fileop,produtil.prog,produtil.mpiprog,produtil.pipeline +from metplus.produtil.fileop import find_exe +from metplus.produtil.prog import Runner, ImmutableRunner +import metplus.produtil.mpiprog as mpiprog +import metplus.produtil.pipeline as pipeline from .mpi_impl_base import MPIMixed,CMDFGen,ImplementationBase, \ MPIThreadsMixed,MPILocalOptsMixed,MPITooManyRanks -from produtil.pipeline import NoMoreProcesses -from produtil.mpiprog import MIXED_VALUES +from metplus.produtil.mpiprog import MIXED_VALUES class Implementation(ImplementationBase): """Adds SLURM srun support to produtil.run @@ -37,12 +39,12 @@ def detect(srun_path=None,mpiserial_path=None,logger=None,force=False,silent=Fal if force: srun_path='srun' else: - srun_path=produtil.fileop.find_exe('srun',raise_missing=True) + srun_path=find_exe('srun',raise_missing=True) if scontrol_path is None: if force: scontrol_path='scontrol' else: - scontrol_path=produtil.fileop.find_exe('scontrol',raise_missing=True) + scontrol_path=find_exe('scontrol',raise_missing=True) if 'SLURM_NODELIST' not in os.environ and not force: return None return Implementation(srun_path,scontrol_path,mpiserial_path,logger,silent,force) @@ -58,8 +60,8 @@ def __init__(self,srun_path,scontrol_path,mpiserial_path,logger,silent,force): def runsync(self,logger=None): """!Runs the "sync" command as an exe().""" if logger is None: logger=self.logger - sync=produtil.prog.Runner(['/bin/sync']) - produtil.pipeline.Pipeline(sync,capture=True,logger=logger) + sync=Runner(['/bin/sync']) + pipeline.Pipeline(sync,capture=True,logger=logger) def openmp(self,arg,threads): """!Adds OpenMP support to the provided object @@ -86,7 +88,7 @@ def make_bigexe(self,exe,**kwargs): @returns an empty list @param exe The executable to run on compute nodes. @param kwargs Ignored.""" - return produtil.prog.ImmutableRunner([str(exe)],**kwargs) + return ImmutableRunner([str(exe)],**kwargs) def mpirunner(self,arg,allranks=False,**kwargs): """!Turns a produtil.mpiprog.MPIRanksBase tree into a produtil.prog.Runner @@ -104,10 +106,10 @@ def mpirunner(self,arg,allranks=False,**kwargs): def _get_available_nodes(self): available_nodes=list() nodeset=set() - scontrol=produtil.prog.Runner([ + scontrol=Runner([ self.scontrol_path,'show','hostnames', os.environ['SLURM_NODELIST']]) - p=produtil.pipeline.Pipeline( + p=pipeline.Pipeline( scontrol,capture=True,logger=self.logger) nodelist=p.to_string() for line in nodelist.splitlines(): @@ -121,7 +123,7 @@ def _get_available_nodes(self): def mpirunner_impl(self,arg,allranks=False,rewrite_nodefile=True,label_io=False,**kwargs): """!This is the underlying implementation of mpirunner and should not be called directly.""" - assert(isinstance(arg,produtil.mpiprog.MPIRanksBase)) + assert(isinstance(arg,mpiprog.MPIRanksBase)) (serial,parallel)=arg.check_serial() if serial and parallel: raise MPIMixed('Cannot mix serial and parallel MPI ranks in the ' @@ -143,7 +145,7 @@ def mpirunner_impl(self,arg,allranks=False,rewrite_nodefile=True,label_io=False, srun_args.append('--distribution=block:block') arglist=[ str(a) for a in arg.to_arglist( pre=srun_args,before=[],between=[])] - return produtil.prog.Runner(arglist) + return Runner(arglist) elif allranks: raise MPIAllRanksError( "When using allranks=True, you must provide an mpi program " @@ -151,9 +153,9 @@ def mpirunner_impl(self,arg,allranks=False,rewrite_nodefile=True,label_io=False, "all ranks).") elif serial: srun_args.append('--distribution=block:block') - arg=produtil.mpiprog.collapse(arg) + arg=mpiprog.collapse(arg) lines=[str(a) for a in arg.to_arglist(to_shell=True,expand=True)] - return produtil.prog.Runner( + return Runner( [self.srun_path,'--ntasks','%s'%(arg.nranks()),self.mpiserial_path], prerun=CMDFGen('serialcmdf',lines,silent=self.silent,**kwargs)) else: @@ -204,4 +206,4 @@ def mpirunner_impl(self,arg,allranks=False,rewrite_nodefile=True,label_io=False, silent=self.silent,filename_option='--nodelist', next_prerun=prerun,**kwargs) - return produtil.prog.Runner(srun_args,prerun=prerun) + return Runner(srun_args,prerun=prerun) diff --git a/produtil/mpiprog.py b/metplus/produtil/mpiprog.py similarity index 99% rename from produtil/mpiprog.py rename to metplus/produtil/mpiprog.py index e7adf6e1f4..ad7609f445 100644 --- a/produtil/mpiprog.py +++ b/metplus/produtil/mpiprog.py @@ -44,12 +44,9 @@ # Ensure nothing is loaded by "from produtil.mpiprog import *" __all__=[] -import sys - import io import logging -import produtil.prog -from produtil.prog import ProgSyntaxError, shbackslash +from metplus.produtil.prog import ProgSyntaxError, shbackslash, Runner, ImmutableRunner class MPIProgSyntaxError(ProgSyntaxError): """!Base class of syntax errors in MPI program specifications""" @@ -831,7 +828,7 @@ def __init__(self,arg,logger=None): self._localopts=list(arg._localopts) self._turbomode=arg.turbomode self._ranks_per_node=arg.ranks_per_node - elif isinstance(arg,produtil.prog.Runner): + elif isinstance(arg,Runner): if arg.isplainexe(): self._args=[x for x in arg.args()] else: @@ -871,7 +868,7 @@ def delthreads(self): def to_shell(self): """!Return a POSIX sh representation of this MPI rank, if possible.""" - return ' '.join([produtil.prog.shbackslash(x) for x in self._args]) + return ' '.join([shbackslash(x) for x in self._args]) def __getitem__(self,args): """!Adds arguments to this MPI rank's program.""" c=self.copy() @@ -995,8 +992,8 @@ def __init__(self,runner,logger=None): self._ranks_per_node=0 def make_runners_immutable(self): """!Creates a version of self with a produtil.prog.ImmutableRunner child.""" - if not isinstance(self._runner,produtil.prog.ImmutableRunner): - return MPISerial(produtil.prog.ImmutableRunner(self._runner),self._logger) + if not isinstance(self._runner,ImmutableRunner): + return MPISerial(ImmutableRunner(self._runner),self._logger) else: return self def copy(self): diff --git a/produtil/numerics.py b/metplus/produtil/numerics.py similarity index 100% rename from produtil/numerics.py rename to metplus/produtil/numerics.py diff --git a/produtil/pipeline.py b/metplus/produtil/pipeline.py similarity index 100% rename from produtil/pipeline.py rename to metplus/produtil/pipeline.py diff --git a/produtil/prog.py b/metplus/produtil/prog.py similarity index 99% rename from produtil/prog.py rename to metplus/produtil/prog.py index f6f2c6fdfc..441cc6bd35 100644 --- a/produtil/prog.py +++ b/metplus/produtil/prog.py @@ -37,11 +37,9 @@ directly, except for type checking (ie.: to see if your argument is a Runner before passing it to produtil.run.checkrun)..""" -import produtil.sigsafety -import io,select,io,re,time,fcntl,os,logging,signal +import re, os -import produtil.mpi_impl -from produtil.pipeline import launch, manage, PIPE, ERR2OUT +from metplus.produtil.pipeline import ERR2OUT ERR2OUT_FCT_STR = '.err2out()' diff --git a/produtil/retry.py b/metplus/produtil/retry.py similarity index 100% rename from produtil/retry.py rename to metplus/produtil/retry.py diff --git a/produtil/run.py b/metplus/produtil/run.py similarity index 97% rename from produtil/run.py rename to metplus/produtil/run.py index a7e469b890..bdd305ea29 100644 --- a/produtil/run.py +++ b/metplus/produtil/run.py @@ -155,15 +155,14 @@ """ import time, logging -import produtil.mpi_impl -import produtil.sigsafety -import produtil.prog as prog -import produtil.mpiprog as mpiprog -import produtil.pipeline as pipeline +import metplus.produtil.mpi_impl as mpi_impl +import metplus.produtil.prog as prog +import metplus.produtil.mpiprog as mpiprog +import metplus.produtil.pipeline as pipeline # These two were moved to produtil.prog to avoid a cyclic import. # They still need to be available from produtil.run: -from produtil.prog import InvalidRunArgument,ExitStatusException +from metplus.produtil.prog import InvalidRunArgument,ExitStatusException ##@var __all__ # List of symbols exported by "from produtil.run import *" @@ -180,7 +179,7 @@ # The cached return value from detect_mpi() _detected_mpi=None -def make_mpi(mpi_name=produtil.mpi_impl.NO_NAME,**kwargs): +def make_mpi(mpi_name=mpi_impl.NO_NAME,**kwargs): """!Creates an MPI implementation object for the specified MPI implementation. @@ -208,7 +207,7 @@ def make_mpi(mpi_name=produtil.mpi_impl.NO_NAME,**kwargs): # situation does not need any initialization of the produtil.run # module: if mpi_name is None: - return produtil.mpi_impl.get_mpi(None,**kwargs) + return mpi_impl.get_mpi(None,**kwargs) # For anything other than "None," we have to ensure the # produtil.run is initialized to something before running @@ -216,7 +215,7 @@ def make_mpi(mpi_name=produtil.mpi_impl.NO_NAME,**kwargs): detect_mpi() # Next, return the requested implementation: - return produtil.mpi_impl.get_mpi(mpi_name,**kwargs) + return mpi_impl.get_mpi(mpi_name,**kwargs) def detect_mpi(): """!Called by functions inside produtil.run to automatically @@ -253,10 +252,10 @@ def detect_mpi(): # First, set the implementation to None so that the mpi_impl # subclasses can use produtil.run. This will only allow serial, # non-OpenMP programs: - _detected_mpi=produtil.mpi_impl.get_mpi(None) + _detected_mpi=mpi_impl.get_mpi(None) - # Next, ask produtil.mpi_impl to detect the MPI implementation: - detected=produtil.mpi_impl.get_mpi() + # Next, ask mpi_impl to detect the MPI implementation: + detected=mpi_impl.get_mpi() # If detection succeeds, override the selected MPI implementation: if detected: diff --git a/produtil/setup.py b/metplus/produtil/setup.py similarity index 85% rename from produtil/setup.py rename to metplus/produtil/setup.py index 9a58e6a8b0..b97bdc8840 100644 --- a/produtil/setup.py +++ b/metplus/produtil/setup.py @@ -7,9 +7,12 @@ # Lists symbols exported by "from produtil.setup import *" __all__=['setup'] -import logging, threading -import produtil.sigsafety, produtil.log, produtil.dbnalert, produtil.cluster -import produtil.batchsystem +import threading +from metplus.produtil.sigsafety import install_handlers +from metplus.produtil.log import configureLogging +from metplus.produtil.dbnalert import init_module +from metplus.produtil.cluster import set_cluster, where +from metplus.produtil.batchsystem import set_default_name def setup(ignore_hup=False,dbnalert_logger=None,jobname=None,cluster=None, send_dbn=None,thread_logger=False,thread_stack=2**24,**kwargs): @@ -61,19 +64,19 @@ def setup(ignore_hup=False,dbnalert_logger=None,jobname=None,cluster=None, # Set the default jobname. This is usually used for manually-run # scripts to ensure they have a "jobname" in the logging system: if jobname is not None: - produtil.batchsystem.set_default_name(jobname) + set_default_name(jobname) # Configure logging next so that the install_handlers will be able # to log. - produtil.log.configureLogging(thread_logger=thread_logger,**kwargs) + configureLogging(thread_logger=thread_logger,**kwargs) # Install signal handlers, and let the caller configure SIGHUP settings: - produtil.sigsafety.install_handlers(ignore_hup=ignore_hup) + install_handlers(ignore_hup=ignore_hup) # Set up dbnalert: - produtil.dbnalert.init_module(logger=dbnalert_logger,jobname=jobname, + init_module(logger=dbnalert_logger,jobname=jobname, send_dbn=send_dbn) # Set up cluster: if cluster is not None: - produtil.cluster.set_cluster(cluster) + set_cluster(cluster) else: - produtil.cluster.where() # guess cluster + where() # guess cluster diff --git a/produtil/sigsafety.py b/metplus/produtil/sigsafety.py similarity index 97% rename from produtil/sigsafety.py rename to metplus/produtil/sigsafety.py index 301e488031..23699d766d 100644 --- a/produtil/sigsafety.py +++ b/metplus/produtil/sigsafety.py @@ -40,7 +40,8 @@ One can call install_handlers directly, though it is recommended to call produtil.setup.setup instead.""" -import produtil.locking, produtil.pipeline +from metplus.produtil.locking import disable_locking +from metplus.produtil.pipeline import kill_all import signal ##@var defaultsigs @@ -135,7 +136,7 @@ def hup_handler(signum,frame): caught_signal=signum caught_class=HangupSignal - produtil.locking.disable_locking() + disable_locking() raise HangupSignal(signum) def term_handler(signum,frame): @@ -145,8 +146,8 @@ def term_handler(signum,frame): caught_signal=signum caught_class=FatalSignal - produtil.locking.disable_locking() # forbid file locks - produtil.pipeline.kill_all() # kill all subprocesses + disable_locking() # forbid file locks + kill_all() # kill all subprocesses uninstall_handlers() raise FatalSignal(signum) diff --git a/metplus/scripts/run_metplus.py b/metplus/scripts/run_metplus.py new file mode 120000 index 0000000000..116d56b438 --- /dev/null +++ b/metplus/scripts/run_metplus.py @@ -0,0 +1 @@ +../../ush/run_metplus.py \ No newline at end of file diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index c0d25deea6..238b2c10b5 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -11,7 +11,6 @@ import os import re -import sys import logging from datetime import datetime, timezone import time @@ -20,7 +19,7 @@ from pathlib import Path import uuid -from produtil.config import ProdConfig +from metplus.produtil.config import ProdConfig from .constants import RUNTIME_CONFS, MISSING_DATA_VALUE from .string_template_substitution import do_string_sub @@ -63,7 +62,7 @@ # set parm base to METPLUS_BASE/parm unless METPLUS_PARM_BASE env var is set PARM_BASE = os.environ.get('METPLUS_PARM_BASE', - os.path.join(METPLUS_BASE, PARM_DIR)) + os.path.join(METPLUS_BASE, 'metplus', PARM_DIR)) # name of directory under PARM_DIR that contains defaults METPLUS_CONFIG_DIR = 'metplus_config' diff --git a/metplus/util/constants.py b/metplus/util/constants.py index 3208aa2a26..3b5c0f7ff2 100644 --- a/metplus/util/constants.py +++ b/metplus/util/constants.py @@ -93,6 +93,14 @@ 'GenEnsProd', ) +# wrappers that support the time_offset_warning global MET config variable +TIME_OFFSET_WARNING_WRAPPERS = ( + 'GridStat', + 'MODE', + 'SeriesAnalysis', + 'WaveletStat', +) + # configuration variables that are specific to a given run # these are copied from [config] to [runtime] at the # end of the run so they will not be read if the final diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index 145468c5a6..90ace07686 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -6,7 +6,7 @@ from logging import Logger import shlex -from produtil.run import exe, run +from metplus.produtil.run import exe, run from .string_manip import get_logfile_info, log_terminal_includes_info from .system_util import get_user_info, write_list_to_file diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index b9ba20f3e3..d47efa2617 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -15,7 +15,8 @@ from abc import ABCMeta from inspect import getframeinfo, stack -from ..util.constants import PYTHON_EMBEDDING_TYPES, COMPRESSION_EXTENSIONS, MULTIPLE_INPUT_WRAPPERS +from ..util.constants import PYTHON_EMBEDDING_TYPES, COMPRESSION_EXTENSIONS +from ..util.constants import MULTIPLE_INPUT_WRAPPERS, TIME_OFFSET_WARNING_WRAPPERS from ..util import getlist, preprocess_file, loop_over_times_and_call from ..util import do_string_sub, ti_calculate, get_seconds_from_string from ..util import get_time_from_file, shift_time_seconds, seconds_to_met_time @@ -103,15 +104,16 @@ def __init__(self, config, instance=None): # add key to list of env vars to set self.env_var_keys.append(self.MET_OVERRIDES_KEY) - # add time_offset_warning env var - self.env_var_keys.append('METPLUS_TIME_OFFSET_WARNING') - time_offset_warning = self.get_wrapper_or_generic_config( - 'TIME_OFFSET_WARNING', var_type='int' - ) - if time_offset_warning != '': - self.env_var_dict['METPLUS_TIME_OFFSET_WARNING'] = ( - f'time_offset_warning = {time_offset_warning};' + if get_wrapper_name(self.app_name) in TIME_OFFSET_WARNING_WRAPPERS: + # add time_offset_warning env var + self.env_var_keys.append('METPLUS_TIME_OFFSET_WARNING') + time_offset_warning = self.get_wrapper_or_generic_config( + 'TIME_OFFSET_WARNING', var_type='int' ) + if time_offset_warning != '': + self.env_var_dict['METPLUS_TIME_OFFSET_WARNING'] = ( + f'time_offset_warning = {time_offset_warning};' + ) # warn if any environment variables set by the wrapper are not # being utilized in the user's config file diff --git a/metplus/wrappers/point2grid_wrapper.py b/metplus/wrappers/point2grid_wrapper.py index c95a9da3d4..11ed39ce0c 100755 --- a/metplus/wrappers/point2grid_wrapper.py +++ b/metplus/wrappers/point2grid_wrapper.py @@ -14,6 +14,8 @@ from ..util import do_string_sub from ..util import remove_quotes +from ..util.met_config import add_met_config_dict_list + from . import ReformatPointWrapper '''!@namespace Point2GridWrapper @@ -26,6 +28,15 @@ class Point2GridWrapper(ReformatPointWrapper): RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + WRAPPER_ENV_VAR_KEYS = [ + 'METPLUS_VALID_TIME', + 'METPLUS_OBS_WINDOW_DICT', + 'METPLUS_MESSAGE_TYPE', + 'METPLUS_VAR_NAME_MAP_LIST', + 'METPLUS_OBS_QUALITY_INC', + 'METPLUS_OBS_QUALITY_EXC', + ] + def __init__(self, config, instance=None): self.app_name = "point2grid" self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), @@ -70,8 +81,12 @@ def create_c_dict(self): 'POINT2GRID_INPUT_LEVEL', '') - c_dict['QC_FLAGS'] = self.config.getraw('config', - 'POINT2GRID_QC_FLAGS') + # support legacy QC_FLAGS and new GOES_QC_FLAGS to set -goes_qc arg + config_name = self.config.get_mp_config_name(['POINT2GRID_GOES_QC_FLAGS', + 'POINT2GRID_QC_FLAGS']) + if config_name: + c_dict['GOES_QC_FLAGS'] = self.config.getraw('config', config_name) + c_dict['ADP'] = self.config.getraw('config', 'POINT2GRID_ADP') c_dict['REGRID_METHOD'] = self.config.getstr('config', @@ -94,6 +109,28 @@ def create_c_dict(self): 'POINT2GRID_VLD_THRESH', '') + # get the MET config file path or use default + c_dict['CONFIG_FILE'] = self.get_config_file('Point2GridConfig_wrapped') + + # read config file variables + self.add_met_config(name='valid_time', data_type='string') + self.add_met_config_window('obs_window') + self.add_met_config(name='message_type', data_type='list') + if not add_met_config_dict_list(config=self.config, + app_name=self.app_name, + output_dict=self.env_var_dict, + dict_name='var_name_map', + dict_items={'key': 'string', + 'val': 'string'}): + self.isOK = False + + self.add_met_config(name='obs_quality_inc', data_type='list', + metplus_configs=['POINT2GRID_OBS_QUALITY_INC', + 'POINT2GRID_OBS_QUALITY_INCLUDE', + 'POINT2GRID_OBS_QUALITY']) + self.add_met_config(name='obs_quality_exc', data_type='list', + metplus_configs=['POINT2GRID_OBS_QUALITY_EXC', + 'POINT2GRID_OBS_QUALITY_EXCLUDE']) return c_dict def find_input_files(self, time_info): @@ -131,8 +168,11 @@ def set_command_line_arguments(self, time_info): #Add either the specified level above or the defauilt blank one self.args.append(f"-field 'name=\"{input_field}\"; level=\"{input_level}\";'") - if self.c_dict['QC_FLAGS'] != '': - self.args.append(f"-qc {self.c_dict['QC_FLAGS']}") + config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) + self.args.append(f'-config {config_file}') + + if self.c_dict.get('GOES_QC_FLAGS', '') != '': + self.args.append(f"-goes_qc {self.c_dict['GOES_QC_FLAGS']}") if self.c_dict['ADP']: self.args.append(f"-adp {self.c_dict['ADP']}") diff --git a/metplus/wrappers/series_analysis_wrapper.py b/metplus/wrappers/series_analysis_wrapper.py index cea1a44b4a..1fb1948a6b 100755 --- a/metplus/wrappers/series_analysis_wrapper.py +++ b/metplus/wrappers/series_analysis_wrapper.py @@ -784,9 +784,7 @@ def build_and_run_series_request(self, time_info, fcst_path, obs_path): add_field_info_to_time_info(time_info, var_info) # get formatted field dictionary to pass into the MET config file - fcst_field, obs_field = ( - self.get_formatted_fields(var_info, time_info, fcst_path, obs_path) - ) + fcst_field, obs_field = self.get_formatted_fields(var_info, time_info) if fcst_field is None: continue @@ -1035,22 +1033,20 @@ def _get_netcdf_min_max(filepath, variable_name): except (FileNotFoundError, KeyError): return None, None - def get_formatted_fields(self, var_info, time_info, fcst_path, obs_path): + def get_formatted_fields(self, var_info, time_info): """! Get forecast and observation field information for var_info and format it so it can be passed into the MET config file @param var_info dictionary containing info to format @param time_info dictionary containing time information - @param fcst_path path to file list file for forecast data - @param obs_path path to file list file for observation data @returns tuple containing strings of the formatted forecast and observation information or (None, None) if something went wrong """ fcst_field_list = ( - self._get_field_list('fcst', var_info, time_info, obs_path) + self._get_field_list('fcst', var_info, time_info) ) obs_field_list = ( - self._get_field_list('obs', var_info, time_info, fcst_path) + self._get_field_list('obs', var_info, time_info) ) if not fcst_field_list or not obs_field_list: @@ -1062,7 +1058,7 @@ def get_formatted_fields(self, var_info, time_info, fcst_path, obs_path): return fcst_fields, obs_fields - def _get_field_list(self, data_type, var_info, time_info, file_list_path): + def _get_field_list(self, data_type, var_info, time_info): """!Get formatted field information in a list. If no time (init/valid/lead) filename template tags were found in the level value or if the time info contains all init/valid/lead values @@ -1075,10 +1071,8 @@ def _get_field_list(self, data_type, var_info, time_info, file_list_path): @param data_type type of data to process, e.g. fcst or obs @param var_info dictionary containing info to format @param time_info dictionary containing time information - @param file_list_path path to file list file to parse @returns list containing formatted field info to pass to MET config """ - other = 'OBS' if data_type == 'fcst' else 'FCST' # if there are no time tags (init/valid/lead) in the field level # or if init, valid, and lead have values in time_info, # get field info for a single field to pass to the MET config file @@ -1088,15 +1082,10 @@ def _get_field_list(self, data_type, var_info, time_info, file_list_path): field_list = [] - # handle multiple templates - templates = [] - for template in self.c_dict[f'{other}_INPUT_TEMPLATE'].split(','): - templates.append(os.path.join(self.c_dict[f'{other}_INPUT_DIR'], template.strip())) - - # loop through fcst/obs files to extract time info + # loop through fcst/obs files to read time info # for each file apply time info to field info and add to list - for file_time_info in self._get_times_from_file_list(file_list_path, - templates): + for file_dict in self.c_dict['ALL_FILES']: + file_time_info = file_dict['time_info'] field = self._get_field_sub_level(data_type, var_info, file_time_info) if field: field_list.extend(field) diff --git a/parm/met_config/Ascii2NcConfig_wrapped b/parm/met_config/Ascii2NcConfig_wrapped index e86116277b..4233450615 100644 --- a/parm/met_config/Ascii2NcConfig_wrapped +++ b/parm/met_config/Ascii2NcConfig_wrapped @@ -38,5 +38,4 @@ message_type_map = [ tmp_dir = "${MET_TMP_DIR}"; -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/EnsembleStatConfig_wrapped b/parm/met_config/EnsembleStatConfig_wrapped index fb505cc09e..9f508ff4c7 100644 --- a/parm/met_config/EnsembleStatConfig_wrapped +++ b/parm/met_config/EnsembleStatConfig_wrapped @@ -242,5 +242,4 @@ ${METPLUS_OUTPUT_PREFIX} tmp_dir = "${MET_TMP_DIR}"; -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/GenEnsProdConfig_wrapped b/parm/met_config/GenEnsProdConfig_wrapped index 0a67225e2a..5659d9330a 100644 --- a/parm/met_config/GenEnsProdConfig_wrapped +++ b/parm/met_config/GenEnsProdConfig_wrapped @@ -118,5 +118,4 @@ ${METPLUS_ENSEMBLE_FLAG_DICT} tmp_dir = "${MET_TMP_DIR}"; -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/GridDiagConfig_wrapped b/parm/met_config/GridDiagConfig_wrapped index 8c5117ddbe..691338acf2 100644 --- a/parm/met_config/GridDiagConfig_wrapped +++ b/parm/met_config/GridDiagConfig_wrapped @@ -41,5 +41,4 @@ ${METPLUS_MASK_DICT} tmp_dir = "${MET_TMP_DIR}"; -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/IODA2NCConfig_wrapped b/parm/met_config/IODA2NCConfig_wrapped index 7f25fb8cf1..ba1faa1695 100644 --- a/parm/met_config/IODA2NCConfig_wrapped +++ b/parm/met_config/IODA2NCConfig_wrapped @@ -114,5 +114,4 @@ tmp_dir = "${MET_TMP_DIR}"; //////////////////////////////////////////////////////////////////////////////// -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/MTDConfig_wrapped b/parm/met_config/MTDConfig_wrapped index f010982346..d521670eac 100644 --- a/parm/met_config/MTDConfig_wrapped +++ b/parm/met_config/MTDConfig_wrapped @@ -257,5 +257,4 @@ tmp_dir = "${MET_TMP_DIR}"; //////////////////////////////////////////////////////////////////////////////// -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/Madis2NcConfig_wrapped b/parm/met_config/Madis2NcConfig_wrapped index 282f6782f1..4cafab0038 100644 --- a/parm/met_config/Madis2NcConfig_wrapped +++ b/parm/met_config/Madis2NcConfig_wrapped @@ -25,6 +25,4 @@ ${METPLUS_TIME_SUMMARY_DICT} // //version = "V12.0.0"; -${METPLUS_TIME_OFFSET_WARNING} - ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/PB2NCConfig_wrapped b/parm/met_config/PB2NCConfig_wrapped index 6174608bb9..8b9057923b 100644 --- a/parm/met_config/PB2NCConfig_wrapped +++ b/parm/met_config/PB2NCConfig_wrapped @@ -129,5 +129,4 @@ tmp_dir = "${MET_TMP_DIR}"; //////////////////////////////////////////////////////////////////////////////// -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/PlotPointObsConfig_wrapped b/parm/met_config/PlotPointObsConfig_wrapped index 8fd113b86d..2194f01fee 100644 --- a/parm/met_config/PlotPointObsConfig_wrapped +++ b/parm/met_config/PlotPointObsConfig_wrapped @@ -96,5 +96,4 @@ tmp_dir = "${MET_TMP_DIR}"; //////////////////////////////////////////////////////////////////////////////// -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/Point2GridConfig_wrapped b/parm/met_config/Point2GridConfig_wrapped new file mode 100644 index 0000000000..5ebc51a460 --- /dev/null +++ b/parm/met_config/Point2GridConfig_wrapped @@ -0,0 +1,54 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Point2Grid configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// NetCDF output variable timing information: +// - valid_time in YYYYMMDD[_HH[MMSS]] format +// +//valid_time = +${METPLUS_VALID_TIME} + +// +// Observation time window +// +//obs_window = { +${METPLUS_OBS_WINDOW_DICT} + +//////////////////////////////////////////////////////////////////////////////// + + +// +// Observation message type +// +//message_type = [ +${METPLUS_MESSAGE_TYPE} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Mapping of input variable names to output variables names. +// +//var_name_map = [ +${METPLUS_VAR_NAME_MAP_LIST} + +//////////////////////////////////////////////////////////////////////////////// + +//obs_quality_inc = [ +${METPLUS_OBS_QUALITY_INC} + +//obs_quality_exc = [ +${METPLUS_OBS_QUALITY_EXC} + +//////////////////////////////////////////////////////////////////////////////// + +tmp_dir = "${MET_TMP_DIR}"; +//version = "V12.0.0"; + +//////////////////////////////////////////////////////////////////////////////// + +${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/PointStatConfig_wrapped b/parm/met_config/PointStatConfig_wrapped index 7b717c6d2c..0d12bd0a73 100644 --- a/parm/met_config/PointStatConfig_wrapped +++ b/parm/met_config/PointStatConfig_wrapped @@ -216,5 +216,4 @@ ${METPLUS_OUTPUT_PREFIX} //////////////////////////////////////////////////////////////////////////////// -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/STATAnalysisConfig_wrapped b/parm/met_config/STATAnalysisConfig_wrapped index e3e32deba4..d38c8def2d 100644 --- a/parm/met_config/STATAnalysisConfig_wrapped +++ b/parm/met_config/STATAnalysisConfig_wrapped @@ -164,5 +164,4 @@ tmp_dir = "${MET_TMP_DIR}"; //version = "V10.0"; -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCDiagConfig_wrapped b/parm/met_config/TCDiagConfig_wrapped index 0407e982c5..8dacb81b26 100644 --- a/parm/met_config/TCDiagConfig_wrapped +++ b/parm/met_config/TCDiagConfig_wrapped @@ -180,5 +180,4 @@ tmp_dir = "${MET_TMP_DIR}"; //////////////////////////////////////////////////////////////////////////////// -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCGenConfig_wrapped b/parm/met_config/TCGenConfig_wrapped index 7d813aa105..c18f895f8b 100644 --- a/parm/met_config/TCGenConfig_wrapped +++ b/parm/met_config/TCGenConfig_wrapped @@ -294,5 +294,4 @@ ${METPLUS_NC_PAIRS_GRID} tmp_dir = "${MET_TMP_DIR}"; -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCPairsConfig_wrapped b/parm/met_config/TCPairsConfig_wrapped index 7b9aa04651..93947c0c18 100644 --- a/parm/met_config/TCPairsConfig_wrapped +++ b/parm/met_config/TCPairsConfig_wrapped @@ -168,5 +168,4 @@ ${METPLUS_DIAG_CONVERT_MAP_LIST} tmp_dir = "${MET_TMP_DIR}"; -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCRMWConfig_wrapped b/parm/met_config/TCRMWConfig_wrapped index 00a7ea0ad7..9df3ddec8f 100644 --- a/parm/met_config/TCRMWConfig_wrapped +++ b/parm/met_config/TCRMWConfig_wrapped @@ -89,5 +89,4 @@ ${METPLUS_RMW_SCALE} tmp_dir = "${MET_TMP_DIR}"; -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCStatConfig_wrapped b/parm/met_config/TCStatConfig_wrapped index a6bd4afb23..dc4d5ec780 100644 --- a/parm/met_config/TCStatConfig_wrapped +++ b/parm/met_config/TCStatConfig_wrapped @@ -233,5 +233,4 @@ ${METPLUS_JOBS} tmp_dir = "${MET_TMP_DIR}"; -${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf b/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf index f1aa331915..b6a0de754e 100644 --- a/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf +++ b/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf @@ -79,5 +79,3 @@ ASCII2NC_TIME_SUMMARY_VAR_NAMES = ASCII2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 ASCII2NC_TIME_SUMMARY_VALID_FREQ = 0 ASCII2NC_TIME_SUMMARY_VALID_THRESH = 0.0 - -#ASCII2NC_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf index 3f911cbf71..3ac91314f2 100644 --- a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf +++ b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf @@ -246,5 +246,3 @@ ENSEMBLE_STAT_NC_ORANK_FLAG_WEIGHT = FALSE #ENSEMBLE_STAT_CONTROL_ID = #ENSEMBLE_STAT_GRID_WEIGHT_FLAG = - -#ENSEMBLE_STAT_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf b/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf index 2d2394d374..55ff515571 100644 --- a/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf +++ b/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf @@ -165,5 +165,3 @@ GEN_ENS_PROD_ENS_THRESH = 0.8 #GEN_ENS_PROD_ENS_MEMBER_IDS = #GEN_ENS_PROD_CONTROL_ID = - -#GEN_ENS_PROD_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf b/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf index 23813198fa..4104ee6016 100644 --- a/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf +++ b/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf @@ -83,5 +83,3 @@ GRID_DIAG_CONFIG_FILE = {PARM_BASE}/met_config/GridDiagConfig_wrapped #GRID_DIAG_REGRID_CENSOR_VAL = GRID_DIAG_MASK_POLY = MET_BASE/poly/SAO.poly - -#GRID_DIAG_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf b/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf index a036901873..0948944a55 100644 --- a/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf +++ b/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf @@ -96,5 +96,3 @@ IODA2NC_TIME_SUMMARY_OBS_VAR = "WIND" IODA2NC_TIME_SUMMARY_TYPE = "min", "max", "range", "mean", "stdev", "median", "p80" IODA2NC_TIME_SUMMARY_VLD_FREQ = 0 IODA2NC_TIME_SUMMARY_VLD_THRESH = 0.0 - -#IODA2NC_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/MADIS2NC/MADIS2NC.conf b/parm/use_cases/met_tool_wrapper/MADIS2NC/MADIS2NC.conf index ac4067dd6c..41b8009075 100644 --- a/parm/use_cases/met_tool_wrapper/MADIS2NC/MADIS2NC.conf +++ b/parm/use_cases/met_tool_wrapper/MADIS2NC/MADIS2NC.conf @@ -83,6 +83,4 @@ MADIS2NC_TYPE = metar #MADIS2NC_TIME_SUMMARY_VLD_FREQ = #MADIS2NC_TIME_SUMMARY_VLD_THRESH = -#MADIS2NC_TIME_OFFSET_WARNING = - #MADIS2NC_MET_CONFIG_OVERRIDES = \ No newline at end of file diff --git a/parm/use_cases/met_tool_wrapper/MTD/MTD.conf b/parm/use_cases/met_tool_wrapper/MTD/MTD.conf index 30d3c1403c..6f615537e0 100644 --- a/parm/use_cases/met_tool_wrapper/MTD/MTD.conf +++ b/parm/use_cases/met_tool_wrapper/MTD/MTD.conf @@ -99,5 +99,3 @@ MTD_REGRID_TO_GRID = OBS MTD_MIN_VOLUME = 2000 MTD_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_NAME}_{CURRENT_FCST_LEVEL} - -#MTD_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf b/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf index 9acd346c18..93651289ea 100644 --- a/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf +++ b/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf @@ -94,5 +94,3 @@ PB2NC_TIME_SUMMARY_VALID_FREQ = 0 PB2NC_TIME_SUMMARY_VALID_THRESH = 0.0 #PB2NC_OBS_BUFR_MAP = - -#PB2NC_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf index 6289088f02..304e1c5253 100644 --- a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf +++ b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf @@ -131,5 +131,3 @@ PLOT_POINT_OBS_POINT_DATA = colorbar_flag = FALSE; } } - -#PLOT_POINT_OBS_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf b/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf index 570ee1cc81..03695fcc67 100644 --- a/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf +++ b/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf @@ -68,14 +68,24 @@ POINT2GRID_REGRID_METHOD = MAX POINT2GRID_INPUT_FIELD =TMP POINT2GRID_INPUT_LEVEL = -#POINT2GRID_QC_FLAGS = +#POINT2GRID_GOES_QC_FLAGS = POINT2GRID_ADP = - POINT2GRID_GAUSSIAN_DX = 81.271 POINT2GRID_GAUSSIAN_RADIUS = 120 POINT2GRID_PROB_CAT_THRESH = POINT2GRID_VLD_THRESH = + +#POINT2GRID_VALID_TIME = +#POINT2GRID_OBS_WINDOW_BEG = +#POINT2GRID_OBS_WINDOW_END = +#POINT2GRID_MESSAGE_TYPE = +#POINT2GRID_VAR_NAME_MAP1_KEY = +#POINT2GRID_VAR_NAME_MAP1_VAL = +#POINT2GRID_OBS_QUALITY_INC = +#POINT2GRID_OBS_QUALITY_EXC = + +#POINT2GRID_MET_CONFIG_OVERRIDES = diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf index 3cfaa3b52e..6d793e2bf9 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf @@ -264,5 +264,3 @@ POINT_STAT_MESSAGE_TYPE = ADPUPA, ADPSFC #POINT_STAT_UGRID_MAX_DISTANCE_KM = #POINT_STAT_UGRID_COORDINATES_FILE = #POINT_STAT_UGRID_CONFIG_FILE = - -#POINT_STAT_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/StatAnalysis/StatAnalysis.conf b/parm/use_cases/met_tool_wrapper/StatAnalysis/StatAnalysis.conf index 59378482fe..14eb180192 100644 --- a/parm/use_cases/met_tool_wrapper/StatAnalysis/StatAnalysis.conf +++ b/parm/use_cases/met_tool_wrapper/StatAnalysis/StatAnalysis.conf @@ -84,5 +84,3 @@ LINE_TYPE_LIST = GROUP_LIST_ITEMS = FCST_INIT_HOUR_LIST LOOP_LIST_ITEMS = FCST_VALID_HOUR_LIST, MODEL_LIST - -#STAT_ANALYSIS_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf b/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf index 078e9c04ae..b10919341f 100644 --- a/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf +++ b/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf @@ -117,5 +117,3 @@ TC_DIAG_DOMAIN_INFO1_OVERRIDE_DIAGS = #TC_DIAG_NC_DIAG_FLAG = #TC_DIAG_CIRA_DIAG_FLAG = #TC_DIAG_OUTPUT_BASE_FORMAT = - -#TC_DIAG_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/TCGen/TCGen.conf b/parm/use_cases/met_tool_wrapper/TCGen/TCGen.conf index 424c3376bb..ace36034eb 100644 --- a/parm/use_cases/met_tool_wrapper/TCGen/TCGen.conf +++ b/parm/use_cases/met_tool_wrapper/TCGen/TCGen.conf @@ -177,5 +177,3 @@ TC_GEN_DLAND_FILE = MET_BASE/tc_data/dland_global_tenth_degree.nc TC_GEN_BASIN_FILE = MET_BASE/tc_data/basin_global_tenth_degree.nc TC_GEN_NC_PAIRS_GRID = G003 - -#TC_GEN_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf index 231138ef23..e5a0bcf1eb 100644 --- a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf +++ b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf @@ -122,5 +122,3 @@ TC_PAIRS_MISSING_VAL = -9999 #TC_PAIRS_DIAG_CONVERT_MAP1_DIAG_SOURCE = #TC_PAIRS_DIAG_CONVERT_MAP1_KEY = #TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT = - -#TC_PAIRS_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf b/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf index a410e91ffe..60b2e494db 100644 --- a/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf +++ b/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf @@ -95,5 +95,3 @@ TC_RMW_CYCLONE = 14 #TC_RMW_VALID_INCLUDE_LIST = #TC_RMW_VALID_EXCLUDE_LIST = #TC_RMW_VALID_HOUR_LIST = - -#TC_RMW_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf b/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf index 7f4be36833..ec1705915b 100644 --- a/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf +++ b/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf @@ -117,5 +117,3 @@ TC_STAT_MATCH_POINTS = false #TC_STAT_EVENT_EQUAL_LEAD = #TC_STAT_OUT_INIT_MASK = #TC_STAT_OUT_VALID_MASK = - -#TC_STAT_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf b/parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf new file mode 100644 index 0000000000..7bd4c7fb7b --- /dev/null +++ b/parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf @@ -0,0 +1,132 @@ +[config] + +# Documentation for this use-case can be found at: +# https://metplus.readthedocs.io/en/latest/generated/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.html + +# Processes to run +PROCESS_LIST = MODE + +# Time Info +LOOP_BY = INIT +INIT_TIME_FMT = %Y%m%d%H +INIT_BEG=2024010905 +INIT_END=2024010905 +INIT_INCREMENT = 1H + +LEAD_SEQ = 9,10 + +MODEL = RRFS +OBTYPE = ANALYSIS + +################################## +# Multivariate MODE Configurations +################################## + +# Run MODE to output super objects +MODE_MULTIVAR_LOGIC = #1 && #2 || #3 +MODE_MULTIVAR_INTENSITY_FLAG = FALSE,TRUE,TRUE + +FCST_MODE_INPUT_DIR = {INPUT_BASE}/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning +FCST_MODE_INPUT_TEMPLATE= {init?fmt=%Y%m%d}.{init?fmt=%H}00.F{lead?fmt=%HH}00.nc,{init?fmt=%Y%m%d}.{init?fmt=%H}00.F{lead?fmt=%HH}00.nc,{init?fmt=%Y%m%d}.{init?fmt=%H}00.F{lead?fmt=%HH}00.nc + +OBS_MODE_INPUT_DIR = {INPUT_BASE}/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning +OBS_MODE_INPUT_TEMPLATE = remap_GOES-16.{valid?fmt=%Y%m%d}.{valid?fmt=%H}0000.nc,{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000_remap.nc,remap_GOES-16_lightning_{valid?fmt=%Y%m%d}-{valid?fmt=%H}00_75W.nc + +MODE_OUTPUT_DIR = {OUTPUT_BASE}/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning +MODE_OUTPUT_TEMPLATE = f{lead?fmt=%2H} + +FCST_MODE_FILE_WINDOW_BEGIN = 0 +FCST_MODE_FILE_WINDOW_END = 0 + +OBS_MODE_FILE_WINDOW_BEGIN = 0 +OBS_MODE_FILE_WINDOW_END = 0 + +### +# Field Info +### + + +FCST_VAR1_NAME = bt13_topofatmosphere_G16 +FCST_VAR1_LEVELS = (*,*) +FCST_VAR1_OPTIONS = conv_radius = 5; conv_thresh = <=230; merge_thresh = <=230; merge_flag = NONE; file_type = NETCDF_MET; filter_attr_name = ["AREA"]; filter_attr_thresh= [ >=1 ]; censor_thresh = [<=0]; censor_val = [9999]; vld_thresh = 0.5; inten_perc_value = 100; inten_perc_thresh = NA + + +OBS_VAR1_NAME = channel_13_brightness_temperature +OBS_VAR1_LEVELS = L0 +OBS_VAR1_OPTIONS = conv_radius = 0; conv_thresh = <=230; merge_thresh = <=230; merge_flag = NONE; file_type = NETCDF_MET; filter_attr_name = ["AREA"]; filter_attr_thresh= [ >=1 ]; censor_thresh = [<=0]; censor_val = [9999]; vld_thresh = 0.5; inten_perc_value = 100; inten_perc_thresh = NA + + +FCST_VAR2_NAME = REFC_entireatmosphere_consideredasinglelayer +FCST_VAR2_LEVELS = (*,*) +FCST_VAR2_OPTIONS = conv_radius = 5; conv_thresh = >=40; merge_thresh = >=40; merge_flag = NONE; file_type = NETCDF_MET; filter_attr_name = ["AREA"]; filter_attr_thresh= [ >=1 ]; censor_thresh = [< -30]; censor_val = [9999]; vld_thresh = 0.5; inten_perc_value = 100; inten_perc_thresh = NA + +OBS_VAR2_NAME = MergedReflectivityQCComposite +OBS_VAR2_LEVELS = (*,*) +OBS_VAR2_OPTIONS = conv_radius = 5; conv_thresh = >=40; merge_thresh = >=40; merge_flag = NONE; file_type = NETCDF_MET; filter_attr_name = ["AREA"]; filter_attr_thresh= [ >=1 ]; censor_thresh = [< -30]; censor_val = [9999]; vld_thresh = 0.5; inten_perc_value = 100; inten_perc_thresh = NA + + +FCST_VAR3_NAME = LTNG_entireatmosphere +FCST_VAR3_LEVELS = (*,*) +FCST_VAR3_OPTIONS = conv_radius = 3; conv_thresh = >SFP10; merge_thresh = >SFP10; merge_flag = THRESH; file_type = NETCDF_MET; censor_thresh = [<= 0]; censor_val = [-9999]; vld_thresh = 0.5; inten_perc_value = 100; inten_perc_thresh = NA + +OBS_VAR3_NAME = flash_extent_density +OBS_VAR3_LEVELS = (*,*) +OBS_VAR3_OPTIONS = conv_radius = 3; conv_thresh = >SOP10; merge_thresh = >SOP10; merge_flag = THRESH; file_type = NETCDF_MET; censor_thresh = [<= 0]; censor_val = [-9999]; vld_thresh = 0.5; inten_perc_value = 100; inten_perc_thresh = NA + + +MODE_MATCH_FLAG = MERGE_BOTH + +MODE_REGRID_TO_GRID = NONE + +LOG_MODE_VERBOSITY = 4 + +MODE_OUTPUT_PREFIX = {MODEL}_or_{OBTYPE} + +MODE_GRID_RES = 3 + +MODE_INTEREST_FUNCTION_CENTROID_DIST = (( 0.0, 1.0 )( 60.0/grid_res, 1.0 )( 450.0/grid_res, 0.0 )) +MODE_INTEREST_FUNCTION_BOUNDARY_DIST = (( 0.0, 1.0 )( 400.0/grid_res, 0.0 )) +MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = (( 0.0, 1.0 )( 400.0/grid_res, 0.0 )) +MODE_INTEREST_FUNCTION_ANGLE_DIFF = (( 0.0, 1.0 )( 30.0, 1.0 )( 90.0, 0.0 )) +MODE_INTEREST_FUNCTION_ASPECT_DIFF = (( 0.00, 1.0 )( 0.10, 1.0 )( 0.75, 0.0 )) +MODE_INTEREST_FUNCTION_AREA_RATIO = (( 0.0, 0.0 )( 0.8, 1.0 )( 1.0, 1.0 )) +MODE_INTEREST_FUNCTION_INT_AREA_RATIO = (( 0.00, 0.00 )( 0.10, 0.50 )( 0.25, 1.00 )( 1.00, 1.00 )) +MODE_INTEREST_FUNCTION_CURVATURE_RATIO = (( 0.0, 0.0 )( 0.8, 1.0 )( 1.0, 1.0 )) +MODE_INTEREST_FUNCTION_COMPLEXITY_RATIO = (( 0.0, 0.0 )( 0.8, 1.0 )( 1.0, 1.0 )) +MODE_INTEREST_FUNCTION_INTEN_PERC_RATIO = (( 0.0, 0.0 )( 0.8, 1.0 )( 1.0, 1.0 )) + +MODE_TOTAL_INTEREST_THRESH = 0.65 + +MODE_MASK_GRID = "" +MODE_MASK_GRID_FLAG = NONE +MODE_MASK_POLY = "" +MODE_MASK_POLY_FLAG = NONE + +MODE_MASK_MISSING_FLAG = BOTH +MODE_MATCH_FLAG = MERGE_BOTH + +MODE_MAX_CENTROID_DIST = 600.0/grid_res + +MODE_WEIGHT_CENTROID_DIST = 4.0 +MODE_WEIGHT_BOUNDARY_DIST = 3.0 +MODE_WEIGHT_CONVEX_HULL_DIST = 1.0 +MODE_WEIGHT_ANGLE_DIFF = 1.0 +MODE_WEIGHT_ASPECT_DIFF = 0.0 +MODE_WEIGHT_AREA_RATIO = 4.0 +MODE_WEIGHT_INT_AREA_RATIO = 3.0 +MODE_WEIGHT_CURVATURE_RATIO = 0.0 +MODE_WEIGHT_COMPLEXITY_RATIO = 0.0 +MODE_WEIGHT_INTEN_PERC_RATIO = 0.0 +MODE_WEIGHT_INTEN_PERC_VALUE = 50 + +MODE_NC_PAIRS_FLAG_LATLON = TRUE +MODE_NC_PAIRS_FLAG_RAW = TRUE +MODE_NC_PAIRS_FLAG_OBJECT_RAW = TRUE +MODE_NC_PAIRS_FLAG_OBJECT_ID = TRUE +MODE_NC_PAIRS_FLAG_CLUSTER_ID = TRUE +MODE_NC_PAIRS_FLAG_POLYLINES = TRUE + +MODE_QUILT = False + +MODE_PS_PLOT_FLAG = TRUE +MODE_CT_STATS_FLAG = TRUE diff --git a/pyproject.toml b/pyproject.toml index b5a3c468d9..30e5629b73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,47 @@ [build-system] requires = [ - "setuptools>=42", - "wheel" + "setuptools", + "wheel", + "setuptools-git-versioning>=2.0,<3", ] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" + +[project] +name = "metplus" +dynamic = ["version", "dependencies"] +description = "METplus Wrappers" +authors = [ + {name = "METplus"}, +] +requires-python = ">=3.10.4" +readme = "README.md" +license = {text = "MIT"} +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://github.com/dtcenter/METplus" + +[project.scripts] +run_metplus = "metplus.scripts.run_metplus:cli_main" + +[tool.setuptools.packages] +find = {include = ["metplus*"], exclude = ["parm"]} + +[tool.setuptools.package-data] +metplus = [ + "VERSION", "RELEASE_DATE", "PYTHON_VERSION", "PYTHON_VERSION_MIN", "parm/**" +] + +[tool.setuptools.exclude-package-data] +metplus = ["README", "__pycache__", "*~"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} + +[tool.setuptools-git-versioning] +enabled = true +version_file = "metplus/VERSION" diff --git a/requirements.txt b/requirements.txt index 169d9d94cb..2c05e903ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -certifi==2024.7.4 -python-dateutil==2.8.2 -six==1.16.0 +certifi>=2024.7.4 +python-dateutil>=2.8.2 +six>=1.16.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 2114a77efe..0000000000 --- a/setup.py +++ /dev/null @@ -1,54 +0,0 @@ -from setuptools import setup, find_packages -from distutils.util import convert_path -import os - -with open("README.md", "r") as fh: - long_description = fh.read() - -with open("metplus/VERSION", "r") as fh: - version = fh.read().strip() - -with open("metplus/PYTHON_VERSION_MIN", "r") as fh: - python_version_req = fh.read().strip() - -# get list of additional files needed to add to package -data_files = [] -# add version and release date files -data_files.append(('metplus', - ['metplus/VERSION', - 'metplus/RELEASE_DATE', - 'metplus/PYTHON_VERSION', - 'metplus/PYTHON_VERSION_MIN', - ])) - -for root, _, files in os.walk('parm'): - parm_files = [] - for filename in files: - filepath = os.path.join(root, filename) - # skip README, tilda, and pycache files - if ('__pycache__' in filepath or - 'README' in filepath or - filepath.endswith('~')): - continue - parm_files.append(filepath) - if parm_files: - data_files.append((root, parm_files)) - -setup( - name="metplus", - version=version, - author="METplus", - description="METplus Wrappers", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/dtcenter/METplus", - packages=find_packages(), - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - ], - python_requires=f'>={python_version_req}', - data_files=data_files, - zip_safe=False, -) diff --git a/ush/run_metplus.py b/ush/run_metplus.py index eb374b4427..6f329867ac 100755 --- a/ush/run_metplus.py +++ b/ush/run_metplus.py @@ -25,7 +25,7 @@ # add metplus directory to path so the wrappers and utilities can be found sys.path.insert(0, abspath(join(dirname(realpath(__file__)), pardir))) -import produtil.setup +from metplus.produtil.setup import setup as produtil_setup from metplus.util import pre_run_setup, run_metplus, post_run_cleanup from metplus import __version__ as metplus_version @@ -110,12 +110,15 @@ def get_config_inputs_from_command_line(): return config_inputs -if __name__ == "__main__": +def cli_main(): try: - produtil.setup.setup(send_dbn=False, jobname='run-METplus') + produtil_setup(send_dbn=False, jobname='run-METplus') if not main(): sys.exit(1) except Exception as exc: print(traceback.format_exc()) print('ERROR: run_metplus failed: %s' % exc) sys.exit(2) + +if __name__ == "__main__": + cli_main()