diff --git a/.github/actions/run_tests/Dockerfile b/.github/actions/run_tests/Dockerfile index f76b50ffae..8e4636d235 100644 --- a/.github/actions/run_tests/Dockerfile +++ b/.github/actions/run_tests/Dockerfile @@ -3,8 +3,6 @@ FROM alpine:latest COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh -RUN apk add --update --no-cache docker python3 py3-pip bash - -RUN pip3 install python-dateutil requests +RUN apk add --update --no-cache docker python3 py3-pip bash py3-dateutil py3-requests ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 2ba2e011c4..4bf11ac758 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -1,7 +1,7 @@ [ { "category": "met_tool_wrapper", - "index_list": "0-29,59-61,63", + "index_list": "0-29,59-63", "run": false }, { @@ -101,7 +101,7 @@ }, { "category": "medium_range", - "index_list": "3-5", + "index_list": "3-5,10", "run": false }, { diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 1ab1e903bf..6a3d717e5b 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -10302,20 +10302,19 @@ METplus Configuration Glossary | *Used by:* TCDiag - TC_DIAG_CENSOR_THRESH - Specify the value for 'censor_thresh' in the MET configuration file for TCDiag. + TC_DIAG_DOMAIN_INFO_OVERRIDE_DIAGS + Specify the value for the nth 'domain_info.override_diags' in the MET configuration file for TCDiag. | *Used by:* TCDiag - TC_DIAG_CENSOR_VAL - Specify the value for 'censor_val' in the MET configuration file for TCDiag. + TC_DIAG_CENSOR_THRESH + .. warning:: **DEPRECATED:** Please use :term:`BOTH_VAR_OPTIONS` instead. - | *Used by:* TCDiag + TC_DIAG_CENSOR_VAL + .. warning:: **DEPRECATED:** Please use :term:`BOTH_VAR_OPTIONS` instead. TC_DIAG_CONVERT - Specify the value for 'convert' in the MET configuration file for TCDiag. - - | *Used by:* TCDiag + .. warning:: **DEPRECATED:** Please use :term:`BOTH_VAR_OPTIONS` instead. TC_DIAG_DATA_DOMAIN Specify the value for 'data.domain' in the MET configuration file for TCDiag. @@ -10407,8 +10406,8 @@ METplus Configuration Glossary | *Used by:* TCDiag - TC_DIAG_NC_RNG_AZI_FLAG - Specify the value for 'nc_rng_azi_flag' in the MET configuration file for TCDiag. + TC_DIAG_NC_CYL_GRID_FLAG + Specify the value for 'nc_cyl_grid_flag' in the MET configuration file for TCDiag. | *Used by:* TCDiag @@ -10423,7 +10422,10 @@ METplus Configuration Glossary | *Used by:* TCDiag TC_DIAG_OUTPUT_PREFIX - Specify the value for 'output_prefix' in the MET configuration file for TCDiag. + .. warning:: **DEPRECATED:** Please use :term:`TC_DIAG_OUTPUT_BASE_FORMAT` instead. + + TC_DIAG_OUTPUT_BASE_FORMAT + Specify the value for 'output_base_format' in the MET configuration file for TCDiag. | *Used by:* TCDiag diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index 28054ff5c2..6042b24fb1 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -197,6 +197,13 @@ the use case will not run until the METplus configuration file has been updated. How to upgrade ^^^^^^^^^^^^^^ +This video provides a demonstration of the process to upgrade a use case. + +.. raw:: html + + + + Removing **_CONFIG_FILE**, e.g. :term:`GRID_STAT_CONFIG_FILE`, from the METplus config file will prevent the errors and allow the use case to run. diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 1078f6466c..380c7e8673 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -8225,9 +8225,7 @@ METplus Configuration | :term:`TC_DIAG_DOMAIN_INFO_N_AZIMUTH` | :term:`TC_DIAG_DOMAIN_INFO_DELTA_RANGE_KM` | :term:`TC_DIAG_DOMAIN_INFO_DIAG_SCRIPT` -| :term:`TC_DIAG_CENSOR_THRESH` -| :term:`TC_DIAG_CENSOR_VAL` -| :term:`TC_DIAG_CONVERT` +| :term:`TC_DIAG_DOMAIN_INFO_OVERRIDE_DIAGS` | :term:`TC_DIAG_INPUT_DATATYPE` | :term:`TC_DIAG_DATA_DOMAIN` | :term:`TC_DIAG_DATA_LEVEL` @@ -8247,10 +8245,10 @@ METplus Configuration | :term:`TC_DIAG_RADIAL_VELOCITY_LONG_FIELD_NAME` | :term:`TC_DIAG_VORTEX_REMOVAL` | :term:`TC_DIAG_ONE_TIME_PER_FILE_FLAG` -| :term:`TC_DIAG_NC_RNG_AZI_FLAG` +| :term:`TC_DIAG_NC_CYL_GRID_FLAG` | :term:`TC_DIAG_NC_DIAG_FLAG` | :term:`TC_DIAG_CIRA_DIAG_FLAG` -| :term:`TC_DIAG_OUTPUT_PREFIX` +| :term:`TC_DIAG_OUTPUT_BASE_FORMAT` | :term:`TC_DIAG_SKIP_IF_OUTPUT_EXISTS` | :term:`TC_DIAG_MET_CONFIG_OVERRIDES` | :term:`TC_DIAG_SKIP_VALID_TIMES` @@ -8447,42 +8445,8 @@ ${METPLUS_DOMAIN_INFO_LIST} - domain_info.delta_range_km * - :term:`TC_DIAG_DOMAIN_INFO_DIAG_SCRIPT` - domain_info.diag_script - -${METPLUS_CENSOR_THRESH} -"""""""""""""""""""""""" - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`TC_DIAG_CENSOR_THRESH` - - censor_thresh - -${METPLUS_CENSOR_VAL} -""""""""""""""""""""" - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`TC_DIAG_CENSOR_VAL` - - censor_val - -${METPLUS_CONVERT} -"""""""""""""""""" - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`TC_DIAG_CONVERT` - - convert + * - :term:`TC_DIAG_DOMAIN_INFO_OVERRIDE_DIAGS` + - domain_info.override_diags ${METPLUS_DATA_FILE_TYPE} """"""""""""""""""""""""" @@ -8670,8 +8634,8 @@ ${METPLUS_ONE_TIME_PER_FILE_FLAG} * - :term:`TC_DIAG_ONE_TIME_PER_FILE_FLAG` - one_time_per_file_flag -${METPLUS_NC_RNG_AZI_FLAG} -"""""""""""""""""""""""""" +${METPLUS_NC_CYL_GRID_FLAG} +""""""""""""""""""""""""""" .. list-table:: :widths: 5 5 @@ -8679,8 +8643,8 @@ ${METPLUS_NC_RNG_AZI_FLAG} * - METplus Config(s) - MET Config File - * - :term:`TC_DIAG_NC_RNG_AZI_FLAG` - - nc_rng_azi_flag + * - :term:`TC_DIAG_NC_CYL_GRID_FLAG` + - nc_cyl_grid_flag ${METPLUS_NC_DIAG_FLAG} """"""""""""""""""""""" @@ -8715,8 +8679,8 @@ ${METPLUS_OUTPUT_PREFIX} * - METplus Config(s) - MET Config File - * - :term:`TC_DIAG_OUTPUT_PREFIX` - - output_prefix + * - :term:`TC_DIAG_OUTPUT_BASE_FORMAT` + - output_base_format ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" diff --git a/docs/_static/medium_range-GridStat_fcstGFS_obsOMI_TotalColumnOzone.png b/docs/_static/medium_range-GridStat_fcstGFS_obsOMI_TotalColumnOzone.png new file mode 100644 index 0000000000..965f3e58db Binary files /dev/null and b/docs/_static/medium_range-GridStat_fcstGFS_obsOMI_TotalColumnOzone.png differ diff --git a/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.py b/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.py new file mode 100644 index 0000000000..521f84f209 --- /dev/null +++ b/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.py @@ -0,0 +1,136 @@ +""" +Grid-Stat: Using Python Embedding for Total Column Ozone +================================================================================ + +model_applications/medium_range/GridStat_fcstGFS_obsOMI +_TotalColumnOzone.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# To provide useful statistical information on the relationship between observation +# data in gridded format to a gridded forecast. The Ozone Monitoring Instrument (OMI) +# data covers a 36 hour period and is compared to the average of the gridded forecast +# files (all from the same initialization time). + +############################################################################## +# Datasets +# -------- +# +# | **Forecast:** GFS (1 degree Lat/Lon grid) +# | **Observation:** NASA's Level-3 Aura/OMI Global Total Ozone Mapping Spectrometer-Like (TOMS-Like) Total Column Ozone gridded product OMTO3e (0.25deg Lat/Lon grid) +# | **Location:** Click here for the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. + +############################################################################## +# METplus Components +# ------------------ +# +# This use case first runs PCPCombine on the forecast data to build a +# 36-hour Total Colum Ozone mean/average file. Then the forecast data +# are compared to the observation data using GridStat. This use case utilizes +# the METplus GridStat wrapper to read in the observation file using Python Embedding. + +############################################################################## +# METplus Workflow +# ---------------- +# +# The following tools are used for each run time: +# +# PCPCombine (forecast) > GridStat +# +# It processes the following run times: +# +# | **Valid:** 2023-12-05 06Z (36 hour period covering 2023-12-03 18Z - 2023-12-05 06Z) +# | **Init:** 2023-12-03 06Z +# | **Forecast lead:** 48 + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads all of the configuration files found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line +# parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf + +############################################################################## +# MET Configuration +# ----------------- +# +# METplus sets environment variables based on user settings in the METplus configuration file. +# See :ref:`How METplus controls MET config file settings` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped + +############################################################################## +# Python Embedding +# ---------------- +# +# This use case uses one Python embedding script with GridStat to read the input observation data +# +# parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/read_omi-aura_l3-omto3e.py +# +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/read_omi-aura_l3-omto3e.py + + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. +# + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in /model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone +# (relative to **OUTPUT_BASE**) +# and will contain the following files: +# +# * PCPCombine/pcp_combine.tozone_l0.mean.f048.init2023120306.nc +# * GridStat/grid_stat_GFS_vs_OMI_480000L_20231205_060000V.stat + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * PCPCombineToolUseCase +# * GridStatToolUseCase +# * MediumRangeAppUseCase +# * GRIBFileUseCase +# * PythonEmbeddingFileUseCase +# * RegriddingInToolUseCase +# * NOAAEMCOrgUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# +# +# sphinx_gallery_thumbnail_path = '_static/medium_range-GridStat_fcstGFS_obsOMI_TotalColumnOzone.png' +# diff --git a/internal/scripts/docker_env/README.md b/internal/scripts/docker_env/README.md index a3f70f3b39..6a121cde50 100644 --- a/internal/scripts/docker_env/README.md +++ b/internal/scripts/docker_env/README.md @@ -416,9 +416,6 @@ docker push dtcenter/metplus-envs:geovista.${METPLUS_ENV_VERSION} ``` export METPLUS_ENV_VERSION=v6.0 ./scripts/geovista_env.sh ${METPLUS_ENV_VERSION} - -cd /home/met_test/geovista -/home/met_test/.conda/envs/geovista.${METPLUS_ENV_VERSION}/bin/pip3 install --no-deps --editable . ``` ## pandac.v6.0 (from metplotpy.v6.0) diff --git a/internal/scripts/docker_env/scripts/cfgrib_env.sh b/internal/scripts/docker_env/scripts/cfgrib_env.sh index 123d99d45d..66799f1aef 100755 --- a/internal/scripts/docker_env/scripts/cfgrib_env.sh +++ b/internal/scripts/docker_env/scripts/cfgrib_env.sh @@ -21,7 +21,4 @@ METPLUS_VERSION=$1 ENV_NAME=cfgrib.${METPLUS_VERSION} mamba create -y --name ${ENV_NAME} -c conda-forge python=3.10.4 -mamba install -y --name ${ENV_NAME} -c conda-forge metpy==1.4.0 -mamba install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.6.2 -mamba install -y --name ${ENV_NAME} -c conda-forge cfgrib==0.9.10.1 -mamba install -y --name ${ENV_NAME} -c conda-forge pygrib==2.1.4 +mamba install -y --name ${ENV_NAME} -c conda-forge metpy==1.4.0 netcdf4==1.6.2 cfgrib==0.9.10.1 pygrib==2.1.4 diff --git a/internal/scripts/docker_env/scripts/cycloneplotter_env.sh b/internal/scripts/docker_env/scripts/cycloneplotter_env.sh index ae742ceaec..d7aa43e863 100755 --- a/internal/scripts/docker_env/scripts/cycloneplotter_env.sh +++ b/internal/scripts/docker_env/scripts/cycloneplotter_env.sh @@ -23,6 +23,4 @@ ENV_NAME=cycloneplotter.${METPLUS_VERSION} BASE_ENV=metplus_base.${METPLUS_VERSION} mamba create -y --clone ${BASE_ENV} --name ${ENV_NAME} -mamba install -y --name ${ENV_NAME} -c conda-forge cartopy==0.20.3 -mamba install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.5.2 -mamba install -y --name ${ENV_NAME} -c conda-forge pandas==1.4.3 +mamba install -y --name ${ENV_NAME} -c conda-forge cartopy==0.20.3 matplotlib==3.5.2 pandas==1.4.3 diff --git a/internal/scripts/docker_env/scripts/diff_env.sh b/internal/scripts/docker_env/scripts/diff_env.sh index b5f62c4e2e..40048e841a 100755 --- a/internal/scripts/docker_env/scripts/diff_env.sh +++ b/internal/scripts/docker_env/scripts/diff_env.sh @@ -24,8 +24,7 @@ ENV_NAME=diff.${METPLUS_VERSION} BASE_ENV=netcdf4.${METPLUS_VERSION} mamba create -y --clone ${BASE_ENV} --name ${ENV_NAME} -mamba install -y --name ${ENV_NAME} -c conda-forge pandas==2.0.2 -mamba install -y --name ${ENV_NAME} -c conda-forge pillow==9.2.0 +mamba install -y --name ${ENV_NAME} -c conda-forge pandas==2.0.2 pillow==9.2.0 apt-get update apt-get install -y poppler-utils diff --git a/internal/scripts/docker_env/scripts/geovista_env.sh b/internal/scripts/docker_env/scripts/geovista_env.sh index 4c0a8a9398..9dce88d63d 100755 --- a/internal/scripts/docker_env/scripts/geovista_env.sh +++ b/internal/scripts/docker_env/scripts/geovista_env.sh @@ -25,6 +25,4 @@ apt install -y libgl1-mesa-glx apt install -y libegl1 mamba create -y --name ${ENV_NAME} -c conda-forge python=3.10.4 -mamba install -y --name ${ENV_NAME} -c conda-forge geovista==0.3.0 -mamba install -y --name ${ENV_NAME} -c conda-forge xarray==2023.8.0 -mamba install -y --name ${ENV_NAME} -c conda-forge iris==3.7.0 +mamba install -y --name ${ENV_NAME} -c conda-forge geovista==0.3.0 xarray==2023.8.0 iris==3.7.0 diff --git a/internal/scripts/docker_env/scripts/icecover_env.sh b/internal/scripts/docker_env/scripts/icecover_env.sh index fc5ea361cf..a622b869e0 100755 --- a/internal/scripts/docker_env/scripts/icecover_env.sh +++ b/internal/scripts/docker_env/scripts/icecover_env.sh @@ -23,7 +23,4 @@ ENV_NAME=icecover.${METPLUS_VERSION} BASE_ENV=py_embed_base.${METPLUS_VERSION} mamba create -y --clone ${BASE_ENV} --name ${ENV_NAME} -mamba install -y --name ${ENV_NAME} -c conda-forge xarray==2023.5.0 -mamba install -y --name ${ENV_NAME} -c conda-forge pyresample==1.27.1 -mamba install -y --name ${ENV_NAME} -c conda-forge scikit-learn==1.3.0 -mamba install -y --name ${ENV_NAME} -c conda-forge pyproj==3.6.0 +mamba install -y --name ${ENV_NAME} -c conda-forge xarray==2023.5.0 pyresample==1.27.1 scikit-learn==1.3.0 pyproj==3.6.0 diff --git a/internal/scripts/docker_env/scripts/metdataio_env.sh b/internal/scripts/docker_env/scripts/metdataio_env.sh index a1e2c291c8..a0eac3a869 100755 --- a/internal/scripts/docker_env/scripts/metdataio_env.sh +++ b/internal/scripts/docker_env/scripts/metdataio_env.sh @@ -23,6 +23,4 @@ BASE_ENV=metplus_base.${METPLUS_VERSION} mamba create -y --clone ${BASE_ENV} --name ${ENV_NAME} -mamba install -y --name ${ENV_NAME} -c conda-forge lxml==4.9.1 -mamba install -y --name ${ENV_NAME} -c conda-forge pymysql==1.0.2 -mamba install -y --name ${ENV_NAME} -c conda-forge pandas==1.5.1 +mamba install -y --name ${ENV_NAME} -c conda-forge lxml==4.9.1 pymysql==1.0.2 pandas==1.5.1 diff --git a/internal/scripts/docker_env/scripts/metplotpy_env.sh b/internal/scripts/docker_env/scripts/metplotpy_env.sh index c571404b65..6a55e6238e 100755 --- a/internal/scripts/docker_env/scripts/metplotpy_env.sh +++ b/internal/scripts/docker_env/scripts/metplotpy_env.sh @@ -33,17 +33,4 @@ BASE_ENV=metplus_base.${METPLUS_VERSION} mamba create -y --clone ${BASE_ENV} --name ${ENV_NAME} -mamba install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.6.3 -mamba install -y --name ${ENV_NAME} -c conda-forge scipy==1.9.3 -mamba install -y --name ${ENV_NAME} -c conda-forge plotly==5.13.0 -mamba install -y --name ${ENV_NAME} -c conda-forge xarray==2023.1.0 -mamba install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.6.2 -mamba install -y --name ${ENV_NAME} -c conda-forge pyyaml==6.0 -#mamba install -y --name ${ENV_NAME} -c conda-forge statsmodels #==0.13.2 -mamba install -y --name ${ENV_NAME} -c conda-forge python-kaleido==0.2.1 -mamba install -y --name ${ENV_NAME} -c conda-forge imageio==2.25.0 -mamba install -y --name ${ENV_NAME} -c conda-forge imutils==0.5.4 -mamba install -y --name ${ENV_NAME} -c conda-forge scikit-image -mamba install -y --name ${ENV_NAME} -c conda-forge pint==0.20.1 -mamba install -y --name ${ENV_NAME} -c conda-forge metpy -mamba install -y --name ${ENV_NAME} -c conda-forge cartopy==0.21.1 +mamba install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.6.3 scipy==1.9.3 plotly==5.13.0 xarray==2023.1.0 netcdf4==1.6.2 pyyaml==6.0 python-kaleido==0.2.1 imageio==2.25.0 imutils==0.5.4 scikit-image pint==0.20.1 metpy cartopy==0.21.1 diff --git a/internal/scripts/docker_env/scripts/pandac_env.sh b/internal/scripts/docker_env/scripts/pandac_env.sh index 05474b936e..571824f19a 100755 --- a/internal/scripts/docker_env/scripts/pandac_env.sh +++ b/internal/scripts/docker_env/scripts/pandac_env.sh @@ -24,20 +24,8 @@ BASE_ENV=metplotpy.${METPLUS_VERSION} mamba create -y --clone ${BASE_ENV} --name ${ENV_NAME} - -#mamba install -y --name ${ENV_NAME} -c conda-forge scipy==1.10.1 -#if [ $? != 0 ]; then -# echo install of scipy==1.10.1 failed -# exit 1 -#fi - mamba install -y --name ${ENV_NAME} -c conda-forge pygrib==2.1.4 if [ $? != 0 ]; then echo install of pygrib==2.1.4 failed exit 1 fi -#mamba install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.6.3 -#if [ $? != 0 ]; then -# echo install of matplotlib==3.6.3 failed -# exit 1 -#fi diff --git a/internal/scripts/docker_env/scripts/py_embed_base_env.sh b/internal/scripts/docker_env/scripts/py_embed_base_env.sh index 1a59cd3b2d..bd28945ea7 100755 --- a/internal/scripts/docker_env/scripts/py_embed_base_env.sh +++ b/internal/scripts/docker_env/scripts/py_embed_base_env.sh @@ -19,5 +19,4 @@ METPLUS_VERSION=$1 ENV_NAME=py_embed_base.${METPLUS_VERSION} mamba create -y --name ${ENV_NAME} -c conda-forge python=3.10.4 -mamba install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0 -mamba install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.6.2 +mamba install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0 netcdf4==1.6.2 yaml==0.2.5 diff --git a/internal/scripts/docker_env/scripts/pygrib_env.sh b/internal/scripts/docker_env/scripts/pygrib_env.sh index 061ff2c951..ab2754ca01 100755 --- a/internal/scripts/docker_env/scripts/pygrib_env.sh +++ b/internal/scripts/docker_env/scripts/pygrib_env.sh @@ -23,5 +23,4 @@ BASE_ENV=py_embed_base.${METPLUS_VERSION} mamba create -y --clone ${BASE_ENV} --name ${ENV_NAME} -mamba install -y --name ${ENV_NAME} -c conda-forge pygrib==2.1.4 -mamba install -y --name ${ENV_NAME} -c conda-forge metpy==1.3.0 +mamba install -y --name ${ENV_NAME} -c conda-forge pygrib==2.1.4 metpy==1.3.0 diff --git a/internal/scripts/docker_env/scripts/spacetime_env.sh b/internal/scripts/docker_env/scripts/spacetime_env.sh index d56c2c3221..36e250496a 100755 --- a/internal/scripts/docker_env/scripts/spacetime_env.sh +++ b/internal/scripts/docker_env/scripts/spacetime_env.sh @@ -23,9 +23,4 @@ ENV_NAME=spacetime.${METPLUS_VERSION} mamba create -y --name ${ENV_NAME} -c conda-forge python=3.10.4 -mamba install -y --name ${ENV_NAME} -c conda-forge netCDF4==1.6.4 -mamba install -y --name ${ENV_NAME} -c conda-forge xarray==2023.8.0 -mamba install -y --name ${ENV_NAME} -c conda-forge scipy==1.11.2 -mamba install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.7.2 -mamba install -y --name ${ENV_NAME} -c conda-forge pyngl==1.6.1 -mamba install -y --name ${ENV_NAME} -c conda-forge pyyaml==6.0.1 +mamba install -y --name ${ENV_NAME} -c conda-forge netCDF4==1.6.4 xarray==2023.8.0 scipy==1.11.2 matplotlib==3.7.2 pyngl==1.6.1 pyyaml==6.0.1 diff --git a/internal/scripts/docker_env/scripts/weatherregime_env.sh b/internal/scripts/docker_env/scripts/weatherregime_env.sh index e00b92e411..74932dc5e9 100755 --- a/internal/scripts/docker_env/scripts/weatherregime_env.sh +++ b/internal/scripts/docker_env/scripts/weatherregime_env.sh @@ -25,6 +25,4 @@ BASE_ENV=metplotpy.${METPLUS_VERSION} mamba create -y --clone ${BASE_ENV} --name ${ENV_NAME} -mamba install -y --name ${ENV_NAME} -c conda-forge scikit-learn==1.1.1 -mamba install -y --name ${ENV_NAME} -c conda-forge eofs==1.4.0 -mamba install -y --name ${ENV_NAME} -c conda-forge cmocean==2.0 +mamba install -y --name ${ENV_NAME} -c conda-forge scikit-learn==1.1.1 eofs==1.4.0 cmocean==2.0 diff --git a/internal/scripts/docker_env/scripts/xesmf_env.sh b/internal/scripts/docker_env/scripts/xesmf_env.sh index 779e0c36ff..3a7965b264 100755 --- a/internal/scripts/docker_env/scripts/xesmf_env.sh +++ b/internal/scripts/docker_env/scripts/xesmf_env.sh @@ -26,8 +26,4 @@ ENV_NAME=xesmf.${METPLUS_VERSION} BASE_ENV=metplus_base.${METPLUS_VERSION} mamba create -y --clone ${BASE_ENV} --name ${ENV_NAME} -mamba install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.6.2 -mamba install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0 -mamba install -y --name ${ENV_NAME} -c conda-forge xesmf==0.8.2 -mamba install -y --name ${ENV_NAME} -c conda-forge esmf==8.3.1 - +mamba install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.6.2 xarray==2022.3.0 xesmf==0.8.2 esmf==8.3.1 diff --git a/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py b/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py index ec3458c8ea..2a0954cb2b 100644 --- a/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py @@ -101,35 +101,31 @@ def set_minimum_config_settings(config): {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{delta_range_km = 10.0;}];'}), ({'TC_DIAG_DOMAIN_INFO1_DIAG_SCRIPT': 'MET_BASE/python/tc_diag/compute_all_diagnostics.py,MET_BASE/python/tc_diag/compute_custom_diagnostics.py', }, {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{diag_script = ["MET_BASE/python/tc_diag/compute_all_diagnostics.py", "MET_BASE/python/tc_diag/compute_custom_diagnostics.py"];}];'}), + ({'TC_DIAG_DOMAIN_INFO1_OVERRIDE_DIAGS': 'RMW,SST', }, + {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{override_diags = ["RMW", "SST"];}];'}), ({'TC_DIAG_DOMAIN_INFO1_DOMAIN': 'parent', 'TC_DIAG_DOMAIN_INFO1_N_RANGE': '150', 'TC_DIAG_DOMAIN_INFO1_N_AZIMUTH': '8', 'TC_DIAG_DOMAIN_INFO1_DELTA_RANGE_KM': '10.0', 'TC_DIAG_DOMAIN_INFO1_DIAG_SCRIPT': 'MET_BASE/python/tc_diag/compute_all_diagnostics.py,MET_BASE/python/tc_diag/compute_custom_diagnostics.py', + 'TC_DIAG_DOMAIN_INFO1_OVERRIDE_DIAGS': 'RMW,SST', }, - {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{domain = "parent";n_range = 150;n_azimuth = 8;delta_range_km = 10.0;diag_script = ["MET_BASE/python/tc_diag/compute_all_diagnostics.py", "MET_BASE/python/tc_diag/compute_custom_diagnostics.py"];}];'}), + {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{domain = "parent";n_range = 150;n_azimuth = 8;delta_range_km = 10.0;diag_script = ["MET_BASE/python/tc_diag/compute_all_diagnostics.py", "MET_BASE/python/tc_diag/compute_custom_diagnostics.py"];override_diags = ["RMW", "SST"];}];'}), # domain_info 2 dictionaries in list ({'TC_DIAG_DOMAIN_INFO1_DOMAIN': 'parent', 'TC_DIAG_DOMAIN_INFO1_N_RANGE': '150', 'TC_DIAG_DOMAIN_INFO1_N_AZIMUTH': '8', 'TC_DIAG_DOMAIN_INFO1_DELTA_RANGE_KM': '10.0', 'TC_DIAG_DOMAIN_INFO1_DIAG_SCRIPT': 'MET_BASE/python/tc_diag/compute_all_diagnostics.py,MET_BASE/python/tc_diag/compute_custom_diagnostics.py', + 'TC_DIAG_DOMAIN_INFO1_OVERRIDE_DIAGS': 'RMW,SST', 'TC_DIAG_DOMAIN_INFO2_DOMAIN': 'nest', 'TC_DIAG_DOMAIN_INFO2_N_RANGE': '100', 'TC_DIAG_DOMAIN_INFO2_N_AZIMUTH': '7', 'TC_DIAG_DOMAIN_INFO2_DELTA_RANGE_KM': '12.0', 'TC_DIAG_DOMAIN_INFO2_DIAG_SCRIPT': 'MET_BASE/python/tc_diag/compute_sst_diagnostics.py', + 'TC_DIAG_DOMAIN_INFO2_OVERRIDE_DIAGS': 'SST', }, - {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{domain = "parent";n_range = 150;n_azimuth = 8;delta_range_km = 10.0;diag_script = ["MET_BASE/python/tc_diag/compute_all_diagnostics.py", "MET_BASE/python/tc_diag/compute_custom_diagnostics.py"];},{domain = "nest";n_range = 100;n_azimuth = 7;delta_range_km = 12.0;diag_script = ["MET_BASE/python/tc_diag/compute_sst_diagnostics.py"];}];'}), - - ({'TC_DIAG_CENSOR_THRESH': '>12000,<5000', }, - {'METPLUS_CENSOR_THRESH': 'censor_thresh = [>12000, <5000];'}), - - ({'TC_DIAG_CENSOR_VAL': '12000,5000', }, - {'METPLUS_CENSOR_VAL': 'censor_val = [12000, 5000];'}), - - ({'TC_DIAG_CONVERT': '2*x', }, - {'METPLUS_CONVERT': 'convert(x) = 2*x;'}), + {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{domain = "parent";n_range = 150;n_azimuth = 8;delta_range_km = 10.0;diag_script = ["MET_BASE/python/tc_diag/compute_all_diagnostics.py", "MET_BASE/python/tc_diag/compute_custom_diagnostics.py"];override_diags = ["RMW", "SST"];},{domain = "nest";n_range = 100;n_azimuth = 7;delta_range_km = 12.0;diag_script = ["MET_BASE/python/tc_diag/compute_sst_diagnostics.py"];override_diags = ["SST"];}];'}), ({'TC_DIAG_DATA_DOMAIN': 'parent,nest', }, {'METPLUS_DATA_DOMAIN': 'domain = ["parent", "nest"];'}), @@ -197,13 +193,13 @@ def set_minimum_config_settings(config): ({'TC_DIAG_VORTEX_REMOVAL': 'False', }, {'METPLUS_VORTEX_REMOVAL': 'vortex_removal = FALSE;'}), - ({'TC_DIAG_NC_RNG_AZI_FLAG': 'true', }, {'METPLUS_NC_RNG_AZI_FLAG': 'nc_rng_azi_flag = TRUE;'}), + ({'TC_DIAG_NC_CYL_GRID_FLAG': 'true', }, {'METPLUS_NC_CYL_GRID_FLAG': 'nc_cyl_grid_flag = TRUE;'}), ({'TC_DIAG_NC_DIAG_FLAG': 'true', }, {'METPLUS_NC_DIAG_FLAG': 'nc_diag_flag = TRUE;'}), ({'TC_DIAG_CIRA_DIAG_FLAG': 'True', }, {'METPLUS_CIRA_DIAG_FLAG': 'cira_diag_flag = TRUE;'}), - ({'TC_DIAG_OUTPUT_PREFIX': 'my_prefix', }, {'METPLUS_OUTPUT_PREFIX': 'output_prefix = "my_prefix";'}), + ({'TC_DIAG_OUTPUT_BASE_FORMAT': 's{storm_id}_{technique}_doper_{init_ymdh}', }, {'METPLUS_OUTPUT_BASE_FORMAT': 'output_base_format = "s{storm_id}_{technique}_doper_{init_ymdh}";'}), ({'TC_DIAG_ONE_TIME_PER_FILE_FLAG': 'false', }, {'METPLUS_ONE_TIME_PER_FILE_FLAG': 'one_time_per_file_flag = FALSE;'}), ] diff --git a/internal/tests/use_cases/all_use_cases.txt b/internal/tests/use_cases/all_use_cases.txt index 235c910c71..68f29e3873 100644 --- a/internal/tests/use_cases/all_use_cases.txt +++ b/internal/tests/use_cases/all_use_cases.txt @@ -115,6 +115,7 @@ Category: medium_range 7::TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics:: model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf:: cfgrib_env,py_embed 8::MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead::model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf:: netcdf4_env 9::GridStat_fcstGEFS_obsCADB_BinaryObsPOE:: model_applications/medium_range/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.conf:: netcdf4_env +10::GridStat_fcstGFS_obsOMI_TotalColumnOzone:: model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf:: py_embed Category: pbl diff --git a/metplus/wrappers/tc_diag_wrapper.py b/metplus/wrappers/tc_diag_wrapper.py index e72f0efffa..554151a0f9 100755 --- a/metplus/wrappers/tc_diag_wrapper.py +++ b/metplus/wrappers/tc_diag_wrapper.py @@ -43,9 +43,6 @@ class TCDiagWrapper(RuntimeFreqWrapper): 'METPLUS_LEAD_LIST', 'METPLUS_DIAG_SCRIPT', 'METPLUS_DOMAIN_INFO_LIST', - 'METPLUS_CENSOR_THRESH', - 'METPLUS_CENSOR_VAL', - 'METPLUS_CONVERT', 'METPLUS_DATA_FILE_TYPE', 'METPLUS_DATA_DOMAIN', 'METPLUS_DATA_LEVEL', @@ -61,15 +58,19 @@ class TCDiagWrapper(RuntimeFreqWrapper): 'METPLUS_VORTEX_REMOVAL', 'METPLUS_VORTEX_REMOVAL', 'METPLUS_NC_DIAG_FLAG', - 'METPLUS_NC_RNG_AZI_FLAG', + 'METPLUS_NC_CYL_GRID_FLAG', 'METPLUS_CIRA_DIAG_FLAG', - 'METPLUS_OUTPUT_PREFIX', + 'METPLUS_OUTPUT_BASE_FORMAT', 'METPLUS_ONE_TIME_PER_FILE_FLAG', ] # deprecated env vars that are no longer supported in the wrapped MET conf DEPRECATED_WRAPPER_ENV_VAR_KEYS = [ 'OUTPUT_PREFIX', + 'METPLUS_OUTPUT_PREFIX', + 'METPLUS_CENSOR_THRESH', + 'METPLUS_CENSOR_VAL', + 'METPLUS_CONVERT', ] def __init__(self, config, instance=None): @@ -168,6 +169,7 @@ def create_c_dict(self): 'n_azimuth': 'int', 'delta_range_km': 'float', 'diag_script': 'list', + 'override_diags': ('list', 'allow_empty'), } if not add_met_config_dict_list(config=self.config, app_name=self.app_name, @@ -176,19 +178,6 @@ def create_c_dict(self): dict_items=dict_items): self.isOK = False - self.add_met_config(name='censor_thresh', - data_type='list', - extra_args={'remove_quotes': True}) - - self.add_met_config(name='censor_val', - data_type='list', - extra_args={'remove_quotes': True}) - - self.add_met_config(name='convert', - data_type='string', - extra_args={'remove_quotes': True, - 'add_x': True}) - # handle data dictionary, including field, domain, level, and file_type c_dict['VAR_LIST_TEMP'] = parse_var_list(self.config, data_type='FCST', @@ -228,11 +217,11 @@ def create_c_dict(self): self.add_met_config(name='one_time_per_file_flag', data_type='bool') - self.add_met_config(name='nc_rng_azi_flag', data_type='bool') + self.add_met_config(name='nc_cyl_grid_flag', data_type='bool') self.add_met_config(name='nc_diag_flag', data_type='bool') self.add_met_config(name='cira_diag_flag', data_type='bool') - self.add_met_config(name='output_prefix', data_type='string') + self.add_met_config(name='output_base_format', data_type='string') return c_dict diff --git a/parm/met_config/TCDiagConfig_wrapped b/parm/met_config/TCDiagConfig_wrapped index d081f2430d..8dacb81b26 100644 --- a/parm/met_config/TCDiagConfig_wrapped +++ b/parm/met_config/TCDiagConfig_wrapped @@ -88,21 +88,17 @@ ${METPLUS_DOMAIN_INFO_LIST} //////////////////////////////////////////////////////////////////////////////// // -// Data censoring and conversion +// Cylindrical coordinate transformation regridding options // May be set separately in each diag_data "field" entry // -// censor_thresh = []; -${METPLUS_CENSOR_THRESH} - -// censor_val = []; -${METPLUS_CENSOR_VAL} - -// convert(x) = x; -${METPLUS_CONVERT} +//regrid = { +${METPLUS_REGRID_DICT} // // Data fields +// censor_thresh, censor_val, and convert(x) may be set separately +// in each data "field" array entry // data = { @@ -122,12 +118,6 @@ data = { //////////////////////////////////////////////////////////////////////////////// -// -// Regridding options -// -//regrid = { -${METPLUS_REGRID_DICT} - // // Optionally convert u/v winds to tangential/radial winds // @@ -168,8 +158,8 @@ ${METPLUS_ONE_TIME_PER_FILE_FLAG} // // Flags to control output files // -//nc_rng_azi_flag = -${METPLUS_NC_RNG_AZI_FLAG} +//nc_cyl_grid_flag = +${METPLUS_NC_CYL_GRID_FLAG} //nc_diag_flag = ${METPLUS_NC_DIAG_FLAG} @@ -177,14 +167,15 @@ ${METPLUS_NC_DIAG_FLAG} //cira_diag_flag = ${METPLUS_CIRA_DIAG_FLAG} +// +// Output base file naming convention +// +//output_base_format = +${METPLUS_OUTPUT_BASE_FORMAT} //////////////////////////////////////////////////////////////////////////////// tmp_dir = "${MET_TMP_DIR}"; - -//output_prefix = -${METPLUS_OUTPUT_PREFIX} - //version = "V11.1.0"; //////////////////////////////////////////////////////////////////////////////// diff --git a/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf b/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf index 385148ee0b..37f191c712 100644 --- a/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf +++ b/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf @@ -45,12 +45,6 @@ TC_DIAG_INPUT1_TEMPLATE = subset.gfs.t12z.pgrb2.0p50.f* TC_DIAG_INPUT1_DOMAIN = parent TC_DIAG_INPUT1_TECH_ID_LIST = AVNO - -TC_DIAG_INPUT2_DIR = {INPUT_BASE}/met_test/new/model_data/grib2/gfs -TC_DIAG_INPUT2_TEMPLATE = subset.gfs.t12z.pgrb2.0p50.f* -TC_DIAG_INPUT2_DOMAIN = nest -TC_DIAG_INPUT2_TECH_ID_LIST = AVNO - TC_DIAG_OUTPUT_DIR = {OUTPUT_BASE}/tc_diag TC_DIAG_OUTPUT_TEMPLATE = {date?fmt=%Y} @@ -96,17 +90,8 @@ TC_DIAG_DOMAIN_INFO1_DOMAIN = parent TC_DIAG_DOMAIN_INFO1_N_RANGE = 150 TC_DIAG_DOMAIN_INFO1_N_AZIMUTH = 8 TC_DIAG_DOMAIN_INFO1_DELTA_RANGE_KM = 10.0 -#TC_DIAG_DOMAIN_INFO1_DIAG_SCRIPT = - -TC_DIAG_DOMAIN_INFO2_DOMAIN = nest -TC_DIAG_DOMAIN_INFO2_N_RANGE = 150 -TC_DIAG_DOMAIN_INFO2_N_AZIMUTH = 8 -TC_DIAG_DOMAIN_INFO2_DELTA_RANGE_KM = 2.0 - - -#TC_DIAG_CENSOR_THRESH = -#TC_DIAG_CENSOR_VAL = -#TC_DIAG_CONVERT = +TC_DIAG_DOMAIN_INFO1_DIAG_SCRIPT = MET_BASE/python/tc_diag/compute_tc_diag.py MET_BASE/python/tc_diag/config/post_resample.yml MET_BASE/tc_data/v2023-04-07_gdland_table.dat +TC_DIAG_DOMAIN_INFO1_OVERRIDE_DIAGS = #TC_DIAG_DATA_DOMAIN = #TC_DIAG_DATA_LEVEL = @@ -128,9 +113,9 @@ TC_DIAG_DOMAIN_INFO2_DELTA_RANGE_KM = 2.0 #TC_DIAG_RADIAL_VELOCITY_LONG_FIELD_NAME = #TC_DIAG_VORTEX_REMOVAL = #TC_DIAG_ONE_TIME_PER_FILE_FLAG = -#TC_DIAG_NC_RNG_AZI_FLAG = +#TC_DIAG_NC_CYL_GRID_FLAG = #TC_DIAG_NC_DIAG_FLAG = #TC_DIAG_CIRA_DIAG_FLAG = -#TC_DIAG_OUTPUT_PREFIX = +#TC_DIAG_OUTPUT_BASE_FORMAT = #LOG_LEVEL=DEBUG \ No newline at end of file diff --git a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf new file mode 100644 index 0000000000..02c1e5b1b8 --- /dev/null +++ b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf @@ -0,0 +1,106 @@ +[config] + +# Documentation for this use case can be found at +# https://metplus.readthedocs.io/en/latest/generated/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.html + +# For additional information, please see the METplus Users Guide. +# https://metplus.readthedocs.io/en/latest/Users_Guide + +### +# Processes to run +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#process-list +### + +PROCESS_LIST = PCPCombine, GridStat + +### +# Time Info +# LOOP_BY options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +# LEAD_SEQ is the list of forecast leads to process +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#timing-control +### + +LOOP_BY = VALID +VALID_TIME_FMT = %Y%m%d%H +VALID_BEG = 2023120506 +VALID_END = 2023120506 +VALID_INCREMENT = 21600 + +LEAD_SEQ = 48 + +### +# File I/O +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info +### + +# PCP_COMBINE +FCST_PCP_COMBINE_RUN = True +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE}/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/fcst +FCST_PCP_COMBINE_INPUT_TEMPLATE = tozne.pgbf{lead?fmt=%HH}.gfs.{init?fmt=%Y%m%d%H} +FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/PCPCombine +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = pcp_combine.tozone_l0.mean.f{lead?fmt=%HHH}.init{init?fmt=%Y%m%d%H}.nc + +# GRID_STAT +FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} +FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} +OBS_GRID_STAT_INPUT_TEMPLATE = PYTHON_NUMPY +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/GridStat +GRID_STAT_OUTPUT_TEMPLATE = + +### +# Field Info +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#field-info +### + +CONFIG_DIR = {PARM_BASE}/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone + +# GRID_STAT +GRID_STAT_ONCE_PER_FIELD = False +FCST_VAR1_NAME = TOZNE_L0_mean +FCST_VAR1_LEVELS = (*,*) +OBS_VAR1_NAME = {CONFIG_DIR}/read_omi-aura_l3-omto3e.py {INPUT_BASE}/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/obs/OMI-Aura_L3-OMTO3e_{valid?fmt=%Y?shift=-86400}m{valid?fmt=%m%d?shift=-86400}_v883-{valid?fmt=%Y}m{valid?fmt=%m%d}t010905.he5 obs + +### +# PCPCombine Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pcpcombine +### +FCST_PCP_COMBINE_METHOD = DERIVE +FCST_PCP_COMBINE_STAT_LIST = mean +FCST_PCP_COMBINE_DERIVE_LOOKBACK = 42H +FCST_PCP_COMBINE_MIN_FORECAST = 12 +FCST_PCP_COMBINE_MAX_FORECAST = 48 +FCST_IS_PROB = false +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB +FCST_PCP_COMBINE_INPUT_ACCUMS = 6H +FCST_PCP_COMBINE_INPUT_NAMES = TOZNE +FCST_PCP_COMBINE_INPUT_LEVELS = L0 +FCST_PCP_COMBINE_INPUT_OPTIONS = GRIB_lvl_typ = 200; +FCST_PCP_COMBINE_OUTPUT_ACCUM = 36H +FCST_PCP_COMBINE_OUTPUT_NAME = + +### +# GridStat Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#gridstat +### + +MODEL = GFS +OBTYPE = OMI +GRID_STAT_OUTPUT_PREFIX = {MODEL}_vs_{OBTYPE} +GRID_STAT_DESC = NA +GRID_STAT_REGRID_TO_GRID = G004 +GRID_STAT_REGRID_METHOD = BILIN +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_MASK_POLY = +GRID_STAT_CLIMO_CDF_WRITE_BINS = False +GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT +GRID_STAT_OUTPUT_FLAG_CNT = STAT +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE +GRID_STAT_GRID_WEIGHT_FLAG = COS_LAT diff --git a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/read_omi-aura_l3-omto3e.py b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/read_omi-aura_l3-omto3e.py new file mode 100644 index 0000000000..38b963e672 --- /dev/null +++ b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/read_omi-aura_l3-omto3e.py @@ -0,0 +1,92 @@ +""" +Mallory Row - SAIC at NOAA/NWS/NCEP/EMC +Designed to read in NASA OMI Total Column Ozone +""" + +import sys +import os +import numpy as np +import netCDF4 as netcdf +import datetime + +print("Python Script:\t" + repr(sys.argv[0])) + +# Process script arguements +if len(sys.argv) != 3: + print("Must specify the following elements: omi_file file_flag") + sys.exit(1) +omi_file = os.path.expandvars(sys.argv[1]) +if not os.path.exists(omi_file): + print(f"OMI file {omi_file} does not exist, exit") + sys.exit(1) +file_flag = sys.argv[2] +if file_flag not in ['fcst', 'obs']: + print(f"File flag {file_flag} not valid (fcst, obs), exit") + sys.exit(1) + +print(f"Processing {omi_file} as {file_flag} data") + +# Read in OMI data +omi_data = netcdf.Dataset(omi_file) +omi_StartUTC = omi_data['/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES'].StartUTC +omi_StartUTC_dt = datetime.datetime.strptime(omi_StartUTC.split(':')[0], + '%Y-%m-%dT%H') +omi_EndUTC = omi_data['/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES'].EndUTC +omi_EndUTC_dt = datetime.datetime.strptime(omi_EndUTC.split(':')[0], + '%Y-%m-%dT%H') +omi_delta_lat = float(eval( + omi_data['/HDFEOS/GRIDS/OMI Column Amount O3'].GridSpacing +)[0]) +omi_delta_lon = float(eval( + omi_data['/HDFEOS/GRIDS/OMI Column Amount O3'].GridSpacing +)[1]) +omi_ColumnAmountO3 = omi_data[ + '/HDFEOS/GRIDS/OMI Column Amount O3/Data Fields/ColumnAmountO3' +] +omi_lat_ll = float(eval( + omi_data['/HDFEOS/GRIDS/OMI Column Amount O3'].GridSpan +)[2]) +omi_lon_ll = float(eval( + omi_data['/HDFEOS/GRIDS/OMI Column Amount O3'].GridSpan +)[0]) +omi_nlat = int( + omi_data['/HDFEOS/GRIDS/OMI Column Amount O3'].NumberOfLatitudesInGrid +) +omi_nlon = int( + omi_data['/HDFEOS/GRIDS/OMI Column Amount O3'].NumberOfLongitudesInGrid +) +omi_ColumnAmountO3_vals = omi_ColumnAmountO3[:] +omi_ColumnAmountO3_Units = omi_ColumnAmountO3.Units + +# There is no geolocation data, so construct it ourselves. +latitude = np.arange(0., omi_nlat) * omi_delta_lat + omi_lat_ll + 0.125 +longitude = np.arange(0., omi_nlon) * omi_delta_lon + omi_lon_ll + 0.125 + +# Set data up for MET +met_data = omi_ColumnAmountO3_vals.copy() +omi_MidPointUTC_dt = omi_StartUTC_dt + ((omi_EndUTC_dt - omi_StartUTC_dt)/2) +print(f"Data runs from {omi_StartUTC_dt:%Y%m%d_%H%M%S} to " + +f"{omi_EndUTC_dt:%Y%m%d_%H%M%S}...setting valid date as " + +f"{omi_EndUTC_dt:%Y%m%d_%H%M%S}") +met_data.attrs = { + 'valid': f"{omi_EndUTC_dt:%Y%m%d_%H%M%S}", + 'init': f"{omi_EndUTC_dt:%Y%m%d_%H%M%S}", + 'lead': '00', + 'accum': '00', + 'name': 'ColumnAmountO3', + 'standard_name': 'total_column_ozone', + 'long_name': 'total_column_ozone', + 'level': 'TotalColumn', + 'units': omi_ColumnAmountO3_Units, + 'grid': { + 'type': 'LatLon', + 'name': 'OMI Grid', + 'lat_ll': omi_lat_ll, + 'lon_ll': omi_lon_ll, + 'delta_lat': omi_delta_lat, + 'delta_lon': omi_delta_lon, + 'Nlat': omi_nlat, + 'Nlon': omi_nlon, + } +} +attrs = met_data.attrs