From 45c3d2f1c8a75315bbfa5e39621c1b603bfecf49 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 22 Jun 2022 14:11:02 -0600 Subject: [PATCH 1/4] feature 1653 develop climo day_interval NA value (#1671) * moved logic to handle climo_mean/stdev dictionaries into met_config utility * moved constant variable to constants and made name uppercase * per #1599, added logic to read VAR variables for climo_mean/stdev (i.e. GRID_STAT_CLIMO_MEAN_VAR1_NAME). Added function to easily format field info for a single field or list of fields with fewer arguments * added function to format field info that is easier to call than get_field_info and can eventually replace it * added line breaks to match pep8 standards, ci-run-all-diff * per 1599, added documentation for new config variables * replaced calls to get_field_info with format_field_info since it is much simpler * fixed bug where field list is never returned from function call * removed whitespace * modified logic to report an error if no commands were run at all * only report an error if the process list contains a wrapper that should generate at least 1 command. Example and CyclonePlotter wrappers do not run any shell commands, so they are excluded * per feedback in PR #1658 review, added glossary entry that is wrapper independent and provide reference to that entry instead of duplicating information for each wrapper * per #1653, add unit tests to check that day_interval value of climo_mean/stdev dictionary can be NA or an integer * per #1653, modified function to read climo variables to support NA instead of just integers * per #1653, updated unit tests to test both climo_mean and climo_stdev variables, added tests for all other climo variables --- .../grid_stat/test_grid_stat_wrapper.py | 3 + .../pytests/met_config/test_met_config.py | 140 +++++++++++++++--- metplus/util/met_config.py | 2 +- 3 files changed, 125 insertions(+), 20 deletions(-) diff --git a/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py b/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py index 34385c23c2..e29aeaae5a 100644 --- a/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py +++ b/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py @@ -516,6 +516,9 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, ({'GRID_STAT_CLIMO_MEAN_DAY_INTERVAL': '30', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {day_interval = 30;}'}), + ({'GRID_STAT_CLIMO_MEAN_DAY_INTERVAL': 'NA', }, + {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {day_interval = NA;}'}), + ({'GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL': '12', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {hour_interval = 12;}'}), diff --git a/internal_tests/pytests/met_config/test_met_config.py b/internal_tests/pytests/met_config/test_met_config.py index c67cf09e0b..0f990e678b 100644 --- a/internal_tests/pytests/met_config/test_met_config.py +++ b/internal_tests/pytests/met_config/test_met_config.py @@ -11,37 +11,139 @@ # 0 no relevant config set ({}, ''), # 1 _FIELD set - ({'GRID_STAT_CLIMO_MEAN_FIELD': '{name="TMP"; level="(*,*)";}'}, + ({'APP_CLIMO__FIELD': '{name="TMP"; level="(*,*)";}'}, '{name="TMP"; level="(*,*)";}'), # 2 VAR1 name/level set - ({'GRID_STAT_CLIMO_MEAN_VAR1_NAME': 'TMP', - 'GRID_STAT_CLIMO_MEAN_VAR1_LEVELS': '"(*,*)"'}, + ({'APP_CLIMO__VAR1_NAME': 'TMP', + 'APP_CLIMO__VAR1_LEVELS': '"(*,*)"'}, '{ name="TMP"; level="(*,*)"; }'), # 3 VAR1/2 name/level set - ({'GRID_STAT_CLIMO_MEAN_VAR1_NAME': 'TMP', - 'GRID_STAT_CLIMO_MEAN_VAR1_LEVELS': '"(*,*)"', - 'GRID_STAT_CLIMO_MEAN_VAR2_NAME': 'PRES', - 'GRID_STAT_CLIMO_MEAN_VAR2_LEVELS': '"(0,*,*)"'}, + ({'APP_CLIMO__VAR1_NAME': 'TMP', + 'APP_CLIMO__VAR1_LEVELS': '"(*,*)"', + 'APP_CLIMO__VAR2_NAME': 'PRES', + 'APP_CLIMO__VAR2_LEVELS': '"(0,*,*)"'}, '{ name="TMP"; level="(*,*)"; },{ name="PRES"; level="(0,*,*)"; }'), # 4 VAR1 name/level and FIELD set - prefer VAR - ({'GRID_STAT_CLIMO_MEAN_FIELD': '{name="TEMP"; level="(0,*,*)";}', - 'GRID_STAT_CLIMO_MEAN_VAR1_NAME': 'TMP', - 'GRID_STAT_CLIMO_MEAN_VAR1_LEVELS': '"(*,*)"'}, + ({'APP_CLIMO__FIELD': '{name="TEMP"; level="(0,*,*)";}', + 'APP_CLIMO__VAR1_NAME': 'TMP', + 'APP_CLIMO__VAR1_LEVELS': '"(*,*)"'}, '{ name="TMP"; level="(*,*)"; }'), ] ) def test_read_climo_field(metplus_config, config_overrides, expected_value): - app_name = 'grid_stat' - climo_type = 'MEAN' - expected_var = f'{app_name}_CLIMO_{climo_type}_FIELD'.upper() - config = metplus_config() + app_name = 'app' + for climo_type in ('MEAN', 'STDEV'): + expected_var = f'{app_name}_CLIMO_{climo_type}_FIELD'.upper() + config = metplus_config() + + # set config values + for key, value in config_overrides.items(): + key_sub = key.replace('', climo_type) + value_sub = value.replace('', climo_type.lower()) + config.set('config', key_sub, value_sub) - # set config values - for key, value in config_overrides.items(): - config.set('config', key, value) + _read_climo_field(climo_type, config, app_name) + assert config.getraw('config', expected_var) == expected_value - _read_climo_field(climo_type, config, app_name) - assert config.getraw('config', expected_var) == expected_value +@pytest.mark.parametrize( + 'config_overrides, expected_value', [ + # 0 no relevant config set + ({}, ''), + # 1 file name single + ({'APP_CLIMO__FILE_NAME': 'some/file/path'}, + 'climo_ = {file_name = ["some/file/path"];}'), + # 2 file name multiple + ({'APP_CLIMO__FILE_NAME': 'some/file/path, other/path'}, + 'climo_ = {file_name = ["some/file/path", "other/path"];}'), + # 3 field single + ({'APP_CLIMO__FIELD': '{name="TMP"; level="(*,*)";}'}, + 'climo_ = {field = [{name="TMP"; level="(*,*)";}];}'), + # 4 field multiple + ({'APP_CLIMO__FIELD': ('{name="TMP"; level="(*,*)";},' + '{name="TEMP"; level="P500";}')}, + ('climo_ = {field = [{name="TMP"; level="(*,*)";}, ' + '{name="TEMP"; level="P500";}];}')), + # 5 use fcst no other climo_ + ({'APP_CLIMO__USE_FCST': 'TRUE'}, + 'climo_ = fcst;'), + # 6 use obs no other climo_ + ({'APP_CLIMO__USE_OBS': 'TRUE'}, + 'climo_ = obs;'), + # 7 use fcst with other climo_ + ({'APP_CLIMO__REGRID_METHOD': 'NEAREST', + 'APP_CLIMO__USE_FCST': 'TRUE'}, + 'climo_ = {regrid = {method = NEAREST;}}climo_ = fcst;'), + # 8 use obs with other climo_ + ({'APP_CLIMO__REGRID_METHOD': 'NEAREST', + 'APP_CLIMO__USE_OBS': 'TRUE'}, + 'climo_ = {regrid = {method = NEAREST;}}climo_ = obs;'), + # 9 regrid method + ({'APP_CLIMO__REGRID_METHOD': 'NEAREST', }, + 'climo_ = {regrid = {method = NEAREST;}}'), + # 10 regrid width + ({'APP_CLIMO__REGRID_WIDTH': '1', }, + 'climo_ = {regrid = {width = 1;}}'), + # 11 regrid vld_thresh + ({'APP_CLIMO__REGRID_VLD_THRESH': '0.5', }, + 'climo_ = {regrid = {vld_thresh = 0.5;}}'), + # 12 regrid shape + ({'APP_CLIMO__REGRID_SHAPE': 'SQUARE', }, + 'climo_ = {regrid = {shape = SQUARE;}}'), + # 13 time_interp_method + ({'APP_CLIMO__TIME_INTERP_METHOD': 'NEAREST', }, + 'climo_ = {time_interp_method = NEAREST;}'), + # 14 match_month + ({'APP_CLIMO__MATCH_MONTH': 'True', }, + 'climo_ = {match_month = TRUE;}'), + # 15 day_interval - int + ({'APP_CLIMO__DAY_INTERVAL': '30', }, + 'climo_ = {day_interval = 30;}'), + # 16 day_interval - NA + ({'APP_CLIMO__DAY_INTERVAL': 'NA', }, + 'climo_ = {day_interval = NA;}'), + # 17 hour_interval + ({'APP_CLIMO__HOUR_INTERVAL': '12', }, + 'climo_ = {hour_interval = 12;}'), + # 18 all + ({ + 'APP_CLIMO__FILE_NAME': '/some/climo_/file.txt', + 'APP_CLIMO__FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', + 'APP_CLIMO__REGRID_METHOD': 'NEAREST', + 'APP_CLIMO__REGRID_WIDTH': '1', + 'APP_CLIMO__REGRID_VLD_THRESH': '0.5', + 'APP_CLIMO__REGRID_SHAPE': 'SQUARE', + 'APP_CLIMO__TIME_INTERP_METHOD': 'NEAREST', + 'APP_CLIMO__MATCH_MONTH': 'True', + 'APP_CLIMO__DAY_INTERVAL': '30', + 'APP_CLIMO__HOUR_INTERVAL': '12', + }, + ('climo_ = {file_name = ' + '["/some/climo_/file.txt"];' + 'field = [{name="CLM_NAME"; level="(0,0,*,*)";}];' + 'regrid = {method = NEAREST;width = 1;' + 'vld_thresh = 0.5;shape = SQUARE;}' + 'time_interp_method = NEAREST;' + 'match_month = TRUE;day_interval = 30;' + 'hour_interval = 12;}')), + ] +) +def test_handle_climo_dict(metplus_config, config_overrides, expected_value): + app_name = 'app' + for climo_type in ('MEAN', 'STDEV'): + expected_var = f'METPLUS_CLIMO_{climo_type}_DICT' + config = metplus_config() + output_dict = {} + + # set config values + for key, value in config_overrides.items(): + key_sub = key.replace('', climo_type) + value_sub = value.replace('', climo_type.lower()) + config.set('config', key_sub, value_sub) + + handle_climo_dict(config, app_name, output_dict) + print(output_dict) + expected_sub = expected_value.replace('', climo_type.lower()) + assert output_dict[expected_var] == expected_sub @pytest.mark.parametrize( 'name, data_type, mp_configs, extra_args', [ diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py index b2cebe49e9..807cf10a3d 100644 --- a/metplus/util/met_config.py +++ b/metplus/util/met_config.py @@ -773,7 +773,7 @@ def handle_climo_dict(config, app_name, output_dict): }), 'time_interp_method': ('string', 'remove_quotes,uppercase'), 'match_month': ('bool', 'uppercase'), - 'day_interval': 'int', + 'day_interval': ('string', 'remove_quotes,uppercase'), 'hour_interval': 'int', 'file_type': ('string', 'remove_quotes'), } From 2912312a988ddb55a67e82784217e714b14bcd06 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 22 Jun 2022 15:55:02 -0600 Subject: [PATCH 2/4] feature #1673 5.0.0-beta1 release (#1674) Co-authored-by: jprestop --- .../update_release_notes_bugfix.rst | 19 +- .../update_release_notes_development.rst | 19 +- .../update_release_notes_official.rst | 19 +- docs/Users_Guide/release-notes.rst | 312 ++---------------- metplus/VERSION | 2 +- 5 files changed, 65 insertions(+), 306 deletions(-) diff --git a/docs/Release_Guide/release_steps/update_release_notes_bugfix.rst b/docs/Release_Guide/release_steps/update_release_notes_bugfix.rst index 54882d1051..e530beadac 100644 --- a/docs/Release_Guide/release_steps/update_release_notes_bugfix.rst +++ b/docs/Release_Guide/release_steps/update_release_notes_bugfix.rst @@ -1,15 +1,26 @@ Update Release Notes -------------------- -You can refer to the GitHub Issues page to see what has changed for this +You can refer to the GitHub Project board to see what has changed for this release. Open the following URL in a browser: .. parsed-literal:: - https://github.com/dtcenter/|projectRepo|/issues + https://github.com/orgs/dtcenter/projects?type=beta -* Click on the Projects tab and select the project (under Repository) that - corresponds to the release you are creating. +* Click on the project that corresponds to support for the release, i.e. + |projectRepo| Version X.Y Support + +* Navigate to the "Closed Issues" tab. + **If this tab does not exist**, follow these instructions to create it: + + * Click on "+ New view" button on the far right side of the view tabs + * Click on "View " (where is an integer) and rename it to + "Closed Issues" + * Click on the down arrow next to the newly created view + * Click on "Search or filter this view" + * Enter the following info into the filter bar: **is:closed is:issue** + * Click on the down arrow next to the view and click "Save changes" * Update the release-notes.rst file found in the User's Guide directory. diff --git a/docs/Release_Guide/release_steps/update_release_notes_development.rst b/docs/Release_Guide/release_steps/update_release_notes_development.rst index 378b17fe00..8cc4021a6c 100644 --- a/docs/Release_Guide/release_steps/update_release_notes_development.rst +++ b/docs/Release_Guide/release_steps/update_release_notes_development.rst @@ -1,15 +1,26 @@ Update Release Notes -------------------- -You can refer to the GitHub Issues page to see what has changed for this +You can refer to the GitHub Project board to see what has changed for this release. Open the following URL in a browser: .. parsed-literal:: - https://github.com/dtcenter/|projectRepo|/issues + https://github.com/orgs/dtcenter/projects?type=beta -* Click on the Projects tab and select the project (under Repository) that - corresponds to the release you are creating. +* Click on the project that corresponds to this release, i.e. + |projectRepo|-X.Y.Z-betaN + +* Navigate to the "Closed Issues" tab. + **If this tab does not exist**, follow these instructions to create it: + + * Click on "+ New view" button on the far right side of the view tabs + * Click on "View " (where is an integer) and rename it to + "Closed Issues" + * Click on the down arrow next to the newly created view + * Click on "Search or filter this view" + * Enter the following info into the filter bar: **is:closed is:issue** + * Click on the down arrow next to the view and click "Save changes" * Update the release-notes.rst file found in the User's Guide directory. diff --git a/docs/Release_Guide/release_steps/update_release_notes_official.rst b/docs/Release_Guide/release_steps/update_release_notes_official.rst index f3c384e7bd..02e077b4f6 100644 --- a/docs/Release_Guide/release_steps/update_release_notes_official.rst +++ b/docs/Release_Guide/release_steps/update_release_notes_official.rst @@ -1,15 +1,26 @@ Update Release Notes -------------------- -You can refer to the GitHub Issues page to see what has changed for this +You can refer to the GitHub Project board to see what has changed for this release. Open the following URL in a browser: .. parsed-literal:: - https://github.com/dtcenter/|projectRepo|/issues + https://github.com/orgs/dtcenter/projects?type=beta -* Click on the Projects tab and select the project (under Repository) that - corresponds to the release you are creating. +* Click on the project that corresponds to this release, i.e. + |projectRepo|-X.Y.Z-rcN + +* Navigate to the "Closed Issues" tab. + **If this tab does not exist**, follow these instructions to create it: + + * Click on "+ New view" button on the far right side of the view tabs + * Click on "View " (where is an integer) and rename it to + "Closed Issues" + * Click on the down arrow next to the newly created view + * Click on "Search or filter this view" + * Enter the following info into the filter bar: **is:closed is:issue** + * Click on the down arrow next to the view and click "Save changes" * Update the release-notes.rst file found in the User's Guide directory. diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index 3443efde4c..a670f9d3aa 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -39,311 +39,37 @@ describes the bugfix, enhancement, or new feature: https://github.com/dtcenter/METplus/issues -METplus Version 4.1.0 Release Notes (2022-03-14) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +METplus Version 5.0.0 Beta 1 Release Notes (2022-06-22) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * Enhancements: * General: - * **Add support for setting control members in EnsembleStat and GenEnsProd** (`#1236 `_) - * **Create an Amazon AMI containing all METplus components** (`#506 `_) - * Modify wrappers that use wrapped MET config files to default to parm/met_config versions if unset (`#931 `_) - * Add support for setting hss_ec_value in MET config files (`#951 `_) - * Added support for setting a dictionary value for time_summary.width (`#1252 `_) - * Properly handle list values that include square braces (`#1212 `_) - * Update wrapped MET config files to reference MET_TMP_DIR in tmp value (`#1101 `_) - * Add support for setting INIT_LIST and VALID_LIST for irregular time intervals (`#1286 `_) - * Support setting the OMP_NUM_THREADS environment variable (`#1320 `_) - * Add support for commonly changed MET config variables (`#896 `_) - * Prevent wildcard character from being used in output file path (`#1291 `_) - * Add support for setting file_type for fcst/obs for applications that process gridded data (`#1165 `_) - * Enhance logic for setting mask.poly to allow MET list characters (square braces and semi-colon) (`#966 `_) - * Add support for new climo_cdf.direct_prob flag (`#1392 `_) - * Implement various enhancements to climatology settings (`#1247 `_) - * Enhance logic to set climatology info for Python embedding (`#944 `_) - * Updated logic for handling _CLIMO_MEAN_FIELD variables for specifying climatology fields (`#1021 `_) - * Incorporate basic zonal and meridional means into METplus (`#1230 `_) - * Add support for explicitly setting file list file paths in wrappers that support multiple input files (`#1289 `_) - * Add support for setting -out argument in TCStat and StatAnalysis wrappers (`#1102 `_) - - * PointStat: - - * Make output_flag.orank configurable for Point-Stat (`#1103 `_) - * Added support for setting obs_quality_inc/exc in PointStat (`#1213 `_) - - * GridStat: - - * Add Grid-Stat configuration options for distance_map dictionary (`#1089 `_) - - * EnsembleStat: - - * Add support for setting grid_weight_flag in EnsembleStat (`#1369 `_) - * Fix logic to use fcst dictionary if ens dictionary is not set in EnsembleStat wrapper (`#1421 `_) - * Add support for probabilistic verification to the Ensemble-Stat wrapper (`#1464 `_) - - * GenEnsProd: - - * Add support for the normalize option to the Gen-Ens-Prod wrapper (`#1445 `_) - - * TCPairs: - - * Enhance TC-Pairs wrapper to make valid_inc, valid_exc, and write_valid configurable options (`#1069 `_) - * Improve logic of TCPairs wrapper (`#749 `_) - * Enhance TCPairs to loop by valid time and allow looping when LOOP_ORDER = processes (`#986 `_) - - * TCGen: - - * Enhance TCGen wrapper to add support for new configurations (`#1273 `_) - - * SeriesAnalysis: - - * **Enhance SeriesAnalysis wrapper to allow different field info values for each file in a list** (`#1166 `_) - * Add support for probability field threshold in SeriesAnalysis (`#875 `_) - - * RegridDataPlane: - - * **Add support for extra field options in RegridDataPlane wrapper** (`#924 `_) - - * PCPCombine: - - * Improve PCPCombine derive mode logic to skip lookback (`#928 `_) - * Add support for using filename templates for defining input level in PCPCombine (`#1062 `_) - * Add option to PCPCombine to force using 0 hr accum in subtract mode (`#1368 `_) - - * GenVxMask: - - * Update GenVxMask wrapper to require setting -type (`#960 `_) - - * UserScript: - - * **Enhance UserScript to get a list of files that match the run times instead of using a wildcard** (`#1002 `_) - - * ExtractTiles: - - * Enhance ExtractTiles using MTD input to properly match times (`#1285 `_) - - * TCMPRPlotter: - - * Improvements to TCMPRPlotter wrapper logging and output control (`#926 `_) - * Add option to TCMPRPlotter to pass in directory to tc_stat instead of individual files (`#1057 `_) - * Add option to pass in the input directory to TCMPRPlotter instead of finding all tcst files and passing the list (`#1084 `_) - - * CyclonePlotter: - - * Update CyclonePlotter for offline/HPC usage (`#933 `_) - * CyclonePlotter, create options to format output grid area to user-desired area (`#1091 `_) - * CyclonePlotter, connected lines run over the Prime Meridian (`#1000 `_) - - * Use Cases: - - * Add stat_analysis to the Blocking and Weather Regime processing (`#1001 `_) - * Modify user diagnostic feature relative use case to use MetPy Python package (`#759 `_) - * Reorganize the Cryosphere and Marine and Coastal use case categories into one group (`#1200 `_) - * Add harmonic pre-processing to the RMM use case (`#1019 `_) - - -* New Wrappers: - - * GenEnsProd (`#1180 `_, `#1266 `_) - * GFDLTracker (`#615 `_) - * IODA2NC (`#1203 `_) - - -* New Use Cases: - - * MET Tool Wrapper: - - * **PointStat: Python Embedding for Point Observations** (`#1490 `_) - * IODA2NC (`#1204 `_) - * GenEnsProd (`#1180 `_, `#1266 `_) - * GFDLTracker for TropicalCyclone (`#615 `_) - * GFDLTracker for TC Genesis (`#616 `_) - * GFDLTracker for Extra-TC Tracking (`#617 `_) - - - * Marine and Cryosphere: - - * GridStat_fcstRTOFS_obsOSTIA_iceCover (`#834 `_) - * Satellite verification of sea surface temperature (GHRSST) against RTOFS output (`#1004 `_) - * Satellite verification of sea surface salinity: SMOS vs RTOFS output (`#1116 `_) - * Satellite verification of sea surface salinity: AVISO vs RTOFS output HYCOM climo (`#1318 `_) - * Satellite verification of sea surface salinity: SMAP vs RTOFS output (`#1216 `_) - - - * Medium Range: - - * Feature Relative using MTD output for feature centroid lat/lon (`#641 `_) - - - * Precipitation: - - * Precipitation-type comparison across 3 models (`#1408 `_) - - - * Seasonal to Subseasonal (S2S): - - * UserScript_fcstGFS_obsERA_OMI (`#892 `_) - * UserScript_fcstGFS_obsERA_PhaseDiagram (`#1019 `_) - * UserScript_fcstGFS_obsERA_RMM (`#892 `_) - * RMM and OMI (driver scripts) (`#892 `_) - - - * Tropical Cyclone and Extra Tropical Cyclone (tc_and_extra_tc): - - * TC Verification Compare ADECK vs BDECK (`#911 `_) - * TCGen Verify Deterministic Genesis Forecasts and Probabilities from ATCF e-deck files (`#1274 `_) - + * **Enhance MODE wrapper to support multi-variate MODE** (`#1585 `_) + * **Allow FCST_IS_PROB variable setting specific to tool (FCST__IS_PROB)** (`#1586 `_) + * **Enhance climatology field settings to be consistent with fcst/obs field** (`#1599 `_) + * Update Hovmoeller Use case to use updated Hovmoeller plotting (`#1650 `_) * Bugfixes: - * Fix read of PB2NC_FILE_WINDOW_[BEGIN/END] configuration variables (`#1486 `_) - * Fix use of current field info in output prefix when using process list instances (`#1471 `_) - * Fix logic to create instances of other wrappers within wrappers to avoid modifying global configurations (`#1356 `_) - + * Add support for the {custom} loop string in the MODEL config variable (`#1382 `_) + * Fix PCPCombine extra options removal of semi-colon (`#1534 `_) + * Fix reset of arguments for some wrappers (i.e. GenEnsProd) after each run (`#1555 `_) + * Enhance METDbLoad Wrapper to find MODE .txt files (`#1608 `_) + * Add missing brackets around list variable values for StatAnalysis wrapper (`#1641 `_) + * Allow NA value for _CLIMO_[MEAN/STDEV]_DAY_INTERVAL (`#1653 `_) -* Documentation: +* New Wrappers: None - * Add list of METplus statistics to documentation (`#1049 `_) - * Update documentation to reference GitHub Discussions instead of MET Help (`#956 `_) - * Fix installation instructions in User's Guide (`#1067 `_) - * Add instructions to update old METplus configuration files that reference user-defined wrapped MET config files (`#1147 `_) - -* Internal: +* New Use Cases: None - * Improve approach to obtain additional python packages needed for some use cases (`#839 `_) - * Make updates to the Release Guide (`#935 `_) - * Clean up GitHub wiki broken links and out-of-date information (`#237 `_) - * Add option to override MET version used for automated tests (`#936 `_) - * Transition Community and Developer Support to Github Discussions (`#932 `_) - * Add documentation about the Release Guide and Verification Datasets Guide (`#874 `_) - * Create guidance for memory-intensive use cases, introduce Python memory profiler (`#1183 `_) - * Identify code throughout METplus components that are common utilities (`#799 `_) - * Add definitions to the Release Guide for the stages of the release cycle (`#934 `_) - * Document Continous Integration Functionality in the METplus Contributor's Guide (`#675 `_) - * Update Contributor's Guide for new removing/adding data protocols (`#1227 `_) - * Add recording of Python packages to Adding Use Cases documentation (`#1374 `_) - * Remove public-facing access to outdated use case categories (Cryosphere, marine_and_coastal) (`#1226 `_) - - -METplus Version 4.0.0 Release Notes (2021-05-10) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -* Bugfixes: - - * **Changed default values in wrapped MET config files to align with actual default values in MET config files** (:ref:`reconcile_default_values`) - * Fix bug causing GridStat fatal error (`#740 `_) - * Add support for comparing inputs using a mix of python embedding and non-embedding (`#684 `_) - * Fix quick search links (`#687 `_) - * Align the user guide with get_relativedelta() in time_util.py (`#579 `_) - * Fix CyclonePlotter cartopy mapping issues (`#850 `_, `#803 `_) - -* Enhancements: - - * **Rename master_metplus.py script to run_metplus.py** (`#794 `_) - * **Update setting of environment variables for MET config files to add support for all to METPLUS\_ vars** (`#768 `_) - * **Add support for many commonly changed MET config variables** (`#779 `_, `#755 `_, `#621 `_, `#620 `_) - * **Add support for a UserScript wrapper** (`#723 `_) - * **Create use case subdirectories** (`#751 `_) - * **Implement [INIT/VALID]EXCLUDE for time looping** (`#307 `_) - * **Add files to allow installation of METplus wrappers as a Python package (beta)** (`#282 `_) - * Generate PDF of User's Guide (`#551 `_) - * Add support for MET tc_gen changes in METplus (`#871 `_, (`#801 `_) - * Add support for 2 fields with same name and different levels in SeriesBy cases (`#852 `_) - * Enhance PCPCombine wrapper to be able to process multiple fields in one command (`#718 `_) - * Update TCStat config options and wrappers to filter data by excluding strings (`#857 `_) - * Support METplus to run from a driver script (`#569 `_) - * Refactor field info parsing to read once then substitute time info for each run time (`#880 `_) - * Enhance Python embedding logic to allow multiple level values (`#719 `_) - * Enhance Python embedding logic to allow multiple fcst and obs variable levels (`#708 `_) - * Add support for a group of files covering multiple run times for a single analysis in GridDiag (`#733 `_) - * Enhance ascii2nc python embedding script for TC dropsonde data (`#734 `_, `#731 `_) - * Support additional configuration variables in EnsembleStat (`#748 `_) - * Ensure backwards compatibility for MET config environment variables (`#760 `_) - * Combine configuration file sections into single config section (`#777 `_) - * Add support for skipping existing output files for all wrappers (`#711 `_) - * Add support for multiple instance of the same tool in the process list (`#670 `_) - * Add GFDL build support in build_components (`#614 `_) - * Decouple PCPCombine, RegridDataPlane, and GridStat wrappers behavior (`#602 `_) - * StatAnalysis run without filtering or config file (`#625 `_) - * Enhance User Diagnostic Feature Relative use case to Run Multiple Diagnostics (`#536 `_) - * Enhance PyEmbedIngest to run RegridDataPlane over Multiple Fields in One Call (`#549 `_) - * Filename templates that have other arguments besides a filename for python embedding fails (`#581 `_) - * Add more logging to tc_gen_wrapper (`#576 `_) - * Prevent crash when improperly formatted filename template is used (`#674 `_) - -* New Wrappers: - - * **PlotDataPlane** - * **UserScript** - * **METdbLoad** - -* New Use Cases: +* Documentation: - * Air Quality and Comp: EnsembleStat_fcstICAP_obsMODIS_aod - * Medium Range: UserScript_fcstGEFS_Difficulty_Index - * Convection Allowing Models: MODE_fcstFV3_obsGOES_BrightnessTemp - * Convection Allowing Models: MODE_fcstFV3_obsGOES_BrightnessTempObjs - * Convection Allowing Models: GridStat_fcstFV3_obsGOES_BrightnessTempDmap - * Data Assimilation: StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface - * Medium Range: SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics - * Precipitation: EnsembleStat_fcstWOFS_obsWOFS - * Seasonal to Subseasonal: TCGen_fcstGFSO_obsBDECKS_GDF_TDF - * Seasonal to Subseasonal: UserScript_fcstGFS_obsERA_Blocking - * Seasonal to Subseasonal: UserScript_obsERA_obsOnly_Blocking - * Seasonal to Subseasonal: UserScript_obsERA_obsOnly_WeatherRegime - * Seasonal to Subseasonal: UserScript_obsPrecip_obsOnly_Hovmoeller - * Seasonal to Subseasonal: UserScript_obsPrecip_obsOnly_CrossSpectraPlot - * TC and Extra TC: CyclonePlotter_fcstGFS_obsGFS_OPC - * TC and Extra TC: UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF - * TC and Extra TC: GridStat_fcstHAFS_obsTDR_NetCDF - * Marine and Coastal: PlotDataPlane_obsHYCOM_coordTripolar - * MET Tool Wrapper: METdbLoad/METdbLoad - * MET Tool Wrapper: PlotDataPlane/PlotDataPlane_grib1 - * MET Tool Wrapper: PlotDataPlane/PlotDataPlane_netcdf - * MET Tool Wrapper: PlotDataPlane/PlotDataPlane_python_embedding - * MET Tool Wrapper: GridStat/GridStat_python_embedding - * MET Tool Wrapper: PointStat/PointStat_python_embedding - * MET Tool Wrapper: MODE/MODE_python_embedding - * MET Tool Wrapper: PyEmbedIngest_multi_field_one_file + * Update documentation to include instructions to disable UserScript wrapper (`dtcenter/METplus-Internal#33 `_) * Internal: - * Append semi-colon to end of _OPTIONS variables if not found (`#707 `_) - * Ensure all wrappers follow the same conventions (`#76 `_) - * Refactor SeriesBy and ExtractTiles wrappers (`#310 `_) - * Refactor SeriesByLead wrapper (`#671 `_, `#76 `_) - * Add the pull request approval process steps to the Contributor's Guide (`#429 `_) - * Remove jlogger and postmsg (`#470 `_) - * Add unit tests for set_met_config_X functions in CommandBuilder (`#682 `_) - * Define a common set of GitHub labels that apply to all of the METplus component repos (`#690 `_) - * Transition from using Travis CI to GitHub Actions (`#721 `_) - * Improve workflow formatting in Contributers Guide (`#688 `_) - * Change INPUT_BASE to optional (`#679 `_) - * Refactor TCStat and ExtractTiles wrappers to conform to current standards - * Automate release date (`#665 `_) - * Add documentation for input verification datasets (`#662 `_) - * Add timing tests for Travis/Docker (`#649 `_) - * Set up encrypted credentials in Travis to push to DockerHub (`#634 `_) - * Add to User's Guide: using environment variables in METplus configuration files (`#594 `_) - * Cleanup version info (`#651 `_) - * Fix Travis tests for pull requests from forks (`#659 `_) - * Enhance the build_docker_images.sh script to support TravisCI updates (`#636 `_) - * Reorganize use case tests so users can add new cases easily (`#648 `_) - * Investigate how to add version selector to documentation (`#653 `_) - * Docker push pull image repository (`#639 `_) - * Tutorial Proofreading (`#534 `_) - * Update METplus data container logic to pull tarballs from dtcenter.org instead of GitHub release assets (`#613 `_) - * Convert Travis Docker files (automated builds) to use Dockerhub data volumes instead of tarballs (`#597 `_) - * Migrate from travis-ci.org to travis-ci.com (`#618 `_) - * Migrate Docker run commands to the METplus ci/jobs scripts/files (`#607 `_) - * Add stage to Travis to update or create data volumes when new sample data is available (`#633 `_) - * Docker data caching (`#623 `_) - * Tutorial testing on supported platforms (`#468 `_) - * Add additional Branch support to the Travis CI pipeline (`#478 `_) - * Change $DOCKER_WORK_DIR from /metplus to /root to be consistent with METplus tutorial (`#595 `_) - * Add all use_cases to automated tests (eg Travis) (`#571 `_) - * Add support to run METplus tests against multiple version of Python (`#483 `_) - * Enhanced testing to use Docker data volumes to supply truth data for output comparisons (`#567 `_) - * Update manage externals for beta5 versions (`#832 `_) - * Create a new METplus GitHub issue template for "New Use Case" (`#726 `_) + * Document GitHub Discussions procedure for the Contributor's Guide (`#1159 `_) + * Create a METplus "Release Guide" describing how to build releases for the METplus components (`#673 `_) + * Update documentation about viewing RTD URLs on branches (`#1512 `_) diff --git a/metplus/VERSION b/metplus/VERSION index 3d298f67af..c138960ea6 100644 --- a/metplus/VERSION +++ b/metplus/VERSION @@ -1 +1 @@ -5.0.0-beta1-dev +5.0.0-beta1 From 2e4b709e5911e40e7e990143599bde78ca7fe376 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 22 Jun 2022 15:57:56 -0600 Subject: [PATCH 3/4] update version for development towards 5.0.0-beta2 --- metplus/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metplus/VERSION b/metplus/VERSION index c138960ea6..c3a40d43b1 100644 --- a/metplus/VERSION +++ b/metplus/VERSION @@ -1 +1 @@ -5.0.0-beta1 +5.0.0-beta2-dev From b69616cd7799b35c5f7adc7cd7ec9bc5b22e8832 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 22 Jun 2022 16:45:18 -0600 Subject: [PATCH 4/4] added logic to switch DockerHub repo used to obtain MET to use dtcenter/met-dev if the tag is develop, otherwise use dtcenter/met --- scripts/docker/hooks/build | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/docker/hooks/build b/scripts/docker/hooks/build index 378a386bbd..472551c107 100644 --- a/scripts/docker/hooks/build +++ b/scripts/docker/hooks/build @@ -4,4 +4,14 @@ met_tag=`$(dirname $DOCKERFILE_PATH)/hooks/get_met_version` echo $met_tag -docker build -t $IMAGE_NAME --build-arg SOURCE_VERSION=$SOURCE_BRANCH --build-arg MET_TAG=$met_tag . + +MET_DOCKER_REPO=met +if [ "$met_tag" == "develop" ]; then + MET_DOCKER_REPO=met-dev +fi + +docker build -t $IMAGE_NAME \ + --build-arg SOURCE_VERSION=$SOURCE_BRANCH \ + --build-arg MET_TAG=$met_tag \ + --build-arg MET_DOCKER_REPO=$MET_DOCKER_REPO \ + .