From 80afcac6e3906ac84b38ee678df640968c47b059 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Sep 2023 10:42:31 -0600 Subject: [PATCH 01/12] Update develop-ref after #2680 and #2683 (#2684) Co-authored-by: jprestop Co-authored-by: Seth Linden Co-authored-by: John Halley Gotway Co-authored-by: Daniel Adriaansen Co-authored-by: John and Cindy Co-authored-by: rgbullock Co-authored-by: Randy Bullock Co-authored-by: Dave Albo Co-authored-by: Howard Soh Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: hsoh-u Co-authored-by: MET Tools Test Account Co-authored-by: Seth Linden Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: davidalbo Co-authored-by: Lisa Goodrich Co-authored-by: metplus-bot <97135045+metplus-bot@users.noreply.github.com> Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Jonathan Vigh Co-authored-by: Tracy Hertneky <39317287+hertneky@users.noreply.github.com> Co-authored-by: David Albo Co-authored-by: Dan Adriaansen fix 2518 dtypes appf docs (#2519) fix 2531 compilation errors (#2533) fix #2531 compilation_errors_configure (#2535) fix #2514 develop clang (#2563) fix #2575 develop python_convert (#2576) Fix Python environment issue (#2407) fix definitions of G172 and G220 based on comments in NOAA-EMC/NCEPLIBS-w3emc#157. (#2406) fix #2380 develop override (#2382) fix #2408 develop empty config (#2410) fix #2390 develop compile zlib (#2404) fix #2412 develop climo (#2422) fix #2437 develop convert (#2439) fix for develop, for #2437, forgot one reference to the search_parent for a dictionary lookup. fix #2452 develop airnow (#2454) fix #2449 develop pdf (#2464) fix #2402 develop sonarqube (#2468) fix #2426 develop buoy (#2475) fix 2596 main v11.1 rpath compilation (#2614) fix #2514 main_v11.1 clang (#2628) fix #2644 develop percentile (#2647) --- data/config/EnsembleStatConfig_default | 2 +- data/config/GenEnsProdConfig_default | 2 +- data/config/GridStatConfig_default | 2 +- data/config/MODEConfig_default | 2 +- data/config/MODEMultivarConfig_default | 2 +- data/config/MTDConfig_default | 2 +- data/config/PointStatConfig_default | 2 +- data/config/SeriesAnalysisConfig_default | 2 +- data/config/TCDiagConfig_default | 12 +- data/config/WaveletStatConfig_default | 2 +- docs/Users_Guide/config_options.rst | 2 +- docs/Users_Guide/config_options_tc.rst | 8 +- docs/Users_Guide/ensemble-stat.rst | 2 +- docs/Users_Guide/gen-ens-prod.rst | 2 +- docs/Users_Guide/grid-stat.rst | 2 +- docs/Users_Guide/mode-td.rst | 2 +- docs/Users_Guide/mode.rst | 2 +- docs/Users_Guide/point-stat.rst | 2 +- docs/Users_Guide/series-analysis.rst | 2 +- docs/Users_Guide/tc-diag.rst | 23 +++- docs/Users_Guide/wavelet-stat.rst | 2 +- internal/test_unit/R_test/test_util.R | 2 +- internal/test_unit/config/EnsembleStatConfig | 2 +- .../config/EnsembleStatConfig_MASK_SID | 2 +- .../test_unit/config/EnsembleStatConfig_climo | 2 +- .../config/EnsembleStatConfig_grid_weight | 2 +- .../config/EnsembleStatConfig_one_cdf_bin | 2 +- .../config/EnsembleStatConfig_qty_inc_exc | 2 +- internal/test_unit/config/GenEnsProdConfig | 2 +- .../config/GenEnsProdConfig_normalize | 2 +- .../config/GridStatConfig_APCP_regrid | 2 +- .../test_unit/config/GridStatConfig_SEEPS | 2 +- internal/test_unit/config/GridStatConfig_rtma | 2 +- .../config/GridStatConfig_rtma_perc_thresh | 2 +- internal/test_unit/config/GridStatConfig_st4 | 2 +- internal/test_unit/config/MODEConfig_cut_line | 2 +- internal/test_unit/config/MODEConfig_hmt | 2 +- .../config/MODEConfig_multivar_fake_data | 2 +- ...Config_multivar_fake_data_with_intensities | 2 +- .../test_unit/config/MODEConfig_perc_thresh | 2 +- internal/test_unit/config/MODEConfig_python | 2 +- .../test_unit/config/MODEConfig_python_mixed | 2 +- internal/test_unit/config/MODEConfig_quilt | 2 +- internal/test_unit/config/MTDConfig_simple | 2 +- .../test_unit/config/PointStatConfig_APCP | 2 +- .../config/PointStatConfig_APCP_HIRA | 2 +- .../config/PointStatConfig_LAND_TOPO_MASK | 2 +- .../test_unit/config/PointStatConfig_MASK_SID | 2 +- .../test_unit/config/PointStatConfig_PHYS | 2 +- .../config/PointStatConfig_PHYS_pint | 2 +- .../test_unit/config/PointStatConfig_WINDS | 2 +- internal/test_unit/config/PointStatConfig_dup | 2 +- .../config/PointStatConfig_obs_summary | 2 +- .../config/PointStatConfig_obs_summary_all | 2 +- .../test_unit/config/PointStatConfig_prob | 2 +- .../test_unit/config/PointStatConfig_python | 2 +- .../config/PointStatConfig_qty_inc_exc | 2 +- .../config/PointStatConfig_sid_inc_exc | 2 +- internal/test_unit/config/TCDiagConfig_ian | 12 +- internal/test_unit/config/WaveletStatConfig | 2 +- .../config/WaveletStatConfig_no_thresholds | 2 +- .../test_unit/config/WaveletStatConfig_python | 2 +- .../config/WaveletStatConfig_python_mixed | 2 +- scripts/config/EnsembleStatConfig | 2 +- scripts/config/GenEnsProdConfig | 2 +- scripts/config/GridStatConfig_APCP_12 | 2 +- scripts/config/GridStatConfig_APCP_24 | 2 +- scripts/config/GridStatConfig_POP_12 | 2 +- scripts/config/GridStatConfig_all | 2 +- scripts/config/MODEConfig_APCP_12 | 2 +- scripts/config/MODEConfig_APCP_24 | 2 +- scripts/config/MODEConfig_RH | 2 +- scripts/config/PointStatConfig | 2 +- scripts/config/WaveletStatConfig_APCP_12 | 2 +- scripts/config/WaveletStatConfig_APCP_12_NC | 2 +- src/basic/vx_config/config_constants.h | 15 ++- src/basic/vx_log/logger.cc | 39 +++++- src/basic/vx_log/logger.h | 5 + src/libcode/vx_data2d/data_class.cc | 43 +++++++ src/libcode/vx_data2d/data_class.h | 5 + src/libcode/vx_data2d_grib/data2d_grib.cc | 2 + src/libcode/vx_data2d_grib/data2d_grib.h | 8 +- src/libcode/vx_data2d_grib2/data2d_grib2.cc | 15 +-- .../vx_data2d_nc_pinterp/data2d_nc_pinterp.cc | 2 + src/libcode/vx_data2d_nccf/data2d_nccf.h | 1 - src/libcode/vx_data2d_python/data2d_python.cc | 1 - src/libcode/vx_series_data/series_data.cc | 121 ++++++++++++++++-- src/libcode/vx_series_data/series_data.h | 14 +- src/tools/tc_utils/tc_diag/tc_diag.cc | 112 +++++++++++++--- src/tools/tc_utils/tc_diag/tc_diag.h | 2 + .../tc_utils/tc_diag/tc_diag_conf_info.cc | 5 + .../tc_utils/tc_diag/tc_diag_conf_info.h | 5 +- 92 files changed, 462 insertions(+), 130 deletions(-) diff --git a/data/config/EnsembleStatConfig_default b/data/config/EnsembleStatConfig_default index d7b003bb02..57f1984807 100644 --- a/data/config/EnsembleStatConfig_default +++ b/data/config/EnsembleStatConfig_default @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/data/config/GenEnsProdConfig_default b/data/config/GenEnsProdConfig_default index e979270203..e2cb994bba 100644 --- a/data/config/GenEnsProdConfig_default +++ b/data/config/GenEnsProdConfig_default @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/data/config/GridStatConfig_default b/data/config/GridStatConfig_default index a18b668419..4bec5ecf4b 100644 --- a/data/config/GridStatConfig_default +++ b/data/config/GridStatConfig_default @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/data/config/MODEConfig_default b/data/config/MODEConfig_default index b96ceb7160..7cfe46a6b4 100644 --- a/data/config/MODEConfig_default +++ b/data/config/MODEConfig_default @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/data/config/MODEMultivarConfig_default b/data/config/MODEMultivarConfig_default index 45c612e497..66ab5dfdf3 100644 --- a/data/config/MODEMultivarConfig_default +++ b/data/config/MODEMultivarConfig_default @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/data/config/MTDConfig_default b/data/config/MTDConfig_default index 1774458415..b5f185ff7e 100644 --- a/data/config/MTDConfig_default +++ b/data/config/MTDConfig_default @@ -10,7 +10,7 @@ // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/data/config/PointStatConfig_default b/data/config/PointStatConfig_default index fde6d0e17f..d4b277c6a3 100644 --- a/data/config/PointStatConfig_default +++ b/data/config/PointStatConfig_default @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/data/config/SeriesAnalysisConfig_default b/data/config/SeriesAnalysisConfig_default index 1ab74aee7b..2b0cdfa53e 100644 --- a/data/config/SeriesAnalysisConfig_default +++ b/data/config/SeriesAnalysisConfig_default @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/data/config/TCDiagConfig_default b/data/config/TCDiagConfig_default index f81ccdf86d..f4b489302e 100644 --- a/data/config/TCDiagConfig_default +++ b/data/config/TCDiagConfig_default @@ -45,9 +45,14 @@ valid_exc = []; // // Subset by the valid hour and lead time. +// By default, compute diagnostics every 6 hours out to 126 hours. // valid_hour = []; -lead = []; +lead = [ "0", "6", "12", "18", "24", + "30", "36", "42", "48", "54", + "60", "66", "72", "78", "84", + "90", "96", "102", "108", "114", + "120", "126" ]; //////////////////////////////////////////////////////////////////////////////// @@ -134,6 +139,11 @@ vortex_removal = FALSE; //////////////////////////////////////////////////////////////////////////////// +// +// Flag to control input files +// +one_time_per_file_flag = TRUE; + // // Flags to control output files // diff --git a/data/config/WaveletStatConfig_default b/data/config/WaveletStatConfig_default index 09c3b4d153..668f0a655d 100644 --- a/data/config/WaveletStatConfig_default +++ b/data/config/WaveletStatConfig_default @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/docs/Users_Guide/config_options.rst b/docs/Users_Guide/config_options.rst index 48be5c556a..2bba1cc0ac 100644 --- a/docs/Users_Guide/config_options.rst +++ b/docs/Users_Guide/config_options.rst @@ -602,7 +602,7 @@ e.g. model = "GFS"; .. code-block:: none - model = "WRF"; + model = "FCST"; desc ^^^^ diff --git a/docs/Users_Guide/config_options_tc.rst b/docs/Users_Guide/config_options_tc.rst index 1d6d098ad8..f3c1592544 100644 --- a/docs/Users_Guide/config_options_tc.rst +++ b/docs/Users_Guide/config_options_tc.rst @@ -108,7 +108,7 @@ For example: valid_beg end inc exc --------------------- -Specify a model valid time window YYYYMMDD[_HH[MMSS]] format or provide a +Specify a model valid time window in YYYYMMDD[_HH[MMSS]] format or provide a list of specific valid times to include (inc) or exclude (exc). If a time window is specified, only tracks for which all points are contained within the window will be used. If valid times to include or exclude are specified, @@ -1077,7 +1077,7 @@ Statistics will be generated separately for each ATCF ID. init_beg, init_end ^^^^^^^^^^^^^^^^^^ -Forecast and operational initialization time window +Forecast and operational initialization time window, as strings in YYYYMMDD[_HH[MMSS]] format .. code-block:: none @@ -1087,7 +1087,7 @@ Forecast and operational initialization time window valid_beg, valid_end ^^^^^^^^^^^^^^^^^^^^ -Forecast, BEST, and operational valid time window +Forecast, BEST, and operational valid time window, as strings in YYYYMMDD[_HH[MMSS]] format .. code-block:: none @@ -1097,7 +1097,7 @@ Forecast, BEST, and operational valid time window lead ^^^^ -Forecast and operational lead times in hours +Forecast and operational lead times, as strings in HH[MMSS] format .. code-block:: none diff --git a/docs/Users_Guide/ensemble-stat.rst b/docs/Users_Guide/ensemble-stat.rst index 0972250627..7cba057a69 100644 --- a/docs/Users_Guide/ensemble-stat.rst +++ b/docs/Users_Guide/ensemble-stat.rst @@ -156,7 +156,7 @@ ____________________ .. code-block:: none - model = "WRF"; + model = "FCST"; desc = "NA"; obtype = "ANALYS"; regrid = { ... } diff --git a/docs/Users_Guide/gen-ens-prod.rst b/docs/Users_Guide/gen-ens-prod.rst index babb8e0ed4..7c581d2baf 100644 --- a/docs/Users_Guide/gen-ens-prod.rst +++ b/docs/Users_Guide/gen-ens-prod.rst @@ -96,7 +96,7 @@ ____________________ .. code-block:: none - model = "WRF"; + model = "FCST"; desc = "NA"; regrid = { ... } censor_thresh = []; diff --git a/docs/Users_Guide/grid-stat.rst b/docs/Users_Guide/grid-stat.rst index 55d9daaa4b..61c6ccb5db 100644 --- a/docs/Users_Guide/grid-stat.rst +++ b/docs/Users_Guide/grid-stat.rst @@ -236,7 +236,7 @@ __________________________ .. code-block:: none - model = "WRF"; + model = "FCST"; desc = "NA"; obtype = "ANALYS"; fcst = { ... } diff --git a/docs/Users_Guide/mode-td.rst b/docs/Users_Guide/mode-td.rst index bbda4dc315..e205ccba36 100644 --- a/docs/Users_Guide/mode-td.rst +++ b/docs/Users_Guide/mode-td.rst @@ -246,7 +246,7 @@ ______________________ .. code-block:: none - model = "WRF"; + model = "FCST"; desc = "NA"; obtype = "ANALYS"; regrid = { ... } diff --git a/docs/Users_Guide/mode.rst b/docs/Users_Guide/mode.rst index f5d9c4e609..98ab658ea4 100644 --- a/docs/Users_Guide/mode.rst +++ b/docs/Users_Guide/mode.rst @@ -203,7 +203,7 @@ _____________________ .. code-block:: none - model = "WRF"; + model = "FCST"; desc = "NA"; obtype = "ANALYS"; regrid = { ... } diff --git a/docs/Users_Guide/point-stat.rst b/docs/Users_Guide/point-stat.rst index e4f0e7eeda..1fd88668a9 100644 --- a/docs/Users_Guide/point-stat.rst +++ b/docs/Users_Guide/point-stat.rst @@ -331,7 +331,7 @@ ________________________ .. code-block:: none - model = "WRF"; + model = "FCST"; desc = "NA"; regrid = { ... } climo_mean = { ... } diff --git a/docs/Users_Guide/series-analysis.rst b/docs/Users_Guide/series-analysis.rst index 2b0b391d24..1dd568fcda 100644 --- a/docs/Users_Guide/series-analysis.rst +++ b/docs/Users_Guide/series-analysis.rst @@ -99,7 +99,7 @@ ____________________ .. code-block:: none - model = "WRF"; + model = "FCST"; desc = "NA"; obtype = "ANALYS"; regrid = { ... } diff --git a/docs/Users_Guide/tc-diag.rst b/docs/Users_Guide/tc-diag.rst index 53c1ee1b0d..5ecfe649f0 100644 --- a/docs/Users_Guide/tc-diag.rst +++ b/docs/Users_Guide/tc-diag.rst @@ -94,12 +94,21 @@ Configuring input tracks and time valid_inc = []; valid_exc = []; valid_hour = []; - lead = []; The TC-Diag tool should be configured to filter the input track data (**-deck**) down to the subset of tracks that correspond to the gridded data files provided (**-data**). The filtered tracks should contain data for only *one initialization time* but may contain tracks for multiple models. The configuration options listed above are used to filter the input track data down to those that should be processed in the current run. These options are common to multiple MET tools and are described in :numref:`config_options_tc`. +.. code-block:: none + + lead = [ "0", "6", "12", "18", "24", + "30", "36", "42", "48", "54", + "60", "66", "72", "78", "84", + "90", "96", "102", "108", "114", + "120", "126" ]; + +The **lead** entry is an array of strings specifying lead times in HH[MMSS] format. By default, diagnostics are computed every 6 hours out to 126 hours. Lead times for which no track point or gridded model data exist produce a warning message and diagnostics set to a missing data value. + Configuring domain information ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -204,8 +213,16 @@ The **vortex_removal** flag entry is a boolean specifying whether or not vortex .. note:: As of MET version 11.1.0, vortex removal logic is not yet supported. -Configuring data output options -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Configuring data input and output options +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: none + + one_time_per_file_flag = TRUE; + +The **one_time_per_file_flag** entry controls the logic for reading data from input files. This option describes how data is stored in the gridded input files specified with the **-data** command line option. Set this to true if each input file contains all of the data for a single initialization time and for a single valid time. If the input files contain data for multiple initialization or valid times, or if data for one valid time is spread across multiple files, set this to false. + +If true, all input fields are read efficiently from each file in a single call. If false, each field is processed separately in a less efficient manner. .. code-block:: none diff --git a/docs/Users_Guide/wavelet-stat.rst b/docs/Users_Guide/wavelet-stat.rst index 3622a4ff7a..b9a1aa7a76 100644 --- a/docs/Users_Guide/wavelet-stat.rst +++ b/docs/Users_Guide/wavelet-stat.rst @@ -211,7 +211,7 @@ _______________________ .. code-block:: none - model = "WRF"; + model = "FCST"; desc = "NA"; obtype = "ANALYS"; fcst = { ... } diff --git a/internal/test_unit/R_test/test_util.R b/internal/test_unit/R_test/test_util.R index bcbeb14e71..7d83360ecd 100644 --- a/internal/test_unit/R_test/test_util.R +++ b/internal/test_unit/R_test/test_util.R @@ -856,7 +856,7 @@ compareDiff = function(file1, file2, verb=0){ else if( 1 <= verb ){ cat("passed diff\n"); } } -listHeaderCols = c("VERSION", "DESC", "MODEL", +listHeaderCols = c("VERSION", "MODEL", "DESC", "FCST_LEAD", "FCST_VALID_BEG", "FCST_VALID_END", "OBS_LEAD", "OBS_VALID_BEG", "OBS_VALID_END", "FCST_VAR", "FCST_UNITS", "FCST_LEV", diff --git a/internal/test_unit/config/EnsembleStatConfig b/internal/test_unit/config/EnsembleStatConfig index fdcd370b7f..80157bed0c 100644 --- a/internal/test_unit/config/EnsembleStatConfig +++ b/internal/test_unit/config/EnsembleStatConfig @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/EnsembleStatConfig_MASK_SID b/internal/test_unit/config/EnsembleStatConfig_MASK_SID index 2a1e778b04..96128cb148 100644 --- a/internal/test_unit/config/EnsembleStatConfig_MASK_SID +++ b/internal/test_unit/config/EnsembleStatConfig_MASK_SID @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/EnsembleStatConfig_climo b/internal/test_unit/config/EnsembleStatConfig_climo index 0d11408904..633ab775f9 100644 --- a/internal/test_unit/config/EnsembleStatConfig_climo +++ b/internal/test_unit/config/EnsembleStatConfig_climo @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/EnsembleStatConfig_grid_weight b/internal/test_unit/config/EnsembleStatConfig_grid_weight index 08bf826bd1..9915c3fa37 100644 --- a/internal/test_unit/config/EnsembleStatConfig_grid_weight +++ b/internal/test_unit/config/EnsembleStatConfig_grid_weight @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/EnsembleStatConfig_one_cdf_bin b/internal/test_unit/config/EnsembleStatConfig_one_cdf_bin index 57031acdce..013763c74d 100644 --- a/internal/test_unit/config/EnsembleStatConfig_one_cdf_bin +++ b/internal/test_unit/config/EnsembleStatConfig_one_cdf_bin @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/EnsembleStatConfig_qty_inc_exc b/internal/test_unit/config/EnsembleStatConfig_qty_inc_exc index 66ff5edb5d..0881ff3b8d 100644 --- a/internal/test_unit/config/EnsembleStatConfig_qty_inc_exc +++ b/internal/test_unit/config/EnsembleStatConfig_qty_inc_exc @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/GenEnsProdConfig b/internal/test_unit/config/GenEnsProdConfig index 326c6b55aa..9a565c2fb4 100644 --- a/internal/test_unit/config/GenEnsProdConfig +++ b/internal/test_unit/config/GenEnsProdConfig @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/GenEnsProdConfig_normalize b/internal/test_unit/config/GenEnsProdConfig_normalize index 7236c64a1c..b23708ab46 100644 --- a/internal/test_unit/config/GenEnsProdConfig_normalize +++ b/internal/test_unit/config/GenEnsProdConfig_normalize @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/GridStatConfig_APCP_regrid b/internal/test_unit/config/GridStatConfig_APCP_regrid index caa5a2a59a..74bbba5b4d 100644 --- a/internal/test_unit/config/GridStatConfig_APCP_regrid +++ b/internal/test_unit/config/GridStatConfig_APCP_regrid @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/GridStatConfig_SEEPS b/internal/test_unit/config/GridStatConfig_SEEPS index e50b248914..8a23c76c45 100644 --- a/internal/test_unit/config/GridStatConfig_SEEPS +++ b/internal/test_unit/config/GridStatConfig_SEEPS @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/GridStatConfig_rtma b/internal/test_unit/config/GridStatConfig_rtma index 03723c0a74..ddef797d2e 100644 --- a/internal/test_unit/config/GridStatConfig_rtma +++ b/internal/test_unit/config/GridStatConfig_rtma @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/GridStatConfig_rtma_perc_thresh b/internal/test_unit/config/GridStatConfig_rtma_perc_thresh index 4e828f4192..44ed5f564f 100644 --- a/internal/test_unit/config/GridStatConfig_rtma_perc_thresh +++ b/internal/test_unit/config/GridStatConfig_rtma_perc_thresh @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/GridStatConfig_st4 b/internal/test_unit/config/GridStatConfig_st4 index daaa43ec02..71d165e98a 100644 --- a/internal/test_unit/config/GridStatConfig_st4 +++ b/internal/test_unit/config/GridStatConfig_st4 @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/MODEConfig_cut_line b/internal/test_unit/config/MODEConfig_cut_line index 8bd98820f2..81658982c4 100644 --- a/internal/test_unit/config/MODEConfig_cut_line +++ b/internal/test_unit/config/MODEConfig_cut_line @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/MODEConfig_hmt b/internal/test_unit/config/MODEConfig_hmt index 60ea3d5043..abfcd961aa 100644 --- a/internal/test_unit/config/MODEConfig_hmt +++ b/internal/test_unit/config/MODEConfig_hmt @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/MODEConfig_multivar_fake_data b/internal/test_unit/config/MODEConfig_multivar_fake_data index f391ee07df..fa46e066b2 100644 --- a/internal/test_unit/config/MODEConfig_multivar_fake_data +++ b/internal/test_unit/config/MODEConfig_multivar_fake_data @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/MODEConfig_multivar_fake_data_with_intensities b/internal/test_unit/config/MODEConfig_multivar_fake_data_with_intensities index 4591156fe5..7230ecf817 100644 --- a/internal/test_unit/config/MODEConfig_multivar_fake_data_with_intensities +++ b/internal/test_unit/config/MODEConfig_multivar_fake_data_with_intensities @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/MODEConfig_perc_thresh b/internal/test_unit/config/MODEConfig_perc_thresh index f8fc43b254..22f8c4938e 100644 --- a/internal/test_unit/config/MODEConfig_perc_thresh +++ b/internal/test_unit/config/MODEConfig_perc_thresh @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/MODEConfig_python b/internal/test_unit/config/MODEConfig_python index 23dea05343..9fd55471b9 100644 --- a/internal/test_unit/config/MODEConfig_python +++ b/internal/test_unit/config/MODEConfig_python @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/MODEConfig_python_mixed b/internal/test_unit/config/MODEConfig_python_mixed index a887a38093..ad59671837 100644 --- a/internal/test_unit/config/MODEConfig_python_mixed +++ b/internal/test_unit/config/MODEConfig_python_mixed @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/MODEConfig_quilt b/internal/test_unit/config/MODEConfig_quilt index 2b98d7988b..a8561aed9d 100644 --- a/internal/test_unit/config/MODEConfig_quilt +++ b/internal/test_unit/config/MODEConfig_quilt @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/MTDConfig_simple b/internal/test_unit/config/MTDConfig_simple index dd90c8806f..920606c433 100644 --- a/internal/test_unit/config/MTDConfig_simple +++ b/internal/test_unit/config/MTDConfig_simple @@ -10,7 +10,7 @@ // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_APCP b/internal/test_unit/config/PointStatConfig_APCP index 7f67f5cf8f..a6a6f06d64 100644 --- a/internal/test_unit/config/PointStatConfig_APCP +++ b/internal/test_unit/config/PointStatConfig_APCP @@ -6,7 +6,7 @@ // //////////////////////////////////////////////////////////////////////////////// -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_APCP_HIRA b/internal/test_unit/config/PointStatConfig_APCP_HIRA index 7debd797d3..3dc823a50b 100644 --- a/internal/test_unit/config/PointStatConfig_APCP_HIRA +++ b/internal/test_unit/config/PointStatConfig_APCP_HIRA @@ -6,7 +6,7 @@ // //////////////////////////////////////////////////////////////////////////////// -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK b/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK index 57841b0dae..48a04ed227 100644 --- a/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK +++ b/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK @@ -6,7 +6,7 @@ // //////////////////////////////////////////////////////////////////////////////// -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_MASK_SID b/internal/test_unit/config/PointStatConfig_MASK_SID index 2dae93f005..a77a8f9008 100644 --- a/internal/test_unit/config/PointStatConfig_MASK_SID +++ b/internal/test_unit/config/PointStatConfig_MASK_SID @@ -6,7 +6,7 @@ // //////////////////////////////////////////////////////////////////////////////// -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_PHYS b/internal/test_unit/config/PointStatConfig_PHYS index 7a1c51134b..3c13262a71 100644 --- a/internal/test_unit/config/PointStatConfig_PHYS +++ b/internal/test_unit/config/PointStatConfig_PHYS @@ -6,7 +6,7 @@ // //////////////////////////////////////////////////////////////////////////////// -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_PHYS_pint b/internal/test_unit/config/PointStatConfig_PHYS_pint index 58b8e5cba6..e246c570b3 100644 --- a/internal/test_unit/config/PointStatConfig_PHYS_pint +++ b/internal/test_unit/config/PointStatConfig_PHYS_pint @@ -6,7 +6,7 @@ // //////////////////////////////////////////////////////////////////////////////// -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_WINDS b/internal/test_unit/config/PointStatConfig_WINDS index d856492fd4..1c87273e89 100644 --- a/internal/test_unit/config/PointStatConfig_WINDS +++ b/internal/test_unit/config/PointStatConfig_WINDS @@ -6,7 +6,7 @@ // //////////////////////////////////////////////////////////////////////////////// -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_dup b/internal/test_unit/config/PointStatConfig_dup index 9f65ae8cfd..8fad74b0e5 100644 --- a/internal/test_unit/config/PointStatConfig_dup +++ b/internal/test_unit/config/PointStatConfig_dup @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_obs_summary b/internal/test_unit/config/PointStatConfig_obs_summary index 275f6fb963..a931ba6b21 100644 --- a/internal/test_unit/config/PointStatConfig_obs_summary +++ b/internal/test_unit/config/PointStatConfig_obs_summary @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_obs_summary_all b/internal/test_unit/config/PointStatConfig_obs_summary_all index 34e01fdd4f..384e4ada19 100644 --- a/internal/test_unit/config/PointStatConfig_obs_summary_all +++ b/internal/test_unit/config/PointStatConfig_obs_summary_all @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_prob b/internal/test_unit/config/PointStatConfig_prob index 1a1c73f749..ab8067a79e 100644 --- a/internal/test_unit/config/PointStatConfig_prob +++ b/internal/test_unit/config/PointStatConfig_prob @@ -6,7 +6,7 @@ // //////////////////////////////////////////////////////////////////////////////// -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_python b/internal/test_unit/config/PointStatConfig_python index 13e736396e..5116179451 100644 --- a/internal/test_unit/config/PointStatConfig_python +++ b/internal/test_unit/config/PointStatConfig_python @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_qty_inc_exc b/internal/test_unit/config/PointStatConfig_qty_inc_exc index 7939151307..1eab2b184c 100644 --- a/internal/test_unit/config/PointStatConfig_qty_inc_exc +++ b/internal/test_unit/config/PointStatConfig_qty_inc_exc @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/PointStatConfig_sid_inc_exc b/internal/test_unit/config/PointStatConfig_sid_inc_exc index 2705529d04..a586064f53 100644 --- a/internal/test_unit/config/PointStatConfig_sid_inc_exc +++ b/internal/test_unit/config/PointStatConfig_sid_inc_exc @@ -6,7 +6,7 @@ // //////////////////////////////////////////////////////////////////////////////// -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/TCDiagConfig_ian b/internal/test_unit/config/TCDiagConfig_ian index 19f2fac84d..98e9fdd114 100644 --- a/internal/test_unit/config/TCDiagConfig_ian +++ b/internal/test_unit/config/TCDiagConfig_ian @@ -45,9 +45,14 @@ valid_exc = []; // // Subset by the valid hour and lead time. +// By default, compute diagnostics every 6 hours out to 126 hours. // valid_hour = []; -lead = [ "0", "6", "12", "18", "24" ]; +lead = [ "0", "6", "12", "18", "24", + "30", "36", "42", "48", "54", + "60", "66", "72", "78", "84", + "90", "96", "102", "108", "114", + "120", "126" ]; //////////////////////////////////////////////////////////////////////////////// @@ -128,6 +133,11 @@ vortex_removal = FALSE; //////////////////////////////////////////////////////////////////////////////// +// +// Flag to control input files +// +one_time_per_file_flag = TRUE; + // // Flags to control output files // diff --git a/internal/test_unit/config/WaveletStatConfig b/internal/test_unit/config/WaveletStatConfig index 66c9fb64ab..d577cdc721 100644 --- a/internal/test_unit/config/WaveletStatConfig +++ b/internal/test_unit/config/WaveletStatConfig @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/WaveletStatConfig_no_thresholds b/internal/test_unit/config/WaveletStatConfig_no_thresholds index 8d51a464d8..ccc5787736 100644 --- a/internal/test_unit/config/WaveletStatConfig_no_thresholds +++ b/internal/test_unit/config/WaveletStatConfig_no_thresholds @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/WaveletStatConfig_python b/internal/test_unit/config/WaveletStatConfig_python index 980c80a2dd..30d963e0d0 100644 --- a/internal/test_unit/config/WaveletStatConfig_python +++ b/internal/test_unit/config/WaveletStatConfig_python @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/internal/test_unit/config/WaveletStatConfig_python_mixed b/internal/test_unit/config/WaveletStatConfig_python_mixed index 1dda90d7ca..d496ebb2df 100644 --- a/internal/test_unit/config/WaveletStatConfig_python_mixed +++ b/internal/test_unit/config/WaveletStatConfig_python_mixed @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/scripts/config/EnsembleStatConfig b/scripts/config/EnsembleStatConfig index 52309f857b..bc84b81d8e 100644 --- a/scripts/config/EnsembleStatConfig +++ b/scripts/config/EnsembleStatConfig @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/scripts/config/GenEnsProdConfig b/scripts/config/GenEnsProdConfig index 27fc32b618..74350a328d 100644 --- a/scripts/config/GenEnsProdConfig +++ b/scripts/config/GenEnsProdConfig @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/scripts/config/GridStatConfig_APCP_12 b/scripts/config/GridStatConfig_APCP_12 index bdbfb78ca2..2358000a81 100644 --- a/scripts/config/GridStatConfig_APCP_12 +++ b/scripts/config/GridStatConfig_APCP_12 @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/scripts/config/GridStatConfig_APCP_24 b/scripts/config/GridStatConfig_APCP_24 index fcd4a0611c..39eaa7d220 100644 --- a/scripts/config/GridStatConfig_APCP_24 +++ b/scripts/config/GridStatConfig_APCP_24 @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/scripts/config/GridStatConfig_POP_12 b/scripts/config/GridStatConfig_POP_12 index 71bb4ed9dd..6fb4775939 100644 --- a/scripts/config/GridStatConfig_POP_12 +++ b/scripts/config/GridStatConfig_POP_12 @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/scripts/config/GridStatConfig_all b/scripts/config/GridStatConfig_all index 8f0d58d274..a165ef836d 100644 --- a/scripts/config/GridStatConfig_all +++ b/scripts/config/GridStatConfig_all @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/scripts/config/MODEConfig_APCP_12 b/scripts/config/MODEConfig_APCP_12 index 733076d4fc..01976934a1 100644 --- a/scripts/config/MODEConfig_APCP_12 +++ b/scripts/config/MODEConfig_APCP_12 @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/scripts/config/MODEConfig_APCP_24 b/scripts/config/MODEConfig_APCP_24 index 3950a6b316..baeb2f27cc 100644 --- a/scripts/config/MODEConfig_APCP_24 +++ b/scripts/config/MODEConfig_APCP_24 @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/scripts/config/MODEConfig_RH b/scripts/config/MODEConfig_RH index dcdd115ae6..8448ec4068 100644 --- a/scripts/config/MODEConfig_RH +++ b/scripts/config/MODEConfig_RH @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/scripts/config/PointStatConfig b/scripts/config/PointStatConfig index ee4915fdc3..3c79f15262 100644 --- a/scripts/config/PointStatConfig +++ b/scripts/config/PointStatConfig @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/scripts/config/WaveletStatConfig_APCP_12 b/scripts/config/WaveletStatConfig_APCP_12 index 51b1d84246..fab8eb940a 100644 --- a/scripts/config/WaveletStatConfig_APCP_12 +++ b/scripts/config/WaveletStatConfig_APCP_12 @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/scripts/config/WaveletStatConfig_APCP_12_NC b/scripts/config/WaveletStatConfig_APCP_12_NC index 2e0fb2efea..2a11427926 100644 --- a/scripts/config/WaveletStatConfig_APCP_12_NC +++ b/scripts/config/WaveletStatConfig_APCP_12_NC @@ -9,7 +9,7 @@ // // Output model name to be written // -model = "WRF"; +model = "FCST"; // // Output description to be written diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index 34da078eb1..bd3ffe0219 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -1206,18 +1206,19 @@ static const char conf_key_radial_velocity_field_name[] = "radial_velocity_field static const char conf_key_tangential_velocity_field_name[] = "tangential_velocity_field_name"; static const char conf_key_radial_velocity_long_field_name[] = "radial_velocity_long_field_name"; static const char conf_key_tangential_velocity_long_field_name[] = "tangential_velocity_long_field_name"; -static const char conf_key_vortex_removal[] = "vortex_removal"; // // TC-Diag specific parameter key names // -static const char conf_key_domain_info[] = "domain_info"; -static const char conf_key_domain[] = "domain"; -static const char conf_key_diag_script[] = "diag_script"; -static const char conf_key_nc_rng_azi_flag[] = "nc_rng_azi_flag"; -static const char conf_key_nc_diag_flag[] = "nc_diag_flag"; -static const char conf_key_cira_diag_flag[] = "cira_diag_flag"; +static const char conf_key_domain_info[] = "domain_info"; +static const char conf_key_domain[] = "domain"; +static const char conf_key_diag_script[] = "diag_script"; +static const char conf_key_vortex_removal[] = "vortex_removal"; +static const char conf_key_one_time_per_file_flag[] = "one_time_per_file_flag"; +static const char conf_key_nc_rng_azi_flag[] = "nc_rng_azi_flag"; +static const char conf_key_nc_diag_flag[] = "nc_diag_flag"; +static const char conf_key_cira_diag_flag[] = "cira_diag_flag"; // // Parameter value names common to multiple tools diff --git a/src/basic/vx_log/logger.cc b/src/basic/vx_log/logger.cc index 5c531468d7..6e8785e041 100644 --- a/src/basic/vx_log/logger.cc +++ b/src/basic/vx_log/logger.cc @@ -37,8 +37,11 @@ static const int ErrorMessageLevel = -1; static const int WarningMessageLevel = 0; +static const bool DefaultPrintWarning = true; + static const bool DefaultExitOnWarning = false; + // // these need external linkage, do not make static or extern // @@ -217,6 +220,8 @@ LoggerWarning::LoggerWarning() { warn = WarningMessageLevel; + PrintWarning = DefaultPrintWarning; + ExitOnWarning = DefaultExitOnWarning; NeedToExit = false; @@ -246,6 +251,8 @@ LoggerWarning::LoggerWarning(const LoggerWarning & lw) { warn = lw.warn; + PrintWarning = lw.PrintWarning; + ExitOnWarning = lw.ExitOnWarning; NeedToExit = lw.NeedToExit; @@ -259,6 +266,8 @@ LoggerWarning & LoggerWarning::operator=(const LoggerWarning & lw) { warn = lw.warn; + PrintWarning = lw.PrintWarning; + ExitOnWarning = lw.ExitOnWarning; NeedToExit = lw.NeedToExit; @@ -491,6 +500,17 @@ void Logger::dump(ostream & dump_out, int depth) const ////////////////////////////////////////////////////////////////// +bool Logger::print_warning() const +{ + + return (Warning.PrintWarning); + +} + + +////////////////////////////////////////////////////////////////// + + void Logger::set_verbosity_level(const int i) { // @@ -509,6 +529,19 @@ void Logger::set_verbosity_level(const int i) ////////////////////////////////////////////////////////////////// +void Logger::set_print_warning(bool b) +{ + // + // if false, do not print warning messages + // + Warning.PrintWarning = b; + +} + + +////////////////////////////////////////////////////////////////// + + void Logger::set_exit_on_warning(bool b) { // @@ -519,7 +552,6 @@ void Logger::set_exit_on_warning(bool b) } - ////////////////////////////////////////////////////////////////// @@ -1604,7 +1636,10 @@ Logger & Logger::operator<<(const LoggerError e) Logger & Logger::operator<<(const LoggerWarning w) { - (*this) << level(WarningMessageLevel); + if (Warning.PrintWarning) + { + (*this) << level(WarningMessageLevel); + } if (Warning.ExitOnWarning) Warning.NeedToExit = true; diff --git a/src/basic/vx_log/logger.h b/src/basic/vx_log/logger.h index 529ffefc1b..3c431e53e4 100644 --- a/src/basic/vx_log/logger.h +++ b/src/basic/vx_log/logger.h @@ -115,6 +115,7 @@ class LoggerWarning { public: int warn; + bool PrintWarning; bool ExitOnWarning; bool NeedToExit; @@ -248,6 +249,8 @@ class Logger int verbosity_level() const; + bool print_warning() const; + bool is_open() const; // @@ -256,6 +259,8 @@ class Logger void set_verbosity_level(const int); + void set_print_warning(bool); + void set_exit_on_warning(bool); // diff --git a/src/libcode/vx_data2d/data_class.cc b/src/libcode/vx_data2d/data_class.cc index c9ab09ee7e..b44b94351e 100644 --- a/src/libcode/vx_data2d/data_class.cc +++ b/src/libcode/vx_data2d/data_class.cc @@ -309,6 +309,49 @@ return; //////////////////////////////////////////////////////////////////////// +int Met2dDataFile::data_planes(vector &vi_list, + vector &dp_list) + +{ + +int n_valid = 0; +DataPlane cur_dp; + + // + // Loop over requested VarInfo objects + // + +for(int i=0; i&, + std::vector&); + // post-process data after reading it bool process_data_plane(VarInfo *, DataPlane &); diff --git a/src/libcode/vx_data2d_grib/data2d_grib.cc b/src/libcode/vx_data2d_grib/data2d_grib.cc index 98720ea120..9ac709ef3a 100644 --- a/src/libcode/vx_data2d_grib/data2d_grib.cc +++ b/src/libcode/vx_data2d_grib/data2d_grib.cc @@ -827,3 +827,5 @@ int MetGrib1DataFile::index(VarInfo &vinfo){ return rec; } + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d_grib/data2d_grib.h b/src/libcode/vx_data2d_grib/data2d_grib.h index 563c09fb4f..ad14d6e8a3 100644 --- a/src/libcode/vx_data2d_grib/data2d_grib.h +++ b/src/libcode/vx_data2d_grib/data2d_grib.h @@ -84,10 +84,16 @@ class MetGrib1DataFile : public Met2dDataFile { int read_record( VarInfoGrib &); // returns match count (>=0), or -1 on error + // retrieve the first matching data plane + bool data_plane(VarInfo &, DataPlane &); + // retrieve all matching data planes + int data_plane_array(VarInfo &, DataPlaneArray &); - + + // retrieve the index of the first matching record + int index(VarInfo &); }; diff --git a/src/libcode/vx_data2d_grib2/data2d_grib2.cc b/src/libcode/vx_data2d_grib2/data2d_grib2.cc index 6a3f40574d..f86516c2b7 100644 --- a/src/libcode/vx_data2d_grib2/data2d_grib2.cc +++ b/src/libcode/vx_data2d_grib2/data2d_grib2.cc @@ -239,8 +239,8 @@ bool MetGrib2DataFile::data_plane(VarInfo &vinfo, DataPlane &plane) { //////////////////////////////////////////////////////////////////////// -int MetGrib2DataFile::data_plane_array( VarInfo &vinfo, - DataPlaneArray &plane_array ){ +int MetGrib2DataFile::data_plane_array(VarInfo &vinfo, + DataPlaneArray &plane_array) { // Initialize plane_array.clear(); @@ -357,10 +357,9 @@ int MetGrib2DataFile::data_plane_array( VarInfo &vinfo, //////////////////////////////////////////////////////////////////////// -void MetGrib2DataFile::find_record_matches( VarInfoGrib2* vinfo, - vector &listMatchExact, - vector &listMatchRange - ){ +void MetGrib2DataFile::find_record_matches(VarInfoGrib2* vinfo, + vector &listMatchExact, + vector &listMatchRange) { const char *method_name = "MetGrib2DataFile::find_record_matches() -> "; @@ -955,7 +954,7 @@ void MetGrib2DataFile::read_grib2_record_list() { //////////////////////////////////////////////////////////////////////// -void MetGrib2DataFile::read_grib2_grid( gribfield *gfld) { +void MetGrib2DataFile::read_grib2_grid(gribfield *gfld) { double d, r_km; int ResCompFlag; @@ -1416,7 +1415,7 @@ void MetGrib2DataFile::read_grib2_grid( gribfield *gfld) { //////////////////////////////////////////////////////////////////////// bool MetGrib2DataFile::read_grib2_record_data_plane(Grib2Record *rec, - DataPlane &plane) { + DataPlane &plane) { // attempt to read the record gribfield *gfld; diff --git a/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc b/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc index dc7d59be00..83872f4614 100644 --- a/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc +++ b/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc @@ -310,3 +310,5 @@ int MetNcPinterpDataFile::data_plane_array(VarInfo &vinfo, int MetNcPinterpDataFile::index(VarInfo &vinfo){ return -1; } + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d_nccf/data2d_nccf.h b/src/libcode/vx_data2d_nccf/data2d_nccf.h index d9a9a2df9c..e6ad78b1eb 100644 --- a/src/libcode/vx_data2d_nccf/data2d_nccf.h +++ b/src/libcode/vx_data2d_nccf/data2d_nccf.h @@ -92,7 +92,6 @@ class MetNcCFDataFile : public Met2dDataFile { int index(VarInfo &); - // // do stuff // diff --git a/src/libcode/vx_data2d_python/data2d_python.cc b/src/libcode/vx_data2d_python/data2d_python.cc index 9251fb0d21..8aa4d33ba3 100644 --- a/src/libcode/vx_data2d_python/data2d_python.cc +++ b/src/libcode/vx_data2d_python/data2d_python.cc @@ -506,4 +506,3 @@ return ( 0 ); //////////////////////////////////////////////////////////////////////// - diff --git a/src/libcode/vx_series_data/series_data.cc b/src/libcode/vx_series_data/series_data.cc index d7a0e587d9..581ea7ff50 100644 --- a/src/libcode/vx_series_data/series_data.cc +++ b/src/libcode/vx_series_data/series_data.cc @@ -8,6 +8,8 @@ //////////////////////////////////////////////////////////////////////// +using namespace std; + #include #include #include @@ -20,12 +22,27 @@ //////////////////////////////////////////////////////////////////////// -void get_series_entry(int i_series, VarInfo* data_info, +static bool read_single_entry(VarInfo*, const ConcatString&, const GrdFileType, + DataPlane&, Grid&); + +static bool read_all_entries(vector&, const ConcatString&, const GrdFileType, + vector&, Grid&); + +//////////////////////////////////////////////////////////////////////// + +bool get_series_entry(int i_series, VarInfo* data_info, const StringArray& search_files, const GrdFileType type, - DataPlane& dp, Grid& grid) { + DataPlane& dp, Grid& grid, + bool error_out, bool print_warning) { int i; bool found; + // Save the log print warning state + bool save_print_warning_state = mlog.print_warning(); + + // Suppress log warnings, if requested + if(!print_warning) mlog.set_print_warning(false); + mlog << Debug(3) << "Processing series entry " << i_series + 1 << ": " << data_info->magic_time_str() << "\n"; @@ -41,19 +58,23 @@ void get_series_entry(int i_series, VarInfo* data_info, found = read_single_entry(data_info, search_files[i_cur], type, dp, grid); + // Break out of the loop if data was found if(found) break; } // end for i - // Error out if not found - if(!found) { + // Error out if not found and specified + if(!found && error_out) { mlog << Error << "\nget_series_entry() -> " << "Could not find data for " << data_info->magic_time_str() - << " in file list:\n:" << write_css(search_files) << "\n\n"; + << " in file list:\n" << write_css(search_files) << "\n\n"; exit(1); } - return; + // Restore warnings to their original state + mlog.set_print_warning(save_print_warning_state); + + return(found); } //////////////////////////////////////////////////////////////////////// @@ -66,9 +87,9 @@ bool read_single_entry(VarInfo* info, const ConcatString& filename, // Check that file exists if(!file_exists(filename.c_str())) { - mlog << Warning << "\nread_single_entry() -> " - << "File does not exist: " << filename << "\n\n"; - return(false); + mlog << Warning << "\nread_single_entry() -> " + << "File does not exist: " << filename << "\n\n"; + return(false); } // Open data file @@ -87,3 +108,85 @@ bool read_single_entry(VarInfo* info, const ConcatString& filename, } //////////////////////////////////////////////////////////////////////// + +bool get_series_entries(int i_series, vector &vi_list, + const StringArray &search_files, const GrdFileType type, + vector &dp_list, Grid &grid, + bool error_out, bool print_warning) { + int i; + bool found; + + // Check for at least one field requested + if(vi_list.size() <= 0) return(false); + + // Save the log print warning state + bool save_print_warning_state = mlog.print_warning(); + + // Suppress log warnings, if requested + if(!print_warning) mlog.set_print_warning(false); + + mlog << Debug(3) + << "Processing series entry " << i_series + 1 << ": " + << vi_list[0]->magic_time_str() << "\n"; + + // Initialize + dp_list.clear(); + + // Search for data, beginning with the i_series index + for(i=0,found=false; i " + << "Could not find data for " << vi_list[0]->magic_time_str() + << " in file list:\n" << write_css(search_files) << "\n\n"; + exit(1); + } + + // Restore warnings to their original state + mlog.set_print_warning(save_print_warning_state); + + return(found); +} + +//////////////////////////////////////////////////////////////////////// + +bool read_all_entries(vector &vi_list, const ConcatString &filename, + const GrdFileType type, vector &dp_list, Grid &grid) { + + Met2dDataFileFactory mtddf_factory; + Met2dDataFile* mtddf = (Met2dDataFile*) 0; + + // Check that file exists + if(!file_exists(filename.c_str())) { + mlog << Warning << "\nread_all_entries() -> " + << "File does not exist: " << filename << "\n\n"; + return(false); + } + + // Open data file + mtddf = mtddf_factory.new_met_2d_data_file(filename.c_str(), type); + + // Attempt to read gridded data + int n_valid = mtddf->data_planes(vi_list, dp_list); + + // Store grid + if(n_valid > 0) grid = mtddf->grid(); + + // Cleanup + if(mtddf) { delete mtddf; mtddf = (Met2dDataFile *) 0; } + + return(n_valid > 0); +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_series_data/series_data.h b/src/libcode/vx_series_data/series_data.h index d809b64edc..fd84a5ea86 100644 --- a/src/libcode/vx_series_data/series_data.h +++ b/src/libcode/vx_series_data/series_data.h @@ -13,6 +13,8 @@ //////////////////////////////////////////////////////////////////////// +#include + #include "vx_util.h" #include "vx_log.h" #include "vx_data2d.h" @@ -21,11 +23,15 @@ //////////////////////////////////////////////////////////////////////// -void get_series_entry(int, VarInfo*, const StringArray&, - const GrdFileType, DataPlane&, Grid&); +bool get_series_entry(int, VarInfo*, const StringArray&, + const GrdFileType, DataPlane&, Grid&, + bool error_out=true, + bool print_warning=true); -bool read_single_entry(VarInfo*, const ConcatString&, const GrdFileType, - DataPlane&, Grid&); +bool get_series_entries(int, std::vector&, const StringArray&, + const GrdFileType, std::vector&, Grid&, + bool error_out=true, + bool print_warning=true); //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/tc_utils/tc_diag/tc_diag.cc b/src/tools/tc_utils/tc_diag/tc_diag.cc index f6ba1926f4..05e8471957 100644 --- a/src/tools/tc_utils/tc_diag/tc_diag.cc +++ b/src/tools/tc_utils/tc_diag/tc_diag.cc @@ -15,6 +15,7 @@ // Mod# Date Name Description // ---- ---- ---- ----------- // 000 09/27/22 Halley Gotway New +// 001 08/17/23 Halley Gotway MET #2609 handle missing data // //////////////////////////////////////////////////////////////////////// @@ -120,7 +121,7 @@ static void compute_lat_lon(TcrmwGrid&, double *, double *); // - Write CIRA ASCII and NetCDF diagnostics output files. // - Add support for $MET_PYTHON_EXE. // - Input data: -// - Instead of reading DataPlanes one at a time, +// - [DONE for #2609] Instead of reading DataPlanes one at a time, // read them all at once or perhaps in groups // (e.g. all pressure levels). // - Parellelize the processing of valid times. @@ -343,6 +344,10 @@ void process_tracks(TrackInfoArray& tracks) { << unix_to_yyyymmddhh(init_ta.max()) << ".\n\n"; exit(1); } + // Store the initialization time + else { + init_ut = init_ta[0]; + } return; } @@ -612,6 +617,14 @@ void set_data(const StringArray& a) { for(int i=2; i " + << "File does not exist: " << info.data_files[i] << "\n\n"; + } + } + // Store the data in the map if(data_opt_map.count(domain) == 0) data_opt_map[domain] = info; else data_opt_map[domain] += info; @@ -956,33 +969,80 @@ void process_track_points(const TrackInfoArray& tracks) { } //////////////////////////////////////////////////////////////////////// +/// + // TODO: Consider adding vortex removal logic here + // Read in the full set of fields required for vortex removal + // Add flag to configure which fields are used for vortex removal void process_fields(const TrackInfoArray &tracks, const unixtime vld_ut, int i_vld, const string &domain, const DomainInfo &di) { int i, j, i_pnt; - DataPlane data_dp; Grid grid_dp; VarInfoFactory vi_factory; VarInfo *vi = (VarInfo *) 0; - StringArray tmp_key_sa; + vector vi_list; + DataPlane dp; + vector dp_list; + StringArray tmp_key_sa, fields_missing; - // TODO: Consider adding vortex removal logic here - // Read in the full set of fields required for vortex removal - // Add flag to configure which fields are used for vortex removal - - // Loop over the VarInfo fields to be processed + // Create vector of VarInfo objects for(i=0; iset_valid(vld_ut); + vi_list.push_back(vi); + } - // Find data for this track point - get_series_entry(i_vld, vi, + // Read all data at the same time if they are all in the same file + if(conf_info.one_time_per_file_flag) { + + // Find all entries for this track point + bool status = get_series_entries(i_vld, vi_list, di.data_files, file_type, - data_dp, grid_dp); + dp_list, grid_dp, + false, false); + + } + // Otherwise, read data one field at a time + else { + + // Loop over the VarInfo fields to be processed + for(i=0; imagic_str()); + + // Store the requested valid time + dp_list[i].set_valid(vld_ut); + + } + } // end for i + + // Loop over the VarInfo fields to be processed + for(i=0; i 0) { + mlog << Warning << "For the " + << domain << " domain, " + << sec_to_hhmmss(vld_ut - init_ut) << " lead time, and " + << unix_to_yyyymmdd_hhmmss(vld_ut) << " valid time, " + << fields_missing.n() << " of " << di.var_info_ptr.size() + << " requested fields missing:\n" + << write_css(fields_missing) << "\n"; + } + // Loop over the current set of temp files for(i=0; i 0) { + dp_out = met_regrid(dp_in, grid_in, grid_out, ri); + } + // Handle empty input fields + else { + dp_out.set_valid(dp_in.valid()); + dp_out.set_size(ra_grid.range_n(), ra_grid.azimuth_n(), + bad_data_double); + } // Logic for pressure level data bool is_prs = (vi->level().type() == LevelType_Pres); diff --git a/src/tools/tc_utils/tc_diag/tc_diag.h b/src/tools/tc_utils/tc_diag/tc_diag.h index 2c79d5f8b2..b768b93864 100644 --- a/src/tools/tc_utils/tc_diag/tc_diag.h +++ b/src/tools/tc_utils/tc_diag/tc_diag.h @@ -91,6 +91,8 @@ static ConcatString out_prefix; // //////////////////////////////////////////////////////////////////////// +static unixtime init_ut = (unixtime) 0; + class OutFileInfo { private: diff --git a/src/tools/tc_utils/tc_diag/tc_diag_conf_info.cc b/src/tools/tc_utils/tc_diag/tc_diag_conf_info.cc index 62c8f374c8..97d14d9e1f 100644 --- a/src/tools/tc_utils/tc_diag/tc_diag_conf_info.cc +++ b/src/tools/tc_utils/tc_diag/tc_diag_conf_info.cc @@ -186,6 +186,7 @@ void TCDiagConfInfo::clear() { domain_info.clear(); vortex_removal_flag = false; + one_time_per_file_flag = true; nc_rng_azi_flag = false; nc_diag_flag = false; @@ -327,6 +328,10 @@ void TCDiagConfInfo::process_config(GrdFileType file_type, vortex_removal_flag = conf.lookup_bool(conf_key_vortex_removal); + // Conf: one_time_per_file_flag + one_time_per_file_flag = + conf.lookup_bool(conf_key_one_time_per_file_flag); + // Conf: nc_rng_azi_flag nc_rng_azi_flag = conf.lookup_bool(conf_key_nc_rng_azi_flag); diff --git a/src/tools/tc_utils/tc_diag/tc_diag_conf_info.h b/src/tools/tc_utils/tc_diag/tc_diag_conf_info.h index 3c2d0b5132..ff0bbc6c6e 100644 --- a/src/tools/tc_utils/tc_diag/tc_diag_conf_info.h +++ b/src/tools/tc_utils/tc_diag/tc_diag_conf_info.h @@ -119,9 +119,12 @@ class TCDiagConfInfo { // Vector of DomainInfo std::vector domain_info; - // Vortext removal settings + // Vortex removal settings bool vortex_removal_flag; + // One time per file settings + bool one_time_per_file_flag; + // Directory for temporary files ConcatString tmp_dir; From bb90592fb8eb3fc33061b2bcc40fbd07c7c035b9 Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Tue, 10 Oct 2023 12:14:22 -0600 Subject: [PATCH 02/12] Feature 2669 proj comp (#2710) * Per #2669, make adjustments to get MET to compile on Jet and Hera * Removed beta1 * Removed beta1 * Update internal/scripts/installation/compile_MET_all.sh Co-authored-by: John Halley Gotway * Per #2669, update SQLITE installation * Per #2669, adding in missing then keyword * Per #2669, fixing typo in value for SQLITE_LIB_DIR * Per #2669, adding SQLITE_INCLUDE_DIR and SQLITE_LIB_DIR * Per #2669, updated installation files for SQLITE changes * Update internal/scripts/installation/compile_MET_all.sh Co-authored-by: John Halley Gotway * Update internal/scripts/installation/compile_MET_all.sh Co-authored-by: John Halley Gotway --------- Co-authored-by: John Halley Gotway --- .../scripts/environment/development.docker | 3 ++ .../scripts/installation/compile_MET_all.sh | 48 ++++++++++++++----- .../installation/config/install_met_env.hera | 5 ++ .../installation/config/install_met_env.jet | 23 ++++++--- .../config/install_met_env_met_only.hera | 5 ++ .../config/install_met_env_met_only.jet | 25 ++++++---- .../installation/modulefiles/11.1.0_jet | 19 -------- .../installation/modulefiles/12.0.0_jet | 17 +++++++ 8 files changed, 98 insertions(+), 47 deletions(-) delete mode 100644 internal/scripts/installation/modulefiles/11.1.0_jet create mode 100644 internal/scripts/installation/modulefiles/12.0.0_jet diff --git a/internal/scripts/environment/development.docker b/internal/scripts/environment/development.docker index b92adf7f14..5011adabe3 100644 --- a/internal/scripts/environment/development.docker +++ b/internal/scripts/environment/development.docker @@ -40,3 +40,6 @@ export COMPILE_HDF=1 export COMPILE_HDFEOS=1 export COMPILE_FREETYPE=1 export COMPILE_CAIRO=1 + +export SQLITE_INCLUDE_DIR=/usr/include +export SQLITE_LIB_DIR=/usr/lib/x86_64-linux-gnu diff --git a/internal/scripts/installation/compile_MET_all.sh b/internal/scripts/installation/compile_MET_all.sh index ecabc9dc49..59fb4f3dcf 100644 --- a/internal/scripts/installation/compile_MET_all.sh +++ b/internal/scripts/installation/compile_MET_all.sh @@ -36,11 +36,11 @@ # # The compile_MET_all.sh script will compile and install MET and its # external library dependencies, if needed, including: -# PROJ, GSL, BUFRLIB, GRIB2C (with dependencies Z, PNG, JASPER), -# HDF5, NETCDF (C and CXX), HDF4 (optional for MODIS-Regrid -# and lidar2nc), HDFEOS (optional for MODIS-Regrid and lidar2nc), -# FREETYPE (optional for MODE Graphics), and CAIRO (optional -# for MODE Graphics). +# PROJ (with dependency SQLITE >= 3.11), GSL, BUFRLIB, +# GRIB2C (with dependencies Z, PNG, JASPER), HDF5, NETCDF (C and CXX), +# HDF4 (optional for MODIS-Regrid and lidar2nc), HDFEOS (optional for +# MODIS-Regrid and lidar2nc), FREETYPE (optional for MODE Graphics), +# and CAIRO (optional for MODE Graphics). # # If these libraries have already been installed and don't need to be # reinstalled or if you are compiling on a machine that uses modulefiles @@ -49,9 +49,9 @@ # need to set to let MET know where the library and header files are. # Please supply values for the following environment variables # in the input environment configuration file (install_met_env.: -# MET_GRIB2CLIB, MET_GRIB2CINC, GRIB2CLIB_NAME, -# MET_BUFRLIB, BUFRLIB_NAME, MET_HDF5, MET_NETCDF, -# MET_PROJ, MET_GSL, LIB_JASPER, LIB_PNG, LIB_Z. +# MET_GRIB2CLIB, MET_GRIB2CINC, GRIB2CLIB_NAME, MET_BUFRLIB, BUFRLIB_NAME, +# MET_HDF5, MET_NETCDF, MET_PROJ, MET_GSL, LIB_JASPER, LIB_PNG, LIB_Z, +# SQLITE_INCLUDE_DIR, SQLITE_LIB_DIR. # # The optional libraries HDF4, HDFEOS, FREETYPE, and CAIRO are # used for the following, not widely used tools, MODIS-Regrid, @@ -155,6 +155,13 @@ if [[ -z "$LIB_Z" ]]; then LIB_Z=${LIB_DIR}/lib fi +# if SQLITE is not defined in the environment file, enable its compilation +if [[ -z ${SQLITE_INCLUDE_DIR} ]] && [[ -z ${SQLITE_LIB_DIR} ]]; then + COMPILE_SQLITE=1 +else + COMPILE_SQLITE=0 +fi + # Constants if [[ -z ${MET_GRIB2CLIB} ]] && [[ -z ${MET_GRIB2C} ]]; then COMPILE_ZLIB=1 @@ -394,6 +401,21 @@ fi # Compile Proj if [ $COMPILE_PROJ -eq 1 ]; then + if [ $COMPILE_SQLITE -eq 1 ]; then + echo + echo "Compiling SQLITE at `date`" + mkdir -p ${LIB_DIR}/sqlite + rm -rf ${LIB_DIR}/sqlite/sqlite* + tar -xf ${TAR_DIR}/sqlite*.tar.gz -C ${LIB_DIR}/sqlite > /dev/null 2>&1 + cd ${LIB_DIR}/sqlite/sqlite* + echo "cd `pwd`" + run_cmd "./configure --enable-shared --prefix=${LIB_DIR} > sqlite.configure.log 2>&1" + run_cmd "make ${MAKE_ARGS} > sqlite.make.log 2>&1" + run_cmd "make ${MAKE_ARGS} install > sqlite.make_install.log 2>&1" + export SQLITE_INCLUDE_DIR=${LIB_DIR}/include + export SQLITE_LIB_DIR=${LIB_DIR}/lib + fi + vrs="9.2.1"; echo @@ -403,8 +425,9 @@ if [ $COMPILE_PROJ -eq 1 ]; then tar -xf ${TAR_DIR}/proj-${vrs}.tar.gz -C ${LIB_DIR}/proj cd ${LIB_DIR}/proj/proj* echo "cd `pwd`" + export PATH=${LIB_DIR}/bin:${PATH} run_cmd "mkdir build; cd build" - run_cmd "cmake -DCMAKE_INSTALL_PREFIX=${LIB_DIR} .." + run_cmd "cmake -DCMAKE_INSTALL_PREFIX=${LIB_DIR} -DSQLITE3_INCLUDE_DIR=${SQLITE_INCLUDE_DIR} -DSQLITE3_LIBRARY=${SQLITE_LIB_DIR}/libsqlite3.so .." run_cmd "cmake --build ." run_cmd "cmake --build . --target install" @@ -716,8 +739,7 @@ if [ -z ${MET_GSL} ]; then fi if [ -z ${MET_PROJ} ]; then - export MET_PROJINC=${LIB_DIR}/include - export MET_PROJLIB=${LIB_DIR}/lib64 + export MET_PROJ=${LIB_DIR} fi export MET_PYTHON_BIN_EXE=${MET_PYTHON_BIN_EXE:=${MET_PYTHON}/bin/python3} @@ -732,10 +754,10 @@ fi # https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html # ${parameter:+word} # If parameter is null or unset, nothing is substituted, otherwise the expansion of word is substituted. -export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_DIR}/lib${MET_PROJ:+:$MET_PROJ/lib64}${LIB_DIR}/lib${MET_NETCDF:+:$MET_NETCDF/lib}${MET_HDF5:+:$MET_HDF5/lib}${MET_BUFRLIB:+:$MET_BUFRLIB}${MET_GRIB2CLIB:+:$MET_GRIB2CLIB}${MET_PYTHON_LIB:+:$MET_PYTHON_LIB}${MET_GSL:+:$MET_GSL/lib}${ADDTL_DIR:+:$ADDTL_DIR}" +export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_DIR}/lib:${MET_PROJ:+:$MET_PROJ/lib64}${LIB_DIR}/lib${MET_NETCDF:+:$MET_NETCDF/lib}${MET_HDF5:+:$MET_HDF5/lib}${MET_BUFRLIB:+:$MET_BUFRLIB}${MET_GRIB2CLIB:+:$MET_GRIB2CLIB}${MET_PYTHON_LIB:+:$MET_PYTHON_LIB}${MET_GSL:+:$MET_GSL/lib}${ADDTL_DIR:+:$ADDTL_DIR}" export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_JASPER:+$LIB_JASPER}${LIB_LIBPNG:+:$LIB_PNG}${LIB_Z:+$LIB_Z}" export LDFLAGS="${LDFLAGS} ${LIB_JASPER:+-L$LIB_JASPER} ${LIB_LIBPNG:+-L$LIB_LIBPNG} ${MET_HDF5:+-L$MET_HDF5/lib} ${ADDTL_DIR:+-L$ADDTL_DIR}" -export LIBS="${LIBS} -lhdf5_hl -lhdf5 -lz" +export LIBS="${LIBS} -lhdf5_hl -lhdf5 -lz -ltiff" export MET_FONT_DIR=${TEST_BASE}/fonts if [ "${SET_D64BIT}" = "TRUE" ]; then diff --git a/internal/scripts/installation/config/install_met_env.hera b/internal/scripts/installation/config/install_met_env.hera index 2b30d35a67..2b7b6ed08f 100755 --- a/internal/scripts/installation/config/install_met_env.hera +++ b/internal/scripts/installation/config/install_met_env.hera @@ -1,5 +1,7 @@ +module load gnu/9.2.0 module load intel/2022.1.2 module load cmake/3.26.4 + export PATH=/scratch1/BMC/dtc/miniconda/miniconda3/envs/metplus_v5.1_py3.10/bin:${PATH} export TEST_BASE=/contrib/met/12.0.0 export COMPILER=intel_2022.1.2 @@ -10,6 +12,7 @@ export MET_PYTHON=/scratch1/BMC/dtc/miniconda/miniconda3/envs/metplus_v5.1_py3.1 export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.10 export MET_PYTHON_LD=`python3-config --ldflags --embed` export SET_D64BIT=FALSE +export ADDTL_DIR=/usr/lib64 export EXTERNAL_LIBS=/contrib/met/12.0.0/external_libs/ #export MET_PROJ=${EXTERNAL_LIBS} #export MET_NETCDF=${EXTERNAL_LIBS} @@ -23,6 +26,8 @@ export EXTERNAL_LIBS=/contrib/met/12.0.0/external_libs/ #export LIB_JASPER=${EXTERNAL_LIBS}/lib #export LIB_LIBPNG=${EXTERNAL_LIBS}/lib #export LIB_Z=${EXTERNAL_LIBS}/lib +#export SQLITE_INCLUDE_DIR=${EXTERNAL_LIBS}/include +#export SQLITE_LIB_DIR=${EXTERNAL_LIBS}/lib #export CFLAGS="-Wall -g" #export CXXFLAGS="-Wall -g -lcurl" export MAKE_ARGS=-j diff --git a/internal/scripts/installation/config/install_met_env.jet b/internal/scripts/installation/config/install_met_env.jet index 0f93cc0615..87442a55fe 100644 --- a/internal/scripts/installation/config/install_met_env.jet +++ b/internal/scripts/installation/config/install_met_env.jet @@ -1,27 +1,36 @@ +module load gnu/9.2.0 module load intel/2022.1.2 -module load netcdf/4.7.0 -module load hdf5/1.10.5 +module load cmake/3.26.4 -export TEST_BASE=/contrib/met/11.1.0 -export COMPILER=intel_18.0.5.274 +export FC=ifort +export F77=ifort +export F90=ifort +export CC=icc +export CXX=icpc +export TEST_BASE=/contrib/met/12.0.0 +export COMPILER=intel_2022.1.2 export MET_SUBDIR=${TEST_BASE} -export MET_TARBALL=v11.1.0.tar.gz +export MET_TARBALL=v12.0.0.tar.gz export USE_MODULES=TRUE export MET_PYTHON=/mnt/lfs1/HFIP/dtc-hurr/METplus/miniconda/miniconda3/envs/metplus_v5.1_py3.10 export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.10 export MET_PYTHON_LD=-L${MET_PYTHON}/lib/python3.10/config-3.10-x86_64-linux-gnu\ -L${MET_PYTHON}/lib\ -lpython3.10\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lrt\ -lm\ -lm -export MET_NETCDF=/apps/netcdf/4.7.0/intel/18.0.5.274 -export MET_HDF5=/apps/hdf5/1.10.5/intel/18.0.5.274 +export ADDTL_DIR=/usr/lib64 export EXTERNAL_LIBS=${TEST_BASE}/external_libs/ +#export MET_PROJ=${EXTERNAL_LIBS} #export MET_GSL=${EXTERNAL_LIBS} #export MET_BUFRLIB=${EXTERNAL_LIBS} #export BUFRLIB_NAME=-lbufr #export MET_GRIB2CLIB=${EXTERNAL_LIBS}/lib #export MET_GRIB2CINC=${EXTERNAL_LIBS}/include #export GRIB2CLIB_NAME=-lgrib2c +#export MET_HDF5=${EXTERNAL_LIBS} +#export MET_NETCDF=${EXTERNAL_LIBS} #export LIB_JASPER=${EXTERNAL_LIBS}/lib #export LIB_LIBPNG=${EXTERNAL_LIBS}/lib #export LIB_Z=${EXTERNAL_LIBS}/lib +#export SQLITE_INCLUDE_DIR=${EXTERNAL_LIBS}/include +#export SQLITE_LIB_DIR=${EXTERNAL_LIBS}/lib export MAKE_ARGS=-j export SET_D64BIT=FALSE export CXXFLAGS="-std=c++11" diff --git a/internal/scripts/installation/config/install_met_env_met_only.hera b/internal/scripts/installation/config/install_met_env_met_only.hera index 7666e92619..b8ac02a3e0 100644 --- a/internal/scripts/installation/config/install_met_env_met_only.hera +++ b/internal/scripts/installation/config/install_met_env_met_only.hera @@ -1,5 +1,7 @@ +module load gnu/9.2.0 module load intel/2022.1.2 module load cmake/3.26.4 + export PATH=/scratch1/BMC/dtc/miniconda/miniconda3/envs/metplus_v5.1_py3.10/bin:${PATH} export TEST_BASE=/contrib/met/12.0.0 export COMPILER=intel_2022.1.2 @@ -10,6 +12,7 @@ export MET_PYTHON=/scratch1/BMC/dtc/miniconda/miniconda3/envs/metplus_v5.1_py3.1 export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.10 export MET_PYTHON_LD=`python3-config --ldflags --embed` export SET_D64BIT=FALSE +export ADDTL_DIR=/usr/lib64 export EXTERNAL_LIBS=/contrib/met/12.0.0/external_libs/ export MET_PROJ=${EXTERNAL_LIBS} export MET_NETCDF=${EXTERNAL_LIBS} @@ -23,6 +26,8 @@ export GRIB2CLIB_NAME=-lgrib2c export LIB_JASPER=${EXTERNAL_LIBS}/lib export LIB_LIBPNG=${EXTERNAL_LIBS}/lib export LIB_Z=${EXTERNAL_LIBS}/lib +export SQLITE_INCLUDE_DIR=${EXTERNAL_LIBS}/include +export SQLITE_LIB_DIR=${EXTERNAL_LIBS}/lib #export CFLAGS="-Wall -g" #export CXXFLAGS="-Wall -g -lcurl" export MAKE_ARGS=-j diff --git a/internal/scripts/installation/config/install_met_env_met_only.jet b/internal/scripts/installation/config/install_met_env_met_only.jet index 99cf673403..00b9d689d2 100644 --- a/internal/scripts/installation/config/install_met_env_met_only.jet +++ b/internal/scripts/installation/config/install_met_env_met_only.jet @@ -1,27 +1,36 @@ +module load gnu/9.2.0 module load intel/2022.1.2 -module load netcdf/4.7.0 -module load hdf5/1.10.5 +module load cmake/3.26.4 -export TEST_BASE=/contrib/met/11.1.0 -export COMPILER=intel_18.0.5.274 +export FC=ifort +export F77=ifort +export F90=ifort +export CC=icc +export CXX=icpc +export TEST_BASE=/contrib/met/12.0.0 +export COMPILER=intel_2022.1.2 export MET_SUBDIR=${TEST_BASE} -export MET_TARBALL=v11.1.0.tar.gz +export MET_TARBALL=v12.0.0.tar.gz export USE_MODULES=TRUE export MET_PYTHON=/mnt/lfs1/HFIP/dtc-hurr/METplus/miniconda/miniconda3/envs/metplus_v5.1_py3.10 export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.10 -export MET_PYTHON_LD=`${MET_PYTHON}/bin/python3-config --ldflags --embed` -export MET_NETCDF=/apps/netcdf/4.7.0/intel/18.0.5.274 -export MET_HDF5=/apps/hdf5/1.10.5/intel/18.0.5.274 +export MET_PYTHON_LD=-L${MET_PYTHON}/lib/python3.10/config-3.10-x86_64-linux-gnu\ -L${MET_PYTHON}/lib\ -lpython3.10\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lrt\ -lm\ -lm +export ADDTL_DIR=/usr/lib64 export EXTERNAL_LIBS=${TEST_BASE}/external_libs/ +export MET_PROJ=${EXTERNAL_LIBS} export MET_GSL=${EXTERNAL_LIBS} export MET_BUFRLIB=${EXTERNAL_LIBS} export BUFRLIB_NAME=-lbufr export MET_GRIB2CLIB=${EXTERNAL_LIBS}/lib export MET_GRIB2CINC=${EXTERNAL_LIBS}/include export GRIB2CLIB_NAME=-lgrib2c +export MET_HDF5=${EXTERNAL_LIBS} +export MET_NETCDF=${EXTERNAL_LIBS} export LIB_JASPER=${EXTERNAL_LIBS}/lib export LIB_LIBPNG=${EXTERNAL_LIBS}/lib export LIB_Z=${EXTERNAL_LIBS}/lib +export SQLITE_INCLUDE_DIR=${EXTERNAL_LIBS}/include +export SQLITE_LIB_DIR=${EXTERNAL_LIBS}/lib export MAKE_ARGS=-j export SET_D64BIT=FALSE export CXXFLAGS="-std=c++11" diff --git a/internal/scripts/installation/modulefiles/11.1.0_jet b/internal/scripts/installation/modulefiles/11.1.0_jet deleted file mode 100644 index fb1190d8f7..0000000000 --- a/internal/scripts/installation/modulefiles/11.1.0_jet +++ /dev/null @@ -1,19 +0,0 @@ -#%Module###################################################################### -## -## Model Evaluation Tools -## -proc ModulesHelp { } { - puts stderr "Sets up the paths and environment variables to use the Model Evaluation Tools v11.1.0 - *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" -} - -prereq intel -prereq netcdf/4.7.0 -prereq hdf5/1.10.5 - -set base /contrib/met/11.1.0 -set ver 11.1.0 -set share $base/share/met -setenv MET_ROOT $base/$ver/MET-11.1.0 - -prepend-path PATH $base/bin:$base/external_libs/bin:/apps/netcdf/4.7.0/intel/18.0.5.274/bin:/apps/hdf5/1.10.5/intel_seq/18.0.5.274/bin:/mnt/lfs1/HFIP/dtc-hurr/METplus/miniconda/miniconda3/envs/metplus_v5.1_py3.10 diff --git a/internal/scripts/installation/modulefiles/12.0.0_jet b/internal/scripts/installation/modulefiles/12.0.0_jet new file mode 100644 index 0000000000..430bd27113 --- /dev/null +++ b/internal/scripts/installation/modulefiles/12.0.0_jet @@ -0,0 +1,17 @@ +#%Module###################################################################### +## +## Model Evaluation Tools +## +proc ModulesHelp { } { + puts stderr "Sets up the paths and environment variables to use the Model Evaluation Tools v12.0.0 + *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" +} + +prereq intel + +set base /contrib/met/12.0.0 +set ver 12.0.0 +set share $base/share/met +setenv MET_ROOT $base/$ver/MET-12.0.0 + +prepend-path PATH $base/bin:$base/external_libs/bin:/mnt/lfs1/HFIP/dtc-hurr/METplus/miniconda/miniconda3/envs/metplus_v5.1_py3.10 From f051fac88e9e5e2632d2c9ef38de8fa8a3315fc3 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 16 Oct 2023 17:53:03 -0600 Subject: [PATCH 03/12] #2687 Saved the PBL input into the vector --- src/tools/other/pb2nc/pb2nc.cc | 114 ++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 50 deletions(-) diff --git a/src/tools/other/pb2nc/pb2nc.cc b/src/tools/other/pb2nc/pb2nc.cc index 92fdc288df..c42fc1bdc7 100644 --- a/src/tools/other/pb2nc/pb2nc.cc +++ b/src/tools/other/pb2nc/pb2nc.cc @@ -372,8 +372,8 @@ static bool keep_level_category(int); static float derive_grib_code(int, float *, float *, double, float&); -static int combine_tqz_and_uv(map, - map, map &); +static int combine_tqz_and_uv(map, map, + vector &); static float compute_pbl(map pqtzuv_map_tq, map pqtzuv_map_uv); static void copy_pqtzuv(float *to_pqtzuv, float *from_pqtzuv, bool copy_all=true); @@ -2959,14 +2959,16 @@ void copy_pqtzuv(float *to_pqtzuv, float *from_pqtzuv, bool copy_all) { //////////////////////////////////////////////////////////////////////// int combine_tqz_and_uv(map pqtzuv_map_tq, - map pqtzuv_map_uv, map &pqtzuv_map_merged) { + map pqtzuv_map_uv, vector &pqtzuv_merged_array) { static const char *method_name = "combine_tqz_and_uv() "; int tq_count = pqtzuv_map_tq.size(); int uv_count = pqtzuv_map_uv.size(); + map pqtzuv_map_merged; + pqtzuv_merged_array.clear(); if (tq_count > 0 && uv_count > 0) { IntArray common_levels, tq_levels; float *pqtzuv_tq, *pqtzuv_uv; - float *pqtzuv_merged = (float *) 0; + float *pqtzuv_merged = (float *) nullptr; float *next_pqtzuv, *prev_pqtzuv; float tq_pres_max, tq_pres_min, uv_pres_max, uv_pres_min; map::iterator it, it_tq, it_uv; @@ -3055,9 +3057,10 @@ int combine_tqz_and_uv(map pqtzuv_map_tq, } interpolate_pqtzuv(prev_pqtzuv, pqtzuv_merged, next_pqtzuv); } - float first_pres = (pqtzuv_merged[0] == 0 ? bad_data_float : pqtzuv_merged[0]); + float first_pres = (pqtzuv_merged[0] < 0 || is_eq(pqtzuv_merged[0], 0.) + ? bad_data_float : pqtzuv_merged[0]); pqtzuv_map_merged[first_pres] = pqtzuv_merged; - mlog << Debug(9) << method_name << "Added " << first_pres << " to merged records\n"; + mlog << Debug(9) << method_name << "Added " << first_pres << " to merged records (first record)\n"; if (pqtzuv_merged != 0) { //Merge UV into TQZ records @@ -3065,6 +3068,12 @@ int combine_tqz_and_uv(map pqtzuv_map_tq, //Merge TQZ into UV records merge_records(pqtzuv_merged, pqtzuv_map_uv, pqtzuv_map_tq, pqtzuv_map_merged); } + for (map::iterator it=pqtzuv_map_merged.begin(); + it!=pqtzuv_map_merged.end(); ++it) { + float *new_pqtzuv = new float[mxr8vt]; + for (int i=0; isecond[i]; + pqtzuv_merged_array.push_back(new_pqtzuv); + } if(mlog.verbosity_level() >= PBL_DEBUG_LEVEL) { log_merged_tqz_uv(pqtzuv_map_tq, pqtzuv_map_uv, pqtzuv_map_merged, method_name); @@ -3072,7 +3081,7 @@ int combine_tqz_and_uv(map pqtzuv_map_tq, delete [] pqtzuv_merged; } - return pqtzuv_map_merged.size(); + return pqtzuv_merged_array.size(); } //////////////////////////////////////////////////////////////////////// @@ -3093,12 +3102,13 @@ float compute_pbl(map pqtzuv_map_tq, mlog << Debug(7) << method_name << "is called: TQZ: " << tq_count << " UV: " << uv_count << "\n"; if (tq_count > 0 || uv_count > 0) { + float *pqtzuv = nullptr; int hgt_cnt, spfh_cnt; IntArray selected_levels; - map pqtzuv_map_merged; + vector pqtzuv_merged_array; pbl_level = combine_tqz_and_uv(pqtzuv_map_tq, pqtzuv_map_uv, - pqtzuv_map_merged); + pqtzuv_merged_array); mlog << Debug(7) << method_name << "pbl_level= " << pbl_level << " from TQ (" << tq_count << ") and UV (" << uv_count << ")\n"; @@ -3112,60 +3122,62 @@ float compute_pbl(map pqtzuv_map_tq, << "Skip CALPBL because of only one available record after combining TQZ and UV\n"; } else { - // Order all observations by pressure from bottom to top - index = pbl_level - 1; + // Reverse the order all observations by pressure from bottom to top + index = 0; hgt_cnt = spfh_cnt = 0; - for (it=pqtzuv_map_merged.begin(); it!=pqtzuv_map_merged.end(); ++it) { - if (index < 0) { - mlog << Error << "\n" << method_name << "negative index: " << index << "\n\n"; - break; - } - - if (index < MAX_PBL_LEVEL) { - float *pqtzuv = it->second; - pbl_data_pres[index] = pqtzuv[0]; - pbl_data_spfh[index] = pqtzuv[1]; - pbl_data_temp[index] = pqtzuv[2]; - pbl_data_hgt[index] = pqtzuv[3]; - pbl_data_ugrd[index] = pqtzuv[4]; - pbl_data_vgrd[index] = pqtzuv[5]; - if (is_valid_pb_data(pbl_data_spfh[index])) spfh_cnt++; - if (is_valid_pb_data(pbl_data_hgt[index])) hgt_cnt++; - selected_levels.add(nint(it->first)); - } - - index--; - } - if (index != -1) { - mlog << Error << "\n" << method_name << "Missing some levels (" << index << ")\n"; - } - - if (pbl_level > MAX_PBL_LEVEL) { - it = pqtzuv_map_tq.begin(); + int start_offset = (MAX_PBL_LEVEL >= pbl_level) ? 0 : (pbl_level-MAX_PBL_LEVEL); + for (int i=(pbl_level-1); i>=start_offset; i--,index++) { + pqtzuv = pqtzuv_merged_array[i]; + pbl_data_pres[index] = pqtzuv[0]; + pbl_data_pres[index] = pqtzuv[0]; + pbl_data_spfh[index] = pqtzuv[1]; + pbl_data_temp[index] = pqtzuv[2]; + pbl_data_hgt[index] = pqtzuv[3]; + pbl_data_ugrd[index] = pqtzuv[4]; + pbl_data_vgrd[index] = pqtzuv[5]; + if (is_valid_pb_data(pbl_data_spfh[index])) spfh_cnt++; + if (is_valid_pb_data(pbl_data_hgt[index])) hgt_cnt++; + selected_levels.add(nint(pqtzuv[0])); + } + if (start_offset > 0) { + // Replace the interpolated records with common records. + mlog << Error << "\n" << method_name << "Excluded " << start_offset << " records\n"; // Find vertical levels with both data float highest_pressure = bad_data_float; - for (; it!=pqtzuv_map_tq.end(); ++it) { + for (it = pqtzuv_map_tq.begin(); it!=pqtzuv_map_tq.end(); ++it) { if (pqtzuv_map_uv.count(it->first) > 0) { highest_pressure = it->first; break; } } if (!is_bad_data(highest_pressure)) { + bool found; + int vector_idx = start_offset - 1; index = MAX_PBL_LEVEL - 1; for (; it!=pqtzuv_map_tq.end(); ++it) { int pres_level = nint(it->first); + // Stop replacing if already exists at input list if (selected_levels.has(pres_level)) break; - float *pqtzuv = pqtzuv_map_merged[it->first]; - pbl_data_pres[index] = pqtzuv[0]; - pbl_data_spfh[index] = pqtzuv[1]; - pbl_data_temp[index] = pqtzuv[2]; - pbl_data_hgt[index] = pqtzuv[3]; - pbl_data_ugrd[index] = pqtzuv[4]; - pbl_data_vgrd[index] = pqtzuv[5]; - mlog << Debug(6) << method_name << "Force to add " - << pres_level << " into " << index << "\n"; - index--; + found = false; + for (; vector_idx>=0; vector_idx--) { + if (is_eq(pqtzuv_merged_array[vector_idx][0], it->first)) { + pqtzuv = pqtzuv_merged_array[vector_idx]; + pbl_data_pres[index] = pqtzuv[0]; + pbl_data_spfh[index] = pqtzuv[1]; + pbl_data_temp[index] = pqtzuv[2]; + pbl_data_hgt[index] = pqtzuv[3]; + pbl_data_ugrd[index] = pqtzuv[4]; + pbl_data_vgrd[index] = pqtzuv[5]; + mlog << Debug(6) << method_name << "Force to add " + << pres_level << " into " << index << "\n"; + vector_idx--; + found = true; + break; + } + } + if (vector_idx < 0) break; + if(found) index--; } } } @@ -3205,9 +3217,11 @@ float compute_pbl(map pqtzuv_map_tq, if (!is_valid_pb_data(hpbl)) mlog << Debug(5) << method_name << " fail to compute PBL. TQ records: " << tq_count << " UV records: " << uv_count << " merged records: " - << pqtzuv_map_merged.size() << "\n"; + << pqtzuv_merged_array.size() << "\n"; } } + for (int i; i Date: Thu, 19 Oct 2023 10:17:16 -0600 Subject: [PATCH 04/12] #2673 Moved a code block under else for SonarQube scanning --- scripts/python/met/dataplane.py | 49 +++++++++++++++++---------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/scripts/python/met/dataplane.py b/scripts/python/met/dataplane.py index e11fc31d50..57c9ac367b 100644 --- a/scripts/python/met/dataplane.py +++ b/scripts/python/met/dataplane.py @@ -180,32 +180,33 @@ def validate_met_data(met_data, fill_value=None): from_ndarray = False if met_data is None: logger.quit(f"{method_name} The met_data is None") - - nx, ny = met_data.shape - met_fill_value = dataplane.MET_FILL_VALUE - if dataplane.is_xarray_dataarray(met_data): - from_xarray = True - attrs = met_data.attrs - met_data = met_data.data - modified_met_data = True - if isinstance(met_data, np.ndarray): - from_ndarray = True - met_data = np.ma.array(met_data) - - if isinstance(met_data, np.ma.MaskedArray): - is_int_data = dataplane.is_integer(met_data[0,0]) or dataplane.is_integer(met_data[int(nx/2),int(ny/2)]) - met_data = np.ma.masked_equal(met_data, float('nan')) - met_data = np.ma.masked_equal(met_data, float('inf')) - if fill_value is not None: - met_data = np.ma.masked_equal(met_data, fill_value) - met_data = met_data.filled(int(met_fill_value) if is_int_data else met_fill_value) else: - logger.log_msg(f"{method_name} unknown datatype {type(met_data)}") + nx, ny = met_data.shape + + met_fill_value = dataplane.MET_FILL_VALUE + if dataplane.is_xarray_dataarray(met_data): + from_xarray = True + attrs = met_data.attrs + met_data = met_data.data + modified_met_data = True + if isinstance(met_data, np.ndarray): + from_ndarray = True + met_data = np.ma.array(met_data) + + if isinstance(met_data, np.ma.MaskedArray): + is_int_data = dataplane.is_integer(met_data[0,0]) or dataplane.is_integer(met_data[int(nx/2),int(ny/2)]) + met_data = np.ma.masked_equal(met_data, float('nan')) + met_data = np.ma.masked_equal(met_data, float('inf')) + if fill_value is not None: + met_data = np.ma.masked_equal(met_data, fill_value) + met_data = met_data.filled(int(met_fill_value) if is_int_data else met_fill_value) + else: + logger.log_msg(f"{method_name} unknown datatype {type(met_data)}") - if dataplane.KEEP_XARRAY: - return xr.DataArray(met_data,attrs=attrs) if from_xarray else met_data - else: - return met_data + if dataplane.KEEP_XARRAY: + return xr.DataArray(met_data,attrs=attrs) if from_xarray else met_data + else: + return met_data def main(argv): From 723bff8cec9a5ab3213927cd574087fa1422a299 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 19 Oct 2023 10:18:42 -0600 Subject: [PATCH 05/12] #2673 Deleted a break statement which will never be executed --- src/basic/vx_util/ascii_table.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/basic/vx_util/ascii_table.cc b/src/basic/vx_util/ascii_table.cc index 9d1911f1de..326d53b978 100644 --- a/src/basic/vx_util/ascii_table.cc +++ b/src/basic/vx_util/ascii_table.cc @@ -1567,7 +1567,6 @@ switch ( just ) { default: mlog << Error << "\njustified_item() -> bad justification value\n\n"; exit ( 1 ); - break; } // switch From 65e2fef24c46c2912d274d27567a6b2a2e8b1aad Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 19 Oct 2023 11:06:15 -0600 Subject: [PATCH 06/12] #2673 Catch exception by reference (SonarQubue) --- src/libcode/vx_nc_util/nc_utils.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index 6350b6576a..b319fbabe9 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -8,21 +8,21 @@ //////////////////////////////////////////////////////////////////////// -using namespace std; - #include #include #include #include -using namespace netCDF; -using namespace netCDF::exceptions; #include "vx_log.h" #include "nc_utils.h" #include "util_constants.h" #include "vx_cal.h" +using namespace std; +using namespace netCDF; +using namespace netCDF::exceptions; + //////////////////////////////////////////////////////////////////////// void patch_nc_name(string *var_name) { @@ -172,7 +172,7 @@ bool get_att_value_chars(const NcAtt *att, ConcatString &value) { att->getValues(att_value); value = att_value; } - catch (exceptions::NcChar ex) { + catch (exceptions::NcChar &ex) { value = ""; // Handle netCDF::exceptions::NcChar: NetCDF: Attempt to convert between text & numbers mlog << Warning << "\n" << method_name @@ -188,7 +188,7 @@ bool get_att_value_chars(const NcAtt *att, ConcatString &value) { att->getValues(att_value); value = att_value; } - catch (exceptions::NcChar ex) { + catch (exceptions::NcChar &ex) { int num_elements_sub = 8096; int num_elements = att->getAttLength();; char *att_value[num_elements]; @@ -199,7 +199,7 @@ bool get_att_value_chars(const NcAtt *att, ConcatString &value) { att->getValues(att_value); value = att_value[0]; } - catch (exceptions::NcException ex) { + catch (exceptions::NcException &ex) { mlog << Warning << "\n" << method_name << "Exception: " << ex.what() << "\n" << "Fail to read " << GET_NC_NAME_P(att) << " attribute (" From 306b08884d44fe846f0de0c8f744bb95ae49bb0b Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 19 Oct 2023 11:07:49 -0600 Subject: [PATCH 07/12] #2673 Move down using statements below include directive. Changed 0 to nullptr (SonarQubue) --- src/tools/core/pcp_combine/pcp_combine.cc | 55 ++++++++++++----------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/src/tools/core/pcp_combine/pcp_combine.cc b/src/tools/core/pcp_combine/pcp_combine.cc index 0efcb42ac9..0e364d08cc 100644 --- a/src/tools/core/pcp_combine/pcp_combine.cc +++ b/src/tools/core/pcp_combine/pcp_combine.cc @@ -80,8 +80,6 @@ // //////////////////////////////////////////////////////////////////////// -using namespace std; - #include #include #include @@ -94,7 +92,6 @@ using namespace std; #include #include -using namespace netCDF; #include "main.h" #include "vx_log.h" @@ -107,6 +104,9 @@ using namespace netCDF; #include "vx_cal.h" #include "vx_math.h" +using namespace std; +using namespace netCDF; + //////////////////////////////////////////////////////////////////////// static ConcatString program_name; @@ -133,7 +133,7 @@ static StringArray req_out_var_name; static int i_out_var = 0; static int n_out_var; static MetConfig config; -static VarInfo * var_info = (VarInfo *) 0; +static VarInfo * var_info = (VarInfo *) nullptr; static double vld_thresh = 1.0; static int compress_level = -1; @@ -151,7 +151,7 @@ static StringArray field_list; static StringArray derive_list; // Output NetCDF file -NcFile *nc_out = (NcFile *) 0; +NcFile *nc_out = (NcFile *) nullptr; NcDim lat_dim; NcDim lon_dim; @@ -217,7 +217,7 @@ int met_main(int argc, char *argv[]) { // Reinitialize for the current loop. // field_string = req_field_list[i]; - if(var_info) { delete var_info; var_info = (VarInfo *) 0; } + if(var_info) { delete var_info; var_info = (VarInfo *) nullptr; } // // Reset when reading multiple fields from the same input files. @@ -241,7 +241,7 @@ int met_main(int argc, char *argv[]) { // close_nc(); - return(0); + return 0; } //////////////////////////////////////////////////////////////////////// @@ -561,9 +561,9 @@ void sum_data_files(Grid & grid, DataPlane & plane) { DataPlane part; double v_sum, v_part; Grid cur_grid; - unixtime * pcp_times = (unixtime *) 0; - int * pcp_recs = (int *) 0; - ConcatString * pcp_files = (ConcatString *) 0; + unixtime * pcp_times = (unixtime *) nullptr; + int * pcp_recs = (int *) nullptr; + ConcatString * pcp_files = (ConcatString *) nullptr; // // Compute the number of forecast precipitation files to be found, @@ -596,6 +596,7 @@ void sum_data_files(Grid & grid, DataPlane & plane) { // // Search in each directory for the current file time. // + if (0 == pcp_dir.n_elements()) pcp_recs[i] = -1; for(j=0; jd_name; Met2dDataFileFactory factory; - Met2dDataFile * mtddf = (Met2dDataFile *) 0; + Met2dDataFile * mtddf = (Met2dDataFile *) nullptr; VarInfoFactory var_fac; - VarInfo * cur_var = (VarInfo *) 0; + VarInfo * cur_var = (VarInfo *) nullptr; // // Create a data file object. @@ -767,7 +768,7 @@ int search_pcp_dir(const char *cur_dir, const unixtime cur_ut, // cur_var = var_fac.new_var_info(mtddf->file_type()); if(!cur_var) { - delete mtddf; mtddf = 0; + delete mtddf; mtddf = nullptr; mlog << Warning << "search_pcp_dir() -> " << "unable to determine filetype of \"" << cur_file << "\"\n"; @@ -796,8 +797,8 @@ int search_pcp_dir(const char *cur_dir, const unixtime cur_ut, // // Cleanup. // - if(mtddf) { delete mtddf; mtddf = (Met2dDataFile *) 0; } - if(cur_var) { delete cur_var; cur_var = (VarInfo *) 0; } + if(mtddf) { delete mtddf; mtddf = (Met2dDataFile *) nullptr; } + if(cur_var) { delete cur_var; cur_var = (VarInfo *) nullptr; } // check for a valid match if( -1 != i_rec ) { met_closedir(dp); break; } @@ -1168,7 +1169,7 @@ void get_field(const char *filename, const char *cur_field, const unixtime get_init_ut, const unixtime get_valid_ut, Grid & grid, DataPlane & plane) { Met2dDataFileFactory factory; - Met2dDataFile *mtddf = (Met2dDataFile *) 0; + Met2dDataFile *mtddf = (Met2dDataFile *) nullptr; GrdFileType ftype; VarInfoFactory var_fac; VarInfo *cur_var; @@ -1244,10 +1245,10 @@ void get_field(const char *filename, const char *cur_field, // // Cleanup. // - if(mtddf) { delete mtddf; mtddf = (Met2dDataFile *) 0; } - if(cur_var) { delete cur_var; cur_var = (VarInfo *) 0; } + if(mtddf) { delete mtddf; mtddf = (Met2dDataFile *) nullptr; } + if(cur_var) { delete cur_var; cur_var = (VarInfo *) nullptr; } - // if ( var ) { delete var; var = 0; } + // if ( var ) { delete var; var = nullptr; } return; @@ -1270,7 +1271,7 @@ void open_nc(const Grid &grid) { << "trouble opening output file " << out_filename << "\n\n"; delete nc_out; - nc_out = (NcFile *) 0; + nc_out = (NcFile *) nullptr; exit(1); } @@ -1452,8 +1453,8 @@ void close_nc() { // // Clean up. // - if(nc_out) { delete nc_out; nc_out = (NcFile *) 0; } - if(var_info ) { delete var_info; var_info = (VarInfo *) 0; } + if(nc_out) { delete nc_out; nc_out = (NcFile *) nullptr; } + if(var_info ) { delete var_info; var_info = (VarInfo *) nullptr; } return; } From 80fdc796c92d7b2ba1db77d5b42b89edcd8434d6 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 19 Oct 2023 11:08:14 -0600 Subject: [PATCH 08/12] #2673 Move down using statements below include directive. Changed 0 to nullptr (SonarQubue) --- src/tools/other/mode_time_domain/3d_conv.cc | 109 ++++++++++---------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/src/tools/other/mode_time_domain/3d_conv.cc b/src/tools/other/mode_time_domain/3d_conv.cc index 0dd0172f22..c2eeeddcfb 100644 --- a/src/tools/other/mode_time_domain/3d_conv.cc +++ b/src/tools/other/mode_time_domain/3d_conv.cc @@ -18,8 +18,6 @@ static const bool do_ppms = false; //////////////////////////////////////////////////////////////////////// -using namespace std; - #include #include #include @@ -29,7 +27,6 @@ using namespace std; #include #include -using namespace netCDF; #include "vx_cal.h" #include "vx_util.h" @@ -38,14 +35,17 @@ using namespace netCDF; #include "mtd_file.h" #include "mtd_nc_defs.h" +using namespace std; +using namespace netCDF; + //////////////////////////////////////////////////////////////////////// static int spatial_conv_radius = -1; -static double * sum_plane_buf = 0; -static bool * ok_sum_plane_buf = 0; +static double * sum_plane_buf = nullptr; +static bool * ok_sum_plane_buf = nullptr; //////////////////////////////////////////////////////////////////////// @@ -66,11 +66,11 @@ struct DataHandle { { - int j, k; + int k; k = 0; - for (j=0; j0) { + value /= n_good; - value *= scale; + value *= scale; - if ( value < min_conv_value ) min_conv_value = value; - if ( value > max_conv_value ) max_conv_value = value; + if ( value < min_conv_value ) min_conv_value = value; + if ( value > max_conv_value ) max_conv_value = value; + } } @@ -429,10 +432,10 @@ for (x=0; x Date: Thu, 19 Oct 2023 11:09:38 -0600 Subject: [PATCH 09/12] #2673 Added std namespace for vector (SonarQubue) --- src/tools/other/mode_time_domain/mtd_file_base.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/other/mode_time_domain/mtd_file_base.h b/src/tools/other/mode_time_domain/mtd_file_base.h index 82f7d40509..659dc784a1 100644 --- a/src/tools/other/mode_time_domain/mtd_file_base.h +++ b/src/tools/other/mode_time_domain/mtd_file_base.h @@ -88,7 +88,7 @@ class MtdFileBase { int DeltaT; // seconds, useful for constant time increments - vector ActualValidTimes; // useful for uneven time increments + std::vector ActualValidTimes; // useful for uneven time increments IntArray Lead_Times; @@ -117,7 +117,7 @@ class MtdFileBase { void set_delta_t (int); // seconds - void init_actual_valid_times(const vector &validTimes); + void init_actual_valid_times(const std::vector &validTimes); void set_lead_time(int index, int value); From 9738e587672514aa9f4cd2a4968e2526ea219904 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 19 Oct 2023 11:10:13 -0600 Subject: [PATCH 10/12] #2673 Changed 0 to nullptr (SonarQubue) --- .../other/mode_time_domain/mtd_file_float.cc | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/src/tools/other/mode_time_domain/mtd_file_float.cc b/src/tools/other/mode_time_domain/mtd_file_float.cc index f0c70f509f..dd407c5061 100644 --- a/src/tools/other/mode_time_domain/mtd_file_float.cc +++ b/src/tools/other/mode_time_domain/mtd_file_float.cc @@ -101,7 +101,7 @@ void MtdFloatFile::float_init_from_scratch() { -Data = 0; +Data = nullptr; clear(); @@ -119,7 +119,7 @@ void MtdFloatFile::clear() MtdFileBase::clear(); -if ( Data ) { delete [] Data; Data = 0; } +if ( Data ) { delete [] Data; Data = nullptr; } DataMin = DataMax = 0; @@ -218,7 +218,7 @@ void MtdFloatFile::set_size(int _nx, int _ny, int _nt) { -if ( Data ) { delete [] Data; Data = 0; } +if ( Data ) { delete [] Data; Data = nullptr; } int j; const int n3 = _nx*_ny*_nt; @@ -1011,18 +1011,6 @@ for (x=0; x Date: Thu, 19 Oct 2023 11:12:19 -0600 Subject: [PATCH 11/12] #2673 Changed a double pointer to vector (SonarQubue) --- src/libcode/vx_statistics/compute_stats.cc | 76 ++++++++++++---------- src/libcode/vx_statistics/compute_stats.h | 5 +- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/src/libcode/vx_statistics/compute_stats.cc b/src/libcode/vx_statistics/compute_stats.cc index 6a66c316b0..fc8a3fbdd2 100644 --- a/src/libcode/vx_statistics/compute_stats.cc +++ b/src/libcode/vx_statistics/compute_stats.cc @@ -1449,7 +1449,7 @@ void compute_aggregated_seeps(const PairDataPoint *pd, SeepsAggScore *seeps) { seeps_mprs.push_back(seeps_mpr); } if (count > 0) { - double *density_vector; + vector density_vector; double pvf[SEEPS_MATRIX_SIZE]; double weighted_score, weight_sum, weight[count]; @@ -1457,41 +1457,47 @@ void compute_aggregated_seeps(const PairDataPoint *pd, SeepsAggScore *seeps) { seeps->mean_fcst = fcst_sum / count; seeps->mean_obs = obs_sum / count; seeps->score = score_sum / count; - density_vector = compute_seeps_density_vector(pd, seeps); weighted_score = 0.; for (int i=0; iscore * weight[i]; - //IDL: svf(cat{i)) = svf(cat{i)) + c(4+cat(i) * w{i) - //IDL: pvf(cat{i)) = pvf(cat{i)) + w{i) - pvf[seeps_mpr->s_idx] += weight[i]; - } - else mlog << Debug(1) << method_name - << "the length of density vector (" << count << ") is less than MPR.\n"; + } + if (!is_eq(weight_sum, 0)) { + //IDL: w = w/sum(w) + for (int i=0; iscore * weight[i]; + //IDL: svf(cat{i)) = svf(cat{i)) + c(4+cat(i) * w{i) + //IDL: pvf(cat{i)) = pvf(cat{i)) + w{i) + pvf[seeps_mpr->s_idx] += weight[i]; + } + else { + mlog << Debug(1) << method_name + << "the length of density vector (" << density_cnt + << ") is less than SEEPS MPR (" << seeps_mprs.size() << ").\n"; + break; } } - - if (density_vector != nullptr) delete [] density_vector; } + + density_vector.clear(); + seeps_mprs.clear(); // The weight for s12 to s32 should come from climo file, but not available yet @@ -1707,7 +1713,7 @@ void compute_aggregated_seeps_grid(const DataPlane &fcst_dp, const DataPlane &ob // ; PV-WAVE prints: 2.00000 4.00000 //////////////////////////////////////////////////////////////////////// -double *compute_seeps_density_vector(const PairDataPoint *pd, SeepsAggScore *seeps) { +void compute_seeps_density_vector(const PairDataPoint *pd, SeepsAggScore *seeps, vector &density_vector) { int seeps_idx; SeepsScore *seeps_mpr; int seeps_cnt = seeps->n_obs; @@ -1725,12 +1731,11 @@ double *compute_seeps_density_vector(const PairDataPoint *pd, SeepsAggScore *see if (seeps_cnt == 0) { mlog << Debug(1) << method_name << "no SEEPS_MPR available.\n"; - return nullptr; + return; } // Get lat/lon & convert them to radian and get sin/cos values seeps_idx = 0; - double *density_vector = new double[seeps_cnt]; for(int i=0; in_obs; i++) { if (i >= pd->seeps_mpr.size()) break; seeps_mpr = pd->seeps_mpr[i]; @@ -1752,10 +1757,9 @@ double *compute_seeps_density_vector(const PairDataPoint *pd, SeepsAggScore *see // Initialize v_count = 0; if (seeps_idx < seeps_cnt) seeps_cnt = seeps_idx; - for(int i=0; i &density_vector); //////////////////////////////////////////////////////////////////////// // From 23f707a1fa2a0924830ece91c6bfc354dc8bc7a0 Mon Sep 17 00:00:00 2001 From: davidalbo Date: Wed, 25 Oct 2023 15:29:39 -0600 Subject: [PATCH 12/12] Feature 2708 mvmode merge flag default (#2713) * mods to use the multivar mode default config (when in multivar mode), and to check for problematic merge config settings * added more meaningful error and debug messages * Modified documentation to better describe multivariate mode including new default values * Bug fix for unit test * cleanup and fix a bug in which the default output path was not set in multivar mode --- data/config/MODEMultivarConfig_default | 7 ++ docs/Users_Guide/mode.rst | 4 +- src/basic/vx_util/file_exists.cc | 2 - src/libcode/vx_shapedata/mode_field_info.cc | 81 ++++++++++++++++++--- src/tools/core/mode/mode.cc | 7 ++ src/tools/core/mode/mode_exec.cc | 11 ++- src/tools/core/mode/mode_exec.h | 2 +- src/tools/core/mode/mode_frontend.cc | 3 +- src/tools/core/mode/multivar_frontend.cc | 14 +++- 9 files changed, 106 insertions(+), 25 deletions(-) diff --git a/data/config/MODEMultivarConfig_default b/data/config/MODEMultivarConfig_default index 66ab5dfdf3..8d4d7f8406 100644 --- a/data/config/MODEMultivarConfig_default +++ b/data/config/MODEMultivarConfig_default @@ -53,6 +53,11 @@ quilt = FALSE; // multivar_logic = "#1 && #2 && #3"; +// +// MODE Multivar intensity computation flag +// +multivar_intensity_flag = [FALSE, TRUE, TRUE]; + // // Forecast and observation fields to be verified // @@ -84,6 +89,8 @@ fcst = { filter_attr_thresh = []; merge_thresh = >=3.5; merge_flag = NONE; + multivar_name = "ALPHA_BETA_GAMMA"; + multivar_level = "LO"; } obs = fcst; diff --git a/docs/Users_Guide/mode.rst b/docs/Users_Guide/mode.rst index 98ab658ea4..2d304b21a1 100644 --- a/docs/Users_Guide/mode.rst +++ b/docs/Users_Guide/mode.rst @@ -197,6 +197,8 @@ mode configuration file The default configuration file for the MODE tool, **MODEConfig_default**, can be found in the installed *share/met/config* directory. Another version of the configuration file is provided in *scripts/config*. We encourage users to make a copy of the configuration files prior to modifying their contents. Descriptions of **MODEConfig_default** and the required variables for any MODE configuration file are also provided below. While the configuration file contains many entries, most users will only need to change a few for their use. Specific options are described in the following subsections. +A second default configuration file for the multivar MODE option, **MODEMultivarConfig_default**, is also found in the installed *share/met/config* directory. We encourage users to make a copy of this default configuration file when setting up a multivar configuration prior to modifying content. The two default config files **MODEConfig_default** and **MODEMultivarConfig_default** are similar, with **MODEMultivarConfig_default** having example multivar specific content. + Note that environment variables may be used when editing configuration files, as described in the :numref:`config_env_vars`. _____________________ @@ -324,7 +326,7 @@ The **merge_flag** entry controls what type of merging techniques will be applie • **BOTH** indicates that both techniques should be used. -By default, the double thresholding merging technique is applied. +By default, the double thresholding **THRESH** merging technique is applied in single variable mode. The merging defaults to **NONE** with multivariate mode. _____________________ diff --git a/src/basic/vx_util/file_exists.cc b/src/basic/vx_util/file_exists.cc index 6d007303a4..a51db6f164 100644 --- a/src/basic/vx_util/file_exists.cc +++ b/src/basic/vx_util/file_exists.cc @@ -1,5 +1,3 @@ - - // *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* // ** Copyright UCAR (c) 1992 - 2023 // ** University Corporation for Atmospheric Research (UCAR) diff --git a/src/libcode/vx_shapedata/mode_field_info.cc b/src/libcode/vx_shapedata/mode_field_info.cc index 6cecb55d07..06311f5d47 100644 --- a/src/libcode/vx_shapedata/mode_field_info.cc +++ b/src/libcode/vx_shapedata/mode_field_info.cc @@ -9,6 +9,7 @@ //////////////////////////////////////////////////////////////////////// +#include #include "vx_config.h" #include "vx_data2d.h" @@ -20,6 +21,7 @@ #include "mode_field_info.h" +using std::string; //////////////////////////////////////////////////////////////////////// @@ -219,32 +221,87 @@ if ( dict->lookup(conf_key_conv_thresh) ) { } -if ( dict->lookup(conf_key_merge_thresh) ) { - - merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh); - -} - if ( dict->lookup(conf_key_vld_thresh) ) { vld_thresh = dict->lookup_double(conf_key_vld_thresh); } -// for the multivar case, go without parent, and error out if merge flag is -// not set for the individual field +// For the multivar case, more complex logic regarding merge_flag and merge_thresh +// If individual entry has a merge_flag, it must have a merge_thresh (unless merge_flag=NONE) +// If individual entry has no merge_flag, check the parent and go with that flag and thresh +// If individual entry has no merge_flag, but individual entry has a merge_thresh, it's an error + if ( _multivar ) { + // set defaults to no merging + merge_thresh.clear(); + merge_flag = MergeType_None; + + // pull out the name + string name = var_info->name(); + if ( dict->lookup(conf_key_merge_flag, false)) { - merge_flag = int_to_mergetype(dict->lookup_int(conf_key_merge_flag)); + // this individual entry has merge_flag + merge_flag = int_to_mergetype(dict->lookup_int(conf_key_merge_flag, + default_dictionary_error_out, + default_dictionary_print_warning, + false)); + string merge_name = mergetype_to_string(merge_flag); + if (dict->lookup(conf_key_merge_thresh, false)) { + // the individual entry also has a merge_thresh, this is good + merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh, + default_dictionary_error_out, + default_dictionary_print_warning, + false); + } else { + // get the parent's merge_thresh, just to have something. Error out if the merge_flag is not none + // because that is inconsistent + merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh); + + if (merge_flag != MergeType_None) { + mlog << Error << "\nMode_Field_Info::set() -> " + << "Field:" << name << ". " + << " When 'merge_flag' is explicitly set, 'merge_thresh' must be explicitly set for multivariate mode\n\n"; + exit ( 1 ); + } + + } } else { - mlog << Error << "\nMode_Field_Info::set() -> " - << "'merge_flag' must be explicitly set for all fields with multivariate mode\n\n"; - exit ( 1 ); + // individual entry does not have a merge_flag, try parent + if ( dict->lookup(conf_key_merge_flag, true)) { + // the parent does have a merge flag + merge_flag = int_to_mergetype(dict->lookup_int(conf_key_merge_flag)); + string merge_name = mergetype_to_string(merge_flag); + if (dict->lookup(conf_key_merge_thresh, false)) { + // individual entry has a merge_thresh but no merge_flag, this is not good + mlog << Error << "\nMode_Field_Info::set() -> " + << "Field:" << name << ". " + << "When 'merge_flag' is not explicitly set, 'merge_thresh' cannot explicitly set for multivariate mode\n\n"; + exit ( 1 ); + } else { + // individual entry doesn't have a merge_thresh, parent has a merge_flag + // expect parent to have a merge_thresh + merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh); + if (merge_thresh_array.n() == 0 && merge_flag != MergeType_None) { + // parent has a merge_flag but no merge_thresh + mlog << Error << "\nMode_Field_Info::set() -> " + << "Field:" << name << ". using parent merge_flag: " << merge_name + << " Parent has no 'merge_thresh', not allowed in multivariate mode\n\n"; + exit ( 1 ); + } else { + string thresh_str = merge_thresh_array.thresh()->get_str(); + mlog << Debug(2) << "Field:" << name << ". Using parent merge_flag: " + << merge_name << " and parent merge_thresh:" << thresh_str + << "\n"; + } + } + } } } else { if ( dict->lookup(conf_key_merge_flag, true)) { merge_flag = int_to_mergetype(dict->lookup_int(conf_key_merge_flag)); } + merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh); } filter_attr_map = parse_conf_filter_attr_map(dict); diff --git a/src/tools/core/mode/mode.cc b/src/tools/core/mode/mode.cc index 079a5db734..88e18d0b40 100644 --- a/src/tools/core/mode/mode.cc +++ b/src/tools/core/mode/mode.cc @@ -115,6 +115,7 @@ const char * const program_name = "mode"; static const char default_config_filename [] = "MET_BASE/config/MODEConfig_default"; +static const char default_multivar_config_filename [] = "MET_BASE/config/MODEMultivarConfig_default"; /////////////////////////////////////////////////////////////////////// @@ -169,6 +170,12 @@ int met_main(int argc, char * argv []) if ( config.is_multivar() ) { + + // read again, this time with the mvmode defaults, which are slightly different + + config.clear(); + config.read_config(default_multivar_config_filename, user_config_filename); + // exit inside this method if something is not implemented config.check_multivar_not_implemented(); diff --git a/src/tools/core/mode/mode_exec.cc b/src/tools/core/mode/mode_exec.cc index b3f130c4ec..f927b637be 100644 --- a/src/tools/core/mode/mode_exec.cc +++ b/src/tools/core/mode/mode_exec.cc @@ -35,6 +35,7 @@ static const char * cts_str[n_cts] = {"RAW", "OBJECT"}; static const char program_name [] = "mode"; static const char * default_config_filename = "MET_BASE/config/MODEConfig_default"; +static const char * default_multivar_config_filename = "MET_BASE/config/MODEMultivarConfig_default"; // took this out of the do_conv_thresh() method static int local_r_index = -1; @@ -143,7 +144,7 @@ void ModeExecutive::clear() /////////////////////////////////////////////////////////////////////// -void ModeExecutive::init(int n_files) +void ModeExecutive::init(int n_files, bool isMultivar) { @@ -152,7 +153,11 @@ void ModeExecutive::init(int n_files) R_index = T_index = 0; // Create the default config file name - default_config_file = replace_path(default_config_filename); + if (isMultivar) { + default_config_file = replace_path(default_multivar_config_filename); + } else { + default_config_file = replace_path(default_config_filename); + } // If the merge config file was not set, use the match config file if(merge_config_file.length() == 0) @@ -226,7 +231,7 @@ void ModeExecutive::init_multivar(GrdFileType ftype, GrdFileType otype) R_index = T_index = 0; // Create the default config file name - default_config_file = replace_path(default_config_filename); + default_config_file = replace_path(default_multivar_config_filename); // If the merge config file was not set, use the match config file if(merge_config_file.length() == 0) diff --git a/src/tools/core/mode/mode_exec.h b/src/tools/core/mode/mode_exec.h index 89d0102d2a..547792c1c6 100644 --- a/src/tools/core/mode/mode_exec.h +++ b/src/tools/core/mode/mode_exec.h @@ -76,7 +76,7 @@ class ModeExecutive { void clear(); - void init(int n_files); + void init(int n_files, bool isMultivar); void init_multivar(GrdFileType ftype, GrdFileType otype); diff --git a/src/tools/core/mode/mode_frontend.cc b/src/tools/core/mode/mode_frontend.cc index 329bcaa0e7..a37bd89b0c 100644 --- a/src/tools/core/mode/mode_frontend.cc +++ b/src/tools/core/mode/mode_frontend.cc @@ -37,7 +37,6 @@ extern const char * const program_name; static ModeExecutive *mode_exec = 0; static int compress_level = -1; - /////////////////////////////////////////////////////////////////////// @@ -78,7 +77,7 @@ int ModeFrontEnd::run(const StringArray & Argv, Processing_t ptype, int field_in process_command_line(Argv, false); - mode_exec->init(n_files); + mode_exec->init(n_files, ptype != SINGLE_VAR); ModeConfInfo & conf = mode_exec->engine.conf_info; if ( field_index >= 0 ) conf.set_field_index(field_index); diff --git a/src/tools/core/mode/multivar_frontend.cc b/src/tools/core/mode/multivar_frontend.cc index 087311dd67..c19a3ad0b0 100644 --- a/src/tools/core/mode/multivar_frontend.cc +++ b/src/tools/core/mode/multivar_frontend.cc @@ -10,7 +10,8 @@ //////////////////////////////////////////////////////////////////////// -static const char mode_default_config [] = "MET_BASE/config/MODEConfig_default"; +// for multivar mode, this is the default file +static const char mode_default_config [] = "MET_BASE/config/MODEMultivarConfig_default"; static const int dir_creation_mode = 0755; @@ -68,6 +69,8 @@ static const char tab [] = " "; static const bool do_clusters = false; +static string default_out_dir = "."; + static ModeConfInfo config; static string mode_path; @@ -394,6 +397,12 @@ void process_command_line(const StringArray & argv) CommandLine cline; + // + // Set the default output directory + // + + outdir = replace_path(default_out_dir); + mode_path = argv[0]; cline.set(argv); @@ -671,9 +680,6 @@ void process_superobjects(ShapeData &f_result, ShapeData &o_result, mode_argv.add(junk); mode_argv.add("-outdir"); mode_argv.add(dir); - // mode_argv.add("-field_index"); - // snprintf(junk, sizeof(junk), "%d", j); - // mode_argv.add(junk); mlog << Debug(1) << "Running superobject mode \n\n";