diff --git a/src/libcode/vx_data2d/data_class.h b/src/libcode/vx_data2d/data_class.h index ca9312880e..f8460cb4ce 100644 --- a/src/libcode/vx_data2d/data_class.h +++ b/src/libcode/vx_data2d/data_class.h @@ -153,6 +153,11 @@ class Met2dDataFile : public Met2dData { virtual int index(VarInfo &) = 0; + // retrieve a vector of all requested data planes + + virtual int data_planes(std::vector&, + std::vector&) = 0; + // post-process data after reading it bool process_data_plane(VarInfo *, DataPlane &); diff --git a/src/libcode/vx_data2d_grib/data2d_grib.cc b/src/libcode/vx_data2d_grib/data2d_grib.cc index 98720ea120..69e0105f67 100644 --- a/src/libcode/vx_data2d_grib/data2d_grib.cc +++ b/src/libcode/vx_data2d_grib/data2d_grib.cc @@ -827,3 +827,18 @@ int MetGrib1DataFile::index(VarInfo &vinfo){ return rec; } + +//////////////////////////////////////////////////////////////////////// + +int MetGrib1DataFile::data_planes(vector &vi_list, + vector &dp_list) { + + // TODO implement this + + mlog << Error << "\nMetGrib1DataFile::data_planes() -> " + << "not yet implemented!\n\n"; + exit(1); + +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d_grib/data2d_grib.h b/src/libcode/vx_data2d_grib/data2d_grib.h index 563c09fb4f..df6e8580e3 100644 --- a/src/libcode/vx_data2d_grib/data2d_grib.h +++ b/src/libcode/vx_data2d_grib/data2d_grib.h @@ -84,12 +84,23 @@ class MetGrib1DataFile : public Met2dDataFile { int read_record( VarInfoGrib &); // returns match count (>=0), or -1 on error + // retrieve the first matching data plane + bool data_plane(VarInfo &, DataPlane &); + // retrieve all matching data planes + int data_plane_array(VarInfo &, DataPlaneArray &); - + + // retrieve the index of the first matching record + int index(VarInfo &); + // retrieve a vector of all requested data planes + + int data_planes(std::vector&, + std::vector&); + }; diff --git a/src/libcode/vx_data2d_grib2/data2d_grib2.cc b/src/libcode/vx_data2d_grib2/data2d_grib2.cc index 6a3f40574d..7a0dfef4fc 100644 --- a/src/libcode/vx_data2d_grib2/data2d_grib2.cc +++ b/src/libcode/vx_data2d_grib2/data2d_grib2.cc @@ -1611,6 +1611,19 @@ int MetGrib2DataFile::index( VarInfo &vinfo ){ return 1 > listMatchExact.size() ? -1 : listMatchExact[0]->Index; } +//////////////////////////////////////////////////////////////////////// + +int MetGrib2DataFile::data_planes(vector &vi_list, + vector &dp_list) { + + // TODO implement this + + mlog << Error << "\nMetGrib2DataFile::data_planes() -> " + << "not yet implemented!\n\n"; + exit(1); + +} + //////////////////////////////////////////////////////////////////////// // // Begin utility functions. diff --git a/src/libcode/vx_data2d_grib2/data2d_grib2.h b/src/libcode/vx_data2d_grib2/data2d_grib2.h index 4191e19728..d04dce33a1 100644 --- a/src/libcode/vx_data2d_grib2/data2d_grib2.h +++ b/src/libcode/vx_data2d_grib2/data2d_grib2.h @@ -142,6 +142,11 @@ class MetGrib2DataFile : public Met2dDataFile { int index(VarInfo &); + // retrieve a vector of all requested data planes + + int data_planes(std::vector&, + std::vector&); + // // do stuff // diff --git a/src/libcode/vx_data2d_nc_met/data2d_nc_met.cc b/src/libcode/vx_data2d_nc_met/data2d_nc_met.cc index 46a39bd502..a269c452cb 100644 --- a/src/libcode/vx_data2d_nc_met/data2d_nc_met.cc +++ b/src/libcode/vx_data2d_nc_met/data2d_nc_met.cc @@ -231,3 +231,16 @@ int MetNcMetDataFile::index(VarInfo &vinfo) { } //////////////////////////////////////////////////////////////////////// + +int MetNcMetDataFile::data_planes(vector &vi_list, + vector &dp_list) { + + // TODO implement this + + mlog << Error << "\nMetNcMetDataFile::data_planes() -> " + << "not yet implemented!\n\n"; + exit(1); + +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d_nc_met/data2d_nc_met.h b/src/libcode/vx_data2d_nc_met/data2d_nc_met.h index e137583b78..f3a5f9c3ed 100644 --- a/src/libcode/vx_data2d_nc_met/data2d_nc_met.h +++ b/src/libcode/vx_data2d_nc_met/data2d_nc_met.h @@ -69,6 +69,11 @@ class MetNcMetDataFile : public Met2dDataFile { int index(VarInfo &); + // retrieve a vector of all requested data planes + + int data_planes(std::vector&, + std::vector&); + // // do stuff // diff --git a/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc b/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc index dc7d59be00..756eab7336 100644 --- a/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc +++ b/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc @@ -310,3 +310,18 @@ int MetNcPinterpDataFile::data_plane_array(VarInfo &vinfo, int MetNcPinterpDataFile::index(VarInfo &vinfo){ return -1; } + +//////////////////////////////////////////////////////////////////////// + +int MetNcPinterpDataFile::data_planes(vector &vi_list, + vector &dp_list) { + + // TODO implement this + + mlog << Error << "\nMetNcPinterpDataFile::data_planes() -> " + << "not yet implemented!\n\n"; + exit(1); + +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.h b/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.h index 40ab3f88f0..712325675b 100644 --- a/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.h +++ b/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.h @@ -71,6 +71,11 @@ class MetNcPinterpDataFile : public Met2dDataFile { int index(VarInfo &); + // retrieve a vector of all requested data planes + + int data_planes(std::vector&, + std::vector&); + // // do stuff // diff --git a/src/libcode/vx_data2d_nccf/data2d_nccf.cc b/src/libcode/vx_data2d_nccf/data2d_nccf.cc index f8b251c6d1..0fca438831 100644 --- a/src/libcode/vx_data2d_nccf/data2d_nccf.cc +++ b/src/libcode/vx_data2d_nccf/data2d_nccf.cc @@ -605,6 +605,19 @@ int MetNcCFDataFile::index(VarInfo &vinfo){ //////////////////////////////////////////////////////////////////////// +int MetNcCFDataFile::data_planes(vector &vi_list, + vector &dp_list) { + + // TODO implement this + + mlog << Error << "\nMetNcCFDataFile::data_planes() -> " + << "not yet implemented!\n\n"; + exit(1); + +} + +//////////////////////////////////////////////////////////////////////// + long MetNcCFDataFile::convert_time_to_offset(long time_value) { bool found = false; bool found_value = false; diff --git a/src/libcode/vx_data2d_nccf/data2d_nccf.h b/src/libcode/vx_data2d_nccf/data2d_nccf.h index d9a9a2df9c..0f0056f1a4 100644 --- a/src/libcode/vx_data2d_nccf/data2d_nccf.h +++ b/src/libcode/vx_data2d_nccf/data2d_nccf.h @@ -92,6 +92,10 @@ class MetNcCFDataFile : public Met2dDataFile { int index(VarInfo &); + // retrieve a vector of all requested data planes + + int data_planes(std::vector&, + std::vector&); // // do stuff diff --git a/src/libcode/vx_data2d_python/data2d_python.cc b/src/libcode/vx_data2d_python/data2d_python.cc index 9251fb0d21..a2e52e9958 100644 --- a/src/libcode/vx_data2d_python/data2d_python.cc +++ b/src/libcode/vx_data2d_python/data2d_python.cc @@ -507,3 +507,17 @@ return ( 0 ); //////////////////////////////////////////////////////////////////////// + +int MetPythonDataFile::data_planes(vector &vi_list, + vector &dp_list) { + + // TODO implement this + + mlog << Error << "\nMetPythonDataFile::data_planes() -> " + << "not yet implemented!\n\n"; + exit(1); + +} + + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d_python/data2d_python.h b/src/libcode/vx_data2d_python/data2d_python.h index d036d2a706..ce36e16c47 100644 --- a/src/libcode/vx_data2d_python/data2d_python.h +++ b/src/libcode/vx_data2d_python/data2d_python.h @@ -92,6 +92,9 @@ class MetPythonDataFile : public Met2dDataFile { int index(VarInfo &); + int data_planes(std::vector&, + std::vector&); + bool data_plane(DataPlane &); }; diff --git a/src/libcode/vx_series_data/series_data.cc b/src/libcode/vx_series_data/series_data.cc index c40955a073..581ea7ff50 100644 --- a/src/libcode/vx_series_data/series_data.cc +++ b/src/libcode/vx_series_data/series_data.cc @@ -8,6 +8,8 @@ //////////////////////////////////////////////////////////////////////// +using namespace std; + #include #include #include @@ -23,6 +25,9 @@ static bool read_single_entry(VarInfo*, const ConcatString&, const GrdFileType, DataPlane&, Grid&); +static bool read_all_entries(vector&, const ConcatString&, const GrdFileType, + vector&, Grid&); + //////////////////////////////////////////////////////////////////////// bool get_series_entry(int i_series, VarInfo* data_info, @@ -103,3 +108,85 @@ bool read_single_entry(VarInfo* info, const ConcatString& filename, } //////////////////////////////////////////////////////////////////////// + +bool get_series_entries(int i_series, vector &vi_list, + const StringArray &search_files, const GrdFileType type, + vector &dp_list, Grid &grid, + bool error_out, bool print_warning) { + int i; + bool found; + + // Check for at least one field requested + if(vi_list.size() <= 0) return(false); + + // Save the log print warning state + bool save_print_warning_state = mlog.print_warning(); + + // Suppress log warnings, if requested + if(!print_warning) mlog.set_print_warning(false); + + mlog << Debug(3) + << "Processing series entry " << i_series + 1 << ": " + << vi_list[0]->magic_time_str() << "\n"; + + // Initialize + dp_list.clear(); + + // Search for data, beginning with the i_series index + for(i=0,found=false; i " + << "Could not find data for " << vi_list[0]->magic_time_str() + << " in file list:\n" << write_css(search_files) << "\n\n"; + exit(1); + } + + // Restore warnings to their original state + mlog.set_print_warning(save_print_warning_state); + + return(found); +} + +//////////////////////////////////////////////////////////////////////// + +bool read_all_entries(vector &vi_list, const ConcatString &filename, + const GrdFileType type, vector &dp_list, Grid &grid) { + + Met2dDataFileFactory mtddf_factory; + Met2dDataFile* mtddf = (Met2dDataFile*) 0; + + // Check that file exists + if(!file_exists(filename.c_str())) { + mlog << Warning << "\nread_all_entries() -> " + << "File does not exist: " << filename << "\n\n"; + return(false); + } + + // Open data file + mtddf = mtddf_factory.new_met_2d_data_file(filename.c_str(), type); + + // Attempt to read gridded data + int n_valid = mtddf->data_planes(vi_list, dp_list); + + // Store grid + if(n_valid > 0) grid = mtddf->grid(); + + // Cleanup + if(mtddf) { delete mtddf; mtddf = (Met2dDataFile *) 0; } + + return(n_valid > 0); +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_series_data/series_data.h b/src/libcode/vx_series_data/series_data.h index bf45c3e22a..fd84a5ea86 100644 --- a/src/libcode/vx_series_data/series_data.h +++ b/src/libcode/vx_series_data/series_data.h @@ -13,6 +13,8 @@ //////////////////////////////////////////////////////////////////////// +#include + #include "vx_util.h" #include "vx_log.h" #include "vx_data2d.h" @@ -26,6 +28,11 @@ bool get_series_entry(int, VarInfo*, const StringArray&, bool error_out=true, bool print_warning=true); +bool get_series_entries(int, std::vector&, const StringArray&, + const GrdFileType, std::vector&, Grid&, + bool error_out=true, + bool print_warning=true); + //////////////////////////////////////////////////////////////////////// #endif // __SERIES_DATA_H__ diff --git a/src/tools/tc_utils/tc_diag/tc_diag.cc b/src/tools/tc_utils/tc_diag/tc_diag.cc index b870c92315..0a6b9efa90 100644 --- a/src/tools/tc_utils/tc_diag/tc_diag.cc +++ b/src/tools/tc_utils/tc_diag/tc_diag.cc @@ -15,6 +15,7 @@ // Mod# Date Name Description // ---- ---- ---- ----------- // 000 09/27/22 Halley Gotway New +// 001 08/17/23 Halley Gotway MET #2609 handle missing data // //////////////////////////////////////////////////////////////////////// @@ -968,44 +969,71 @@ void process_track_points(const TrackInfoArray& tracks) { } //////////////////////////////////////////////////////////////////////// +/// + // TODO: Consider adding vortex removal logic here + // Read in the full set of fields required for vortex removal + // Add flag to configure which fields are used for vortex removal void process_fields(const TrackInfoArray &tracks, const unixtime vld_ut, int i_vld, const string &domain, const DomainInfo &di) { int i, j, i_pnt; - DataPlane data_dp; Grid grid_dp; VarInfoFactory vi_factory; VarInfo *vi = (VarInfo *) 0; + vector vi_list; + DataPlane dp; + vector dp_list; StringArray tmp_key_sa, fields_missing; - // TODO: Consider adding vortex removal logic here - // Read in the full set of fields required for vortex removal - // Add flag to configure which fields are used for vortex removal - - // TODO: Add logic to support one_time_per_file_flag set to true. - - // Loop over the VarInfo fields to be processed + // Create vector of VarInfo objects for(i=0; iset_valid(vld_ut); + vi_list.push_back(vi); + } + + // Read all data at the same time if they are all in the same file + if(conf_info.one_time_per_file_flag) { - // Find data for this track point - bool status = get_series_entry(i_vld, vi, + // Find all entries for this track point + bool status = get_series_entries(i_vld, vi_list, di.data_files, file_type, - data_dp, grid_dp, + dp_list, grid_dp, false, false); - // Keep track of missing fields - if(!status) { - fields_missing.add(vi->magic_str()); + } + // Otherwise, read data one at a time + else { + + // Loop over the VarInfo fields to be processed + for(i=0; imagic_str()); + + // Store the requested valid time + dp.set_valid(vld_ut); + } + + // Append current DataPlane to the vector + dp_list.push_back(dp); + + } // end for i + } + + // Loop over the VarInfo fields to be processed + for(i=0; i