From 5a01b1c26d8d079fc6cfa83acc423ad9feb96c7d Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Tue, 5 Apr 2022 17:42:22 -0600 Subject: [PATCH 1/6] #2123 Support int64 data type --- met/src/libcode/vx_nc_util/nc_utils.cc | 57 +++++++++++++++++++++----- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/met/src/libcode/vx_nc_util/nc_utils.cc b/met/src/libcode/vx_nc_util/nc_utils.cc index 8d8e5b1087..36e97e40b7 100644 --- a/met/src/libcode/vx_nc_util/nc_utils.cc +++ b/met/src/libcode/vx_nc_util/nc_utils.cc @@ -1533,6 +1533,7 @@ bool get_nc_data(NcVar *var, float *data) { // Note: missing data was checked here // int type_id = GET_NC_TYPE_ID_P(var); + return_status = true; if (NcType::nc_FLOAT == type_id) { var->getVar(data); } @@ -1565,9 +1566,26 @@ bool get_nc_data(NcVar *var, float *data) { if (IS_VALID_NC_P(att_scale_factor)) scale_factor = get_att_value_float(att_scale_factor); mlog << Debug(4) << method_name << "add_offset = " << add_offset << ", scale_factor=" << scale_factor << ", cell_count=" << cell_count - << ", is_unsigned_value: " << unsigned_value << "\n"; + << ", is_unsigned_value: " << unsigned_value << " for " << GET_NC_NAME_P(var) << "\n"; switch ( type_id ) { + case NcType::nc_INT64: + { + long long fill_value = bad_data_int; + long long min_value = 2147483647; + long long max_value = -2147483648; + long long *packed_data = new long long[cell_count]; + + if (IS_VALID_NC_P(att_fill_value)) + fill_value = get_att_value_int(att_fill_value); + + var->getVar(packed_data); + _apply_scale_factor(data, packed_data, cell_count, + fill_value, min_value, max_value, "int", + add_offset, scale_factor); + delete [] packed_data; + } + break; case NcType::nc_INT: { int fill_value = bad_data_int; @@ -1720,15 +1738,15 @@ bool get_nc_data(NcVar *var, float *data) { } break; default: - mlog << Debug(1) << method_name << "type_id: " - << type_id << " type name: " << GET_NC_TYPE_NAME_P(var) - << "\n"; + return_status = false; + mlog << Debug(1) << method_name << "Did not read data because of unsupported data type (" + << type_id << ", type name: " << GET_NC_TYPE_NAME_P(var) + << ") for " << GET_NC_NAME_P(var) << "\n"; } if(att_add_offset) delete att_add_offset; if(att_scale_factor) delete att_scale_factor; if(att_fill_value) delete att_fill_value; } - return_status = true; } mlog << Debug(6) << method_name << "took " @@ -1821,6 +1839,7 @@ bool get_nc_data(NcVar *var, double *data) { // int unpacked_count = 0; int type_id = GET_NC_TYPE_ID_P(var); + return_status = true; if ((NcType::nc_DOUBLE == type_id) || (NcType::nc_FLOAT == type_id)){ var->getVar(data); } @@ -1844,9 +1863,26 @@ bool get_nc_data(NcVar *var, double *data) { } mlog << Debug(4) << method_name << "add_offset = " << add_offset << ", scale_factor=" << scale_factor << ", cell_count=" << cell_count - << ", is_unsigned_value: " << unsigned_value << "\n"; + << ", is_unsigned_value: " << unsigned_value << " for " << GET_NC_NAME_P(var) << "\n"; switch ( type_id ) { + case NcType::nc_INT64: + { + long long fill_value = bad_data_int; + long long min_value = 2147483647; + long long max_value = -2147483648; + long long *packed_data = new long long[cell_count]; + + if (IS_VALID_NC_P(att_fill_value)) + fill_value = get_att_value_int(att_fill_value); + + var->getVar(packed_data); + _apply_scale_factor(data, packed_data, cell_count, + fill_value, min_value, max_value, "int", + add_offset, scale_factor); + delete [] packed_data; + } + break; case NcType::nc_INT: { int fill_value = bad_data_int; @@ -2001,15 +2037,16 @@ bool get_nc_data(NcVar *var, double *data) { } break; default: - mlog << Debug(1) << method_name << "type_id: " - << type_id << " type name: " << GET_NC_TYPE_NAME_P(var) - << "\n"; + return_status = false; + mlog << Debug(1) << method_name << "Did not read data because of unsupported data type (" + << type_id << ", type name: " << GET_NC_TYPE_NAME_P(var) + << ") for " << GET_NC_NAME_P(var) << "\n"; + } if(att_add_offset) delete att_add_offset; if(att_scale_factor) delete att_scale_factor; if(att_fill_value) delete att_fill_value; } - return_status = true; } return(return_status); } From 5b500e2ffc5e8b6a6dc057d6156a57a9fa42b84e Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Tue, 5 Apr 2022 17:43:16 -0600 Subject: [PATCH 2/6] #2123 Initialize ValidTime if failed to read time variable --- met/src/libcode/vx_data2d_nccf/nccf_file.cc | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/met/src/libcode/vx_data2d_nccf/nccf_file.cc b/met/src/libcode/vx_data2d_nccf/nccf_file.cc index dbe5e4e323..7dcb7dc99a 100644 --- a/met/src/libcode/vx_data2d_nccf/nccf_file.cc +++ b/met/src/libcode/vx_data2d_nccf/nccf_file.cc @@ -296,17 +296,15 @@ bool NcCfFile::open(const char * filepath) } // Parse the units for the time variable. + ut = sec_per_unit = 0; NcVarAtt *units_att = get_nc_att(valid_time_var, (string)"units", false); if (IS_VALID_NC_P(units_att)) { - get_att_value_chars(units_att, units); - - if (units.length() == 0) + if (!get_att_value_chars(units_att, units) || units.length() == 0) { mlog << Warning << "\n" << method_name << "the \"time\" variable must contain a \"units\" attribute. " << "Using valid time of 0\n\n"; - ut = sec_per_unit = 0; } else { @@ -315,14 +313,10 @@ bool NcCfFile::open(const char * filepath) parse_cf_time_string(units.c_str(), ut, sec_per_unit); } } - else - { - ut = sec_per_unit = 0; - } + if (units_att) delete units_att; NcVar bounds_time_var; NcVar *nc_time_var = (NcVar *)0; - nc_time_var = valid_time_var; bool use_bounds_var = false; ConcatString bounds_var_name; nc_time_var = valid_time_var; @@ -338,12 +332,12 @@ bool NcCfFile::open(const char * filepath) } if (bounds_att) delete bounds_att; - if (units_att) delete units_att; // Determine the number of times present. int n_times = (int) get_data_size(valid_time_var); int tim_buf_size = n_times; if (use_bounds_var) tim_buf_size *= 2; double *time_values = new double[tim_buf_size]; + if( get_nc_data(nc_time_var, time_values) ) { bool no_leap_year = get_att_no_leap_year(valid_time_var); if( time_dim_count > 1 ) { @@ -375,6 +369,7 @@ bool NcCfFile::open(const char * filepath) } } } + else ValidTime.add(0); //Initialize delete [] time_values; } From 391c58a32e814e4d3a09ca983bf5beab36725cdb Mon Sep 17 00:00:00 2001 From: hsoh-u Date: Tue, 5 Apr 2022 20:07:14 -0600 Subject: [PATCH 3/6] #2123 Support int64 data type --- met/src/libcode/vx_nc_util/nc_utils.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/met/src/libcode/vx_nc_util/nc_utils.cc b/met/src/libcode/vx_nc_util/nc_utils.cc index 36e97e40b7..0fa9f87df9 100644 --- a/met/src/libcode/vx_nc_util/nc_utils.cc +++ b/met/src/libcode/vx_nc_util/nc_utils.cc @@ -1581,7 +1581,7 @@ bool get_nc_data(NcVar *var, float *data) { var->getVar(packed_data); _apply_scale_factor(data, packed_data, cell_count, - fill_value, min_value, max_value, "int", + fill_value, min_value, max_value, "int64", add_offset, scale_factor); delete [] packed_data; } @@ -1878,7 +1878,7 @@ bool get_nc_data(NcVar *var, double *data) { var->getVar(packed_data); _apply_scale_factor(data, packed_data, cell_count, - fill_value, min_value, max_value, "int", + fill_value, min_value, max_value, "int64", add_offset, scale_factor); delete [] packed_data; } From c0574abd7f9ce455a22c28fcbd8b3ef86ee31c96 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Fri, 8 Apr 2022 13:06:34 -0600 Subject: [PATCH 4/6] #2123 Added plot_data_plane_NCCF_time_int64 to test int64 time variable --- test/xml/unit_plot_data_plane.xml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/xml/unit_plot_data_plane.xml b/test/xml/unit_plot_data_plane.xml index 3db1b4bfa0..0681bc741f 100644 --- a/test/xml/unit_plot_data_plane.xml +++ b/test/xml/unit_plot_data_plane.xml @@ -271,13 +271,27 @@ + + &MET_BIN;/plot_data_plane + \ + &DATA_DIR_MODEL;/nccf/precipitation_int64_time.nc \ + &OUTPUT_DIR;/plot_data_plane/plot_data_plane_NCCF_int64_time.ps \ + 'name="precipitation"; level="(*,*)";' \ + -title "NCCF GloTEC - Total Electron Content" \ + -v 1 + + + &OUTPUT_DIR;/plot_data_plane/plot_data_plane_NCCF_int64_time.ps + + + &MET_BIN;/plot_data_plane \ &DATA_DIR_MODEL;/nccf/prods_op_ukv_rotated_latlon_20220301_00_000.nc \ &OUTPUT_DIR;/plot_data_plane/plot_data_plane_NCCF_rotlatlon.ps \ 'name="surface_altitude"; level="(*,*)";' \ - -title "Rotated Lat/Lon Surface Altitude" \ + -title "Observation precipitation" \ -v 1 From 62b33a4956ee40ae5c7604aeefe5cfc50d1e81e1 Mon Sep 17 00:00:00 2001 From: johnhg Date: Fri, 8 Apr 2022 14:15:51 -0600 Subject: [PATCH 5/6] Correct name of plot in unit_plot_data_plane.xml --- test/xml/unit_plot_data_plane.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/xml/unit_plot_data_plane.xml b/test/xml/unit_plot_data_plane.xml index 0681bc741f..1ba52f44bf 100644 --- a/test/xml/unit_plot_data_plane.xml +++ b/test/xml/unit_plot_data_plane.xml @@ -291,7 +291,7 @@ &DATA_DIR_MODEL;/nccf/prods_op_ukv_rotated_latlon_20220301_00_000.nc \ &OUTPUT_DIR;/plot_data_plane/plot_data_plane_NCCF_rotlatlon.ps \ 'name="surface_altitude"; level="(*,*)";' \ - -title "Observation precipitation" \ + -title "Rotated Lat/Lon Surface Altitude" \ -v 1 From 30664098c17bfae22ee751e9bd57f9ceb045f299 Mon Sep 17 00:00:00 2001 From: hsoh-u Date: Fri, 8 Apr 2022 16:05:19 -0600 Subject: [PATCH 6/6] Correct name of plot in unit_plot_data_plane.xml --- test/xml/unit_plot_data_plane.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/xml/unit_plot_data_plane.xml b/test/xml/unit_plot_data_plane.xml index 1ba52f44bf..4ff6c14c56 100644 --- a/test/xml/unit_plot_data_plane.xml +++ b/test/xml/unit_plot_data_plane.xml @@ -277,7 +277,7 @@ &DATA_DIR_MODEL;/nccf/precipitation_int64_time.nc \ &OUTPUT_DIR;/plot_data_plane/plot_data_plane_NCCF_int64_time.ps \ 'name="precipitation"; level="(*,*)";' \ - -title "NCCF GloTEC - Total Electron Content" \ + -title "Observation precipitation" \ -v 1