Merge branch 'develop' into feature_2708_mvmode_merge_flag_default

internal/scripts/environment/development.docker

@@ -40,3 +40,6 @@ export COMPILE_HDF=1
 export COMPILE_HDFEOS=1
 export COMPILE_FREETYPE=1
 export COMPILE_CAIRO=1
+
+export SQLITE_INCLUDE_DIR=/usr/include
+export SQLITE_LIB_DIR=/usr/lib/x86_64-linux-gnu

internal/scripts/installation/compile_MET_all.sh

@@ -36,11 +36,11 @@
 #
 # The compile_MET_all.sh script will compile and install MET and its
 # external library dependencies, if needed, including:
-# PROJ, GSL, BUFRLIB, GRIB2C (with dependencies Z, PNG, JASPER),
-# HDF5, NETCDF (C and CXX), HDF4 (optional for MODIS-Regrid
-# and lidar2nc), HDFEOS (optional for MODIS-Regrid and lidar2nc),
-# FREETYPE (optional for MODE Graphics), and CAIRO (optional
-# for MODE Graphics).
+# PROJ (with dependency SQLITE >= 3.11), GSL, BUFRLIB, 
+# GRIB2C (with dependencies Z, PNG, JASPER), HDF5, NETCDF (C and CXX), 
+# HDF4 (optional for MODIS-Regrid and lidar2nc), HDFEOS (optional for
+# MODIS-Regrid and lidar2nc), FREETYPE (optional for MODE Graphics),
+# and CAIRO (optional for MODE Graphics).
 #
 # If these libraries have already been installed and don't need to be
 # reinstalled or if you are compiling on a machine that uses modulefiles
@@ -49,9 +49,9 @@
 # need to set to let MET know where the library and header files are.
 # Please supply values for the following environment variables
 # in the input environment configuration file (install_met_env.<machine_name>:
-# MET_GRIB2CLIB, MET_GRIB2CINC, GRIB2CLIB_NAME,
-# MET_BUFRLIB, BUFRLIB_NAME, MET_HDF5, MET_NETCDF,
-# MET_PROJ, MET_GSL, LIB_JASPER, LIB_PNG, LIB_Z.
+# MET_GRIB2CLIB, MET_GRIB2CINC, GRIB2CLIB_NAME, MET_BUFRLIB, BUFRLIB_NAME, 
+# MET_HDF5, MET_NETCDF, MET_PROJ, MET_GSL, LIB_JASPER, LIB_PNG, LIB_Z,
+# SQLITE_INCLUDE_DIR, SQLITE_LIB_DIR.
 #
 # The optional libraries HDF4, HDFEOS, FREETYPE, and CAIRO are
 # used for the following, not widely used tools, MODIS-Regrid,
@@ -155,6 +155,13 @@ if [[ -z "$LIB_Z" ]]; then
   LIB_Z=${LIB_DIR}/lib
 fi
 
+# if SQLITE is not defined in the environment file, enable its compilation
+if [[ -z ${SQLITE_INCLUDE_DIR} ]] && [[ -z ${SQLITE_LIB_DIR} ]]; then
+   COMPILE_SQLITE=1
+else
+   COMPILE_SQLITE=0       
+fi
+
 # Constants
 if [[ -z ${MET_GRIB2CLIB} ]] && [[ -z ${MET_GRIB2C} ]]; then
   COMPILE_ZLIB=1
@@ -394,6 +401,21 @@ fi
 # Compile Proj
 if [ $COMPILE_PROJ -eq 1 ]; then
 
+  if [ $COMPILE_SQLITE -eq 1 ]; then
+    echo
+    echo "Compiling SQLITE at `date`"
+    mkdir -p ${LIB_DIR}/sqlite
+    rm -rf ${LIB_DIR}/sqlite/sqlite*
+    tar -xf ${TAR_DIR}/sqlite*.tar.gz -C ${LIB_DIR}/sqlite > /dev/null 2>&1
+    cd ${LIB_DIR}/sqlite/sqlite*
+    echo "cd `pwd`"
+    run_cmd "./configure --enable-shared --prefix=${LIB_DIR} > sqlite.configure.log 2>&1"
+    run_cmd "make ${MAKE_ARGS} > sqlite.make.log 2>&1"
+    run_cmd "make ${MAKE_ARGS} install > sqlite.make_install.log 2>&1"
+    export SQLITE_INCLUDE_DIR=${LIB_DIR}/include
+    export SQLITE_LIB_DIR=${LIB_DIR}/lib
+  fi
+
   vrs="9.2.1";
 
   echo
@@ -403,8 +425,9 @@ if [ $COMPILE_PROJ -eq 1 ]; then
   tar -xf ${TAR_DIR}/proj-${vrs}.tar.gz -C ${LIB_DIR}/proj
   cd ${LIB_DIR}/proj/proj*
   echo "cd `pwd`"
+  export PATH=${LIB_DIR}/bin:${PATH}
   run_cmd "mkdir build; cd build"
-  run_cmd "cmake -DCMAKE_INSTALL_PREFIX=${LIB_DIR} .."
+  run_cmd "cmake -DCMAKE_INSTALL_PREFIX=${LIB_DIR} -DSQLITE3_INCLUDE_DIR=${SQLITE_INCLUDE_DIR} -DSQLITE3_LIBRARY=${SQLITE_LIB_DIR}/libsqlite3.so .."
   run_cmd "cmake --build ."
   run_cmd "cmake --build . --target install"
 
@@ -716,8 +739,7 @@ if [ -z ${MET_GSL} ]; then
 fi
 
 if [ -z ${MET_PROJ} ]; then
-  export MET_PROJINC=${LIB_DIR}/include
-  export MET_PROJLIB=${LIB_DIR}/lib64
+  export MET_PROJ=${LIB_DIR}
 fi
 
 export MET_PYTHON_BIN_EXE=${MET_PYTHON_BIN_EXE:=${MET_PYTHON}/bin/python3}
@@ -732,10 +754,10 @@ fi
 # https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html
 # ${parameter:+word}
 # If parameter is null or unset, nothing is substituted, otherwise the expansion of word is substituted.
-export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_DIR}/lib${MET_PROJ:+:$MET_PROJ/lib64}${LIB_DIR}/lib${MET_NETCDF:+:$MET_NETCDF/lib}${MET_HDF5:+:$MET_HDF5/lib}${MET_BUFRLIB:+:$MET_BUFRLIB}${MET_GRIB2CLIB:+:$MET_GRIB2CLIB}${MET_PYTHON_LIB:+:$MET_PYTHON_LIB}${MET_GSL:+:$MET_GSL/lib}${ADDTL_DIR:+:$ADDTL_DIR}"
+export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_DIR}/lib:${MET_PROJ:+:$MET_PROJ/lib64}${LIB_DIR}/lib${MET_NETCDF:+:$MET_NETCDF/lib}${MET_HDF5:+:$MET_HDF5/lib}${MET_BUFRLIB:+:$MET_BUFRLIB}${MET_GRIB2CLIB:+:$MET_GRIB2CLIB}${MET_PYTHON_LIB:+:$MET_PYTHON_LIB}${MET_GSL:+:$MET_GSL/lib}${ADDTL_DIR:+:$ADDTL_DIR}"
 export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_JASPER:+$LIB_JASPER}${LIB_LIBPNG:+:$LIB_PNG}${LIB_Z:+$LIB_Z}"
 export LDFLAGS="${LDFLAGS} ${LIB_JASPER:+-L$LIB_JASPER} ${LIB_LIBPNG:+-L$LIB_LIBPNG} ${MET_HDF5:+-L$MET_HDF5/lib} ${ADDTL_DIR:+-L$ADDTL_DIR}"
-export LIBS="${LIBS} -lhdf5_hl -lhdf5 -lz"
+export LIBS="${LIBS} -lhdf5_hl -lhdf5 -lz -ltiff"
 export MET_FONT_DIR=${TEST_BASE}/fonts
 
 if [ "${SET_D64BIT}" = "TRUE" ]; then

internal/scripts/installation/config/install_met_env.hera

@@ -1,5 +1,7 @@
+module load gnu/9.2.0
 module load intel/2022.1.2
 module load cmake/3.26.4
+
 export PATH=/scratch1/BMC/dtc/miniconda/miniconda3/envs/metplus_v5.1_py3.10/bin:${PATH}
 export TEST_BASE=/contrib/met/12.0.0
 export COMPILER=intel_2022.1.2
@@ -10,6 +12,7 @@ export MET_PYTHON=/scratch1/BMC/dtc/miniconda/miniconda3/envs/metplus_v5.1_py3.1
 export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.10
 export MET_PYTHON_LD=`python3-config --ldflags --embed`
 export SET_D64BIT=FALSE
+export ADDTL_DIR=/usr/lib64
 export EXTERNAL_LIBS=/contrib/met/12.0.0/external_libs/
 #export MET_PROJ=${EXTERNAL_LIBS}
 #export MET_NETCDF=${EXTERNAL_LIBS}
@@ -23,6 +26,8 @@ export EXTERNAL_LIBS=/contrib/met/12.0.0/external_libs/
 #export LIB_JASPER=${EXTERNAL_LIBS}/lib
 #export LIB_LIBPNG=${EXTERNAL_LIBS}/lib
 #export LIB_Z=${EXTERNAL_LIBS}/lib
+#export SQLITE_INCLUDE_DIR=${EXTERNAL_LIBS}/include
+#export SQLITE_LIB_DIR=${EXTERNAL_LIBS}/lib
 #export CFLAGS="-Wall -g"
 #export CXXFLAGS="-Wall -g -lcurl"
 export MAKE_ARGS=-j

internal/scripts/installation/config/install_met_env.jet

@@ -1,27 +1,36 @@
+module load gnu/9.2.0
 module load intel/2022.1.2
-module load netcdf/4.7.0
-module load hdf5/1.10.5
+module load cmake/3.26.4
 
-export TEST_BASE=/contrib/met/11.1.0
-export COMPILER=intel_18.0.5.274
+export FC=ifort
+export F77=ifort
+export F90=ifort
+export CC=icc
+export CXX=icpc
+export TEST_BASE=/contrib/met/12.0.0
+export COMPILER=intel_2022.1.2
 export MET_SUBDIR=${TEST_BASE}
-export MET_TARBALL=v11.1.0.tar.gz
+export MET_TARBALL=v12.0.0.tar.gz
 export USE_MODULES=TRUE
 export MET_PYTHON=/mnt/lfs1/HFIP/dtc-hurr/METplus/miniconda/miniconda3/envs/metplus_v5.1_py3.10
 export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.10
 export MET_PYTHON_LD=-L${MET_PYTHON}/lib/python3.10/config-3.10-x86_64-linux-gnu\ -L${MET_PYTHON}/lib\ -lpython3.10\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lrt\ -lm\ -lm
-export MET_NETCDF=/apps/netcdf/4.7.0/intel/18.0.5.274
-export MET_HDF5=/apps/hdf5/1.10.5/intel/18.0.5.274
+export ADDTL_DIR=/usr/lib64
 export EXTERNAL_LIBS=${TEST_BASE}/external_libs/
+#export MET_PROJ=${EXTERNAL_LIBS}
 #export MET_GSL=${EXTERNAL_LIBS}
 #export MET_BUFRLIB=${EXTERNAL_LIBS}
 #export BUFRLIB_NAME=-lbufr
 #export MET_GRIB2CLIB=${EXTERNAL_LIBS}/lib
 #export MET_GRIB2CINC=${EXTERNAL_LIBS}/include
 #export GRIB2CLIB_NAME=-lgrib2c
+#export MET_HDF5=${EXTERNAL_LIBS}
+#export MET_NETCDF=${EXTERNAL_LIBS}
 #export LIB_JASPER=${EXTERNAL_LIBS}/lib
 #export LIB_LIBPNG=${EXTERNAL_LIBS}/lib
 #export LIB_Z=${EXTERNAL_LIBS}/lib
+#export SQLITE_INCLUDE_DIR=${EXTERNAL_LIBS}/include
+#export	SQLITE_LIB_DIR=${EXTERNAL_LIBS}/lib
 export MAKE_ARGS=-j
 export SET_D64BIT=FALSE
 export CXXFLAGS="-std=c++11"

internal/scripts/installation/config/install_met_env_met_only.hera

@@ -1,5 +1,7 @@
+module load gnu/9.2.0
 module load intel/2022.1.2
 module load cmake/3.26.4
+
 export PATH=/scratch1/BMC/dtc/miniconda/miniconda3/envs/metplus_v5.1_py3.10/bin:${PATH}
 export TEST_BASE=/contrib/met/12.0.0
 export COMPILER=intel_2022.1.2
@@ -10,6 +12,7 @@ export MET_PYTHON=/scratch1/BMC/dtc/miniconda/miniconda3/envs/metplus_v5.1_py3.1
 export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.10
 export MET_PYTHON_LD=`python3-config --ldflags --embed`
 export SET_D64BIT=FALSE
+export ADDTL_DIR=/usr/lib64
 export EXTERNAL_LIBS=/contrib/met/12.0.0/external_libs/
 export MET_PROJ=${EXTERNAL_LIBS}
 export MET_NETCDF=${EXTERNAL_LIBS}
@@ -23,6 +26,8 @@ export GRIB2CLIB_NAME=-lgrib2c
 export LIB_JASPER=${EXTERNAL_LIBS}/lib
 export LIB_LIBPNG=${EXTERNAL_LIBS}/lib
 export LIB_Z=${EXTERNAL_LIBS}/lib
+export SQLITE_INCLUDE_DIR=${EXTERNAL_LIBS}/include
+export SQLITE_LIB_DIR=${EXTERNAL_LIBS}/lib
 #export CFLAGS="-Wall -g"
 #export CXXFLAGS="-Wall -g -lcurl"
 export MAKE_ARGS=-j

internal/scripts/installation/config/install_met_env_met_only.jet

@@ -1,27 +1,36 @@
+module load gnu/9.2.0
 module load intel/2022.1.2
-module load netcdf/4.7.0
-module load hdf5/1.10.5
+module load cmake/3.26.4
 
-export TEST_BASE=/contrib/met/11.1.0
-export COMPILER=intel_18.0.5.274
+export FC=ifort
+export F77=ifort
+export F90=ifort
+export CC=icc
+export CXX=icpc
+export TEST_BASE=/contrib/met/12.0.0
+export COMPILER=intel_2022.1.2
 export MET_SUBDIR=${TEST_BASE}
-export MET_TARBALL=v11.1.0.tar.gz
+export MET_TARBALL=v12.0.0.tar.gz
 export USE_MODULES=TRUE
 export MET_PYTHON=/mnt/lfs1/HFIP/dtc-hurr/METplus/miniconda/miniconda3/envs/metplus_v5.1_py3.10
 export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.10
-export MET_PYTHON_LD=`${MET_PYTHON}/bin/python3-config --ldflags --embed`
-export MET_NETCDF=/apps/netcdf/4.7.0/intel/18.0.5.274
-export MET_HDF5=/apps/hdf5/1.10.5/intel/18.0.5.274
+export MET_PYTHON_LD=-L${MET_PYTHON}/lib/python3.10/config-3.10-x86_64-linux-gnu\ -L${MET_PYTHON}/lib\ -lpython3.10\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lrt\ -lm\ -lm
+export ADDTL_DIR=/usr/lib64
 export EXTERNAL_LIBS=${TEST_BASE}/external_libs/
+export MET_PROJ=${EXTERNAL_LIBS}
 export MET_GSL=${EXTERNAL_LIBS}
 export MET_BUFRLIB=${EXTERNAL_LIBS}
 export BUFRLIB_NAME=-lbufr
 export MET_GRIB2CLIB=${EXTERNAL_LIBS}/lib
 export MET_GRIB2CINC=${EXTERNAL_LIBS}/include
 export GRIB2CLIB_NAME=-lgrib2c
+export MET_HDF5=${EXTERNAL_LIBS}
+export MET_NETCDF=${EXTERNAL_LIBS}
 export LIB_JASPER=${EXTERNAL_LIBS}/lib
 export LIB_LIBPNG=${EXTERNAL_LIBS}/lib
 export LIB_Z=${EXTERNAL_LIBS}/lib
+export SQLITE_INCLUDE_DIR=${EXTERNAL_LIBS}/include
+export SQLITE_LIB_DIR=${EXTERNAL_LIBS}/lib
 export MAKE_ARGS=-j
 export SET_D64BIT=FALSE
 export CXXFLAGS="-std=c++11"

internal/scripts/installation/modulefiles/12.0.0_jet

@@ -0,0 +1,17 @@
+#%Module######################################################################
+##
+##      Model Evaluation Tools
+##
+proc ModulesHelp { } {
+  puts stderr "Sets up the paths and environment variables to use the Model Evaluation Tools v12.0.0
+     *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***"
+}
+
+prereq intel
+
+set base /contrib/met/12.0.0
+set ver 12.0.0
+set share $base/share/met
+setenv MET_ROOT $base/$ver/MET-12.0.0
+
+prepend-path PATH $base/bin:$base/external_libs/bin:/mnt/lfs1/HFIP/dtc-hurr/METplus/miniconda/miniconda3/envs/metplus_v5.1_py3.10

scripts/python/met/dataplane.py

@@ -180,32 +180,33 @@ def validate_met_data(met_data, fill_value=None):
       from_ndarray = False
       if met_data is None:
          logger.quit(f"{method_name} The met_data is None")
-
-      nx, ny = met_data.shape
-      met_fill_value = dataplane.MET_FILL_VALUE
-      if dataplane.is_xarray_dataarray(met_data):
-         from_xarray = True
-         attrs = met_data.attrs
-         met_data = met_data.data
-         modified_met_data = True
-      if isinstance(met_data, np.ndarray):
-         from_ndarray = True
-         met_data = np.ma.array(met_data)
-
-      if isinstance(met_data, np.ma.MaskedArray):
-         is_int_data = dataplane.is_integer(met_data[0,0]) or dataplane.is_integer(met_data[int(nx/2),int(ny/2)])
-         met_data = np.ma.masked_equal(met_data, float('nan'))
-         met_data = np.ma.masked_equal(met_data, float('inf'))
-         if fill_value is not None:
-            met_data = np.ma.masked_equal(met_data, fill_value)
-         met_data = met_data.filled(int(met_fill_value) if is_int_data else met_fill_value)
       else:
-         logger.log_msg(f"{method_name} unknown datatype {type(met_data)}")
+         nx, ny = met_data.shape
+
+         met_fill_value = dataplane.MET_FILL_VALUE
+         if dataplane.is_xarray_dataarray(met_data):
+            from_xarray = True
+            attrs = met_data.attrs
+            met_data = met_data.data
+            modified_met_data = True
+         if isinstance(met_data, np.ndarray):
+            from_ndarray = True
+            met_data = np.ma.array(met_data)
+
+         if isinstance(met_data, np.ma.MaskedArray):
+            is_int_data = dataplane.is_integer(met_data[0,0]) or dataplane.is_integer(met_data[int(nx/2),int(ny/2)])
+            met_data = np.ma.masked_equal(met_data, float('nan'))
+            met_data = np.ma.masked_equal(met_data, float('inf'))
+            if fill_value is not None:
+               met_data = np.ma.masked_equal(met_data, fill_value)
+            met_data = met_data.filled(int(met_fill_value) if is_int_data else met_fill_value)
+         else:
+            logger.log_msg(f"{method_name} unknown datatype {type(met_data)}")
 
-      if dataplane.KEEP_XARRAY:
-         return xr.DataArray(met_data,attrs=attrs) if from_xarray else met_data
-      else:
-         return met_data
+         if dataplane.KEEP_XARRAY:
+            return xr.DataArray(met_data,attrs=attrs) if from_xarray else met_data
+         else:
+            return met_data
 
 
 def main(argv):

src/basic/vx_util/ascii_table.cc

@@ -1567,7 +1567,6 @@ switch ( just )  {
    default:
       mlog << Error << "\njustified_item() -> bad justification value\n\n";
       exit ( 1 );
-      break;
 
 }   //  switch