Port to Hercules (#880)

- Update build. 
- Add regression test driver scripts.
- Add grid generation driver script.
- Update link_fixdirs.sh.
- Fix the GitHub workflow to use the latest splib version.

Fixes #878.

.github/PULL_REQUEST_TEMPLATE

@@ -16,7 +16,7 @@ One or more paragraphs describing the problem, solution, and required changes.
 ## TESTS CONDUCTED: 
 If there are changes to the build or source code, the tests below must be conducted. Contact a repository manager if you need assistance.
 
-- [ ] Compile branch on all Tier 1 machines using Intel (Orion, Jet, Hera and WCOSS2).
+- [ ] Compile branch on all Tier 1 machines using Intel (Orion, Jet, Hera, Hercules and WCOSS2).
 - [ ] Compile branch on Hera using GNU.
 - [ ] Compile branch in 'Debug' mode on WCOSS2.
 - [ ] Run unit tests locally on any Tier 1 machine.

.github/workflows/ubuntu_intel.yaml

@@ -50,14 +50,14 @@ jobs:
           sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
           echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
           sudo apt-get update
-          sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
+          sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran-2023.2.1 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.1
           echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile
 
       # Install dependencies using Spack
       - name: install-dependencies-with-spack
         if: steps.cache-env.outputs.cache-hit != 'true'
         run: |
-          git clone -c feature.manyFiles=true https://github.com/NOAA-EMC/spack.git
+          git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
           source spack/share/spack/setup-env.sh
           sed "s/\[intel, gcc@10:10, apple-clang@14\]/\[intel\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
           spack env create ufs_utils-env spack_ci.yaml

README.md

@@ -59,7 +59,7 @@ It also uses the following repositories:
 
 ## Installing
 
-On Orion, Jet, Hera and WCOSS2 do the following:
+On Orion, Hercules, Jet, Hera and WCOSS2 do the following:
 
 1) Set the 'fixed' directories using the `link_fixdirs.sh`
 script in `./fix`. Usage: `./link_fixdirs.sh $RUN_ENVIR $machine`,

ci/spack.yaml

@@ -10,7 +10,7 @@ spack:
   - [email protected]
   - [email protected] precision=d
   - [email protected]
-  - sp@2.3.3
+  - sp@2.5.0
   - [email protected]
   - [email protected]
   - [email protected]
@@ -19,4 +19,4 @@ spack:
   - [email protected]~debug~xerces+external-parallelio
   view: true
   concretizer:
-    unify: true
+    unify: when_possible

cmake/LibMPI.cmake

@@ -94,6 +94,17 @@ function (platform_name RETURN_VARIABLE)
 
     set (${RETURN_VARIABLE} "orion" PARENT_SCOPE)
 
+  elseif (SITENAME MATCHES "^Hercules-login-1.HPC.MsState.Edu" OR
+      SITENAME MATCHES "^Hercules-login-2.HPC.MsState.Edu" OR
+      SITENAME MATCHES "^Hercules-login-3.HPC.MsState.Edu" OR
+      SITENAME MATCHES "^Hercules-login-4.HPC.MsState.Edu" OR
+      SITENAME MATCHES "^hercules-login-1.hpc.msstate.edu" OR
+      SITENAME MATCHES "^hercules-login-2.hpc.msstate.edu" OR
+      SITENAME MATCHES "^hercules-login-3.hpc.msstate.edu" OR
+      SITENAME MATCHES "^hercules-login-4.hps.msstate.edu")
+
+    set (${RETURN_VARIABLE} "hercules" PARENT_SCOPE)
+
   elseif (SITENAME MATCHES "^cheyenne1.cheyenne.ucar.edu" OR
       SITENAME MATCHES "^cheyenne1.cheyenne.ucar.edu" OR
       SITENAME MATCHES "^cheyenne2.cheyenne.ucar.edu" OR

cmake/mpiexec.hercules

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# Arguments:
+#
+#   $1  - Number of MPI Tasks
+#   $2+ - Executable and its arguments
+#
+
+ACCOUNT=
+QOS=debug
+
+NP=$1
+shift
+
+srun -A $ACCOUNT -q $QOS -n $NP $@

driver_scripts/driver_grid.hercules.sh

@@ -0,0 +1,178 @@
+#!/bin/bash
+
+#SBATCH -J fv3_grid_driver
+#SBATCH -A fv3-cpu
+#SBATCH --open-mode=truncate
+#SBATCH -o log.fv3_grid_driver
+#SBATCH -e log.fv3_grid_driver
+#SBATCH --nodes=2 --ntasks-per-node=15
+#SBATCH -q debug
+#SBATCH -t 00:30:00
+
+#-----------------------------------------------------------------------
+# Driver script to create a cubic-sphere based model grid on Hercules.
+#
+# Produces the following files (netcdf, each tile in separate file):
+#   1) 'mosaic' and 'grid' files containing lat/lon and other
+#      records that describe the model grid.
+#   2) 'oro' files containing land mask, terrain and gravity
+#      wave drag fields.
+#   3) Surface climo fields, such as soil type, vegetation
+#      greenness and albedo.
+#
+# Note: The sfc_climo_gen program only runs with an
+#       mpi task count that is a multiple of six.  This is
+#       an ESMF library requirement.  Large grids may require
+#       tasks spread across multiple nodes. The orography code
+#       benefits from threads.
+#
+# To run, do the following:
+#
+#   1) Set "C" resolution, "res" - Example: res=96.
+#   2) Set grid type ("gtype").  Valid choices are
+#         "uniform"  - global uniform grid
+#         "stretch"  - global stretched grid
+#         "nest"     - global stretched grid with nest
+#         "regional_gfdl" - stand-alone gfdl regional grid
+#         "regional_esg"  - stand-alone extended Schmidt gnomonic
+#                           (esg) regional grid
+#   3) For "uniform" and "regional_gfdl" grids - to include lake
+#      fraction and depth, set "add_lake" to true, and the
+#      "lake_cutoff" value.
+#   4) For "stretch" and "nest" grids, set the stretching factor -
+#       "stretch_fac", and center lat/lon of highest resolution
+#      tile - "target_lat" and "target_lon".
+#   5) For "nest" grids, set the refinement ratio - "refine_ratio",
+#      the starting/ending i/j index location within the parent
+#      tile - "istart_nest", "jstart_nest", "iend_nest", "jend_nest"
+#   6) For "regional_gfdl" grids, set the "halo".  Default is three
+#      rows/columns.
+#   7) For "regional_esg" grids, set center lat/lon of grid,
+#      - "target_lat/lon" - the i/j dimensions - "i/jdim", the
+#      x/y grid spacing - "delx/y", and halo.
+#   8) Set working directory - TEMP_DIR - and path to the repository
+#      clone - home_dir.
+#   9) To use the GSL orographic drag suite, set 'make_gsl_orog' to true.
+#  10) Set 'soil_veg_src' and 'veg_type_src' to choose the
+#      soil type and vegetation type data.
+#  11) Submit script: "sbatch $script".
+#  12) All files will be placed in "out_dir".
+#
+#-----------------------------------------------------------------------
+
+set -x
+
+source ../sorc/machine-setup.sh > /dev/null 2>&1
+module use ../modulefiles
+module load build.$target.intel
+module list
+
+#-----------------------------------------------------------------------
+# Set grid specs here.
+#-----------------------------------------------------------------------
+
+export gtype=uniform     # 'uniform', 'stretch', 'nest', 
+                               # 'regional_gfdl', 'regional_esg'
+
+export make_gsl_orog=false    # When 'true' will output 'oro' files for 
+                              # the GSL orographic drag suite.
+
+export vegsoilt_frac='.false.'  # When true, outputs percent of each
+                               # soil and veg type category and a
+                               # dominant category. When false, only
+                               # outputs the dominant category. A
+                               # Fortran logical, so include the dots.
+
+export veg_type_src="viirs.v3.igbp.30s" #  Vegetation type data.
+                                # For viirs-based vegetation type data, set to:
+                                # 1) "viirs.v3.igbp.30s" for global 30s data
+                                # For the modis-based data, set to:
+                                # 1) "modis.igbp.0.05" for global 0.05-deg data
+                                # 2) "modis.igbp.0.03" for global 0.03-deg data
+                                # 3) "modis.igbp.conus.30s" for CONUS 30s data
+                                # 4) "modis.igbp.nh.30s" for N Hemis 30s data
+                                # 5) "modis.igbp.30s" for global 30s data
+
+export soil_type_src="bnu.v3.30s" #  Soil type data.
+                                # For Beijing Normal Univ. data, set to:
+                                # 1) "bnu.v3.30s" for global 30s data.
+                                # For STATSGO soil type data, set to:
+                                # 1) "statsgo.0.05" for global 0.05-deg data
+                                # 2) "statsgo.0.03" for global 0.03-deg data
+                                # 3) "statsgo.conus.30s" for CONUS 30s data
+                                # 4) "statsgo.nh.30s" for NH 30s data
+                                # 5) "statsgo.30s" for global 30s data
+
+# choose dataset sources for lakefrac & lakedepth so that lake_data_srce=LakeFrac_LakeDepth; 
+# available options are 'MODISP_GLDBV3', 'MODISP_GLOBATHY', 'VIIRS_GLDBV3', 'VIIRS_GLOBATHY' & 'GLDBV3'
+export lake_data_srce=MODISP_GLDBV3 
+
+if [ $gtype = uniform ]; then
+  export res=96
+  export add_lake=true         # Add lake frac and depth to orography data.
+  export lake_cutoff=0.50      # return 0 if lake_frac <  lake_cutoff & add_lake=T
+  export binary_lake=1         # return 1 if lake_frac >= lake_cutoff & add_lake=T
+  export ocn=${ocn:-"025"}     # use one of  "025", "050", "100", "500". Cannot be empty
+elif [ $gtype = stretch ]; then
+  export res=96
+  export stretch_fac=1.5       # Stretching factor for the grid
+  export target_lon=-97.5      # Center longitude of the highest resolution tile
+  export target_lat=35.5       # Center latitude of the highest resolution tile
+elif [ $gtype = nest ] || [ $gtype = regional_gfdl ]; then
+  export add_lake=false        # Add lake frac and depth to orography data.
+  export lake_cutoff=0.20      # lake frac < lake_cutoff ignored when add_lake=T
+  export res=768
+  export stretch_fac=1.5       # Stretching factor for the grid
+  export target_lon=-97.5      # Center longitude of the highest resolution tile
+  export target_lat=38.5       # Center latitude of the highest resolution tile
+  export refine_ratio=3        # The refinement ratio
+  export istart_nest=123       # Starting i-direction index of nest grid in parent tile supergrid
+  export jstart_nest=331       # Starting j-direction index of nest grid in parent tile supergrid
+  export iend_nest=1402        # Ending i-direction index of nest grid in parent tile supergrid
+  export jend_nest=1194        # Ending j-direction index of nest grid in parent tile supergrid
+  export halo=3                # Lateral boundary halo
+elif [ $gtype = regional_esg ] ; then
+  export res=-999              # equivalent resolution is computed
+  export target_lon=-97.5      # Center longitude of grid
+  export target_lat=35.5       # Center latitude of grid
+  export idim=301              # Dimension of grid in 'i' direction
+  export jdim=200              # Dimension of grid in 'j' direction
+  export delx=0.0585           # Grid spacing (in degrees) in the 'i' direction
+                               # on the SUPERGRID (which has twice the resolution of
+                               # the model grid).  The physical grid spacing in the 'i'
+                               # direction is related to delx as follows:
+                               #    distance = 2*delx*(circumf_Earth/360 deg)
+  export dely=0.0585           # Grid spacing (in degrees) in the 'j' direction.
+  export halo=3                # number of row/cols for halo
+fi
+
+#-----------------------------------------------------------------------
+# Check paths.
+#   home_dir - location of repository.
+#   TEMP_DIR - working directory.
+#   out_dir  - where files will be placed upon completion.
+#-----------------------------------------------------------------------
+
+export home_dir=$SLURM_SUBMIT_DIR/..
+export TEMP_DIR=/work/noaa/stmp/$LOGNAME/fv3_grid.$gtype
+export out_dir=/work/noaa/stmp/$LOGNAME/my_grids
+
+#-----------------------------------------------------------------------
+# Should not need to change anything below here.
+#-----------------------------------------------------------------------
+
+export APRUN=time
+export APRUN_SFC=srun
+export OMP_NUM_THREADS=24
+export OMP_STACKSIZE=2048m
+
+ulimit -a
+ulimit -s unlimited
+
+#-----------------------------------------------------------------------
+# Start script.
+#-----------------------------------------------------------------------
+
+$home_dir/ush/fv3gfs_driver_grid.sh
+
+exit

fix/link_fixdirs.sh

@@ -9,15 +9,15 @@ set -ex
 #               'nco' (copies data).
 #
 #  $machine - is the machine. Choices are:
-#             'wcoss2', 'hera', 'jet', 'orion', 's4'
+#             'wcoss2', 'hera', 'jet', 'orion', 'hercules', 's4'
 
 RUN_ENVIR=${1}
 machine=${2}
 
 if [ $# -lt 2 ]; then
     set +x
     echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine'
-    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 |  hera  | jet | orion | s4 )'
+    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 |  hera  | jet | orion | hercules | s4 )'
     exit 1
 fi
 
@@ -28,10 +28,10 @@ if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then
     exit 1
 fi
 
-if [ $machine != wcoss2 -a $machine != hera -a $machine != jet -a $machine != orion -a $machine != s4 ]; then
+if [ $machine != wcoss2 -a $machine != hera -a $machine != jet -a $machine != orion -a $machine != s4 -a $machine != hercules ]; then
     set +x
     echo '***ERROR*** unsupported machine'
-    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | hera | jet | orion | s4 )'
+    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | hera | jet | orion | hercules | s4 )'
     exit 1
 fi
 
@@ -48,7 +48,7 @@ if [ $machine = "hera" ]; then
     FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix"
 elif [ $machine = "jet" ]; then
     FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix"
-elif [ $machine = "orion" ]; then
+elif [ $machine = "orion" -o $machine = "hercules" ]; then
     FIX_DIR="/work/noaa/global/glopara/fix"
 elif [ $machine = "wcoss2" ]; then
     FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix"

modulefiles/build.hercules.intel.lua

@@ -0,0 +1,61 @@
+help([[
+Load environment to compile UFS_UTILS on Hercules using Intel
+]])
+
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+bacio_ver=os.getenv("bacio_ver") or "2.4.1"
+load(pathJoin("bacio", bacio_ver))
+
+g2_ver=os.getenv("g2_ver") or "3.4.5"
+load(pathJoin("g2", g2_ver))
+
+ip_ver=os.getenv("ip_ver") or "4.3.0"
+load(pathJoin("ip", ip_ver))
+
+nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
+load(pathJoin("nemsio", nemsio_ver))
+
+sp_ver=os.getenv("sp_ver") or "2.3.3"
+load(pathJoin("sp", sp_ver))
+
+w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
+load(pathJoin("w3emc", w3emc_ver))
+
+sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
+load(pathJoin("sfcio", sfcio_ver))
+
+sigio_ver=os.getenv("sigio_ver") or "2.3.2"
+load(pathJoin("sigio", sigio_ver))
+
+zlib_ver=os.getenv("zlib_ver") or "1.2.13"
+load(pathJoin("zlib", zlib_ver))
+
+png_ver=os.getenv("png_ver") or "1.6.37"
+load(pathJoin("libpng", png_ver))
+
+netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
+load(pathJoin("netcdf-c", netcdf_c_ver))
+
+netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0"
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+esmf_ver=os.getenv("esmf_ver") or "8.4.2"
+load(pathJoin("esmf", esmf_ver))
+
+nco_ver=os.getenv("nco_ver") or "5.0.6"
+load(pathJoin("nco", nco_ver))
+
+whatis("Description: UFS_UTILS build environment")