Port UFS_UTILS to WCOSS2 (#642)

Add the following for WCOSS2:

Build modules.
Consistency test driver scripts.
Driver script for creating new grids.
Driver script for the GDAS initialization utility.

Fixes #559.

README.md

@@ -15,7 +15,7 @@ https://ufs-community.github.io/UFS_UTILS/.
 Utility | Programmer(s)
 --------|----------
 chgres_cube | George Gayno, Jeff Beck, Larissa Reames
-cpld_gridgen | Denise Worthen, Minsuk Ji
+cpld_gridgen | Denise Worthen
 emcsfc_ice_blend | George Gayno
 emcsfc_snow2mdl | George Gayno
 fre-nctools | GFDL progammer
@@ -58,6 +58,14 @@ It also uses the following repositories:
 
 ## Installing
 
+On Orion, Jet, Hera and WCOSS2, invoke the build script:
+
+```
+./build_all.sh
+```
+
+Otherwise, do:
+
 ```
 mkdir build
 cd build

build_all.sh

@@ -30,7 +30,7 @@ fi
 # access the EMC ftp site, so turn off the build (-DBUILD_TESTING=OFF) of the units tests accordingly.
 # Those with access to the EMC ftp site are: Orion, Hera, WCOSS-Cray, WCOSS-Dell.
 
-if [[ "$target" == "hera" || "$target" == "orion" || "$target" == "wcoss_cray" || "$target" == "wcoss_dell_p3" ]]; then
+if [[ "$target" == "hera" || "$target" == "orion" || "$target" == "wcoss_cray" || "$target" == "wcoss_dell_p3" || "$target" == "wcoss2" ]]; then
   CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DBUILD_TESTING=OFF"
    #CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DBUILD_TESTING=ON"
    #CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DENABLE_DOCS=ON -DBUILD_TESTING=ON"

driver_scripts/driver_grid.wcoss2.sh

@@ -0,0 +1,159 @@
+#!/bin/bash
+
+#PBS -o log
+#PBS -e log
+#PBS -q debug
+#PBS -A GFS-DEV
+#PBS -l walltime=00:05:00
+#PBS -N make_grid
+#PBS -l select=1:ncpus=24:mem=100GB
+
+#-----------------------------------------------------------------------
+# Driver script to create a cubic-sphere based model grid on WCOSS2.
+#
+# Produces the following files (netcdf, each tile in separate file):
+#   1) 'mosaic' and 'grid' files containing lat/lon and other
+#      records that describe the model grid.
+#   2) 'oro' files containing land mask, terrain and gravity
+#      wave drag fields.
+#   3) Surface climo fields, such as soil type, vegetation
+#      greenness and albedo.
+#
+# Note: The sfc_climo_gen program only runs with an
+#       mpi task count that is a multiple of six.  This is
+#       an ESMF library requirement.  Large grids may require
+#       tasks spread across multiple nodes. The orography code
+#       benefits from threads.
+#
+# To run, do the following:
+#
+#   1) Set "C" resolution, "res" - Example: res=96.
+#   2) Set grid type ("gtype").  Valid choices are
+#         "uniform"  - global uniform grid
+#         "stretch"  - global stretched grid
+#         "nest"     - global stretched grid with nest
+#         "regional_gfdl" - stand-alone gfdl regional grid
+#         "regional_esg"  - stand-alone extended Schmidt gnomonic
+#                           (esg) regional grid
+#   3) For "uniform" and "regional_gfdl" grids - to include lake
+#      fraction and depth, set "add_lake" to true, and the
+#      "lake_cutoff" value.
+#   4) For "stretch" and "nest" grids, set the stretching factor -
+#       "stretch_fac", and center lat/lon of highest resolution
+#      tile - "target_lat" and "target_lon".
+#   5) For "nest" grids, set the refinement ratio - "refine_ratio",
+#      the starting/ending i/j index location within the parent
+#      tile - "istart_nest", "jstart_nest", "iend_nest", "jend_nest"
+#   6) For "regional_gfdl" grids, set the "halo".  Default is three
+#      rows/columns.
+#   7) For "regional_esg" grids, set center lat/lon of grid,
+#      - "target_lat/lon" - the i/j dimensions - "i/jdim", the
+#      x/y grid spacing - "delx/y", and halo.
+#   8) Set working directory - TEMP_DIR - and path to the repository
+#      clone - home_dir.
+#   9) Check settings for 'make_gsl_orog' and 'veg_type_src'
+#      below.
+#  10) Submit script: "cat $script | bsub".
+#  11) All files will be placed in "out_dir".
+#
+#-----------------------------------------------------------------------
+
+cd $PBS_O_WORKDIR
+
+source ../sorc/machine-setup.sh > /dev/null 2>&1
+module use ../modulefiles
+module load build.$target.intel
+module list
+
+#-----------------------------------------------------------------------
+# Set grid specs here.
+#-----------------------------------------------------------------------
+
+export gtype=uniform           # 'uniform', 'stretch', 'nest', 
+                               # 'regional_gfdl', 'regional_esg'
+export make_gsl_orog=false     # 'true' if user needs 'oro' files for GSL
+                               # orographic drag suite
+export veg_type_src="modis.igbp.0.05" #  veg type data.
+                                # For viirs-based vegetation type data, set to:
+                                # 1) "viirs.igbp.0.05" for global 5km data
+                                # 2) "viirs.igbp.0.1" for global 10km data
+                                # 3) "viirs.igbp.0.03" for global 3km data
+                                # 4) "viirs.igbp.conus.0.01" for regional 1km data
+                                # For the modis-based data, set to:
+                                # 1) "modis.igbp.0.05" for global 5km data
+                                # 2) "modis.igbp.0.03" for global 3km data
+                                # 3) "modis.igbp.conus.0.01" for regional 1km data
+
+if [ $gtype = uniform ]; then
+  export res=96
+  export add_lake=false        # Add lake frac and depth to orography data.
+  export lake_cutoff=0.20      # lake frac < lake_cutoff ignored when add_lake=T
+elif [ $gtype = stretch ]; then
+  export res=96
+  export stretch_fac=1.5       # Stretching factor for the grid
+  export target_lon=-97.5      # Center longitude of the highest resolution tile
+  export target_lat=35.5       # Center latitude of the highest resolution tile
+elif [ $gtype = nest ] || [ $gtype = regional_gfdl ]; then
+  export add_lake=false        # Add lake frac and depth to orography data.
+  export lake_cutoff=0.20      # lake frac < lake_cutoff ignored when add_lake=T
+  export res=768
+  export stretch_fac=1.5       # Stretching factor for the grid
+  export target_lon=-97.5      # Center longitude of the highest resolution tile
+  export target_lat=38.5       # Center latitude of the highest resolution tile
+  export refine_ratio=3        # The refinement ratio
+  export istart_nest=123       # Starting i-direction index of nest grid in parent tile supergrid
+  export jstart_nest=331       # Starting j-direction index of nest grid in parent tile supergrid
+  export iend_nest=1402        # Ending i-direction index of nest grid in parent tile supergrid
+  export jend_nest=1194        # Ending j-direction index of nest grid in parent tile supergrid
+  export halo=3                # Lateral boundary halo
+elif [ $gtype = regional_esg ] ; then
+  export res=-999              # equivalent resolution is computed
+  export target_lon=-97.5      # Center longitude of grid
+  export target_lat=35.5       # Center latitude of grid
+  export idim=301              # Dimension of grid in 'i' direction
+  export jdim=200              # Dimension of grid in 'j' direction
+  export delx=0.0585           # Grid spacing (in degrees) in the 'i' direction
+                               # on the SUPERGRID (which has twice the resolution of
+                               # the model grid).  The physical grid spacing in the 'i'
+                               # direction is related to delx as follows:
+                               #    distance = 2*delx*(circumf_Earth/360 deg)
+  export dely=0.0585           # Grid spacing (in degrees) in the 'j' direction.
+  export halo=3                # number of row/cols for halo
+fi
+
+#-----------------------------------------------------------------------
+# Check paths.
+#   home_dir - location of repository.
+#   TEMP_DIR - working directory.
+#   out_dir  - where files will be placed upon completion.
+#-----------------------------------------------------------------------
+
+export home_dir=$PBS_O_WORKDIR/..
+export TEMP_DIR=/lfs/h2/emc/stmp/$LOGNAME/fv3_grid.$gtype
+export out_dir=/lfs/h2/emc/stmp/$LOGNAME/my_grids
+
+#-----------------------------------------------------------------------
+# Should not need to change anything below here unless you want to
+# to change the job card for the number of tasks to use. Then,
+# you will need to check APRUN_SFC and OMP_NUM_THREADS.
+#-----------------------------------------------------------------------
+
+set -x
+
+export APRUN=time
+export APRUN_SFC="mpiexec -n 24 -ppn 24 -cpu-bind core"
+export OMP_NUM_THREADS=24 # orog code worked best with 24 threads.
+export OMP_PLACES=cores
+export OMP_STACKSIZE=2048m
+export machine=WCOSS2_CRAY
+
+ulimit -a
+ulimit -s unlimited
+
+#-----------------------------------------------------------------------
+# Start script.
+#-----------------------------------------------------------------------
+
+$home_dir/ush/fv3gfs_driver_grid.sh
+
+exit

fix/link_fixdirs.sh

@@ -9,17 +9,17 @@ machine=${2}
 if [ $# -lt 2 ]; then
     set +x
     echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine'
-    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera  | jet | orion | s4 )'
+    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | cray | dell | hera  | jet | orion | s4 )'
     exit 1
 fi
 
 if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then
     set +x
     echo '***ERROR*** unsupported run environment'
-    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion | s4 )'
+    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | cray | dell | hera | jet | orion | s4 )'
     exit 1
 fi
-if [ $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet -a $machine != orion -a $machine != s4 ]; then
+if [ $machine != wcoss2 -a $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet -a $machine != orion -a $machine != s4 ]; then
     set +x
     echo '***ERROR*** unsupported machine'
     echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion | s4 )'
@@ -45,6 +45,8 @@ elif [ $machine = "jet" ]; then
     FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix"
 elif [ $machine = "orion" ]; then
     FIX_DIR="/work/noaa/global/glopara/fix"
+elif [ $machine = "wcoss2" ]; then
+    FIX_DIR="/lfs/h2/emc/global/noscrub/kate.friedman/glopara/FIX/fix"
 elif [ $machine = "s4" ]; then
     FIX_DIR="/data/prod/glopara/fix"
 fi

modulefiles/build.wcoss2.intel.lua

@@ -0,0 +1,79 @@
+help([[
+Load environment to compile UFS_UTILS on WCOSS2
+]])
+
+cmake_ver=os.getenv("cmake_ver") or "3.20.2"
+load(pathJoin("cmake", cmake_ver))
+
+PrgEnv_intel_ver=os.getenv("PrgEnv_intel_ver") or "8.1.0"
+load(pathJoin("PrgEnv-intel", PrgEnv_intel_ver))
+
+craype_ver=os.getenv("craype_ver") or "2.7.13"
+load(pathJoin("craype", craype_ver))
+
+intel_ver=os.getenv("intel_ver") or "19.1.3.304"
+load(pathJoin("intel", intel_ver))
+
+cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.7"
+load(pathJoin("cray-mpich", cray_mpich_ver))
+
+
+libjpeg_ver=os.getenv("libjpeg_ver") or "9c"
+load(pathJoin("libjpeg", libjpeg_ver))
+
+zlib_ver=os.getenv("zlib_ver") or "1.2.11"
+load(pathJoin("zlib", zlib_ver))
+
+libpng_ver=os.getenv("libpng_ver") or "1.6.37"
+load(pathJoin("libpng", libpng_ver))
+
+hdf5_ver=os.getenv("hdf5_ver") or "1.10.6"
+load(pathJoin("hdf5", hdf5_ver))
+
+netcdf_ver=os.getenv("netcdf_ver") or "4.7.4"
+load(pathJoin("netcdf", netcdf_ver))
+
+bacio_ver=os.getenv("bacio_ver") or "2.4.1"
+load(pathJoin("bacio", bacio_ver))
+
+sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
+load(pathJoin("sfcio", sfcio_ver))
+
+w3nco_ver=os.getenv("w3nco_ver") or "2.4.1"
+load(pathJoin("w3nco", w3nco_ver))
+
+nemsio_ver=os.getenv("nemsio_ver") or "2.5.2"
+load(pathJoin("nemsio", nemsio_ver))
+
+sigio_ver=os.getenv("sigio_ver") or "2.3.2"
+load(pathJoin("sigio", sigio_ver))
+
+sp_ver=os.getenv("sp_ver") or "2.3.3"
+load(pathJoin("sp", sp_ver))
+
+ip_ver=os.getenv("ip_ver") or "3.3.3"
+load(pathJoin("ip", ip_ver))
+
+g2_ver=os.getenv("g2_ver") or "3.4.5"
+load(pathJoin("g2", g2_ver))
+
+-- for mpiexec command
+cray_pals_ver=os.getenv("cray_pals_ver") or "1.0.12"
+load(pathJoin("cray-pals", cray_pals_ver))
+
+udunits_ver=os.getenv("udunits_ver") or "2.2.28"
+load(pathJoin("udunits", udunits_ver))
+
+gsl_ver=os.getenv("gsl_ver") or "2.7"
+load(pathJoin("gsl", gsl_ver))
+
+nco_ver=os.getenv("nco_ver") or "4.9.7"
+load(pathJoin("nco", nco_ver))
+
+setenv("HPC_OPT","/apps/ops/para/libs")
+prepend_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304")
+prepend_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.7")
+esmf_ver=os.getenv("esmf_ver") or "8.2.1b04"
+load(pathJoin("esmf", esmf_ver))
+
+whatis("Description: UFS_UTILS build environment")

reg_tests/chgres_cube/README

@@ -7,18 +7,16 @@ To run the consistency tests:
 
 1) Build chgres_cube program.  Go to ./sorc and
    invoke 'build_all.sh' with no arguments.
-2) Invoke driver script for your machine.  See script
-   prolog for details.  Supported machines are:
-   - Hera (driver.hera.sh)
-   - WCOSS-Cray (driver.cray.sh)
-   - WCOSS-Dell (driver.dell.sh)
-   - Jet (driver.jet.sh)
-   - Orion (driver.orion.sh)
+2) Set the fixed directories. Go to ./fix and
+   invoke the script.
+3) Invoke driver script for your machine.  See script
+   prolog for details.
 
-A series of daisy-chained tests will run.  A test fails 
-if the output files differ from the baseline set of files
-as determined by the 'nccmp' utility.
+A series of tests will run.  A test fails if the output
+files differ from the baseline set of files as determined
+by the 'nccmp' utility.
 
-Log output from the tests will be in "regression.log".
+Log output from each test will be placed in its own
+logfile - "consistency.log??".
 
 A summary of results will be in "summary.log".