Merge branch 'develop' into feature/wcoss2

Fixes ufs-community#559.
GeorgeGayno-NOAA · May 23, 2022 · db9b31f · db9b31f
2 parents 9f82f4d + 6322952
commit db9b31f
Show file tree

Hide file tree

Showing 15 changed files with 359 additions and 136 deletions.
diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh
@@ -75,7 +75,7 @@ cd fix
 cd ../reg_tests
 
 sleep_time=0
-for dir in global_cycle chgres_cube grid_gen; do
+for dir in snow2mdl global_cycle chgres_cube grid_gen; do
     cd $dir
     ./driver.$target.sh
     # Wait for job to complete
@@ -96,8 +96,7 @@ elif [[ $target == "wcoss_cray" ]]; then
     module load xt-lsfhpc/9.1.3
 fi
 
-
-for dir in snow2mdl ice_blend; do
+for dir in ice_blend; do
     cd $dir
     if [[ $target == "hera" ]] || [[ $target == "jet" ]] || [[ $target == "orion" ]]; then
         sbatch -A ${PROJECT_CODE} ./driver.$target.sh

diff --git a/reg_tests/snow2mdl/driver.hera.sh b/reg_tests/snow2mdl/driver.hera.sh
@@ -2,12 +2,12 @@
 
 #-----------------------------------------------------------------------------
 #
-# Run snow2mdl consistency test on Hera.
+# Run snow2mdl consistency tests on Hera.
 #
-# Set $DATA to your working directory.  Set the project code (SBATCH -A)
+# Set $DATA_ROOT to your working directory.  Set the project code (SBATCH -A)
 # and queue (SBATCH -q) as appropriate.
 #
-# Invoke the script as follows:  sbatch $script
+# Invoke the script from the command line as follows:  ./$script
 #
 # Log output is placed in consistency.log.  A summary is
 # placed in summary.log
@@ -18,15 +18,6 @@
 #
 #-----------------------------------------------------------------------------
 
-#SBATCH -J snow
-#SBATCH -A fv3-cpu
-#SBATCH --open-mode=truncate
-#SBATCH -o consistency.log
-#SBATCH -e consistency.log
-#SBATCH --ntasks=1
-#SBATCH -q debug
-#SBATCH -t 00:03:00
-
 set -x
 
 compiler=${compiler:-"intel"}
@@ -36,8 +27,13 @@ module use ../../modulefiles
 module load build.$target.$compiler
 module list
 
-export DATA="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}"
-export DATA="${DATA}/reg-tests/snow2mdl"
+DATA_ROOT="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}"
+DATA_ROOT="${DATA_ROOT}/reg-tests/snow2mdl"
+
+rm -fr $DATA_ROOT
+
+PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}"
+QUEUE="${QUEUE:-batch}"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below.
@@ -50,13 +46,29 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then
   source ../get_hash.sh
 fi
 
-rm -fr $DATA
-
 export HOMEreg=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/snow2mdl
 export HOMEgfs=$PWD/../..
 export WGRIB=/scratch2/NCEPDEV/nwprod/NCEPLIBS/utils/grib_util.v1.1.1/exec/wgrib
 export WGRIB2=/scratch2/NCEPDEV/nwprod/NCEPLIBS/utils/grib_util.v1.1.1/exec/wgrib2
 
-./snow2mdl.sh
+# The first test mimics GFS OPS.
+
+export DATA="${DATA_ROOT}/test.ops"
+TEST1=$(sbatch --parsable -J snow.ops -A ${PROJECT_CODE} -o consistency.log -e consistency.log \
+      --ntasks=1 -q ${QUEUE} -t 00:03:00 ./snow2mdl.ops.sh)
+
+# The second test is for the new AFWA global GRIB2 data.
+
+export DATA="${DATA_ROOT}/test.global"
+TEST2=$(sbatch --parsable -J snow.global -A ${PROJECT_CODE} -o consistency.log -e consistency.log \
+      --ntasks=1 -q ${QUEUE} -t 00:03:00 -d afterok:$TEST1 ./snow2mdl.global.sh)
+
+# Create summary file.
+
+sbatch --nodes=1 -t 0:01:00 -A ${PROJECT_CODE} -J snow_summary -o consistency.log -e consistency.log \
+       --open-mode=append -q ${QUEUE} -d afterok:$TEST2 << EOF
+#!/bin/bash
+grep -a '<<<' consistency.log  > summary.log
+EOF
 
 exit 0
diff --git a/reg_tests/snow2mdl/driver.jet.sh b/reg_tests/snow2mdl/driver.jet.sh
@@ -2,12 +2,12 @@
 
 #-----------------------------------------------------------------------------
 #
-# Run snow2mdl consistency test on Jet.
+# Run snow2mdl consistency tests on Jet.
 #
-# Set $DATA to your working directory.  Set the project code (SBATCH -A)
-# and queue (SBATCH -q) as appropriate.
+# Set $DATA_ROOT to your working directory.  Set the project code and
+# and queue as appropriate.
 #
-# Invoke the script as follows:  sbatch $script
+# Invoke the script as follows:  ./$script
 #
 # Log output is placed in consistency.log.  A summary is
 # placed in summary.log
@@ -18,13 +18,6 @@
 #
 #-----------------------------------------------------------------------------
 
-#SBATCH --nodes=1
-#SBATCH --partition=sjet
-#SBATCH --time 0:01
-#SBATCH --account=emcda
-#SBATCH --job-name=snow2mdl
-#SBATCH -o consistency.log
-#SBATCH -e consistency.log
 
 set -x
 
@@ -33,8 +26,11 @@ module use ../../modulefiles
 module load build.$target.intel
 module list
 
-export DATA="${WORK_DIR:-/lfs4/HFIP/emcda/$LOGNAME/stmp}"
-export DATA="${DATA}/reg-tests/snow2mdl"
+DATA_ROOT="${WORK_DIR:-/lfs4/HFIP/emcda/$LOGNAME/stmp}"
+DATA_ROOT="${DATA_ROOT}/reg-tests/snow2mdl"
+
+PROJECT_CODE="${PROJECT_CODE:-hfv3gfs}"
+QUEUE="${QUEUE:-batch}"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below.
@@ -52,8 +48,25 @@ export HOMEgfs=$PWD/../..
 export WGRIB=/apps/wgrib/1.8.1.0b/bin/wgrib
 export WGRIB2=/apps/wgrib2/0.1.9.6a/bin/wgrib2
 
-rm -fr $DATA
+rm -fr $DATA_ROOT
+
+# This tests the OPS GFS snow processing.
+
+export DATA="${DATA_ROOT}/test.ops"
+TEST1=$(sbatch --parsable --nodes=1 --partition=xjet --time 0:02 -J snow.ops -o consistency.log \
+        -e consistency.log -A $PROJECT_CODE -q $QUEUE ./snow2mdl.ops.sh)
 
-./snow2mdl.sh
+# Test the new global afwa grib2 data.
 
+export DATA="${DATA_ROOT}/test.global"
+TEST2=$(sbatch --parsable --nodes=1 --partition=xjet --time 0:02 -J snow.global -o consistency.log \
+        -e consistency.log -A $PROJECT_CODE -q $QUEUE -d afterok:$TEST1 ./snow2mdl.global.sh)
+
+# Create summary file.
+
+sbatch --nodes=1 --partition=xjet -t 0:01:00 -A $PROJECT_CODE -J snow.summary -o consistency.log \
+       -e consistency.log --open-mode=append -q $QUEUE -d afterok:$TEST2 << EOF
+#!/bin/bash
+grep -a '<<<' consistency.log  > summary.log
+EOF
 exit 0
diff --git a/reg_tests/snow2mdl/driver.orion.sh b/reg_tests/snow2mdl/driver.orion.sh
@@ -2,31 +2,22 @@
 
 #-----------------------------------------------------------------------------
 #
-# Run snow2mdl consistency test on Orion.
+# Run snow2mdl consistency tests on Orion.
 #
-# Set $DATA to your working directory.  Set the project code (SBATCH -A)
-# and queue (SBATCH -q) as appropriate.
+# Set $DATA_ROOT to your working directory.  Set the project code
+# and queue as appropriate.
 #
-# Invoke the script as follows:  sbatch $script
+# Invoke the script as follows:  ./$script
 #
 # Log output is placed in consistency.log.  A summary is
 # placed in summary.log
 #
 # The test fails when its output does not match the baseline file
-# as determined by the 'cmp' command.  The baseline file is
+# as determined by the 'cmp' command.  The baseline files are
 # stored in HOMEreg.
 #
 #-----------------------------------------------------------------------------
 
-#SBATCH -J snow
-#SBATCH -A fv3-cpu
-#SBATCH --open-mode=truncate
-#SBATCH -o consistency.log
-#SBATCH -e consistency.log
-#SBATCH --ntasks=1
-#SBATCH -q debug
-#SBATCH -t 00:03:00
-
 set -x
 
 source ../../sorc/machine-setup.sh > /dev/null 2>&1
@@ -36,8 +27,11 @@ module list
 
 ulimit -s unlimited
 
-export DATA="${WORK_DIR:-/work/noaa/stmp/$LOGNAME}"
-export DATA="${DATA}/reg-tests/snow2mdl"
+export DATA_ROOT="${WORK_DIR:-/work/noaa/stmp/$LOGNAME}"
+export DATA_ROOT="${DATA_ROOT}/reg-tests/snow2mdl"
+
+PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}"
+QUEUE="${QUEUE:-batch}"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below.
@@ -50,13 +44,32 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then
   source ../get_hash.sh
 fi
 
-rm -fr $DATA
+rm -fr $DATA_ROOT
 
 export HOMEreg=/work/noaa/nems/role-nems/ufs_utils/reg_tests/snow2mdl
 export HOMEgfs=$PWD/../..
 export WGRIB=/apps/contrib/NCEPLIBS/orion/utils/grib_util.v1.2.0/exec/wgrib
 export WGRIB2=/apps/contrib/NCEPLIBS/orion/utils/grib_util.v1.2.0/exec/wgrib2
 
-./snow2mdl.sh
+# The first test mimics GFS OPS.
+
+export DATA="${DATA_ROOT}/test.ops"
+TEST1=$(sbatch --parsable -J snow.ops -A $PROJECT_CODE -o consistency.log \
+        -e consistency.log --ntasks=1 -q $QUEUE -t 00:03:00 ./snow2mdl.ops.sh)
+
+# This tests the afwa global grib2 data. 
+
+export DATA="${DATA_ROOT}/test.global"
+TEST2=$(sbatch --parsable -J snow.global -A $PROJECT_CODE -o consistency.log \
+        -e consistency.log --ntasks=1 -q $QUEUE -t 00:03:00 --open-mode=append \
+        -d afterok:$TEST1 ./snow2mdl.global.sh)
+
+# Create the summary file.
+
+sbatch --ntasks=1 -t 0:01:00 -A $PROJECT_CODE -J snow_summary -o consistency.log -e consistency.log \
+       --open-mode=append -q $QUEUE -d afterok:$TEST2 << EOF
+#!/bin/bash
+grep -a '<<<' consistency.log  > summary.log
+EOF
 
 exit 0
diff --git a/reg_tests/snow2mdl/driver.wcoss_cray.sh b/reg_tests/snow2mdl/driver.wcoss_cray.sh
@@ -2,39 +2,31 @@
 
 #-----------------------------------------------------------------------------
 #
-# Run snow2mdl consistency test on WCOSS-Cray.
+# Run snow2mdl consistency tests on WCOSS-Cray.
 #
-# Set $DATA to your working directory.  Set the project code (BSUB -P)
-# and queue (BSUB -q) as appropriate.
+# Set $DATA_ROOT to your working directory.  Set the project code
+# and queue as appropriate.
 #
-# Invoke the script as follows:  cat $script | bsub
+# Invoke the script as follows:  ./$script
 #
 # Log output is placed in consistency.log.  A summary is
 # placed in summary.log
 #
 # The test fails when its output does not match the baseline file
-# as determined by the 'cmp' command.  The baseline file is
+# as determined by the 'cmp' command.  The baseline files are
 # stored in HOMEreg.
 #
 #-----------------------------------------------------------------------------
 
-#BSUB -W 0:02
-#BSUB -o consistency.log
-#BSUB -e consistency.log
-#BSUB -J s2m_regt
-#BSUB -q debug
-#BSUB -R "rusage[mem=2000]"
-#BSUB -P GFS-DEV
-
 set -x
 
 source ../../sorc/machine-setup.sh > /dev/null 2>&1
 module use ../../modulefiles
 module load build.$target.intel
 module list
 
-export DATA="${WORK_DIR:-/gpfs/hps3/stmp/$LOGNAME}"
-export DATA="${DATA}/reg-tests/snow2mdl"
+DATA_ROOT="${WORK_DIR:-/gpfs/hps3/stmp/$LOGNAME}"
+DATA_ROOT="${DATA_ROOT}/reg-tests/snow2mdl"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below.
@@ -47,13 +39,34 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then
   source ../get_hash.sh
 fi
 
+PROJECT_CODE=${PROJECT_CODE:-GFS-DEV}
+QUEUE=${QUEUE:-dev}
+
 export HOMEreg=/gpfs/hps3/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/snow2mdl
 export HOMEgfs=$PWD/../..
 export WGRIB=/gpfs/hps/nco/ops/nwprod/grib_util.v1.0.2/exec/wgrib
 export WGRIB2=/gpfs/hps/nco/ops/nwprod/grib_util.v1.0.2/exec/wgrib2
 
-rm -fr $DATA
+rm -fr $DATA_ROOT
+
+LOG_FILE="consistency.log"
+SUM_FILE="summary.log"
+
+# Test the ops function of snow2mdl.
+
+export DATA=$DATA_ROOT/test.ops
+bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J snow.ops -W 0:02 \
+        -R "rusage[mem=2000]" "$PWD/snow2mdl.ops.sh"
+
+# Test the afwa global snow data.
+
+export DATA=$DATA_ROOT/test.global
+bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J snow.global -W 0:02 \
+        -R "rusage[mem=2000]" -w 'ended(snow.ops)' "$PWD/snow2mdl.global.sh"
+
+# Create a summary file.
 
-./snow2mdl.sh
+bsub -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J summary -R "rusage[mem=100]" -W 0:01 \
+     -w 'ended(snow.global)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE"
 
 exit 0