The new CCPP suite FV3_GFS_v16b_ugwpv1 and regression tests

.gitmodules

@@ -1,14 +1,14 @@
 [submodule "FV3"]
   path = FV3
-  url = https://github.com/NOAA-EMC/fv3atm
+  url = https://github.com/ValeryYudin-NOAA/fv3atm
   branch = develop
 [submodule "NEMS"]
   path = NEMS
   url = https://github.com/NOAA-EMC/NEMS
   branch = develop
 [submodule "FMS"]
   path = FMS
-  url = https://github.com/NOAA-GFDL/FMS
+  url = https://github.com/ValeryYudin-NOAA/FMS
   branch = master
 [submodule "WW3"]
   path = WW3

tests/detect_machine.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Default account "nems"
-export ACCNR=${ACCNR:-nems}
+export ACCNR=${ACCNR:-swpc}
 
 case $(hostname -f) in
 

tests/detect_machine.sh_nems

@@ -0,0 +1,106 @@
+#!/bin/bash
+
+# Default account "nems"
+export ACCNR=${ACCNR:-nems}
+
+case $(hostname -f) in
+
+  llogin1)                 MACHINE_ID=wcoss_cray ;; ### luna
+  llogin2)                 MACHINE_ID=wcoss_cray ;; ### luna
+  llogin3)                 MACHINE_ID=wcoss_cray ;; ### luna
+
+  slogin1)                 MACHINE_ID=wcoss_cray ;; ### surge
+  slogin2)                 MACHINE_ID=wcoss_cray ;; ### surge
+  slogin3)                 MACHINE_ID=wcoss_cray ;; ### surge
+
+  v71a1.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### venus
+  v71a2.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### venus
+  v71a3.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### venus
+  v72a1.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### venus
+  v72a2.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### venus
+  v72a3.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### venus
+
+  m71a1.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### mars
+  m71a2.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### mars
+  m71a3.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### mars
+  m72a1.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### mars
+  m72a2.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### mars
+  m72a3.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### mars
+
+  alogin01)                MACHINE_ID=wcoss2 ;; ### acorn
+  alogin02)                MACHINE_ID=wcoss2 ;; ### acorn
+
+  gaea9)                   MACHINE_ID=gaea ;; ### gaea9
+  gaea10)                  MACHINE_ID=gaea ;; ### gaea10
+  gaea11)                  MACHINE_ID=gaea ;; ### gaea11
+  gaea12)                  MACHINE_ID=gaea ;; ### gaea12
+  gaea13)                  MACHINE_ID=gaea ;; ### gaea13
+  gaea14)                  MACHINE_ID=gaea ;; ### gaea14
+  gaea15)                  MACHINE_ID=gaea ;; ### gaea15
+  gaea16)                  MACHINE_ID=gaea ;; ### gaea16
+  gaea9.ncrc.gov)          MACHINE_ID=gaea ;; ### gaea9
+  gaea10.ncrc.gov)         MACHINE_ID=gaea ;; ### gaea10
+  gaea11.ncrc.gov)         MACHINE_ID=gaea ;; ### gaea11
+  gaea12.ncrc.gov)         MACHINE_ID=gaea ;; ### gaea12
+  gaea13.ncrc.gov)         MACHINE_ID=gaea ;; ### gaea13
+  gaea14.ncrc.gov)         MACHINE_ID=gaea ;; ### gaea14
+  gaea15.ncrc.gov)         MACHINE_ID=gaea ;; ### gaea15
+  gaea16.ncrc.gov)         MACHINE_ID=gaea ;; ### gaea16
+
+  hfe01)                   MACHINE_ID=hera ;; ### hera01
+  hfe02)                   MACHINE_ID=hera ;; ### hera02
+  hfe03)                   MACHINE_ID=hera ;; ### hera03
+  hfe04)                   MACHINE_ID=hera ;; ### hera04
+  hfe05)                   MACHINE_ID=hera ;; ### hera05
+  hfe06)                   MACHINE_ID=hera ;; ### hera06
+  hfe07)                   MACHINE_ID=hera ;; ### hera07
+  hfe08)                   MACHINE_ID=hera ;; ### hera08
+  hfe09)                   MACHINE_ID=hera ;; ### hera09
+  hfe10)                   MACHINE_ID=hera ;; ### hera10
+  hfe11)                   MACHINE_ID=hera ;; ### hera11
+  hfe12)                   MACHINE_ID=hera ;; ### hera12
+
+  fe1)                     MACHINE_ID=jet ;; ### jet01
+  fe2)                     MACHINE_ID=jet ;; ### jet02
+  fe3)                     MACHINE_ID=jet ;; ### jet03
+  fe4)                     MACHINE_ID=jet ;; ### jet04
+  fe5)                     MACHINE_ID=jet ;; ### jet05
+  fe6)                     MACHINE_ID=jet ;; ### jet06
+  fe7)                     MACHINE_ID=jet ;; ### jet07
+  fe8)                     MACHINE_ID=jet ;; ### jet08
+  tfe1)                    MACHINE_ID=jet ;; ### jet09
+  tfe2)                    MACHINE_ID=jet ;; ### jet10
+
+  Orion-login-1.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1
+  Orion-login-2.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion2
+  Orion-login-3.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion3
+  Orion-login-4.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion4
+
+  cheyenne1.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1
+  cheyenne2.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne2
+  cheyenne3.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne3
+  cheyenne4.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne4
+  cheyenne5.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne5
+  cheyenne6.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne6
+  cheyenne1.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1
+  cheyenne2.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne2
+  cheyenne3.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne3
+  cheyenne4.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne4
+  cheyenne5.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne5
+  cheyenne6.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne6
+
+  login1.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1
+  login2.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede2
+  login3.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede3
+  login4.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede4
+esac
+
+# Overwrite auto-detect with RT_MACHINE if set
+MACHINE_ID=${RT_MACHINE:-${MACHINE_ID}}
+
+# Append compiler
+if [ $MACHINE_ID = orion ] || [ $MACHINE_ID = hera ] || [ $MACHINE_ID = cheyenne ] || [ $MACHINE_ID = jet ] || [ $MACHINE_ID = gaea ] || [ $MACHINE_ID = stampede ] ; then
+    MACHINE_ID=${MACHINE_ID}.${RT_COMPILER}
+fi
+
+echo "Machine: " $MACHINE_ID "    Account: " $ACCNR

tests/fv3_conf/ccpp_gfsv16_ugwpv1_run.IN

@@ -0,0 +1,22 @@
+rm -rf INPUT RESTART
+
+if [ $WARM_START = .F. ]; then
+  cp -r @[RTPWD127]/FV3_input_data/INPUT ./
+  mkdir RESTART
+else
+  mkdir INPUT RESTART
+  cp -r @[RTPWD127]/FV3_input_data/RESTART/* ./INPUT
+fi
+cp    @[RTPWD127]/FV3_input_data//aerosol.dat .
+cp    @[RTPWD127]/FV3_input_data/co2historicaldata_201*.txt .
+cp    @[RTPWD127]/FV3_input_data/sfc_emissivity_idx.txt .
+cp    @[RTPWD127]/FV3_input_data/solarconstant_noaa_an.txt .
+cp    @[RTPWD127]/FV3_input_data/global_o3prdlos.f77 ./global_o3prdlos.f77
+cp    @[RTPWD127]/FV3_input_data/global_h2oprdlos.f77 ./global_h2oprdlos.f77
+cp    @[RTPWD127]/FV3_input_data/*grb .
+cp    @[RTPWD127]/FV3_input_data/*_table ./
+cp    @[RTPWD127]/FV3_input_data/*configure ./
+cp    @[RTPWD127]/FV3_input_data/ugwp_c384_tau.nc   ./ugwp_limb_tau.nc 
+if [ $DO_RRTMGP = .T. ]; then
+cp    @[RTPWD]/FV3_input_data_RRTMGP/* .
+fi

tests/fv3_conf/ccpp_gfsv16_ugwpv1ws_run.IN

@@ -0,0 +1,25 @@
+rm -rf INPUT RESTART
+
+if [ $WARM_START = .F. ]; then
+  cp -r @[RTPWD127]/FV3_input_data/INPUT ./
+  mkdir RESTART
+  cp    @[RTPWD127]/FV3_input_data/model_cold_configure ./model_configure  
+else
+  mkdir INPUT RESTART
+  cp  @[RTPWD127]/FV3_input_data/INPUT/*   ./INPUT 
+  cp  @[RTPWD127]/FV3_input_data/RESTART/* ./INPUT
+  cp    @[RTPWD127]/FV3_input_data/model_warm_configure ./model_configure  
+fi
+cp    @[RTPWD127]/FV3_input_data//aerosol.dat .
+cp    @[RTPWD127]/FV3_input_data/co2historicaldata_201*.txt .
+cp    @[RTPWD127]/FV3_input_data/sfc_emissivity_idx.txt .
+cp    @[RTPWD127]/FV3_input_data/solarconstant_noaa_an.txt .
+cp    @[RTPWD127]/FV3_input_data/global_o3prdlos.f77 ./global_o3prdlos.f77
+cp    @[RTPWD127]/FV3_input_data/global_h2oprdlos.f77 ./global_h2oprdlos.f77
+cp    @[RTPWD127]/FV3_input_data/*grb .
+cp    @[RTPWD127]/FV3_input_data/*_table ./
+cp    @[RTPWD127]/FV3_input_data/nems.configure ./
+cp    @[RTPWD127]/FV3_input_data/ugwp_c384_tau.nc   ./ugwp_limb_tau.nc 
+if [ $DO_RRTMGP = .T. ]; then
+cp    @[RTPWD]/FV3_input_data_RRTMGP/* .
+fi

tests/fv3_conf/compile_slurm.IN

@@ -0,0 +1,17 @@
+#!/bin/sh
+#SBATCH -e err
+#SBATCH -o out
+#SBATCH --account=@[ACCNR]
+#SBATCH --qos=@[QUEUE]
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=8
+#SBATCH --time=30
+#SBATCH --job-name="@[JBNME]"
+
+set -eux
+
+echo "Compile started:  " `date`
+
+@[PATHRT]/compile.sh @[MACHINE_ID] "@[MAKE_OPT]" @[COMPILE_NR]
+
+echo "Compile ended:    " `date`

tests/fv3_conf/fv3_slurm.IN

@@ -0,0 +1,39 @@
+#!/bin/sh
+#SBATCH -e err
+#SBATCH -o out
+#SBATCH --account=@[ACCNR]
+#SBATCH --qos=@[QUEUE]
+### #SBATCH --ntasks=@[TASKS]
+#SBATCH --nodes=@[NODES]
+#SBATCH --ntasks-per-node=@[TPN]
+#SBATCH --time=@[WLCLK]
+#SBATCH --job-name="@[JBNME]"
+### #SBATCH --exclusive
+
+set -eux
+
+set +x
+source ./module-setup.sh
+module use $( pwd -P )
+module load modules.fv3
+module list
+
+set -x
+
+echo "Model started:  " `date`
+
+export MPI_TYPE_DEPTH=20
+export OMP_STACKSIZE=512M
+export OMP_NUM_THREADS=@[THRD]
+export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4
+export PSM_RANKS_PER_CONTEXT=4
+export PSM_SHAREDCONTEXTS=1
+
+# Avoid job errors because of filesystem synchronization delays
+sync && sleep 1
+
+srun --label -n @[TASKS] ./fv3.exe
+
+echo "Model ended:    " `date`
+
+exit