Merge pull request #1 from DusanJovic-NOAA/wcoss2

Port to WCOSS2 TDS (acorn)
mark-a-potts · Dec 8, 2020 · 01dc7a2 · 01dc7a2
2 parents 7771a20 + 7e327db
commit 01dc7a2
Show file tree

Hide file tree

Showing 12 changed files with 238 additions and 4 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -194,6 +194,8 @@ if(WW3)
       set(WW3_COMP "hera")
     elseif(${CMAKE_Platform} STREQUAL "orion.intel")
       set(WW3_COMP "orion")
+    elseif(${CMAKE_Platform} STREQUAL "wcoss2")
+      set(WW3_COMP "wcoss_cray")
     elseif(${CMAKE_Platform} MATCHES "linux*" OR ${CMAKE_Platform} MATCHES "macosx*")
       if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
         set(WW3_COMP "gnu")

diff --git a/cmake/Intel.cmake b/cmake/Intel.cmake
@@ -21,8 +21,10 @@ elseif(DEBUG)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align")
 else()
     if(AVX2)
-        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
+        #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
+        #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
+        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2 -qno-opt-dynamic-align")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx2 -qno-opt-dynamic-align")
     elseif(SIMDMULTIARCH)
         set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align")
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align")

diff --git a/cmake/configure_wcoss2.cmake b/cmake/configure_wcoss2.cmake
@@ -0,0 +1,3 @@
+set(INLINE_POST     ON  CACHE BOOL "Enable inline post" FORCE)
+set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
+set(DEBUG_LINKMPI   OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE)
diff --git a/modulefiles/wcoss2/fv3 b/modulefiles/wcoss2/fv3
@@ -0,0 +1,42 @@
+#%Module
+
+proc ModulesHelp {} {
+  puts stderr "\tcit - loads modules required for building and running UFS Model on the Dell side of WCOSS"
+}
+
+module-whatis "loads UFS Model prerequisites on Venus and Mars"
+
+module unload cpe-cray cce
+module load cpe-intel intel
+
+module load cmake/3.17.3
+setenv CMAKE_C_COMPILER cc
+setenv CMAKE_CXX_COMPILER CC
+setenv CMAKE_Fortran_COMPILER ftn
+setenv CMAKE_Platform wcoss2
+
+module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack
+
+module load hpc/1.0.0-beta1
+module load hpc-intel/19.1.1.217
+module load hpc-cray-mpich/8.0.15
+
+module load jasper/2.0.22
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load pio/2.5.1
+module load esmf/8_1_0_beta_snapshot_27
+
+module load bacio/2.4.1
+module load crtm/2.3.0
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.0
diff --git a/modulefiles/wcoss2/fv3_debug b/modulefiles/wcoss2/fv3_debug
@@ -0,0 +1,42 @@
+#%Module
+
+proc ModulesHelp {} {
+  puts stderr "\tcit - loads modules required for building and running UFS Model on the Dell side of WCOSS"
+}
+
+module-whatis "loads UFS Model prerequisites on Venus and Mars"
+
+module unload cpe-cray cce
+module load cpe-intel intel
+
+module load cmake/3.17.3
+setenv CMAKE_C_COMPILER cc
+setenv CMAKE_CXX_COMPILER CC
+setenv CMAKE_Fortran_COMPILER ftn
+setenv CMAKE_Platform wcoss2
+
+module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack
+
+module load hpc/1.0.0-beta1
+module load hpc-intel/19.1.1.217
+module load hpc-cray-mpich/8.0.15
+
+module load jasper/2.0.22
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load pio/2.5.1
+module load esmf/8_1_0_beta_snapshot_27
+
+module load bacio/2.4.1
+module load crtm/2.3.0
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.0
diff --git a/tests/compile.sh b/tests/compile.sh
@@ -56,6 +56,9 @@ set +x
 if [[ $MACHINE_ID == macosx.* ]] || [[ $MACHINE_ID == linux.* ]]; then
   source $PATHTR/modulefiles/${MACHINE_ID}/fv3
 else
+  if [[ $MACHINE_ID == wcoss2 ]]; then
+    source /apps/prod/lmodules/startLmod
+  fi
   # Activate lua environment for gaea
   if [[ $MACHINE_ID == gaea.* ]] ; then
     source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh

diff --git a/tests/default_vars.sh b/tests/default_vars.sh
@@ -7,14 +7,14 @@
 #
 ###############################################################################
 
-if [ $MACHINE_ID = wcoss_cray ]; then
+if [[ $MACHINE_ID = wcoss_cray ]]; then
 
   TASKS_dflt=150 ; TPN_dflt=24 ; INPES_dflt=3 ; JNPES_dflt=8
   TASKS_thrd=84  ; TPN_thrd=12 ; INPES_thrd=3 ; JNPES_thrd=4
   TASKS_stretch=48 ; TPN_stretch=24 ; INPES_stretch=2 ; JNPES_stretch=4
   TASKS_strnest=96 ; TPN_strnest=24 ; INPES_strnest=2 ; JNPES_strnest=4
 
-elif [ $MACHINE_ID = wcoss_dell_p3 ]; then
+elif [[ $MACHINE_ID = wcoss_dell_p3 || $MACHINE_ID = wcoss2 ]]; then
 
   TASKS_dflt=150 ; TPN_dflt=28 ; INPES_dflt=3 ; JNPES_dflt=8
   TASKS_thrd=84  ; TPN_thrd=14 ; INPES_thrd=3 ; JNPES_thrd=4

diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh
@@ -27,6 +27,9 @@ case $(hostname -f) in
   m72a2.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### mars
   m72a3.ncep.noaa.gov)     MACHINE_ID=wcoss_dell_p3 ;; ### mars
 
+  alogin01)                MACHINE_ID=wcoss2 ;; ### acorn
+  alogin02)                MACHINE_ID=wcoss2 ;; ### acorn
+
   gaea9)                   MACHINE_ID=gaea ;; ### gaea9
   gaea10)                  MACHINE_ID=gaea ;; ### gaea10
   gaea11)                  MACHINE_ID=gaea ;; ### gaea11

diff --git a/tests/fv3_conf/compile_qsub.IN_wcoss2 b/tests/fv3_conf/compile_qsub.IN_wcoss2
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+#PBS -o out
+#PBS -e err
+#PBS -N @[JBNME]
+# #PBS -A @[ACCNR]
+#PBS -q @[QUEUE]
+#PBS -l select=1:ncpus=8:mpiprocs=1
+#PBS -l walltime=00:30:00
+
+set -eux
+
+cd $PBS_O_WORKDIR
+export CRAY_CONFIG_DIR=$HOME
+
+echo "Compile started:  " `date`
+
+@[PATHRT]/compile.sh @[MACHINE_ID] "@[MAKE_OPT]" @[COMPILE_NR]
+
+echo "Compile ended:    " `date`
diff --git a/tests/fv3_conf/fv3_qsub.IN_wcoss2 b/tests/fv3_conf/fv3_qsub.IN_wcoss2
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+#PBS -o out
+#PBS -e err
+#PBS -N @[JBNME]
+# #PBS -A @[ACCNR]
+#PBS -q @[QUEUE]
+#PBS -l place=vscatter,select=@[NODES]:ncpus=@[TPN]:mpiprocs=@[TPN]
+#PBS -l walltime=00:@[WLCLK]:00
+
+set -eux
+
+cd $PBS_O_WORKDIR
+export CRAY_CONFIG_DIR=$HOME
+
+source /apps/prod/lmodules/startLmod
+module use $( pwd -P )
+module load modules.fv3
+module list
+
+echo "Model started:  " `date`
+
+export MPI_TYPE_DEPTH=20
+export OMP_STACKSIZE=512M
+export OMP_NUM_THREADS=@[THRD]
+export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4
+
+cray aprun -n @[TASKS] ./fv3.exe
+
+echo "Model ended:    " `date`
+
+exit
diff --git a/tests/rt.sh b/tests/rt.sh
@@ -173,6 +173,33 @@ elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then
   cp fv3_conf/fv3_bsub.IN_wcoss_dell_p3 fv3_conf/fv3_bsub.IN
   cp fv3_conf/compile_bsub.IN_wcoss_dell_p3 fv3_conf/compile_bsub.IN
 
+elif [[ $MACHINE_ID = wcoss2 ]]; then
+
+  source /apps/prod/lmodules/startLmod
+
+  #module use /usrx/local/dev/emc_rocoto/modulefiles
+  #module load ruby/2.5.1 rocoto/1.3.0rc2
+  #ROCOTORUN=$(which rocotorun)
+  #ROCOTOSTAT=$(which rocotostat)
+  #ROCOTOCOMPLETE=$(which rocotocomplete)
+  #ROCOTO_SCHEDULER=lsf
+
+  #module load ips/18.0.1.163
+  #module load ecflow/4.7.1
+  #ECFLOW_START=${ECF_ROOT}/intel/bin/ecflow_start.sh
+  #ECF_PORT=$(grep $USER /usrx/local/sys/ecflow/assigned_ports.txt | awk '{print $2}')
+
+  DISKNM=/lfs/h1/emc/ptmp/Dusan.Jovic/RT
+  QUEUE=workq
+  COMPILE_QUEUE=workq
+  PARTITION=
+  ACCNR=GFS-DEV
+  STMP=/lfs/h1/emc/stmp
+  PTMP=/lfs/h1/emc/ptmp
+  SCHEDULER=pbs
+  cp fv3_conf/fv3_qsub.IN_wcoss2 fv3_conf/fv3_qsub.IN
+  cp fv3_conf/compile_qsub.IN_wcoss2 fv3_conf/compile_qsub.IN
+
 elif [[ $MACHINE_ID = gaea.* ]]; then
 
   module load cray-python/3.7.3.2
@@ -451,6 +478,10 @@ if [[ $ROCOTO == true ]]; then
     QUEUE=dev
     COMPILE_QUEUE=dev_transfer
     ROCOTO_SCHEDULER=lsf
+  elif [[ $MACHINE_ID = wcoss2 ]]; then
+    QUEUE=workq
+    COMPILE_QUEUE=workq
+    ROCOTO_SCHEDULER=pbs
   elif [[ $MACHINE_ID = hera.* ]]; then
     QUEUE=batch
     COMPILE_QUEUE=batch
@@ -512,6 +543,8 @@ EOF
     QUEUE=dev
   elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then
     QUEUE=dev
+  elif [[ $MACHINE_ID = wcoss2 ]]; then
+    QUEUE=workq
   elif [[ $MACHINE_ID = hera.* ]]; then
     QUEUE=batch
   elif [[ $MACHINE_ID = orion.* ]]; then

diff --git a/tests/rt_acorn.conf b/tests/rt_acorn.conf
@@ -0,0 +1,52 @@
+#######################################################################################################################################################################################
+# CCPP PROD tests                                                                                                                                                                     #
+#######################################################################################################################################################################################
+
+COMPILE | CCPP=Y SUITES=FV3_GFS_2017                                                                                                     | standard    |                | fv3         |
+
+RUN     | fv3_ccpp_control                                                                                                               | standard    |                | fv3         |
+RUN     | fv3_ccpp_decomp                                                                                                                | standard    |                |             |
+RUN     | fv3_ccpp_2threads                                                                                                              | standard    |                |             |
+RUN     | fv3_ccpp_restart                                                                                                               | standard    |                | fv3         |
+RUN     | fv3_ccpp_read_inc                                                                                                              | standard    |                | fv3         |
+RUN     | fv3_ccpp_wrtGauss_netcdf_esmf                                                                                                  | standard    |                | fv3         |
+RUN     | fv3_ccpp_wrtGauss_netcdf                                                                                                       | standard    |                | fv3         |
+RUN     | fv3_ccpp_wrtGlatlon_netcdf                                                                                                     | standard    |                | fv3         |
+RUN     | fv3_ccpp_wrtGauss_nemsio                                                                                                       | standard    |                | fv3         |
+RUN     | fv3_ccpp_wrtGauss_nemsio_c192                                                                                                  | standard    |                | fv3         |
+RUN     | fv3_ccpp_stochy                                                                                                                | standard    |                | fv3         |
+RUN     | fv3_ccpp_iau                                                                                                                   | standard    |                | fv3         |
+RUN     | fv3_ccpp_lheatstrg                                                                                                             | standard    |                | fv3         |
+
+COMPILE | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y                                           | standard    |                | fv3         |
+RUN     | fv3_ccpp_regional_control                                                                                                      | standard    |                | fv3         |
+RUN     | fv3_ccpp_regional_restart                                                                                                      | standard    |                | fv3         | fv3_ccpp_regional_control
+RUN     | fv3_ccpp_regional_quilt                                                                                                        | standard    |                | fv3         |
+
+COMPILE | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp                                                                   | standard    |                | fv3         |
+RUN     | fv3_ccpp_gfdlmp                                                                                                                | standard    |                | fv3         |
+RUN     | fv3_ccpp_gfdlmprad_gwd                                                                                                         | standard    |                | fv3         |
+RUN     | fv3_ccpp_gfdlmprad_noahmp                                                                                                      | standard    |                | fv3         |
+
+COMPILE | CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq                         | standard    |                | fv3         |
+RUN     | fv3_ccpp_csawmg                                                                                                                | standard    |                | fv3         |
+RUN     | fv3_ccpp_satmedmf                                                                                                              | standard    |                | fv3         |
+RUN     | fv3_ccpp_satmedmfq                                                                                                             | standard    |                | fv3         |
+
+COMPILE | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson,FV3_RAP,FV3_HRRR,FV3_RRFS_v1beta 32BIT=Y          | standard    |                | fv3         |
+RUN     | fv3_ccpp_gfdlmp_32bit                                                                                                          | standard    |                | fv3         |
+RUN     | fv3_ccpp_gfdlmprad_32bit_post                                                                                                  | standard    |                | fv3         |
+RUN     | fv3_ccpp_cpt                                                                                                                   | standard    |                | fv3         |
+RUN     | fv3_ccpp_gsd                                                                                                                   | standard    |                | fv3         |
+RUN     | fv3_ccpp_thompson                                                                                                              | standard    |                | fv3         |
+RUN     | fv3_ccpp_thompson_no_aero                                                                                                      | standard    |                | fv3         |
+RUN     | fv3_ccpp_rrfs_v1beta                                                                                                           | standard    |                | fv3         |
+
+#######################################################################################################################################################################################
+# CPLD tests
+#######################################################################################################################################################################################
+
+COMPILE | CCPP=Y SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled S2S=Y							|  standard |  |	   fv3 |
+RUN     | cpld_control																											|  standard |  |	   fv3 |
+RUN     | cpld_2threads																											|  standard |  |		   |
+RUN     | cpld_decomp																											|  standard |  |		   |