diff --git a/CMakeLists.txt b/CMakeLists.txt index a3559bcd4c..2221cf24e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -194,6 +194,8 @@ if(WW3) set(WW3_COMP "hera") elseif(${CMAKE_Platform} STREQUAL "orion.intel") set(WW3_COMP "orion") + elseif(${CMAKE_Platform} STREQUAL "wcoss2") + set(WW3_COMP "wcoss_cray") elseif(${CMAKE_Platform} MATCHES "linux*" OR ${CMAKE_Platform} MATCHES "macosx*") if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU") set(WW3_COMP "gnu") diff --git a/cmake/Intel.cmake b/cmake/Intel.cmake index 6d2eb74537..d070ac2927 100644 --- a/cmake/Intel.cmake +++ b/cmake/Intel.cmake @@ -21,8 +21,10 @@ elseif(DEBUG) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align") else() if(AVX2) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align") + #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align") + #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2 -qno-opt-dynamic-align") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx2 -qno-opt-dynamic-align") elseif(SIMDMULTIARCH) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align") diff --git a/cmake/configure_wcoss2.cmake b/cmake/configure_wcoss2.cmake new file mode 100644 index 0000000000..c8a689be6c --- /dev/null +++ b/cmake/configure_wcoss2.cmake @@ -0,0 +1,3 @@ +set(INLINE_POST ON CACHE BOOL "Enable inline post" FORCE) +set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE) +set(DEBUG_LINKMPI OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE) diff --git a/modulefiles/wcoss2/fv3 b/modulefiles/wcoss2/fv3 new file mode 100644 index 0000000000..fbb682affc --- /dev/null +++ b/modulefiles/wcoss2/fv3 @@ -0,0 +1,42 @@ +#%Module + +proc ModulesHelp {} { + puts stderr "\tcit - loads modules required for building and running UFS Model on the Dell side of WCOSS" +} + +module-whatis "loads UFS Model prerequisites on Venus and Mars" + +module unload cpe-cray cce +module load cpe-intel intel + +module load cmake/3.17.3 +setenv CMAKE_C_COMPILER cc +setenv CMAKE_CXX_COMPILER CC +setenv CMAKE_Fortran_COMPILER ftn +setenv CMAKE_Platform wcoss2 + +module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack + +module load hpc/1.0.0-beta1 +module load hpc-intel/19.1.1.217 +module load hpc-cray-mpich/8.0.15 + +module load jasper/2.0.22 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.1 +module load esmf/8_1_0_beta_snapshot_27 + +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load upp/10.0.0 diff --git a/modulefiles/wcoss2/fv3_debug b/modulefiles/wcoss2/fv3_debug new file mode 100644 index 0000000000..fbb682affc --- /dev/null +++ b/modulefiles/wcoss2/fv3_debug @@ -0,0 +1,42 @@ +#%Module + +proc ModulesHelp {} { + puts stderr "\tcit - loads modules required for building and running UFS Model on the Dell side of WCOSS" +} + +module-whatis "loads UFS Model prerequisites on Venus and Mars" + +module unload cpe-cray cce +module load cpe-intel intel + +module load cmake/3.17.3 +setenv CMAKE_C_COMPILER cc +setenv CMAKE_CXX_COMPILER CC +setenv CMAKE_Fortran_COMPILER ftn +setenv CMAKE_Platform wcoss2 + +module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack + +module load hpc/1.0.0-beta1 +module load hpc-intel/19.1.1.217 +module load hpc-cray-mpich/8.0.15 + +module load jasper/2.0.22 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.1 +module load esmf/8_1_0_beta_snapshot_27 + +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load upp/10.0.0 diff --git a/tests/compile.sh b/tests/compile.sh index 8d64b33394..4e77b4d6d1 100755 --- a/tests/compile.sh +++ b/tests/compile.sh @@ -56,6 +56,9 @@ set +x if [[ $MACHINE_ID == macosx.* ]] || [[ $MACHINE_ID == linux.* ]]; then source $PATHTR/modulefiles/${MACHINE_ID}/fv3 else + if [[ $MACHINE_ID == wcoss2 ]]; then + source /apps/prod/lmodules/startLmod + fi # Activate lua environment for gaea if [[ $MACHINE_ID == gaea.* ]] ; then source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh diff --git a/tests/default_vars.sh b/tests/default_vars.sh index 1e7a4c4632..94a1ef9464 100755 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -7,14 +7,14 @@ # ############################################################################### -if [ $MACHINE_ID = wcoss_cray ]; then +if [[ $MACHINE_ID = wcoss_cray ]]; then TASKS_dflt=150 ; TPN_dflt=24 ; INPES_dflt=3 ; JNPES_dflt=8 TASKS_thrd=84 ; TPN_thrd=12 ; INPES_thrd=3 ; JNPES_thrd=4 TASKS_stretch=48 ; TPN_stretch=24 ; INPES_stretch=2 ; JNPES_stretch=4 TASKS_strnest=96 ; TPN_strnest=24 ; INPES_strnest=2 ; JNPES_strnest=4 -elif [ $MACHINE_ID = wcoss_dell_p3 ]; then +elif [[ $MACHINE_ID = wcoss_dell_p3 || $MACHINE_ID = wcoss2 ]]; then TASKS_dflt=150 ; TPN_dflt=28 ; INPES_dflt=3 ; JNPES_dflt=8 TASKS_thrd=84 ; TPN_thrd=14 ; INPES_thrd=3 ; JNPES_thrd=4 diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index 8877530d7b..c4bea08a2e 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -27,6 +27,9 @@ case $(hostname -f) in m72a2.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### mars m72a3.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### mars + alogin01) MACHINE_ID=wcoss2 ;; ### acorn + alogin02) MACHINE_ID=wcoss2 ;; ### acorn + gaea9) MACHINE_ID=gaea ;; ### gaea9 gaea10) MACHINE_ID=gaea ;; ### gaea10 gaea11) MACHINE_ID=gaea ;; ### gaea11 diff --git a/tests/fv3_conf/compile_qsub.IN_wcoss2 b/tests/fv3_conf/compile_qsub.IN_wcoss2 new file mode 100644 index 0000000000..ee2b789758 --- /dev/null +++ b/tests/fv3_conf/compile_qsub.IN_wcoss2 @@ -0,0 +1,20 @@ +#!/bin/bash + +#PBS -o out +#PBS -e err +#PBS -N @[JBNME] +# #PBS -A @[ACCNR] +#PBS -q @[QUEUE] +#PBS -l select=1:ncpus=8:mpiprocs=1 +#PBS -l walltime=00:30:00 + +set -eux + +cd $PBS_O_WORKDIR +export CRAY_CONFIG_DIR=$HOME + +echo "Compile started: " `date` + +@[PATHRT]/compile.sh @[MACHINE_ID] "@[MAKE_OPT]" @[COMPILE_NR] + +echo "Compile ended: " `date` diff --git a/tests/fv3_conf/fv3_qsub.IN_wcoss2 b/tests/fv3_conf/fv3_qsub.IN_wcoss2 new file mode 100644 index 0000000000..d000b7e493 --- /dev/null +++ b/tests/fv3_conf/fv3_qsub.IN_wcoss2 @@ -0,0 +1,32 @@ +#!/bin/bash + +#PBS -o out +#PBS -e err +#PBS -N @[JBNME] +# #PBS -A @[ACCNR] +#PBS -q @[QUEUE] +#PBS -l place=vscatter,select=@[NODES]:ncpus=@[TPN]:mpiprocs=@[TPN] +#PBS -l walltime=00:@[WLCLK]:00 + +set -eux + +cd $PBS_O_WORKDIR +export CRAY_CONFIG_DIR=$HOME + +source /apps/prod/lmodules/startLmod +module use $( pwd -P ) +module load modules.fv3 +module list + +echo "Model started: " `date` + +export MPI_TYPE_DEPTH=20 +export OMP_STACKSIZE=512M +export OMP_NUM_THREADS=@[THRD] +export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4 + +cray aprun -n @[TASKS] ./fv3.exe + +echo "Model ended: " `date` + +exit diff --git a/tests/rt.sh b/tests/rt.sh index 1bb2eff817..3379c2a914 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -173,6 +173,33 @@ elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then cp fv3_conf/fv3_bsub.IN_wcoss_dell_p3 fv3_conf/fv3_bsub.IN cp fv3_conf/compile_bsub.IN_wcoss_dell_p3 fv3_conf/compile_bsub.IN +elif [[ $MACHINE_ID = wcoss2 ]]; then + + source /apps/prod/lmodules/startLmod + + #module use /usrx/local/dev/emc_rocoto/modulefiles + #module load ruby/2.5.1 rocoto/1.3.0rc2 + #ROCOTORUN=$(which rocotorun) + #ROCOTOSTAT=$(which rocotostat) + #ROCOTOCOMPLETE=$(which rocotocomplete) + #ROCOTO_SCHEDULER=lsf + + #module load ips/18.0.1.163 + #module load ecflow/4.7.1 + #ECFLOW_START=${ECF_ROOT}/intel/bin/ecflow_start.sh + #ECF_PORT=$(grep $USER /usrx/local/sys/ecflow/assigned_ports.txt | awk '{print $2}') + + DISKNM=/lfs/h1/emc/ptmp/Dusan.Jovic/RT + QUEUE=workq + COMPILE_QUEUE=workq + PARTITION= + ACCNR=GFS-DEV + STMP=/lfs/h1/emc/stmp + PTMP=/lfs/h1/emc/ptmp + SCHEDULER=pbs + cp fv3_conf/fv3_qsub.IN_wcoss2 fv3_conf/fv3_qsub.IN + cp fv3_conf/compile_qsub.IN_wcoss2 fv3_conf/compile_qsub.IN + elif [[ $MACHINE_ID = gaea.* ]]; then module load cray-python/3.7.3.2 @@ -451,6 +478,10 @@ if [[ $ROCOTO == true ]]; then QUEUE=dev COMPILE_QUEUE=dev_transfer ROCOTO_SCHEDULER=lsf + elif [[ $MACHINE_ID = wcoss2 ]]; then + QUEUE=workq + COMPILE_QUEUE=workq + ROCOTO_SCHEDULER=pbs elif [[ $MACHINE_ID = hera.* ]]; then QUEUE=batch COMPILE_QUEUE=batch @@ -512,6 +543,8 @@ EOF QUEUE=dev elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then QUEUE=dev + elif [[ $MACHINE_ID = wcoss2 ]]; then + QUEUE=workq elif [[ $MACHINE_ID = hera.* ]]; then QUEUE=batch elif [[ $MACHINE_ID = orion.* ]]; then diff --git a/tests/rt_acorn.conf b/tests/rt_acorn.conf new file mode 100644 index 0000000000..a5df552904 --- /dev/null +++ b/tests/rt_acorn.conf @@ -0,0 +1,52 @@ +####################################################################################################################################################################################### +# CCPP PROD tests # +####################################################################################################################################################################################### + +COMPILE | CCPP=Y SUITES=FV3_GFS_2017 | standard | | fv3 | + +RUN | fv3_ccpp_control | standard | | fv3 | +RUN | fv3_ccpp_decomp | standard | | | +RUN | fv3_ccpp_2threads | standard | | | +RUN | fv3_ccpp_restart | standard | | fv3 | +RUN | fv3_ccpp_read_inc | standard | | fv3 | +RUN | fv3_ccpp_wrtGauss_netcdf_esmf | standard | | fv3 | +RUN | fv3_ccpp_wrtGauss_netcdf | standard | | fv3 | +RUN | fv3_ccpp_wrtGlatlon_netcdf | standard | | fv3 | +RUN | fv3_ccpp_wrtGauss_nemsio | standard | | fv3 | +RUN | fv3_ccpp_wrtGauss_nemsio_c192 | standard | | fv3 | +RUN | fv3_ccpp_stochy | standard | | fv3 | +RUN | fv3_ccpp_iau | standard | | fv3 | +RUN | fv3_ccpp_lheatstrg | standard | | fv3 | + +COMPILE | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y | standard | | fv3 | +RUN | fv3_ccpp_regional_control | standard | | fv3 | +RUN | fv3_ccpp_regional_restart | standard | | fv3 | fv3_ccpp_regional_control +RUN | fv3_ccpp_regional_quilt | standard | | fv3 | + +COMPILE | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp | standard | | fv3 | +RUN | fv3_ccpp_gfdlmp | standard | | fv3 | +RUN | fv3_ccpp_gfdlmprad_gwd | standard | | fv3 | +RUN | fv3_ccpp_gfdlmprad_noahmp | standard | | fv3 | + +COMPILE | CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq | standard | | fv3 | +RUN | fv3_ccpp_csawmg | standard | | fv3 | +RUN | fv3_ccpp_satmedmf | standard | | fv3 | +RUN | fv3_ccpp_satmedmfq | standard | | fv3 | + +COMPILE | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson,FV3_RAP,FV3_HRRR,FV3_RRFS_v1beta 32BIT=Y | standard | | fv3 | +RUN | fv3_ccpp_gfdlmp_32bit | standard | | fv3 | +RUN | fv3_ccpp_gfdlmprad_32bit_post | standard | | fv3 | +RUN | fv3_ccpp_cpt | standard | | fv3 | +RUN | fv3_ccpp_gsd | standard | | fv3 | +RUN | fv3_ccpp_thompson | standard | | fv3 | +RUN | fv3_ccpp_thompson_no_aero | standard | | fv3 | +RUN | fv3_ccpp_rrfs_v1beta | standard | | fv3 | + +####################################################################################################################################################################################### +# CPLD tests +####################################################################################################################################################################################### + +COMPILE | CCPP=Y SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled S2S=Y | standard | | fv3 | +RUN | cpld_control | standard | | fv3 | +RUN | cpld_2threads | standard | | | +RUN | cpld_decomp | standard | | |