From 3003c67f23aef2aeff9a02e78a8929d4c59e2f82 Mon Sep 17 00:00:00 2001 From: apcraig Date: Thu, 30 Dec 2021 04:17:50 +0000 Subject: [PATCH 01/14] Add narwhal intel, gnu, cray, aocc Add perf_suite.ts --- configuration/scripts/cice.batch.csh | 21 +++++- configuration/scripts/cice.launch.csh | 2 +- .../scripts/machines/Macros.narwhal_aocc | 59 ++++++++++++++++ .../scripts/machines/Macros.narwhal_cray | 60 +++++++++++++++++ .../scripts/machines/Macros.narwhal_gnu | 67 +++++++++++++++++++ .../scripts/machines/Macros.narwhal_intel | 57 ++++++++++++++++ .../scripts/machines/env.narwhal_aocc | 52 ++++++++++++++ .../scripts/machines/env.narwhal_cray | 52 ++++++++++++++ .../scripts/machines/env.narwhal_gnu | 52 ++++++++++++++ .../scripts/machines/env.narwhal_intel | 52 ++++++++++++++ configuration/scripts/tests/perf_suite.ts | 27 ++++++++ 11 files changed, 499 insertions(+), 2 deletions(-) create mode 100644 configuration/scripts/machines/Macros.narwhal_aocc create mode 100644 configuration/scripts/machines/Macros.narwhal_cray create mode 100644 configuration/scripts/machines/Macros.narwhal_gnu create mode 100644 configuration/scripts/machines/Macros.narwhal_intel create mode 100755 configuration/scripts/machines/env.narwhal_aocc create mode 100755 configuration/scripts/machines/env.narwhal_cray create mode 100755 configuration/scripts/machines/env.narwhal_gnu create mode 100755 configuration/scripts/machines/env.narwhal_intel create mode 100644 configuration/scripts/tests/perf_suite.ts diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 024270039..f86b55502 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -91,7 +91,26 @@ cat >> ${jobfile} << EOFB #PBS -l walltime=${batchtime} EOFB -else if (${ICE_MACHINE} =~ gordon* || ${ICE_MACHINE} =~ conrad* || ${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr* || ${ICE_MACHINE} =~ mustang) then +else if (${ICE_MACHINE} =~ gordon* || ${ICE_MACHINE} =~ conrad* || ${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr* || ${ICE_MACHINE} =~ mustang*) then +cat >> ${jobfile} << EOFB +#PBS -N ${shortcase} +#PBS -q ${queue} +#PBS -A ${acct} +#PBS -l select=${nnodes}:ncpus=${maxtpn}:mpiprocs=${taskpernode} +#PBS -l walltime=${batchtime} +#PBS -j oe +#PBS -W umask=022 +###PBS -M username@domain.com +###PBS -m be +EOFB + +else if (${ICE_MACHINE} =~ narwhal*) then +if (${runlength} <= 0) then + set batchtime = "00:29:59" + set queue = "debug" +else + set queue = "standard" +endif cat >> ${jobfile} << EOFB #PBS -N ${shortcase} #PBS -q ${queue} diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index 40b8996b4..a63c802ed 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -58,7 +58,7 @@ EOFR endif #======= -else if (${ICE_MACHINE} =~ onyx*) then +else if (${ICE_MACHINE} =~ onyx* || ${ICE_MACHINE} =~ narwhal) then cat >> ${jobfile} << EOFR aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE EOFR diff --git a/configuration/scripts/machines/Macros.narwhal_aocc b/configuration/scripts/machines/Macros.narwhal_aocc new file mode 100644 index 000000000..44b1dc2f6 --- /dev/null +++ b/configuration/scripts/machines/Macros.narwhal_aocc @@ -0,0 +1,59 @@ +#============================================================================== +# Macros file for NAVYDSRC narwhal, aocc compiler +#============================================================================== + +CPP := ftn -E +CPPDEFS := -DNO_R16 -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 + +FIXEDFLAGS := -ffixed-form +FREEFLAGS := -ffree-form +FFLAGS := -byteswapio +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fsanitize=integer-divide-by-zero,float-divide-by-zero,bounds +else + FFLAGS += -O2 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +# defined by module +#NETCDF_PATH := $(NETCDF) +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default +#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) +#INCLDIR += -I$(NETCDF_PATH)/include + +#LIB_NETCDF := $(NETCDF_PATH)/lib +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) +#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -mp + CFLAGS += -mp + FFLAGS += -mp +else + LDFLAGS += -nomp +# CFLAGS += -nomp + FFLAGS += -nomp +endif + diff --git a/configuration/scripts/machines/Macros.narwhal_cray b/configuration/scripts/machines/Macros.narwhal_cray new file mode 100644 index 000000000..ab0e6378e --- /dev/null +++ b/configuration/scripts/machines/Macros.narwhal_cray @@ -0,0 +1,60 @@ +#============================================================================== +# Macros file for NAVYDSRC narwhal, cray compiler +#============================================================================== + +CPP := ftn -e P +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 + +FIXEDFLAGS := -132 +FREEFLAGS := +FFLAGS := -hbyteswapio +FFLAGS_NOOPT:= -O0 +LDFLAGS := -hbyteswapio + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -hfp0 -g -Rbcdps -Ktrap=fp +else + FFLAGS += -O2 -hfp0 # -eo +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +# defined by module +#NETCDF_PATH := $(NETCDF) +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default +#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) +#INCLDIR += -I$(NETCDF_PATH)/include + +#LIB_NETCDF := $(NETCDF_PATH)/lib +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) +#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +else + LDFLAGS += -hnoomp +# CFLAGS += -hnoomp + FFLAGS += -hnoomp +endif + diff --git a/configuration/scripts/machines/Macros.narwhal_gnu b/configuration/scripts/machines/Macros.narwhal_gnu new file mode 100644 index 000000000..e980c1e29 --- /dev/null +++ b/configuration/scripts/machines/Macros.narwhal_gnu @@ -0,0 +1,67 @@ +#============================================================================== +# Macros file for NAVYDSRC narwhal, gnu compiler +#============================================================================== + +CPP := ftn -E +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c + +FIXEDFLAGS := -ffixed-line-length-132 +FREEFLAGS := -ffree-form +FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none -fallow-argument-mismatch +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + CFLAGS += -O0 +endif + +ifeq ($(ICE_COVERAGE), true) + FFLAGS += -O0 -g -fprofile-arcs -ftest-coverage + CFLAGS += -O0 -g -coverage + LDFLAGS += -g -ftest-coverage -fprofile-arcs +endif + +ifneq ($(ICE_BLDDEBUG), true) +ifneq ($(ICE_COVERAGE), true) + FFLAGS += -O2 + CFLAGS += -O2 +endif +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +# defined by module +#NETCDF_PATH := $(NETCDF) +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default +#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) +#INCLDIR += -I$(NETCDF_PATH)/include + +#LIB_NETCDF := $(NETCDF_PATH)/lib +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) +#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif + diff --git a/configuration/scripts/machines/Macros.narwhal_intel b/configuration/scripts/machines/Macros.narwhal_intel new file mode 100644 index 000000000..c7c103b24 --- /dev/null +++ b/configuration/scripts/machines/Macros.narwhal_intel @@ -0,0 +1,57 @@ +#============================================================================== +# Macros file for NAVYDSRC narwhal, intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise -fcommon + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback +# -mcmodel medium -shared-intel +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays +else + FFLAGS += -O2 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +# defined by module +#NETCDF_PATH := $(NETCDF) +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default +#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) +#INCLDIR += -I$(NETCDF_PATH)/include + +#LIB_NETCDF := $(NETCDF_PATH)/lib +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) +#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/env.narwhal_aocc b/configuration/scripts/machines/env.narwhal_aocc new file mode 100755 index 000000000..a392f9363 --- /dev/null +++ b/configuration/scripts/machines/env.narwhal_aocc @@ -0,0 +1,52 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source ${MODULESHOME}/init/csh + +module unload PrgEnv-aocc +module unload PrgEnv-cray +module unload PrgEnv-gnu +module unload PrgEnv-intel +module unload PrgEnv-nvidia +module load PrgEnv-aocc/8.1.0 +module load cray-pals/1.0.17 +module load bct-env/0.1 +module unload aocc +module load aocc/2.2.0.1 +module unload cray-mpich +module load cray-mpich/8.1.5 + +module unload cray-hdf5 +module unload cray-hdf5-parallel +module unload cray-netcdf-hdf5parallel +module unload cray-parallel-netcdf +module unload netcdf +module load cray-netcdf/4.7.4.4 +module load cray-hdf5/1.12.0.4 + +setenv NETCDF_PATH ${NETCDF_DIR} +limit coredumpsize unlimited +limit stacksize unlimited + +endif + +setenv ICE_MACHINE_MACHNAME narwhal +setenv ICE_MACHINE_MACHINFO "Cray EX AMD EPYC 7H12" +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_ENVINFO "aocc_3.0.0-Build#78 2020_12_10 clang/flang 12.0.0, cray-mpich/8.1.9, netcdf/4.7.4.4" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /p/work1/projects/RASM/cice_consortium +setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub " +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_QUEUE "debug" +setenv ICE_MACHINE_TPNODE 128 # tasks per node +setenv ICE_MACHINE_BLDTHRDS 12 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.narwhal_cray b/configuration/scripts/machines/env.narwhal_cray new file mode 100755 index 000000000..91a06081a --- /dev/null +++ b/configuration/scripts/machines/env.narwhal_cray @@ -0,0 +1,52 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source ${MODULESHOME}/init/csh + +module unload PrgEnv-aocc +module unload PrgEnv-cray +module unload PrgEnv-gnu +module unload PrgEnv-intel +module unload PrgEnv-nvidia +module load PrgEnv-cray/8.1.0 +module load cray-pals/1.0.17 +module load bct-env/0.1 +module unload cce +module load cce/12.0.3 +module unload cray-mpich +module load cray-mpich/8.1.9 + +module unload cray-hdf5 +module unload cray-hdf5-parallel +module unload cray-netcdf-hdf5parallel +module unload cray-parallel-netcdf +module unload netcdf +module load cray-netcdf/4.7.4.4 +module load cray-hdf5/1.12.0.4 + +setenv NETCDF_PATH ${NETCDF_DIR} +limit coredumpsize unlimited +limit stacksize unlimited + +endif + +setenv ICE_MACHINE_MACHNAME narwhal +setenv ICE_MACHINE_MACHINFO "Cray EX AMD EPYC 7H12" +setenv ICE_MACHINE_ENVNAME cray +setenv ICE_MACHINE_ENVINFO "cce 12.0.3, cray-mpich/8.1.9, netcdf/4.7.4.4" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /p/work1/projects/RASM/cice_consortium +setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub " +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_QUEUE "debug" +setenv ICE_MACHINE_TPNODE 128 # tasks per node +setenv ICE_MACHINE_BLDTHRDS 12 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.narwhal_gnu b/configuration/scripts/machines/env.narwhal_gnu new file mode 100755 index 000000000..4df81b957 --- /dev/null +++ b/configuration/scripts/machines/env.narwhal_gnu @@ -0,0 +1,52 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source ${MODULESHOME}/init/csh + +module unload PrgEnv-aocc +module unload PrgEnv-cray +module unload PrgEnv-gnu +module unload PrgEnv-intel +module unload PrgEnv-nvidia +module load PrgEnv-gnu/8.1.0 +module load cray-pals/1.0.17 +module load bct-env/0.1 +module unload gcc +module load gcc/11.2.0 +module unload cray-mpich +module load cray-mpich/8.1.9 + +module unload cray-hdf5 +module unload cray-hdf5-parallel +module unload cray-netcdf-hdf5parallel +module unload cray-parallel-netcdf +module unload netcdf +module load cray-netcdf/4.7.4.4 +module load cray-hdf5/1.12.0.4 + +setenv NETCDF_PATH ${NETCDF_DIR} +limit coredumpsize unlimited +limit stacksize unlimited + +endif + +setenv ICE_MACHINE_MACHNAME narwhal +setenv ICE_MACHINE_MACHINFO "Cray EX AMD EPYC 7H12" +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_ENVINFO "gnu fortran/c 11.2.0, cray-mpich/8.1.9, netcdf/4.7.4.4" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /p/work1/projects/RASM/cice_consortium +setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub " +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_QUEUE "debug" +setenv ICE_MACHINE_TPNODE 128 # tasks per node +setenv ICE_MACHINE_BLDTHRDS 12 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.narwhal_intel b/configuration/scripts/machines/env.narwhal_intel new file mode 100755 index 000000000..2cdf4f93c --- /dev/null +++ b/configuration/scripts/machines/env.narwhal_intel @@ -0,0 +1,52 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source ${MODULESHOME}/init/csh + +module unload PrgEnv-aocc +module unload PrgEnv-cray +module unload PrgEnv-gnu +module unload PrgEnv-intel +module unload PrgEnv-nvidia +module load PrgEnv-intel/8.0.0 +module load cray-pals/1.0.17 +module load bct-env/0.1 +module unload intel +module load intel/2021.1 +module unload cray-mpich +module load cray-mpich/8.1.9 + +module unload cray-hdf5 +module unload cray-hdf5-parallel +module unload cray-netcdf-hdf5parallel +module unload cray-parallel-netcdf +module unload netcdf +module load cray-netcdf/4.7.4.4 +module load cray-hdf5/1.12.0.4 + +setenv NETCDF_PATH ${NETCDF_DIR} +limit coredumpsize unlimited +limit stacksize unlimited + +endif + +setenv ICE_MACHINE_MACHNAME narwhal +setenv ICE_MACHINE_MACHINFO "Cray EX AMD EPYC 7H12" +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_ENVINFO "ifort 2021.1 Beta 20201112, cray-mpich/8.1.9, netcdf/4.7.4.4" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /p/work1/projects/RASM/cice_consortium +setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub " +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_QUEUE "debug" +setenv ICE_MACHINE_TPNODE 128 # tasks per node +setenv ICE_MACHINE_BLDTHRDS 12 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/tests/perf_suite.ts b/configuration/scripts/tests/perf_suite.ts new file mode 100644 index 000000000..cc5c48440 --- /dev/null +++ b/configuration/scripts/tests/perf_suite.ts @@ -0,0 +1,27 @@ +# Test Grid PEs Sets BFB-compare +smoke gx1 1x1x320x384x1 run2day,droundrobin +smoke gx1 64x1x16x16x8 run2day,droundrobin,thread +sleep 180 +# +smoke gx1 1x1x320x384x1 run2day,droundrobin +smoke gx1 1x1x160x192x4 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +smoke gx1 1x1x80x96x16 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +smoke gx1 1x1x40x48x64 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +smoke gx1 1x1x20x24x256 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +# +smoke gx1 1x1x16x16x480 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +smoke gx1 2x1x16x16x240 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +smoke gx1 4x1x16x16x120 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +smoke gx1 8x1x16x16x60 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +smoke gx1 16x1x16x16x30 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +smoke gx1 32x1x16x16x15 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +smoke gx1 64x1x16x16x8 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +smoke gx1 128x1x16x16x4 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +# +smoke gx1 64x1x16x16x8 run2day,droundrobin smoke_gx1_1x1x320x384x1_droundrobin_run2day +smoke gx1 64x1x16x16x8 run2day,droundrobin,thread +smoke gx1 32x2x16x16x16 run2day,droundrobin smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread +smoke gx1 16x4x16x16x32 run2day,droundrobin smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread +smoke gx1 8x8x16x16x64 run2day,droundrobin smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread +smoke gx1 8x16x16x16x128 run2day,droundrobin smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread +# From 3525b14d30f2016c100c47bc5d3875f72acfe2a9 Mon Sep 17 00:00:00 2001 From: apcraig Date: Fri, 31 Dec 2021 21:04:40 +0000 Subject: [PATCH 02/14] update narwhal_cray and perf_suite --- configuration/scripts/machines/env.narwhal_cray | 1 + configuration/scripts/tests/perf_suite.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/configuration/scripts/machines/env.narwhal_cray b/configuration/scripts/machines/env.narwhal_cray index 91a06081a..eb9e42bb2 100755 --- a/configuration/scripts/machines/env.narwhal_cray +++ b/configuration/scripts/machines/env.narwhal_cray @@ -33,6 +33,7 @@ module load cray-hdf5/1.12.0.4 setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited limit stacksize unlimited +setenv OMP_WAIT_POLICY passive endif diff --git a/configuration/scripts/tests/perf_suite.ts b/configuration/scripts/tests/perf_suite.ts index cc5c48440..859b9f21b 100644 --- a/configuration/scripts/tests/perf_suite.ts +++ b/configuration/scripts/tests/perf_suite.ts @@ -23,5 +23,5 @@ smoke gx1 64x1x16x16x8 run2day,droundrobin,thread smoke gx1 32x2x16x16x16 run2day,droundrobin smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread smoke gx1 16x4x16x16x32 run2day,droundrobin smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread smoke gx1 8x8x16x16x64 run2day,droundrobin smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread -smoke gx1 8x16x16x16x128 run2day,droundrobin smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread +smoke gx1 4x16x16x16x128 run2day,droundrobin smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread # From 80e8caa9981f00a09380405bd41e2d50816772f9 Mon Sep 17 00:00:00 2001 From: apcraig Date: Fri, 7 Jan 2022 12:16:31 -0700 Subject: [PATCH 03/14] Review and update OMP implementation - Fix call to timers in block loops - Fix some OMP Private variables - Test OMP Scheduling, add SCHEDULE(runtime) to some OMP loops - Review column and advection OMP implementation - ADD OMP_TIMERS CPP option (temporary) to time threaded sections - Add timer_tmp timers (temporary) - Add omp_suite.ts test suite - Add ability to set OMP_SCHEDULE via options (ompscheds, ompscheds1, ompschedd1) --- cicecore/cicedynB/dynamics/ice_dyn_eap.F90 | 10 +- cicecore/cicedynB/dynamics/ice_dyn_evp.F90 | 2 +- cicecore/cicedynB/dynamics/ice_dyn_vp.F90 | 32 ++--- .../dynamics/ice_transport_driver.F90 | 114 ++++++++++++++-- .../cicedynB/dynamics/ice_transport_remap.F90 | 87 ++++++++++-- cicecore/cicedynB/general/ice_step_mod.F90 | 26 ++-- .../infrastructure/comm/mpi/ice_timers.F90 | 32 ++++- .../infrastructure/comm/serial/ice_timers.F90 | 30 ++++- cicecore/drivers/nuopc/dmi/CICE_RunMod.F90 | 4 +- .../drivers/standalone/cice/CICE_RunMod.F90 | 124 +++++++++++++++--- configuration/scripts/cice.run.setup.csh | 6 +- configuration/scripts/cice.settings | 1 + .../scripts/options/set_env.ompschedd1 | 1 + .../scripts/options/set_env.ompscheds | 1 + .../scripts/options/set_env.ompscheds1 | 1 + configuration/scripts/tests/omp_suite.ts | 48 +++++++ configuration/scripts/tests/perf_suite.ts | 3 + 17 files changed, 437 insertions(+), 85 deletions(-) create mode 100644 configuration/scripts/options/set_env.ompschedd1 create mode 100644 configuration/scripts/options/set_env.ompscheds create mode 100644 configuration/scripts/options/set_env.ompscheds1 create mode 100644 configuration/scripts/tests/omp_suite.ts diff --git a/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 b/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 index 83374d4dd..e95e4cdef 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 @@ -277,7 +277,7 @@ subroutine eap (dt) ! tcraig, tcx, turned off this threaded region, in evp, this block and ! the icepack_ice_strength call seems to not be thread safe. more ! debugging needed - !$TCXOMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block) + !$TCXOMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block,ij) do iblk = 1, nblocks !----------------------------------------------------------------- @@ -429,7 +429,7 @@ subroutine eap (dt) !$TCXOMP PARALLEL DO PRIVATE(iblk,strtmp) do iblk = 1, nblocks -! call ice_timer_start(timer_tmp1) ! dynamics +! call ice_timer_start(timer_tmp1,iblk) ! dynamics call stress_eap (nx_block, ny_block, & ksub, ndte, & icellt(iblk), & @@ -462,7 +462,7 @@ subroutine eap (dt) ! rdg_conv (:,:,iblk), rdg_shear (:,:,iblk), & rdg_conv (:,:,iblk), & strtmp (:,:,:)) -! call ice_timer_stop(timer_tmp1) ! dynamics +! call ice_timer_stop(timer_tmp1,iblk) ! dynamics !----------------------------------------------------------------- ! momentum equation @@ -488,7 +488,7 @@ subroutine eap (dt) ! evolution of structure tensor A !----------------------------------------------------------------- -! call ice_timer_start(timer_tmp3) ! dynamics +! call ice_timer_start(timer_tmp3,iblk) ! dynamics if (mod(ksub,10) == 1) then ! only called every 10th timestep call stepa (nx_block, ny_block, & dtei, icellt (iblk), & @@ -505,7 +505,7 @@ subroutine eap (dt) stress12_1(:,:,iblk), stress12_2(:,:,iblk), & stress12_3(:,:,iblk), stress12_4(:,:,iblk)) endif -! call ice_timer_stop(timer_tmp3) ! dynamics +! call ice_timer_stop(timer_tmp3,iblk) ! dynamics enddo !$TCXOMP END PARALLEL DO diff --git a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 index 8f3fc4910..143c3a8dd 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 @@ -241,7 +241,7 @@ subroutine evp (dt) ! tcraig, tcx, threading here leads to some non-reproducbile results and failures in icepack_ice_strength ! need to do more debugging - !$TCXOMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) + !$TCXOMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,ij,i,j) do iblk = 1, nblocks !----------------------------------------------------------------- diff --git a/cicecore/cicedynB/dynamics/ice_dyn_vp.F90 b/cicecore/cicedynB/dynamics/ice_dyn_vp.F90 index 2f1285084..a6cf2aebd 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_vp.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_vp.F90 @@ -343,7 +343,7 @@ subroutine implicit_solver (dt) ! tcraig, tcx, threading here leads to some non-reproducbile results and failures in icepack_ice_strength ! need to do more debugging - !$TCXOMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) + !$TCXOMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,ij,i,j) do iblk = 1, nblocks !----------------------------------------------------------------- @@ -814,7 +814,7 @@ subroutine anderson_solver (icellt , icellu, & !----------------------------------------------------------------- ! Calc zetax2, etax2, dPr/dx, dPr/dy, Cb and vrel = f(uprev_k, vprev_k) !----------------------------------------------------------------- - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk,stress_Pr) do iblk = 1, nblocks if (use_mean_vrel) then @@ -912,7 +912,7 @@ subroutine anderson_solver (icellt , icellu, & ! Prepare diagonal for preconditioner if (precond == 'diag' .or. precond == 'pgmres') then - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk,diag_rheo) do iblk = 1, nblocks ! first compute diagonal contributions due to rheology term call formDiag_step1 (nx_block , ny_block , & @@ -2851,7 +2851,7 @@ subroutine fgmres (zetax2 , etax2 , & ! Normalize the first Arnoldi vector inverse_norm = c1 / norm_residual - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) @@ -2947,7 +2947,7 @@ subroutine fgmres (zetax2 , etax2 , & if (.not. almost_zero( hessenberg(nextit,initer) ) ) then ! Normalize next Arnoldi vector inverse_norm = c1 / hessenberg(nextit,initer) - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) @@ -3013,7 +3013,7 @@ subroutine fgmres (zetax2 , etax2 , & ! Form linear combination to get new solution iterate do it = 1, initer t = rhs_hess(it) - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) @@ -3057,7 +3057,7 @@ subroutine fgmres (zetax2 , etax2 , & workspace_x = c0 workspace_y = c0 do it = 1, nextit - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) @@ -3244,7 +3244,7 @@ subroutine pgmres (zetax2 , etax2 , & ! Normalize the first Arnoldi vector inverse_norm = c1 / norm_residual - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) @@ -3329,7 +3329,7 @@ subroutine pgmres (zetax2 , etax2 , & if (.not. almost_zero( hessenberg(nextit,initer) ) ) then ! Normalize next Arnoldi vector inverse_norm = c1 / hessenberg(nextit,initer) - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) @@ -3397,7 +3397,7 @@ subroutine pgmres (zetax2 , etax2 , & workspace_y = c0 do it = 1, initer t = rhs_hess(it) - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) @@ -3453,7 +3453,7 @@ subroutine pgmres (zetax2 , etax2 , & workspace_x = c0 workspace_y = c0 do it = 1, nextit - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) @@ -3534,7 +3534,7 @@ subroutine precondition(zetax2 , etax2, & wx = vx wy = vy elseif (precond_type == 'diag') then ! Jacobi preconditioner (diagonal) - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) @@ -3617,7 +3617,7 @@ subroutine orthogonalize(ortho_type , initer , & do it = 1, initer local_dot = c0 - !$OMP PARALLEL DO PRIVATE(iblk, ij, i, j) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) @@ -3637,7 +3637,7 @@ subroutine orthogonalize(ortho_type , initer , & ! Second loop of Gram-Schmidt (orthonormalize) do it = 1, initer - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) @@ -3656,7 +3656,7 @@ subroutine orthogonalize(ortho_type , initer , & do it = 1, initer local_dot = c0 - !$OMP PARALLEL DO PRIVATE(iblk, ij, i, j) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) @@ -3671,7 +3671,7 @@ subroutine orthogonalize(ortho_type , initer , & hessenberg(it,initer) = global_sum(sum(local_dot), distrb_info) - !$OMP PARALLEL DO PRIVATE(iblk, ij, i, j) + !$OMP PARALLEL DO PRIVATE(iblk,ij,i,j) do iblk = 1, nblocks do ij = 1, icellu(iblk) i = indxui(ij, iblk) diff --git a/cicecore/cicedynB/dynamics/ice_transport_driver.F90 b/cicecore/cicedynB/dynamics/ice_transport_driver.F90 index f2dff2367..2ac99d640 100644 --- a/cicecore/cicedynB/dynamics/ice_transport_driver.F90 +++ b/cicecore/cicedynB/dynamics/ice_transport_driver.F90 @@ -267,6 +267,12 @@ subroutine transport_remap (dt) timer_advect, timer_bound use ice_transport_remap, only: horizontal_remap, make_masks + use ice_fileunits, only: nu_diag, flush_fileunit + use ice_communicate, only: my_task, master_task +#ifdef OMP_TIMERS + use OMP_LIB +#endif + real (kind=dbl_kind), intent(in) :: & dt ! time step @@ -331,6 +337,11 @@ subroutine transport_remap (dt) character(len=char_len_long) :: fieldid +#ifdef OMP_TIMERS + real (kind=dbl_kind) :: & + dtimes(nblocks),dtimee(nblocks) +#endif + character(len=*), parameter :: subname = '(transport_remap)' call ice_timer_start(timer_advect) ! advection @@ -355,7 +366,7 @@ subroutine transport_remap (dt) ! Here we assume that aice0 is up to date. !------------------------------------------------------------------- -! !$OMP PARALLEL DO PRIVATE(i,j,iblk) +! !$OMP PARALLEL DO PRIVATE(i,j,iblk) SCHEDULE(runtime) ! do iblk = 1, nblocks ! do j = 1, ny_block ! do i = 1, nx_block @@ -397,9 +408,15 @@ subroutine transport_remap (dt) ! call ice_timer_stop(timer_bound) -! MHRI: CHECK THIS OMP ... maybe ok: Were trcrn(:,:,1:ntrcr,:,iblk) in my testcode - !$OMP PARALLEL DO PRIVATE(iblk) +#ifdef OMP_TIMERS + dtimes = 0. + dtimee = 0. +#endif + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks +#ifdef OMP_TIMERS + dtimes(iblk) = OMP_GET_WTIME() +#endif !------------------------------------------------------------------- ! Fill arrays with fields to be remapped. @@ -412,8 +429,18 @@ subroutine transport_remap (dt) vicen(:,:,:,iblk), vsnon(:,:, :,iblk), & aim (:,:,:,iblk), trm (:,:,:,:,iblk)) +#ifdef OMP_TIMERS + dtimee(iblk) = OMP_GET_WTIME() +#endif enddo !$OMP END PARALLEL DO +#ifdef OMP_TIMERS + if (my_task == master_task) then + do iblk = 1,nblocks + write(nu_diag,*) subname,'t1',iblk,dtimes(iblk),dtimee(iblk)-dtimes(iblk) + enddo + endif +#endif !---!------------------------------------------------------------------- !---! Optional conservation and monotonicity checks. @@ -471,8 +498,15 @@ subroutine transport_remap (dt) tmin(:,:,:,:,:) = c0 tmax(:,:,:,:,:) = c0 - !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,n) +#ifdef OMP_TIMERS + dtimes = 0. + dtimee = 0. +#endif + !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,n) SCHEDULE(runtime) do iblk = 1, nblocks +#ifdef OMP_TIMERS + dtimes(iblk) = OMP_GET_WTIME() +#endif this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo ihi = this_block%ihi @@ -506,8 +540,18 @@ subroutine transport_remap (dt) tmin(:,:,:,n,iblk), tmax (:,:,:,n,iblk), & aimask(:,:,n,iblk), trmask(:,:,:,n,iblk)) enddo +#ifdef OMP_TIMERS + dtimee(iblk) = OMP_GET_WTIME() +#endif enddo !$OMP END PARALLEL DO +#ifdef OMP_TIMERS + if (my_task == master_task) then + do iblk = 1,nblocks + write(nu_diag,*) subname,'t2',iblk,dtimes(iblk),dtimee(iblk)-dtimes(iblk) + enddo + endif +#endif call ice_timer_start(timer_bound) call ice_HaloUpdate (tmin, halo_info, & @@ -516,8 +560,15 @@ subroutine transport_remap (dt) field_loc_center, field_type_scalar) call ice_timer_stop(timer_bound) - !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,n) +#ifdef OMP_TIMERS + dtimes = 0. + dtimee = 0. +#endif + !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,n) SCHEDULE(runtime) do iblk = 1, nblocks +#ifdef OMP_TIMERS + dtimes(iblk) = OMP_GET_WTIME() +#endif this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo ihi = this_block%ihi @@ -530,8 +581,18 @@ subroutine transport_remap (dt) tmin(:,:,:,n,iblk), & tmax(:,:,:,n,iblk)) enddo +#ifdef OMP_TIMERS + dtimee(iblk) = OMP_GET_WTIME() +#endif enddo !$OMP END PARALLEL DO +#ifdef OMP_TIMERS + if (my_task == master_task) then + do iblk = 1,nblocks + write(nu_diag,*) subname,'t3',iblk,dtimes(iblk),dtimee(iblk)-dtimes(iblk) + enddo + endif +#endif endif ! l_monotonicity_check @@ -551,10 +612,16 @@ subroutine transport_remap (dt) ! Given new fields, recompute state variables. !------------------------------------------------------------------- -! MHRI: CHECK THIS OMP ... maybe ok: Were trcrn(:,:,1:ntrcr,:,iblk) in my testcode - !$OMP PARALLEL DO PRIVATE(iblk) +#ifdef OMP_TIMERS + dtimes = 0. + dtimee = 0. +#endif + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks +#ifdef OMP_TIMERS + dtimes(iblk) = OMP_GET_WTIME() +#endif call tracers_to_state (nx_block, ny_block, & ntrcr, ntrace, & aim (:,:,:,iblk), trm (:,:,:,:,iblk), & @@ -562,8 +629,18 @@ subroutine transport_remap (dt) trcrn(:,:,:,:,iblk), & vicen(:,:,:,iblk), vsnon(:,:, :,iblk)) +#ifdef OMP_TIMERS + dtimee(iblk) = OMP_GET_WTIME() +#endif enddo ! iblk !$OMP END PARALLEL DO +#ifdef OMP_TIMERS + if (my_task == master_task) then + do iblk = 1,nblocks + write(nu_diag,*) subname,'t4',iblk,dtimes(iblk),dtimee(iblk)-dtimes(iblk) + enddo + endif +#endif !------------------------------------------------------------------- ! Ghost cell updates for state variables. @@ -655,9 +732,16 @@ subroutine transport_remap (dt) ! Check tracer monotonicity. (Optional) !------------------------------------------------------------------- +#ifdef OMP_TIMERS + dtimes = 0. + dtimee = 0. +#endif if (l_monotonicity_check) then - !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,n,ckflag,istop,jstop) + !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,n,ckflag,istop,jstop) SCHEDULE(runtime) do iblk = 1, nblocks +#ifdef OMP_TIMERS + dtimes(iblk) = OMP_GET_WTIME() +#endif this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo ihi = this_block%ihi @@ -684,8 +768,18 @@ subroutine transport_remap (dt) endif enddo ! n +#ifdef OMP_TIMERS + dtimee(iblk) = OMP_GET_WTIME() +#endif enddo ! iblk !$OMP END PARALLEL DO +#ifdef OMP_TIMERS + if (my_task == master_task) then + do iblk = 1,nblocks + write(nu_diag,*) subname,'t5',iblk,dtimes(iblk),dtimee(iblk)-dtimes(iblk) + enddo + endif +#endif deallocate(tmin, tmax, STAT=alloc_error) if (alloc_error /= 0) call abort_ice (subname//'ERROR: deallocation error') @@ -762,7 +856,7 @@ subroutine transport_upwind (dt) ! Average corner velocities to edges. !------------------------------------------------------------------- - !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block) + !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block) SCHEDULE(runtime) do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -786,7 +880,7 @@ subroutine transport_upwind (dt) field_loc_Nface, field_type_vector) call ice_timer_stop(timer_bound) - !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) + !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) SCHEDULE(runtime) do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo diff --git a/cicecore/cicedynB/dynamics/ice_transport_remap.F90 b/cicecore/cicedynB/dynamics/ice_transport_remap.F90 index 89c0609ef..bb36607ca 100644 --- a/cicecore/cicedynB/dynamics/ice_transport_remap.F90 +++ b/cicecore/cicedynB/dynamics/ice_transport_remap.F90 @@ -270,7 +270,7 @@ subroutine init_remap ! Note: On a rectangular grid, the integral of any odd function ! of x or y = 0. - !$OMP PARALLEL DO PRIVATE(iblk,i,j) + !$OMP PARALLEL DO PRIVATE(iblk,i,j) SCHEDULE(runtime) do iblk = 1, nblocks do j = 1, ny_block do i = 1, nx_block @@ -331,6 +331,11 @@ subroutine horizontal_remap (dt, ntrace, & ! xyav, xxxav, xxyav, xyyav, yyyav use ice_calendar, only: istep1 use ice_timers, only: ice_timer_start, ice_timer_stop, timer_bound + use ice_fileunits, only: nu_diag, flush_fileunit + use ice_communicate, only: my_task, master_task +#ifdef OMP_TIMERS + use OMP_LIB +#endif real (kind=dbl_kind), intent(in) :: & dt ! time step @@ -448,6 +453,11 @@ subroutine horizontal_remap (dt, ntrace, & type (block) :: & this_block ! block information for current block +#ifdef OMP_TIMERS + real (kind=dbl_kind) :: & + time0,time1 +#endif + character(len=*), parameter :: subname = '(horizontal_remap)' !---!------------------------------------------------------------------- @@ -455,10 +465,22 @@ subroutine horizontal_remap (dt, ntrace, & !---! Remap the open water area (without tracers). !---!------------------------------------------------------------------- +#ifdef OMP_TIMERS + time0 = OMP_GET_WTIME() + + if (my_task == master_task) then + time1 = OMP_GET_WTIME() + write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk,time1-time0,'t1' + time0 = time1 + call flush_fileunit(nu_diag) + endif +#endif + !--- tcraig, tcx, this omp loop leads to a seg fault in gnu !--- need to check private variables and debug further - !$TCXOMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,n,m, & - !$TCXOMP indxinc,indxjnc,mmask,tmask,istop,jstop,l_stop) + !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,n, & + !$OMP indxinc,indxjnc,mmask,tmask,istop,jstop,l_stop) & + !$OMP SCHEDULE(runtime) do iblk = 1, nblocks l_stop = .false. @@ -560,7 +582,16 @@ subroutine horizontal_remap (dt, ntrace, & endif enddo ! iblk - !$TCXOMP END PARALLEL DO + !$OMP END PARALLEL DO + +#ifdef OMP_TIMERS + if (my_task == master_task) then + time1 = OMP_GET_WTIME() + write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk,time1-time0,'t2' + time0 = time1 + call flush_fileunit(nu_diag) + endif +#endif !------------------------------------------------------------------- ! Ghost cell updates @@ -585,10 +616,18 @@ subroutine horizontal_remap (dt, ntrace, & call ice_HaloUpdate (my, halo_info, & field_loc_center, field_type_vector) +#ifdef OMP_TIMERS + if (my_task == master_task) then + time1 = OMP_GET_WTIME() + write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk,time1-time0,'t3' + time0 = time1 + call flush_fileunit(nu_diag) + endif +#endif ! tracer fields if (maskhalo_remap) then halomask(:,:,:) = 0 - !$OMP PARALLEL DO PRIVATE(iblk,this_block,ilo,ihi,jlo,jhi,n,m,j,i) + !$OMP PARALLEL DO PRIVATE(iblk,this_block,ilo,ihi,jlo,jhi,n,m,j,i) SCHEDULE(runtime) do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -609,6 +648,14 @@ subroutine horizontal_remap (dt, ntrace, & enddo enddo !$OMP END PARALLEL DO +#ifdef OMP_TIMERS + if (my_task == master_task) then + time1 = OMP_GET_WTIME() + write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk,time1-time0,'t4' + time0 = time1 + call flush_fileunit(nu_diag) + endif +#endif call ice_HaloUpdate(halomask, halo_info, & field_loc_center, field_type_scalar) call ice_HaloMask(halo_info_tracer, halo_info, halomask) @@ -632,12 +679,21 @@ subroutine horizontal_remap (dt, ntrace, & endif ! nghost - !--- tcraig, tcx, this omp loop leads to a seg fault in gnu +#ifdef OMP_TIMERS + if (my_task == master_task) then + time1 = OMP_GET_WTIME() + write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk,time1-time0,'t5' + time0 = time1 + call flush_fileunit(nu_diag) + endif +#endif + !--- need to check private variables and debug further - !$TCXOMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block,n,m, & - !$TCXOMP edgearea_e,edgearea_n,edge,iflux,jflux, & - !$TCXOMP xp,yp,indxing,indxjng,mflxe,mflxn, & - !$TCXOMP mtflxe,mtflxn,triarea,istop,jstop,l_stop) + !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block,n, & + !$OMP edgearea_e,edgearea_n,edge,iflux,jflux, & + !$OMP xp,yp,indxing,indxjng,mflxe,mflxn, & + !$OMP mtflxe,mtflxn,triarea,istop,jstop,l_stop) & + !$OMP SCHEDULE(runtime) do iblk = 1, nblocks l_stop = .false. @@ -845,8 +901,17 @@ subroutine horizontal_remap (dt, ntrace, & enddo ! n enddo ! iblk - !$TCXOMP END PARALLEL DO + !$OMP END PARALLEL DO +#ifdef OMP_TIMERS + if (my_task == master_task) then + time1 = OMP_GET_WTIME() + write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk,time1-time0,'t6' + time0 = time1 + call flush_fileunit(nu_diag) + endif +#endif + end subroutine horizontal_remap !======================================================================= diff --git a/cicecore/cicedynB/general/ice_step_mod.F90 b/cicecore/cicedynB/general/ice_step_mod.F90 index 976e95361..bfe29efcd 100644 --- a/cicecore/cicedynB/general/ice_step_mod.F90 +++ b/cicecore/cicedynB/general/ice_step_mod.F90 @@ -98,7 +98,7 @@ subroutine prep_radiation (iblk) character(len=*), parameter :: subname = '(prep_radiation)' - call ice_timer_start(timer_sw) ! shortwave + call ice_timer_start(timer_sw,iblk) ! shortwave alvdr_init(:,:,iblk) = c0 alvdf_init(:,:,iblk) = c0 @@ -146,7 +146,7 @@ subroutine prep_radiation (iblk) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & file=__FILE__, line=__LINE__) - call ice_timer_stop(timer_sw) ! shortwave + call ice_timer_stop(timer_sw,iblk) ! shortwave end subroutine prep_radiation @@ -716,7 +716,7 @@ subroutine update_state (dt, daidt, dvidt, dagedt, offset) use ice_state, only: aicen, trcrn, vicen, vsnon, & aice, trcr, vice, vsno, aice0, trcr_depend, & bound_state, trcr_base, nt_strata, n_trcr_strata - use ice_timers, only: ice_timer_start, ice_timer_stop, timer_bound + use ice_timers, only: ice_timer_start, ice_timer_stop, timer_bound, timer_updstate real (kind=dbl_kind), intent(in) :: & dt ! time step @@ -740,6 +740,7 @@ subroutine update_state (dt, daidt, dvidt, dagedt, offset) character(len=*), parameter :: subname='(update_state)' + call ice_timer_start(timer_updstate) call icepack_query_tracer_flags(tr_iage_out=tr_iage) call icepack_query_tracer_sizes(ntrcr_out=ntrcr) call icepack_query_tracer_indices(nt_iage_out=nt_iage) @@ -757,7 +758,7 @@ subroutine update_state (dt, daidt, dvidt, dagedt, offset) ntrcr, trcrn) call ice_timer_stop(timer_bound) - !$OMP PARALLEL DO PRIVATE(iblk,i,j) + !$OMP PARALLEL DO PRIVATE(iblk,i,j) SCHEDULE(runtime) do iblk = 1, nblocks do j = 1, ny_block do i = 1, nx_block @@ -811,6 +812,7 @@ subroutine update_state (dt, daidt, dvidt, dagedt, offset) call icepack_warnings_flush(nu_diag) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & file=__FILE__, line=__LINE__) + call ice_timer_stop(timer_updstate) end subroutine update_state @@ -986,8 +988,8 @@ subroutine step_dyn_ridge (dt, ndtd, iblk) ! Ridging !----------------------------------------------------------------- - call ice_timer_start(timer_column) - call ice_timer_start(timer_ridge) + call ice_timer_start(timer_column,iblk) + call ice_timer_start(timer_ridge,iblk) call icepack_query_tracer_sizes(ntrcr_out=ntrcr, nbtrcr_out=nbtrcr) call icepack_warnings_flush(nu_diag) @@ -1056,8 +1058,8 @@ subroutine step_dyn_ridge (dt, ndtd, iblk) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & file=__FILE__, line=__LINE__) - call ice_timer_stop(timer_ridge) - call ice_timer_stop(timer_column) + call ice_timer_stop(timer_ridge,iblk) + call ice_timer_stop(timer_column,iblk) end subroutine step_dyn_ridge @@ -1244,7 +1246,7 @@ subroutine step_radiation (dt, iblk) character(len=*), parameter :: subname = '(step_radiation)' - call ice_timer_start(timer_sw) ! shortwave + call ice_timer_start(timer_sw,iblk) ! shortwave call icepack_query_tracer_sizes(ntrcr_out=ntrcr, & nbtrcr_out=nbtrcr, nbtrcr_sw_out=nbtrcr_sw) @@ -1363,7 +1365,7 @@ subroutine step_radiation (dt, iblk) deallocate(ztrcr_sw) deallocate(rsnow) - call ice_timer_stop(timer_sw) ! shortwave + call ice_timer_stop(timer_sw,iblk) ! shortwave end subroutine step_radiation @@ -1591,7 +1593,7 @@ subroutine biogeochemistry (dt, iblk) if (tr_brine .or. skl_bgc) then - call ice_timer_start(timer_bgc) ! biogeochemistry + call ice_timer_start(timer_bgc,iblk) ! biogeochemistry this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -1684,7 +1686,7 @@ subroutine biogeochemistry (dt, iblk) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & file=__FILE__, line=__LINE__) - call ice_timer_stop(timer_bgc) ! biogeochemistry + call ice_timer_stop(timer_bgc,iblk) ! biogeochemistry endif ! tr_brine .or. skl_bgc diff --git a/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90 b/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90 index 046cf9336..51c1dbff5 100644 --- a/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90 +++ b/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90 @@ -62,8 +62,17 @@ module ice_timers timer_bgc, &! biogeochemistry timer_forcing, &! forcing timer_evp_1d, &! timer only loop - timer_evp_2d ! timer including conversion 1d/2d -! timer_tmp ! for temporary timings + timer_evp_2d, &! timer including conversion 1d/2d + timer_updstate, &! update state + timer_tmp1, &! for temporary timings + timer_tmp2, &! for temporary timings + timer_tmp3, &! for temporary timings + timer_tmp4, &! for temporary timings + timer_tmp5, &! for temporary timings + timer_tmp6, &! for temporary timings + timer_tmp7, &! for temporary timings + timer_tmp8, &! for temporary timings + timer_tmp9 ! for temporary timings !----------------------------------------------------------------------- ! @@ -173,7 +182,7 @@ subroutine init_ice_timers ! call get_ice_timer(timer_ponds, 'Meltponds',nblocks,distrb_info%nprocs) call get_ice_timer(timer_ridge, 'Ridging', nblocks,distrb_info%nprocs) ! call get_ice_timer(timer_catconv, 'Cat Conv', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_fsd, 'Floe size',nblocks,distrb_info%nprocs) + call get_ice_timer(timer_fsd, 'FloeSize', nblocks,distrb_info%nprocs) call get_ice_timer(timer_couple, 'Coupling', nblocks,distrb_info%nprocs) call get_ice_timer(timer_readwrite,'ReadWrite',nblocks,distrb_info%nprocs) call get_ice_timer(timer_diags, 'Diags ',nblocks,distrb_info%nprocs) @@ -187,9 +196,18 @@ subroutine init_ice_timers call get_ice_timer(timer_cplsend, 'Cpl-Send', nblocks,distrb_info%nprocs) call get_ice_timer(timer_sndrcv, 'Snd->Rcv', nblocks,distrb_info%nprocs) #endif - call get_ice_timer(timer_evp_1d, '1d-evp', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_evp_2d, '2d-evp', nblocks,distrb_info%nprocs) -! call get_ice_timer(timer_tmp, ' ',nblocks,distrb_info%nprocs) + call get_ice_timer(timer_evp_1d, '1d-evp', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_evp_2d, '2d-evp', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_updstate, 'UpdState', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp1, 'tmp1', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp2, 'tmp2', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp3, 'tmp3', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp4, 'tmp4', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp5, 'tmp5', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp6, 'tmp6', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp7, 'tmp7', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp8, 'tmp8', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp9, 'tmp9', nblocks,distrb_info%nprocs) !----------------------------------------------------------------------- @@ -333,6 +351,7 @@ subroutine ice_timer_start(timer_id, block_id) character(len=*), parameter :: subname = '(ice_timer_start)' +! if (my_task == master_task) write(nu_diag,*) subname,trim(all_timers(timer_id)%name) !----------------------------------------------------------------------- ! ! if timer is defined, start it up @@ -433,6 +452,7 @@ subroutine ice_timer_stop(timer_id, block_id) character(len=*), parameter :: subname = '(ice_timer_stop)' +! if (my_task == master_task) write(nu_diag,*) subname,trim(all_timers(timer_id)%name) !----------------------------------------------------------------------- ! ! get end cycles diff --git a/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90 b/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90 index 4599de42e..7ae981e2c 100644 --- a/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90 +++ b/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90 @@ -54,8 +54,17 @@ module ice_timers timer_bgc, &! biogeochemistry timer_forcing, &! forcing timer_evp_1d, &! timer only loop - timer_evp_2d ! timer including conversion 1d/2d -! timer_tmp ! for temporary timings + timer_evp_2d, &! timer including conversion 1d/2d + timer_updstate, &! update state + timer_tmp1, &! for temporary timings + timer_tmp2, &! for temporary timings + timer_tmp3, &! for temporary timings + timer_tmp4, &! for temporary timings + timer_tmp5, &! for temporary timings + timer_tmp6, &! for temporary timings + timer_tmp7, &! for temporary timings + timer_tmp8, &! for temporary timings + timer_tmp9 ! for temporary timings !----------------------------------------------------------------------- ! @@ -187,7 +196,7 @@ subroutine init_ice_timers ! call get_ice_timer(timer_ponds, 'Meltponds',nblocks,distrb_info%nprocs) call get_ice_timer(timer_ridge, 'Ridging', nblocks,distrb_info%nprocs) ! call get_ice_timer(timer_catconv, 'Cat Conv', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_fsd, 'Floe size',nblocks,distrb_info%nprocs) + call get_ice_timer(timer_fsd, 'FloeSize', nblocks,distrb_info%nprocs) call get_ice_timer(timer_couple, 'Coupling', nblocks,distrb_info%nprocs) call get_ice_timer(timer_readwrite,'ReadWrite',nblocks,distrb_info%nprocs) call get_ice_timer(timer_diags, 'Diags ',nblocks,distrb_info%nprocs) @@ -197,7 +206,16 @@ subroutine init_ice_timers call get_ice_timer(timer_forcing, 'Forcing', nblocks,distrb_info%nprocs) call get_ice_timer(timer_evp_1d, '1d-evp', nblocks,distrb_info%nprocs) call get_ice_timer(timer_evp_2d, '2d-evp', nblocks,distrb_info%nprocs) -! call get_ice_timer(timer_tmp, ' ',nblocks,distrb_info%nprocs) + call get_ice_timer(timer_updstate, 'UpdState', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp1, 'tmp1', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp2, 'tmp2', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp3, 'tmp3', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp4, 'tmp4', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp5, 'tmp5', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp6, 'tmp6', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp7, 'tmp7', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp8, 'tmp8', nblocks,distrb_info%nprocs) + call get_ice_timer(timer_tmp9, 'tmp9', nblocks,distrb_info%nprocs) !----------------------------------------------------------------------- @@ -341,6 +359,8 @@ subroutine ice_timer_start(timer_id, block_id) character(len=*), parameter :: subname = '(ice_timer_start)' +! if (my_task == master_task) write(nu_diag,*) subname,trim(all_timers(timer_id)%name) + !----------------------------------------------------------------------- ! ! if timer is defined, start it up @@ -444,6 +464,8 @@ subroutine ice_timer_stop(timer_id, block_id) character(len=*), parameter :: subname = '(ice_timer_stop)' +! if (my_task == master_task) write(nu_diag,*) subname,trim(all_timers(timer_id)%name) + !----------------------------------------------------------------------- ! ! get end cycles diff --git a/cicecore/drivers/nuopc/dmi/CICE_RunMod.F90 b/cicecore/drivers/nuopc/dmi/CICE_RunMod.F90 index 2d3e22973..33b9a165c 100644 --- a/cicecore/drivers/nuopc/dmi/CICE_RunMod.F90 +++ b/cicecore/drivers/nuopc/dmi/CICE_RunMod.F90 @@ -496,7 +496,7 @@ subroutine coupling_prep (iblk) enddo enddo - call ice_timer_start(timer_couple) ! atm/ocn coupling + call ice_timer_start(timer_couple,iblk) ! atm/ocn coupling if (oceanmixed_ice) & call ocean_mixed_layer (dt,iblk) ! ocean surface fluxes and sst @@ -663,7 +663,7 @@ subroutine coupling_prep (iblk) endif !echmod #endif - call ice_timer_stop(timer_couple) ! atm/ocn coupling + call ice_timer_stop(timer_couple,iblk) ! atm/ocn coupling end subroutine coupling_prep diff --git a/cicecore/drivers/standalone/cice/CICE_RunMod.F90 b/cicecore/drivers/standalone/cice/CICE_RunMod.F90 index 0fde18e04..3557177fc 100644 --- a/cicecore/drivers/standalone/cice/CICE_RunMod.F90 +++ b/cicecore/drivers/standalone/cice/CICE_RunMod.F90 @@ -15,8 +15,9 @@ module CICE_RunMod use ice_kinds_mod - use ice_fileunits, only: nu_diag + use ice_fileunits, only: nu_diag, flush_fileunit use ice_arrays_column, only: oceanmixed_ice + use ice_communicate, only: my_task, master_task, ice_barrier use ice_constants, only: c0, c1 use ice_constants, only: field_loc_center, field_type_scalar use ice_exit, only: abort_ice @@ -160,7 +161,11 @@ subroutine ice_step biogeochemistry, save_init, step_dyn_wave, step_snow use ice_timers, only: ice_timer_start, ice_timer_stop, & timer_diags, timer_column, timer_thermo, timer_bound, & - timer_hist, timer_readwrite + timer_hist, timer_readwrite, & + timer_tmp1, timer_tmp2, timer_tmp3, timer_tmp4, timer_tmp5, timer_tmp6 +#ifdef OMP_TIMERS + use OMP_LIB +#endif integer (kind=int_kind) :: & iblk , & ! block index @@ -170,6 +175,13 @@ subroutine ice_step real (kind=dbl_kind) :: & offset ! d(age)/dt time offset +#ifdef OMP_TIMERS + real (kind=dbl_kind) :: & + dtimes(nblocks,9), dtimee(nblocks,9) + integer(kind=omp_sched_kind) :: ompsk + integer :: ompcs, itim +#endif + logical (kind=log_kind) :: & tr_iage, tr_FY, tr_lvl, tr_fsd, tr_snow, & tr_pond_cesm, tr_pond_lvl, tr_pond_topo, tr_brine, tr_iso, tr_aero, & @@ -213,21 +225,46 @@ subroutine ice_step call init_history_bgc call ice_timer_stop(timer_diags) ! diagnostics/history +#ifdef OMP_TIMERS + call ice_barrier() +#endif call ice_timer_start(timer_column) ! column physics call ice_timer_start(timer_thermo) ! thermodynamics call save_init - !$OMP PARALLEL DO PRIVATE(iblk) +#ifdef OMP_TIMERS + dtimes = 0. + dtimee = 0. + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks + if (my_task == master_task) then + call omp_get_schedule(ompsk,ompcs) + if (iblk == 1) write(nu_diag,*) subname,'tcx omp3',ompsk,ompcs + write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk + call flush_fileunit(nu_diag) + endif + enddo + !$OMP END PARALLEL DO +#endif - if (ktherm >= 0) then + if (ktherm >= 0) then + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) + do iblk = 1, nblocks !----------------------------------------------------------------- ! scale radiation fields !----------------------------------------------------------------- +#ifdef OMP_TIMERS + dtimes(iblk,1) = OMP_GET_WTIME() + call ice_timer_start(timer_tmp1,iblk) +#endif if (calc_Tsfc) call prep_radiation (iblk) +#ifdef OMP_TIMERS + call ice_timer_stop(timer_tmp1,iblk) + dtimee(iblk,1) = OMP_GET_WTIME() +#endif if (debug_model) then plabeld = 'post prep_radiation' @@ -237,32 +274,56 @@ subroutine ice_step !----------------------------------------------------------------- ! thermodynamics and biogeochemistry !----------------------------------------------------------------- - + +#ifdef OMP_TIMERS + dtimes(iblk,2) = OMP_GET_WTIME() + call ice_timer_start(timer_tmp2,iblk) +#endif call step_therm1 (dt, iblk) ! vertical thermodynamics +#ifdef OMP_TIMERS + call ice_timer_stop(timer_tmp2,iblk) + dtimee(iblk,2) = OMP_GET_WTIME() +#endif if (debug_model) then plabeld = 'post step_therm1' call debug_ice (iblk, plabeld) endif +#ifdef OMP_TIMERS + dtimes(iblk,3) = OMP_GET_WTIME() + call ice_timer_start(timer_tmp3,iblk) +#endif call biogeochemistry (dt, iblk) ! biogeochemistry +#ifdef OMP_TIMERS + call ice_timer_stop(timer_tmp3,iblk) + dtimee(iblk,3) = OMP_GET_WTIME() +#endif if (debug_model) then plabeld = 'post biogeochemistry' call debug_ice (iblk, plabeld) endif +#ifdef OMP_TIMERS + dtimes(iblk,4) = OMP_GET_WTIME() + call ice_timer_start(timer_tmp4,iblk) +#endif call step_therm2 (dt, iblk) ! ice thickness distribution thermo +#ifdef OMP_TIMERS + call ice_timer_stop(timer_tmp4,iblk) + dtimee(iblk,4) = OMP_GET_WTIME() +#endif if (debug_model) then plabeld = 'post step_therm2' call debug_ice (iblk, plabeld) endif - endif ! ktherm > 0 + enddo + !$OMP END PARALLEL DO - enddo ! iblk - !$OMP END PARALLEL DO + endif ! ktherm > 0 ! clean up, update tendency diagnostics offset = dt @@ -271,6 +332,10 @@ subroutine ice_step call ice_timer_stop(timer_thermo) ! thermodynamics call ice_timer_stop(timer_column) ! column physics +#ifdef OMP_TIMERS + call ice_barrier() +#endif + !----------------------------------------------------------------- ! dynamics, transport, ridging !----------------------------------------------------------------- @@ -292,7 +357,7 @@ subroutine ice_step endif ! ridging - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks if (kridge > 0) call step_dyn_ridge (dt_dyn, ndtd, iblk) enddo @@ -318,6 +383,9 @@ subroutine ice_step enddo endif +#ifdef OMP_TIMERS + call ice_barrier() +#endif call ice_timer_start(timer_column) ! column physics call ice_timer_start(timer_thermo) ! thermodynamics @@ -326,21 +394,30 @@ subroutine ice_step !----------------------------------------------------------------- if (tr_snow) then ! advanced snow physics + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks call step_snow (dt, iblk) enddo + !$OMP END PARALLEL DO call update_state (dt) ! clean up endif -!MHRI: CHECK THIS OMP - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks !----------------------------------------------------------------- ! albedo, shortwave radiation !----------------------------------------------------------------- +#ifdef OMP_TIMERS + dtimes(iblk,5) = OMP_GET_WTIME() + call ice_timer_start(timer_tmp5,iblk) +#endif if (ktherm >= 0) call step_radiation (dt, iblk) +#ifdef OMP_TIMERS + call ice_timer_stop(timer_tmp5,iblk) + dtimee(iblk,5) = OMP_GET_WTIME() +#endif if (debug_model) then plabeld = 'post step_radiation' @@ -351,14 +428,22 @@ subroutine ice_step ! get ready for coupling and the next time step !----------------------------------------------------------------- +#ifdef OMP_TIMERS + dtimes(iblk,6) = OMP_GET_WTIME() + call ice_timer_start(timer_tmp6,iblk) +#endif call coupling_prep (iblk) +#ifdef OMP_TIMERS + call ice_timer_stop(timer_tmp6,iblk) + dtimee(iblk,6) = OMP_GET_WTIME() +#endif if (debug_model) then plabeld = 'post coupling_prep' call debug_ice (iblk, plabeld) endif - enddo ! iblk + enddo !$OMP END PARALLEL DO call ice_timer_start(timer_bound) @@ -369,6 +454,15 @@ subroutine ice_step call ice_timer_stop(timer_thermo) ! thermodynamics call ice_timer_stop(timer_column) ! column physics +#ifdef OMP_TIMERS + do itim = 1,6 + do iblk = 1,nblocks + write(nu_diag,'(2a,2i6,2g15.7)') subname,' omptime',itim,iblk,dtimes(iblk,itim),dtimee(iblk,itim)-dtimes(iblk,itim) + enddo + enddo + call ice_barrier() +#endif + !----------------------------------------------------------------- ! write data !----------------------------------------------------------------- @@ -386,6 +480,7 @@ subroutine ice_step call accum_hist (dt) ! history file call ice_timer_stop(timer_hist) ! history + call ice_barrier() call ice_timer_start(timer_readwrite) ! reading/writing if (write_restart == 1) then call dumpfile ! core variables for restarting @@ -405,7 +500,6 @@ subroutine ice_step if (kdyn == 2) call write_restart_eap call final_restart endif - call ice_timer_stop(timer_readwrite) ! reading/writing end subroutine ice_step @@ -488,7 +582,7 @@ subroutine coupling_prep (iblk) enddo enddo - call ice_timer_start(timer_couple) ! atm/ocn coupling + call ice_timer_start(timer_couple,iblk) ! atm/ocn coupling if (oceanmixed_ice) & call ocean_mixed_layer (dt,iblk) ! ocean surface fluxes and sst @@ -655,7 +749,7 @@ subroutine coupling_prep (iblk) endif !echmod #endif - call ice_timer_stop(timer_couple) ! atm/ocn coupling + call ice_timer_stop(timer_couple,iblk) ! atm/ocn coupling end subroutine coupling_prep diff --git a/configuration/scripts/cice.run.setup.csh b/configuration/scripts/cice.run.setup.csh index aa578b5ca..58c4ebe66 100755 --- a/configuration/scripts/cice.run.setup.csh +++ b/configuration/scripts/cice.run.setup.csh @@ -9,8 +9,6 @@ echo "running cice.run.setup.csh" set jobfile = cice.run set subfile = cice.submit -set nthrds = ${ICE_NTHRDS} - #========================================== # Write the batch code into the job file @@ -43,7 +41,9 @@ set ICE_RUNLOG_FILE = "cice.runlog.\${stamp}" #-------------------------------------------- cd \${ICE_RUNDIR} -setenv OMP_NUM_THREADS ${nthrds} +setenv OMP_NUM_THREADS \${ICE_NTHRDS} +setenv OMP_SCHEDULE "\${ICE_OMPSCHED}" +#setenv OMP_DISPLAY_ENV TRUE cp -f \${ICE_CASEDIR}/ice_in \${ICE_RUNDIR} cp -f \${ICE_CASEDIR}/env.${ICE_MACHCOMP} \${ICE_RUNDIR} diff --git a/configuration/scripts/cice.settings b/configuration/scripts/cice.settings index 1faf2c5be..8c9f6cbeb 100755 --- a/configuration/scripts/cice.settings +++ b/configuration/scripts/cice.settings @@ -21,6 +21,7 @@ setenv ICE_QUIETMODE false setenv ICE_GRID undefined setenv ICE_NTASKS undefined setenv ICE_NTHRDS undefined +setenv ICE_OMPSCHED "static,1" setenv ICE_TEST undefined setenv ICE_TESTNAME undefined setenv ICE_TESTID undefined diff --git a/configuration/scripts/options/set_env.ompschedd1 b/configuration/scripts/options/set_env.ompschedd1 new file mode 100644 index 000000000..a4d255f48 --- /dev/null +++ b/configuration/scripts/options/set_env.ompschedd1 @@ -0,0 +1 @@ +setenv ICE_OMPSCHED "dynamic,1" diff --git a/configuration/scripts/options/set_env.ompscheds b/configuration/scripts/options/set_env.ompscheds new file mode 100644 index 000000000..b9a4f58b0 --- /dev/null +++ b/configuration/scripts/options/set_env.ompscheds @@ -0,0 +1 @@ +setenv ICE_OMPSCHED "static" diff --git a/configuration/scripts/options/set_env.ompscheds1 b/configuration/scripts/options/set_env.ompscheds1 new file mode 100644 index 000000000..a9ca4874d --- /dev/null +++ b/configuration/scripts/options/set_env.ompscheds1 @@ -0,0 +1 @@ +setenv ICE_OMPSCHED "static,1" diff --git a/configuration/scripts/tests/omp_suite.ts b/configuration/scripts/tests/omp_suite.ts new file mode 100644 index 000000000..893daf88e --- /dev/null +++ b/configuration/scripts/tests/omp_suite.ts @@ -0,0 +1,48 @@ +# Test Grid PEs Sets BFB-compare + +smoke gx3 8x4 diag1 +smoke gx3 6x2 alt01 +smoke gx3 8x2 alt02 +smoke gx3 12x2 alt03 +smoke gx3 4x4 alt04 +smoke gx3 4x4 alt05 +smoke gx3 8x2 alt06 +smoke gx3 8x2 bgcz +smoke gx1 15x2 seabedprob +smoke gx3 14x2 fsd12 +smoke gx3 11x2 isotope +smoke gx3 8x4 snwitdrdg,snwgrain,icdefault +smoke gx3 6x4 dynpicard +smoke gx3 8x3 zsal + +smoke gbox128 8x2 short +smoke gbox128 12x2 boxnodyn,short +smoke gbox128 9x2 boxadv,short +smoke gbox128 14x2 boxrestore,short +smoke gbox80 4x5 box2001 +smoke gbox80 11x3 boxslotcyl + +smoke gx3 4x2 diag1 smoke_gx3_8x2_diag1 +smoke gx3 4x1 diag1,thread smoke_gx3_8x2_diag1 +smoke gx3 8x1 alt01,thread smoke_gx3_6x2_alt01 +smoke gx3 16x1 alt02,thread smoke_gx3_8x2_alt02 +smoke gx3 24x1 alt03,thread smoke_gx3_12x2_alt03 +smoke gx3 24x1 alt04,thread smoke_gx3_4x4_alt04 +smoke gx3 14x1 alt05,thread smoke_gx3_4x4_alt05 +smoke gx3 24x1 alt06,thread smoke_gx3_8x2_alt06 +smoke gx3 12x1 bgcz,thread smoke_gx3_8x2_bgcz +smoke gx1 28x1 seabedprob,thread smoke_gx1_15x2_seabedprob +smoke gx3 30x1 fsd12,thread smoke_gx3_14x2_fsd12 +smoke gx3 16x1 isotope,thread smoke_gx3_11x2_isotope +smoke gx3 28x1 snwitdrdg,snwgrain,icdefault,thread smoke_gx3_8x4_icdefault_snwitdrdg_snwgrain +smoke gx3 18x1 dynpicard,thread smoke_gx3_6x4_dynpicard +smoke gx3 20x1 zsal,thread smoke_gx3_8x3_zsal + +smoke gbox128 20x1 short,thread smoke_gbox128_8x2_short +smoke gbox128 16x1 boxnodyn,short,thread smoke_gbox128_12x2_boxnodyn_short +smoke gbox128 14x1 boxadv,short,thread smoke_gbox128_9x2_boxadv_short +smoke gbox128 24x1 boxrestore,short,thread smoke_gbox128_14x2_boxrestore_short +smoke gbox80 19x1 box2001,thread smoke_gbox80_4x5_box2001 +smoke gbox80 8x4 boxslotcyl,thread smoke_gbox80_11x3_boxslotcyl + + diff --git a/configuration/scripts/tests/perf_suite.ts b/configuration/scripts/tests/perf_suite.ts index 859b9f21b..9a17d8a55 100644 --- a/configuration/scripts/tests/perf_suite.ts +++ b/configuration/scripts/tests/perf_suite.ts @@ -24,4 +24,7 @@ smoke gx1 32x2x16x16x16 run2day,droundrobin smoke_gx1_64x1x16x16x8 smoke gx1 16x4x16x16x32 run2day,droundrobin smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread smoke gx1 8x8x16x16x64 run2day,droundrobin smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread smoke gx1 4x16x16x16x128 run2day,droundrobin smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread +smoke gx1 32x2x16x16x16 run2day,droundrobin,ompscheds smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread +smoke gx1 32x2x16x16x16 run2day,droundrobin,ompschedd1 smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread +smoke gx1 32x2x16x16x16 run2day,droundrobin,ompscheds1 smoke_gx1_64x1x16x16x8_droundrobin_run2day_thread # From 9c442bc33c282e15032fd75324b7ac6dc1f52798 Mon Sep 17 00:00:00 2001 From: apcraig Date: Mon, 10 Jan 2022 11:57:50 -0700 Subject: [PATCH 04/14] - Review diagnostics OMP implementation - Add timer_stats namelist to turn on extra timer output information - Add ICE_BFBTYPE and update bit-for-bit comparison logic in scripts - Update qc and logbfb testing - Remove logbfb and qchkf tests, add cmplog, cmplogrest, cmprest set_env files to set ICE_BFBTYPE - Update documentation --- .../cicedynB/analysis/ice_diagnostics.F90 | 11 ++- cicecore/cicedynB/general/ice_init.F90 | 6 +- .../infrastructure/comm/mpi/ice_timers.F90 | 3 + .../infrastructure/comm/serial/ice_timers.F90 | 3 + cicecore/drivers/mct/cesm1/CICE_RunMod.F90 | 1 - .../drivers/standalone/cice/CICE_FinalMod.F90 | 5 +- configuration/scripts/cice.settings | 1 + configuration/scripts/ice_in | 1 + .../scripts/machines/env.cheyenne_gnu | 4 +- .../scripts/machines/env.cheyenne_intel | 4 +- .../scripts/machines/env.cheyenne_pgi | 4 +- configuration/scripts/options/set_env.cmplog | 1 + .../scripts/options/set_env.cmplogrest | 1 + configuration/scripts/options/set_env.cmprest | 1 + configuration/scripts/options/set_env.qcchk | 1 + configuration/scripts/options/set_env.qcchkf | 1 + configuration/scripts/options/set_nml.dt3456s | 1 + .../scripts/options/set_nml.qcnonbfb | 16 ---- configuration/scripts/tests/baseline.script | 54 ++++++++++-- configuration/scripts/tests/first_suite.ts | 2 +- configuration/scripts/tests/nothread_suite.ts | 14 ++-- configuration/scripts/tests/omp_suite.ts | 84 +++++++++---------- configuration/scripts/tests/prod_suite.ts | 8 +- configuration/scripts/tests/reprosum_suite.ts | 20 ++--- .../scripts/tests/test_logbfb.script | 33 -------- .../scripts/tests/test_qcchkf.script | 36 -------- doc/source/cice_index.rst | 1 + doc/source/user_guide/ug_case_settings.rst | 27 ++++-- doc/source/user_guide/ug_implementation.rst | 29 ++++--- 29 files changed, 181 insertions(+), 192 deletions(-) create mode 100644 configuration/scripts/options/set_env.cmplog create mode 100644 configuration/scripts/options/set_env.cmplogrest create mode 100644 configuration/scripts/options/set_env.cmprest create mode 100644 configuration/scripts/options/set_env.qcchk create mode 100644 configuration/scripts/options/set_env.qcchkf create mode 100644 configuration/scripts/options/set_nml.dt3456s delete mode 100644 configuration/scripts/options/set_nml.qcnonbfb delete mode 100644 configuration/scripts/tests/test_logbfb.script delete mode 100644 configuration/scripts/tests/test_qcchkf.script diff --git a/cicecore/cicedynB/analysis/ice_diagnostics.F90 b/cicecore/cicedynB/analysis/ice_diagnostics.F90 index 23f39634f..ec041483c 100644 --- a/cicecore/cicedynB/analysis/ice_diagnostics.F90 +++ b/cicecore/cicedynB/analysis/ice_diagnostics.F90 @@ -720,7 +720,7 @@ subroutine runtime_diags (dt) fisoon(n) = fisoon(n)*dt fisoos(n) = fisoos(n)*dt - !$OMP PARALLEL DO PRIVATE(iblk,i,j) + !$OMP PARALLEL DO PRIVATE(iblk,i,j,k) do iblk = 1, nblocks do j = 1, ny_block do i = 1, nx_block @@ -1214,7 +1214,7 @@ subroutine init_mass_diags if (tr_iso) then do n=1,n_iso - !$OMP PARALLEL DO PRIVATE(iblk,i,j) + !$OMP PARALLEL DO PRIVATE(iblk,i,j,k) do iblk = 1, nblocks do j = 1, ny_block do i = 1, nx_block @@ -1317,7 +1317,6 @@ subroutine total_energy (work) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & file=__FILE__, line=__LINE__) -! MHRI: CHECK THIS OMP !$OMP PARALLEL DO PRIVATE(iblk,i,j,n,k,ij,icells,indxi,indxj) do iblk = 1, nblocks @@ -1405,7 +1404,6 @@ subroutine total_salt (work) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & file=__FILE__, line=__LINE__) -! MHRI: CHECK THIS OMP !$OMP PARALLEL DO PRIVATE(iblk,i,j,n,k,ij,icells,indxi,indxj) do iblk = 1, nblocks @@ -1513,7 +1511,8 @@ subroutine init_diags if (abs(latpnt(n)) < c360 .and. abs(lonpnt(n)) < c360) then ! MDT, 09/2017: Comment out OpenMP directives since loop is not thread-safe - !!$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,latdis,londis,totdis) + ! This is computing closest point, Could add a CRITICAL but it's just initialization + !!$XXXOMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,latdis,londis,totdis) do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -1538,7 +1537,7 @@ subroutine init_diags enddo ! i enddo ! j enddo ! iblk - !!$OMP END PARALLEL DO + !!$XXXOMP END PARALLEL DO endif diff --git a/cicecore/cicedynB/general/ice_init.F90 b/cicecore/cicedynB/general/ice_init.F90 index e3030ec55..123f51f26 100644 --- a/cicecore/cicedynB/general/ice_init.F90 +++ b/cicecore/cicedynB/general/ice_init.F90 @@ -116,6 +116,7 @@ subroutine input_data damping_andacc, start_andacc, use_mean_vrel, ortho_type use ice_transport_driver, only: advection, conserv_check use ice_restoring, only: restore_ice + use ice_timers, only: timer_stats #ifdef CESMCOUPLED use shr_file_mod, only: shr_file_setIO #endif @@ -174,7 +175,7 @@ subroutine input_data print_global, print_points, latpnt, lonpnt, & debug_forcing, histfreq, histfreq_n, hist_avg, & history_dir, history_file, history_precision, cpl_bgc, & - histfreq_base, dumpfreq_base, & + histfreq_base, dumpfreq_base, timer_stats, & conserv_check, debug_model, debug_model_step, & debug_model_i, debug_model_j, debug_model_iblk, debug_model_task, & year_init, month_init, day_init, sec_init, & @@ -292,6 +293,7 @@ subroutine input_data debug_model_task = -1 ! debug model local task number print_points = .false. ! if true, print point data print_global = .true. ! if true, print global diagnostic data + timer_stats = .false. ! if true, print out detailed timer statistics bfbflag = 'off' ! off = optimized diag_type = 'stdout' diag_file = 'ice_diag.d' @@ -763,6 +765,7 @@ subroutine input_data call broadcast_scalar(debug_model_task, master_task) call broadcast_scalar(print_points, master_task) call broadcast_scalar(print_global, master_task) + call broadcast_scalar(timer_stats, master_task) call broadcast_scalar(bfbflag, master_task) call broadcast_scalar(diag_type, master_task) call broadcast_scalar(diag_file, master_task) @@ -2009,6 +2012,7 @@ subroutine input_data write(nu_diag,1021) ' debug_model_i = ', debug_model_j write(nu_diag,1021) ' debug_model_iblk = ', debug_model_iblk write(nu_diag,1021) ' debug_model_task = ', debug_model_task + write(nu_diag,1011) ' timer_stats = ', timer_stats write(nu_diag,1031) ' bfbflag = ', trim(bfbflag) write(nu_diag,1021) ' numin = ', numin write(nu_diag,1021) ' numax = ', numax diff --git a/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90 b/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90 index 51c1dbff5..cdbe72ef2 100644 --- a/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90 +++ b/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90 @@ -32,6 +32,9 @@ module ice_timers ice_timer_print_all, & ice_timer_check + logical(log_kind), public :: & + timer_stats ! controls printing of timer statistics + !----------------------------------------------------------------------- ! public timers !----------------------------------------------------------------------- diff --git a/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90 b/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90 index 7ae981e2c..41c511ea5 100644 --- a/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90 +++ b/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90 @@ -30,6 +30,9 @@ module ice_timers ice_timer_print_all, & ice_timer_check + logical(log_kind), public :: & + timer_stats ! controls printing of timer statistics + !----------------------------------------------------------------------- ! public timers !----------------------------------------------------------------------- diff --git a/cicecore/drivers/mct/cesm1/CICE_RunMod.F90 b/cicecore/drivers/mct/cesm1/CICE_RunMod.F90 index e9ab0d7e4..610b146a6 100644 --- a/cicecore/drivers/mct/cesm1/CICE_RunMod.F90 +++ b/cicecore/drivers/mct/cesm1/CICE_RunMod.F90 @@ -287,7 +287,6 @@ subroutine ice_step call ice_timer_start(timer_column) ! column physics call ice_timer_start(timer_thermo) ! thermodynamics -!MHRI: CHECK THIS OMP !$OMP PARALLEL DO PRIVATE(iblk) do iblk = 1, nblocks diff --git a/cicecore/drivers/standalone/cice/CICE_FinalMod.F90 b/cicecore/drivers/standalone/cice/CICE_FinalMod.F90 index a59c210aa..28811c3cd 100644 --- a/cicecore/drivers/standalone/cice/CICE_FinalMod.F90 +++ b/cicecore/drivers/standalone/cice/CICE_FinalMod.F90 @@ -31,7 +31,8 @@ module CICE_FinalMod subroutine CICE_Finalize use ice_restart_shared, only: runid - use ice_timers, only: ice_timer_stop, ice_timer_print_all, timer_total + use ice_timers, only: ice_timer_stop, ice_timer_print_all, timer_total, & + timer_stats character(len=*), parameter :: subname = '(CICE_Finalize)' @@ -40,7 +41,7 @@ subroutine CICE_Finalize !------------------------------------------------------------------- call ice_timer_stop(timer_total) ! stop timing entire run - call ice_timer_print_all(stats=.false.) ! print timing information + call ice_timer_print_all(stats=timer_stats) ! print timing information call icepack_warnings_flush(nu_diag) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & diff --git a/configuration/scripts/cice.settings b/configuration/scripts/cice.settings index 8c9f6cbeb..9b57aab3f 100755 --- a/configuration/scripts/cice.settings +++ b/configuration/scripts/cice.settings @@ -29,6 +29,7 @@ setenv ICE_BASELINE undefined setenv ICE_BASEGEN undefined setenv ICE_BASECOM undefined setenv ICE_BFBCOMP undefined +setenv ICE_BFBTYPE restart setenv ICE_SPVAL undefined setenv ICE_RUNLENGTH -1 setenv ICE_ACCOUNT undefined diff --git a/configuration/scripts/ice_in b/configuration/scripts/ice_in index bb44663eb..7ed806edf 100644 --- a/configuration/scripts/ice_in +++ b/configuration/scripts/ice_in @@ -38,6 +38,7 @@ debug_forcing = .false. print_global = .true. print_points = .true. + timer_stats = .false. conserv_check = .false. latpnt(1) = 90. lonpnt(1) = 0. diff --git a/configuration/scripts/machines/env.cheyenne_gnu b/configuration/scripts/machines/env.cheyenne_gnu index 8ddc443b1..e731b8634 100755 --- a/configuration/scripts/machines/env.cheyenne_gnu +++ b/configuration/scripts/machines/env.cheyenne_gnu @@ -29,8 +29,8 @@ if ($ICE_IOTYPE =~ pio*) then endif endif -if ($?ICE_TEST) then -if ($ICE_TEST =~ qcchk*) then +if ($?ICE_BFBTYPE) then +if ($ICE_BFBTYPE =~ qcchk*) then module load python source /glade/u/apps/opt/ncar_pylib/ncar_pylib.csh default endif diff --git a/configuration/scripts/machines/env.cheyenne_intel b/configuration/scripts/machines/env.cheyenne_intel index 28df6647d..c8cc06523 100755 --- a/configuration/scripts/machines/env.cheyenne_intel +++ b/configuration/scripts/machines/env.cheyenne_intel @@ -29,8 +29,8 @@ if ($ICE_IOTYPE =~ pio*) then endif endif -if ($?ICE_TEST) then -if ($ICE_TEST =~ qcchk*) then +if ($?ICE_BFBTYPE) then +if ($ICE_BFBTYPE =~ qcchk*) then module load python source /glade/u/apps/opt/ncar_pylib/ncar_pylib.csh default endif diff --git a/configuration/scripts/machines/env.cheyenne_pgi b/configuration/scripts/machines/env.cheyenne_pgi index d492129fb..1510d64ad 100755 --- a/configuration/scripts/machines/env.cheyenne_pgi +++ b/configuration/scripts/machines/env.cheyenne_pgi @@ -29,8 +29,8 @@ if ($ICE_IOTYPE =~ pio*) then endif endif -if ($?ICE_TEST) then -if ($ICE_TEST =~ qcchk*) then +if ($?ICE_BFBTYPE) then +if ($ICE_BFBTYPE =~ qcchk*) then module load python source /glade/u/apps/opt/ncar_pylib/ncar_pylib.csh default endif diff --git a/configuration/scripts/options/set_env.cmplog b/configuration/scripts/options/set_env.cmplog new file mode 100644 index 000000000..b59c1cb6d --- /dev/null +++ b/configuration/scripts/options/set_env.cmplog @@ -0,0 +1 @@ +setenv ICE_BFBTYPE log diff --git a/configuration/scripts/options/set_env.cmplogrest b/configuration/scripts/options/set_env.cmplogrest new file mode 100644 index 000000000..118986199 --- /dev/null +++ b/configuration/scripts/options/set_env.cmplogrest @@ -0,0 +1 @@ +setenv ICE_BFBTYPE logrest diff --git a/configuration/scripts/options/set_env.cmprest b/configuration/scripts/options/set_env.cmprest new file mode 100644 index 000000000..7258fa058 --- /dev/null +++ b/configuration/scripts/options/set_env.cmprest @@ -0,0 +1 @@ +setenv ICE_BFBTYPE restart diff --git a/configuration/scripts/options/set_env.qcchk b/configuration/scripts/options/set_env.qcchk new file mode 100644 index 000000000..9b9fbbd2e --- /dev/null +++ b/configuration/scripts/options/set_env.qcchk @@ -0,0 +1 @@ +setenv ICE_BFBTYPE qcchk diff --git a/configuration/scripts/options/set_env.qcchkf b/configuration/scripts/options/set_env.qcchkf new file mode 100644 index 000000000..589e60772 --- /dev/null +++ b/configuration/scripts/options/set_env.qcchkf @@ -0,0 +1 @@ +setenv ICE_BFBTYPE qcchkf diff --git a/configuration/scripts/options/set_nml.dt3456s b/configuration/scripts/options/set_nml.dt3456s new file mode 100644 index 000000000..74e5482d7 --- /dev/null +++ b/configuration/scripts/options/set_nml.dt3456s @@ -0,0 +1 @@ +dt = 3456.0 diff --git a/configuration/scripts/options/set_nml.qcnonbfb b/configuration/scripts/options/set_nml.qcnonbfb deleted file mode 100644 index a965b863c..000000000 --- a/configuration/scripts/options/set_nml.qcnonbfb +++ /dev/null @@ -1,16 +0,0 @@ -dt = 3456.0 -npt_unit = 'y' -npt = 5 -year_init = 2005 -month_init = 1 -day_init = 1 -sec_init = 0 -use_leap_years = .false. -fyear_init = 2005 -ycycle = 1 -dumpfreq = 'm' -dumpfreq_n = 12 -diagfreq = 24 -histfreq = 'd','x','x','x','x' -f_hi = 'd' -hist_avg = .false. diff --git a/configuration/scripts/tests/baseline.script b/configuration/scripts/tests/baseline.script index ac69d49a0..bb8f50a1f 100644 --- a/configuration/scripts/tests/baseline.script +++ b/configuration/scripts/tests/baseline.script @@ -36,7 +36,7 @@ if (${ICE_BASECOM} != ${ICE_SPVAL}) then ${ICE_CASEDIR}/casescripts/comparelog.csh ${base_file} ${test_file} notcicefile set bfbstatus = $status - else if (${ICE_TEST} =~ qcchk*) then + else if (${ICE_BFBTYPE} =~ qcchk*) then set test_dir = ${ICE_RUNDIR} set base_dir = ${ICE_BASELINE}/${ICE_BASECOM}/${ICE_TESTNAME} ${ICE_SANDBOX}/configuration/scripts/tests/QC/cice.t-test.py ${base_dir} ${test_dir} @@ -151,7 +151,7 @@ if (${ICE_BFBCOMP} != ${ICE_SPVAL}) then endif endif - if (${ICE_TEST} == "logbfb") then + if (${ICE_BFBTYPE} == "log") then set test_file = `ls -1t ${ICE_RUNDIR}/cice.runlog* | head -1` set base_file = `ls -1t ${ICE_RUNDIR}/../${ICE_BFBCOMP}.${ICE_TESTID}/cice.runlog* | head -1` @@ -163,21 +163,61 @@ if (${ICE_BFBCOMP} != ${ICE_SPVAL}) then ${ICE_CASEDIR}/casescripts/comparelog.csh ${base_file} ${test_file} set bfbstatus = $status - else if (${ICE_TEST} =~ qcchk*) then + else if (${ICE_BFBTYPE} == "logrest") then + set test_file = `ls -1t ${ICE_RUNDIR}/cice.runlog* | head -1` + set base_file = `ls -1t ${ICE_RUNDIR}/../${ICE_BFBCOMP}.${ICE_TESTID}/cice.runlog* | head -1` + + echo "" + echo "bfb Log Compare Mode:" + echo "base_file: ${base_file}" + echo "test_file: ${test_file}" + + ${ICE_CASEDIR}/casescripts/comparelog.csh ${base_file} ${test_file} + set bfbstatusl = $status + + set test_dir = ${ICE_RUNDIR}/restart + set base_dir = ${ICE_RUNDIR}/../${ICE_BFBCOMP}.${ICE_TESTID}/restart + + echo "" + echo "bfb Restart Compare Mode:" + echo "base_dir: ${base_dir}" + echo "test_dir: ${test_dir}" + + ${ICE_CASEDIR}/casescripts/comparebfb.csh ${base_dir} ${test_dir} + set bfbstatusr = $status + + if ({$bfbstatusl} == ${bfbstatusr}) then + set bfbstatus = ${bfbstatusl} + else if (${bfbstatusl} == 1 || ${bfbstatusr} == 1) then + set bfbstatus = 1 + else if ({$bfbstatusl} > ${bfbstatusr}) then + set bfbstatus = ${bfbstatusl} + else + set bfbstatus = ${bfbstatusr} + endif + + echo "bfb log, rest, combined status = ${bfbstatusl},${bfbstatusr},${bfbstatus}" + + else if (${ICE_BFBTYPE} =~ qcchk*) then set test_dir = ${ICE_RUNDIR} set base_dir = ${ICE_RUNDIR}/../${ICE_BFBCOMP}.${ICE_TESTID} + echo "" + echo "qcchk Compare Mode:" + echo "base_dir: ${base_dir}" + echo "test_dir: ${test_dir}" ${ICE_SANDBOX}/configuration/scripts/tests/QC/cice.t-test.py ${base_dir} ${test_dir} set bfbstatus = $status # expecting failure, so switch value - if (${ICE_TEST} =~ qcchkf*) then + if (${ICE_BFBTYPE} == "qcchkf") then @ bfbstatus = 1 - $bfbstatus endif + else set test_dir = ${ICE_RUNDIR}/restart set base_dir = ${ICE_RUNDIR}/../${ICE_BFBCOMP}.${ICE_TESTID}/restart echo "" - echo "bfb Compare Mode:" + echo "bfb Restart Compare Mode:" echo "base_dir: ${base_dir}" echo "test_dir: ${test_dir}" @@ -190,10 +230,10 @@ if (${ICE_BFBCOMP} != ${ICE_SPVAL}) then rm -f ${ICE_CASEDIR}/test_output.prev if (${bfbstatus} == 0) then echo "PASS ${ICE_TESTNAME} bfbcomp ${ICE_BFBCOMP}" >> ${ICE_CASEDIR}/test_output - echo "bfb baseline and test dataset are identical" + echo "bfbcomp baseline and test dataset pass" else if (${bfbstatus} == 1) then echo "FAIL ${ICE_TESTNAME} bfbcomp ${ICE_BFBCOMP} different-data" >> ${ICE_CASEDIR}/test_output - echo "bfbcomp and test dataset are different" + echo "bfbcomp baseline and test dataset fail" else if (${bfbstatus} == 2) then echo "MISS ${ICE_TESTNAME} bfbcomp ${ICE_BFBCOMP} missing-data" >> ${ICE_CASEDIR}/test_output echo "Missing data" diff --git a/configuration/scripts/tests/first_suite.ts b/configuration/scripts/tests/first_suite.ts index 31eba9fb7..b42d917ea 100644 --- a/configuration/scripts/tests/first_suite.ts +++ b/configuration/scripts/tests/first_suite.ts @@ -2,5 +2,5 @@ smoke gx3 8x2 diag1,run5day restart gx3 4x2x25x29x4 dslenderX2 smoke gx3 4x2x25x29x4 debug,run2day,dslenderX2 -logbfb gx3 4x2x25x29x4 dslenderX2,diag1,reprosum +smoke gx3 4x2x25x29x4 dslenderX2,diag1,reprosum,cmplog smoke gx3 1x2 run2day diff --git a/configuration/scripts/tests/nothread_suite.ts b/configuration/scripts/tests/nothread_suite.ts index 616741aa2..12fd03662 100644 --- a/configuration/scripts/tests/nothread_suite.ts +++ b/configuration/scripts/tests/nothread_suite.ts @@ -1,7 +1,7 @@ # Test Grid PEs Sets BFB-compare restart gx3 8x1x25x29x2 dslenderX2 -logbfb gx3 8x1x25x29x2 dslenderX2,diag1,reprosum +smoke gx3 8x1x25x29x2 dslenderX2,diag1,reprosum smoke gx3 16x1 diag1,run5day smoke gx3 1x1 debug,diag1,run2day @@ -70,9 +70,9 @@ restart gx3 32x1x5x10x12 drakeX2 restart_gx3_8x1x25x29x2_ restart gx3 16x1x8x10x10 droundrobin,maskhalo restart_gx3_8x1x25x29x2_dslenderX2 restart gx3 4x1x25x29x4 droundrobin restart_gx3_8x1x25x29x2_dslenderX2 -logbfb gx3 1x1x50x58x4 droundrobin,diag1,maskhalo,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum -logbfb gx3 4x1x25x116x1 dslenderX1,diag1,maskhalo,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum -logbfb gx3 20x1x5x29x20 dsectrobin,diag1,short,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum -logbfb gx3 16x1x8x10x10 droundrobin,diag1,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum -logbfb gx3 6x1x50x58x1 droundrobin,diag1,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum -logbfb gx3 12x1x4x29x9 dspacecurve,diag1,maskhalo,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +smoke gx3 1x1x50x58x4 droundrobin,diag1,maskhalo,reprosum,cmplog smoke_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +smoke gx3 4x1x25x116x1 dslenderX1,diag1,maskhalo,reprosum,cmplog smoke_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +smoke gx3 20x1x5x29x20 dsectrobin,diag1,short,reprosum,cmplog smoke_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +smoke gx3 16x1x8x10x10 droundrobin,diag1,reprosum,cmplog smoke_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +smoke gx3 6x1x50x58x1 droundrobin,diag1,reprosum,cmplog smoke_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +smoke gx3 12x1x4x29x9 dspacecurve,diag1,maskhalo,reprosum,cmplog smoke_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum diff --git a/configuration/scripts/tests/omp_suite.ts b/configuration/scripts/tests/omp_suite.ts index 893daf88e..9202b06e5 100644 --- a/configuration/scripts/tests/omp_suite.ts +++ b/configuration/scripts/tests/omp_suite.ts @@ -1,48 +1,46 @@ # Test Grid PEs Sets BFB-compare -smoke gx3 8x4 diag1 -smoke gx3 6x2 alt01 -smoke gx3 8x2 alt02 -smoke gx3 12x2 alt03 -smoke gx3 4x4 alt04 -smoke gx3 4x4 alt05 -smoke gx3 8x2 alt06 -smoke gx3 8x2 bgcz -smoke gx1 15x2 seabedprob -smoke gx3 14x2 fsd12 -smoke gx3 11x2 isotope -smoke gx3 8x4 snwitdrdg,snwgrain,icdefault -smoke gx3 6x4 dynpicard -smoke gx3 8x3 zsal +smoke gx3 8x4 diag1,reprosum,run10day +smoke gx3 6x2 alt01,reprosum,run10day +smoke gx3 8x2 alt02,reprosum,run10day +smoke gx3 12x2 alt03,droundrobin,reprosum,run10day +smoke gx3 4x4 alt04,reprosum,run10day +smoke gx3 4x4 alt05,reprosum,run10day +smoke gx3 8x2 alt06,reprosum,run10day +smoke gx3 8x2 bgcz,reprosum,run10day +smoke gx1 15x2 seabedprob,reprosum,run10day +smoke gx3 14x2 fsd12,reprosum,run10day +smoke gx3 11x2 isotope,reprosum,run10day +smoke gx3 8x4 snwitdrdg,snwgrain,icdefault,reprosum,run10day +smoke gx3 6x4 dynpicard,reprosum,run10day +smoke gx3 8x3 zsal,reprosum,run10day -smoke gbox128 8x2 short -smoke gbox128 12x2 boxnodyn,short -smoke gbox128 9x2 boxadv,short -smoke gbox128 14x2 boxrestore,short -smoke gbox80 4x5 box2001 -smoke gbox80 11x3 boxslotcyl - -smoke gx3 4x2 diag1 smoke_gx3_8x2_diag1 -smoke gx3 4x1 diag1,thread smoke_gx3_8x2_diag1 -smoke gx3 8x1 alt01,thread smoke_gx3_6x2_alt01 -smoke gx3 16x1 alt02,thread smoke_gx3_8x2_alt02 -smoke gx3 24x1 alt03,thread smoke_gx3_12x2_alt03 -smoke gx3 24x1 alt04,thread smoke_gx3_4x4_alt04 -smoke gx3 14x1 alt05,thread smoke_gx3_4x4_alt05 -smoke gx3 24x1 alt06,thread smoke_gx3_8x2_alt06 -smoke gx3 12x1 bgcz,thread smoke_gx3_8x2_bgcz -smoke gx1 28x1 seabedprob,thread smoke_gx1_15x2_seabedprob -smoke gx3 30x1 fsd12,thread smoke_gx3_14x2_fsd12 -smoke gx3 16x1 isotope,thread smoke_gx3_11x2_isotope -smoke gx3 28x1 snwitdrdg,snwgrain,icdefault,thread smoke_gx3_8x4_icdefault_snwitdrdg_snwgrain -smoke gx3 18x1 dynpicard,thread smoke_gx3_6x4_dynpicard -smoke gx3 20x1 zsal,thread smoke_gx3_8x3_zsal - -smoke gbox128 20x1 short,thread smoke_gbox128_8x2_short -smoke gbox128 16x1 boxnodyn,short,thread smoke_gbox128_12x2_boxnodyn_short -smoke gbox128 14x1 boxadv,short,thread smoke_gbox128_9x2_boxadv_short -smoke gbox128 24x1 boxrestore,short,thread smoke_gbox128_14x2_boxrestore_short -smoke gbox80 19x1 box2001,thread smoke_gbox80_4x5_box2001 -smoke gbox80 8x4 boxslotcyl,thread smoke_gbox80_11x3_boxslotcyl +smoke gbox128 8x2 reprosum,run10day +smoke gbox128 12x2 boxnodyn,reprosum,run10day +smoke gbox128 9x2 boxadv,reprosum,run10day +smoke gbox128 14x2 boxrestore,reprosum,run10day +smoke gbox80 4x5 box2001,reprosum,run10day +smoke gbox80 11x3 boxslotcyl,reprosum,run10day +smoke gx3 4x2 diag1,reprosum,run10day,cmplogrest smoke_gx3_8x4_diag1_reprosum_run10day +smoke gx3 4x1 diag1,reprosum,run10day,cmplogrest,thread smoke_gx3_8x4_diag1_reprosum_run10day +smoke gx3 8x1 alt01,reprosum,run10day,cmplogrest,thread smoke_gx3_6x2_alt01_reprosum_run10day +smoke gx3 16x1 alt02,reprosum,run10day,cmplogrest,thread smoke_gx3_8x2_alt02_reprosum_run10day +smoke gx3 24x1 alt03,reprosum,run10day,cmplogrest,thread smoke_gx3_12x2_alt03_droundrobin_reprosum_run10day +smoke gx3 24x1 alt04,reprosum,run10day,cmplogrest,thread smoke_gx3_4x4_alt04_reprosum_run10day +smoke gx3 14x1 alt05,reprosum,run10day,cmplogrest,thread smoke_gx3_4x4_alt05_reprosum_run10day +smoke gx3 24x1 alt06,reprosum,run10day,cmplogrest,thread smoke_gx3_8x2_alt06_reprosum_run10day +smoke gx3 12x1 bgcz,reprosum,run10day,cmplogrest,thread smoke_gx3_8x2_bgcz_reprosum_run10day +smoke gx1 28x1 seabedprob,reprosum,run10day,cmplogrest,thread smoke_gx1_15x2_reprosum_run10day_seabedprob +smoke gx3 30x1 fsd12,reprosum,run10day,cmplogrest,thread smoke_gx3_14x2_fsd12_reprosum_run10day +smoke gx3 16x1 isotope,reprosum,run10day,cmplogrest,thread smoke_gx3_11x2_isotope_reprosum_run10day +smoke gx3 28x1 snwitdrdg,snwgrain,icdefault,reprosum,run10day,cmplogrest,thread smoke_gx3_8x4_icdefault_reprosum_run10day_snwitdrdg_snwgrain +smoke gx3 18x1 dynpicard,reprosum,run10day,cmplogrest,thread smoke_gx3_6x4_dynpicard_reprosum_run10day +smoke gx3 20x1 zsal,reprosum,run10day,cmplogrest,thread smoke_gx3_8x3_reprosum_run10day_zsal +smoke gbox128 20x1 reprosum,run10day,cmplogrest,thread smoke_gbox128_8x2_reprosum_run10day +smoke gbox128 16x1 boxnodyn,reprosum,run10day,cmplogrest,thread smoke_gbox128_12x2_boxnodyn_reprosum_run10day +smoke gbox128 14x1 boxadv,reprosum,run10day,cmplogrest,thread smoke_gbox128_9x2_boxadv_reprosum_run10day +smoke gbox128 24x1 boxrestore,reprosum,run10day,cmplogrest,thread smoke_gbox128_14x2_boxrestore_reprosum_run10day +smoke gbox80 19x1 box2001,reprosum,run10day,cmplogrest,thread smoke_gbox80_4x5_box2001_reprosum_run10day +smoke gbox80 8x4 boxslotcyl,reprosum,run10day,cmplogrest,thread smoke_gbox80_11x3_boxslotcyl_reprosum_run10day diff --git a/configuration/scripts/tests/prod_suite.ts b/configuration/scripts/tests/prod_suite.ts index 04982adb1..877fa1ce6 100644 --- a/configuration/scripts/tests/prod_suite.ts +++ b/configuration/scripts/tests/prod_suite.ts @@ -1,6 +1,6 @@ # Test Grid PEs Sets BFB-compare -qcchk gx3 72x1 qc,medium qcchk_gx3_72x1_medium_qc -qcchk gx1 144x1 qc,medium +qcchk gx3 72x1 qc,qcchk,medium qcchk_gx3_72x1_medium_qc_qcchk +qcchk gx1 144x1 qc,qcchk,medium smoke gx1 144x2 gx1prod,long,run10year -qcchkf gx3 72x1 qc,medium,alt02 qcchk_gx3_72x1_medium_qc -qcchk gx3 72x1 qcnonbfb,medium qcchk_gx3_72x1_medium_qc +qcchk gx3 72x1 qc,qcchkf,medium,alt02 qcchk_gx3_72x1_medium_qc_qcchk +qcchk gx3 72x1 qc,qcchk,dt3456s,medium qcchk_gx3_72x1_medium_qc_qcchk diff --git a/configuration/scripts/tests/reprosum_suite.ts b/configuration/scripts/tests/reprosum_suite.ts index a7f3fe5bc..417a7de2e 100644 --- a/configuration/scripts/tests/reprosum_suite.ts +++ b/configuration/scripts/tests/reprosum_suite.ts @@ -1,11 +1,11 @@ # Test Grid PEs Sets BFB-compare -logbfb gx3 4x2x25x29x4 dslenderX2,diag1,reprosum -#logbfb gx3 4x2x25x29x4 dslenderX2,diag1 -logbfb gx3 1x1x50x58x4 droundrobin,diag1,thread,maskhalo,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum -logbfb gx3 4x1x25x116x1 dslenderX1,diag1,thread,maskhalo,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum -logbfb gx3 1x20x5x29x80 dsectrobin,diag1,short,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum -logbfb gx3 8x2x8x10x20 droundrobin,diag1,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum -logbfb gx3 6x2x50x58x1 droundrobin,diag1,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum -logbfb gx3 6x2x4x29x18 dspacecurve,diag1,maskhalo,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum -logbfb gx3 17x2x1x1x800 droundrobin,diag1,maskhalo,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum -#logbfb gx3 8x2x8x10x20 droundrobin,diag1 logbfb_gx3_4x2x25x29x4_diag1_dslenderX2 +smoke gx3 4x2x25x29x4 dslenderX2,diag1,reprosum +#smoke gx3 4x2x25x29x4 dslenderX2,diag1 +smoke gx3 1x1x50x58x4 droundrobin,diag1,thread,maskhalo,reprosum,cmplog smoke_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +smoke gx3 4x1x25x116x1 dslenderX1,diag1,thread,maskhalo,reprosum,cmplog smoke_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +smoke gx3 1x20x5x29x80 dsectrobin,diag1,short,reprosum,cmplog smoke_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +smoke gx3 8x2x8x10x20 droundrobin,diag1,reprosum,cmplog smoke_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +smoke gx3 6x2x50x58x1 droundrobin,diag1,reprosum,cmplog smoke_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +smoke gx3 6x2x4x29x18 dspacecurve,diag1,maskhalo,reprosum,cmplog smoke_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +smoke gx3 17x2x1x1x800 droundrobin,diag1,maskhalo,reprosum,cmplog smoke_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +#smoke gx3 8x2x8x10x20 droundrobin,diag1,cmplog smoke_gx3_4x2x25x29x4_diag1_dslenderX2 diff --git a/configuration/scripts/tests/test_logbfb.script b/configuration/scripts/tests/test_logbfb.script deleted file mode 100644 index 0ac1ed224..000000000 --- a/configuration/scripts/tests/test_logbfb.script +++ /dev/null @@ -1,33 +0,0 @@ -# This is identical to a smoke test, but triggers bfbcompare with log files instead of restarts -#---------------------------------------------------- -# Run the CICE model -# cice.run returns -1 if run did not complete successfully - -./cice.run -set res="$status" - -set log_file = `ls -t1 ${ICE_RUNDIR}/cice.runlog* | head -1` -set ttimeloop = `grep TimeLoop ${log_file} | grep Timer | cut -c 22-32` -set tdynamics = `grep Dynamics ${log_file} | grep Timer | cut -c 22-32` -set tcolumn = `grep Column ${log_file} | grep Timer | cut -c 22-32` -if (${ttimeloop} == "") set ttimeloop = -1 -if (${tdynamics} == "") set tdynamics = -1 -if (${tcolumn} == "") set tcolumn = -1 - -mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev -cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output -mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev -cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} test" >! ${ICE_CASEDIR}/test_output -rm -f ${ICE_CASEDIR}/test_output.prev - -set grade = PASS -if ( $res != 0 ) then - set grade = FAIL - echo "$grade ${ICE_TESTNAME} run ${ttimeloop} ${tdynamics} ${tcolumn}" >> ${ICE_CASEDIR}/test_output - echo "$grade ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output - exit 99 -endif - -echo "$grade ${ICE_TESTNAME} run ${ttimeloop} ${tdynamics} ${tcolumn}" >> ${ICE_CASEDIR}/test_output -echo "$grade ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output - diff --git a/configuration/scripts/tests/test_qcchkf.script b/configuration/scripts/tests/test_qcchkf.script deleted file mode 100644 index 81b5f05fc..000000000 --- a/configuration/scripts/tests/test_qcchkf.script +++ /dev/null @@ -1,36 +0,0 @@ - -cp ${ICE_SANDBOX}/configuration/scripts/tests/QC/CICE_t_critical_p0.8.nc . -cp ${ICE_SANDBOX}/configuration/scripts/tests/QC/CICE_Lookup_Table_p0.8_n1825.nc . - -#---------------------------------------------------- -# Run the CICE model -# cice.run returns -1 if run did not complete successfully - -./cice.run -set res="$status" - -set log_file = `ls -t1 ${ICE_RUNDIR}/cice.runlog* | head -1` -set ttimeloop = `grep TimeLoop ${log_file} | grep Timer | cut -c 22-32` -set tdynamics = `grep Dynamics ${log_file} | grep Timer | cut -c 22-32` -set tcolumn = `grep Column ${log_file} | grep Timer | cut -c 22-32` -if (${ttimeloop} == "") set ttimeloop = -1 -if (${tdynamics} == "") set tdynamics = -1 -if (${tcolumn} == "") set tcolumn = -1 - -mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev -cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output -mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev -cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} test" >! ${ICE_CASEDIR}/test_output -rm -f ${ICE_CASEDIR}/test_output.prev - -set grade = PASS -if ( $res != 0 ) then - set grade = FAIL - echo "$grade ${ICE_TESTNAME} run ${ttimeloop} ${tdynamics} ${tcolumn}" >> ${ICE_CASEDIR}/test_output - echo "$grade ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output - exit 99 -endif - -echo "$grade ${ICE_TESTNAME} run ${ttimeloop} ${tdynamics} ${tcolumn}" >> ${ICE_CASEDIR}/test_output -echo "$grade ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output - diff --git a/doc/source/cice_index.rst b/doc/source/cice_index.rst index 85acbece3..d216f7849 100644 --- a/doc/source/cice_index.rst +++ b/doc/source/cice_index.rst @@ -650,6 +650,7 @@ either Celsius or Kelvin units). "Tffresh", "freezing temp of fresh ice", "273.15 K" "tfrz_option", "form of ocean freezing temperature", "" "thinS", "minimum ice thickness for brine tracer", "" + "timer_stats", "logical to turn on extra timer statistics", ".false." "timesecs", "total elapsed time in seconds", "s" "time_beg", "beginning time for history averages", "" "time_bounds", "beginning and ending time for history averages", "" diff --git a/doc/source/user_guide/ug_case_settings.rst b/doc/source/user_guide/ug_case_settings.rst index 23e6951fc..eed9c8c5f 100644 --- a/doc/source/user_guide/ug_case_settings.rst +++ b/doc/source/user_guide/ug_case_settings.rst @@ -81,25 +81,34 @@ can be modified as needed. "ICE_TARGET", "string", "build target", "set by cice.setup" "ICE_IOTYPE", "string", "I/O format", "set by cice.setup" " ", "netcdf", "serial netCDF" - " ", "pio", "parallel netCDF" " ", "none", "netCDF library is not available" + " ", "pio", "parallel netCDF" "ICE_CLEANBUILD", "true, false", "automatically clean before building", "true" "ICE_CPPDEFS", "user defined preprocessor macros for build", "null" "ICE_QUIETMODE", "true, false", "reduce build output to the screen", "false" "ICE_GRID", "string (see below)", "grid", "set by cice.setup" - " ", "gx3", "3-deg displace-pole (Greenland) global grid", " " - " ", "gx1", "1-deg displace-pole (Greenland) global grid", " " - " ", "tx1", "1-deg tripole global grid", " " " ", "gbox80", "80x80 box", " " " ", "gbox128", "128x128 box", " " - "ICE_NTASKS", "integer", "number of tasks, must be set to 1", "set by cice.setup" - "ICE_NTHRDS", "integer", "number of threads per task, must be set to 1", "set by cice.setup" + " ", "gbox180", "180x180 box", " " + " ", "gx1", "1-deg displace-pole (Greenland) global grid", " " + " ", "gx3", "3-deg displace-pole (Greenland) global grid", " " + " ", "tx1", "1-deg tripole global grid", " " + "ICE_NTASKS", "integer", "number of MPI tasks", "set by cice.setup" + "ICE_NTHRDS", "integer", "number of threads per task", "set by cice.setup" + "ICE_OMPSCHED", "string", "OpenMP SCHEDULE env setting", "static,1" "ICE_TEST", "string", "test setting if using a test", "set by cice.setup" "ICE_TESTNAME", "string", "test name if using a test", "set by cice.setup" - "ICE_BASELINE", "string", "baseline directory name, associated with cice.setup -bdir ", "set by cice.setup" + "ICE_TESTID", "string", "test name testid", "set by cice.setup" + "ICE_BASELINE", "string", "baseline directory name, associated with cice.setup --bdir ", "set by cice.setup" "ICE_BASEGEN", "string", "baseline directory name for regression generation, associated with cice.setup -bgen ", "set by cice.setup" "ICE_BASECOM", "string", "baseline directory name for regression comparison, associated with cice.setup -bcmp ", "set by cice.setup" - "ICE_BFBCOMP", "string", "location of case for comparison, associated with cice.setup -td", "set by cice.setup" + "ICE_BFBCOMP", "string", "location of case for comparison, associated with cice.setup --bcmp", "set by cice.setup" + "ICE_BFBTYPE", "string", "type and files used in BFBCOMP", "restart" + " ", "log", "log file comparison for bit for bit", " " + " ", "logrest", "log and restart files for bit for bit", " " + " ", "qcchk", "QC test for same climate", " " + " ", "qcchkf", "QC test for different climate", " " + " ", "restart", "restart files for bit for bit", " " "ICE_SPVAL", "string", "special value for cice.settings strings", "set by cice.setup" "ICE_RUNLENGTH", "integer (see below)", "batch run length default", "set by cice.setup" " ", "-1", "15 minutes (default)", " " @@ -111,6 +120,7 @@ can be modified as needed. "ICE_ACCOUNT", "string", "batch account number", "set by cice.setup, .cice_proj or by default" "ICE_QUEUE", "string", "batch queue name", "set by cice.setup or by default" "ICE_THREADED", "true, false", "force threading in compile, will always compile threaded if ICE_NTHRDS :math:`> 1`", "false" + "ICE_COMMDIR", "mpi, serial", "specify infrastructure comm version", "set by ICE_NTASKS" "ICE_BLDDEBUG", "true, false", "turn on compile debug flags", "false" "ICE_COVERAGE", "true, false", "turn on code coverage flags", "false" @@ -214,6 +224,7 @@ setup_nml "``runtype``", "``continue``", "restart using ``pointer_file``", "``initial``" "", "``initial``", "start from ``ice_ic``", "" "``sec_init``", "integer", "the initial second if not using restart", "0" + "``timer_stats``", "logical", "controls extra timer output", "``.false.``" "``use_leap_years``", "logical", "include leap days", "``.false.``" "``use_restart_time``", "logical", "set initial date using restart file on initial runtype only", "``.false.``" "``version_name``", "string", "model version", "'unknown_version_name'" diff --git a/doc/source/user_guide/ug_implementation.rst b/doc/source/user_guide/ug_implementation.rst index 650d3d6c7..c4a8575d6 100644 --- a/doc/source/user_guide/ug_implementation.rst +++ b/doc/source/user_guide/ug_implementation.rst @@ -1056,7 +1056,8 @@ Timers are declared and initialized in **ice\_timers.F90**, and the code to be timed is wrapped with calls to *ice\_timer\_start* and *ice\_timer\_stop*. Finally, *ice\_timer\_print* writes the results to the log file. The optional “stats" argument (true/false) prints -additional statistics. Calling *ice\_timer\_print\_all* prints all of +additional statistics. The "stats" argument can be set by the ``timer_stats`` +namelist. Calling *ice\_timer\_print\_all* prints all of the timings at once, rather than having to call each individually. Currently, the timers are set up as in :ref:`timers`. Section :ref:`addtimer` contains instructions for adding timers. @@ -1080,7 +1081,7 @@ The timers use *MPI\_WTIME* for parallel runs and the F90 intrinsic +--------------+-------------+----------------------------------------------------+ | 1 | Total | the entire run | +--------------+-------------+----------------------------------------------------+ - | 2 | Step | total minus initialization and exit | + | 2 | Timeloop | total minus initialization and exit | +--------------+-------------+----------------------------------------------------+ | 3 | Dynamics | EVP | +--------------+-------------+----------------------------------------------------+ @@ -1092,23 +1093,29 @@ The timers use *MPI\_WTIME* for parallel runs and the F90 intrinsic +--------------+-------------+----------------------------------------------------+ | 7 | Shortwave | SW radiation and albedo | +--------------+-------------+----------------------------------------------------+ - | 8 | Meltponds | melt ponds | + | 8 | Ridging | mechanical redistribution | +--------------+-------------+----------------------------------------------------+ - | 9 | Ridging | mechanical redistribution | + | 9 | FloeSize | flow size | +--------------+-------------+----------------------------------------------------+ - | 10 | Cat Conv | transport in thickness space | + | 10 | Coupling | sending/receiving coupler messages | +--------------+-------------+----------------------------------------------------+ - | 11 | Coupling | sending/receiving coupler messages | + | 11 | ReadWrite | reading/writing files | +--------------+-------------+----------------------------------------------------+ - | 12 | ReadWrite | reading/writing files | + | 12 | Diags | diagnostics (log file) | +--------------+-------------+----------------------------------------------------+ - | 13 | Diags | diagnostics (log file) | + | 13 | History | history output | +--------------+-------------+----------------------------------------------------+ - | 14 | History | history output | + | 14 | Bound | boundary conditions and subdomain communications | +--------------+-------------+----------------------------------------------------+ - | 15 | Bound | boundary conditions and subdomain communications | + | 15 | BGC | biogeochemistry | +--------------+-------------+----------------------------------------------------+ - | 16 | BGC | biogeochemistry | + | 16 | Forcing | forcing | + +--------------+-------------+----------------------------------------------------+ + | 17 | 1d-evp | 1d evp | + +--------------+-------------+----------------------------------------------------+ + | 18 | 2d-evp | 2d evp | + +--------------+-------------+----------------------------------------------------+ + | 19 | UpdState | update state | +--------------+-------------+----------------------------------------------------+ .. _restartfiles: From 3446cf632793432536dc14f080d6d80c5df4bf5c Mon Sep 17 00:00:00 2001 From: apcraig Date: Tue, 11 Jan 2022 11:57:51 -0700 Subject: [PATCH 05/14] Update EVP OMP implementation --- cicecore/cicedynB/dynamics/ice_dyn_evp.F90 | 12 ++++------ configuration/scripts/cice.batch.csh | 1 - .../scripts/options/set_nml.timerstats | 1 + doc/source/user_guide/ug_implementation.rst | 24 ++++++++++--------- 4 files changed, 19 insertions(+), 19 deletions(-) create mode 100644 configuration/scripts/options/set_nml.timerstats diff --git a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 index 143c3a8dd..e117d85c3 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 @@ -239,9 +239,7 @@ subroutine evp (dt) call t2ugrid_vector(strairy) endif -! tcraig, tcx, threading here leads to some non-reproducbile results and failures in icepack_ice_strength -! need to do more debugging - !$TCXOMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,ij,i,j) + !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,ij,i,j) do iblk = 1, nblocks !----------------------------------------------------------------- @@ -301,7 +299,7 @@ subroutine evp (dt) enddo ! ij enddo ! iblk - !$TCXOMP END PARALLEL DO + !$OMP END PARALLEL DO call icepack_warnings_flush(nu_diag) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & @@ -354,7 +352,7 @@ subroutine evp (dt) endif enddo - !$OMP END PARALLEL DO + !$OMP END PARALLEL DO endif call ice_timer_start(timer_evp_2d) @@ -399,7 +397,7 @@ subroutine evp (dt) ! stress tensor equation, total surface stress !----------------------------------------------------------------- - !$TCXOMP PARALLEL DO PRIVATE(iblk,strtmp) + !$OMP PARALLEL DO PRIVATE(iblk,strtmp) do iblk = 1, nblocks ! if (trim(yield_curve) == 'ellipse') then @@ -445,7 +443,7 @@ subroutine evp (dt) Tbu (:,:,iblk)) enddo - !$TCXOMP END PARALLEL DO + !$OMP END PARALLEL DO call stack_velocity_field(uvel, vvel, fld2) call ice_timer_start(timer_bound) diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index f86b55502..e53b04533 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -107,7 +107,6 @@ EOFB else if (${ICE_MACHINE} =~ narwhal*) then if (${runlength} <= 0) then set batchtime = "00:29:59" - set queue = "debug" else set queue = "standard" endif diff --git a/configuration/scripts/options/set_nml.timerstats b/configuration/scripts/options/set_nml.timerstats new file mode 100644 index 000000000..723891b7b --- /dev/null +++ b/configuration/scripts/options/set_nml.timerstats @@ -0,0 +1 @@ +timer_stats = .true. diff --git a/doc/source/user_guide/ug_implementation.rst b/doc/source/user_guide/ug_implementation.rst index c4a8575d6..66b145762 100644 --- a/doc/source/user_guide/ug_implementation.rst +++ b/doc/source/user_guide/ug_implementation.rst @@ -1063,9 +1063,11 @@ Currently, the timers are set up as in :ref:`timers`. Section :ref:`addtimer` contains instructions for adding timers. The timings provided by these timers are not mutually exclusive. For -example, the column timer (5) includes the timings from 6–10, and -subroutine *bound* (timer 15) is called from many different places in -the code, including the dynamics and advection routines. +example, the Column timer includes the timings from several other +timers, while timer Bound is called from many different places in +the code, including the dynamics and advection routines. The +Dynamics, Advection, and Column timers do not overlap and represent +most of the overall model work. The timers use *MPI\_WTIME* for parallel runs and the F90 intrinsic *system\_clock* for single-processor runs. @@ -1083,19 +1085,19 @@ The timers use *MPI\_WTIME* for parallel runs and the F90 intrinsic +--------------+-------------+----------------------------------------------------+ | 2 | Timeloop | total minus initialization and exit | +--------------+-------------+----------------------------------------------------+ - | 3 | Dynamics | EVP | + | 3 | Dynamics | dynamics | +--------------+-------------+----------------------------------------------------+ | 4 | Advection | horizontal transport | +--------------+-------------+----------------------------------------------------+ | 5 | Column | all vertical (column) processes | +--------------+-------------+----------------------------------------------------+ - | 6 | Thermo | vertical thermodynamics | + | 6 | Thermo | vertical thermodynamics, part of Column timer | +--------------+-------------+----------------------------------------------------+ - | 7 | Shortwave | SW radiation and albedo | + | 7 | Shortwave | SW radiation and albedo, part of Thermo timer | +--------------+-------------+----------------------------------------------------+ - | 8 | Ridging | mechanical redistribution | + | 8 | Ridging | mechanical redistribution, part of Column timer | +--------------+-------------+----------------------------------------------------+ - | 9 | FloeSize | flow size | + | 9 | FloeSize | flow size, part of Column timer | +--------------+-------------+----------------------------------------------------+ | 10 | Coupling | sending/receiving coupler messages | +--------------+-------------+----------------------------------------------------+ @@ -1107,13 +1109,13 @@ The timers use *MPI\_WTIME* for parallel runs and the F90 intrinsic +--------------+-------------+----------------------------------------------------+ | 14 | Bound | boundary conditions and subdomain communications | +--------------+-------------+----------------------------------------------------+ - | 15 | BGC | biogeochemistry | + | 15 | BGC | biogeochemistry, part of Thermo timer | +--------------+-------------+----------------------------------------------------+ | 16 | Forcing | forcing | +--------------+-------------+----------------------------------------------------+ - | 17 | 1d-evp | 1d evp | + | 17 | 1d-evp | 1d evp, part of Dynamics timer | +--------------+-------------+----------------------------------------------------+ - | 18 | 2d-evp | 2d evp | + | 18 | 2d-evp | 2d evp, part of Dynamics timer | +--------------+-------------+----------------------------------------------------+ | 19 | UpdState | update state | +--------------+-------------+----------------------------------------------------+ From f91f4597b8d0740227a6172b9b496a2f1976ac29 Mon Sep 17 00:00:00 2001 From: apcraig Date: Tue, 11 Jan 2022 16:17:16 -0700 Subject: [PATCH 06/14] - Refactor puny/pi scalars in eap dynamics to improve performance - Update OMP in evp and eap --- cicecore/cicedynB/dynamics/ice_dyn_eap.F90 | 165 +++++++----------- cicecore/cicedynB/dynamics/ice_dyn_evp.F90 | 10 +- cicecore/cicedynB/dynamics/ice_dyn_shared.F90 | 6 +- 3 files changed, 73 insertions(+), 108 deletions(-) diff --git a/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 b/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 index e95e4cdef..5324a9a8f 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 @@ -25,6 +25,11 @@ module ice_dyn_eap p001, p027, p055, p111, p166, p222, p25, p333 use ice_fileunits, only: nu_diag, nu_dump_eap, nu_restart_eap use ice_exit, only: abort_ice +! use ice_timers, only: & +! ice_timer_start, ice_timer_stop, & +! timer_tmp1, timer_tmp2, timer_tmp3, timer_tmp4, & +! timer_tmp5, timer_tmp6, timer_tmp7, timer_tmp8, timer_tmp9 + use icepack_intfc, only: icepack_warnings_flush, icepack_warnings_aborted use icepack_intfc, only: icepack_query_parameters use icepack_intfc, only: icepack_ice_strength @@ -61,6 +66,11 @@ module ice_dyn_eap a11 , & ! components of structure tensor () a12 + ! private for reuse, set in init_eap + + real (kind=dbl_kind) :: & + puny, pi, pi2, piq, pih + !======================================================================= contains @@ -137,9 +147,6 @@ subroutine eap (dt) tarear, uarear, to_ugrid, t2ugrid_vector, u2tgrid_vector use ice_state, only: aice, vice, vsno, uvel, vvel, divu, shear, & aice_init, aice0, aicen, vicen, strength -! use ice_timers, only: timer_dynamics, timer_bound, & -! ice_timer_start, ice_timer_stop, & -! timer_tmp1, timer_tmp2, timer_tmp3 use ice_timers, only: timer_dynamics, timer_bound, & ice_timer_start, ice_timer_stop @@ -204,7 +211,7 @@ subroutine eap (dt) ! This call is needed only if dt changes during runtime. ! call set_evp_parameters (dt) - !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block) + !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block) SCHEDULE(runtime) do iblk = 1, nblocks do j = 1, ny_block do i = 1, nx_block @@ -274,10 +281,7 @@ subroutine eap (dt) call t2ugrid_vector(strairy) endif -! tcraig, tcx, turned off this threaded region, in evp, this block and -! the icepack_ice_strength call seems to not be thread safe. more -! debugging needed - !$TCXOMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block,ij) + !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,ij,i,j) SCHEDULE(runtime) do iblk = 1, nblocks !----------------------------------------------------------------- @@ -364,7 +368,7 @@ subroutine eap (dt) strength = strength(i,j, iblk) ) enddo ! ij enddo ! iblk - !$TCXOMP END PARALLEL DO + !$OMP END PARALLEL DO call icepack_warnings_flush(nu_diag) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & @@ -396,7 +400,7 @@ subroutine eap (dt) if (seabed_stress) then - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks if ( seabed_stress_method == 'LKD' ) then @@ -426,10 +430,10 @@ subroutine eap (dt) ! stress tensor equation, total surface stress !----------------------------------------------------------------- - !$TCXOMP PARALLEL DO PRIVATE(iblk,strtmp) + !$OMP PARALLEL DO PRIVATE(iblk,strtmp) SCHEDULE(runtime) do iblk = 1, nblocks -! call ice_timer_start(timer_tmp1,iblk) ! dynamics +! call ice_timer_start(timer_tmp1,iblk) call stress_eap (nx_block, ny_block, & ksub, ndte, & icellt(iblk), & @@ -462,12 +466,13 @@ subroutine eap (dt) ! rdg_conv (:,:,iblk), rdg_shear (:,:,iblk), & rdg_conv (:,:,iblk), & strtmp (:,:,:)) -! call ice_timer_stop(timer_tmp1,iblk) ! dynamics +! call ice_timer_stop(timer_tmp1,iblk) !----------------------------------------------------------------- ! momentum equation !----------------------------------------------------------------- +! call ice_timer_start(timer_tmp2,iblk) call stepu (nx_block, ny_block, & icellu (iblk), Cdn_ocn (:,:,iblk), & indxui (:,iblk), indxuj (:,iblk), & @@ -483,12 +488,13 @@ subroutine eap (dt) uvel_init(:,:,iblk), vvel_init(:,:,iblk),& uvel (:,:,iblk), vvel (:,:,iblk), & Tbu (:,:,iblk)) +! call ice_timer_stop(timer_tmp2,iblk) !----------------------------------------------------------------- ! evolution of structure tensor A !----------------------------------------------------------------- -! call ice_timer_start(timer_tmp3,iblk) ! dynamics +! call ice_timer_start(timer_tmp3,iblk) if (mod(ksub,10) == 1) then ! only called every 10th timestep call stepa (nx_block, ny_block, & dtei, icellt (iblk), & @@ -505,9 +511,9 @@ subroutine eap (dt) stress12_1(:,:,iblk), stress12_2(:,:,iblk), & stress12_3(:,:,iblk), stress12_4(:,:,iblk)) endif -! call ice_timer_stop(timer_tmp3,iblk) ! dynamics +! call ice_timer_stop(timer_tmp3,iblk) enddo - !$TCXOMP END PARALLEL DO + !$OMP END PARALLEL DO call stack_velocity_field(uvel, vvel, fld2) call ice_timer_start(timer_bound) @@ -530,7 +536,7 @@ subroutine eap (dt) ! ice-ocean stress !----------------------------------------------------------------- - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks call dyn_finish & @@ -583,17 +589,19 @@ subroutine init_eap real (kind=dbl_kind) :: & ainit, xinit, yinit, zinit, & da, dx, dy, dz, & - pi, pih, piq, phi + phi character(len=*), parameter :: subname = '(init_eap)' - call icepack_query_parameters(pi_out=pi, pih_out=pih, piq_out=piq) + call icepack_query_parameters(puny_out=puny, & + pi_out=pi, pi2_out=pi2, piq_out=piq, pih_out=pih) call icepack_warnings_flush(nu_diag) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & file=__FILE__, line=__LINE__) + phi = pi/c12 ! diamond shaped floe smaller angle (default phi = 30 deg) - !$OMP PARALLEL DO PRIVATE(iblk,i,j) + !$OMP PARALLEL DO PRIVATE(iblk,i,j) SCHEDULE(runtime) do iblk = 1, nblocks do j = 1, ny_block do i = 1, nx_block @@ -747,14 +755,9 @@ FUNCTION s11kr(x,y,z,phi) d11, d12, d22, & IIn1t2, IIn2t1, & ! IIt1t2, & - Hen1t2, Hen2t1, & - pih, puny - character(len=*), parameter :: subname = '(s11kr)' + Hen1t2, Hen2t1 - call icepack_query_parameters(pih_out=pih, puny_out=puny) - call icepack_warnings_flush(nu_diag) - if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & - file=__FILE__, line=__LINE__) + character(len=*), parameter :: subname = '(s11kr)' p = phi @@ -818,14 +821,9 @@ FUNCTION s12kr(x,y,z,phi) d11, d12, d22, & IIn1t2, IIn2t1, & ! IIt1t2, & - Hen1t2, Hen2t1, & - pih, puny - character(len=*), parameter :: subname = '(s12kr)' + Hen1t2, Hen2t1 - call icepack_query_parameters(pih_out=pih, puny_out=puny) - call icepack_warnings_flush(nu_diag) - if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & - file=__FILE__, line=__LINE__) + character(len=*), parameter :: subname = '(s12kr)' p = phi @@ -889,14 +887,9 @@ FUNCTION s22kr(x,y,z,phi) d11, d12, d22, & IIn1t2, IIn2t1, & ! IIt1t2, & - Hen1t2, Hen2t1, & - pih, puny - character(len=*), parameter :: subname = '(s22kr)' + Hen1t2, Hen2t1 - call icepack_query_parameters(pih_out=pih, puny_out=puny) - call icepack_warnings_flush(nu_diag) - if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & - file=__FILE__, line=__LINE__) + character(len=*), parameter :: subname = '(s22kr)' p = phi @@ -959,14 +952,9 @@ FUNCTION s11ks(x,y,z,phi) ! t2t1i12, t2t1i21, t2t1i22, & d11, d12, d22, & IIn1t2, IIn2t1, IIt1t2, & - Hen1t2, Hen2t1, & - pih, puny - character(len=*), parameter :: subname = '(s11ks)' + Hen1t2, Hen2t1 - call icepack_query_parameters(pih_out=pih, puny_out=puny) - call icepack_warnings_flush(nu_diag) - if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & - file=__FILE__, line=__LINE__) + character(len=*), parameter :: subname = '(s11ks)' p = phi @@ -1028,14 +1016,9 @@ FUNCTION s12ks(x,y,z,phi) t2t1i12, t2t1i21, & d11, d12, d22, & IIn1t2, IIn2t1, IIt1t2, & - Hen1t2, Hen2t1, & - pih, puny - character(len=*), parameter :: subname = '(s12ks)' + Hen1t2, Hen2t1 - call icepack_query_parameters(pih_out=pih, puny_out=puny) - call icepack_warnings_flush(nu_diag) - if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & - file=__FILE__, line=__LINE__) + character(len=*), parameter :: subname = '(s12ks)' p =phi @@ -1099,14 +1082,9 @@ FUNCTION s22ks(x,y,z,phi) t2t1i22, & d11, d12, d22, & IIn1t2, IIn2t1, IIt1t2, & - Hen1t2, Hen2t1, & - pih, puny - character(len=*), parameter :: subname = '(s22ks)' + Hen1t2, Hen2t1 - call icepack_query_parameters(pih_out=pih, puny_out=puny) - call icepack_warnings_flush(nu_diag) - if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & - file=__FILE__, line=__LINE__) + character(len=*), parameter :: subname = '(s22ks)' p = phi @@ -1188,11 +1166,6 @@ subroutine stress_eap (nx_block, ny_block, & rdg_conv, & strtmp) -!echmod tmp -! use ice_timers, only: & -! ice_timer_start, ice_timer_stop, & -! timer_tmp1, timer_tmp2, timer_tmp3 - integer (kind=int_kind), intent(in) :: & nx_block, ny_block, & ! block dimensions ksub , & ! subcycling step @@ -1270,7 +1243,7 @@ subroutine stress_eap (nx_block, ny_block, & csigmne, csigmnw, csigmse, csigmsw , & csig12ne, csig12nw, csig12se, csig12sw , & str12ew, str12we, str12ns, str12sn , & - strp_tmp, strm_tmp, puny + strp_tmp, strm_tmp real (kind=dbl_kind) :: & alpharne, alpharnw, alpharsw, alpharse, & @@ -1282,11 +1255,6 @@ subroutine stress_eap (nx_block, ny_block, & ! Initialize !----------------------------------------------------------------- - call icepack_query_parameters(puny_out=puny) - call icepack_warnings_flush(nu_diag) - if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & - file=__FILE__, line=__LINE__) - strtmp(:,:,:) = c0 do ij = 1, icellt @@ -1330,7 +1298,6 @@ subroutine stress_eap (nx_block, ny_block, & !----------------------------------------------------------------- ! Stress updated depending on strain rate and structure tensor !----------------------------------------------------------------- -! call ice_timer_start(timer_tmp2) ! dynamics ! ne call update_stress_rdg (ksub, ndte, divune, tensionne, & @@ -1357,7 +1324,6 @@ subroutine stress_eap (nx_block, ny_block, & stress12tmp_4, strength(i,j), & alpharse, alphasse) -! call ice_timer_stop(timer_tmp2) ! dynamics !----------------------------------------------------------------- ! on last subcycle, save quantities for mechanical redistribution !----------------------------------------------------------------- @@ -1609,10 +1575,14 @@ subroutine update_stress_rdg (ksub, ndte, divu, tension, & Angle_denom_gamma, Angle_denom_alpha, & Tany_1, Tany_2, & x, y, dx, dy, da, & - invdx, invdy, invda, invsin, & dtemp1, dtemp2, atempprime, & - kxw, kyw, kaw, & - puny, pi, pi2, piq, pih + kxw, kyw, kaw + + real (kind=dbl_kind), save :: & + invdx, invdy, invda, invsin + + logical (kind=log_kind), save :: & + first_call = .true. real (kind=dbl_kind), parameter :: & kfriction = 0.45_dbl_kind @@ -1624,17 +1594,13 @@ subroutine update_stress_rdg (ksub, ndte, divu, tension, & character(len=*), parameter :: subname = '(update_stress_rdg)' - call icepack_query_parameters(puny_out=puny, & - pi_out=pi, pi2_out=pi2, piq_out=piq, pih_out=pih) - call icepack_warnings_flush(nu_diag) - if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & - file=__FILE__, line=__LINE__) - ! Factor to maintain the same stress as in EVP (see Section 3) ! Can be set to 1 otherwise - invstressconviso = c1/(c1+kfriction*kfriction) - invsin = c1/sin(pi2/c12) * invstressconviso + if (first_call) then + invstressconviso = c1/(c1+kfriction*kfriction) + invsin = c1/sin(pi2/c12) * invstressconviso + endif ! compute eigenvalues, eigenvectors and angles for structure tensor, strain rates @@ -1642,7 +1608,7 @@ subroutine update_stress_rdg (ksub, ndte, divu, tension, & a22 = c1-a11 -! gamma: angle between general coordiantes and principal axis of A +! gamma: angle between general coordinates and principal axis of A ! here Tan2gamma = 2 a12 / (a11 - a22) Q11Q11 = c1 @@ -1733,12 +1699,14 @@ subroutine update_stress_rdg (ksub, ndte, divu, tension, & if (y < 0) y = y + pi ! Now calculate updated stress tensor - dx = pi/real(nx_yield-1,kind=dbl_kind) - dy = pi/real(ny_yield-1,kind=dbl_kind) - da = p5/real(na_yield-1,kind=dbl_kind) - invdx = c1/dx - invdy = c1/dy - invda = c1/da + if (first_call) then + dx = pi/real(nx_yield-1,kind=dbl_kind) + dy = pi/real(ny_yield-1,kind=dbl_kind) + da = p5/real(na_yield-1,kind=dbl_kind) + invdx = c1/dx + invdy = c1/dy + invda = c1/da + endif if (interpolate_stress_rdg) then @@ -1876,6 +1844,8 @@ subroutine update_stress_rdg (ksub, ndte, divu, tension, & + rotstemp22s*dtemp22 endif + first_call = .false. + end subroutine update_stress_rdg !======================================================================= @@ -2006,7 +1976,7 @@ subroutine calc_ffrac (stressp, stressm, & real (kind=dbl_kind) :: & sigma11, sigma12, sigma22, & - gamma, sigma_1, sigma_2, pih, & + gamma, sigma_1, sigma_2, & Q11, Q12, Q11Q11, Q11Q12, Q12Q12 real (kind=dbl_kind), parameter :: & @@ -2015,11 +1985,6 @@ subroutine calc_ffrac (stressp, stressm, & character(len=*), parameter :: subname = '(calc_ffrac)' - call icepack_query_parameters(pih_out=pih) - call icepack_warnings_flush(nu_diag) - if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & - file=__FILE__, line=__LINE__) - sigma11 = p5*(stressp+stressm) sigma12 = stress12 sigma22 = p5*(stressp-stressm) @@ -2182,7 +2147,7 @@ subroutine read_restart_eap() ! Ensure unused values in west and south ghost cells are 0 !----------------------------------------------------------------- - !$OMP PARALLEL DO PRIVATE(iblk,i,j) + !$OMP PARALLEL DO PRIVATE(iblk,i,j) SCHEDULE(runtime) do iblk = 1, nblocks do j = 1, nghost do i = 1, nx_block diff --git a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 index e117d85c3..1e5cdecff 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 @@ -177,7 +177,7 @@ subroutine evp (dt) ! field_loc_center, field_type_scalar) ! call ice_timer_stop(timer_bound) - !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block) + !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block) SCHEDULE(runtime) do iblk = 1, nblocks do j = 1, ny_block @@ -239,7 +239,7 @@ subroutine evp (dt) call t2ugrid_vector(strairy) endif - !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,ij,i,j) + !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,ij,i,j) SCHEDULE(runtime) do iblk = 1, nblocks !----------------------------------------------------------------- @@ -331,7 +331,7 @@ subroutine evp (dt) if (seabed_stress) then - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks if ( seabed_stress_method == 'LKD' ) then @@ -397,7 +397,7 @@ subroutine evp (dt) ! stress tensor equation, total surface stress !----------------------------------------------------------------- - !$OMP PARALLEL DO PRIVATE(iblk,strtmp) + !$OMP PARALLEL DO PRIVATE(iblk,strtmp) SCHEDULE(runtime) do iblk = 1, nblocks ! if (trim(yield_curve) == 'ellipse') then @@ -537,7 +537,7 @@ subroutine evp (dt) ! ice-ocean stress !----------------------------------------------------------------- - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks call dyn_finish & diff --git a/cicecore/cicedynB/dynamics/ice_dyn_shared.F90 b/cicecore/cicedynB/dynamics/ice_dyn_shared.F90 index 23251b2d1..8946e644a 100755 --- a/cicecore/cicedynB/dynamics/ice_dyn_shared.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_shared.F90 @@ -157,7 +157,7 @@ subroutine init_dyn (dt) allocate(fcor_blk(nx_block,ny_block,max_blocks)) - !$OMP PARALLEL DO PRIVATE(iblk,i,j) + !$OMP PARALLEL DO PRIVATE(iblk,i,j) SCHEDULE(runtime) do iblk = 1, nblocks do j = 1, ny_block do i = 1, nx_block @@ -1464,7 +1464,7 @@ subroutine stack_velocity_field(uvel, vvel, fld2) character(len=*), parameter :: subname = '(stack_velocity_field)' ! load velocity into array for boundary updates - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks fld2(:,:,1,iblk) = uvel(:,:,iblk) fld2(:,:,2,iblk) = vvel(:,:,iblk) @@ -1496,7 +1496,7 @@ subroutine unstack_velocity_field(fld2, uvel, vvel) character(len=*), parameter :: subname = '(unstack_velocity_field)' ! Unload velocity from array after boundary updates - !$OMP PARALLEL DO PRIVATE(iblk) + !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks uvel(:,:,iblk) = fld2(:,:,1,iblk) vvel(:,:,iblk) = fld2(:,:,2,iblk) From 7905bf69d444a84e9f111e2a75543fb49058c526 Mon Sep 17 00:00:00 2001 From: apcraig Date: Tue, 11 Jan 2022 18:31:04 -0700 Subject: [PATCH 07/14] Clean up --- cicecore/cicedynB/dynamics/ice_dyn_vp.F90 | 2 +- .../dynamics/ice_transport_driver.F90 | 96 ----------------- .../cicedynB/dynamics/ice_transport_remap.F90 | 67 ------------ cicecore/cicedynB/infrastructure/ice_grid.F90 | 31 ++++++ .../drivers/standalone/cice/CICE_RunMod.F90 | 102 +----------------- 5 files changed, 35 insertions(+), 263 deletions(-) diff --git a/cicecore/cicedynB/dynamics/ice_dyn_vp.F90 b/cicecore/cicedynB/dynamics/ice_dyn_vp.F90 index a6cf2aebd..6fa6ab108 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_vp.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_vp.F90 @@ -816,7 +816,7 @@ subroutine anderson_solver (icellt , icellu, & !----------------------------------------------------------------- !$OMP PARALLEL DO PRIVATE(iblk,stress_Pr) do iblk = 1, nblocks - + if (use_mean_vrel) then ulin(:,:,iblk) = p5*uprev_k(:,:,iblk) + p5*uvel(:,:,iblk) vlin(:,:,iblk) = p5*vprev_k(:,:,iblk) + p5*vvel(:,:,iblk) diff --git a/cicecore/cicedynB/dynamics/ice_transport_driver.F90 b/cicecore/cicedynB/dynamics/ice_transport_driver.F90 index 2ac99d640..c160f867f 100644 --- a/cicecore/cicedynB/dynamics/ice_transport_driver.F90 +++ b/cicecore/cicedynB/dynamics/ice_transport_driver.F90 @@ -267,12 +267,6 @@ subroutine transport_remap (dt) timer_advect, timer_bound use ice_transport_remap, only: horizontal_remap, make_masks - use ice_fileunits, only: nu_diag, flush_fileunit - use ice_communicate, only: my_task, master_task -#ifdef OMP_TIMERS - use OMP_LIB -#endif - real (kind=dbl_kind), intent(in) :: & dt ! time step @@ -337,11 +331,6 @@ subroutine transport_remap (dt) character(len=char_len_long) :: fieldid -#ifdef OMP_TIMERS - real (kind=dbl_kind) :: & - dtimes(nblocks),dtimee(nblocks) -#endif - character(len=*), parameter :: subname = '(transport_remap)' call ice_timer_start(timer_advect) ! advection @@ -408,15 +397,8 @@ subroutine transport_remap (dt) ! call ice_timer_stop(timer_bound) -#ifdef OMP_TIMERS - dtimes = 0. - dtimee = 0. -#endif !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks -#ifdef OMP_TIMERS - dtimes(iblk) = OMP_GET_WTIME() -#endif !------------------------------------------------------------------- ! Fill arrays with fields to be remapped. @@ -429,18 +411,8 @@ subroutine transport_remap (dt) vicen(:,:,:,iblk), vsnon(:,:, :,iblk), & aim (:,:,:,iblk), trm (:,:,:,:,iblk)) -#ifdef OMP_TIMERS - dtimee(iblk) = OMP_GET_WTIME() -#endif enddo !$OMP END PARALLEL DO -#ifdef OMP_TIMERS - if (my_task == master_task) then - do iblk = 1,nblocks - write(nu_diag,*) subname,'t1',iblk,dtimes(iblk),dtimee(iblk)-dtimes(iblk) - enddo - endif -#endif !---!------------------------------------------------------------------- !---! Optional conservation and monotonicity checks. @@ -498,15 +470,8 @@ subroutine transport_remap (dt) tmin(:,:,:,:,:) = c0 tmax(:,:,:,:,:) = c0 -#ifdef OMP_TIMERS - dtimes = 0. - dtimee = 0. -#endif !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,n) SCHEDULE(runtime) do iblk = 1, nblocks -#ifdef OMP_TIMERS - dtimes(iblk) = OMP_GET_WTIME() -#endif this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo ihi = this_block%ihi @@ -540,18 +505,8 @@ subroutine transport_remap (dt) tmin(:,:,:,n,iblk), tmax (:,:,:,n,iblk), & aimask(:,:,n,iblk), trmask(:,:,:,n,iblk)) enddo -#ifdef OMP_TIMERS - dtimee(iblk) = OMP_GET_WTIME() -#endif enddo !$OMP END PARALLEL DO -#ifdef OMP_TIMERS - if (my_task == master_task) then - do iblk = 1,nblocks - write(nu_diag,*) subname,'t2',iblk,dtimes(iblk),dtimee(iblk)-dtimes(iblk) - enddo - endif -#endif call ice_timer_start(timer_bound) call ice_HaloUpdate (tmin, halo_info, & @@ -560,15 +515,8 @@ subroutine transport_remap (dt) field_loc_center, field_type_scalar) call ice_timer_stop(timer_bound) -#ifdef OMP_TIMERS - dtimes = 0. - dtimee = 0. -#endif !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,n) SCHEDULE(runtime) do iblk = 1, nblocks -#ifdef OMP_TIMERS - dtimes(iblk) = OMP_GET_WTIME() -#endif this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo ihi = this_block%ihi @@ -581,18 +529,8 @@ subroutine transport_remap (dt) tmin(:,:,:,n,iblk), & tmax(:,:,:,n,iblk)) enddo -#ifdef OMP_TIMERS - dtimee(iblk) = OMP_GET_WTIME() -#endif enddo !$OMP END PARALLEL DO -#ifdef OMP_TIMERS - if (my_task == master_task) then - do iblk = 1,nblocks - write(nu_diag,*) subname,'t3',iblk,dtimes(iblk),dtimee(iblk)-dtimes(iblk) - enddo - endif -#endif endif ! l_monotonicity_check @@ -612,16 +550,9 @@ subroutine transport_remap (dt) ! Given new fields, recompute state variables. !------------------------------------------------------------------- -#ifdef OMP_TIMERS - dtimes = 0. - dtimee = 0. -#endif !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks -#ifdef OMP_TIMERS - dtimes(iblk) = OMP_GET_WTIME() -#endif call tracers_to_state (nx_block, ny_block, & ntrcr, ntrace, & aim (:,:,:,iblk), trm (:,:,:,:,iblk), & @@ -629,18 +560,8 @@ subroutine transport_remap (dt) trcrn(:,:,:,:,iblk), & vicen(:,:,:,iblk), vsnon(:,:, :,iblk)) -#ifdef OMP_TIMERS - dtimee(iblk) = OMP_GET_WTIME() -#endif enddo ! iblk !$OMP END PARALLEL DO -#ifdef OMP_TIMERS - if (my_task == master_task) then - do iblk = 1,nblocks - write(nu_diag,*) subname,'t4',iblk,dtimes(iblk),dtimee(iblk)-dtimes(iblk) - enddo - endif -#endif !------------------------------------------------------------------- ! Ghost cell updates for state variables. @@ -732,16 +653,9 @@ subroutine transport_remap (dt) ! Check tracer monotonicity. (Optional) !------------------------------------------------------------------- -#ifdef OMP_TIMERS - dtimes = 0. - dtimee = 0. -#endif if (l_monotonicity_check) then !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,n,ckflag,istop,jstop) SCHEDULE(runtime) do iblk = 1, nblocks -#ifdef OMP_TIMERS - dtimes(iblk) = OMP_GET_WTIME() -#endif this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo ihi = this_block%ihi @@ -768,18 +682,8 @@ subroutine transport_remap (dt) endif enddo ! n -#ifdef OMP_TIMERS - dtimee(iblk) = OMP_GET_WTIME() -#endif enddo ! iblk !$OMP END PARALLEL DO -#ifdef OMP_TIMERS - if (my_task == master_task) then - do iblk = 1,nblocks - write(nu_diag,*) subname,'t5',iblk,dtimes(iblk),dtimee(iblk)-dtimes(iblk) - enddo - endif -#endif deallocate(tmin, tmax, STAT=alloc_error) if (alloc_error /= 0) call abort_ice (subname//'ERROR: deallocation error') diff --git a/cicecore/cicedynB/dynamics/ice_transport_remap.F90 b/cicecore/cicedynB/dynamics/ice_transport_remap.F90 index bb36607ca..75489bd5e 100644 --- a/cicecore/cicedynB/dynamics/ice_transport_remap.F90 +++ b/cicecore/cicedynB/dynamics/ice_transport_remap.F90 @@ -331,11 +331,6 @@ subroutine horizontal_remap (dt, ntrace, & ! xyav, xxxav, xxyav, xyyav, yyyav use ice_calendar, only: istep1 use ice_timers, only: ice_timer_start, ice_timer_stop, timer_bound - use ice_fileunits, only: nu_diag, flush_fileunit - use ice_communicate, only: my_task, master_task -#ifdef OMP_TIMERS - use OMP_LIB -#endif real (kind=dbl_kind), intent(in) :: & dt ! time step @@ -453,11 +448,6 @@ subroutine horizontal_remap (dt, ntrace, & type (block) :: & this_block ! block information for current block -#ifdef OMP_TIMERS - real (kind=dbl_kind) :: & - time0,time1 -#endif - character(len=*), parameter :: subname = '(horizontal_remap)' !---!------------------------------------------------------------------- @@ -465,19 +455,6 @@ subroutine horizontal_remap (dt, ntrace, & !---! Remap the open water area (without tracers). !---!------------------------------------------------------------------- -#ifdef OMP_TIMERS - time0 = OMP_GET_WTIME() - - if (my_task == master_task) then - time1 = OMP_GET_WTIME() - write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk,time1-time0,'t1' - time0 = time1 - call flush_fileunit(nu_diag) - endif -#endif - - !--- tcraig, tcx, this omp loop leads to a seg fault in gnu - !--- need to check private variables and debug further !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,n, & !$OMP indxinc,indxjnc,mmask,tmask,istop,jstop,l_stop) & !$OMP SCHEDULE(runtime) @@ -584,15 +561,6 @@ subroutine horizontal_remap (dt, ntrace, & enddo ! iblk !$OMP END PARALLEL DO -#ifdef OMP_TIMERS - if (my_task == master_task) then - time1 = OMP_GET_WTIME() - write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk,time1-time0,'t2' - time0 = time1 - call flush_fileunit(nu_diag) - endif -#endif - !------------------------------------------------------------------- ! Ghost cell updates ! If nghost >= 2, these calls are not needed @@ -616,14 +584,6 @@ subroutine horizontal_remap (dt, ntrace, & call ice_HaloUpdate (my, halo_info, & field_loc_center, field_type_vector) -#ifdef OMP_TIMERS - if (my_task == master_task) then - time1 = OMP_GET_WTIME() - write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk,time1-time0,'t3' - time0 = time1 - call flush_fileunit(nu_diag) - endif -#endif ! tracer fields if (maskhalo_remap) then halomask(:,:,:) = 0 @@ -648,14 +608,6 @@ subroutine horizontal_remap (dt, ntrace, & enddo enddo !$OMP END PARALLEL DO -#ifdef OMP_TIMERS - if (my_task == master_task) then - time1 = OMP_GET_WTIME() - write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk,time1-time0,'t4' - time0 = time1 - call flush_fileunit(nu_diag) - endif -#endif call ice_HaloUpdate(halomask, halo_info, & field_loc_center, field_type_scalar) call ice_HaloMask(halo_info_tracer, halo_info, halomask) @@ -679,16 +631,6 @@ subroutine horizontal_remap (dt, ntrace, & endif ! nghost -#ifdef OMP_TIMERS - if (my_task == master_task) then - time1 = OMP_GET_WTIME() - write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk,time1-time0,'t5' - time0 = time1 - call flush_fileunit(nu_diag) - endif -#endif - - !--- need to check private variables and debug further !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block,n, & !$OMP edgearea_e,edgearea_n,edge,iflux,jflux, & !$OMP xp,yp,indxing,indxjng,mflxe,mflxn, & @@ -903,15 +845,6 @@ subroutine horizontal_remap (dt, ntrace, & enddo ! iblk !$OMP END PARALLEL DO -#ifdef OMP_TIMERS - if (my_task == master_task) then - time1 = OMP_GET_WTIME() - write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk,time1-time0,'t6' - time0 = time1 - call flush_fileunit(nu_diag) - endif -#endif - end subroutine horizontal_remap !======================================================================= diff --git a/cicecore/cicedynB/infrastructure/ice_grid.F90 b/cicecore/cicedynB/infrastructure/ice_grid.F90 index 18dbaaefe..2cf7775ab 100644 --- a/cicecore/cicedynB/infrastructure/ice_grid.F90 +++ b/cicecore/cicedynB/infrastructure/ice_grid.F90 @@ -358,6 +358,9 @@ subroutine init_grid2 field_loc_center, field_loc_NEcorner, & field_type_scalar, field_type_vector, field_type_angle use ice_domain_size, only: max_blocks +#if defined (_OPENMP) + use OMP_LIB +#endif integer (kind=int_kind) :: & i, j, iblk, & @@ -376,6 +379,11 @@ subroutine init_grid2 type (block) :: & this_block ! block information for current block +#if defined (_OPENMP) + integer(kind=omp_sched_kind) :: ompsk ! openmp schedule + integer(kind=int_kind) :: ompcs ! openmp schedule count +#endif + character(len=*), parameter :: subname = '(init_grid2)' !----------------------------------------------------------------- @@ -406,6 +414,29 @@ subroutine init_grid2 call rectgrid ! regular rectangular grid endif + !----------------------------------------------------------------- + ! Diagnose OpenMP thread schedule, force order in output + !----------------------------------------------------------------- + +#if defined (_OPENMP) + !$OMP PARALLEL DO ORDERED PRIVATE(iblk) SCHEDULE(runtime) + do iblk = 1, nblocks + if (my_task == master_task) then + !$OMP ORDERED + if (iblk == 1) then + call omp_get_schedule(ompsk,ompcs) + write(nu_diag,*) '' + write(nu_diag,*) subname,' OpenMP runtime thread schedule:' + write(nu_diag,*) subname,' omp schedule = ',ompsk,ompcs + endif + write(nu_diag,*) subname,' block, thread = ',iblk,OMP_GET_THREAD_NUM() + call flush_fileunit(nu_diag) + !$OMP END ORDERED + endif + enddo + !$OMP END PARALLEL DO +#endif + !----------------------------------------------------------------- ! T-grid cell and U-grid cell quantities !----------------------------------------------------------------- diff --git a/cicecore/drivers/standalone/cice/CICE_RunMod.F90 b/cicecore/drivers/standalone/cice/CICE_RunMod.F90 index 3557177fc..aef6a1ec1 100644 --- a/cicecore/drivers/standalone/cice/CICE_RunMod.F90 +++ b/cicecore/drivers/standalone/cice/CICE_RunMod.F90 @@ -15,9 +15,8 @@ module CICE_RunMod use ice_kinds_mod - use ice_fileunits, only: nu_diag, flush_fileunit + use ice_fileunits, only: nu_diag use ice_arrays_column, only: oceanmixed_ice - use ice_communicate, only: my_task, master_task, ice_barrier use ice_constants, only: c0, c1 use ice_constants, only: field_loc_center, field_type_scalar use ice_exit, only: abort_ice @@ -161,11 +160,7 @@ subroutine ice_step biogeochemistry, save_init, step_dyn_wave, step_snow use ice_timers, only: ice_timer_start, ice_timer_stop, & timer_diags, timer_column, timer_thermo, timer_bound, & - timer_hist, timer_readwrite, & - timer_tmp1, timer_tmp2, timer_tmp3, timer_tmp4, timer_tmp5, timer_tmp6 -#ifdef OMP_TIMERS - use OMP_LIB -#endif + timer_hist, timer_readwrite integer (kind=int_kind) :: & iblk , & ! block index @@ -175,13 +170,6 @@ subroutine ice_step real (kind=dbl_kind) :: & offset ! d(age)/dt time offset -#ifdef OMP_TIMERS - real (kind=dbl_kind) :: & - dtimes(nblocks,9), dtimee(nblocks,9) - integer(kind=omp_sched_kind) :: ompsk - integer :: ompcs, itim -#endif - logical (kind=log_kind) :: & tr_iage, tr_FY, tr_lvl, tr_fsd, tr_snow, & tr_pond_cesm, tr_pond_lvl, tr_pond_topo, tr_brine, tr_iso, tr_aero, & @@ -225,29 +213,11 @@ subroutine ice_step call init_history_bgc call ice_timer_stop(timer_diags) ! diagnostics/history -#ifdef OMP_TIMERS - call ice_barrier() -#endif call ice_timer_start(timer_column) ! column physics call ice_timer_start(timer_thermo) ! thermodynamics call save_init -#ifdef OMP_TIMERS - dtimes = 0. - dtimee = 0. - !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) - do iblk = 1, nblocks - if (my_task == master_task) then - call omp_get_schedule(ompsk,ompcs) - if (iblk == 1) write(nu_diag,*) subname,'tcx omp3',ompsk,ompcs - write(nu_diag,*) trim(subname),OMP_GET_THREAD_NUM(),iblk - call flush_fileunit(nu_diag) - endif - enddo - !$OMP END PARALLEL DO -#endif - if (ktherm >= 0) then !$OMP PARALLEL DO PRIVATE(iblk) SCHEDULE(runtime) do iblk = 1, nblocks @@ -256,15 +226,7 @@ subroutine ice_step ! scale radiation fields !----------------------------------------------------------------- -#ifdef OMP_TIMERS - dtimes(iblk,1) = OMP_GET_WTIME() - call ice_timer_start(timer_tmp1,iblk) -#endif if (calc_Tsfc) call prep_radiation (iblk) -#ifdef OMP_TIMERS - call ice_timer_stop(timer_tmp1,iblk) - dtimee(iblk,1) = OMP_GET_WTIME() -#endif if (debug_model) then plabeld = 'post prep_radiation' @@ -275,45 +237,21 @@ subroutine ice_step ! thermodynamics and biogeochemistry !----------------------------------------------------------------- -#ifdef OMP_TIMERS - dtimes(iblk,2) = OMP_GET_WTIME() - call ice_timer_start(timer_tmp2,iblk) -#endif call step_therm1 (dt, iblk) ! vertical thermodynamics -#ifdef OMP_TIMERS - call ice_timer_stop(timer_tmp2,iblk) - dtimee(iblk,2) = OMP_GET_WTIME() -#endif if (debug_model) then plabeld = 'post step_therm1' call debug_ice (iblk, plabeld) endif -#ifdef OMP_TIMERS - dtimes(iblk,3) = OMP_GET_WTIME() - call ice_timer_start(timer_tmp3,iblk) -#endif call biogeochemistry (dt, iblk) ! biogeochemistry -#ifdef OMP_TIMERS - call ice_timer_stop(timer_tmp3,iblk) - dtimee(iblk,3) = OMP_GET_WTIME() -#endif if (debug_model) then plabeld = 'post biogeochemistry' call debug_ice (iblk, plabeld) endif -#ifdef OMP_TIMERS - dtimes(iblk,4) = OMP_GET_WTIME() - call ice_timer_start(timer_tmp4,iblk) -#endif call step_therm2 (dt, iblk) ! ice thickness distribution thermo -#ifdef OMP_TIMERS - call ice_timer_stop(timer_tmp4,iblk) - dtimee(iblk,4) = OMP_GET_WTIME() -#endif if (debug_model) then plabeld = 'post step_therm2' @@ -322,7 +260,6 @@ subroutine ice_step enddo !$OMP END PARALLEL DO - endif ! ktherm > 0 ! clean up, update tendency diagnostics @@ -332,10 +269,6 @@ subroutine ice_step call ice_timer_stop(timer_thermo) ! thermodynamics call ice_timer_stop(timer_column) ! column physics -#ifdef OMP_TIMERS - call ice_barrier() -#endif - !----------------------------------------------------------------- ! dynamics, transport, ridging !----------------------------------------------------------------- @@ -383,9 +316,6 @@ subroutine ice_step enddo endif -#ifdef OMP_TIMERS - call ice_barrier() -#endif call ice_timer_start(timer_column) ! column physics call ice_timer_start(timer_thermo) ! thermodynamics @@ -409,15 +339,7 @@ subroutine ice_step ! albedo, shortwave radiation !----------------------------------------------------------------- -#ifdef OMP_TIMERS - dtimes(iblk,5) = OMP_GET_WTIME() - call ice_timer_start(timer_tmp5,iblk) -#endif if (ktherm >= 0) call step_radiation (dt, iblk) -#ifdef OMP_TIMERS - call ice_timer_stop(timer_tmp5,iblk) - dtimee(iblk,5) = OMP_GET_WTIME() -#endif if (debug_model) then plabeld = 'post step_radiation' @@ -428,22 +350,14 @@ subroutine ice_step ! get ready for coupling and the next time step !----------------------------------------------------------------- -#ifdef OMP_TIMERS - dtimes(iblk,6) = OMP_GET_WTIME() - call ice_timer_start(timer_tmp6,iblk) -#endif call coupling_prep (iblk) -#ifdef OMP_TIMERS - call ice_timer_stop(timer_tmp6,iblk) - dtimee(iblk,6) = OMP_GET_WTIME() -#endif if (debug_model) then plabeld = 'post coupling_prep' call debug_ice (iblk, plabeld) endif - enddo + enddo ! iblk !$OMP END PARALLEL DO call ice_timer_start(timer_bound) @@ -454,15 +368,6 @@ subroutine ice_step call ice_timer_stop(timer_thermo) ! thermodynamics call ice_timer_stop(timer_column) ! column physics -#ifdef OMP_TIMERS - do itim = 1,6 - do iblk = 1,nblocks - write(nu_diag,'(2a,2i6,2g15.7)') subname,' omptime',itim,iblk,dtimes(iblk,itim),dtimee(iblk,itim)-dtimes(iblk,itim) - enddo - enddo - call ice_barrier() -#endif - !----------------------------------------------------------------- ! write data !----------------------------------------------------------------- @@ -480,7 +385,6 @@ subroutine ice_step call accum_hist (dt) ! history file call ice_timer_stop(timer_hist) ! history - call ice_barrier() call ice_timer_start(timer_readwrite) ! reading/writing if (write_restart == 1) then call dumpfile ! core variables for restarting From 667e998c9ac61bf66eb781659e0385fd693aecf9 Mon Sep 17 00:00:00 2001 From: apcraig Date: Wed, 12 Jan 2022 10:53:05 -0700 Subject: [PATCH 08/14] Comment out temporary timers --- .../infrastructure/comm/mpi/ice_timers.F90 | 39 ++++++++++--------- .../infrastructure/comm/serial/ice_timers.F90 | 39 ++++++++++--------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90 b/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90 index cdbe72ef2..bc14e30d3 100644 --- a/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90 +++ b/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90 @@ -66,16 +66,17 @@ module ice_timers timer_forcing, &! forcing timer_evp_1d, &! timer only loop timer_evp_2d, &! timer including conversion 1d/2d - timer_updstate, &! update state - timer_tmp1, &! for temporary timings - timer_tmp2, &! for temporary timings - timer_tmp3, &! for temporary timings - timer_tmp4, &! for temporary timings - timer_tmp5, &! for temporary timings - timer_tmp6, &! for temporary timings - timer_tmp7, &! for temporary timings - timer_tmp8, &! for temporary timings - timer_tmp9 ! for temporary timings + timer_updstate ! update state +! timer_updstate, &! update state +! timer_tmp1, &! for temporary timings +! timer_tmp2, &! for temporary timings +! timer_tmp3, &! for temporary timings +! timer_tmp4, &! for temporary timings +! timer_tmp5, &! for temporary timings +! timer_tmp6, &! for temporary timings +! timer_tmp7, &! for temporary timings +! timer_tmp8, &! for temporary timings +! timer_tmp9 ! for temporary timings !----------------------------------------------------------------------- ! @@ -202,15 +203,15 @@ subroutine init_ice_timers call get_ice_timer(timer_evp_1d, '1d-evp', nblocks,distrb_info%nprocs) call get_ice_timer(timer_evp_2d, '2d-evp', nblocks,distrb_info%nprocs) call get_ice_timer(timer_updstate, 'UpdState', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp1, 'tmp1', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp2, 'tmp2', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp3, 'tmp3', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp4, 'tmp4', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp5, 'tmp5', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp6, 'tmp6', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp7, 'tmp7', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp8, 'tmp8', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp9, 'tmp9', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp1, 'tmp1', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp2, 'tmp2', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp3, 'tmp3', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp4, 'tmp4', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp5, 'tmp5', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp6, 'tmp6', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp7, 'tmp7', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp8, 'tmp8', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp9, 'tmp9', nblocks,distrb_info%nprocs) !----------------------------------------------------------------------- diff --git a/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90 b/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90 index 41c511ea5..b18c35040 100644 --- a/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90 +++ b/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90 @@ -58,16 +58,17 @@ module ice_timers timer_forcing, &! forcing timer_evp_1d, &! timer only loop timer_evp_2d, &! timer including conversion 1d/2d - timer_updstate, &! update state - timer_tmp1, &! for temporary timings - timer_tmp2, &! for temporary timings - timer_tmp3, &! for temporary timings - timer_tmp4, &! for temporary timings - timer_tmp5, &! for temporary timings - timer_tmp6, &! for temporary timings - timer_tmp7, &! for temporary timings - timer_tmp8, &! for temporary timings - timer_tmp9 ! for temporary timings + timer_updstate ! update state +! timer_updstate, &! update state +! timer_tmp1, &! for temporary timings +! timer_tmp2, &! for temporary timings +! timer_tmp3, &! for temporary timings +! timer_tmp4, &! for temporary timings +! timer_tmp5, &! for temporary timings +! timer_tmp6, &! for temporary timings +! timer_tmp7, &! for temporary timings +! timer_tmp8, &! for temporary timings +! timer_tmp9 ! for temporary timings !----------------------------------------------------------------------- ! @@ -210,15 +211,15 @@ subroutine init_ice_timers call get_ice_timer(timer_evp_1d, '1d-evp', nblocks,distrb_info%nprocs) call get_ice_timer(timer_evp_2d, '2d-evp', nblocks,distrb_info%nprocs) call get_ice_timer(timer_updstate, 'UpdState', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp1, 'tmp1', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp2, 'tmp2', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp3, 'tmp3', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp4, 'tmp4', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp5, 'tmp5', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp6, 'tmp6', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp7, 'tmp7', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp8, 'tmp8', nblocks,distrb_info%nprocs) - call get_ice_timer(timer_tmp9, 'tmp9', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp1, 'tmp1', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp2, 'tmp2', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp3, 'tmp3', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp4, 'tmp4', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp5, 'tmp5', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp6, 'tmp6', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp7, 'tmp7', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp8, 'tmp8', nblocks,distrb_info%nprocs) +! call get_ice_timer(timer_tmp9, 'tmp9', nblocks,distrb_info%nprocs) !----------------------------------------------------------------------- From 6bb8e22efa8fec5846d5434c519d5274996a9a6e Mon Sep 17 00:00:00 2001 From: apcraig Date: Thu, 13 Jan 2022 02:35:25 +0000 Subject: [PATCH 09/14] Update OMP env variables on Narwhal --- configuration/scripts/machines/env.narwhal_aocc | 2 ++ configuration/scripts/machines/env.narwhal_cray | 3 ++- configuration/scripts/machines/env.narwhal_gnu | 2 ++ configuration/scripts/machines/env.narwhal_intel | 2 ++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/configuration/scripts/machines/env.narwhal_aocc b/configuration/scripts/machines/env.narwhal_aocc index a392f9363..6d6822f46 100755 --- a/configuration/scripts/machines/env.narwhal_aocc +++ b/configuration/scripts/machines/env.narwhal_aocc @@ -33,6 +33,8 @@ module load cray-hdf5/1.12.0.4 setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited limit stacksize unlimited +setenv OMP_STACKSIZE 128M +setenv OMP_WAIT_POLICY PASSIVE endif diff --git a/configuration/scripts/machines/env.narwhal_cray b/configuration/scripts/machines/env.narwhal_cray index eb9e42bb2..d0fcc9ba7 100755 --- a/configuration/scripts/machines/env.narwhal_cray +++ b/configuration/scripts/machines/env.narwhal_cray @@ -33,7 +33,8 @@ module load cray-hdf5/1.12.0.4 setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited limit stacksize unlimited -setenv OMP_WAIT_POLICY passive +setenv OMP_STACKSIZE 128M +setenv OMP_WAIT_POLICY PASSIVE endif diff --git a/configuration/scripts/machines/env.narwhal_gnu b/configuration/scripts/machines/env.narwhal_gnu index 4df81b957..51a272f4e 100755 --- a/configuration/scripts/machines/env.narwhal_gnu +++ b/configuration/scripts/machines/env.narwhal_gnu @@ -33,6 +33,8 @@ module load cray-hdf5/1.12.0.4 setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited limit stacksize unlimited +setenv OMP_STACKSIZE 128M +setenv OMP_WAIT_POLICY PASSIVE endif diff --git a/configuration/scripts/machines/env.narwhal_intel b/configuration/scripts/machines/env.narwhal_intel index 2cdf4f93c..f79d962ff 100755 --- a/configuration/scripts/machines/env.narwhal_intel +++ b/configuration/scripts/machines/env.narwhal_intel @@ -33,6 +33,8 @@ module load cray-hdf5/1.12.0.4 setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited limit stacksize unlimited +setenv OMP_STACKSIZE 128M +setenv OMP_WAIT_POLICY PASSIVE endif From 0d4c2e39f4308c89a1905b7e05ba2dd723828b2d Mon Sep 17 00:00:00 2001 From: apcraig Date: Fri, 14 Jan 2022 02:21:01 +0000 Subject: [PATCH 10/14] Update gaffney OMP_STACKSIZE --- configuration/scripts/machines/env.gaffney_gnu | 1 + configuration/scripts/machines/env.gaffney_intel | 1 + 2 files changed, 2 insertions(+) diff --git a/configuration/scripts/machines/env.gaffney_gnu b/configuration/scripts/machines/env.gaffney_gnu index a63ee2ae4..dd889c5af 100755 --- a/configuration/scripts/machines/env.gaffney_gnu +++ b/configuration/scripts/machines/env.gaffney_gnu @@ -24,6 +24,7 @@ setenv MPI_DSM_DISTRIBUTE 0 setenv KMP_AFFINITY disabled limit coredumpsize unlimited limit stacksize unlimited +setenv OMP_STACKSIZE 64M endif diff --git a/configuration/scripts/machines/env.gaffney_intel b/configuration/scripts/machines/env.gaffney_intel index 9fa11d16e..c7fd0f6b3 100755 --- a/configuration/scripts/machines/env.gaffney_intel +++ b/configuration/scripts/machines/env.gaffney_intel @@ -24,6 +24,7 @@ setenv MPI_DSM_DISTRIBUTE 0 setenv KMP_AFFINITY disabled limit coredumpsize unlimited limit stacksize unlimited +setenv OMP_STACKSIZE 64M endif From d56737c27d4aa0b114b89d1d2dcba97f0319ec59 Mon Sep 17 00:00:00 2001 From: apcraig Date: Thu, 13 Jan 2022 19:33:26 -0800 Subject: [PATCH 11/14] update OMP_STACKSIZE on cori --- configuration/scripts/machines/env.cori_intel | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/scripts/machines/env.cori_intel b/configuration/scripts/machines/env.cori_intel index ed01928f4..734b2edf3 100755 --- a/configuration/scripts/machines/env.cori_intel +++ b/configuration/scripts/machines/env.cori_intel @@ -39,6 +39,7 @@ module load craype/2.6.2 setenv NETCDF_PATH ${NETCDF_DIR} setenv OMP_PROC_BIND true setenv OMP_PLACES threads +setenv OMP_STACKSIZE 32M limit coredumpsize unlimited limit stacksize unlimited From 021e0f343a998c41f3d450d718471d48e98c07d1 Mon Sep 17 00:00:00 2001 From: apcraig Date: Fri, 14 Jan 2022 11:10:35 -0600 Subject: [PATCH 12/14] Update Onyx OMP_STACKSIZE Update documentation --- configuration/scripts/machines/env.onyx_cray | 1 + configuration/scripts/machines/env.onyx_gnu | 1 + configuration/scripts/machines/env.onyx_intel | 1 + doc/source/user_guide/ug_implementation.rst | 18 ++++++++++++++++-- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/configuration/scripts/machines/env.onyx_cray b/configuration/scripts/machines/env.onyx_cray index 38785a27d..e696d1b98 100755 --- a/configuration/scripts/machines/env.onyx_cray +++ b/configuration/scripts/machines/env.onyx_cray @@ -39,6 +39,7 @@ module load craype-broadwell setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited limit stacksize unlimited +setenv OMP_STACKSIZE 64M endif diff --git a/configuration/scripts/machines/env.onyx_gnu b/configuration/scripts/machines/env.onyx_gnu index 699c01559..80ebb8e43 100755 --- a/configuration/scripts/machines/env.onyx_gnu +++ b/configuration/scripts/machines/env.onyx_gnu @@ -39,6 +39,7 @@ module load craype-broadwell setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited limit stacksize unlimited +setenv OMP_STACKSIZE 64M endif diff --git a/configuration/scripts/machines/env.onyx_intel b/configuration/scripts/machines/env.onyx_intel index 39f25e8e5..362454dd4 100755 --- a/configuration/scripts/machines/env.onyx_intel +++ b/configuration/scripts/machines/env.onyx_intel @@ -39,6 +39,7 @@ module load craype-broadwell setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited limit stacksize unlimited +setenv OMP_STACKSIZE 64M endif diff --git a/doc/source/user_guide/ug_implementation.rst b/doc/source/user_guide/ug_implementation.rst index 66b145762..a74e13ecf 100644 --- a/doc/source/user_guide/ug_implementation.rst +++ b/doc/source/user_guide/ug_implementation.rst @@ -169,7 +169,8 @@ and chooses a block size ``block_size_x`` :math:`\times`\ ``block_size_y``, and ``distribution_type`` in **ice\_in**. That information is used to determine how the blocks are distributed across the processors, and how the processors are -distributed across the grid domain. Recommended combinations of these +distributed across the grid domain. The model is parallelized over blocks +for both MPI and OpenMP. Some suggested combinations for these parameters for best performance are given in Section :ref:`performance`. The script **cice.setup** computes some default decompositions and layouts but the user can overwrite the defaults by manually changing the values in @@ -384,7 +385,8 @@ The user specifies the total number of tasks and threads in **cice.settings** and the block size and decompostion in the namelist file. The main trades offs are the relative efficiency of large square blocks versus model internal load balance -as CICE computation cost is very small for ice-free blocks. +as CICE computation cost is very small for ice-free blocks. The code +is parallelized over blocks for both MPI and OpenMP. Smaller, more numerous blocks provides an opportunity for better load balance by allocating each processor both ice-covered and ice-free blocks. But smaller, more numerous blocks becomes @@ -395,6 +397,18 @@ volume-to-surface ratio important for communication cost. Often 3 to 8 blocks per processor provide the decompositions flexiblity to create reasonable load balance configurations. +Like MPI, load balance +of blocks across threads is important for efficient performance. Most of the OpenMP +threading is implemented with ``SCHEDULE(runtime)``, so the OMP_SCHEDULE env +variable can be used to set the OpenMPI schedule. The default ``OMP_SCHEDULE`` +setting is defined by the +variable ``ICE_OMPSCHE`` in **cice.settings**. ``OMP_SCHEDULE`` values of "STATIC,1" +and "DYNAMIC,1" are worth testing. The OpenMP implementation in +CICE is constantly under review, but users should validate results and +performance on their machine. CICE should be bit-for-bit with different block sizes, +different decompositions, different MPI task counts, and different OpenMP threads. +Finally, we recommend the ``OMP_STACKSIZE`` env variable should be set to 32M or greater. + The ``distribution_type`` options allow standard cartesian distributions of blocks, redistribution via a ‘rake’ algorithm for improved load balancing across processors, and redistribution based on space-filling From 3a00b69faf0aabb733a43e9a10f24945c69bec8b Mon Sep 17 00:00:00 2001 From: apcraig Date: Mon, 17 Jan 2022 18:01:58 -0500 Subject: [PATCH 13/14] Update OMP_STACKSIZE on mustang --- configuration/scripts/machines/env.mustang_intel18 | 2 +- configuration/scripts/machines/env.mustang_intel19 | 2 +- configuration/scripts/machines/env.mustang_intel20 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configuration/scripts/machines/env.mustang_intel18 b/configuration/scripts/machines/env.mustang_intel18 index f420ec7ff..45e5b6518 100755 --- a/configuration/scripts/machines/env.mustang_intel18 +++ b/configuration/scripts/machines/env.mustang_intel18 @@ -21,7 +21,7 @@ module load netcdf-fortran/intel/4.4.2 setenv NETCDF_PATH /app/COST/netcdf-fortran/4.4.2/intel -#setenv OMP_STACKSIZE 256M +setenv OMP_STACKSIZE 64M #setenv MP_LABELIO yes #setenv MP_INFOLEVEL 2 #setenv MP_SHARED_MEMORY yes diff --git a/configuration/scripts/machines/env.mustang_intel19 b/configuration/scripts/machines/env.mustang_intel19 index 0fc0458fd..438bc1111 100755 --- a/configuration/scripts/machines/env.mustang_intel19 +++ b/configuration/scripts/machines/env.mustang_intel19 @@ -21,7 +21,7 @@ module load netcdf-fortran/intel/4.4.2 setenv NETCDF_PATH /app/COST/netcdf-fortran/4.4.2/intel -#setenv OMP_STACKSIZE 256M +setenv OMP_STACKSIZE 64M #setenv MP_LABELIO yes #setenv MP_INFOLEVEL 2 #setenv MP_SHARED_MEMORY yes diff --git a/configuration/scripts/machines/env.mustang_intel20 b/configuration/scripts/machines/env.mustang_intel20 index 00c4a250d..cca0b3019 100755 --- a/configuration/scripts/machines/env.mustang_intel20 +++ b/configuration/scripts/machines/env.mustang_intel20 @@ -21,7 +21,7 @@ module load netcdf-fortran/intel/4.4.2 setenv NETCDF_PATH /app/COST/netcdf-fortran/4.4.2/intel -#setenv OMP_STACKSIZE 256M +setenv OMP_STACKSIZE 64M #setenv MP_LABELIO yes #setenv MP_INFOLEVEL 2 #setenv MP_SHARED_MEMORY yes From c8b9f17ff8f08536da0e3bc33230f89adc430a22 Mon Sep 17 00:00:00 2001 From: apcraig Date: Wed, 19 Jan 2022 00:52:54 +0000 Subject: [PATCH 14/14] - Update Tsfc values on land in various places in the code, was affecting testing. Specifically fix upwind advection. - Comment out OMP in ice_dyn_evp_1d_kernel, was producing non bit-for-bit results with different thread counts --- cicecore/cicedynB/analysis/ice_history.F90 | 1 - cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 | 17 ++++++----- .../dynamics/ice_transport_driver.F90 | 28 ++++++++++++++++++- cicecore/cicedynB/general/ice_init.F90 | 7 +++-- .../infrastructure/ice_restart_driver.F90 | 11 ++++++++ .../cicedynB/infrastructure/ice_restoring.F90 | 6 +++- 6 files changed, 58 insertions(+), 12 deletions(-) diff --git a/cicecore/cicedynB/analysis/ice_history.F90 b/cicecore/cicedynB/analysis/ice_history.F90 index dea07af63..c32f4b78d 100644 --- a/cicecore/cicedynB/analysis/ice_history.F90 +++ b/cicecore/cicedynB/analysis/ice_history.F90 @@ -1889,7 +1889,6 @@ subroutine accum_hist (dt) ! increment field !--------------------------------------------------------------- -! MHRI: CHECK THIS OMP ... Maybe ok after "dfresh,dfsalt" added !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block, & !$OMP k,n,qn,ns,sn,rho_ocn,rho_ice,Tice,Sbr,phi,rhob,dfresh,dfsalt, & !$OMP worka,workb,worka3,Tinz4d,Sinz4d,Tsnz4d) diff --git a/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 b/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 index c691453cb..99e0aee85 100755 --- a/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 @@ -1306,7 +1306,10 @@ subroutine ice_dyn_evp_1d_kernel if (ndte < 2) call abort_ice(subname & // ' ERROR: ndte must be 2 or higher for this kernel') - !$OMP PARALLEL PRIVATE(ksub) + ! tcraig, turn off the OMP directives here, Jan, 2022 + ! This produces non bit-for-bit results with different thread counts. + ! Seems like there isn't an opportunity for safe threading here ??? + !$XXXOMP PARALLEL PRIVATE(ksub) do ksub = 1, ndte - 1 call evp1d_stress(NA_len, ee, ne, se, 1, NA_len, uvel, & vvel, dxt, dyt, hte, htn, htem1, htnm1, strength, & @@ -1314,15 +1317,15 @@ subroutine ice_dyn_evp_1d_kernel stressm_2, stressm_3, stressm_4, stress12_1, & stress12_2, stress12_3, stress12_4, str1, str2, str3, & str4, str5, str6, str7, str8, skiptcell) - !$OMP BARRIER + !$XXXOMP BARRIER call evp1d_stepu(NA_len, rhow, 1, NA_len, cdn_ocn, aiu, & uocn, vocn, forcex, forcey, umassdti, fm, uarear, Tbu, & uvel_init, vvel_init, uvel, vvel, str1, str2, str3, & str4, str5, str6, str7, str8, nw, sw, sse, skipucell) - !$OMP BARRIER + !$XXXOMP BARRIER call evp1d_halo_update(NAVEL_len, 1, NA_len, uvel, vvel, & halo_parent) - !$OMP BARRIER + !$XXXOMP BARRIER end do call evp1d_stress(NA_len, ee, ne, se, 1, NA_len, uvel, vvel, & @@ -1331,16 +1334,16 @@ subroutine ice_dyn_evp_1d_kernel stressm_3, stressm_4, stress12_1, stress12_2, stress12_3, & stress12_4, str1, str2, str3, str4, str5, str6, str7, & str8, skiptcell, tarear, divu, rdg_conv, rdg_shear, shear) - !$OMP BARRIER + !$XXXOMP BARRIER call evp1d_stepu(NA_len, rhow, 1, NA_len, cdn_ocn, aiu, uocn, & vocn, forcex, forcey, umassdti, fm, uarear, Tbu, & uvel_init, vvel_init, uvel, vvel, str1, str2, str3, str4, & str5, str6, str7, str8, nw, sw, sse, skipucell, strintx, & strinty, taubx, tauby) - !$OMP BARRIER + !$XXXOMP BARRIER call evp1d_halo_update(NAVEL_len, 1, NA_len, uvel, vvel, & halo_parent) - !$OMP END PARALLEL + !$XXXOMP END PARALLEL end if ! master task diff --git a/cicecore/cicedynB/dynamics/ice_transport_driver.F90 b/cicecore/cicedynB/dynamics/ice_transport_driver.F90 index c160f867f..ac88d8ff4 100644 --- a/cicecore/cicedynB/dynamics/ice_transport_driver.F90 +++ b/cicecore/cicedynB/dynamics/ice_transport_driver.F90 @@ -17,6 +17,7 @@ module ice_transport_driver use ice_constants, only: c0, c1, p5, & field_loc_center, & field_type_scalar, field_type_vector, & + field_loc_NEcorner, & field_loc_Nface, field_loc_Eface use ice_fileunits, only: nu_diag use ice_exit, only: abort_ice @@ -708,7 +709,7 @@ subroutine transport_upwind (dt) use ice_state, only: aice0, aicen, vicen, vsnon, trcrn, & uvel, vvel, trcr_depend, bound_state, trcr_base, & n_trcr_strata, nt_strata - use ice_grid, only: HTE, HTN, tarea + use ice_grid, only: HTE, HTN, tarea, tmask use ice_timers, only: ice_timer_start, ice_timer_stop, & timer_bound, timer_advect @@ -756,6 +757,13 @@ subroutine transport_upwind (dt) ! vicen, vsnon, & ! ntrcr, trcrn) +! call ice_timer_start(timer_bound) +! call ice_HaloUpdate (uvel, halo_info, & +! field_loc_NEcorner, field_type_vector) +! call ice_HaloUpdate (vvel, halo_info, & +! field_loc_NEcorner, field_type_vector) +! call ice_timer_stop(timer_bound) + !------------------------------------------------------------------- ! Average corner velocities to edges. !------------------------------------------------------------------- @@ -824,6 +832,7 @@ subroutine transport_upwind (dt) ntrcr, narr, & trcr_depend(:), trcr_base(:,:), & n_trcr_strata(:), nt_strata(:,:), & + tmask(:,:, iblk), & aicen(:,:, :,iblk), trcrn (:,:,:,:,iblk), & vicen(:,:, :,iblk), vsnon (:,:, :,iblk), & aice0(:,:, iblk), works (:,:, :,iblk)) @@ -1628,6 +1637,7 @@ subroutine work_to_state (nx_block, ny_block, & trcr_base, & n_trcr_strata, & nt_strata, & + tmask, & aicen, trcrn, & vicen, vsnon, & aice0, works) @@ -1650,6 +1660,9 @@ subroutine work_to_state (nx_block, ny_block, & integer (kind=int_kind), dimension (ntrcr,2), intent(in) :: & nt_strata ! indices of underlying tracer layers + logical (kind=log_kind), intent (in) :: & + tmask (nx_block,ny_block) + real (kind=dbl_kind), intent (in) :: & works (nx_block,ny_block,narr) @@ -1669,6 +1682,7 @@ subroutine work_to_state (nx_block, ny_block, & integer (kind=int_kind) :: & i, j, ij, n ,&! counting indices narrays ,&! counter for number of state variable arrays + nt_Tsfc ,&! Tsfc tracer number icells ! number of ocean/ice cells integer (kind=int_kind), dimension (nx_block*ny_block) :: & @@ -1679,6 +1693,11 @@ subroutine work_to_state (nx_block, ny_block, & character(len=*), parameter :: subname = '(work_to_state)' + call icepack_query_tracer_indices(nt_Tsfc_out=nt_Tsfc) + call icepack_warnings_flush(nu_diag) + if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & + file=__FILE__, line=__LINE__) + ! for call to compute_tracers icells = 0 do j = 1, ny_block @@ -1721,7 +1740,14 @@ subroutine work_to_state (nx_block, ny_block, & n_trcr_strata = n_trcr_strata(:), & nt_strata = nt_strata(:,:), & trcrn = trcrn(i,j,:,n)) + + ! tcraig, don't let land points get non-zero Tsfc + if (.not.tmask(i,j)) then + trcrn(i,j,nt_Tsfc,n) = c0 + endif + enddo + narrays = narrays + ntrcr enddo ! ncat diff --git a/cicecore/cicedynB/general/ice_init.F90 b/cicecore/cicedynB/general/ice_init.F90 index 123f51f26..0275ca67c 100644 --- a/cicecore/cicedynB/general/ice_init.F90 +++ b/cicecore/cicedynB/general/ice_init.F90 @@ -2413,7 +2413,6 @@ subroutine init_state ! Set state variables !----------------------------------------------------------------- -!MHRI: CHECK THIS OMP !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block, & !$OMP iglob,jglob) do iblk = 1, nblocks @@ -2627,7 +2626,11 @@ subroutine set_state_var (nx_block, ny_block, & aicen(i,j,n) = c0 vicen(i,j,n) = c0 vsnon(i,j,n) = c0 - trcrn(i,j,nt_Tsfc,n) = Tf(i,j) ! surface temperature + if (tmask(i,j)) then + trcrn(i,j,nt_Tsfc,n) = Tf(i,j) ! surface temperature + else + trcrn(i,j,nt_Tsfc,n) = c0 ! at land grid cells (for clean history/restart files) + endif if (ntrcr >= 2) then do it = 2, ntrcr trcrn(i,j,it,n) = c0 diff --git a/cicecore/cicedynB/infrastructure/ice_restart_driver.F90 b/cicecore/cicedynB/infrastructure/ice_restart_driver.F90 index 1a5681b38..ebbef60e0 100644 --- a/cicecore/cicedynB/infrastructure/ice_restart_driver.F90 +++ b/cicecore/cicedynB/infrastructure/ice_restart_driver.F90 @@ -416,6 +416,17 @@ subroutine restartfile (ice_ic) enddo !$OMP END PARALLEL DO + ! set Tsfcn to c0 on land + !$OMP PARALLEL DO PRIVATE(iblk,i,j) + do iblk = 1, nblocks + do j = 1, ny_block + do i = 1, nx_block + if (.not. tmask(i,j,iblk)) trcrn(i,j,nt_Tsfc,:,iblk) = c0 + enddo + enddo + enddo + !$OMP END PARALLEL DO + ! for mixed layer model if (oceanmixed_ice) then diff --git a/cicecore/cicedynB/infrastructure/ice_restoring.F90 b/cicecore/cicedynB/infrastructure/ice_restoring.F90 index c7254cd80..f21e50513 100644 --- a/cicecore/cicedynB/infrastructure/ice_restoring.F90 +++ b/cicecore/cicedynB/infrastructure/ice_restoring.F90 @@ -394,7 +394,11 @@ subroutine set_restore_var (nx_block, ny_block, & aicen(i,j,n) = c0 vicen(i,j,n) = c0 vsnon(i,j,n) = c0 - trcrn(i,j,nt_Tsfc,n) = Tf(i,j) ! surface temperature + if (tmask(i,j)) then + trcrn(i,j,nt_Tsfc,n) = Tf(i,j) ! surface temperature + else + trcrn(i,j,nt_Tsfc,n) = c0 ! on land gridcells + endif if (ntrcr >= 2) then do it = 2, ntrcr trcrn(i,j,it,n) = c0