Merge pull request #33 from NOAA-GFDL/dev/gfdl

Merge in latest updates

.testing/Makefile

@@ -1,8 +1,12 @@
 SHELL = bash
-MPIRUN ?= mpirun
 
+# User-defined configuration
 -include config.mk
 
+# Default configurations
+MPIRUN ?= mpirun
+DO_REPRO_TESTS ?= true
+
 #---
 # Dependencies
 BASE = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/..
@@ -38,36 +42,45 @@ MKMF_TEMPLATE ?= $(DEPS)/mkmf/templates/ncrc-gnu.mk
 # Executables
 BUILDS = symmetric asymmetric repro openmp
 CONFIGS := $(wildcard tc*)
-TESTS = grids layouts restarts repros nans dims
+TESTS = grids layouts restarts nans dims openmps
+
+# REPRO tests enable reproducibility with optimization, and often do not match
+# the DEBUG results in older GCCs and vendor compilers, so we can optionally
+# disable them.
+ifeq ($(DO_REPRO_TESTS), true)
+  BUILDS += repro
+  TESTS += repros
+endif
 
 # The following variables are configured by Travis:
 #   DO_REGRESSION_TESTS: true if $(TRAVIS_PULL_REQUEST) is a PR number
 #   MOM_TARGET_SLUG: TRAVIS_REPO_SLUG
 #   MOM_TARGET_LOCAL_BRANCH: TRAVIS_BRANCH
-#
-# These are set to true by Travis if testing a pull request
+
+# These are set to true by our Travis configuration if testing a pull request
 DO_REGRESSION_TESTS ?=
 REPORT_COVERAGE ?=
 
 ifeq ($(DO_REGRESSION_TESTS), true)
-	BUILDS += target
-	TEST += regressions
+  BUILDS += target
+  TESTS += regressions
 
-	MOM_TARGET_SLUG ?= NOAA-GFDL/MOM6
-	MOM_TARGET_URL ?= https://github.com/$(MOM_TARGET_SLUG)
+  MOM_TARGET_SLUG ?= NOAA-GFDL/MOM6
+  MOM_TARGET_URL ?= https://github.com/$(MOM_TARGET_SLUG)
 
-	MOM_TARGET_LOCAL_BRANCH ?= dev/gfdl
-	MOM_TARGET_BRANCH := origin/$(MOM_TARGET_LOCAL_BRANCH)
+  MOM_TARGET_LOCAL_BRANCH ?= dev/gfdl
+  MOM_TARGET_BRANCH := origin/$(MOM_TARGET_LOCAL_BRANCH)
 
-	TARGET_CODEBASE = $(BUILD)/target_codebase
+  TARGET_CODEBASE = $(BUILD)/target_codebase
 else
-	MOM_TARGET_URL =
-	MOM_TARGET_BRANCH =
-	TARGET_CODEBASE =
+  MOM_TARGET_URL =
+  MOM_TARGET_BRANCH =
+  TARGET_CODEBASE =
 endif
 
 SOURCE = $(wildcard $(BASE)/src/*/*.F90 $(BASE)/src/*/*/*.F90 $(BASE)/config_src/solo_driver/*.F90)
 
+
 #---
 # Rules
 
@@ -85,7 +98,7 @@ $(BUILD)/target/MOM6: MOMFLAGS=NETCDF=3 DEBUG=1
 $(BUILD)/symmetric/MOM6: MOMFLAGS=NETCDF=3 DEBUG=1 $(COVFLAG)
 $(BUILD)/asymmetric/MOM6: MOMFLAGS=NETCDF=3 DEBUG=1
 $(BUILD)/repro/MOM6: MOMFLAGS=NETCDF=3 REPRO=1
-$(BUILD)/openmp/MOM6: MOMFLAGS=NETCDF=3 OPENMP=1
+$(BUILD)/openmp/MOM6: MOMFLAGS=NETCDF=3 DEBUG=1 OPENMP=1
 
 $(BUILD)/asymmetric/path_names: GRID_SRC=config_src/dynamic
 $(BUILD)/%/path_names: GRID_SRC=config_src/dynamic_symmetric
@@ -175,15 +188,14 @@ test.openmps: $(foreach c,$(CONFIGS),$(c).openmp $(c).openmp.diag)
 test.nans: $(foreach c,$(CONFIGS),$(c).nan $(c).nan.diag)
 test.dims: $(foreach c,$(CONFIGS),$(foreach d,t l h z,$(c).dim.$(d) $(c).dim.$(d).diag))
 
-# NOTE: chksum_diag return code of cmp is currently ignored since many fail!
 define CMP_RULE
 .PRECIOUS: $(foreach b,$(2),results/%/ocean.stats.$(b))
 %.$(1): $(foreach b,$(2),results/%/ocean.stats.$(b))
-	cmp $$^
+	cmp $$^ || diff $$^
 
 .PRECIOUS: $(foreach b,$(2),results/%/chksum_diag.$(b))
 %.$(1).diag: $(foreach b,$(2),results/%/chksum_diag.$(b))
-	cmp $$^
+	cmp $$^ || diff $$^
 endef
 
 $(eval $(call CMP_RULE,regression,symmetric target))
@@ -197,20 +209,23 @@ $(foreach d,t l h z,$(eval $(call CMP_RULE,dim.$(d),symmetric dim.$(d))))
 # Restart tests only compare the final stat record
 .PRECIOUS: $(foreach b,symmetric restart,results/%/ocean.stats.$(b))
 %.restart: $(foreach b,symmetric restart,results/%/ocean.stats.$(b))
-	cmp $(foreach f,$^,<(tr -s ' ' < $(f) | cut -d ' ' -f3- | tail -n 1))
+	cmp $(foreach f,$^,<(tr -s ' ' < $(f) | cut -d ' ' -f3- | tail -n 1)) \
+		|| diff $^
 
 # TODO: chksum_diag parsing of restart files
 
 
 #---
 # Test run output files
 
-# Simple function for generalized Slurm (srun) and OpenMPI (mpirun) support
+# Generalized MPI environment variable support
 # $(1): Environment variables
-ifeq ($(MPIRUN), srun)
-MPIRUN_CMD=$(1) $(MPIRUN)
+ifeq ($(shell $(MPIRUN) -x tmp=1 true 2> /dev/null ; echo $$?), 0)
+  MPIRUN_CMD=$(MPIRUN) $(if $(1),-x $(1),)
+else ifeq ($(shell $(MPIRUN) -env tmp=1 true 2> /dev/null ; echo $$?), 0)
+  MPIRUN_CMD=$(MPIRUN) $(if $(1),-env $(1),)
 else
-MPIRUN_CMD=$(MPIRUN) $(if $(1),-x $(1),)
+  MPIRUN_CMD=$(1) $(MPIRUN)
 endif
 
 # Rule to build results/<tc>/{ocean.stats,chksum_diag}.<tag>

.testing/README.md

@@ -56,7 +56,7 @@ Model state is currently defined by the `ocean.stats` output file, which
 reports the total energy (per unit mass) at machine precision alongside similar
 global metrics, such as mass or mean sea level, at lower precision.
 
-Clhecksums for every available diagnostic are also compared and the Makefile
+Checksums for every available diagnostic are also compared and the Makefile
 will report any differences, but such differences are not yet considered a fail
 condition.
 
@@ -138,7 +138,7 @@ This will run through the following tests:
 - `test.restarts`: Resubmission by restarts
 - `test.repros`: Optimized (REPRO) and unoptimized (DEBUG) compilation
 - `test.nans`: NaN initialization of allocated arrays
-- `test.dims`: Dimensional scaling (length, time, thichkness, depth)
+- `test.dims`: Dimensional scaling (length, time, thickness, depth)
 
 To enable the regression tests, use `DO_REGRESSION_TEST=true`.
 ```
@@ -159,18 +159,21 @@ fail if the answers differ from this build.
 
 The following test configurations (TCs) are supported:
 
-- TC0: Unit testing of various model components, based on `unit_tests`
-- TC1: A low-resolution version of the `benchmark` configuration
-- TC2: An ALE configuration based on TC1
-- TC3: An open-boundary condition (OBC) test based on `circle_obcs`
+- tc0: Unit testing of various model components, based on `unit_tests`
+- tc1: A low-resolution version of the `benchmark` configuration
+  - tc1.a: Use the un-split mode with Runge-Kutta 3 time integration
+  - tc1.b: Use the un-split mode with Runge-Kutta 2 time integration
+- tc2: An ALE configuration based on tc1 with tides
+  - tc2.a: Use sigma, PPM_H4 and no tides
+- tc3: An open-boundary condition (OBC) test based on `circle_obcs`
 
 
 ## Code coverage
 
 Code coverage reports the lines of code which have been tested, and can
 explicitly demonstrate when a particular operation is untested.
 
-Coverage is measued using `gcov` and is reported for TCs using the `symmetric`
+Coverage is measured using `gcov` and is reported for TCs using the `symmetric`
 executable.
 
 Coverage reporting is optionally sent to the `codecov.io` site.

.testing/tc1.b/MOM_input

@@ -0,0 +1 @@
+../tc1/MOM_input

.testing/tc1.b/MOM_tc_variant

@@ -0,0 +1,2 @@
+#override SPLIT=False
+#override USE_RK2=True

.testing/tc1.b/diag_table

@@ -0,0 +1 @@
+../tc1/diag_table

.testing/tc1.b/input.nml

@@ -0,0 +1,20 @@
+&mom_input_nml
+    output_directory = './'
+    input_filename = 'n'
+    restart_input_dir = 'INPUT/'
+    restart_output_dir = 'RESTART/'
+    parameter_filename =
+        'MOM_input',
+        'MOM_tc_variant',
+        'MOM_override',
+/
+
+&diag_manager_nml
+/
+
+&fms_nml
+    clock_grain = 'ROUTINE'
+    clock_flags = 'SYNC'
+    domains_stack_size = 955296
+    stack_size = 0
+/

.testing/tc2.a/MOM_input

@@ -0,0 +1 @@
+../tc2/MOM_input

.testing/tc2.a/MOM_tc_variant

@@ -0,0 +1,3 @@
+#override TOPO_CONFIG = "spoon"
+#override REMAPPING_SCHEME = "PPM_H4"
+#override REGRIDDING_COORDINATE_MODE = "SIGMA"

.testing/tc2.a/diag_table

@@ -0,0 +1 @@
+../tc2/diag_table

.testing/tc2.a/input.nml

@@ -0,0 +1,20 @@
+&mom_input_nml
+    output_directory = './'
+    input_filename = 'n'
+    restart_input_dir = 'INPUT/'
+    restart_output_dir = 'RESTART/'
+    parameter_filename =
+        'MOM_input',
+        'MOM_tc_variant',
+        'MOM_override',
+/
+
+&diag_manager_nml
+/
+
+&fms_nml
+    clock_grain = 'ROUTINE'
+    clock_flags = 'SYNC'
+    domains_stack_size = 955296
+    stack_size = 0
+/

.testing/tc2/MOM_input

@@ -319,18 +319,6 @@ SMAG_BI_CONST = 0.06            !   [nondim] default = 0.0
 ENERGETICS_SFC_PBL = True
 DO_GEOTHERMAL = True
 GEOTHERMAL_SCALE = 0.05
-TIDES = True
-TIDE_M2 = True
-TIDE_S2 = True
-TIDE_N2 = True
-TIDE_K2 = True
-TIDE_K1 = True
-TIDE_O1 = True
-TIDE_P1 = True
-TIDE_Q1 = True
-TIDE_MF = True
-TIDE_MM = True
-TIDE_SAL_SCALAR_VALUE = 1.
 USE_NEUTRAL_DIFFUSION = True
 DYNAMIC_VISCOUS_ML = True       !   [Boolean] default = False
                                 ! If true, use a bulk Richardson number criterion to

.testing/tc2/MOM_tc_variant

@@ -0,0 +1,12 @@
+TIDES = True
+TIDE_M2 = True
+TIDE_S2 = True
+TIDE_N2 = True
+TIDE_K2 = True
+TIDE_K1 = True
+TIDE_O1 = True
+TIDE_P1 = True
+TIDE_Q1 = True
+TIDE_MF = True
+TIDE_MM = True
+TIDE_SAL_SCALAR_VALUE = 1.

.testing/tc2/input.nml

@@ -5,6 +5,7 @@
     restart_output_dir = 'RESTART/'
     parameter_filename =
         'MOM_input',
+        'MOM_tc_variant',
         'MOM_override',
 /
 

config_src/mct_driver/mom_ocean_model_mct.F90

@@ -56,7 +56,6 @@ module MOM_ocean_model_mct
 use coupler_types_mod,       only : coupler_type_set_diags, coupler_type_send_data
 use mpp_domains_mod,         only : domain2d, mpp_get_layout, mpp_get_global_domain
 use mpp_domains_mod,         only : mpp_define_domains, mpp_get_compute_domain, mpp_get_data_domain
-use atmos_ocean_fluxes_mod,  only : aof_set_coupler_flux
 use fms_mod,                 only : stdout
 use mpp_mod,                 only : mpp_chksum
 use MOM_EOS,                 only : gsw_sp_from_sr, gsw_pt_from_ct
@@ -525,7 +524,7 @@ subroutine update_ocean_model(Ice_ocean_boundary, OS, Ocean_sfc, &
       if (do_thermo) &
         call shelf_calc_flux(OS%sfc_state, OS%fluxes, OS%Time, dt_coupling, OS%Ice_shelf_CSp)
       if (do_dyn) &
-        call add_shelf_forces(OS%grid, OS%Ice_shelf_CSp, OS%forces)
+        call add_shelf_forces(OS%grid, OS%US, OS%Ice_shelf_CSp, OS%forces)
     endif
     if (OS%icebergs_alter_ocean)  then
       if (do_dyn) &
@@ -560,7 +559,7 @@ subroutine update_ocean_model(Ice_ocean_boundary, OS, Ocean_sfc, &
       if (do_thermo) &
         call shelf_calc_flux(OS%sfc_state, OS%flux_tmp, OS%Time, dt_coupling, OS%Ice_shelf_CSp)
       if (do_dyn) &
-        call add_shelf_forces(OS%grid, OS%Ice_shelf_CSp, OS%forces)
+        call add_shelf_forces(OS%grid, OS%US, OS%Ice_shelf_CSp, OS%forces)
     endif
     if (OS%icebergs_alter_ocean)  then
       if (do_dyn) &

config_src/mct_driver/mom_surface_forcing_mct.F90

@@ -145,8 +145,8 @@ module MOM_surface_forcing_mct
 !> Structure corresponding to forcing, but with the elements, units, and conventions
 !! that exactly conform to the use for MOM-based coupled models.
 type, public :: ice_ocean_boundary_type
-  real, pointer, dimension(:,:) :: rofl_flux         =>NULL() !< liquid runoff [W/m2]
-  real, pointer, dimension(:,:) :: rofi_flux         =>NULL() !< ice runoff [W/m2]
+  real, pointer, dimension(:,:) :: rofl_flux         =>NULL() !< liquid runoff [kg/m2/s]
+  real, pointer, dimension(:,:) :: rofi_flux         =>NULL() !< ice runoff [kg/m2/s]
   real, pointer, dimension(:,:) :: u_flux            =>NULL() !< i-direction wind stress [Pa]
   real, pointer, dimension(:,:) :: v_flux            =>NULL() !< j-direction wind stress [Pa]
   real, pointer, dimension(:,:) :: t_flux            =>NULL() !< sensible heat flux [W/m2]

config_src/mct_driver/ocn_comp_mct.F90

@@ -109,8 +109,9 @@ subroutine ocn_init_mct( EClock, cdata_o, x2o_o, o2x_o, NLFilename )
   type(mct_aVect)             , intent(inout) :: o2x_o       !< Fluxes from ocean to coupler, computed by ocean
   character(len=*), optional  , intent(in)    :: NLFilename  !< Namelist filename
 
-  !  local variables
-  type(time_type)         :: time0                !< Model start time
+  !  local variable
+  type(time_type)         :: time0                !< Start time of coupled model's calendar.
+  type(time_type)         :: time_start           !< The time at which to initialize the ocean model
   type(ESMF_time)         :: time_var             !< ESMF_time variable to query time
   type(ESMF_time)         :: time_in_ESMF         !< Initial time for ocean
   type(ESMF_timeInterval) :: ocn_cpl_interval     !< Ocean coupling interval
@@ -202,11 +203,16 @@ subroutine ocn_init_mct( EClock, cdata_o, x2o_o, o2x_o, NLFilename )
 
   call set_calendar_type(NOLEAP)  !TODO: confirm this
 
-  ! Get the initial time
-  call ESMF_ClockGet(EClock, currTime=time_var, rc=rc)
+  ! Get start time
+  call ESMF_ClockGet(EClock, StartTime=time_var, rc=rc)
   call ESMF_TimeGet(time_var, yy=year, mm=month, dd=day, h=hour, m=minute, s=seconds, rc=rc)
   time0 = set_date(year, month, day, hour, minute, seconds, err_msg=err_msg)
 
+  ! Get current time
+  call ESMF_ClockGet(EClock, currTime=time_var, rc=rc)
+  call ESMF_TimeGet(time_var, yy=year, mm=month, dd=day, h=hour, m=minute, s=seconds, rc=rc)
+  time_start = set_date(year, month, day, hour, minute, seconds, err_msg=err_msg)
+
   ! Debugging clocks
   if (debug .and. is_root_pe()) then
     write(glb%stdout,*) 'ocn_init_mct, current time: y,m,d-',year,month,day,'h,m,s=',hour,minute,seconds
@@ -279,7 +285,7 @@ subroutine ocn_init_mct( EClock, cdata_o, x2o_o, o2x_o, NLFilename )
   runtype = get_runtype()
   if (runtype == "initial") then
     ! startup (new run) - 'n' is needed below since we don't specify input_filename in input.nml
-    call ocean_model_init(glb%ocn_public, glb%ocn_state, time0, time0, input_restart_file = 'n')
+    call ocean_model_init(glb%ocn_public, glb%ocn_state, time0, time_start, input_restart_file = 'n')
   else  ! hybrid or branch or continuos runs
     ! get output path root
     call seq_infodata_GetData( glb%infodata, outPathRoot=restartpath )
@@ -295,7 +301,7 @@ subroutine ocn_init_mct( EClock, cdata_o, x2o_o, o2x_o, NLFilename )
       write(glb%stdout,*) 'Reading restart file: ',trim(restartfile)
     end if
     call shr_file_freeUnit(nu)
-    call ocean_model_init(glb%ocn_public, glb%ocn_state, time0, time0, input_restart_file=trim(restartfile))
+    call ocean_model_init(glb%ocn_public, glb%ocn_state, time0, time_start, input_restart_file=trim(restartfile))
   endif
   if (is_root_pe()) then
     write(glb%stdout,'(/12x,a/)') '======== COMPLETED MOM INITIALIZATION ========'