Merge branch 'rljacob/driver-moab/add-mb-intf' (PR #5979)

Add interfaces to driver-moab in WaterCycle components: EAM, ELM, MOSART, MPAS-Ocean,
MPAS-SeaIce. Summary of changes is below.

MOAB interfaces and support routines in components are isolated with a "HAVE_MOAB" ifdef that
will be automatically defined if the MOAB coupler is chosen. This allows components to use either
mct or moab couplers for now. Choice is made during create_newcase. There are also cpp defines
for debugging. MOABDEBUG will output .h5m files throughout the coupling processs and additional
log info (very slow). MOABCOMP will calculate diffs between MCT and MOAB coupling fields. These
defines need to be set by hand if you wish to use them. These will be removed later. When using
default driver-mct, there is no impact on answers and no dependency on MOAB.

Also bring in latest version of driver-moab which is feature-complete for the watercycle case.
All maps (except r2o) are calculated online. Merging, fluxes and all other coupler calculations are
performed with moab data structures. Currently includes MCT-scaffolding for verification
(data moving between components follows both MCT- and MOAB- paths; offline maps still read by MCT;
all mct-based datatypes still present).

[BFB]

cime_config/machines/cmake_macros/gnu_chrysalis.cmake

@@ -2,3 +2,4 @@ if (COMP_NAME STREQUAL gptl)
   string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
 endif()
 set(PIO_FILESYSTEM_HINTS "gpfs")
+set(MOAB_PATH "/lcrc/soft/climate/moab/chrysalis/gnu")

cime_config/machines/cmake_macros/intel_chrysalis.cmake

@@ -12,6 +12,7 @@ string(APPEND CMAKE_Fortran_FLAGS " -axCORE-AVX2")
 string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O3 -qno-opt-dynamic-align")
 set(PIO_FILESYSTEM_HINTS "gpfs")
 string(APPEND CMAKE_EXE_LINKER_FLAGS " -static-intel")
+set(MOAB_PATH "/lcrc/soft/climate/moab/chrysalis/intel")
 if (MPILIB STREQUAL impi)
   set(MPICC "mpiicc")
   set(MPICXX "mpiicpc")

cime_config/machines/config_machines.xml

@@ -1956,6 +1956,8 @@
       <env name="HDF5_ROOT">/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/hdf5/1.12.1/mpich-4.0/gcc-11.1.0</env>
       <env name="NETCDF_PATH">/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/mpich-4.0/gcc-11.1.0</env>
       <env name="PNETCDF_PATH">/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/pnetcdf/1.12.2/mpich-4.0/gcc-11.1.0</env>
+      <env name="MOAB_ROOT">$SHELL{if [ -z "$MOAB_ROOT" ]; then echo /nfs/gce/projects/climate/software/moab/devel/mpich-4.0/gcc-11.1.0; else echo "$MOAB_ROOT"; fi}</env>
+      <env name="MOAB_PATH">/nfs/gce/projects/climate/software/moab/devel/mpich-4.0/gcc-11.1.0</env>
     </environment_variables>
     <environment_variables mpilib="openmpi">
       <!-- We currently don't have modules for HDF5, NetCDF & PnetCDF -->
@@ -2372,6 +2374,12 @@
     <environment_variables SMP_PRESENT="TRUE" compiler="gnu">
       <env name="OMP_PLACES">cores</env>
     </environment_variables>
+    <environment_variables compiler="intel" mpilib="openmpi">
+      <env name="MOAB_ROOT">$SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/chrysalis/intel; else echo "$MOAB_ROOT"; fi}</env>
+    </environment_variables>
+    <environment_variables compiler="gnu" mpilib="openmpi">
+      <env name="MOAB_ROOT">$SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/chrysalis/gnu; else echo "$MOAB_ROOT"; fi}</env>
+    </environment_variables>
   </machine>
 
   <machine MACH="blues">

components/CMakeLists.txt

@@ -82,6 +82,11 @@ endif()
 
 project(E3SM C CXX Fortran)
 
+if (COMP_INTERFACE STREQUAL "moab")
+  set(USE_MOAB True)
+  set(CPPDEFS "${CPPDEFS} -DHAVE_MOAB")
+endif()
+
 if(USE_CUDA)
   enable_language(CUDA)
 elseif(USE_HIP)

components/cice/bld/config_files/definition.xml

@@ -4,7 +4,7 @@
 
 <config_definition>
 
-<entry id="comp_intf" valid_values="MCT,ESMF" value="MCT">
+<entry id="comp_intf" valid_values="MCT,ESMF,MOAB" value="MOAB">
 CICE build directory; contains .o and .mod files.
 </entry>
 

components/cmake/build_model.cmake

@@ -276,6 +276,10 @@ macro(build_model COMP_CLASS COMP_NAME)
     if (COMP_NAME STREQUAL "csm_share")
       find_package(NETCDF REQUIRED)
       target_link_libraries(${TARGET_NAME} PRIVATE netcdf)
+      if (USE_MOAB)
+        target_link_libraries(${TARGET_NAME} PRIVATE ${MOAB_LIBRARIES})
+        target_include_directories(${TARGET_NAME} PRIVATE ${MOAB_INCLUDE_DIRS})
+      endif()
     else()
       target_link_libraries(${TARGET_NAME} PRIVATE csm_share)
       if (COMP_NAME STREQUAL "eam")

components/cmake/find_dep_packages.cmake

@@ -30,6 +30,10 @@ if (USE_ALBANY OR USE_TRILINOS)
   find_package(Trilinos REQUIRED)
 endif()
 
+if (USE_MOAB)
+  find_package(MOAB REQUIRED)
+endif()
+
 if (USE_ALBANY)
   find_package(Albany REQUIRED)
 endif()

components/cmake/modules/FindCsmShare.cmake

@@ -26,14 +26,21 @@ set(CSM_SHARE "${INSTALL_SHAREDPATH}/${COMP_INTERFACE}/${ESMFDIR}/${NINST_VALUE}
 # Look for libcsm_share in the complex path we built above
 find_library(CSM_SHARE_LIB csm_share REQUIRED PATHS ${CSM_SHARE})
 
+set(CSM_LIBS  ${CSM_SHARE_LIB};mct;spio)
+set(CSM_INCLS ${CSM_SHARE})
+if (USE_MOAB)
+  list(APPEND CSM_LIBS ${MOAB_LIBRARIES})
+  list(APPEND CSM_INCLS ${MOAB_INCLUDE_DIRS})
+endif()
+
 # Grab mct and spio if we don't already have it
 find_package(MCT REQUIRED)
 find_package(PIO REQUIRED)
 
 # Create the interface library, and set target properties
 add_library (csm_share INTERFACE)
-target_link_libraries (csm_share INTERFACE ${CSM_SHARE_LIB};mct;spio)
-target_include_directories(csm_share INTERFACE ${CSM_SHARE})
+target_link_libraries (csm_share INTERFACE ${CSM_LIBS})
+target_include_directories(csm_share INTERFACE ${CSM_INCLS})
 
 # Link against piof. Don't worry about this for now. Fix once spio
 # has a proper config.cmake.

components/eam/bld/config_files/definition.xml

@@ -23,8 +23,8 @@ are specified as a comma separated list with no embedded white space.
 <entry id="ccsm_seq" valid_values="0,1" value="0">
 Switch specifies whether CAM is being built by the CCSM sequential scripts.  0 => no, 1 => yes.
 </entry>
-<entry id="comp_intf" valid_values="mct,esmf" value="mct">
-Component interfaces: mct or esmf.  Default: mct.
+<entry id="comp_intf" valid_values="mct,esmf,moab" value="mct">
+Component interfaces: mct, esmf, or moab.  Default: mct.
 </entry>
 <entry id="dyn" valid_values="eul,sld,fv,homme,se" value="">
 Dynamics package: eul, sld, fv, or se.

components/eam/bld/configure

@@ -131,7 +131,7 @@ OPTIONS
                         clubb_single_prec (Run CLUBB in single precision)
      -co2_cycle         This option is meant to be used with the -ccsm_seq option.  It modifies the
                         EAM configuration by increasing the number of advected constituents by 4.
-     -comp_intf         Specify the component interfaces [mct | esmf] (default: mct).
+     -comp_intf         Specify the component interfaces [mct | esmf | moab] (default: mct).
      -cppdefs <string>  A string of user specified CPP defines.  Appended to
                         Makefile defaults.  E.g. -cppdefs '-DVAR1 -DVAR2'
      -dyn <name>        Build EAM with specified dynamical core [eul | sld | fv | se].

components/eam/cime_config/buildnml

@@ -97,6 +97,7 @@ def buildnml(case, caseroot, compname):
         comp = ""
         if comp_interface == "mct":  comp = "mct"
         if comp_interface == "esmf": comp = "esmf"
+        if comp_interface == "moab": comp = "moab"
 
         cam_lib_dirs = "-cosp_libdir {}/atm/obj/cosp".format(exeroot) if "cosp" in cam_config_opts else ""