Merge branch 'main' into sfcflxrollover

.github/workflows/srt.yml

@@ -26,8 +26,8 @@ jobs:
       CPPFLAGS: "-I/usr/include -I/usr/local/include "
       LDFLAGS: "-L/usr/lib/x86_64-linux-gnu -lnetcdf -lnetcdff -lpnetcdf"     
       # Versions of all dependencies can be updated here
-      ESMF_VERSION: v8.4.0
-      PARALLELIO_VERSION: pio2_5_10
+      ESMF_VERSION: v8.5.0
+      PARALLELIO_VERSION: pio2_6_0
       CIME_MODEL: cesm
       CIME_DRIVER: nuopc
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -153,6 +153,7 @@ jobs:
           mkdir -p $HOME/cesm/scratch
           mkdir -p $HOME/cesm/inputdata
           pushd $GITHUB_WORKSPACE/cesm/cime/CIME/tests
+          export SRCROOT=$GITHUB_WORKSPACE/cesm/
           export CIME_TEST_PLATFORM=ubuntu-latest
           export PIO_INCDIR=$HOME/pio/include
           export PIO_LIBDIR=$HOME/pio/lib
@@ -175,6 +176,6 @@ jobs:
           popd
 #     the following can be used by developers to login to the github server in case of errors
 #     see https://github.com/marketplace/actions/debugging-with-tmate for further details
-#      - name: Setup tmate session
-#        if: ${{ failure() }}
-#        uses: mxschmitt/action-tmate@v3
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3

cime_config/config_component.xml

@@ -784,6 +784,34 @@
       If TRUE, the component libraries are always built with OpenMP capability.</desc>
   </entry>
 
+  <entry id="GPU_TYPE">
+    <type>char</type>
+    <valid_values></valid_values>
+    <default_value></default_value>
+    <group>build_def</group>
+    <file>env_build.xml</file>
+    <desc>If set will compile and submit with this gpu type enabled </desc>
+  </entry>
+
+  <entry id="GPU_OFFLOAD">
+    <type>char</type>
+    <valid_values></valid_values>
+    <default_value></default_value>
+    <group>build_def</group>
+    <file>env_build.xml</file>
+    <desc>If set will compile and submit with this gpu offload method enabled </desc>
+  </entry>
+
+  <entry id="MPI_GPU_WRAPPER_SCRIPT">
+    <type>char</type>
+    <valid_values></valid_values>
+    <default_value></default_value>
+    <group>build_def</group>
+    <file>env_build.xml</file>
+    <desc>If set will attach this script to the MPI run command, mapping
+    different MPI ranks to different GPUs within the same compute node</desc>
+  </entry>
+
   <entry id="SMP_PRESENT">
     <type>logical</type>
     <valid_values>TRUE,FALSE</valid_values>
@@ -1798,12 +1826,22 @@
     <desc>pes or cores per node for accounting purposes </desc>
   </entry>
 
+  <entry id="MAX_CPUTASKS_PER_GPU_NODE">
+    <type>integer</type>
+    <default_value>0</default_value>
+    <values>
+      <value compiler="nvhpc">1</value>
+    </values>
+    <group>mach_pes_last</group>
+    <file>env_mach_pes.xml</file>
+    <desc> Number of CPU cores per GPU node used for simulation </desc>
+  </entry>
+
   <entry id="NGPUS_PER_NODE">
     <type>integer</type>
     <default_value>0</default_value>
     <values>
-      <value compiler="pgi-gpu">1</value>
-      <value compiler="nvhpc-gpu">1</value>
+      <value compiler="nvhpc">1</value>
     </values>
     <group>mach_pes</group>
     <file>env_mach_pes.xml</file>

mediator/esmFldsExchange_cesm_mod.F90

@@ -2158,7 +2158,7 @@ subroutine esmFldsExchange_cesm(gcomp, phase, rc)
           ! liquid from river and possibly flood from river to ocean
           if (fldchk(is_local%wrap%FBImp(comprof, comprof), 'Forr_rofl' , rc=rc)) then
              if (trim(rof2ocn_liq_rmap) == 'unset') then
-                call addmap_from(comprof, 'Forr_rofl', compocn, mapconsd, 'none', 'unset')
+                call addmap_from(comprof, 'Forr_rofl', compocn, mapconsd, 'one', 'unset')
              else
                 call addmap_from(comprof, 'Forr_rofl', compocn, map_rof2ocn_liq, 'none', rof2ocn_liq_rmap)
              end if
@@ -2182,7 +2182,7 @@ subroutine esmFldsExchange_cesm(gcomp, phase, rc)
           ! ice from river to ocean
           if (fldchk(is_local%wrap%FBImp(comprof, comprof), 'Forr_rofi' , rc=rc)) then
              if (trim(rof2ocn_ice_rmap) == 'unset') then
-                call addmap_from(comprof, 'Forr_rofi', compocn, mapconsd, 'none', 'unset')
+                call addmap_from(comprof, 'Forr_rofi', compocn, mapconsd, 'one', 'unset')
              else
                 call addmap_from(comprof, 'Forr_rofi', compocn, map_rof2ocn_ice, 'none', rof2ocn_ice_rmap)
              end if

mediator/esmFldsExchange_nems_mod.F90

@@ -38,6 +38,8 @@ subroutine esmFldsExchange_nems(gcomp, phase, rc)
     use esmFlds               , only : addmap_from => med_fldList_addmap_from
     use esmFlds               , only : addfld_aoflux => med_fldList_addfld_aoflux
     use esmFlds               , only : addmap_aoflux => med_fldList_addmap_aoflux
+    use esmFlds               , only : addfld_ocnalb => med_fldList_addfld_ocnalb
+    use esmFlds               , only : addmap_ocnalb => med_fldList_addmap_ocnalb
 
     ! input/output parameters:
     type(ESMF_GridComp)              :: gcomp
@@ -172,6 +174,14 @@ subroutine esmFldsExchange_nems(gcomp, phase, rc)
        call addfld_from(compice, 'mean_sw_pen_to_ocn')
     end if
 
+    ! Advertise the ocean albedos. These are not sent to the ATM in UFS.
+    if (phase == 'advertise') then
+       call addfld_ocnalb('So_avsdr')
+       call addfld_ocnalb('So_avsdf')
+       call addfld_ocnalb('So_anidr')
+       call addfld_ocnalb('So_anidf')
+    end if
+
     !=====================================================================
     ! FIELDS TO ATMOSPHERE (compatm)
     !=====================================================================
@@ -306,7 +316,7 @@ subroutine esmFldsExchange_nems(gcomp, phase, rc)
     else
        if ( fldchk(is_local%wrap%FBexp(compatm)        , 'Sw_z0', rc=rc) .and. &
             fldchk(is_local%wrap%FBImp(compwav,compwav), 'Sw_z0', rc=rc)) then
-          call addmap_from(compwav, 'Sw_z0', compatm, mapnstod_consf, 'one', 'unset')
+          call addmap_from(compwav, 'Sw_z0', compatm, mapbilnr_nstod, 'one', 'unset')
           call addmrg_to(compatm, 'Sw_z0', mrg_from=compwav, mrg_fld='Sw_z0', mrg_type='copy')
        end if
     end if
@@ -453,13 +463,13 @@ subroutine esmFldsExchange_nems(gcomp, phase, rc)
        ! to ocn: evaporation water flux (custom merge in med_phases_prep_ocn)
        if (phase == 'advertise') then
           if (is_local%wrap%comp_present(compatm) .and. is_local%wrap%comp_present(compocn)) then
-             call addfld_from(compatm, 'Faxa_lat')
+             call addfld_from(compatm, 'Faxa_evap')
              call addfld_to(compocn, 'Faxa_evap')
           end if
        else
           if ( fldchk(is_local%wrap%FBexp(compocn)        , 'Faxa_evap', rc=rc) .and. &
-               fldchk(is_local%wrap%FBImp(compatm,compatm), 'Faxa_lat' , rc=rc)) then
-             call addmap_from(compatm, 'Faxa_lat', compocn, mapconsf_aofrac, 'aofrac', 'unset')
+               fldchk(is_local%wrap%FBImp(compatm,compatm), 'Faxa_evap' , rc=rc)) then
+             call addmap_from(compatm, 'Faxa_evap', compocn, mapconsf_aofrac, 'aofrac', 'unset')
           end if
        end if
     else if (trim(coupling_mode) == 'nems_orig_data' .or. trim(coupling_mode) == 'nems_frac_aoflux') then
@@ -698,7 +708,7 @@ subroutine esmFldsExchange_nems(gcomp, phase, rc)
        else
           if ( fldchk(is_local%wrap%FBexp(compwav)        , trim(fldname), rc=rc) .and. &
                fldchk(is_local%wrap%FBImp(compatm,compatm), trim(fldname), rc=rc)) then
-             call addmap_from(compatm, trim(fldname), compwav, mapnstod_consf, 'one', 'unset')
+             call addmap_from(compatm, trim(fldname), compwav, mapbilnr_nstod, 'one', 'unset')
              call addmrg_to(compwav, trim(fldname), mrg_from=compatm, mrg_fld=trim(fldname), mrg_type='copy')
           end if
        end if

mediator/med.F90

@@ -661,7 +661,7 @@ subroutine AdvertiseFields(gcomp, importState, exportState, clock, rc)
     use NUOPC , only : NUOPC_CompAttributeGet, NUOPC_CompAttributeSet, NUOPC_CompAttributeAdd
     use esmFlds, only : med_fldlist_init1, med_fld_GetFldInfo, med_fldList_entry_type
     use med_phases_history_mod, only : med_phases_history_init
-    use med_methods_mod       , only : mediator_checkfornans  
+    use med_methods_mod       , only : mediator_checkfornans
 
     ! input/output variables
     type(ESMF_GridComp)  :: gcomp
@@ -921,7 +921,7 @@ subroutine AdvertiseFields(gcomp, importState, exportState, clock, rc)
     call NUOPC_CompAttributeGet(gcomp, name="check_for_nans", value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
     if(isPresent .and. isSet) then
-       read(cvalue, *) mediator_checkfornans       
+       read(cvalue, *) mediator_checkfornans
     else
        mediator_checkfornans = .false.
     endif
@@ -1804,7 +1804,8 @@ subroutine DataInitialize(gcomp, rc)
          call esmFldsExchange_cesm(gcomp, phase='initialize', rc=rc)
          if (ChkErr(rc,__LINE__,u_FILE_u)) return
       else if (trim(coupling_mode(1:4)) == 'nems') then
-       call esmFldsExchange_nems(gcomp, phase='initialize', rc=rc)
+         call esmFldsExchange_nems(gcomp, phase='initialize', rc=rc)
+         if (ChkErr(rc,__LINE__,u_FILE_u)) return
       else if (trim(coupling_mode) == 'hafs') then
          call esmFldsExchange_hafs(gcomp, phase='initialize', rc=rc)
          if (ChkErr(rc,__LINE__,u_FILE_u)) return
@@ -1939,14 +1940,12 @@ subroutine DataInitialize(gcomp, rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
     !----------------------------------------------------------
-    ! Initialize ocean albedos (this is needed for cesm and hafs)
+    ! Initialize ocean albedos
     !----------------------------------------------------------
 
-    if (trim(coupling_mode(1:5)) /= 'nems_') then
-       if (is_local%wrap%comp_present(compocn) .or. is_local%wrap%comp_present(compatm)) then
-          call med_phases_ocnalb_run(gcomp, rc=rc)
-          if (ChkErr(rc,__LINE__,u_FILE_u)) return
-       end if
+    if (is_local%wrap%comp_present(compocn) .or. is_local%wrap%comp_present(compatm)) then
+       call med_phases_ocnalb_run(gcomp, rc=rc)
+       if (ChkErr(rc,__LINE__,u_FILE_u)) return
     end if
 
     !---------------------------------------

mediator/med_internalstate_mod.F90

@@ -262,7 +262,6 @@ subroutine med_internalstate_init(gcomp, rc)
        end do
     end if
     is_local%wrap%num_icesheets = num_icesheets
-
     call NUOPC_CompAttributeGet(gcomp, name='mediator_present', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
     if (isPresent .and. isSet) then

mediator/med_io_mod.F90

@@ -13,7 +13,7 @@ module med_io_mod
   use NUOPC                 , only : NUOPC_FieldDictionaryGetEntry
   use NUOPC                 , only : NUOPC_FieldDictionaryHasEntry
   use pio                   , only : file_desc_t, iosystem_desc_t
-  use med_internalstate_mod , only : logunit, med_id
+  use med_internalstate_mod , only : logunit, med_id, maintask
   use med_constants_mod     , only : dbug_flag    => med_constants_dbug_flag
   use med_methods_mod       , only : FB_getFieldN => med_methods_FB_getFieldN
   use med_methods_mod       , only : FB_getFldPtr => med_methods_FB_getFldPtr
@@ -75,7 +75,7 @@ module med_io_mod
   character(*),parameter         :: prefix    = "med_io_"
   character(*),parameter         :: modName   = "(med_io_mod) "
   character(*),parameter         :: version   = "cmeps0"
-  
+
   integer                        :: pio_iotype
   integer                        :: pio_ioformat
   type(iosystem_desc_t), pointer :: io_subsystem
@@ -1737,7 +1737,10 @@ subroutine med_io_read_init_iodesc(FB, name1, pioid, iodesc, rc)
        deallocate(dof)
 
        deallocate(minIndexPTile, maxIndexPTile)
-
+    else
+       if(maintask) write(logunit,'(a)') trim(subname)//' ERROR: '//trim(name1)//' is not present, aborting '
+       call ESMF_LogWrite(trim(subname)//' ERROR: '//trim(name1)//' is not present, aborting ', ESMF_LOGMSG_ERROR)
+       rc = ESMF_FAILURE
     end if ! end if rcode check
 
   end subroutine med_io_read_init_iodesc

mediator/med_map_mod.F90

@@ -111,7 +111,7 @@ subroutine med_map_RouteHandles_initfrom_esmflds(gcomp, flds_scalar_name, llogun
     type(ESMF_Mesh)           :: mesh_dst
     type(med_fldlist_type), pointer :: FldListFr
     type(med_fldlist_entry_type), pointer :: fldptr
-    character(len=*), parameter :: subname=' (module_med_map: RouteHandles_init) '
+    character(len=*), parameter :: subname=' (med_map_mod: RouteHandles_init) '
     !-----------------------------------------------------------
 
     call t_startf('MED:'//subname)
@@ -259,7 +259,8 @@ subroutine med_map_RouteHandles_initfrom_esmflds(gcomp, flds_scalar_name, llogun
                       if (chkerr(rc,__LINE__,u_FILE_u)) return
                       if (maintask) then
                          write(logunit,'(a)') trim(subname)//' created field_NormOne for '&
-                              //compname(n1)//'->'//compname(n2)//' with mapping '//trim(mapnames(mapindex))
+                              //trim(compname(n1))//'->'//trim(compname(n2))//' with mapping '&
+                              //trim(mapnames(mapindex))
                       end if
                    end if
                 end do ! end of loop over map_indiex mappers
@@ -304,7 +305,7 @@ subroutine med_map_routehandles_initfrom_fieldbundle(n1, n2, FBsrc, FBdst, mapin
     ! local variables
     type(ESMF_Field)   :: fldsrc
     type(ESMF_Field)   :: flddst
-    character(len=*), parameter :: subname=' (module_MED_map:med_map_routehandles_initfrom_fieldbundle) '
+    character(len=*), parameter :: subname=' (med_map_mod:med_map_routehandles_initfrom_fieldbundle) '
     !---------------------------------------------
 
     rc = ESMF_SUCCESS
@@ -653,7 +654,7 @@ logical function med_map_RH_is_created_RH3d(RHs,n1,n2,mapindex,rc)
     integer                , intent(out)   :: rc
 
     ! local variables
-    character(len=*), parameter :: subname=' (module_MED_map:med_map_RH_is_created_RH3d) '
+    character(len=*), parameter :: subname=' (med_map_mod:med_map_RH_is_created_RH3d) '
     !-----------------------------------------------------------
 
     rc = ESMF_SUCCESS
@@ -678,7 +679,7 @@ logical function med_map_RH_is_created_RH1d(RHs,mapindex,rc)
     ! local variables
     integer :: rc1, rc2
     logical :: mapexists
-    character(len=*), parameter :: subname=' (module_MED_map:med_map_RH_is_created_RH1d) '
+    character(len=*), parameter :: subname=' (med_map_mod:med_map_RH_is_created_RH1d) '
     !-----------------------------------------------------------
 
     rc  = ESMF_SUCCESS
@@ -750,7 +751,7 @@ subroutine med_map_packed_field_create(destcomp, flds_scalar_name, &
     character(CL), allocatable :: fieldNameList(:)
     character(CS)              :: mapnorm_mapindex
     character(len=CX)          :: tmpstr
-    character(len=*), parameter :: subname=' (module_MED_map:med_packed_field_create) '
+    character(len=*), parameter :: subname=' (med_map_mod:med_packed_field_create) '
     !-----------------------------------------------------------
 
     rc = ESMF_SUCCESS
@@ -818,6 +819,7 @@ subroutine med_map_packed_field_create(destcomp, flds_scalar_name, &
                       //'  '//trim(fieldnamelist(nf))
                    call ESMF_LogWrite(trim(tmpstr), ESMF_LOGMSG_INFO)
                 else
+                   !if(rof_name .ne. 'xrof' .and. compname(destcomp) .ne. 'ocn') then
                    if (mapnorm_mapindex /= packed_data(mapindex)%mapnorm) then
                      write(tmpstr,*)'Map type '//trim(mapnames(mapindex)) &
                         //', destcomp '//trim(compname(destcomp)) &
@@ -953,7 +955,7 @@ subroutine med_map_field_packed(FBSrc, FBDst, FBFracSrc, field_normOne, packed_d
     type(ESMF_Field), pointer  :: fieldlist_dst(:)
     real(r8), pointer          :: data_norm(:)
     real(r8), pointer          :: data_dst(:,:)
-    character(len=*), parameter  :: subname=' (module_MED_map:med_map_field_packed) '
+    character(len=*), parameter  :: subname=' (med_map_mod:med_map_field_packed) '
     !-----------------------------------------------------------
 
     call t_startf('MED:'//subname)
@@ -1165,7 +1167,7 @@ subroutine med_map_field_normalized(field_src, field_dst, routehandles, maptype,
     integer           :: ungriddedUBound(1)     ! currently the size must equal 1 for rank 2 fields
     integer           :: lsize_src
     integer           :: lsize_dst
-    character(len=*), parameter  :: subname=' (module_MED_map:med_map_field_normalized) '
+    character(len=*), parameter  :: subname=' (med_map_mod:med_map_field_normalized) '
     !-----------------------------------------------------------
 
     rc = ESMF_SUCCESS
@@ -1278,7 +1280,7 @@ subroutine med_map_field(field_src, field_dst, routehandles, maptype, fldname, r
     logical :: checkflag = .false.
     character(len=CS) :: lfldname
     real(ESMF_KIND_R8), parameter :: fillValue = 9.99e20_ESMF_KIND_R8
-    character(len=*), parameter :: subname='(module_MED_map:med_map_field) '
+    character(len=*), parameter :: subname='(med_map_mod:med_map_field) '
     !---------------------------------------------------
 
     rc = ESMF_SUCCESS
@@ -1381,7 +1383,7 @@ subroutine med_map_uv_cart3d(FBsrc, FBdst, routehandles, mapindex, rc)
     integer             :: spatialDim
     real(r8), parameter :: deg2rad = shr_const_pi/180.0_R8  ! deg to rads
     logical             :: first_time = .true.
-    character(len=*), parameter :: subname=' (module_MED_map:med_map_uv_cart3d) '
+    character(len=*), parameter :: subname=' (med_map_mod:med_map_uv_cart3d) '
     !-------------------------------------------------------------------------------
 
     rc = ESMF_SUCCESS

mediator/med_methods_mod.F90

@@ -1354,7 +1354,10 @@ subroutine med_methods_FB_accum(FBout, FBin, copy, rc)
         call med_methods_Field_GetFldPtr(lfield, fldptr1=dataptro1, fldptr2=dataptro2, rank=lranko, rc=rc)
         if (chkerr(rc,__LINE__,u_FILE_u)) return
 
-        if (lranki == 1 .and. lranko == 1) then
+        if (lranki == 0 .and. lranko == 0) then
+           ! do nothing
+          call ESMF_LogWrite(trim(subname)//": Both ranki and ranko are 0", ESMF_LOGMSG_INFO)
+        elseif (lranki == 1 .and. lranko == 1) then
 
           if (.not.med_methods_FieldPtr_Compare(dataPtro1, dataPtri1, subname, rc)) then
             call ESMF_LogWrite(trim(subname)//": ERROR in dataPtr1 size ", ESMF_LOGMSG_ERROR)
@@ -1397,7 +1400,7 @@ subroutine med_methods_FB_accum(FBout, FBin, copy, rc)
         else
 
           write(msgString,'(a,2i8)') trim(subname)//": ranki, ranko = ",lranki,lranko
-          call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO)
+          call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_ERROR)
           call ESMF_LogWrite(trim(subname)//": ERROR ranki ranko not supported "//trim(lfieldnamelist(n)), &
                ESMF_LOGMSG_ERROR)
           rc = ESMF_FAILURE

mediator/med_phases_history_mod.F90

@@ -25,7 +25,7 @@ module med_phases_history_mod
   use med_io_mod            , only : med_io_write, med_io_wopen, med_io_enddef, med_io_close
   use perf_mod              , only : t_startf, t_stopf
   use pio                   , only : file_desc_t
-  
+
   implicit none
   private