From b5d1e12a94ccbf153f15b2a7ebdd249a3d4f7c9b Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 7 May 2024 16:07:39 -0700 Subject: [PATCH 01/48] update RCEMIP zenith angle --- components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl index 3ecd465f7a14..e3d4433d4a8d 100644 --- a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl +++ b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl @@ -31,4 +31,4 @@ seq_flux_mct_albdif = 0.07 seq_flux_mct_albdir = 0.07 seq_flux_atmocn_minwind = 1 -constant_zenith_deg = 42.05 \ No newline at end of file +constant_zenith_deg = 42.04 \ No newline at end of file From b33fa7dacce0a31f9552649dc14d0f774ead8780 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 7 May 2024 16:08:06 -0700 Subject: [PATCH 02/48] update RCE use case comment --- components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml b/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml index c1c8e9e027ad..146a9483b36f 100644 --- a/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml +++ b/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml @@ -35,7 +35,7 @@ 0.0 - + From f3278f20b89fe1ed80681e0c8d2935c911e70adb Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 14 May 2024 12:43:37 -0700 Subject: [PATCH 03/48] add RCEMIP2 compsets --- .../eam/cime_config/config_compsets.xml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index 39971fbb0945..01e99a9a1489 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -229,6 +229,50 @@ 2000_EAM%RCE-MMF2_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV + + + FRCE-MW_295dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP11_SROF_SGLC_SWAV + + + FRCE-MW_300dT0p625 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP12_SROF_SGLC_SWAV + + + FRCE-MW_300dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP13_SROF_SGLC_SWAV + + + FRCE-MW_300dT2p5 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP14_SROF_SGLC_SWAV + + + FRCE-MW_305dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP15_SROF_SGLC_SWAV + + + + FRCE-MW-MMF1_295dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP11_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT0p625 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP12_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP13_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT2p5 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP14_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_305dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP15_SROF_SGLC_SWAV + + + From b63cbcadde51a50160d1221a740d142d20ea5c0c Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 14 May 2024 12:44:11 -0700 Subject: [PATCH 04/48] add new data ocean SST patterns or RCEMIP2 --- .../docn/cime_config/config_component.xml | 10 +++- .../cime_config/namelist_definition_docn.xml | 7 ++- .../data_comps/docn/src/docn_comp_mod.F90 | 54 ++++++++++++++++++- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/components/data_comps/docn/cime_config/config_component.xml b/components/data_comps/docn/cime_config/config_component.xml index 5a30c69df6cf..431d358f9951 100644 --- a/components/data_comps/docn/cime_config/config_component.xml +++ b/components/data_comps/docn/cime_config/config_component.xml @@ -13,7 +13,7 @@ This file may have ocn desc entries. --> - DOCN + DOCN null mode prescribed ocean mode slab ocean mode @@ -45,7 +45,7 @@ char - prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,sst_aquapfile,som,som_aquap,sst_aquap_constant,interannual,null + prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,sst_aquap11,sst_aquap12,sst_aquap13,sst_aquap14,sst_aquap15,sst_aquapfile,som,som_aquap,sst_aquap_constant,interannual,null prescribed null @@ -63,6 +63,12 @@ sst_aquap8 sst_aquap9 sst_aquap10 + + sst_aquap11 + sst_aquap12 + sst_aquap13 + sst_aquap14 + sst_aquap15 sst_aquapfile sst_aquap_constant diff --git a/components/data_comps/docn/cime_config/namelist_definition_docn.xml b/components/data_comps/docn/cime_config/namelist_definition_docn.xml index 948902e37324..a191d088d7f5 100644 --- a/components/data_comps/docn/cime_config/namelist_definition_docn.xml +++ b/components/data_comps/docn/cime_config/namelist_definition_docn.xml @@ -257,7 +257,7 @@ char streams shr_strdata_nml - SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SST_AQUAPFILE,SST_AQUAP_CONSTANT,SOM,SOM_AQUAP,IAF,NULL,COPYALL + SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SST_AQUAP11,SST_AQUAP12,SST_AQUAP13,SST_AQUAP14,SST_AQUAP15,SST_AQUAPFILE,SST_AQUAP_CONSTANT,SOM,SOM_AQUAP,IAF,NULL,COPYALL General method that operates on the data. This is generally implemented in the data models but is set in the strdata method for @@ -323,6 +323,11 @@ SST_AQUAP8 SST_AQUAP9 SST_AQUAP10 + SST_AQUAP11 + SST_AQUAP12 + SST_AQUAP13 + SST_AQUAP14 + SST_AQUAP15 SST_AQUAPFILE SST_AQUAP_CONSTANT SOM diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index e692882c9db1..002ed655b404 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -984,6 +984,7 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) integer :: i real(r8) :: tmp, tmp1, pi real(r8) :: rlon(lsize), rlat(lsize) + real(r8) :: mean_SST, delta_SST real(r8), parameter :: pio180 = SHR_CONST_PI/180._r8 @@ -1013,8 +1014,8 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) ! Control - if (sst_option < 1 .or. sst_option > 10) then - call shr_sys_abort ('prescribed_sst: ERROR: sst_option must be between 1 and 10') + if (sst_option < 1 .or. sst_option > 15) then + call shr_sys_abort ('prescribed_sst: ERROR: sst_option must be between 1 and 15') end if if (sst_option == 1 .or. sst_option == 6 .or. sst_option == 7 .or. sst_option == 8) then @@ -1174,6 +1175,55 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) end do end if + !------------------------------------------------------------------------------- + ! RCEMIP phase 2 - Mock-Walker + + ! MW_295dT1p25 - mean SST = 295 / dSST = 1.25 K + + if (sst_option == 11) then + mean_SST = 295 + delta_SST = 1.25 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + + ! MW_300dT0p625 - mean SST = 300 / dSST = 0.625 K + if (sst_option == 12) then + mean_SST = 300 + delta_SST = 0.625 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + + ! MW_300dT1p25 - mean SST = 300 / dSST = 1.25 K + if (sst_option == 13) then + mean_SST = 300 + delta_SST = 1.25 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + + ! MW_300dT2p5 - mean SST = 300 / dSST = 2.5 K + if (sst_option == 14) then + mean_SST = 300 + delta_SST = 2.5 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + + ! MW_305dT1p25 - mean SST = 305 / dSST = 1.25 K + if (sst_option == 15) then + mean_SST = 305 + delta_SST = 1.25 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + end subroutine prescribed_sst end module docn_comp_mod From 3803486652580fc8f983ec5af650c4399fa21870 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 14 May 2024 12:45:03 -0700 Subject: [PATCH 05/48] add TMQS diagnostic for RCEMIP2 --- components/eam/src/physics/cam/cam_diagnostics.F90 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/components/eam/src/physics/cam/cam_diagnostics.F90 b/components/eam/src/physics/cam/cam_diagnostics.F90 index ead4f558f05b..0b4abed44720 100644 --- a/components/eam/src/physics/cam/cam_diagnostics.F90 +++ b/components/eam/src/physics/cam/cam_diagnostics.F90 @@ -306,6 +306,7 @@ subroutine diag_init() call addfld ('MQ',(/ 'lev' /), 'A','kg/m2','Water vapor mass in layer') call addfld ('TMQ',horiz_only, 'A','kg/m2','Total (vertically integrated) precipitable water', & standard_name='atmosphere_mass_content_of_water_vapor') + call addfld ('TMQS',horiz_only, 'A','kg/m2','Total (vertically integrated) saturated precipitable water') call addfld ('TTQ',horiz_only, 'A', 'kg/m/s','Total (vertically integrated) vapor transport') call addfld ('TUQ',horiz_only, 'A','kg/m/s','Total (vertically integrated) zonal water flux') call addfld ('TVQ',horiz_only, 'A','kg/m/s','Total (vertically integrated) meridional water flux') @@ -1335,6 +1336,15 @@ subroutine diag_phys_writeout(state, psl) ftem(:ncol,1) = ftem(:ncol,1) + ftem(:ncol,k) end do call outfld ('TMQ ',ftem, pcols ,lchnk ) +! +! Mass of saturated q vertically integrated +! + call qsat(state%t(:ncol,:), state%pmid(:ncol,:), tem2(:ncol,:), ftem(:ncol,:)) + ftem(:ncol,:) = ftem(:ncol,:) * state%pdel(:ncol,:) * rga + do k=2,pver + ftem(:ncol,1) = ftem(:ncol,1) + ftem(:ncol,k) + end do + call outfld ('TMQS ',ftem, pcols ,lchnk ) ! ! Mass of vertically integrated water vapor flux ! From 53b636a5cbd3d5796021f4983b62d3d73d27e770 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 3 Jul 2024 14:33:44 -0700 Subject: [PATCH 06/48] update docn_comp_mod.F90 --- .../data_comps/docn/src/docn_comp_mod.F90 | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index 002ed655b404..ac1278673656 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -1178,46 +1178,46 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) !------------------------------------------------------------------------------- ! RCEMIP phase 2 - Mock-Walker - ! MW_295dT1p25 - mean SST = 295 / dSST = 1.25 K + ! MW_295dT1p25 - mean SST = 295 K / dSST = 1.25 K if (sst_option == 11) then - mean_SST = 295 + mean_SST = 295 - TkFrz delta_SST = 1.25 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if - ! MW_300dT0p625 - mean SST = 300 / dSST = 0.625 K + ! MW_300dT0p625 - mean SST = 300 K / dSST = 0.625 K if (sst_option == 12) then - mean_SST = 300 + mean_SST = 300 - TkFrz delta_SST = 0.625 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if - ! MW_300dT1p25 - mean SST = 300 / dSST = 1.25 K + ! MW_300dT1p25 - mean SST = 300 K / dSST = 1.25 K if (sst_option == 13) then - mean_SST = 300 + mean_SST = 300 - TkFrz delta_SST = 1.25 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if - ! MW_300dT2p5 - mean SST = 300 / dSST = 2.5 K + ! MW_300dT2p5 - mean SST = 300 K / dSST = 2.5 K if (sst_option == 14) then - mean_SST = 300 + mean_SST = 300 - TkFrz delta_SST = 2.5 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if - ! MW_305dT1p25 - mean SST = 305 / dSST = 1.25 K + ! MW_305dT1p25 - mean SST = 305 K / dSST = 1.25 K if (sst_option == 15) then - mean_SST = 305 + mean_SST = 305 - TkFrz delta_SST = 1.25 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) From 57fee69019b3ccc1ed5682966c486cf59bd430ad Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 3 Jul 2024 14:34:37 -0700 Subject: [PATCH 07/48] add TOA rad history variables --- .../eam/src/physics/crm/rrtmgp/radiation.F90 | 19 +++++++++++++++++++ .../eam/src/physics/rrtmgp/radiation.F90 | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/components/eam/src/physics/crm/rrtmgp/radiation.F90 b/components/eam/src/physics/crm/rrtmgp/radiation.F90 index b7b253d1b6c4..b80ea0c2fe10 100644 --- a/components/eam/src/physics/crm/rrtmgp/radiation.F90 +++ b/components/eam/src/physics/crm/rrtmgp/radiation.F90 @@ -768,6 +768,18 @@ subroutine radiation_init(state) call addfld('FLNTC'//diag(icall), horiz_only, 'A', 'W/m2', & 'Clearsky net longwave flux at top of model', & sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) call addfld('LWCF'//diag(icall), horiz_only, 'A', 'W/m2', & 'Longwave cloud forcing', & sampling_seq='rad_lwsw', flag_xyfill=.true.) @@ -2494,6 +2506,7 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) ! Working arrays real(r8), dimension(pcols,pver+1) :: flux_up, flux_dn, flux_net integer :: ncol + integer :: ktop_rad = 1 ncol = state%ncol @@ -2531,6 +2544,12 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) call outfld('FLUTC'//diag(icall), flux_clr%flux_up(1:ncol,ktop), ncol, state%lchnk) call outfld('FLDSC'//diag(icall), flux_clr%flux_dn(1:ncol,kbot+1), ncol, state%lchnk) + ! TOA fluxes (above model top, use index to rad top) + call outfld('FLUTOA'//diag(icall), flux_all%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOA'//diag(icall), flux_all%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLUTOAC'//diag(icall), flux_clr%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOAC'//diag(icall), flux_clr%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + ! Calculate and output the cloud radiative effect (LWCF in history) cloud_radiative_effect(1:ncol) = flux_all%flux_net(1:ncol,ktop) - flux_clr%flux_net(1:ncol,ktop) call outfld('LWCF'//diag(icall), cloud_radiative_effect, ncol, state%lchnk) diff --git a/components/eam/src/physics/rrtmgp/radiation.F90 b/components/eam/src/physics/rrtmgp/radiation.F90 index 0c715000951d..5c87c3376d9d 100644 --- a/components/eam/src/physics/rrtmgp/radiation.F90 +++ b/components/eam/src/physics/rrtmgp/radiation.F90 @@ -767,6 +767,18 @@ subroutine radiation_init(state,pbuf) call addfld('FLNTC'//diag(icall), horiz_only, 'A', 'W/m2', & 'Clearsky net longwave flux at top of model', & sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) call addfld('LWCF'//diag(icall), horiz_only, 'A', 'W/m2', & 'Longwave cloud forcing', & sampling_seq='rad_lwsw', flag_xyfill=.true.) @@ -2375,6 +2387,7 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) ! Working arrays real(r8), dimension(pcols,pver+1) :: flux_up, flux_dn, flux_net integer :: ncol + integer :: ktop_rad = 1 ncol = state%ncol @@ -2412,6 +2425,12 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) call outfld('FLUTC'//diag(icall), flux_clr%flux_up(1:ncol,ktop), ncol, state%lchnk) call outfld('FLDSC'//diag(icall), flux_clr%flux_dn(1:ncol,kbot+1), ncol, state%lchnk) + ! TOA fluxes (above model top, use index to rad top) + call outfld('FLUTOA'//diag(icall), flux_all%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOA'//diag(icall), flux_all%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLUTOAC'//diag(icall), flux_clr%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOAC'//diag(icall), flux_clr%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + ! Calculate and output the cloud radiative effect (LWCF in history) cloud_radiative_effect(1:ncol) = flux_all%flux_net(1:ncol,ktop) - flux_clr%flux_net(1:ncol,ktop) call outfld('LWCF'//diag(icall), cloud_radiative_effect, ncol, state%lchnk) From 7c182a67279b6499b34ba9d2d4aee6953b221c1e Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Thu, 27 Jun 2024 17:26:26 -0500 Subject: [PATCH 08/48] data land for moab driver use existing lnd_domain attribute in infodata we do not differentiate on coupler side if data land or not still need to send the proper land domain file --- .../data_comps/dlnd/src/dlnd_comp_mod.F90 | 181 +++++++++++++++++- .../data_comps/dlnd/src/lnd_comp_mct.F90 | 40 +++- 2 files changed, 216 insertions(+), 5 deletions(-) diff --git a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 index bf723259be48..7768b2dfe815 100644 --- a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 +++ b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 @@ -29,6 +29,10 @@ module dlnd_comp_mod use dlnd_shr_mod , only: domain_fracname ! namelist input use dlnd_shr_mod , only: nullstr +#ifdef HAVE_MOAB + use seq_comm_mct, only : mlnid ! id of moab lnd app + use iso_c_binding +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -100,6 +104,12 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & scmMode, scmlat, scmlon) ! !DESCRIPTION: initialize dlnd model +#ifdef HAVE_MOAB + use iMOAB, only: iMOAB_DefineTagStorage, iMOAB_GetDoubleTagStorage, & + iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & + iMOAB_ResolveSharedEntities, iMOAB_CreateVertices, & + iMOAB_GetMeshInfo, iMOAB_UpdateMeshInfo, iMOAB_WriteMesh +#endif implicit none ! !INPUT/OUTPUT PARAMETERS: @@ -135,6 +145,18 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & character(nec_len) :: nec_str ! elevation class, as character string character(*), parameter :: domain_fracname_unset = 'null' +#ifdef HAVE_MOAB + character*400 tagname + real(R8) latv, lonv + integer iv, tagindex, ilat, ilon !, arrsize, nfields ! ierr is already defined as integer above + real(R8), allocatable, target :: data(:) + integer(IN), pointer :: idata(:) ! temporary + real(r8), dimension(:), allocatable :: moab_vert_coords ! temporary +#ifdef MOABDEBUG + character*100 outfile, wopts +#endif +#endif + !--- formats --- character(*), parameter :: F00 = "('(dlnd_comp_init) ',8a)" character(*), parameter :: F0L = "('(dlnd_comp_init) ',a, l2)" @@ -256,6 +278,119 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & call t_stopf('dlnd_initmctdom') +#ifdef HAVE_MOAB + ilat = mct_aVect_indexRA(ggrid%data,'lat') + ilon = mct_aVect_indexRA(ggrid%data,'lon') + allocate(moab_vert_coords(lsize*3)) + do iv = 1, lsize + lonv = ggrid%data%rAttr(ilon, iv) * SHR_CONST_PI/180. + latv = ggrid%data%rAttr(ilat, iv) * SHR_CONST_PI/180. + moab_vert_coords(3*iv-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*iv-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*iv )=SIN(latv) + enddo + + ! create the vertices with coordinates from MCT domain + ierr = iMOAB_CreateVertices(mlnid, lsize*3, 3, moab_vert_coords) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to create MOAB vertices in data lnd model') + + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mlnid, tagname, & + 0, & ! dense, integer + 1, & ! number of components + tagindex ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to retrieve GLOBAL_ID tag ') + + ! get list of global IDs for Dofs + call mct_gsMap_orderedPoints(gsMap, my_task, idata) + + ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsize, & + 0, & ! vertex type + idata) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to set GLOBAL_ID tag ') + + ierr = iMOAB_ResolveSharedEntities( mlnid, lsize, idata ); + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to resolve shared entities') + + deallocate(moab_vert_coords) + deallocate(idata) + + ierr = iMOAB_UpdateMeshInfo( mlnid ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to update mesh info ') + + allocate(data(lsize)) + ierr = iMOAB_DefineTagStorage( mlnid, "area:aream:frac:mask"//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create tag: area:aream:frac:mask' ) + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'area'),:) + tagname='area'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to get area tag ') + + ! set the same data for aream (model area) as area + ! data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'aream'),:) + tagname='aream'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set aream tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'mask'),:) + tagname='mask'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set mask tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'frac'),:) + tagname='frac'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set frac tag ') + + deallocate(data) + + ! define tags + ierr = iMOAB_DefineTagStorage( mlnid, trim(seq_flds_x2l_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_x2l_fields tags ') + + ierr = iMOAB_DefineTagStorage( mlnid, trim(seq_flds_l2x_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_l2x_fields tags ') +#ifdef MOABDEBUG + ! debug test + outfile = 'LndDataMesh.h5m'//C_NULL_CHAR + wopts = ';PARALLEL=WRITE_PART'//C_NULL_CHAR ! + ! write out the mesh file to disk + ierr = iMOAB_WriteMesh(mlnid, trim(outfile), trim(wopts)) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' ERROR in writing data mesh lnd ') + endif +#endif +#endif !---------------------------------------------------------------------------- ! Initialize MCT attribute vectors !---------------------------------------------------------------------------- @@ -339,8 +474,15 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & inst_suffix, logunit, case_name) ! !DESCRIPTION: run method for dlnd model - implicit none +#ifdef HAVE_MOAB +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif + use seq_flds_mod , only: seq_flds_l2x_fields + use seq_flds_mod , only: moab_set_tag_from_av +#endif + implicit none ! !INPUT/OUTPUT PARAMETERS: type(ESMF_Clock) , intent(in) :: EClock type(mct_aVect) , intent(inout) :: x2l @@ -366,6 +508,17 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & integer(IN) :: nu ! unit number logical :: write_restart ! restart now character(len=18) :: date_str +#ifdef HAVE_MOAB + real(R8), allocatable, target :: datam(:) + type(mct_list) :: temp_list + integer :: size_list, index_list, lsize + type(mct_string) :: mctOStr ! + character*400 tagname, mct_field +#ifdef MOABDEBUG + integer :: cur_dlnd_stepno, ierr + character*100 outfile, wopts, lnum +#endif +#endif character(*), parameter :: F00 = "('(dlnd_comp_run) ',8a)" character(*), parameter :: F04 = "('(dlnd_comp_run) ',2a,2i8,'s')" @@ -464,6 +617,32 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & call t_stopf('DLND_RUN') +#ifdef HAVE_MOAB + lsize = mct_avect_lsize(l2x) ! is it the same as mct_avect_lsize(avstrm) ? + allocate(datam(lsize)) ! + call mct_list_init(temp_list ,seq_flds_l2x_fields) + size_list=mct_list_nitem (temp_list) + do index_list = 1, size_list + call mct_list_get(mctOStr,index_list,temp_list) + mct_field = mct_string_toChar(mctOStr) + tagname= trim(mct_field)//C_NULL_CHAR + call moab_set_tag_from_av(tagname, l2x, index_list, mlnid, datam, lsize) ! loop over all a2x fields, not just a few + enddo + call mct_list_clean(temp_list) + deallocate(datam) ! maybe we should keep it around, deallocate at the final only? + +#ifdef MOABDEBUG + call seq_timemgr_EClockGetData( EClock, stepno=cur_dlnd_stepno ) + write(lnum,"(I0.2)")cur_dlnd_stepno + outfile = 'dlnd_comp_run_'//trim(lnum)//'.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mlnid, outfile, wopts) + if (ierr > 0 ) then + write(logunit,*) 'Failed to write data lnd component state ' + endif +#endif +#endif + end subroutine dlnd_comp_run !=============================================================================== diff --git a/components/data_comps/dlnd/src/lnd_comp_mct.F90 b/components/data_comps/dlnd/src/lnd_comp_mct.F90 index f5193ca84580..8361a8522d8a 100644 --- a/components/data_comps/dlnd/src/lnd_comp_mct.F90 +++ b/components/data_comps/dlnd/src/lnd_comp_mct.F90 @@ -16,7 +16,11 @@ module lnd_comp_mct use dlnd_comp_mod , only: dlnd_comp_init, dlnd_comp_run, dlnd_comp_final use dlnd_shr_mod , only: dlnd_shr_read_namelists use seq_flds_mod , only: seq_flds_x2l_fields, seq_flds_l2x_fields - +#ifdef HAVE_MOAB + use seq_comm_mct, only : mlnid ! iMOAB app id for lnd + use iso_c_binding + use iMOAB , only: iMOAB_RegisterApplication +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -52,7 +56,9 @@ module lnd_comp_mct !=============================================================================== subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) - +#ifdef HAVE_MOAB + use shr_stream_mod, only: shr_stream_getDomainInfo, shr_stream_getFile +#endif ! !DESCRIPTION: initialize dlnd model implicit none @@ -78,6 +84,16 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) real(R8) :: scmLon = shr_const_SPVAL ! single column lon character(*), parameter :: subName = "(lnd_init_mct) " !------------------------------------------------------------------------------- +#ifdef HAVE_MOAB + character(CL) :: filePath ! generic file path + character(CL) :: fileName ! generic file name + character(CS) :: timeName ! domain file: time variable name + character(CS) :: lonName ! domain file: lon variable name + character(CS) :: latName ! domain file: lat variable name + character(CS) :: hgtName ! domain file: hgt variable name + character(CS) :: maskName ! domain file: mask variable name + character(CS) :: areaName ! domain file: area variable name +#endif ! Set cdata pointers call seq_cdata_setptrs(cdata, & @@ -146,13 +162,29 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) !---------------------------------------------------------------------------- ! Initialize dlnd !---------------------------------------------------------------------------- - +#ifdef HAVE_MOAB + ierr = iMOAB_RegisterApplication(trim("DLND")//C_NULL_CHAR, mpicom, compid, mlnid) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in registering data lnd comp' + call shr_sys_abort(subname//' ERROR in registering data lnd comp') + endif +#endif call dlnd_comp_init(Eclock, x2l, l2x, & seq_flds_x2l_fields, seq_flds_l2x_fields, & SDLND, gsmap, ggrid, mpicom, compid, my_task, master_task, & inst_suffix, inst_name, logunit, read_restart, & scmMode, scmlat, scmlon) - +#ifdef HAVE_MOAB + if (my_task == master_task) then + call shr_stream_getDomainInfo(SDLND%stream(1), filePath,fileName,timeName,lonName, & + latName,hgtName,maskName,areaName) + call shr_stream_getFile(filePath,fileName) + ! send path of data lnd domain to MOAB coupler. + call seq_infodata_PutData( infodata, lnd_domain=fileName) ! we use the same one for regular case + ! in regular case, it is copied from fatmlndfrc ; so we don't know if it is data land or not + write(logunit,*), ' filename: ', filename + endif +#endif !---------------------------------------------------------------------------- ! Fill infodata that needs to be returned from dlnd !---------------------------------------------------------------------------- From 2ad116ba0b787f1c9ab391eb3d95025a27cb19db Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 28 Jun 2024 09:12:11 -0500 Subject: [PATCH 09/48] domain file or land data case domain file is actually set, in shr_strdata_type%domainFile all the other data models use the first stream file for domain information land is the one that actually has that member set in the SDLND structure it is easier to get it from there, then from stream the domain file in regular land case is defined by fatmlndfrac file --- .../data_comps/dlnd/src/lnd_comp_mct.F90 | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/components/data_comps/dlnd/src/lnd_comp_mct.F90 b/components/data_comps/dlnd/src/lnd_comp_mct.F90 index 8361a8522d8a..b699ec217f00 100644 --- a/components/data_comps/dlnd/src/lnd_comp_mct.F90 +++ b/components/data_comps/dlnd/src/lnd_comp_mct.F90 @@ -84,16 +84,6 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) real(R8) :: scmLon = shr_const_SPVAL ! single column lon character(*), parameter :: subName = "(lnd_init_mct) " !------------------------------------------------------------------------------- -#ifdef HAVE_MOAB - character(CL) :: filePath ! generic file path - character(CL) :: fileName ! generic file name - character(CS) :: timeName ! domain file: time variable name - character(CS) :: lonName ! domain file: lon variable name - character(CS) :: latName ! domain file: lat variable name - character(CS) :: hgtName ! domain file: hgt variable name - character(CS) :: maskName ! domain file: mask variable name - character(CS) :: areaName ! domain file: area variable name -#endif ! Set cdata pointers call seq_cdata_setptrs(cdata, & @@ -176,13 +166,9 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) scmMode, scmlat, scmlon) #ifdef HAVE_MOAB if (my_task == master_task) then - call shr_stream_getDomainInfo(SDLND%stream(1), filePath,fileName,timeName,lonName, & - latName,hgtName,maskName,areaName) - call shr_stream_getFile(filePath,fileName) - ! send path of data lnd domain to MOAB coupler. - call seq_infodata_PutData( infodata, lnd_domain=fileName) ! we use the same one for regular case + call seq_infodata_PutData( infodata, lnd_domain=SDLND%domainFile) ! we use the same one for regular case ! in regular case, it is copied from fatmlndfrc ; so we don't know if it is data land or not - write(logunit,*), ' filename: ', filename + write(logunit,*), ' use this land domain file: ', SDLND%domainFile endif #endif !---------------------------------------------------------------------------- From 9f0a28c35186b7f68daf7cf79b1643258ac73485 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 15 Jul 2024 10:33:50 -0500 Subject: [PATCH 10/48] apply weights is not used anymore in fractions iMOAB_ApplyScalarProjectionWeights is not used anymore all maps are done now with seq_map method, and there is only one instance of iMOAB_ApplyScalarProjectionWeights in E3SM using moab driver --- driver-moab/main/seq_frac_mct.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-moab/main/seq_frac_mct.F90 b/driver-moab/main/seq_frac_mct.F90 index a89fc0b9aa4f..031845a102ef 100644 --- a/driver-moab/main/seq_frac_mct.F90 +++ b/driver-moab/main/seq_frac_mct.F90 @@ -180,7 +180,7 @@ module seq_frac_mct use iMOAB, only : iMOAB_DefineTagStorage, iMOAB_SetDoubleTagStorage, & iMOAB_GetMeshInfo, iMOAB_SetDoubleTagStorageWithGid, iMOAB_WriteMesh, & - iMOAB_ApplyScalarProjectionWeights, iMOAB_SendElementTag, iMOAB_ReceiveElementTag, & + iMOAB_SendElementTag, iMOAB_ReceiveElementTag, & iMOAB_FreeSenderBuffers, iMOAB_GetDoubleTagStorage #ifdef MOABDEBUG use seq_comm_mct, only : num_moab_exports From ffb3b6103bc182b235a0c9392e56be41c801dc85 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 27 Aug 2024 15:10:10 -0500 Subject: [PATCH 11/48] initialize factors in area correction computations to 1.0 only intel on chrysalis complained about it It could be a cause for trouble later on --- driver-moab/main/component_mod.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/driver-moab/main/component_mod.F90 b/driver-moab/main/component_mod.F90 index 88d152a95685..3e8ba9042a58 100644 --- a/driver-moab/main/component_mod.F90 +++ b/driver-moab/main/component_mod.F90 @@ -748,6 +748,7 @@ subroutine component_init_areacor_moab (comp, mbccid, mbcxid, seq_flds_c2x_fluxe lsize = comp(1)%mblsize allocate(areas (lsize, 3)) ! lsize is along grid; read mask too allocate(factors (lsize, 2)) + factors = 1.0 ! initialize with 1.0 all factors; then maybe correct them ! get areas tagname='area:aream:mask'//C_NULL_CHAR arrsize = 3 * lsize From c99179079aee1d1b793ecf07530252046e2e40d6 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:19:19 -0600 Subject: [PATCH 12/48] HOMME: add aurora, polaris, spot machine files --- .../homme/cmake/machineFiles/aurora-aot.cmake | 64 ++++++++++++++++ .../homme/cmake/machineFiles/aurora-jit.cmake | 58 +++++++++++++++ .../cmake/machineFiles/chrysalis-bfb.cmake | 2 + .../homme/cmake/machineFiles/chrysalis.cmake | 2 + .../homme/cmake/machineFiles/polaris-a100.sh | 74 +++++++++++++++++++ .../cmake/machineFiles/spot-aot-AB2.cmake | 63 ++++++++++++++++ 6 files changed, 263 insertions(+) create mode 100644 components/homme/cmake/machineFiles/aurora-aot.cmake create mode 100644 components/homme/cmake/machineFiles/aurora-jit.cmake create mode 100644 components/homme/cmake/machineFiles/polaris-a100.sh create mode 100644 components/homme/cmake/machineFiles/spot-aot-AB2.cmake diff --git a/components/homme/cmake/machineFiles/aurora-aot.cmake b/components/homme/cmake/machineFiles/aurora-aot.cmake new file mode 100644 index 000000000000..b6fe34a78d72 --- /dev/null +++ b/components/homme/cmake/machineFiles/aurora-aot.cmake @@ -0,0 +1,64 @@ +#module restore +#module load oneapi/eng-compiler/2022.12.30.005 +#module load intel_compute_runtime/release/agama-devel-627 +#module load spack cmake +#module list + + +SET (SUNSPOT_MACHINE TRUE CACHE BOOL "") + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") +SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +#set(KOKKOS_HOME "/home/onguba/kokkos-build/mar05-aot/install" CACHE STRING "") +#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +SET(CMAKE_CXX_STANDARD 17) + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +# -fsycl-link-huge-device-code for theta to get build +#JIT flags +#SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +#SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64") + +#AOT flags +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fsycl-link-huge-device-code -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") + +SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 4 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") + + diff --git a/components/homme/cmake/machineFiles/aurora-jit.cmake b/components/homme/cmake/machineFiles/aurora-jit.cmake new file mode 100644 index 000000000000..1941fa9eb3f3 --- /dev/null +++ b/components/homme/cmake/machineFiles/aurora-jit.cmake @@ -0,0 +1,58 @@ +#module restore +#module load oneapi/eng-compiler/2022.12.30.005 +#module load intel_compute_runtime/release/agama-devel-627 +#module load spack cmake +#module list + + + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") +SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +#set(KOKKOS_HOME "/home/onguba/kokkos-build/jan03-2024/install" CACHE STRING "") +#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +SET(CMAKE_CXX_STANDARD 17) + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +# -fsycl-link-huge-device-code for theta to get build +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64") + +SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 4 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") + + diff --git a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake index b9f0d41a0606..fa1f1ac545c5 100644 --- a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake +++ b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake @@ -17,6 +17,8 @@ ENDIF() SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") +SET (CHRYSALIS_MACHINE TRUE CACHE BOOL "") + # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/chrysalis.cmake b/components/homme/cmake/machineFiles/chrysalis.cmake index 68ff76ec8082..97bc682c9546 100644 --- a/components/homme/cmake/machineFiles/chrysalis.cmake +++ b/components/homme/cmake/machineFiles/chrysalis.cmake @@ -17,6 +17,8 @@ ENDIF() SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") +SET (CHRYSALIS_MACHINE TRUE CACHE BOOL "") + # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/polaris-a100.sh b/components/homme/cmake/machineFiles/polaris-a100.sh new file mode 100644 index 000000000000..2b63c61a55e7 --- /dev/null +++ b/components/homme/cmake/machineFiles/polaris-a100.sh @@ -0,0 +1,74 @@ +#Currently Loaded Modules: +# 1) craype-x86-rome 6) craype/2.7.15 11) cray-libpals/1.1.7 16) nvhpc-mixed/21.9 +# 2) libfabric/1.11.0.4.125 7) cray-dsmml/0.2.2 12) PrgEnv-gnu/8.3.3 17) cudatoolkit-standalone/11.6.2 +# 3) craype-network-ofi 8) cray-pmi/6.1.2 13) gnu-parallel/2021-09-22 18) cmake/3.23.2 +# 4) perftools-base/22.05.0 9) cray-pmi-lib/6.0.17 14) gcc/11.2.0 +# 5) craype-accel-nvidia80 10) cray-pals/1.1.7 15) cray-mpich/8.1.16 + + + +#SET(HOMMEXX_EXEC_SPACE CUDA CACHE STRING "") +#SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") +#SET(HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING "") + +# cray-hdf5-parallel/1.12.0.6 cray-netcdf-hdf5parallel/4.7.4.6 cray-parallel-netcdf/1.12.1.6 +#SET(NETCDF_DIR $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") +#SET(PNETCDF_DIR $ENV{CRAY_PARALLEL_NETCDF_DIR} CACHE FILEPATH "") +#SET(HDF5_DIR $ENV{CRAY_HDF5_PARALLEL_PREFIX} CACHE FILEPATH "") + +#for scorpio +#SET (NetCDF_C_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") +#SET (NetCDF_Fortran_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK FALSE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +SET(CUDA_BUILD TRUE CACHE BOOL "") + +#SET(HOMMEXX_BFB_TESTING TRUE CACHE BOOL "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "") +SET(Kokkos_ENABLE_CUDA ON CACHE BOOL "") +SET(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "") +SET(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "") +#SET(Kokkos_ARCH_ZEN2 ON CACHE BOOL "") # works, and perf same if both AMPERE80 and ZEN2 are on +#SET(Kokkos_ENABLE_CUDA_UVM ON CACHE BOOL "") +SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") +#SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") + +#SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +#SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +#SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") +SET(CMAKE_C_COMPILER "cc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "CC" CACHE STRING "") + +#SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +#SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +#SET(CMAKE_CXX_COMPILER "${CMAKE_CURRENT_SOURCE_DIR}/../../externals/kokkos/bin/nvcc_wrapper" CACHE STRING "") + +# Note: need to set MPICH_CXX env variable and perhaps NVCC_WRAPPER_DEFAULT_COMPILER + +SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "") + +SET(ENABLE_OPENMP OFF CACHE BOOL "") +SET(ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +SET(ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +SET(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "") + +#SET(HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +SET(USE_NUM_PROCS 4 CACHE STRING "") + +SET(USE_MPIEXEC "srun" CACHE STRING "") +#SET(CPRNC_DIR /global/cfs/cdirs/e3sm/tools/cprnc CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/spot-aot-AB2.cmake b/components/homme/cmake/machineFiles/spot-aot-AB2.cmake new file mode 100644 index 000000000000..23fad2361ccf --- /dev/null +++ b/components/homme/cmake/machineFiles/spot-aot-AB2.cmake @@ -0,0 +1,63 @@ +#module restore +#module load oneapi/eng-compiler/2022.12.30.005 +#module load intel_compute_runtime/release/agama-devel-627 +#module load spack cmake +#module list + +SET (SUNSPOT_MACHINE TRUE CACHE BOOL "") + +SET (HOMMEXX_MPI_ON_DEVICE TRUE CACHE BOOL "") + +#SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +#set(KOKKOS_HOME "/home/onguba/kokkos-build/june22-2024-aot/install" CACHE STRING "") +#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") + +SET (NetCDF_Fortran_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +SET (NetCDF_C_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +#SET(CMAKE_CXX_STANDARD 17) +SET(CMAKE_CXX_STANDARD 17 CACHE STRING "CXX Standard") + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") + +#-fpscomp does not actually solve the issue with bools in here,another suggestion was -fp-model=precise, not working either +SET(ADD_Fortran_FLAGS " -fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS " -std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 4 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") + + From 1379db52a29ff31764b6268eae1f88fbedbae252 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:29:27 -0600 Subject: [PATCH 13/48] HOMME: CMake and CPP mods to support SYCL backend --- components/homme/CMakeLists.txt | 8 +++++--- components/homme/cmake/HommeMacros.cmake | 8 +++++++- components/homme/src/share/cxx/Config.hpp | 2 +- components/homme/src/share/cxx/ExecSpaceDefs.hpp | 4 ++++ .../homme/test_execs/share_kokkos_ut/CMakeLists.txt | 4 ++-- .../homme/test_execs/thetal_kokkos_ut/CMakeLists.txt | 3 +++ 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index 80a89a296910..6fe81180ab54 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -206,7 +206,9 @@ IF (HOMME_USE_KOKKOS) STRING (TOUPPER ${HOMMEXX_EXEC_SPACE} HOMMEXX_EXEC_SPACE_UPPER) - IF (HOMMEXX_EXEC_SPACE_UPPER STREQUAL "HIP") + IF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "SYCL") + SET (HOMMEXX_SYCL_SPACE ON) + ELSEIF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "HIP") SET (HOMMEXX_HIP_SPACE ON) ELSEIF (HOMMEXX_EXEC_SPACE_UPPER STREQUAL "CUDA") SET (HOMMEXX_CUDA_SPACE ON) @@ -303,7 +305,7 @@ SET (HOMMEXX_ENABLE_GPU_F90 FALSE) IF (HOMME_USE_KOKKOS) - IF (CUDA_BUILD OR HIP_BUILD) + IF (CUDA_BUILD OR HIP_BUILD OR SYCL_BUILD) SET (DEFAULT_VECTOR_SIZE 1) SET (HOMMEXX_ENABLE_GPU TRUE) SET (HOMMEXX_ENABLE_GPU_F90 TRUE) @@ -312,7 +314,7 @@ IF (HOMME_USE_KOKKOS) ENDIF() SET (HOMMEXX_VECTOR_SIZE ${DEFAULT_VECTOR_SIZE} CACHE STRING - "If AVX or Cuda or HIP don't take priority, use this software vector size.") + "If AVX or Cuda or HIP or SYCL don't take priority, use this software vector size.") IF (CMAKE_BUILD_TYPE_UPPER MATCHES "DEBUG" OR CMAKE_BUILD_TYPE_UPPER MATCHES "RELWITHDEBINFO") SET (HOMMEXX_DEBUG ON) diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index 6d073dbbe83b..5610947cb299 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -112,7 +112,13 @@ macro(createTestExec execName execType macroNP macroNC ADD_DEFINITIONS(-DHAVE_CONFIG_H) ADD_EXECUTABLE(${execName} ${EXEC_SOURCES}) - SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) + + if(SUNSPOT_MACHINE) + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE CXX) + else() + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) + endif() + IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY) ENDIF() diff --git a/components/homme/src/share/cxx/Config.hpp b/components/homme/src/share/cxx/Config.hpp index 684f9143beaf..b204b1dbd047 100644 --- a/components/homme/src/share/cxx/Config.hpp +++ b/components/homme/src/share/cxx/Config.hpp @@ -21,7 +21,7 @@ # endif #endif -#if ! defined HOMMEXX_CUDA_SPACE && ! defined HOMMEXX_OPENMP_SPACE && ! defined HOMMEXX_THREADS_SPACE && ! defined HOMMEXX_SERIAL_SPACE && ! defined HOMMEXX_HIP_SPACE +#if ! defined HOMMEXX_CUDA_SPACE && ! defined HOMMEXX_OPENMP_SPACE && ! defined HOMMEXX_THREADS_SPACE && ! defined HOMMEXX_SERIAL_SPACE && ! defined HOMMEXX_HIP_SPACE && ! defined HOMMEXX_SYCL_SPACE # define HOMMEXX_DEFAULT_SPACE #endif diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.hpp b/components/homme/src/share/cxx/ExecSpaceDefs.hpp index cd6649c7ab2d..82f5e803801c 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.hpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.hpp @@ -34,6 +34,10 @@ using HommexxGPU = Kokkos::Cuda; using HommexxGPU = Kokkos::Experimental::HIP; #endif +#ifdef KOKKOS_ENABLE_SYCL +using HommexxGPU = Kokkos::Experimental::SYCL; +#endif + #else using HommexxGPU = void; #endif diff --git a/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt index 3fbeff9f6f2c..bc788462ce6e 100644 --- a/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt @@ -10,7 +10,7 @@ SET(UTILS_TIMING_DIRS ${UTILS_TIMING_SRC_DIR} ${UTILS_TIMING_BIN_DIR}) # Note: need CUDA_BUILD and HOMMEXX_BFB_TESTING here, since the share # unit tests do not include a config.h file SET (COMMON_DEFINITIONS NP=4 NC=4) -IF (CUDA_BUILD OR HIP_BUILD) +IF (CUDA_BUILD OR HIP_BUILD OR SYCL_BUILD) SET(COMMON_DEFINITIONS ${COMMON_DEFINITIONS} HOMMEXX_ENABLE_GPU_F90) ENDIF() IF (HOMMEXX_BFB_TESTING) @@ -158,7 +158,7 @@ ELSE() SET (NUM_CPUS 1) ENDIF() cxx_unit_test (sphere_op_ut "${SPHERE_OP_UT_F90_SRCS}" "${SPHERE_OP_UT_CXX_SRCS}" "${SPHERE_OP_UT_INCLUDE_DIRS}" "${CONFIG_DEFINES}" ${NUM_CPUS}) -endif () +endif () #BFB ### Limiters unit test ### diff --git a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt index 205635e918cc..e8bf5e20bd03 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt @@ -11,6 +11,8 @@ SET(UTILS_TIMING_BIN_DIR ${HOMME_BINARY_DIR}/utils/cime/CIME/non_py/src/timing) THETAL_KOKKOS_SETUP() # This is needed to compile the lib and test executables with the correct options +#these vars shared between all targets, so changing one var +#for one test only won't work, config is built once and for the last test SET(THIS_CONFIG_IN ${HOMME_SOURCE_DIR}/src/theta-l_kokkos/config.h.cmake.in) SET(THIS_CONFIG_HC ${CMAKE_CURRENT_BINARY_DIR}/config.h.c) SET(THIS_CONFIG_H ${CMAKE_CURRENT_BINARY_DIR}/config.h) @@ -18,6 +20,7 @@ SET (NUM_POINTS 4) SET (NUM_PLEV 12) SET (QSIZE_D 4) SET (PIO_INTERP TRUE) + HommeConfigFile (${THIS_CONFIG_IN} ${THIS_CONFIG_HC} ${THIS_CONFIG_H} ) ADD_LIBRARY(thetal_kokkos_ut_lib From c7320a6c5e31c161cf8c48ecb72d0ad792506ddf Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:30:22 -0600 Subject: [PATCH 14/48] HOMME: replace some printf with Kokkos::printf --- components/homme/src/share/cxx/utilities/BfbUtils.hpp | 2 +- .../homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp | 4 ++-- .../homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/homme/src/share/cxx/utilities/BfbUtils.hpp b/components/homme/src/share/cxx/utilities/BfbUtils.hpp index e3570874e266..7fb4d042f7f2 100644 --- a/components/homme/src/share/cxx/utilities/BfbUtils.hpp +++ b/components/homme/src/share/cxx/utilities/BfbUtils.hpp @@ -64,7 +64,7 @@ KOKKOS_INLINE_FUNCTION ScalarType int_pow (ScalarType val, int k) { constexpr int max_shift = 30; if (k<0) { - printf ("k = %d\n",k); + Kokkos::printf ("k = %d\n",k); Kokkos::abort("int_pow implemented only for k>=0.\n"); } diff --git a/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp b/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp index ace1ba920141..d16769079729 100644 --- a/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp @@ -382,8 +382,8 @@ struct DirkFunctorImpl { kv.team_barrier(); if (it >= maxiter) { - printf("[DIRK] WARNING! Newton reached max iteration count," - " with deltaerr = %3.17f\n", deltaerr); + Kokkos::printf("[DIRK] WARNING! Newton reached max iteration count," + " with deltaerr = %3.17f\n", deltaerr); nerr = 1; } diff --git a/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp index cd3bf7c32526..7914c0a60e3a 100644 --- a/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp @@ -141,8 +141,8 @@ struct LimiterFunctor { [&](const int k,Real& result) { #ifndef HOMMEXX_BFB_TESTING if(diff_as_real(k) < 0){ - printf("WARNING:CAAR: dp3d too small. k=%d, dp3d(k)=%f, dp0=%f \n", - k+1,dp_as_real(k),dp0_as_real(k)); + Kokkos::printf("WARNING:CAAR: dp3d too small. k=%d, dp3d(k)=%f, dp0=%f \n", + k+1,dp_as_real(k),dp0_as_real(k)); } #endif result = result<=diff_as_real(k) ? result : diff_as_real(k); @@ -202,8 +202,8 @@ struct LimiterFunctor { for (int ivec=0; ivec Date: Thu, 5 Sep 2024 16:33:09 -0600 Subject: [PATCH 15/48] HOMME: prefer to use int to bool in a few places --- components/homme/src/share/control_mod.F90 | 1 + components/homme/src/share/cxx/GllFvRemap.cpp | 4 +- components/homme/src/share/cxx/GllFvRemap.hpp | 4 +- .../homme/src/share/cxx/GllFvRemapImpl.cpp | 8 ++-- .../homme/src/share/cxx/GllFvRemapImpl.hpp | 5 ++- .../homme/src/share/cxx/SimulationParams.hpp | 4 +- components/homme/src/share/gllfvremap_mod.F90 | 14 +++---- components/homme/src/share/namelist_mod.F90 | 6 +++ .../src/theta-l_kokkos/cxx/CamForcing.cpp | 2 +- .../theta-l_kokkos/cxx/EquationOfState.hpp | 4 +- .../src/theta-l_kokkos/cxx/ForcingFunctor.hpp | 4 +- .../cxx/HyperviscosityFunctorImpl.cpp | 8 +++- .../cxx/HyperviscosityFunctorImpl.hpp | 2 +- .../cxx/cxx_f90_interface_theta.cpp | 19 +++++----- .../src/theta-l_kokkos/prim_driver_mod.F90 | 38 ++++++++++++------- .../src/theta-l_kokkos/theta_f2c_mod.F90 | 12 +++--- 16 files changed, 79 insertions(+), 56 deletions(-) diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 0e9494f5a6cd..9c3c599b2324 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -43,6 +43,7 @@ module control_mod ! flag used by preqx, theta-l and theta-c models ! should be renamed to "hydrostatic_mode" logical, public :: theta_hydrostatic_mode + integer, public :: theta_hydrostatic_mode_integer integer, public :: tstep_type= 5 ! preqx timestepping options diff --git a/components/homme/src/share/cxx/GllFvRemap.cpp b/components/homme/src/share/cxx/GllFvRemap.cpp index e36dbc14d74f..a8f564958d46 100644 --- a/components/homme/src/share/cxx/GllFvRemap.cpp +++ b/components/homme/src/share/cxx/GllFvRemap.cpp @@ -16,7 +16,7 @@ namespace Homme { void init_gllfvremap_c (int nelemd, int np, int nf, int nf_max, - bool theta_hydrostatic_mode, + int theta_hydrostatic_mode, CF90Ptr fv_metdet, CF90Ptr g2f_remapd, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f) { auto& c = Context::singleton(); @@ -52,7 +52,7 @@ void GllFvRemap::init_boundary_exchanges () { } void GllFvRemap -::init_data (const int nf, const int nf_max, bool theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const int theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f) { m_impl->init_data(nf, nf_max, theta_hydrostatic_mode, fv_metdet, diff --git a/components/homme/src/share/cxx/GllFvRemap.hpp b/components/homme/src/share/cxx/GllFvRemap.hpp index 07e4bf58a903..2adff0aeaa96 100644 --- a/components/homme/src/share/cxx/GllFvRemap.hpp +++ b/components/homme/src/share/cxx/GllFvRemap.hpp @@ -40,7 +40,7 @@ class GllFvRemap { typedef Phys2T::const_type CPhys2T; typedef Phys3T::const_type CPhys3T; - void init_data(const int nf, const int nf_max, bool theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const int theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f); @@ -81,7 +81,7 @@ class GllFvRemap { extern "C" void init_gllfvremap_c(int nelemd, int np, int nf, int nf_max, - const bool theta_hydrostatic_mode, + const int theta_hydrostatic_mode, CF90Ptr fv_metdet, CF90Ptr g2f_remapd, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f); diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.cpp b/components/homme/src/share/cxx/GllFvRemapImpl.cpp index 6148f69cfa9c..d4ab5c89f510 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.cpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.cpp @@ -131,7 +131,7 @@ void GllFvRemapImpl::init_boundary_exchanges () { template using FV = Kokkos::View; void GllFvRemapImpl -::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const int theta_hydrostatic_mode, const Real* fv_metdet_r, const Real* g2f_remapd_r, const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r) { using Kokkos::create_mirror_view; @@ -142,7 +142,7 @@ ::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, " nf must be > 1.", Errors::err_not_implemented); auto& sp = Context::singleton().get(); - m_data.use_moisture = sp.moisture == MoistDry::MOIST; + m_data.use_moisture = sp.use_moisture; // Only in the unit test gllfvremap_ut does theta_hydrostatic_mode not already // == sp.theta_hydrostatic_mode. m_data.theta_hydrostatic_mode = sp.theta_hydrostatic_mode = theta_hydrostatic_mode; @@ -395,7 +395,7 @@ ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, c const auto hvcoord = m_hvcoord; const bool use_moisture = m_data.use_moisture; - const bool theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; + const int theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; const bool want_dp_fv_out = dp_fv_out_ptr != nullptr; VPhys2T dp_fv_out; @@ -605,7 +605,7 @@ run_fv_phys_to_dyn (const int timeidx, const CPhys2T& Ts, const CPhys3T& uvs, const auto fT = m_forcing.m_ft; const auto hvcoord = m_hvcoord; const auto dp3d = m_state.m_dp3d; - const bool theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; + const int theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; EquationOfState eos; eos.init(theta_hydrostatic_mode, hvcoord); ElementOps ops; ops.init(hvcoord); const auto tu_ne = m_tu_ne; diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.hpp b/components/homme/src/share/cxx/GllFvRemapImpl.hpp index 11738b2bf455..7388fddb1231 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.hpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.hpp @@ -60,7 +60,8 @@ struct GllFvRemapImpl { struct Data { int nelemd, qsize, nf2, n_dss_fld; - bool use_moisture, theta_hydrostatic_mode; + bool use_moisture; + int theta_hydrostatic_mode; static constexpr int nbuf1 = 2, nbuf2 = 1; Buf1 buf1[nbuf1]; @@ -107,7 +108,7 @@ struct GllFvRemapImpl { void init_buffers(const FunctorsBuffersManager& fbm); void init_boundary_exchanges(); - void init_data(const int nf, const int nf_max, const bool theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const int theta_hydrostatic_mode, const Real* fv_metdet_r, const Real* g2f_remapd_r, const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r); diff --git a/components/homme/src/share/cxx/SimulationParams.hpp b/components/homme/src/share/cxx/SimulationParams.hpp index b435911da2e6..4f36962b16c3 100644 --- a/components/homme/src/share/cxx/SimulationParams.hpp +++ b/components/homme/src/share/cxx/SimulationParams.hpp @@ -23,7 +23,7 @@ struct SimulationParams void print(std::ostream& out = std::cout); TimeStepType time_step_type; - MoistDry moisture; + bool use_moisture; RemapAlg remap_alg; TestCase test_case; ForcingAlg ftype = ForcingAlg::FORCING_OFF; @@ -77,7 +77,7 @@ inline void SimulationParams::print (std::ostream& out) { out << "\n************** CXX SimulationParams **********************\n\n"; out << " time_step_type: " << etoi(time_step_type) << "\n"; - out << " moisture: " << (moisture==MoistDry::DRY ? "dry" : "moist") << "\n"; + out << " use_moisture: " << (use_moisture ? "moist" : "dry") << "\n"; out << " remap_alg: " << etoi(remap_alg) << "\n"; out << " test case: " << etoi(test_case) << "\n"; out << " ftype: " << etoi(ftype) << "\n"; diff --git a/components/homme/src/share/gllfvremap_mod.F90 b/components/homme/src/share/gllfvremap_mod.F90 index e0e0fa6c4daa..e927f04aba06 100644 --- a/components/homme/src/share/gllfvremap_mod.F90 +++ b/components/homme/src/share/gllfvremap_mod.F90 @@ -265,22 +265,22 @@ end subroutine gfr_init subroutine gfr_init_hxx() bind(c) #if KOKKOS_TARGET - use control_mod, only: theta_hydrostatic_mode - use iso_c_binding, only: c_bool + use control_mod, only: theta_hydrostatic_mode_integer + use iso_c_binding, only: c_int interface - subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode, & + subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode_integer, & fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f) bind(c) - use iso_c_binding, only: c_bool, c_int, c_double + use iso_c_binding, only: c_int, c_double integer (c_int), value, intent(in) :: nelemd, np, nf, nf_max - logical (c_bool), value, intent(in) :: theta_hydrostatic_mode + integer (c_int), value, intent(in) :: theta_hydrostatic_mode_integer real (c_double), dimension(nf*nf,nelemd), intent(in) :: fv_metdet real (c_double), dimension(np,np,nf_max*nf_max), intent(in) :: g2f_remapd real (c_double), dimension(nf_max*nf_max,np,np), intent(in) :: f2g_remapd real (c_double), dimension(nf*nf,2,2,nelemd), intent(in) :: D_f, Dinv_f end subroutine init_gllfvremap_c end interface - logical (c_bool) :: thm - thm = theta_hydrostatic_mode + integer (c_int) :: thm + thm = theta_hydrostatic_mode_integer call init_gllfvremap_c(nelemd, np, gfr%nphys, nphys_max, thm, & gfr%fv_metdet, gfr%g2f_remapd, gfr%f2g_remapd, gfr%D_f, gfr%Dinv_f) #endif diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index 1d47090182ba..a3edaa07e235 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -41,6 +41,7 @@ module namelist_mod runtype, & integration, & ! integration method theta_hydrostatic_mode, & + theta_hydrostatic_mode_integer, & transport_alg , & ! SE Eulerian, classical SL, cell-integrated SL semi_lagrange_cdr_alg, & ! see control_mod for semi_lagrange_* descriptions semi_lagrange_cdr_check, & @@ -452,8 +453,10 @@ subroutine readnl(par) planar_slice = .false. theta_hydrostatic_mode = .true. ! for preqx, this must be .true. + theta_hydrostatic_mode_integer = 1 ! for preqx, this must be .true. #if ( defined MODEL_THETA_C || defined MODEL_THETA_L ) theta_hydrostatic_mode = .false. ! default NH + theta_hydrostatic_mode_integer = 0 ! default NH #endif @@ -850,7 +853,10 @@ subroutine readnl(par) call MPI_bcast(case_planar_bubble,1,MPIlogical_t,par%root,par%comm,ierr) #endif +if(theta_hydrostatic_mode) theta_hydrostatic_mode_integer = 1 +if(.not. theta_hydrostatic_mode) theta_hydrostatic_mode_integer = 0 call MPI_bcast(theta_hydrostatic_mode ,1,MPIlogical_t,par%root,par%comm,ierr) + call MPI_bcast(theta_hydrostatic_mode_integer ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(transport_alg ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_cdr_alg ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_cdr_check ,1,MPIlogical_t,par%root,par%comm,ierr) diff --git a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp index 02b999db16e9..bd7cee3e7c0a 100644 --- a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp @@ -33,7 +33,7 @@ static void apply_cam_forcing_tracers(const Real dt, ForcingFunctor& ff, if ( p.ftype == ForcingAlg::FORCING_2) adjustment = true; #endif - ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, adjustment, p.moisture); + ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, adjustment, p.use_moisture); GPTLstop("ApplyCAMForcing_tracers"); } diff --git a/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp b/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp index dd97720f1be2..a50a28d58f55 100644 --- a/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp @@ -23,7 +23,7 @@ class EquationOfState { EquationOfState () = default; - void init (const bool theta_hydrostatic_mode, + void init (const int theta_hydrostatic_mode, const HybridVCoord& hvcoord) { m_theta_hydrostatic_mode = theta_hydrostatic_mode; m_hvcoord = hvcoord; @@ -250,7 +250,7 @@ class EquationOfState { public: - bool m_theta_hydrostatic_mode; + int m_theta_hydrostatic_mode; HybridVCoord m_hvcoord; }; diff --git a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp index 28a702c1d273..00fa1deef667 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp @@ -236,7 +236,7 @@ class ForcingFunctor }); } - void tracers_forcing (const Real dt, const int np1, const int np1_qdp, const bool adjustment, const MoistDry moisture) { + void tracers_forcing (const Real dt, const int np1, const int np1_qdp, const bool adjustment, const bool use_moisture) { // The Functor needs to be fully setup to use this function assert (is_setup); @@ -245,7 +245,7 @@ class ForcingFunctor m_np1_qdp = np1_qdp; m_adjustment = adjustment; - m_moist = (moisture==MoistDry::MOIST); + m_moist = use_moisture; Kokkos::parallel_for("temperature, NH perturb press, FQps",m_policy_tracers_pre,*this); Kokkos::fence(); diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp index 046e6f9956d4..55792051d33b 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp @@ -118,9 +118,13 @@ void HyperviscosityFunctorImpl::init_params(const SimulationParams& params) m_eos.init(params.theta_hydrostatic_mode,m_hvcoord); #ifdef HOMMEXX_BFB_TESTING - m_process_nh_vars = true; + m_process_nh_vars = 1; #else - m_process_nh_vars = !params.theta_hydrostatic_mode; + if (params.theta_hydrostatic_mode){ + m_process_nh_vars = 0; + }else{ + m_process_nh_vars = 1; + } #endif } diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp index a55ecbb365f9..993d525422f5 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp @@ -397,7 +397,7 @@ class HyperviscosityFunctorImpl Buffers m_buffers; HybridVCoord m_hvcoord; - bool m_process_nh_vars; + int m_process_nh_vars; // Policies Kokkos::TeamPolicy m_policy_update_states; diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index ec4e2cbe6328..40c4ae64dc98 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -43,12 +43,13 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, const Real& nu, const Real& nu_p, const Real& nu_q, const Real& nu_s, const Real& nu_div, const Real& nu_top, const int& hypervis_order, const int& hypervis_subcycle, const int& hypervis_subcycle_tom, const double& hypervis_scaling, const double& dcmip16_mu, - const int& ftype, const int& theta_adv_form, const bool& prescribed_wind, const bool& moisture, const bool& disable_diagnostics, - const bool& use_cpstar, const int& transport_alg, const bool& theta_hydrostatic_mode, const char** test_case, + const int& ftype, const int& theta_adv_form, const int& prescribed_wind, const int& use_moisture, const int& disable_diagnostics, + const int& use_cpstar, const int& transport_alg, const int& theta_hydrostatic_mode, const char** test_case, const int& dt_remap_factor, const int& dt_tracer_factor, - const double& scale_factor, const double& laplacian_rigid_factor, const int& nsplit, const bool& pgrad_correction, + const double& scale_factor, const double& laplacian_rigid_factor, const int& nsplit, const int& pgrad_correction, const double& dp3d_thresh, const double& vtheta_thresh, const int& internal_diagnostics_level) { + // Check that the simulation options are supported. This helps us in the future, since we // are currently 'assuming' some option have/not have certain values. As we support for more // options in the C++ build, we will remove some checks @@ -111,16 +112,16 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_subcycle = hypervis_subcycle; params.hypervis_subcycle_tom = hypervis_subcycle_tom; params.hypervis_scaling = hypervis_scaling; - params.disable_diagnostics = disable_diagnostics; - params.moisture = (moisture ? MoistDry::MOIST : MoistDry::DRY); - params.use_cpstar = use_cpstar; + params.disable_diagnostics = (bool)disable_diagnostics; + params.use_moisture = (bool)use_moisture; + params.use_cpstar = (bool)use_cpstar; params.transport_alg = transport_alg; - params.theta_hydrostatic_mode = theta_hydrostatic_mode; + params.theta_hydrostatic_mode = (bool)theta_hydrostatic_mode; params.dcmip16_mu = dcmip16_mu; params.nsplit = nsplit; params.scale_factor = scale_factor; params.laplacian_rigid_factor = laplacian_rigid_factor; - params.pgrad_correction = pgrad_correction; + params.pgrad_correction = (bool)pgrad_correction; params.dp3d_thresh = dp3d_thresh; params.vtheta_thresh = vtheta_thresh; params.internal_diagnostics_level = internal_diagnostics_level; @@ -304,7 +305,7 @@ void init_elements_c (const int& num_elems) c.create_ref(e.m_forcing); } -void init_functors_c (const bool& allocate_buffer) +void init_functors_c (const int& allocate_buffer) { auto& c = Context::singleton(); diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index 96b42314453f..262ba19f4b7a 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -64,7 +64,7 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) end subroutine prim_init2 subroutine prim_create_c_data_structures (tl, hvcoord, mp) - use iso_c_binding, only : c_loc, c_ptr, c_bool, C_NULL_CHAR + use iso_c_binding, only : c_loc, c_ptr, C_NULL_CHAR use theta_f2c_mod, only : init_reference_element_c, init_simulation_params_c, & init_time_level_c, init_hvcoord_c, init_elements_c use time_mod, only : TimeLevel_t, nsplit @@ -73,7 +73,7 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) nu, nu_p, nu_q, nu_s, nu_div, nu_top, vert_remap_q_alg, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom,& hypervis_scaling, & - ftype, prescribed_wind, moisture, disable_diagnostics, & + ftype, prescribed_wind, use_moisture, disable_diagnostics, & use_cpstar, transport_alg, theta_hydrostatic_mode, & dcmip16_mu, theta_advect_form, test_case, & MAX_STRING_LEN, dt_remap_factor, dt_tracer_factor, & @@ -93,6 +93,8 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) type (c_ptr) :: hybrid_am_ptr, hybrid_ai_ptr, hybrid_bm_ptr, hybrid_bi_ptr character(len=MAX_STRING_LEN), target :: test_name + integer :: disable_diagnostics_int, theta_hydrostatic_mode_int, use_moisture_int + ! Initialize the C++ reference element structure (i.e., pseudo-spectral deriv matrix and ref element mass matrix) dvv = deriv1%dvv elem_mp = mp @@ -100,22 +102,30 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) ! Fill the simulation params structures in C++ test_name = TRIM(test_case) // C_NULL_CHAR + + if (disable_diagnostics) disable_diagnostics_int=1 + if (.not.disable_diagnostics) disable_diagnostics_int=0 + if (use_moisture) use_moisture_int=1 + if (.not.use_moisture) use_moisture_int=0 + if(theta_hydrostatic_mode) theta_hydrostatic_mode_int=1 + if(.not.theta_hydrostatic_mode) theta_hydrostatic_mode_int=0 + call init_simulation_params_c (vert_remap_q_alg, limiter_option, rsplit, qsplit, tstep_type, & qsize, statefreq, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom, & hypervis_scaling, & dcmip16_mu, ftype, theta_advect_form, & - LOGICAL(prescribed_wind==1,c_bool), & - LOGICAL(moisture/="dry",c_bool), & - LOGICAL(disable_diagnostics,c_bool), & - LOGICAL(use_cpstar==1,c_bool), & + prescribed_wind, & + use_moisture_int, & + disable_diagnostics_int, & + use_cpstar, & transport_alg, & - LOGICAL(theta_hydrostatic_mode,c_bool), & + theta_hydrostatic_mode_int, & c_loc(test_name), & dt_remap_factor, dt_tracer_factor, & scale_factor, laplacian_rigid_factor, & nsplit, & - LOGICAL(pgrad_correction==1,c_bool), & + pgrad_correction, & dp3d_thresh, vtheta_thresh, internal_diagnostics_level) ! Initialize time level structure in C++ @@ -343,21 +353,21 @@ subroutine prim_init_elements_views (elem) end subroutine prim_init_elements_views subroutine prim_init_kokkos_functors (allocate_buffer) - use iso_c_binding, only : c_bool + use iso_c_binding, only : c_int use theta_f2c_mod, only : init_functors_c, init_boundary_exchanges_c - ! ! Optional Input ! - logical(kind=c_bool), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally - + integer, intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally + integer(kind=c_int) :: dummy ! Initialize the C++ functors in the C++ context ! If no argument allocate_buffer is present, ! let Homme internally allocate buffers if (present(allocate_buffer)) then - call init_functors_c (logical(allocate_buffer,c_bool)) + call init_functors_c (allocate_buffer) else - call init_functors_c (logical(.true.,c_bool)) + dummy=1; + call init_functors_c (dummy) endif ! Initialize boundary exchange structure in C++ diff --git a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 index 7a4c0424807b..ba39bb03c22b 100644 --- a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 +++ b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 @@ -11,14 +11,14 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, qsize, state_frequency, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom, & hypervis_scaling, & - dcmip16_mu, ftype, theta_adv_form, prescribed_wind, moisture, & + dcmip16_mu, ftype, theta_adv_form, prescribed_wind, use_moisture, & disable_diagnostics, use_cpstar, transport_alg, & theta_hydrostatic_mode, test_case_name, dt_remap_factor, & dt_tracer_factor, scale_factor, laplacian_rigid_factor, & nsplit, pgrad_correction, dp3d_thresh, vtheta_thresh, & internal_diagnostics_level) bind(c) - use iso_c_binding, only: c_int, c_bool, c_double, c_ptr + use iso_c_binding, only: c_int, c_double, c_ptr ! ! Inputs ! @@ -29,8 +29,8 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, scale_factor, laplacian_rigid_factor, dp3d_thresh, vtheta_thresh integer(kind=c_int), intent(in) :: hypervis_order, hypervis_subcycle, hypervis_subcycle_tom integer(kind=c_int), intent(in) :: ftype, theta_adv_form - logical(kind=c_bool), intent(in) :: prescribed_wind, moisture, disable_diagnostics, use_cpstar - logical(kind=c_bool), intent(in) :: theta_hydrostatic_mode, pgrad_correction + integer(kind=c_int), intent(in) :: prescribed_wind, use_moisture, disable_diagnostics, use_cpstar + integer(kind=c_int), intent(in) :: theta_hydrostatic_mode, pgrad_correction type(c_ptr), intent(in) :: test_case_name end subroutine init_simulation_params_c @@ -138,11 +138,11 @@ end subroutine init_reference_element_c ! Create C++ functors subroutine init_functors_c (allocate_buffer) bind(c) - use iso_c_binding, only: c_bool + use iso_c_binding, only: c_int ! ! Inputs ! - logical(kind=c_bool), intent(in) :: allocate_buffer + integer(kind=c_int), intent(in) :: allocate_buffer end subroutine init_functors_c ! Initialize C++ boundary exchange structures From 0200065081910233ce6c861471b062d3e0608a2a Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:33:51 -0600 Subject: [PATCH 16/48] HOMME: some SYCL related mods in kokkos initialization and team policy defaults --- .../homme/src/share/cxx/ExecSpaceDefs.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp index 8d496bff5d16..c9ca8a0ecd93 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.cpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.cpp @@ -21,6 +21,10 @@ #include #endif +#ifdef KOKKOS_ENABLE_SYCL +#include +#endif + namespace Homme { // Since we're initializing from inside a Fortran code and don't have access to @@ -52,7 +56,16 @@ void initialize_kokkos () { // It isn't a big deal if we can't get the device count. nd = 1; } +#elif defined(KOKKOS_ENABLE_SYCL) + +//https://developer.codeplay.com/products/computecpp/ce/2.11.0/guides/sycl-for-cuda-developers/migrating-from-cuda-to-sycl + +//to make it build + int nd = 1; + #endif + + #ifdef HOMMEXX_ENABLE_GPU std::stringstream ss; ss << "--kokkos-num-devices=" << nd; @@ -117,6 +130,7 @@ team_num_threads_vectors_for_gpu ( assert(num_warps_total >= max_num_warps); assert(tp.max_threads_usable >= 1 && tp.max_vectors_usable >= 1); +#ifndef KOKKOS_ENABLE_SYCL int num_warps; if (tp.prefer_larger_team) { const int num_warps_usable = @@ -161,6 +175,9 @@ team_num_threads_vectors_for_gpu ( return std::make_pair( num_device_threads / num_vectors, num_vectors ); } +#else + return std::make_pair(4,16); +#endif } } // namespace Parallel From 5a43946dcf66085a54e64af76c8c5fade9a0e575 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:34:39 -0600 Subject: [PATCH 17/48] HOMME: do not add dependency on cprnc if building without PIO --- components/homme/test_execs/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/homme/test_execs/CMakeLists.txt b/components/homme/test_execs/CMakeLists.txt index a3113921b029..a007a5532b61 100644 --- a/components/homme/test_execs/CMakeLists.txt +++ b/components/homme/test_execs/CMakeLists.txt @@ -142,8 +142,11 @@ ADD_CUSTOM_TARGET(test-execs) ADD_CUSTOM_TARGET(check COMMAND ${CMAKE_CTEST_COMMAND} "--output-on-failure") +if(NOT BUILD_HOMME_WITHOUT_PIOLIBRARY) # Force cprnc to be built when make check is run ADD_DEPENDENCIES(check cprnc) +endif() + # Create a target for making the reference data ADD_CUSTOM_TARGET(baseline From 132b3a506b8a90274bcb9cc5f7c67cf2eafdfb20 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 5 Sep 2024 16:30:50 -0600 Subject: [PATCH 18/48] HOMME: minor GPTL timing related mods * Disable timing in prim_main for the first two teps * Add some timers in EulerStepFunctor --- components/homme/src/prim_main.F90 | 7 ++++++- components/homme/src/share/cxx/EulerStepFunctorImpl.hpp | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/components/homme/src/prim_main.F90 b/components/homme/src/prim_main.F90 index bfbe57e8b317..d6901151d365 100644 --- a/components/homme/src/prim_main.F90 +++ b/components/homme/src/prim_main.F90 @@ -20,7 +20,7 @@ program prim_main use element_mod, only: element_t use common_io_mod, only: output_dir, infilenames use common_movie_mod, only: nextoutputstep - use perf_mod, only: t_initf, t_prf, t_finalizef, t_startf, t_stopf ! _EXTERNAL + use perf_mod, only: t_initf, t_prf, t_finalizef, t_startf, t_stopf, t_disablef, t_enablef ! _EXTERNAL use restart_io_mod , only: restartheader_t, writerestart use hybrid_mod, only: hybrid_create #if (defined MODEL_THETA_L && defined ARKODE) @@ -240,6 +240,11 @@ end subroutine finalize_kokkos_f90 nstep = nextoutputstep(tl) do while(tl%nstep= 2) call t_enablef() call t_startf('prim_run') call prim_run_subcycle(elem, hybrid,nets,nete, tstep, .false., tl, hvcoord,1) call t_stopf('prim_run') diff --git a/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp b/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp index f3029764dac3..f87bb108bebf 100644 --- a/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp +++ b/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp @@ -652,7 +652,10 @@ class EulerStepFunctorImpl { minmax_and_biharmonic(); } } + + GPTLstart("tl-at adv-n-limit"); advect_and_limit(); + GPTLstop("tl-at adv-n-limit"); exchange_qdp_dss_var(); } @@ -667,6 +670,7 @@ class EulerStepFunctorImpl { void run_tracer_phase (const KernelVariables& kv) const { compute_qtens(kv); kv.team_barrier(); + if (m_data.limiter_option == 8) { limiter_optim_iter_full(kv); kv.team_barrier(); @@ -674,6 +678,7 @@ class EulerStepFunctorImpl { limiter_clip_and_sum(kv); kv.team_barrier(); } + apply_spheremp(kv); } From 30b9f60a441723c13a6e111767b234f8eb482f73 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Mon, 9 Sep 2024 14:14:42 -0500 Subject: [PATCH 19/48] Hommexx: Isolate int-bool workaround to just the C++-F90 interface code. Also fix preqx's use of use_moisture. Remove MOIST-DRY enum to avoid confusion, since it's no longer used. --- .../homme/src/preqx_kokkos/cxx/CamForcing.cpp | 6 ++--- .../cxx/cxx_f90_interface_preqx.cpp | 4 +-- .../src/preqx_kokkos/cxx/prim_advance_exp.cpp | 2 +- components/homme/src/share/control_mod.F90 | 1 - components/homme/src/share/cxx/GllFvRemap.cpp | 6 ++--- components/homme/src/share/cxx/GllFvRemap.hpp | 2 +- .../homme/src/share/cxx/GllFvRemapImpl.cpp | 6 ++--- .../homme/src/share/cxx/GllFvRemapImpl.hpp | 5 ++-- .../homme/src/share/cxx/HommexxEnums.hpp | 5 ---- components/homme/src/share/gllfvremap_mod.F90 | 9 ++++--- components/homme/src/share/namelist_mod.F90 | 6 ----- .../theta-l_kokkos/cxx/EquationOfState.hpp | 4 +-- .../cxx/HyperviscosityFunctorImpl.cpp | 6 +---- .../cxx/HyperviscosityFunctorImpl.hpp | 2 +- .../src/theta-l_kokkos/prim_driver_mod.F90 | 25 +++++++++---------- .../thetal_kokkos_ut/forcing_ut.cpp | 8 +++--- .../thetal_kokkos_ut/gllfvremap_ut.cpp | 4 +-- 17 files changed, 42 insertions(+), 59 deletions(-) diff --git a/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp b/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp index 2b1e6514389e..36ca5f4a95f4 100644 --- a/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp +++ b/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp @@ -51,7 +51,7 @@ void state_forcing( void tracer_forcing( const ExecViewUnmanaged &f_q, const HybridVCoord &hvcoord, const TimeLevel &tl, const int &num_q, - const MoistDry &moisture, const double &dt, + const bool &use_moisture, const double &dt, const ExecViewManaged &ps_v, const ExecViewManaged< Scalar * [Q_NUM_TIME_LEVELS][QSIZE_D][NP][NP][NUM_LEV]> &qdp, @@ -61,7 +61,7 @@ void tracer_forcing( const int np1 = tl.n0; const int np1_qdp = tl.n0_qdp; - if (moisture == MoistDry::MOIST) { + if (use_moisture) { // Remove the m_fq_ps_v buffer since it's not actually needed. // Instead apply the forcing to m_ps_v directly // Bonus - one less parallel reduce in dry cases! @@ -161,7 +161,7 @@ void apply_cam_forcing(const Real &dt) { tracers.fq = decltype(tracers.fq)("fq", elems.num_elems(),tracers.num_tracers()); } tracer_forcing(tracers.fq, hvcoord, tl, tracers.num_tracers(), - sim_params.moisture, dt, elems.m_state.m_ps_v, tracers.qdp, tracers.Q); + sim_params.use_moisture, dt, elems.m_state.m_ps_v, tracers.qdp, tracers.Q); GPTLstop("ApplyCAMForcing"); } diff --git a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp index c75143a9836a..b433a48c2abc 100644 --- a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp +++ b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp @@ -37,7 +37,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, const int& time_step_type, const int& qsize, const int& state_frequency, const Real& nu, const Real& nu_p, const Real& nu_q, const Real& nu_s, const Real& nu_div, const Real& nu_top, const int& hypervis_order, const int& hypervis_subcycle, const double& hypervis_scaling, - const int& ftype, const bool& prescribed_wind, const bool& moisture, const bool& disable_diagnostics, + const int& ftype, const bool& prescribed_wind, const bool& use_moisture, const bool& disable_diagnostics, const bool& use_cpstar, const int& transport_alg, const int& dt_remap_factor, const int& dt_tracer_factor, const double& scale_factor, const double& laplacian_rigid_factor) @@ -90,7 +90,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_subcycle = hypervis_subcycle; params.hypervis_scaling = hypervis_scaling; params.disable_diagnostics = disable_diagnostics; - params.moisture = (moisture ? MoistDry::MOIST : MoistDry::DRY); + params.use_moisture = use_moisture; params.use_cpstar = use_cpstar; params.transport_alg = transport_alg; // SphereOperators parameters; preqx supports only the sphere. diff --git a/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp b/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp index f7c7600aab8d..58e58f0160bf 100644 --- a/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp +++ b/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp @@ -34,7 +34,7 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost // Determine the tracers time level tl.n0_qdp= -1; - if (params.moisture == MoistDry::MOIST) { + if (params.use_moisture) { tl.update_tracers_levels(params.qsplit); } diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 9c3c599b2324..0e9494f5a6cd 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -43,7 +43,6 @@ module control_mod ! flag used by preqx, theta-l and theta-c models ! should be renamed to "hydrostatic_mode" logical, public :: theta_hydrostatic_mode - integer, public :: theta_hydrostatic_mode_integer integer, public :: tstep_type= 5 ! preqx timestepping options diff --git a/components/homme/src/share/cxx/GllFvRemap.cpp b/components/homme/src/share/cxx/GllFvRemap.cpp index a8f564958d46..7b0400427f38 100644 --- a/components/homme/src/share/cxx/GllFvRemap.cpp +++ b/components/homme/src/share/cxx/GllFvRemap.cpp @@ -21,8 +21,8 @@ void init_gllfvremap_c (int nelemd, int np, int nf, int nf_max, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f) { auto& c = Context::singleton(); auto& g = c.get(); - g.init_data(nf, nf_max, theta_hydrostatic_mode, fv_metdet, g2f_remapd, - f2g_remapd, D_f, Dinv_f); + const bool thm = static_cast(theta_hydrostatic_mode); + g.init_data(nf, nf_max, thm, fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f); } GllFvRemap::GllFvRemap () { @@ -52,7 +52,7 @@ void GllFvRemap::init_boundary_exchanges () { } void GllFvRemap -::init_data (const int nf, const int nf_max, const int theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f) { m_impl->init_data(nf, nf_max, theta_hydrostatic_mode, fv_metdet, diff --git a/components/homme/src/share/cxx/GllFvRemap.hpp b/components/homme/src/share/cxx/GllFvRemap.hpp index 2adff0aeaa96..7ebf5a82b71a 100644 --- a/components/homme/src/share/cxx/GllFvRemap.hpp +++ b/components/homme/src/share/cxx/GllFvRemap.hpp @@ -40,7 +40,7 @@ class GllFvRemap { typedef Phys2T::const_type CPhys2T; typedef Phys3T::const_type CPhys3T; - void init_data(const int nf, const int nf_max, const int theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f); diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.cpp b/components/homme/src/share/cxx/GllFvRemapImpl.cpp index d4ab5c89f510..ea1a52f5efdf 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.cpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.cpp @@ -131,7 +131,7 @@ void GllFvRemapImpl::init_boundary_exchanges () { template using FV = Kokkos::View; void GllFvRemapImpl -::init_data (const int nf, const int nf_max, const int theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet_r, const Real* g2f_remapd_r, const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r) { using Kokkos::create_mirror_view; @@ -395,7 +395,7 @@ ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, c const auto hvcoord = m_hvcoord; const bool use_moisture = m_data.use_moisture; - const int theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; + const bool theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; const bool want_dp_fv_out = dp_fv_out_ptr != nullptr; VPhys2T dp_fv_out; @@ -605,7 +605,7 @@ run_fv_phys_to_dyn (const int timeidx, const CPhys2T& Ts, const CPhys3T& uvs, const auto fT = m_forcing.m_ft; const auto hvcoord = m_hvcoord; const auto dp3d = m_state.m_dp3d; - const int theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; + const bool theta_hydrostatic_mode = m_data.theta_hydrostatic_mode; EquationOfState eos; eos.init(theta_hydrostatic_mode, hvcoord); ElementOps ops; ops.init(hvcoord); const auto tu_ne = m_tu_ne; diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.hpp b/components/homme/src/share/cxx/GllFvRemapImpl.hpp index 7388fddb1231..11738b2bf455 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.hpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.hpp @@ -60,8 +60,7 @@ struct GllFvRemapImpl { struct Data { int nelemd, qsize, nf2, n_dss_fld; - bool use_moisture; - int theta_hydrostatic_mode; + bool use_moisture, theta_hydrostatic_mode; static constexpr int nbuf1 = 2, nbuf2 = 1; Buf1 buf1[nbuf1]; @@ -108,7 +107,7 @@ struct GllFvRemapImpl { void init_buffers(const FunctorsBuffersManager& fbm); void init_boundary_exchanges(); - void init_data(const int nf, const int nf_max, const int theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet_r, const Real* g2f_remapd_r, const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r); diff --git a/components/homme/src/share/cxx/HommexxEnums.hpp b/components/homme/src/share/cxx/HommexxEnums.hpp index 59c8f3c9652c..06abbf35adbc 100644 --- a/components/homme/src/share/cxx/HommexxEnums.hpp +++ b/components/homme/src/share/cxx/HommexxEnums.hpp @@ -47,11 +47,6 @@ enum class ForcingAlg : int { FORCING_2 = 2, // TODO: Rename FORCING_1 and FORCING_2 to something more descriptive }; -enum class MoistDry { - MOIST, - DRY -}; - enum class AdvectionForm { Conservative, NonConservative diff --git a/components/homme/src/share/gllfvremap_mod.F90 b/components/homme/src/share/gllfvremap_mod.F90 index e927f04aba06..a5f9b3033c96 100644 --- a/components/homme/src/share/gllfvremap_mod.F90 +++ b/components/homme/src/share/gllfvremap_mod.F90 @@ -265,14 +265,14 @@ end subroutine gfr_init subroutine gfr_init_hxx() bind(c) #if KOKKOS_TARGET - use control_mod, only: theta_hydrostatic_mode_integer + use control_mod, only: theta_hydrostatic_mode use iso_c_binding, only: c_int interface - subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode_integer, & + subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode, & fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f) bind(c) use iso_c_binding, only: c_int, c_double integer (c_int), value, intent(in) :: nelemd, np, nf, nf_max - integer (c_int), value, intent(in) :: theta_hydrostatic_mode_integer + integer (c_int), value, intent(in) :: theta_hydrostatic_mode real (c_double), dimension(nf*nf,nelemd), intent(in) :: fv_metdet real (c_double), dimension(np,np,nf_max*nf_max), intent(in) :: g2f_remapd real (c_double), dimension(nf_max*nf_max,np,np), intent(in) :: f2g_remapd @@ -280,7 +280,8 @@ subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode_inte end subroutine init_gllfvremap_c end interface integer (c_int) :: thm - thm = theta_hydrostatic_mode_integer + thm = 0 + if (theta_hydrostatic_mode) thm = 1 call init_gllfvremap_c(nelemd, np, gfr%nphys, nphys_max, thm, & gfr%fv_metdet, gfr%g2f_remapd, gfr%f2g_remapd, gfr%D_f, gfr%Dinv_f) #endif diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index a3edaa07e235..1d47090182ba 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -41,7 +41,6 @@ module namelist_mod runtype, & integration, & ! integration method theta_hydrostatic_mode, & - theta_hydrostatic_mode_integer, & transport_alg , & ! SE Eulerian, classical SL, cell-integrated SL semi_lagrange_cdr_alg, & ! see control_mod for semi_lagrange_* descriptions semi_lagrange_cdr_check, & @@ -453,10 +452,8 @@ subroutine readnl(par) planar_slice = .false. theta_hydrostatic_mode = .true. ! for preqx, this must be .true. - theta_hydrostatic_mode_integer = 1 ! for preqx, this must be .true. #if ( defined MODEL_THETA_C || defined MODEL_THETA_L ) theta_hydrostatic_mode = .false. ! default NH - theta_hydrostatic_mode_integer = 0 ! default NH #endif @@ -853,10 +850,7 @@ subroutine readnl(par) call MPI_bcast(case_planar_bubble,1,MPIlogical_t,par%root,par%comm,ierr) #endif -if(theta_hydrostatic_mode) theta_hydrostatic_mode_integer = 1 -if(.not. theta_hydrostatic_mode) theta_hydrostatic_mode_integer = 0 call MPI_bcast(theta_hydrostatic_mode ,1,MPIlogical_t,par%root,par%comm,ierr) - call MPI_bcast(theta_hydrostatic_mode_integer ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(transport_alg ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_cdr_alg ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_cdr_check ,1,MPIlogical_t,par%root,par%comm,ierr) diff --git a/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp b/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp index a50a28d58f55..dd97720f1be2 100644 --- a/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/EquationOfState.hpp @@ -23,7 +23,7 @@ class EquationOfState { EquationOfState () = default; - void init (const int theta_hydrostatic_mode, + void init (const bool theta_hydrostatic_mode, const HybridVCoord& hvcoord) { m_theta_hydrostatic_mode = theta_hydrostatic_mode; m_hvcoord = hvcoord; @@ -250,7 +250,7 @@ class EquationOfState { public: - int m_theta_hydrostatic_mode; + bool m_theta_hydrostatic_mode; HybridVCoord m_hvcoord; }; diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp index 55792051d33b..d160e114475b 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp @@ -120,11 +120,7 @@ void HyperviscosityFunctorImpl::init_params(const SimulationParams& params) #ifdef HOMMEXX_BFB_TESTING m_process_nh_vars = 1; #else - if (params.theta_hydrostatic_mode){ - m_process_nh_vars = 0; - }else{ - m_process_nh_vars = 1; - } + m_process_nh_vars = not params.theta_hydrostatic_mode; #endif } diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp index 993d525422f5..a55ecbb365f9 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp @@ -397,7 +397,7 @@ class HyperviscosityFunctorImpl Buffers m_buffers; HybridVCoord m_hvcoord; - int m_process_nh_vars; + bool m_process_nh_vars; // Policies Kokkos::TeamPolicy m_policy_update_states; diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index 262ba19f4b7a..eae8544ca865 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -103,12 +103,12 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) ! Fill the simulation params structures in C++ test_name = TRIM(test_case) // C_NULL_CHAR - if (disable_diagnostics) disable_diagnostics_int=1 - if (.not.disable_diagnostics) disable_diagnostics_int=0 - if (use_moisture) use_moisture_int=1 - if (.not.use_moisture) use_moisture_int=0 - if(theta_hydrostatic_mode) theta_hydrostatic_mode_int=1 - if(.not.theta_hydrostatic_mode) theta_hydrostatic_mode_int=0 + disable_diagnostics_int = 0 + if (disable_diagnostics) disable_diagnostics_int = 1 + use_moisture_int = 0 + if (use_moisture) use_moisture_int = 1 + theta_hydrostatic_mode_int = 0 + if (theta_hydrostatic_mode) theta_hydrostatic_mode_int = 1 call init_simulation_params_c (vert_remap_q_alg, limiter_option, rsplit, qsplit, tstep_type, & qsize, statefreq, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & @@ -358,17 +358,16 @@ subroutine prim_init_kokkos_functors (allocate_buffer) ! ! Optional Input ! - integer, intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally - integer(kind=c_int) :: dummy + logical, intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally + integer(kind=c_int) :: ab ! Initialize the C++ functors in the C++ context ! If no argument allocate_buffer is present, ! let Homme internally allocate buffers + ab = 1 if (present(allocate_buffer)) then - call init_functors_c (allocate_buffer) - else - dummy=1; - call init_functors_c (dummy) - endif + if (.not. allocate_buffer) ab = 0 + end if + call init_functors_c (ab) ! Initialize boundary exchange structure in C++ call init_boundary_exchanges_c () diff --git a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp index 5e4c51c7ca11..fb301166f429 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp @@ -160,8 +160,8 @@ TEST_CASE("forcing", "forcing") { std::cout << "Testing tracers forcing.\n"; for (const bool hydrostatic : {true,false}) { std::cout << " -> hydrostatic mode: " << (hydrostatic ? "true" : "false") << "\n"; - for (const MoistDry moisture : {MoistDry::DRY,MoistDry::MOIST}) { - std::cout << " -> moisture: " << (moisture==MoistDry::MOIST ? "moist" : "dry") << "\n"; + for (const bool use_moisture: {false,true}) { + std::cout << " -> moisture: " << (use_moisture ? "moist" : "dry") << "\n"; for (const bool adjustment : {true,false}) { std::cout << " -> adjustment: " << (adjustment ? "true" : "false") << "\n"; @@ -200,8 +200,8 @@ TEST_CASE("forcing", "forcing") { ff.init_buffers(fbm); // Run tracers forcing (cxx and f90) - ff.tracers_forcing(dt,np1,np1_qdp,adjustment,moisture); - tracers_forcing_f90(dt,np1+1,np1_qdp+1,hydrostatic,moisture==MoistDry::MOIST,adjustment); + ff.tracers_forcing(dt,np1,np1_qdp,adjustment,use_moisture); + tracers_forcing_f90(dt,np1+1,np1_qdp+1,hydrostatic,use_moisture,adjustment); // Compare answers Kokkos::deep_copy(h_dp,state.m_dp3d); diff --git a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp index 0f14b0c3e55a..cf9db941ea1c 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp @@ -183,7 +183,7 @@ struct Session { p.qsize = qsize; p.hypervis_scaling = 0; p.transport_alg = 0; - p.moisture = MoistDry::MOIST; + p.use_moisture = true; p.theta_hydrostatic_mode = false; p.scale_factor = is_sphere ? PhysicalConstants::rearth0 : 1; p.laplacian_rigid_factor = is_sphere ? 1/p.scale_factor : 0; @@ -725,7 +725,7 @@ static void test_get_temperature (Session& s) { const auto& sp = c.get(); EquationOfState eos; eos.init(theta_hydrostatic_mode, s.h); ElementOps ops; ops.init(s.h); - const bool use_moisture = sp.moisture == MoistDry::MOIST; + const bool use_moisture = sp.use_moisture; const auto state = c.get(); const auto tracers = c.get(); const auto dp3d = state.m_dp3d; From f38fe0e46ab0a225a126f793e5fbcee910dd7a78 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 10 Sep 2024 15:56:02 -0500 Subject: [PATCH 20/48] big flow error compute rof lnd interaction only if rof_c2_lnd is true we were computing an unnecessary map always even if rof_c2_lnd is false (for most coupled cases it is true) --- driver-moab/main/prep_lnd_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-moab/main/prep_lnd_mod.F90 b/driver-moab/main/prep_lnd_mod.F90 index 44ae5490e2c6..0272e835ea03 100644 --- a/driver-moab/main/prep_lnd_mod.F90 +++ b/driver-moab/main/prep_lnd_mod.F90 @@ -222,7 +222,6 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln call seq_map_init_rcfile(mapper_Fr2l, rof(1), lnd(1), & 'seq_maps.rc','rof2lnd_fmapname:','rof2lnd_fmaptype:',samegrid_lr, & string='mapper_Fr2l initialization',esmf_map=esmf_map_flag) - end if ! symmetric of l2r, from prep_rof #ifdef HAVE_MOAB ! Call moab intx only if land and river are init in moab @@ -381,6 +380,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln end if ! if ((mbrxid .ge. 0) .and. (mblxid .ge. 0)) ! endif HAVE_MOAB #endif + end if call shr_sys_flush(logunit) if (atm_c2_lnd) then From 1892887ede89b000b3165ea5f76a05c5e1a3dbe2 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 10 Sep 2024 16:15:26 -0500 Subject: [PATCH 21/48] MPI group needs to be retrieved before use in case rof c2 lnd, we were not retrieving the the MPI group needed for comm graph computation, for same grid path we were retrieving it for intx only how come we never had samegrid land rof before ? --- driver-moab/main/prep_rof_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-moab/main/prep_rof_mod.F90 b/driver-moab/main/prep_rof_mod.F90 index 59cc61c58890..a6a9184aefb5 100644 --- a/driver-moab/main/prep_rof_mod.F90 +++ b/driver-moab/main/prep_rof_mod.F90 @@ -327,6 +327,7 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) write(logunit,*) subname,' error in defining tags for seq_flds_a2x_fields on rof cpl' call shr_sys_abort(subname//' ERROR in defining tags for seq_flds_a2x_fields on rof cpl') endif + call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) if (samegrid_lr) then ! the same mesh , lnd and rof use the same dofs, but restricted ! we do not compute intersection, so we will have to just send data from lnd to rof and viceversa, by GLOBAL_ID matching @@ -363,7 +364,6 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) ! we also need to compute the comm graph for the second hop, from the lnd on coupler to the ! lnd for the intx lnd-rof context (coverage) ! - call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) type1 = 3 ! land is FV now on coupler side type2 = 3; From e4b86b6e707aa57589e4361001f1540ca4b8542b Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 10 Sep 2024 18:35:51 -0500 Subject: [PATCH 22/48] reviews from Vijay --- .../data_comps/dlnd/src/dlnd_comp_mod.F90 | 4 +- driver-moab/main/prep_lnd_mod.F90 | 8 +-- driver-moab/main/prep_ocn_mod.F90 | 66 +++++++++---------- driver-moab/main/prep_rof_mod.F90 | 32 ++++----- 4 files changed, 55 insertions(+), 55 deletions(-) diff --git a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 index 7768b2dfe815..4d76c0fe1a1f 100644 --- a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 +++ b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 @@ -148,10 +148,10 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & #ifdef HAVE_MOAB character*400 tagname real(R8) latv, lonv - integer iv, tagindex, ilat, ilon !, arrsize, nfields ! ierr is already defined as integer above + integer iv, tagindex, ilat, ilon real(R8), allocatable, target :: data(:) integer(IN), pointer :: idata(:) ! temporary - real(r8), dimension(:), allocatable :: moab_vert_coords ! temporary + real(R8), dimension(:), allocatable :: moab_vert_coords ! temporary #ifdef MOABDEBUG character*100 outfile, wopts #endif diff --git a/driver-moab/main/prep_lnd_mod.F90 b/driver-moab/main/prep_lnd_mod.F90 index 0272e835ea03..7bdb12376a3d 100644 --- a/driver-moab/main/prep_lnd_mod.F90 +++ b/driver-moab/main/prep_lnd_mod.F90 @@ -1,6 +1,6 @@ module prep_lnd_mod - use shr_kind_mod , only: r8 => SHR_KIND_R8 + use shr_kind_mod , only: R8 => SHR_KIND_R8 use shr_kind_mod , only: cs => SHR_KIND_CS use shr_kind_mod , only: cl => SHR_KIND_CL use shr_kind_mod , only: cxx => SHR_KIND_CXX @@ -162,7 +162,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln integer nrflds ! number of rof fields projected on land integer arrsize ! for setting the r2x fields on land to 0 integer ent_type ! for setting tags - real (kind=r8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 + real (kind=R8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 #endif character(*), parameter :: subname = '(prep_lnd_init)' @@ -369,7 +369,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln arrsize = nrflds*mlsize allocate (tmparray(arrsize)) ! mlsize is the size of local land ! do we need to zero out others or just river ? - tmparray = 0._r8 + tmparray = 0._R8 ierr = iMOAB_SetDoubleTagStorage(mblxid, tagname, arrsize , ent_type, tmparray) if (ierr .ne. 0) then write(logunit,*) subname,' cant zero out r2x tags on land' @@ -693,7 +693,7 @@ subroutine prep_lnd_mrg_moab (infodata) #endif #ifdef MOABCOMP character(CXX) :: tagname, mct_field - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list, ent_type type(mct_string) :: mctOStr ! diff --git a/driver-moab/main/prep_ocn_mod.F90 b/driver-moab/main/prep_ocn_mod.F90 index 44ec0c6a3c7a..b13cce62f95d 100644 --- a/driver-moab/main/prep_ocn_mod.F90 +++ b/driver-moab/main/prep_ocn_mod.F90 @@ -1,6 +1,6 @@ module prep_ocn_mod - use shr_kind_mod, only: r8 => SHR_KIND_R8 + use shr_kind_mod, only: R8 => SHR_KIND_R8 use shr_kind_mod, only: cs => SHR_KIND_CS use shr_kind_mod, only: cl => SHR_KIND_CL use shr_kind_mod, only: CX => shr_kind_CX, CXX => shr_kind_CXX @@ -137,7 +137,7 @@ module prep_ocn_mod integer , target :: x2oacc_ox_cnt ! x2oacc_ox: number of time samples accumulated ! accumulation variables for moab data - real (kind=r8) , allocatable, private, target :: x2oacc_om (:,:) ! Ocn import, ocn grid, cpl pes, moab array + real (kind=R8) , allocatable, private, target :: x2oacc_om (:,:) ! Ocn import, ocn grid, cpl pes, moab array integer , target :: x2oacc_om_cnt ! x2oacc_ox: number of time samples accumulated, in moab array integer :: arrSize_x2o_om ! this will be a module variable, size moabLocal_size * nof @@ -154,20 +154,20 @@ module prep_ocn_mod !================================================================================================ - real (kind=r8) , allocatable, private :: fractions_om (:,:) ! will retrieve the fractions from ocean, and use them + real (kind=R8) , allocatable, private :: fractions_om (:,:) ! will retrieve the fractions from ocean, and use them ! they were init with ! character(*),parameter :: fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' in moab, on the fractions - real (kind=r8) , allocatable, private :: x2o_om (:,:) - real (kind=r8) , allocatable, private :: a2x_om (:,:) - real (kind=r8) , allocatable, private :: i2x_om (:,:) - real (kind=r8) , allocatable, private :: r2x_om (:,:) - real (kind=r8) , allocatable, private :: xao_om (:,:) + real (kind=R8) , allocatable, private :: x2o_om (:,:) + real (kind=R8) , allocatable, private :: a2x_om (:,:) + real (kind=R8) , allocatable, private :: i2x_om (:,:) + real (kind=R8) , allocatable, private :: r2x_om (:,:) + real (kind=R8) , allocatable, private :: xao_om (:,:) ! this will be constructed first time, and be used to copy fields for shared indices ! between xao and x2o character(CXX) :: shared_fields_xao_x2o ! will need some array to hold the data for copying - real(r8) , allocatable, save :: shared_values(:) ! will be the size of shared indices * lsize + real(R8) , allocatable, save :: shared_values(:) ! will be the size of shared indices * lsize integer :: size_of_shared_values logical :: iamin_CPLALLICEID ! pe associated with CPLALLICEID @@ -268,7 +268,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc integer arrsize ! for setting the r2x fields on land to 0 integer ent_type ! for setting tags integer noflds ! used for number of fields in allocating moab accumulated array x2oacc_om - real (kind=r8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 + real (kind=R8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 !--------------------------------------------------------------- @@ -786,7 +786,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc arrsize = nrflds*mlsize allocate (tmparray(arrsize)) ! mlsize is the size of local land ! do we need to zero out others or just river ? - tmparray = 0._r8 + tmparray = 0._R8 ierr = iMOAB_SetDoubleTagStorage(mboxid, tagname, arrsize , ent_type, tmparray) if (ierr .ne. 0) then write(logunit,*) subname,' cant zero out r2x tags on ocn' @@ -1095,7 +1095,7 @@ subroutine prep_ocn_mrg(infodata, fractions_ox, xao_ox, timer_mrg) ! ! Local Variables integer :: eii, ewi, egi, eoi, eai, eri, exi, efi, emi - real(r8) :: flux_epbalfact ! adjusted precip factor + real(R8) :: flux_epbalfact ! adjusted precip factor type(mct_avect), pointer :: x2o_ox integer :: cnt character(*), parameter :: subname = '(prep_ocn_mrg)' @@ -1177,7 +1177,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) !--------------------------------------------------------------- - real(r8) :: flux_epbalfact ! adjusted precip factor + real(R8) :: flux_epbalfact ! adjusted precip factor ! will build x2o_om , similar to x2o_ox ! no averages, just one ocn instance @@ -1187,11 +1187,11 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) integer :: kof,kif integer :: lsize, arrsize ! for double arrays integer , save :: noflds,naflds,niflds,nrflds,nxflds! ,ngflds,nwflds, no glacier or wave model - real(r8) :: ifrac,ifracr - real(r8) :: afrac,afracr - real(r8) :: frac_sum - real(r8) :: avsdr, anidr, avsdf, anidf ! albedos - real(r8) :: fswabsv, fswabsi ! sw + real(R8) :: ifrac,ifracr + real(R8) :: afrac,afracr + real(R8) :: frac_sum + real(R8) :: avsdr, anidr, avsdf, anidf ! albedos + real(R8) :: fswabsv, fswabsi ! sw character(CL),allocatable :: field_ocn(:) ! string converted to char character(CL),allocatable :: field_atm(:) ! string converted to char character(CL),allocatable :: field_ice(:) ! string converted to char @@ -1289,7 +1289,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) #endif #ifdef MOABCOMP character(CXX) :: mct_field - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list type(mct_string) :: mctOStr ! @@ -1345,7 +1345,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) allocate(a2x_om (lsize, naflds)) allocate(i2x_om (lsize, niflds)) allocate(r2x_om (lsize, nrflds)) - r2x_om = 0._r8 ! should we zero out all of them ? + r2x_om = 0._R8 ! should we zero out all of them ? allocate(xao_om (lsize, nxflds)) ! allocate fractions too ! use the fraclist fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' @@ -1769,7 +1769,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifrac = fractions_om(n,kif) ! fo_kif_ifrac(n) ! fractions_o%rAttr(kif,n) afrac = fractions_om(n,kof) ! fo_kof_ofrac(n) ! fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -1777,7 +1777,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifracr = fractions_om(n,kir) ! fo_kir_ifrad(n) ! fractions_o%rAttr(kir,n) afracr = fractions_om(n,kor) ! fo_kor_ofrad(n) ! fractions_o%rAttr(kor,n) frac_sum = ifracr + afracr - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifracr = ifracr / (frac_sum) afracr = afracr / (frac_sum) endif @@ -1913,7 +1913,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifrac = fractions_om(n,kif) !fo_kif_ifrac(n) ! fractions_o%rAttr(kif) afrac = fractions_om(n,kof) ! fo_kof_ofrac(n) ! fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -2046,7 +2046,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa !----------------------------------------------------------------------- ! ! Arguments - real(r8) , intent(in) :: flux_epbalfact + real(R8) , intent(in) :: flux_epbalfact type(mct_aVect), intent(in) :: a2x_o type(mct_aVect), intent(in) :: i2x_o type(mct_aVect), intent(in) :: r2x_o @@ -2061,11 +2061,11 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa integer :: kof,kif integer :: lsize integer :: noflds,naflds,niflds,nrflds,nwflds,nxflds,ngflds - real(r8) :: ifrac,ifracr - real(r8) :: afrac,afracr - real(r8) :: frac_sum - real(r8) :: avsdr, anidr, avsdf, anidf ! albedos - real(r8) :: fswabsv, fswabsi ! sw + real(R8) :: ifrac,ifracr + real(R8) :: afrac,afracr + real(R8) :: frac_sum + real(R8) :: avsdr, anidr, avsdf, anidf ! albedos + real(R8) :: fswabsv, fswabsi ! sw character(CL),allocatable :: field_ocn(:) ! string converted to char character(CL),allocatable :: field_atm(:) ! string converted to char character(CL),allocatable :: field_ice(:) ! string converted to char @@ -2584,7 +2584,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifrac = fractions_o%rAttr(kif,n) afrac = fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -2592,7 +2592,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifracr = fractions_o%rAttr(kir,n) afracr = fractions_o%rAttr(kor,n) frac_sum = ifracr + afracr - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifracr = ifracr / (frac_sum) afracr = afracr / (frac_sum) endif @@ -2738,7 +2738,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifrac = fractions_o%rAttr(kif,n) afrac = fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -3068,7 +3068,7 @@ function prep_ocn_get_mapper_Sw2o() prep_ocn_get_mapper_Sw2o => mapper_Sw2o end function prep_ocn_get_mapper_Sw2o function prep_ocn_get_x2oacc_om() - real(r8), DIMENSION(:, :), pointer :: prep_ocn_get_x2oacc_om + real(R8), DIMENSION(:, :), pointer :: prep_ocn_get_x2oacc_om prep_ocn_get_x2oacc_om => x2oacc_om end function prep_ocn_get_x2oacc_om function prep_ocn_get_x2oacc_om_cnt() diff --git a/driver-moab/main/prep_rof_mod.F90 b/driver-moab/main/prep_rof_mod.F90 index a6a9184aefb5..83a8e331e916 100644 --- a/driver-moab/main/prep_rof_mod.F90 +++ b/driver-moab/main/prep_rof_mod.F90 @@ -1,7 +1,7 @@ module prep_rof_mod #include "shr_assert.h" - use shr_kind_mod, only: r8 => SHR_KIND_R8 + use shr_kind_mod, only: R8 => SHR_KIND_R8 use shr_kind_mod, only: cs => SHR_KIND_CS use shr_kind_mod, only: cl => SHR_KIND_CL use shr_kind_mod, only: cxx => SHR_KIND_CXX @@ -112,22 +112,22 @@ module prep_rof_mod ! accumulation variables over moab fields character(CXX) :: sharedFieldsLndRof ! used in moab to define l2racc_lm - real (kind=r8) , allocatable, private, target :: l2racc_lm(:,:) ! lnd export, lnd grid, cpl pes - real (kind=r8) , allocatable, private :: l2x_lm2(:,:) ! basically l2x_lm, but in another copy, on rof module + real (kind=R8) , allocatable, private, target :: l2racc_lm(:,:) ! lnd export, lnd grid, cpl pes + real (kind=R8) , allocatable, private :: l2x_lm2(:,:) ! basically l2x_lm, but in another copy, on rof module integer , target :: l2racc_lm_cnt ! l2racc_lm: number of time samples accumulated integer :: nfields_sh_lr ! number of fields in sharedFieldsLndRof integer :: lsize_lm ! size of land in moab, local character(CXX) :: sharedFieldsAtmRof ! used in moab to define a2racc_am - real (kind=r8) , allocatable, private :: a2racc_am(:,:) ! atm export, atm grid, cpl pes - real (kind=r8) , allocatable, private :: a2x_am2(:,:) ! basically a2x_am, but in another copy, on rof module + real (kind=R8) , allocatable, private :: a2racc_am(:,:) ! atm export, atm grid, cpl pes + real (kind=R8) , allocatable, private :: a2x_am2(:,:) ! basically a2x_am, but in another copy, on rof module integer , target :: a2racc_am_cnt ! a2racc_am: number of time samples accumulated integer :: nfields_sh_ar ! number of fields in sharedFieldsAtmRof integer :: lsize_am ! size of atm in moab, local character(CXX) :: sharedFieldsOcnRof ! used in moab to define o2racc_om - real (kind=r8) , allocatable, private, target :: o2racc_om(:,:) ! ocn export, ocn grid, cpl pes - real (kind=r8) , allocatable, private :: o2r_om2(:,:) ! basically o2x_om, but in another copy, on rof module, only shared with rof + real (kind=R8) , allocatable, private, target :: o2racc_om(:,:) ! ocn export, ocn grid, cpl pes + real (kind=R8) , allocatable, private :: o2r_om2(:,:) ! basically o2x_om, but in another copy, on rof module, only shared with rof integer , target :: o2racc_om_cnt ! o2racc_om: number of time samples accumulated integer :: nfields_sh_or ! number of fields in sharedFieldsOcnRof integer :: lsize_om ! size of ocn in moab, local @@ -145,12 +145,12 @@ module prep_rof_mod logical :: samegrid_al ! samegrid atm and lnd ! moab stuff - real (kind=r8) , allocatable, private :: fractions_rm (:,:) ! will retrieve the fractions from rof, and use them + real (kind=R8) , allocatable, private :: fractions_rm (:,:) ! will retrieve the fractions from rof, and use them ! they were init with ! character(*),parameter :: fraclist_r = 'lfrac:lfrin:rfrac' in moab, on the fractions - real (kind=r8) , allocatable, private :: x2r_rm (:,:) ! result of merge - real (kind=r8) , allocatable, private :: a2x_rm (:,:) - real (kind=r8) , allocatable, private :: l2x_rm (:,:) + real (kind=R8) , allocatable, private :: x2r_rm (:,:) ! result of merge + real (kind=R8) , allocatable, private :: a2x_rm (:,:) + real (kind=R8) , allocatable, private :: l2x_rm (:,:) !================================================================================================ @@ -1223,7 +1223,7 @@ subroutine prep_rof_merge(l2x_r, a2x_r, fractions_r, x2r_r, cime_model,o2x_r) integer, save :: index_x2r_coszen_str integer, save :: index_frac - real(r8) :: frac + real(R8) :: frac character(CL) :: fracstr logical, save :: first_time = .true. logical, save :: flds_wiso_rof = .false. @@ -1529,7 +1529,7 @@ subroutine prep_rof_mrg_moab (infodata, cime_model) integer, save :: index_x2r_coszen_str integer, save :: index_frac - real(r8) :: frac + real(R8) :: frac character(CL) :: fracstr logical, save :: first_time = .true. logical, save :: flds_wiso_rof = .false. @@ -1548,7 +1548,7 @@ subroutine prep_rof_mrg_moab (infodata, cime_model) character*32 :: outfile, wopts, lnum #endif #ifdef MOABCOMP - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list type(mct_string) :: mctOStr ! @@ -1995,7 +1995,7 @@ function prep_rof_get_o2racc_om_cnt() end function prep_rof_get_o2racc_om_cnt function prep_rof_get_o2racc_om() - real(r8), DIMENSION(:, :), pointer :: prep_rof_get_o2racc_om + real(R8), DIMENSION(:, :), pointer :: prep_rof_get_o2racc_om prep_rof_get_o2racc_om => o2racc_om end function prep_rof_get_o2racc_om @@ -2011,7 +2011,7 @@ function prep_rof_get_l2racc_lm_cnt() end function prep_rof_get_l2racc_lm_cnt function prep_rof_get_l2racc_lm() - real(r8), DIMENSION(:, :), pointer :: prep_rof_get_l2racc_lm + real(R8), DIMENSION(:, :), pointer :: prep_rof_get_l2racc_lm prep_rof_get_l2racc_lm => l2racc_lm end function prep_rof_get_l2racc_lm From f30afb2f1db294c3ddda8e9dcb7bd9ffdba1a3fb Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 10 Sep 2024 19:04:55 +0000 Subject: [PATCH 23/48] switch linker back to F --- components/homme/cmake/HommeMacros.cmake | 8 +------- components/homme/cmake/machineFiles/chrysalis-bfb.cmake | 2 -- components/homme/cmake/machineFiles/chrysalis.cmake | 2 -- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index 5610947cb299..6d073dbbe83b 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -112,13 +112,7 @@ macro(createTestExec execName execType macroNP macroNC ADD_DEFINITIONS(-DHAVE_CONFIG_H) ADD_EXECUTABLE(${execName} ${EXEC_SOURCES}) - - if(SUNSPOT_MACHINE) - SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE CXX) - else() - SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) - endif() - + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY) ENDIF() diff --git a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake index fa1f1ac545c5..b9f0d41a0606 100644 --- a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake +++ b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake @@ -17,8 +17,6 @@ ENDIF() SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") -SET (CHRYSALIS_MACHINE TRUE CACHE BOOL "") - # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/chrysalis.cmake b/components/homme/cmake/machineFiles/chrysalis.cmake index 97bc682c9546..68ff76ec8082 100644 --- a/components/homme/cmake/machineFiles/chrysalis.cmake +++ b/components/homme/cmake/machineFiles/chrysalis.cmake @@ -17,8 +17,6 @@ ENDIF() SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") -SET (CHRYSALIS_MACHINE TRUE CACHE BOOL "") - # Set kokkos arch, to get correct avx flags SET (Kokkos_ARCH_ZEN2 ON CACHE BOOL "") From a4c907fea9039e608d04799ce725de2a8f10eb1a Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Thu, 12 Sep 2024 18:21:11 -0500 Subject: [PATCH 24/48] more reviews --- components/data_comps/dlnd/src/dlnd_comp_mod.F90 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 index 4d76c0fe1a1f..fbd6e35cf87d 100644 --- a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 +++ b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 @@ -105,10 +105,13 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & ! !DESCRIPTION: initialize dlnd model #ifdef HAVE_MOAB - use iMOAB, only: iMOAB_DefineTagStorage, iMOAB_GetDoubleTagStorage, & + use iMOAB, only: iMOAB_DefineTagStorage, & iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & iMOAB_ResolveSharedEntities, iMOAB_CreateVertices, & - iMOAB_GetMeshInfo, iMOAB_UpdateMeshInfo, iMOAB_WriteMesh + iMOAB_UpdateMeshInfo +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif #endif implicit none From 2c103c0e757ab099d82746e734dead767aa6742e Mon Sep 17 00:00:00 2001 From: Charlie Zender Date: Fri, 13 Sep 2024 09:26:28 -0700 Subject: [PATCH 25/48] Allow interpinic to accept any netCDF filetype for output. Print warning and continue (instead of failing) when input has month dimension and output does not. --- .../elm/tools/interpinic/src/interpinic.F90 | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/components/elm/tools/interpinic/src/interpinic.F90 b/components/elm/tools/interpinic/src/interpinic.F90 index b75afd3a411d..0b8af6155775 100644 --- a/components/elm/tools/interpinic/src/interpinic.F90 +++ b/components/elm/tools/interpinic/src/interpinic.F90 @@ -146,9 +146,22 @@ subroutine interp_filei (fin, fout, cmdline) call check_ret (nf90_open(fin, NF90_NOWRITE, ncidi )) call check_ret (nf90_open(fout, NF90_NOWRITE, ncido )) call check_ret (nf_inq_format( ncido, ncformat )) - if ( ncformat /= NF_FORMAT_64BIT )then - write (6,*) 'error: output file is NOT in NetCDF large-file format!' - stop + ! 20240822 csz++ + ! Allow any format for output dataset + ! if ( ncformat /= NF_FORMAT_64BIT )then + ! write (6,*) 'error: output file is NOT in NetCDF large-file format!' + ! stop + ! 20240822 csz-- + if ( ncformat == NF_FORMAT_CLASSIC )then + write (6,*) 'info: output file is NF_FORMAT_CLASSIC' + else if ( ncformat == NF_FORMAT_64BIT_OFFSET )then + write (6,*) 'info: output file is NF_FORMAT_64BIT_OFFSET' + else if ( ncformat == NF_FORMAT_64BIT_DATA )then + write (6,*) 'info: output file is NF_FORMAT_64BIT_DATA' + else if ( ncformat == NF_FORMAT_NETCDF4 )then + write (6,*) 'info: output file is NF_FORMAT_NETCDF4' + else if ( ncformat == NF_FORMAT_NETCDF4_CLASSIC )then + write (6,*) 'info: output file is NF_FORMAT_NETCDF4_CLASSIC' end if call check_ret (nf90_inq_dimid(ncidi, "column", dimidcols )) @@ -214,13 +227,27 @@ subroutine interp_filei (fin, fout, cmdline) ret = nf90_inq_dimid(ncidi, "month", dimidmon) if (ret == NF90_NOERR) then call check_ret (nf90_inquire_dimension(ncidi, dimidmon, len=nlevmon)) - call check_ret (nf90_inq_dimid(ncido, "month", dimid )) - call check_ret (nf90_inquire_dimension(ncido, dimid, len=dimlen)) - if (dimlen/=nlevmon) then - write (6,*) 'error: input and output nlevmon values disagree' - write (6,*) 'input nlevmon = ',nlevmon,' output nlevmon = ',dimlen - stop + ! 20240912 csz++ + ! Many restart files have "month" dimension in input dataset + ! It is only necessary that the output dataset contains "month" dimension + ! when a variable in the input dataset contains the "month" dimension + ! Otherwise, the "month" dimension will never be used + ! Warn rather than die when input has "month" and output does not + !call check_ret (nf90_inq_dimid(ncido, "month", dimid )) + ret = nf90_inq_dimid(ncido, "month", dimid ) + if ( ret == nf_ebaddim ) then + write (6,*) 'warning: input has "month" dimension and output does not' + write (6,*) 'warning: interpolation will fail if any input variable uses "month" dimension' + write (6,*) 'chill: many times the "month" dimension is superfluous so this might work...' + else + call check_ret (nf90_inquire_dimension(ncido, dimid, len=dimlen)) + if (dimlen/=nlevmon) then + write (6,*) 'error: input and output nlevmon values disagree' + write (6,*) 'input nlevmon = ',nlevmon,' output nlevmon = ',dimlen + stop + end if end if + ! 20240912 csz-- else write (6,*) 'month dimension does NOT exist on the input dataset' dimidmon = -9999 @@ -321,7 +348,11 @@ subroutine interp_filei (fin, fout, cmdline) ! OK now, open the output file for writing ! call check_ret(nf90_close( ncido)) - call check_ret (nf90_open(fout, ior(NF90_WRITE, NF_64BIT_OFFSET), ncido )) + ! 20240822 csz++ + ! Allow any format for output dataset + ! call check_ret (nf90_open(fout, ior(NF90_WRITE, NF_64BIT_OFFSET), ncido )) + call check_ret (nf90_open(fout, NF90_WRITE, ncido )) + ! csz-- call addglobal (ncido, cmdline) @@ -1503,8 +1534,7 @@ subroutine addglobal (ncid, cmdline) character(len=10) :: time character(len= 5) :: zone character(len=18) :: datetime - character(len=256):: version = & - "$HeadURL: https://svn-ccsm-models.cgd.ucar.edu/clm2/trunk_tags/clm4_5_1_r085/models/lnd/clm/tools/clm4_5/interpinic/src/interpinic.F90 $" + character(len=256):: version = "" character(len=256) :: revision_id = "$Id: interpinic.F90 54953 2013-11-06 16:29:45Z sacks $" character(len=16) :: logname character(len=16) :: hostname From 4d1c5ea1a6af9249eb616ba408f6c4a0e36e293c Mon Sep 17 00:00:00 2001 From: Charlie Zender Date: Fri, 13 Sep 2024 09:42:06 -0700 Subject: [PATCH 26/48] Place hexadecimal parameters within int() intrinsic in NaN module for modern compilers (per Xylar) --- .../tools/interpinic/src/shr_infnan_mod.F90 | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/components/elm/tools/interpinic/src/shr_infnan_mod.F90 b/components/elm/tools/interpinic/src/shr_infnan_mod.F90 index 638cad84d20e..31ffb1dff329 100644 --- a/components/elm/tools/interpinic/src/shr_infnan_mod.F90 +++ b/components/elm/tools/interpinic/src/shr_infnan_mod.F90 @@ -2,11 +2,11 @@ module shr_infnan_mod -!! Inf_NaN_Detection module +!! Inf_NaN_Detection module !! Copyright(c) 2003, Lahey Computer Systems, Inc. -!! Copies of this source code, or standalone compiled files +!! Copies of this source code, or standalone compiled files !! derived from this source may not be sold without permission -!! from Lahey Computers Systems. All or part of this module may be +!! from Lahey Computers Systems. All or part of this module may be !! freely incorporated into executable programs which are offered !! for sale. Otherwise, distribution of all or part of this file is !! permitted, provided this copyright notice and header are included. @@ -22,12 +22,12 @@ module shr_infnan_mod !! isneginf(x) - test for a negative "infinite" value !! !! Each function accepts a single or double precision real argument, and -!! returns a true or false value to indicate the presence of the value +!! returns a true or false value to indicate the presence of the value !! being tested for. If the argument is array valued, the function returns !! a conformable logical array, suitable for use with the ANY function, or !! as a logical mask. !! -!! Each function operates by transferring the bit pattern from a real +!! Each function operates by transferring the bit pattern from a real !! variable to an integer container. Unless testing for + or - infinity, !! the sign bit is cleared to zero. The value is exclusive ORed with !! the value being tested for. The integer result of the IEOR function is @@ -48,14 +48,14 @@ module shr_infnan_mod integer, parameter :: Double = selected_int_kind(precision(1.0_r8)) ! Single precision IEEE values - integer(Single), parameter :: sNaN = Z"7FC00000" - integer(Single), parameter :: sPosInf = Z"7F800000" - integer(Single), parameter :: sNegInf = Z"FF800000" + integer(Single), parameter :: sNaN = int(Z"7FC00000") + integer(Single), parameter :: sPosInf = int(Z"7F800000") + integer(Single), parameter :: sNegInf = int(Z"FF800000") ! Double precision IEEE values - integer(Double), parameter :: dNaN = Z"7FF8000000000000" - integer(Double), parameter :: dPosInf = Z"7FF0000000000000" - integer(Double), parameter :: dNegInf = Z"FFF0000000000000" + integer(Double), parameter :: dNaN = int(Z"7FF8000000000000") + integer(Double), parameter :: dPosInf = int(Z"7FF0000000000000") + integer(Double), parameter :: dNegInf = int(Z"FFF0000000000000") ! Locatation of single and double precision sign bit (Intel) ! Subtract one because bit numbering starts at zero @@ -84,22 +84,22 @@ module shr_infnan_mod module procedure sisnan module procedure disnan #endif - end interface + end interface interface shr_infnan_isinf module procedure sisinf module procedure disinf - end interface - + end interface + interface shr_infnan_isposinf module procedure sisposinf module procedure disposinf - end interface - + end interface + interface shr_infnan_isneginf module procedure sisneginf module procedure disneginf - end interface + end interface integer :: shr_sisnan @@ -107,7 +107,7 @@ module shr_infnan_mod integer :: shr_disnan external :: shr_disnan -contains +contains ! ! If FORTRAN intrinsic's exist use them @@ -134,7 +134,7 @@ elemental function sisnan(x) result(res) res = isnan(x) #endif - end function + end function ! Double precision test for NaN elemental function disnan(d) result(res) @@ -156,7 +156,7 @@ elemental function disnan(d) result(res) res = isnan(d) #endif - end function + end function ! ! Otherwise link to a C function call that either uses the C90 isnan function or a x != x check @@ -176,13 +176,13 @@ function c_sisnan_1D(x) result(res) real(r4), intent(in) :: x(:) logical :: res(size(x)) - integer :: i + integer :: i do i = 1, size(x) res(i) = (shr_sisnan(x(i)) /= 0) end do end function c_sisnan_1D - + function c_sisnan_2D(x) result(res) real(r4), intent(in) :: x(:,:) logical :: res(size(x,1),size(x,2)) @@ -195,7 +195,7 @@ function c_sisnan_2D(x) result(res) end do end do end function c_sisnan_2D - + function c_sisnan_3D(x) result(res) real(r4), intent(in) :: x(:,:,:) logical :: res(size(x,1),size(x,2),size(x,3)) @@ -210,7 +210,7 @@ function c_sisnan_3D(x) result(res) end do end do end function c_sisnan_3D - + function c_sisnan_4D(x) result(res) real(r4), intent(in) :: x(:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4)) @@ -227,7 +227,7 @@ function c_sisnan_4D(x) result(res) end do end do end function c_sisnan_4D - + function c_sisnan_5D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5)) @@ -246,7 +246,7 @@ function c_sisnan_5D(x) result(res) end do end do end function c_sisnan_5D - + function c_sisnan_6D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6)) @@ -267,7 +267,7 @@ function c_sisnan_6D(x) result(res) end do end do end function c_sisnan_6D - + function c_sisnan_7D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6),size(x,7)) @@ -290,7 +290,7 @@ function c_sisnan_7D(x) result(res) end do end do end function c_sisnan_7D - + function c_disnan_scalar(x) result(res) real(r8), intent(in) :: x logical :: res @@ -302,13 +302,13 @@ function c_disnan_1D(x) result(res) real(r8), intent(in) :: x(:) logical :: res(size(x)) - integer :: i + integer :: i do i = 1, size(x) res(i) = (shr_disnan(x(i)) /= 0) end do end function c_disnan_1D - + function c_disnan_2D(x) result(res) real(r8), intent(in) :: x(:,:) logical :: res(size(x,1),size(x,2)) @@ -321,7 +321,7 @@ function c_disnan_2D(x) result(res) end do end do end function c_disnan_2D - + function c_disnan_3D(x) result(res) real(r8), intent(in) :: x(:,:,:) logical :: res(size(x,1),size(x,2),size(x,3)) @@ -336,7 +336,7 @@ function c_disnan_3D(x) result(res) end do end do end function c_disnan_3D - + function c_disnan_4D(x) result(res) real(r8), intent(in) :: x(:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4)) @@ -353,7 +353,7 @@ function c_disnan_4D(x) result(res) end do end do end function c_disnan_4D - + function c_disnan_5D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5)) @@ -372,7 +372,7 @@ function c_disnan_5D(x) result(res) end do end do end function c_disnan_5D - + function c_disnan_6D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6)) @@ -393,7 +393,7 @@ function c_disnan_6D(x) result(res) end do end do end function c_disnan_6D - + function c_disnan_7D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6),size(x,7)) @@ -418,48 +418,48 @@ function c_disnan_7D(x) result(res) end function c_disnan_7D #endif - + ! Single precision test for Inf elemental function sisinf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(ibclr(transfer(x,sPosInf),SPSB), sPosInf) == 0 - end function + end function ! Double precision test for Inf elemental function disinf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(ibclr(transfer(d,dPosInf),DPSB), dPosInf) == 0 - end function - + end function + ! Single precision test for +Inf elemental function sisposinf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(transfer(x,sPosInf), sPosInf) == 0 - end function + end function ! Double precision test for +Inf elemental function disposinf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(transfer(d,dPosInf), dPosInf) == 0 - end function - + end function + ! Single precision test for -Inf elemental function sisneginf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(transfer(x,sNegInf), sNegInf) == 0 - end function + end function ! Double precision test for -Inf elemental function disneginf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(transfer(d,dNegInf), dNegInf) == 0 - end function + end function end module shr_infnan_mod From 25fcf92996d401b7c674c8a7f18abad1c798583e Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 13 Sep 2024 14:21:12 -0700 Subject: [PATCH 27/48] add newline at end of file --- components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl index e3d4433d4a8d..3a47a8bb2ff9 100644 --- a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl +++ b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl @@ -31,4 +31,5 @@ seq_flux_mct_albdif = 0.07 seq_flux_mct_albdir = 0.07 seq_flux_atmocn_minwind = 1 -constant_zenith_deg = 42.04 \ No newline at end of file +constant_zenith_deg = 42.04 + From ef7aeca234bdd44b9bce518671e69489e13df6b6 Mon Sep 17 00:00:00 2001 From: Charlie Zender Date: Fri, 13 Sep 2024 16:38:59 -0700 Subject: [PATCH 28/48] remove external qualifier from iargc definition --- components/elm/tools/interpinic/src/fmain.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/tools/interpinic/src/fmain.F90 b/components/elm/tools/interpinic/src/fmain.F90 index 0f55c12c67c9..a67e6e2627fb 100644 --- a/components/elm/tools/interpinic/src/fmain.F90 +++ b/components/elm/tools/interpinic/src/fmain.F90 @@ -14,7 +14,7 @@ program fmain character(len= 256) :: arg integer :: n !index integer :: nargs !number of arguments - integer, external :: iargc !number of arguments function + integer :: iargc !number of arguments function character(len=256) :: finidati !input initial dataset to read character(len=256) :: finidato !output initial dataset to create character(len=256) :: cmdline !input command line From ece56d136c5861288291243cae2f49e73414c250 Mon Sep 17 00:00:00 2001 From: Charlie Zender Date: Fri, 13 Sep 2024 16:59:21 -0700 Subject: [PATCH 29/48] Remove commented-out code, CSZ markers, and dates last changed --- .../elm/tools/interpinic/src/interpinic.F90 | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/components/elm/tools/interpinic/src/interpinic.F90 b/components/elm/tools/interpinic/src/interpinic.F90 index 0b8af6155775..3d890a14b895 100644 --- a/components/elm/tools/interpinic/src/interpinic.F90 +++ b/components/elm/tools/interpinic/src/interpinic.F90 @@ -146,12 +146,9 @@ subroutine interp_filei (fin, fout, cmdline) call check_ret (nf90_open(fin, NF90_NOWRITE, ncidi )) call check_ret (nf90_open(fout, NF90_NOWRITE, ncido )) call check_ret (nf_inq_format( ncido, ncformat )) - ! 20240822 csz++ + ! Allow any format for output dataset - ! if ( ncformat /= NF_FORMAT_64BIT )then - ! write (6,*) 'error: output file is NOT in NetCDF large-file format!' - ! stop - ! 20240822 csz-- + if ( ncformat == NF_FORMAT_CLASSIC )then write (6,*) 'info: output file is NF_FORMAT_CLASSIC' else if ( ncformat == NF_FORMAT_64BIT_OFFSET )then @@ -227,13 +224,13 @@ subroutine interp_filei (fin, fout, cmdline) ret = nf90_inq_dimid(ncidi, "month", dimidmon) if (ret == NF90_NOERR) then call check_ret (nf90_inquire_dimension(ncidi, dimidmon, len=nlevmon)) - ! 20240912 csz++ + ! Many restart files have "month" dimension in input dataset ! It is only necessary that the output dataset contains "month" dimension ! when a variable in the input dataset contains the "month" dimension ! Otherwise, the "month" dimension will never be used ! Warn rather than die when input has "month" and output does not - !call check_ret (nf90_inq_dimid(ncido, "month", dimid )) + ret = nf90_inq_dimid(ncido, "month", dimid ) if ( ret == nf_ebaddim ) then write (6,*) 'warning: input has "month" dimension and output does not' @@ -247,7 +244,6 @@ subroutine interp_filei (fin, fout, cmdline) stop end if end if - ! 20240912 csz-- else write (6,*) 'month dimension does NOT exist on the input dataset' dimidmon = -9999 @@ -348,11 +344,9 @@ subroutine interp_filei (fin, fout, cmdline) ! OK now, open the output file for writing ! call check_ret(nf90_close( ncido)) - ! 20240822 csz++ + ! Allow any format for output dataset - ! call check_ret (nf90_open(fout, ior(NF90_WRITE, NF_64BIT_OFFSET), ncido )) call check_ret (nf90_open(fout, NF90_WRITE, ncido )) - ! csz-- call addglobal (ncido, cmdline) From dfeefffa109c1136068eefce1881121f67951a4f Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Mon, 16 Sep 2024 08:31:31 -0700 Subject: [PATCH 30/48] clean up new mock-walker SST section to make it more compact and less redundant --- .../data_comps/docn/src/docn_comp_mod.F90 | 49 +++---------------- 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index ac1278673656..43bac32bff73 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -1177,52 +1177,17 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) !------------------------------------------------------------------------------- ! RCEMIP phase 2 - Mock-Walker - - ! MW_295dT1p25 - mean SST = 295 K / dSST = 1.25 K - - if (sst_option == 11) then - mean_SST = 295 - TkFrz - delta_SST = 1.25 - do i = 1, lsize - sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) - end do - end if - - ! MW_300dT0p625 - mean SST = 300 K / dSST = 0.625 K - if (sst_option == 12) then - mean_SST = 300 - TkFrz - delta_SST = 0.625 - do i = 1, lsize - sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) - end do - end if - - ! MW_300dT1p25 - mean SST = 300 K / dSST = 1.25 K - if (sst_option == 13) then - mean_SST = 300 - TkFrz - delta_SST = 1.25 - do i = 1, lsize - sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) - end do - end if - - ! MW_300dT2p5 - mean SST = 300 K / dSST = 2.5 K - if (sst_option == 14) then - mean_SST = 300 - TkFrz - delta_SST = 2.5 - do i = 1, lsize - sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) - end do - end if - - ! MW_305dT1p25 - mean SST = 305 K / dSST = 1.25 K - if (sst_option == 15) then - mean_SST = 305 - TkFrz - delta_SST = 1.25 + if (sst_option>=11 .and. sst_option<=15) then + if (sst_option==11) then; mean_SST = 295 - TkFrz; delta_SST = 1.250; end if ! MW_295dT1p25 + if (sst_option==12) then; mean_SST = 300 - TkFrz; delta_SST = 0.625; end if ! MW_300dT0p625 + if (sst_option==13) then; mean_SST = 300 - TkFrz; delta_SST = 1.250; end if ! MW_300dT1p25 + if (sst_option==14) then; mean_SST = 300 - TkFrz; delta_SST = 2.500; end if ! MW_300dT2p5 + if (sst_option==15) then; mean_SST = 305 - TkFrz; delta_SST = 1.250; end if ! MW_305dT1p25 do i = 1, lsize sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) end do end if + !------------------------------------------------------------------------------- end subroutine prescribed_sst From a56bd4e2efdc2e14b975d2efe8272484be8f2a9e Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 16 Sep 2024 17:47:30 -0400 Subject: [PATCH 31/48] fix outdated path of output yaml files --- .../testdefs/testmods_dirs/eamxx/prod/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands index 435556401a2b..a5ad8d6c668e 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands @@ -61,7 +61,7 @@ else fi # set the output yaml files -output_yaml_files=$(find ${cime_root}/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/ -maxdepth 1 -type f) +output_yaml_files=$(find ${cime_root}/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/ -maxdepth 1 -type f) for file in ${output_yaml_files[@]}; do # if the word "coarse" is in the file name, do nothing if [[ "${file}" == *"_coarse.yaml" && "${hmapfile}" == "not-supported-yet" ]]; then From 6a6415a80c6bf1342c19abe789bfcac4dbf50d50 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 16 Sep 2024 19:45:04 -0400 Subject: [PATCH 32/48] edit logic to accomodate eamxx specs --- cime_config/config_archive.xml | 2 +- .../testdefs/testmods_dirs/eamxx/prod/shell_commands | 3 +++ ..._native.yaml => eamxx_output.decadal.1dailyAVG_native.yaml} | 2 +- ..._native.yaml => eamxx_output.decadal.1dailyMAX_native.yaml} | 2 +- ..._native.yaml => eamxx_output.decadal.1dailyMIN_native.yaml} | 2 +- ...INST_arm.yaml => eamxx_output.decadal.1hourlyINST_arm.yaml} | 2 +- ...ative.yaml => eamxx_output.decadal.1hourlyINST_native.yaml} | 2 +- ...coarse.yaml => eamxx_output.decadal.3hourlyAVG_coarse.yaml} | 2 +- ...oarse.yaml => eamxx_output.decadal.3hourlyINST_coarse.yaml} | 2 +- ...coarse.yaml => eamxx_output.decadal.6hourlyAVG_coarse.yaml} | 2 +- ...oarse.yaml => eamxx_output.decadal.6hourlyINST_coarse.yaml} | 2 +- ...ative.yaml => eamxx_output.decadal.6hourlyINST_native.yaml} | 2 +- ...G_coarse.yaml => eamxx_output.decadal.dailyAVG_coarse.yaml} | 2 +- 13 files changed, 15 insertions(+), 12 deletions(-) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1dailyAVG_native.yaml => eamxx_output.decadal.1dailyAVG_native.yaml} (81%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1dailyMAX_native.yaml => eamxx_output.decadal.1dailyMAX_native.yaml} (82%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1dailyMIN_native.yaml => eamxx_output.decadal.1dailyMIN_native.yaml} (80%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1hourlyINST_arm.yaml => eamxx_output.decadal.1hourlyINST_arm.yaml} (92%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.1hourlyINST_native.yaml => eamxx_output.decadal.1hourlyINST_native.yaml} (85%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.3hourlyAVG_coarse.yaml => eamxx_output.decadal.3hourlyAVG_coarse.yaml} (96%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.3hourlyINST_coarse.yaml => eamxx_output.decadal.3hourlyINST_coarse.yaml} (96%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.6hourlyAVG_coarse.yaml => eamxx_output.decadal.6hourlyAVG_coarse.yaml} (94%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.6hourlyINST_coarse.yaml => eamxx_output.decadal.6hourlyINST_coarse.yaml} (91%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.6hourlyINST_native.yaml => eamxx_output.decadal.6hourlyINST_native.yaml} (90%) rename components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/{scream_output.decadal.dailyAVG_coarse.yaml => eamxx_output.decadal.dailyAVG_coarse.yaml} (97%) diff --git a/cime_config/config_archive.xml b/cime_config/config_archive.xml index f343fce81ba0..8cb7a5305386 100644 --- a/cime_config/config_archive.xml +++ b/cime_config/config_archive.xml @@ -29,7 +29,7 @@ r\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* rhist\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* - hi\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d*\.\d{4}-\d{2}-\d{2}-\d{5}\.nc$ + .*\.h\.(?!rhist\.).*\.nc$ diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands index a5ad8d6c668e..2520f7b6b12b 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands @@ -3,6 +3,7 @@ cime_root=$(./xmlquery --value CIMEROOT) input_data_dir=$(./xmlquery --value DIN_LOC_ROOT) atmchange=$cime_root/../components/eamxx/scripts/atmchange +case_name=$(./xmlquery --value CASE) # Change run length ./xmlchange RUN_STARTDATE="1994-10-01" @@ -82,6 +83,8 @@ for file in ${output_yaml_files[@]}; do sed -i "s|horiz_remap_file:.*_to_ne30.*|horiz_remap_file: ${hmapfile}|" ./$(basename ${file}) sed -i "s|horiz_remap_file:.*_to_DecadalSites.*|horiz_remap_file: ${armmapfile}|" ./$(basename ${file}) fi + # replace all filename prefixes so that st_archive works... + sed -i "s|eamxx_output.decadal|${case_name}.scream|" ./$(basename ${file}) done # TODO: diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml similarity index 81% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml index 4e1239b5c33c..181f841eeb81 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyAVG_native.h +filename_prefix: eamxx_output.decadal.1dailyAVG_native.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml similarity index 82% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml index a7d10efe0707..a8974c75b35e 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyMAX_native.h +filename_prefix: eamxx_output.decadal.1dailyMAX_native.h iotype: pnetcdf Averaging Type: Max Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml similarity index 80% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml index 653e194d278b..8d48a1bedf63 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyMIN_native.h +filename_prefix: eamxx_output.decadal.1dailyMIN_native.h iotype: pnetcdf Averaging Type: Min Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml similarity index 92% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml index 5bb07048aed6..52fc391ca4d9 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1hourlyINST_arm.h +filename_prefix: eamxx_output.decadal.1hourlyINST_arm.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 24 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml similarity index 85% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml index 7a221e89f1c3..0aba4827ead7 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1hourlyINST_native.h +filename_prefix: eamxx_output.decadal.1hourlyINST_native.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 24 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml similarity index 96% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml index 665294c62273..d429b11ebd1f 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.3hourlyAVG_coarse.h +filename_prefix: eamxx_output.decadal.3hourlyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 8 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml similarity index 96% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml index 42c649545088..a2faa1b971c1 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.3hourlyINST_coarse.h +filename_prefix: eamxx_output.decadal.3hourlyINST_coarse.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 8 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml similarity index 94% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml index 5e4aaed07384..437142ba5599 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyAVG_coarse.h +filename_prefix: eamxx_output.decadal.6hourlyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml similarity index 91% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml index e9e0f34d5e0f..bb83718a8eb0 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyINST_coarse.h +filename_prefix: eamxx_output.decadal.6hourlyINST_coarse.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml similarity index 90% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml index bb7fd275abf4..c69dc4b2212b 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyINST_native.h +filename_prefix: eamxx_output.decadal.6hourlyINST_native.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml similarity index 97% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml index 7c1990a7b56e..2d1e6e7221ef 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.dailyAVG_coarse.h +filename_prefix: eamxx_output.decadal.dailyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 1 From 30ffb16650cfeeb54098495f630fd9698586d16c Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 17 Sep 2024 01:02:49 +0000 Subject: [PATCH 33/48] switch to 16,8 pair --- components/homme/src/share/cxx/ExecSpaceDefs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp index c9ca8a0ecd93..4f3d97135fea 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.cpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.cpp @@ -176,7 +176,7 @@ team_num_threads_vectors_for_gpu ( num_vectors ); } #else - return std::make_pair(4,16); + return std::make_pair(16,8); #endif } From 112fe79e60eeaa4d7ca0d85a54f24640b6a974f6 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 17 Sep 2024 01:03:44 +0000 Subject: [PATCH 34/48] adjust aurora flags --- components/homme/cmake/machineFiles/aurora-aot.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/homme/cmake/machineFiles/aurora-aot.cmake b/components/homme/cmake/machineFiles/aurora-aot.cmake index b6fe34a78d72..094ec8882784 100644 --- a/components/homme/cmake/machineFiles/aurora-aot.cmake +++ b/components/homme/cmake/machineFiles/aurora-aot.cmake @@ -43,9 +43,9 @@ SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") #AOT flags SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") -SET(SYCL_LINK_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fsycl-link-huge-device-code -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") -SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") From 0fcac3cc2906dee8a391f2d446bcf5286db38919 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 17 Sep 2024 01:04:25 +0000 Subject: [PATCH 35/48] adopt ekat sycl changes, TeamVectorRange --- .../share/cxx/utilities/scream_tridiag.hpp | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp index e18bbc4e7e27..26221db39552 100644 --- a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp +++ b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp @@ -128,6 +128,10 @@ int get_thread_id_within_team_gpu (const TeamMember& team) { // Can't use team.team_rank() here because vector direction also uses physical // threads but TeamMember types don't expose that information. return blockDim.x * threadIdx.y + threadIdx.x; +#elif defined(__SYCL_DEVICE_ONLY__) + auto item = team.item(); + return static_cast(item.get_local_range(1) * item.get_local_id(0) + + item.get_local_id(1)); #else assert(0); return -1; @@ -138,6 +142,9 @@ template KOKKOS_FORCEINLINE_FUNCTION int get_team_nthr_gpu (const TeamMember& team) { #if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ return blockDim.x * blockDim.y; +#elif defined __SYCL_DEVICE_ONLY__ + auto item = team.item(); + return static_cast(item.get_local_range(0) * item.get_local_range(1)); #else assert(0); return -1; @@ -161,6 +168,16 @@ KOKKOS_FORCEINLINE_FUNCTION int get_team_nthr (const Kokkos::Impl::HIPTeamMember& team) { return get_team_nthr_gpu(team); } #endif // KOKKOS_ENABLE_HIP + +#ifdef KOKKOS_ENABLE_SYCL +KOKKOS_FORCEINLINE_FUNCTION +int get_thread_id_within_team (const Kokkos::Impl::SYCLTeamMember& team) +{ return get_thread_id_within_team_gpu(team); } +KOKKOS_FORCEINLINE_FUNCTION +int get_team_nthr (const Kokkos::Impl::SYCLTeamMember& team) +{ return get_team_nthr_gpu(team); } +#endif // KOKKOS_ENABLE_SYCL + template KOKKOS_INLINE_FUNCTION const T& min (const T& a, const T& b) { return a < b ? a : b; } @@ -634,7 +651,7 @@ void bfb (const TeamMember& team, const auto f = [&] (const int& j) { impl::bfb_thomas_solve(dl, d, du, Kokkos::subview(X , Kokkos::ALL(), j)); }; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nrhs), f); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nrhs), f); } template @@ -664,7 +681,7 @@ void bfb (const TeamMember& team, subview(du, ALL(), j), subview(X , ALL(), j)); }; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nrhs), f); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nrhs), f); } } // namespace tridiag From a8763c20197cee06943a527eba000a6e0b93fb34 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 19 Sep 2024 08:58:52 -0700 Subject: [PATCH 36/48] Cleanup to fix issues from earlier PR causing sporadic test failures --- .../bld/namelist_files/namelist_defaults_mali.xml | 2 +- components/mpas-albany-landice/cime_config/buildnml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml index 0450eb44f116..8dc305533ca3 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml @@ -18,7 +18,7 @@ 'fo' -'none' +'fo' 3 0.25 .false. diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 9489b6dfa8fd..3b38ed39c438 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -280,7 +280,9 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + if mali_use_albany: + lines.append(' ') + lines.append('') lines.append('') lines.append(' - - squeue - sbatch - scancel - #SBATCH - (\d+)$ - --dependency=afterok:jobid - --dependency=afterany:jobid - : - %H:%M:%S - --mail-user - --mail-type - none, all, begin, end, fail - - --export=ALL - -p {{ job_queue }} - -J {{ job_id }} - -N {{ num_nodes }} - -n {{ total_tasks }} - -t {{ job_wallclock_time }} - -o {{ job_id }}.out - -e {{ job_id }}.err - -A {{ project }} - + + + pbatch + pdebug + + + + - pbatch - pdebug + pbatch + pdebug diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index ff0c583fc93a..249cfdfc5a88 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3065,9 +3065,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/tools/cprnc + /usr/workspace/e3sm/apps/cprnc 8 - lc_slurm + slurm boutte3 -at- llnl.gov 56 56 @@ -3076,8 +3076,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss srun + + --mpi=pmi2 + --export=ALL + -n {{ total_tasks }} -N {{ num_nodes }} + -c 1 + --cpu_bind=cores + -m plane={{ tasks_per_node }} + - + /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh @@ -3089,24 +3097,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss python/3.9.12 git + subversion + cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic - mvapich2/2.3.7 - cmake/3.19.2 - /usr/workspace/e3sm/install/quartz/modulefiles - hdf5/1.12.2 - netcdf-c/4.9.0 - netcdf-fortran/4.6.0 - parallel-netcdf/1.12.3 - screamML-venv/0.0.1 - subversion + /usr/workspace/e3sm/spack/modules/ruby/linux-rhel8-x86_64/Core + mvapich2/2.3.7-ll7cmqm + hdf5/1.10.7-ewjpbjd + netcdf-c/4.4.1.1-vaxofek + netcdf-fortran/4.4.4-3pzbx2u + parallel-netcdf/1.11.0-tzgdala $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/workspace/e3sm/install/quartz/netcdf-fortran/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + 128M + FALSE + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2 + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5 @@ -3121,9 +3132,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/tools/cprnc + /usr/workspace/e3sm/apps/cprnc 8 - lc_slurm + slurm boutte3 -at- llnl.gov 224 112 @@ -3132,8 +3143,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss srun + + --mpi=pmi2 + --export=ALL + -n {{ total_tasks }} -N {{ num_nodes }} + -c 1 + --cpu_bind=cores + -m plane={{ tasks_per_node }} + - + /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh @@ -3145,24 +3164,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss python/3.9.12 git + subversion mkl/2022.1.0 intel-classic/2021.6.0-magic - mvapich2/2.3.7 cmake/3.19.2 - /usr/workspace/e3sm/install/quartz/modulefiles - hdf5/1.12.2 - netcdf-c/4.9.0 - netcdf-fortran/4.6.0 - parallel-netcdf/1.12.3 - screamML-venv/0.0.1 - subversion + /usr/workspace/e3sm/spack/modules/dane/linux-rhel8-x86_64/Core + mvapich2/2.3.7-27jao34 + hdf5/1.10.7-766kapa + netcdf-c/4.4.1.1-2uznnlw + netcdf-fortran/4.4.4-itpstyo + parallel-netcdf/1.11.0-26sxm4m $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/workspace/e3sm/install/quartz/netcdf-fortran/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + 128M + FALSE + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck From 506cd0112299d251bd24f34f028cbf37bd5a61eb Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 15 Apr 2024 15:36:13 -0500 Subject: [PATCH 40/48] Add freshwater tracer without full capability --- components/mpas-ocean/src/shared/Makefile | 5 +- .../mpas-ocean/src/shared/mpas_ocn_tendency.F | 24 +++ .../src/shared/mpas_ocn_tracer_freshwater.F | 174 ++++++++++++++++++ .../Registry_freshwaterTracers.xml | 174 ++++++++++++++++++ .../src/tracer_groups/Registry_tracers.xml | 1 + 5 files changed, 377 insertions(+), 1 deletion(-) create mode 100644 components/mpas-ocean/src/shared/mpas_ocn_tracer_freshwater.F create mode 100644 components/mpas-ocean/src/tracer_groups/Registry_freshwaterTracers.xml diff --git a/components/mpas-ocean/src/shared/Makefile b/components/mpas-ocean/src/shared/Makefile index 35017551c754..8a7e3057e547 100644 --- a/components/mpas-ocean/src/shared/Makefile +++ b/components/mpas-ocean/src/shared/Makefile @@ -51,6 +51,7 @@ OBJS = mpas_ocn_init_routines.o \ mpas_ocn_tracer_ideal_age.o \ mpas_ocn_tracer_TTD.o \ mpas_ocn_tracer_ecosys.o \ + mpas_ocn_tracer_freshwater.o \ mpas_ocn_tracer_DMS.o \ mpas_ocn_tracer_MacroMolecules.o \ mpas_ocn_tracer_CFC.o \ @@ -83,7 +84,7 @@ all: $(OBJS) mpas_ocn_init_routines.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_mesh.o mpas_ocn_diagnostics.o mpas_ocn_diagnostics_variables.o mpas_ocn_gm.o mpas_ocn_submesoscale_eddies.o mpas_ocn_forcing.o mpas_ocn_surface_land_ice_fluxes.o -mpas_ocn_tendency.o: mpas_ocn_high_freq_thickness_hmix_del2.o mpas_ocn_tracer_surface_restoring.o mpas_ocn_thick_surface_flux.o mpas_ocn_tracer_short_wave_absorption.o mpas_ocn_tracer_advection.o mpas_ocn_tracer_hmix.o mpas_ocn_tracer_nonlocalflux.o mpas_ocn_surface_bulk_forcing.o mpas_ocn_surface_land_ice_fluxes.o mpas_ocn_tracer_surface_flux_to_tend.o mpas_ocn_tracer_interior_restoring.o mpas_ocn_tracer_exponential_decay.o mpas_ocn_tracer_ideal_age.o mpas_ocn_tracer_TTD.o mpas_ocn_vmix.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_frazil_forcing.o mpas_ocn_tidal_forcing.o mpas_ocn_tracer_ecosys.o mpas_ocn_tracer_DMS.o mpas_ocn_tracer_MacroMolecules.o mpas_ocn_tracer_CFC.o mpas_ocn_diagnostics.o mpas_ocn_wetting_drying.o mpas_ocn_vel_self_attraction_loading.o mpas_ocn_vel_tidal_potential.o mpas_ocn_mesh.o mpas_ocn_diagnostics_variables.o mpas_ocn_thick_hadv.o mpas_ocn_thick_vadv.o mpas_ocn_vel_hadv_coriolis.o mpas_ocn_vel_pressure_grad.o mpas_ocn_vel_vadv.o mpas_ocn_vel_hmix.o mpas_ocn_vel_forcing.o mpas_ocn_manufactured_solution.o +mpas_ocn_tendency.o: mpas_ocn_high_freq_thickness_hmix_del2.o mpas_ocn_tracer_surface_restoring.o mpas_ocn_thick_surface_flux.o mpas_ocn_tracer_short_wave_absorption.o mpas_ocn_tracer_advection.o mpas_ocn_tracer_hmix.o mpas_ocn_tracer_nonlocalflux.o mpas_ocn_surface_bulk_forcing.o mpas_ocn_surface_land_ice_fluxes.o mpas_ocn_tracer_surface_flux_to_tend.o mpas_ocn_tracer_interior_restoring.o mpas_ocn_tracer_exponential_decay.o mpas_ocn_tracer_ideal_age.o mpas_ocn_tracer_TTD.o mpas_ocn_vmix.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_frazil_forcing.o mpas_ocn_tidal_forcing.o mpas_ocn_tracer_ecosys.o mpas_ocn_tracer_freshwater.o mpas_ocn_tracer_DMS.o mpas_ocn_tracer_MacroMolecules.o mpas_ocn_tracer_CFC.o mpas_ocn_diagnostics.o mpas_ocn_wetting_drying.o mpas_ocn_vel_self_attraction_loading.o mpas_ocn_vel_tidal_potential.o mpas_ocn_mesh.o mpas_ocn_diagnostics_variables.o mpas_ocn_thick_hadv.o mpas_ocn_thick_vadv.o mpas_ocn_vel_hadv_coriolis.o mpas_ocn_vel_pressure_grad.o mpas_ocn_vel_vadv.o mpas_ocn_vel_hmix.o mpas_ocn_vel_forcing.o mpas_ocn_manufactured_solution.o mpas_ocn_diagnostics.o: mpas_ocn_thick_ale.o mpas_ocn_equation_of_state.o mpas_ocn_gm.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_mesh.o mpas_ocn_diagnostics_variables.o mpas_ocn_surface_land_ice_fluxes.o mpas_ocn_vertical_advection.o mpas_ocn_submesoscale_eddies.o mpas_ocn_subgrid.o @@ -205,6 +206,8 @@ mpas_ocn_tracer_TTD.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_tracer_ecosys.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_framework_forcing.o mpas_ocn_diagnostics_variables.o +mpas_ocn_tracer_freshwater.o: mpas_ocn_constants.o mpas_ocn_config.o + mpas_ocn_tracer_DMS.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_tracer_MacroMolecules.o: mpas_ocn_constants.o mpas_ocn_config.o diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index f85427913b09..561581a45474 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -46,6 +46,7 @@ module ocn_tendency use ocn_tracer_surface_restoring use ocn_tracer_interior_restoring use ocn_tracer_exponential_decay + use ocn_tracer_freshwater use ocn_tracer_ideal_age use ocn_tracer_TTD use ocn_tracer_surface_flux_to_tend @@ -802,6 +803,7 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & ! Get tracers from pool, determine number of tracers in group call mpas_pool_get_array(tracersPool, groupName, & tracerGroup, timeLevel) + !call mpas_pool_print_summary(tracersPool, MPAS_POOL_FIELD, .true.) nTracersGroup = size(tracerGroup, dim=1) ! Get Tendency array @@ -809,12 +811,18 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & call mpas_pool_get_array(tracersTendPool, & modifiedGroupName, & tracerGroupTend) + call mpas_pool_print_summary(tracersTendPool, MPAS_POOL_FIELD, .true.) ! Get surface flux array modifiedGroupName = groupName // "SurfaceFlux" call mpas_pool_get_array(tracersSurfaceFluxPool, & modifiedGroupName, & tracerGroupSurfaceFlux) + if (associated(tracerGroupSurfaceFlux)) then + call mpas_log_write(modifiedGroupName // " is associated") + else + call mpas_log_write(modifiedGroupName // " is not associated") + endif ! Get Array of total surface temp/salt flux ! (includes thickness tendencies) @@ -840,6 +848,12 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & tracerGroupSurfaceFluxSubglacialRunoff) end if + if (associated(tracerGroupSurfaceFluxRunoff)) then + call mpas_log_write(modifiedGroupName // " is associated") + else + call mpas_log_write(modifiedGroupName // " is not associated") + endif + call mpas_pool_print_summary(tracersSurfaceFluxPool, MPAS_POOL_FIELD, .true.) ! Get surface flux removed array to keep track of how much ! flux is ignored modifiedGroupName = groupName // "SurfaceFluxRemoved" @@ -1042,6 +1056,16 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & call ocn_surface_land_ice_fluxes_tracers(meshPool, groupName, & forcingPool, tracerGroupSurfaceFlux, err) + ! + ! compute freshwaterTracers surface fluxes, there are no interior sources/sinks + ! + if ( groupName == 'freshwaterTracers' ) then + + call ocn_tracer_freshwater_surface_flux_compute(tracersPool, forcingPool, & + nTracersGroup, nCellsOwned, & + tracerGroupSurfaceFlux, tracerGroupSurfaceFluxRunoff, err) + endif ! freshwater + ! ! other additions to tracerGroupSurfaceFlux should be added here ! diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_freshwater.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_freshwater.F new file mode 100644 index 000000000000..8674e31c1876 --- /dev/null +++ b/components/mpas-ocean/src/shared/mpas_ocn_tracer_freshwater.F @@ -0,0 +1,174 @@ +! Copyright (c) 2013, Los Alamos National Security, LLC (LANS) +! and the University Corporation for Atmospheric Research (UCAR). +! +! Unless noted otherwise source code is licensed under the BSD license. +! Additional copyright and license information can be found in the LICENSE file +! distributed with this code, or at http://mpas-dev.github.io/license.html +! +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ocn_tracer_freshwater +! +!> \brief MPAS freshwater tracer routines +!> \author Mathew Maltrud +!> \date 08/19/2021 +!> \details +!> This module contains routines for computing the tracer tendency due to freshwater fluxes +! +!----------------------------------------------------------------------- + +module ocn_tracer_freshwater + + use mpas_kind_types + use mpas_derived_types + use mpas_pool_routines + use ocn_constants + use ocn_config + + implicit none + private + save + + !-------------------------------------------------------------------- + ! + ! Public parameters + ! + !-------------------------------------------------------------------- + + !-------------------------------------------------------------------- + ! + ! Public member functions + ! + !-------------------------------------------------------------------- + + public :: ocn_tracer_freshwater_surface_flux_compute, & + ocn_tracer_freshwater_init + + !-------------------------------------------------------------------- + ! + ! Private module variables + ! + !-------------------------------------------------------------------- + +!*********************************************************************** + +contains + +!*********************************************************************** +! +! routine ocn_tracer_freshwater_surface_flux_compute +! +!> \brief computes a tracer tendency due to CFC surface fluxes +!> \author Mathew Maltrud +!> \date 12/22/2016 +!> \details +!> This routine computes a tracer tendency due to CFC surface fluxes +! +!----------------------------------------------------------------------- + + subroutine ocn_tracer_freshwater_surface_flux_compute(tracersPool, forcingPool, nTracers, & + nCellsSolve, freshwaterSurfaceFlux, freshwaterSurfaceFluxRunoff, err)!{{{ + + + use ocn_config + + !----------------------------------------------------------------- + ! + ! input variables + ! + !----------------------------------------------------------------- + + ! scalars + integer, intent(in) :: nTracers, nCellsSolve + + !----------------------------------------------------------------- + ! + ! input/output variables + ! + !----------------------------------------------------------------- + + type (mpas_pool_type), intent(inout) :: tracersPool, forcingPool + + ! two dimensional arrays + real (kind=RKIND), dimension(:,:), intent(inout) :: & + freshwaterSurfaceFlux, freshwaterSurfaceFluxRunoff + + !----------------------------------------------------------------- + ! + ! output variables + ! + !----------------------------------------------------------------- + + integer, intent(out) :: err !< Output: Error flag + + !----------------------------------------------------------------- + ! + ! local variables + ! + !----------------------------------------------------------------- + real (kind=RKIND), dimension(:), pointer :: & + rainFlux, snowFlux, riverRunoffFlux, iceRunoffFlux, seaIceFreshWaterFlux, icebergFreshWaterFlux, landIceFreshwaterFlux + + integer :: iCell + integer, pointer :: indexRain, indexSnow, indexRiverRunoff, indexIceRunoff, indexSeaIceFreshWater, indexIcebergFreshWater, indexLandIceFreshwater + + err = 0 + + call mpas_pool_get_dimension(tracersPool, 'index_rainConcentration', indexRain) + call mpas_pool_get_dimension(tracersPool, 'index_snowConcentration', indexSnow) + call mpas_pool_get_dimension(tracersPool, 'index_riverRunoffConcentration', indexRiverRunoff) + call mpas_pool_get_dimension(tracersPool, 'index_iceRunoffConcentration', indexIceRunoff) + call mpas_pool_get_dimension(tracersPool, 'index_seaIceFreshWaterConcentration', indexSeaIceFreshWater) + call mpas_pool_get_dimension(tracersPool, 'index_icebergFreshWaterConcentration', indexIcebergFreshWater) + call mpas_pool_get_dimension(tracersPool, 'index_landIceFreshwaterConcentration', indexLandIceFreshwater) + + call mpas_pool_get_array(forcingPool, 'rainFlux', rainFlux) + call mpas_pool_get_array(forcingPool, 'snowFlux', snowFlux) + call mpas_pool_get_array(forcingPool, 'riverRunoffFlux', riverRunoffFlux) + call mpas_pool_get_array(forcingPool, 'iceRunoffFlux', iceRunoffFlux) + call mpas_pool_get_array(forcingPool, 'seaIceFreshWaterFlux', seaIceFreshWaterFlux) + call mpas_pool_get_array(forcingPool, 'icebergFreshWaterFlux', icebergFreshWaterFlux) + call mpas_pool_get_array(forcingPool, 'landIceFreshWaterFlux', landIceFreshwaterFlux) + + do iCell=1,nCellsSolve + freshwaterSurfaceFlux(indexRain, iCell) = rainFlux(iCell) / rho_sw + freshwaterSurfaceFlux(indexSnow, iCell) = snowFlux(iCell) / rho_sw + freshwaterSurfaceFlux(indexIceRunoff, iCell) = iceRunoffFlux(iCell) / rho_sw + freshwaterSurfaceFlux(indexSeaIceFreshWater, iCell) = seaIceFreshWaterFlux(iCell) / rho_sw + freshwaterSurfaceFlux(indexIcebergFreshWater, iCell) = icebergFreshWaterFlux(iCell) / rho_sw + freshwaterSurfaceFlux(indexLandIceFreshwater, iCell) = landIceFreshwaterFlux(iCell) / rho_sw + freshwaterSurfaceFluxRunoff(indexRiverRunoff, iCell) = riverRunoffFlux(iCell) / rho_sw + enddo ! iCell + + end subroutine ocn_tracer_freshwater_surface_flux_compute!}}} + +!*********************************************************************** +! +! routine ocn_tracer_freshwater_init +! +!> \brief Initializes ocean ideal age +!> \author Mathew Maltrud +!> \date 08/19/2021 +!> \details +!> This routine initializes fields required for tracer ideal age +! +!----------------------------------------------------------------------- + + subroutine ocn_tracer_freshwater_init(domain,err)!{{{ + + type (domain_type), intent(inout) :: domain !< Input/Output: domain information + + integer, intent(out) :: err !< Output: error flag + + err = 0 + + if (.not. config_use_freshwaterTracers) return + + end subroutine ocn_tracer_freshwater_init!}}} + +!*********************************************************************** + +end module ocn_tracer_freshwater + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! vim: foldmethod=marker diff --git a/components/mpas-ocean/src/tracer_groups/Registry_freshwaterTracers.xml b/components/mpas-ocean/src/tracer_groups/Registry_freshwaterTracers.xml new file mode 100644 index 000000000000..068c455943e7 --- /dev/null +++ b/components/mpas-ocean/src/tracer_groups/Registry_freshwaterTracers.xml @@ -0,0 +1,174 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/components/mpas-ocean/src/tracer_groups/Registry_tracers.xml b/components/mpas-ocean/src/tracer_groups/Registry_tracers.xml index b8f0d1f519f5..8dd20a59acac 100644 --- a/components/mpas-ocean/src/tracer_groups/Registry_tracers.xml +++ b/components/mpas-ocean/src/tracer_groups/Registry_tracers.xml @@ -1,6 +1,7 @@ #include "Registry_activeTracers.xml" #include "Registry_debugTracers.xml" #include "Registry_ecosys.xml" +#include "Registry_freshwaterTracers.xml" #include "Registry_DMS.xml" #include "Registry_MacroMolecules.xml" #include "Registry_idealAge.xml" From 6c1247d71865ec5ee661f21c8a60c1aa5f940a35 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 26 Apr 2024 09:49:30 -0500 Subject: [PATCH 41/48] Move freshwater tracer surface fluxes to bulk forcing --- components/mpas-ocean/src/shared/Makefile | 5 +- .../shared/mpas_ocn_surface_bulk_forcing.F | 117 ++++++++++++ .../mpas-ocean/src/shared/mpas_ocn_tendency.F | 35 ++-- .../src/shared/mpas_ocn_tracer_freshwater.F | 174 ------------------ .../Registry_freshwaterTracers.xml | 20 +- 5 files changed, 143 insertions(+), 208 deletions(-) delete mode 100644 components/mpas-ocean/src/shared/mpas_ocn_tracer_freshwater.F diff --git a/components/mpas-ocean/src/shared/Makefile b/components/mpas-ocean/src/shared/Makefile index 8a7e3057e547..35017551c754 100644 --- a/components/mpas-ocean/src/shared/Makefile +++ b/components/mpas-ocean/src/shared/Makefile @@ -51,7 +51,6 @@ OBJS = mpas_ocn_init_routines.o \ mpas_ocn_tracer_ideal_age.o \ mpas_ocn_tracer_TTD.o \ mpas_ocn_tracer_ecosys.o \ - mpas_ocn_tracer_freshwater.o \ mpas_ocn_tracer_DMS.o \ mpas_ocn_tracer_MacroMolecules.o \ mpas_ocn_tracer_CFC.o \ @@ -84,7 +83,7 @@ all: $(OBJS) mpas_ocn_init_routines.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_mesh.o mpas_ocn_diagnostics.o mpas_ocn_diagnostics_variables.o mpas_ocn_gm.o mpas_ocn_submesoscale_eddies.o mpas_ocn_forcing.o mpas_ocn_surface_land_ice_fluxes.o -mpas_ocn_tendency.o: mpas_ocn_high_freq_thickness_hmix_del2.o mpas_ocn_tracer_surface_restoring.o mpas_ocn_thick_surface_flux.o mpas_ocn_tracer_short_wave_absorption.o mpas_ocn_tracer_advection.o mpas_ocn_tracer_hmix.o mpas_ocn_tracer_nonlocalflux.o mpas_ocn_surface_bulk_forcing.o mpas_ocn_surface_land_ice_fluxes.o mpas_ocn_tracer_surface_flux_to_tend.o mpas_ocn_tracer_interior_restoring.o mpas_ocn_tracer_exponential_decay.o mpas_ocn_tracer_ideal_age.o mpas_ocn_tracer_TTD.o mpas_ocn_vmix.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_frazil_forcing.o mpas_ocn_tidal_forcing.o mpas_ocn_tracer_ecosys.o mpas_ocn_tracer_freshwater.o mpas_ocn_tracer_DMS.o mpas_ocn_tracer_MacroMolecules.o mpas_ocn_tracer_CFC.o mpas_ocn_diagnostics.o mpas_ocn_wetting_drying.o mpas_ocn_vel_self_attraction_loading.o mpas_ocn_vel_tidal_potential.o mpas_ocn_mesh.o mpas_ocn_diagnostics_variables.o mpas_ocn_thick_hadv.o mpas_ocn_thick_vadv.o mpas_ocn_vel_hadv_coriolis.o mpas_ocn_vel_pressure_grad.o mpas_ocn_vel_vadv.o mpas_ocn_vel_hmix.o mpas_ocn_vel_forcing.o mpas_ocn_manufactured_solution.o +mpas_ocn_tendency.o: mpas_ocn_high_freq_thickness_hmix_del2.o mpas_ocn_tracer_surface_restoring.o mpas_ocn_thick_surface_flux.o mpas_ocn_tracer_short_wave_absorption.o mpas_ocn_tracer_advection.o mpas_ocn_tracer_hmix.o mpas_ocn_tracer_nonlocalflux.o mpas_ocn_surface_bulk_forcing.o mpas_ocn_surface_land_ice_fluxes.o mpas_ocn_tracer_surface_flux_to_tend.o mpas_ocn_tracer_interior_restoring.o mpas_ocn_tracer_exponential_decay.o mpas_ocn_tracer_ideal_age.o mpas_ocn_tracer_TTD.o mpas_ocn_vmix.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_frazil_forcing.o mpas_ocn_tidal_forcing.o mpas_ocn_tracer_ecosys.o mpas_ocn_tracer_DMS.o mpas_ocn_tracer_MacroMolecules.o mpas_ocn_tracer_CFC.o mpas_ocn_diagnostics.o mpas_ocn_wetting_drying.o mpas_ocn_vel_self_attraction_loading.o mpas_ocn_vel_tidal_potential.o mpas_ocn_mesh.o mpas_ocn_diagnostics_variables.o mpas_ocn_thick_hadv.o mpas_ocn_thick_vadv.o mpas_ocn_vel_hadv_coriolis.o mpas_ocn_vel_pressure_grad.o mpas_ocn_vel_vadv.o mpas_ocn_vel_hmix.o mpas_ocn_vel_forcing.o mpas_ocn_manufactured_solution.o mpas_ocn_diagnostics.o: mpas_ocn_thick_ale.o mpas_ocn_equation_of_state.o mpas_ocn_gm.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_mesh.o mpas_ocn_diagnostics_variables.o mpas_ocn_surface_land_ice_fluxes.o mpas_ocn_vertical_advection.o mpas_ocn_submesoscale_eddies.o mpas_ocn_subgrid.o @@ -206,8 +205,6 @@ mpas_ocn_tracer_TTD.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_tracer_ecosys.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_framework_forcing.o mpas_ocn_diagnostics_variables.o -mpas_ocn_tracer_freshwater.o: mpas_ocn_constants.o mpas_ocn_config.o - mpas_ocn_tracer_DMS.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_tracer_MacroMolecules.o: mpas_ocn_constants.o mpas_ocn_config.o diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index 11d70ae9c40d..48e9038496bf 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -126,6 +126,9 @@ subroutine ocn_surface_bulk_forcing_tracers(meshPool, groupName, forcingPool, tr if ( trim(groupName) == 'activeTracers' ) then call ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracerGroup, & tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxRemoved, layerThickness, dt, err) + elseif ( trim(groupName) == 'freshwaterTracers' ) then + call ocn_surface_bulk_forcing_freshwater_tracers(meshPool, forcingPool, tracerGroup, & + tracersSurfaceFlux, tracersSurfaceFluxRunoff, tracersSurfaceFluxRemoved, err) end if call mpas_timer_stop("bulk_" // trim(groupName)) @@ -866,6 +869,120 @@ subroutine ocn_surface_bulk_forcing_active_tracers_subglacial_runoff(meshPool, f end subroutine ocn_surface_bulk_forcing_active_tracers_subglacial_runoff!}}} +!*********************************************************************** +! +! routine ocn_surface_bulk_forcing_freshwater_tracers +! +!> \brief computes a tracer tendency due to freshwater surface fluxes +!> \author Carolyn Branecky Begeman +!> \date June 2024 +!> \details +!> This routine computes a tracer tendency due to freshwater surface fluxes +! +!----------------------------------------------------------------------- + + subroutine ocn_surface_bulk_forcing_freshwater_tracers(meshPool, forcingPool, tracerGroup, & + freshwaterSurfaceFlux, freshwaterSurfaceFluxRunoff, freshwaterSurfaceFluxRemoved, err)!{{{ + + !----------------------------------------------------------------- + ! + ! input variables + ! + !----------------------------------------------------------------- + type (mpas_pool_type), intent(in) :: meshPool !< Input: mesh information + + !----------------------------------------------------------------- + ! + ! input/output variables + ! + !----------------------------------------------------------------- + + type (mpas_pool_type), intent(inout) :: forcingPool + + ! two dimensional arrays + real (kind=RKIND), dimension(:,:), intent(inout) :: & + freshwaterSurfaceFlux, freshwaterSurfaceFluxRunoff, freshwaterSurfaceFluxRemoved + real (kind=RKIND), dimension(:,:,:), intent(inout) :: tracerGroup + + !----------------------------------------------------------------- + ! + ! output variables + ! + !----------------------------------------------------------------- + + integer, intent(out) :: err !< Output: Error flag + + !----------------------------------------------------------------- + ! + ! local variables + ! + !----------------------------------------------------------------- + + integer :: iCell, nCells + integer, dimension(:), pointer :: nCellsArray + + type(mpas_pool_type),pointer :: tracersSurfaceFluxPool + integer, pointer :: indexRain, indexSnow, indexRiverRunoff, indexIceRunoff, indexSeaIceFreshWater, & + indexIcebergFreshWater, indexLandIceFreshwater + real (kind=RKIND), dimension(:), pointer :: & + rainFlux, snowFlux, riverRunoffFlux, iceRunoffFlux, seaIceFreshWaterFlux, & + icebergFreshWaterFlux, landIceFreshwaterFlux + + err = 0 + + call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) + + call mpas_pool_get_subpool(forcingPool, 'tracersSurfaceFlux',tracersSurfaceFluxPool) + call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_rainSurfaceFlux', indexRain) + call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_snowSurfaceFlux', indexSnow) + call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_riverRunoffFreshWaterSurfaceFlux', indexRiverRunoff) + call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_iceRunoffFreshWaterSurfaceFlux', indexIceRunoff) + call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_seaIceFreshWaterSurfaceFlux', indexSeaIceFreshWater) + call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_icebergFreshWaterSurfaceFlux', indexIcebergFreshWater) + call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_landIceFreshwaterSurfaceFlux', indexLandIceFreshwater) + + call mpas_pool_get_array(forcingPool, 'rainFlux', rainFlux) + call mpas_pool_get_array(forcingPool, 'snowFlux', snowFlux) + call mpas_pool_get_array(forcingPool, 'riverRunoffFlux', riverRunoffFlux) + call mpas_pool_get_array(forcingPool, 'iceRunoffFlux', iceRunoffFlux) + call mpas_pool_get_array(forcingPool, 'seaIceFreshWaterFlux', seaIceFreshWaterFlux) + call mpas_pool_get_array(forcingPool, 'icebergFreshWaterFlux', icebergFreshWaterFlux) + call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFlux) + + nCells = nCellsArray( 3 ) + do iCell=1,nCells + freshwaterSurfaceFlux(indexRain, iCell) = rainFlux(iCell) / rho_sw + freshwaterSurfaceFlux(indexSnow, iCell) = snowFlux(iCell) / rho_sw + freshwaterSurfaceFlux(indexIceRunoff, iCell) = iceRunoffFlux(iCell) / rho_sw + freshwaterSurfaceFluxRunoff(indexRiverRunoff, iCell) = riverRunoffFlux(iCell) / rho_sw + enddo ! iCell + if (associated(indexSeaIceFreshWater)) then + do iCell=1,nCells + freshwaterSurfaceFlux(indexSeaIceFreshWater, iCell) = seaIceFreshWaterFlux(iCell) / rho_sw + enddo ! iCell + else + call mpas_log_write('indexSeaIceFreshWater is not associated') + freshwaterSurfaceFlux(indexSeaIceFreshWater, :) = 0.0_RKIND + endif + if (associated(indexIcebergFreshWater)) then + do iCell=1,nCells + freshwaterSurfaceFlux(indexIcebergFreshWater, iCell) = icebergFreshWaterFlux(iCell) / rho_sw + enddo ! iCell + else + call mpas_log_write('indexIcebergFreshWater is not associated') + freshwaterSurfaceFlux(indexIcebergFreshWater, :) = 0.0_RKIND + endif + if (associated(indexLandIceFreshwater) .and. associated(landIceFreshwaterFlux)) then + do iCell=1,nCells + freshwaterSurfaceFlux(indexLandIceFreshwater, iCell) = landIceFreshwaterFlux(iCell) / rho_sw + enddo ! iCell + else + call mpas_log_write('indexLandIceFreshwater or landIceFreshwaterFlux is not associated') + freshwaterSurfaceFlux(indexLandIceFreshwater, :) = 0.0_RKIND + endif + + end subroutine ocn_surface_bulk_forcing_freshwater_tracers!}}} + end module ocn_surface_bulk_forcing diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index 561581a45474..ece706bc2a8c 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -46,7 +46,6 @@ module ocn_tendency use ocn_tracer_surface_restoring use ocn_tracer_interior_restoring use ocn_tracer_exponential_decay - use ocn_tracer_freshwater use ocn_tracer_ideal_age use ocn_tracer_TTD use ocn_tracer_surface_flux_to_tend @@ -799,11 +798,22 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & modifiedConfigName = configBase // '_ttd_forcing' call mpas_pool_get_config(ocnConfigs, modifiedConfigName, & config_use_tracerGroup_ttd_forcing) - + if ( groupName == 'freshwaterTracers' ) then + if (config_use_tracerGroup_surface_restoring) & + call mpas_log_write('surface restoring not supported for freshwaterTracers group') + if (config_use_tracerGroup_interior_restoring) & + call mpas_log_write('interior restoring not supported for freshwaterTracers group') + if (config_use_tracerGroup_exponential_decay) & + call mpas_log_write('exponential decay not supported for freshwaterTracers group') + if (config_use_tracerGroup_idealAge_forcing) & + ! TODO support this + call mpas_log_write('idealAge forcing not supported for freshwaterTracers group') + if (config_use_tracerGroup_ttd_forcing) & + call mpas_log_write('TTD forcing not supported for freshwaterTracers group') + endif ! Get tracers from pool, determine number of tracers in group call mpas_pool_get_array(tracersPool, groupName, & tracerGroup, timeLevel) - !call mpas_pool_print_summary(tracersPool, MPAS_POOL_FIELD, .true.) nTracersGroup = size(tracerGroup, dim=1) ! Get Tendency array @@ -811,18 +821,12 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & call mpas_pool_get_array(tracersTendPool, & modifiedGroupName, & tracerGroupTend) - call mpas_pool_print_summary(tracersTendPool, MPAS_POOL_FIELD, .true.) ! Get surface flux array modifiedGroupName = groupName // "SurfaceFlux" call mpas_pool_get_array(tracersSurfaceFluxPool, & modifiedGroupName, & tracerGroupSurfaceFlux) - if (associated(tracerGroupSurfaceFlux)) then - call mpas_log_write(modifiedGroupName // " is associated") - else - call mpas_log_write(modifiedGroupName // " is not associated") - endif ! Get Array of total surface temp/salt flux ! (includes thickness tendencies) @@ -853,6 +857,7 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & else call mpas_log_write(modifiedGroupName // " is not associated") endif + call mpas_pool_print_summary(tracersSurfaceFluxPool, MPAS_POOL_FIELD, .true.) ! Get surface flux removed array to keep track of how much ! flux is ignored @@ -902,7 +907,7 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & ! compute surface tracer flux from bulk forcing ! if (config_use_tracerGroup_surface_bulk_forcing) then - + ! This applies surface fluxes if groupName is activeTracers call ocn_surface_bulk_forcing_tracers(meshPool, groupName, & forcingPool, tracerGroup, & tracerGroupSurfaceFlux, & @@ -1056,16 +1061,6 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & call ocn_surface_land_ice_fluxes_tracers(meshPool, groupName, & forcingPool, tracerGroupSurfaceFlux, err) - ! - ! compute freshwaterTracers surface fluxes, there are no interior sources/sinks - ! - if ( groupName == 'freshwaterTracers' ) then - - call ocn_tracer_freshwater_surface_flux_compute(tracersPool, forcingPool, & - nTracersGroup, nCellsOwned, & - tracerGroupSurfaceFlux, tracerGroupSurfaceFluxRunoff, err) - endif ! freshwater - ! ! other additions to tracerGroupSurfaceFlux should be added here ! diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_freshwater.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_freshwater.F deleted file mode 100644 index 8674e31c1876..000000000000 --- a/components/mpas-ocean/src/shared/mpas_ocn_tracer_freshwater.F +++ /dev/null @@ -1,174 +0,0 @@ -! Copyright (c) 2013, Los Alamos National Security, LLC (LANS) -! and the University Corporation for Atmospheric Research (UCAR). -! -! Unless noted otherwise source code is licensed under the BSD license. -! Additional copyright and license information can be found in the LICENSE file -! distributed with this code, or at http://mpas-dev.github.io/license.html -! -!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| -! -! ocn_tracer_freshwater -! -!> \brief MPAS freshwater tracer routines -!> \author Mathew Maltrud -!> \date 08/19/2021 -!> \details -!> This module contains routines for computing the tracer tendency due to freshwater fluxes -! -!----------------------------------------------------------------------- - -module ocn_tracer_freshwater - - use mpas_kind_types - use mpas_derived_types - use mpas_pool_routines - use ocn_constants - use ocn_config - - implicit none - private - save - - !-------------------------------------------------------------------- - ! - ! Public parameters - ! - !-------------------------------------------------------------------- - - !-------------------------------------------------------------------- - ! - ! Public member functions - ! - !-------------------------------------------------------------------- - - public :: ocn_tracer_freshwater_surface_flux_compute, & - ocn_tracer_freshwater_init - - !-------------------------------------------------------------------- - ! - ! Private module variables - ! - !-------------------------------------------------------------------- - -!*********************************************************************** - -contains - -!*********************************************************************** -! -! routine ocn_tracer_freshwater_surface_flux_compute -! -!> \brief computes a tracer tendency due to CFC surface fluxes -!> \author Mathew Maltrud -!> \date 12/22/2016 -!> \details -!> This routine computes a tracer tendency due to CFC surface fluxes -! -!----------------------------------------------------------------------- - - subroutine ocn_tracer_freshwater_surface_flux_compute(tracersPool, forcingPool, nTracers, & - nCellsSolve, freshwaterSurfaceFlux, freshwaterSurfaceFluxRunoff, err)!{{{ - - - use ocn_config - - !----------------------------------------------------------------- - ! - ! input variables - ! - !----------------------------------------------------------------- - - ! scalars - integer, intent(in) :: nTracers, nCellsSolve - - !----------------------------------------------------------------- - ! - ! input/output variables - ! - !----------------------------------------------------------------- - - type (mpas_pool_type), intent(inout) :: tracersPool, forcingPool - - ! two dimensional arrays - real (kind=RKIND), dimension(:,:), intent(inout) :: & - freshwaterSurfaceFlux, freshwaterSurfaceFluxRunoff - - !----------------------------------------------------------------- - ! - ! output variables - ! - !----------------------------------------------------------------- - - integer, intent(out) :: err !< Output: Error flag - - !----------------------------------------------------------------- - ! - ! local variables - ! - !----------------------------------------------------------------- - real (kind=RKIND), dimension(:), pointer :: & - rainFlux, snowFlux, riverRunoffFlux, iceRunoffFlux, seaIceFreshWaterFlux, icebergFreshWaterFlux, landIceFreshwaterFlux - - integer :: iCell - integer, pointer :: indexRain, indexSnow, indexRiverRunoff, indexIceRunoff, indexSeaIceFreshWater, indexIcebergFreshWater, indexLandIceFreshwater - - err = 0 - - call mpas_pool_get_dimension(tracersPool, 'index_rainConcentration', indexRain) - call mpas_pool_get_dimension(tracersPool, 'index_snowConcentration', indexSnow) - call mpas_pool_get_dimension(tracersPool, 'index_riverRunoffConcentration', indexRiverRunoff) - call mpas_pool_get_dimension(tracersPool, 'index_iceRunoffConcentration', indexIceRunoff) - call mpas_pool_get_dimension(tracersPool, 'index_seaIceFreshWaterConcentration', indexSeaIceFreshWater) - call mpas_pool_get_dimension(tracersPool, 'index_icebergFreshWaterConcentration', indexIcebergFreshWater) - call mpas_pool_get_dimension(tracersPool, 'index_landIceFreshwaterConcentration', indexLandIceFreshwater) - - call mpas_pool_get_array(forcingPool, 'rainFlux', rainFlux) - call mpas_pool_get_array(forcingPool, 'snowFlux', snowFlux) - call mpas_pool_get_array(forcingPool, 'riverRunoffFlux', riverRunoffFlux) - call mpas_pool_get_array(forcingPool, 'iceRunoffFlux', iceRunoffFlux) - call mpas_pool_get_array(forcingPool, 'seaIceFreshWaterFlux', seaIceFreshWaterFlux) - call mpas_pool_get_array(forcingPool, 'icebergFreshWaterFlux', icebergFreshWaterFlux) - call mpas_pool_get_array(forcingPool, 'landIceFreshWaterFlux', landIceFreshwaterFlux) - - do iCell=1,nCellsSolve - freshwaterSurfaceFlux(indexRain, iCell) = rainFlux(iCell) / rho_sw - freshwaterSurfaceFlux(indexSnow, iCell) = snowFlux(iCell) / rho_sw - freshwaterSurfaceFlux(indexIceRunoff, iCell) = iceRunoffFlux(iCell) / rho_sw - freshwaterSurfaceFlux(indexSeaIceFreshWater, iCell) = seaIceFreshWaterFlux(iCell) / rho_sw - freshwaterSurfaceFlux(indexIcebergFreshWater, iCell) = icebergFreshWaterFlux(iCell) / rho_sw - freshwaterSurfaceFlux(indexLandIceFreshwater, iCell) = landIceFreshwaterFlux(iCell) / rho_sw - freshwaterSurfaceFluxRunoff(indexRiverRunoff, iCell) = riverRunoffFlux(iCell) / rho_sw - enddo ! iCell - - end subroutine ocn_tracer_freshwater_surface_flux_compute!}}} - -!*********************************************************************** -! -! routine ocn_tracer_freshwater_init -! -!> \brief Initializes ocean ideal age -!> \author Mathew Maltrud -!> \date 08/19/2021 -!> \details -!> This routine initializes fields required for tracer ideal age -! -!----------------------------------------------------------------------- - - subroutine ocn_tracer_freshwater_init(domain,err)!{{{ - - type (domain_type), intent(inout) :: domain !< Input/Output: domain information - - integer, intent(out) :: err !< Output: error flag - - err = 0 - - if (.not. config_use_freshwaterTracers) return - - end subroutine ocn_tracer_freshwater_init!}}} - -!*********************************************************************** - -end module ocn_tracer_freshwater - -!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| -! vim: foldmethod=marker diff --git a/components/mpas-ocean/src/tracer_groups/Registry_freshwaterTracers.xml b/components/mpas-ocean/src/tracer_groups/Registry_freshwaterTracers.xml index 068c455943e7..9c0dc63f2272 100644 --- a/components/mpas-ocean/src/tracer_groups/Registry_freshwaterTracers.xml +++ b/components/mpas-ocean/src/tracer_groups/Registry_freshwaterTracers.xml @@ -52,7 +52,7 @@ - - - - - - - - - - Date: Fri, 26 Apr 2024 10:22:37 -0500 Subject: [PATCH 42/48] Fixup ice runoff to runoff surface flux array --- .../mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index 48e9038496bf..a35a76d3ea81 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -953,7 +953,7 @@ subroutine ocn_surface_bulk_forcing_freshwater_tracers(meshPool, forcingPool, tr do iCell=1,nCells freshwaterSurfaceFlux(indexRain, iCell) = rainFlux(iCell) / rho_sw freshwaterSurfaceFlux(indexSnow, iCell) = snowFlux(iCell) / rho_sw - freshwaterSurfaceFlux(indexIceRunoff, iCell) = iceRunoffFlux(iCell) / rho_sw + freshwaterSurfaceFluxRunoff(indexIceRunoff, iCell) = iceRunoffFlux(iCell) / rho_sw freshwaterSurfaceFluxRunoff(indexRiverRunoff, iCell) = riverRunoffFlux(iCell) / rho_sw enddo ! iCell if (associated(indexSeaIceFreshWater)) then From 51783c95ee1328d52a592f38cec1607716cdaeab Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Tue, 2 Jul 2024 18:18:03 -0500 Subject: [PATCH 43/48] Change namelist bld files --- components/mpas-ocean/bld/build-namelist | 13 +++++++++++++ components/mpas-ocean/bld/build-namelist-group-list | 1 + components/mpas-ocean/bld/build-namelist-section | 13 +++++++++++++ 3 files changed, 27 insertions(+) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index 460b9ecda428..4e5193ad4a34 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -1157,6 +1157,19 @@ if ($ice_bgc eq 'ice_bgc' ) { } add_default($nl, 'config_use_ecosysTracers_river_inputs_from_coupler'); +#################################################### +# Namelist group: tracer_forcing_freshwaterTracers # +#################################################### + +add_default($nl, 'config_use_freshwaterTracers'); +add_default($nl, 'config_use_freshwaterTracers_surface_bulk_forcing'); +add_default($nl, 'config_use_freshwaterTracers_surface_restoring'); +add_default($nl, 'config_use_freshwaterTracers_interior_restoring'); +add_default($nl, 'config_use_freshwaterTracers_exponential_decay'); +add_default($nl, 'config_use_freshwaterTracers_idealAge_forcing'); +add_default($nl, 'config_use_freshwaterTracers_ttd_forcing'); +add_default($nl, 'config_use_freshwaterTracers_surface_value'); + ############################################# # Namelist group: tracer_forcing_DMSTracers # ############################################# diff --git a/components/mpas-ocean/bld/build-namelist-group-list b/components/mpas-ocean/bld/build-namelist-group-list index c109ba9990e3..2adb0b92288b 100644 --- a/components/mpas-ocean/bld/build-namelist-group-list +++ b/components/mpas-ocean/bld/build-namelist-group-list @@ -44,6 +44,7 @@ my @groups = qw(run_modes tracer_forcing_activetracers tracer_forcing_debugtracers tracer_forcing_ecosystracers + tracer_forcing_freshwatertracers tracer_forcing_dmstracers tracer_forcing_macromoleculestracers tracer_forcing_idealagetracers diff --git a/components/mpas-ocean/bld/build-namelist-section b/components/mpas-ocean/bld/build-namelist-section index c5dad5d935a9..cb7a747d4156 100644 --- a/components/mpas-ocean/bld/build-namelist-section +++ b/components/mpas-ocean/bld/build-namelist-section @@ -598,6 +598,19 @@ add_default($nl, 'config_ecosysTracers_diagnostic_fields_level3'); add_default($nl, 'config_ecosysTracers_diagnostic_fields_level4'); add_default($nl, 'config_ecosysTracers_diagnostic_fields_level5'); +#################################################### +# Namelist group: tracer_forcing_freshwaterTracers # +#################################################### + +add_default($nl, 'config_use_freshwaterTracers'); +add_default($nl, 'config_use_freshwaterTracers_surface_bulk_forcing'); +add_default($nl, 'config_use_freshwaterTracers_surface_restoring'); +add_default($nl, 'config_use_freshwaterTracers_interior_restoring'); +add_default($nl, 'config_use_freshwaterTracers_exponential_decay'); +add_default($nl, 'config_use_freshwaterTracers_idealAge_forcing'); +add_default($nl, 'config_use_freshwaterTracers_ttd_forcing'); +add_default($nl, 'config_use_freshwaterTracers_surface_value'); + ############################################# # Namelist group: tracer_forcing_DMSTracers # ############################################# From 41b46ffef705e8a46f26c46a8ea248afaf67737f Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Thu, 11 Jul 2024 16:11:58 -0500 Subject: [PATCH 44/48] Update e3sm namelist files --- .../namelist_defaults_mpaso.xml | 10 +++ .../namelist_definition_mpaso.xml | 63 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index fd2785616655..5888bea24714 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -724,6 +724,16 @@ .false. .false. + +.false. +.false. +.false. +.false. +.false. +.false. +.false. +.false. + .false. .false. diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index bea1e98d9de8..fd167226b6c2 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -3099,6 +3099,69 @@ Valid values: .true. or .false. Default: Defined in namelist_defaults.xml + +if true, the 'freshwaterTracers' category is enabled for the run + +Valid Values: .true. or .false. +Default: Defined in namelist_defaults.xml + + + +if true, surface bulk forcing from coupler is added to surfaceTracerFlux in 'freshwaterTracers' category. True by default because the main function of this tracer group is tracking freshwater fluxes + +Valid Values: .true. or .false. +Default: Defined in namelist_defaults.xml + + + +if true, surface restoring source is applied to tracers in 'freshwaterTracers' category + +Valid Values: .true. or .false. +Default: Defined in namelist_defaults.xml + + + +if true, interior restoring source is applied to tracers in 'freshwaterTracers' category + +Valid Values: .true. or .false. +Default: Defined in namelist_defaults.xml + + + +if true, exponential decay source is applied to tracers in 'freshwaterTracers' category + +Valid Values: .true. or .false. +Default: Defined in namelist_defaults.xml + + + +if true, idealAge forcing source is applied to tracers in 'freshwaterTracers' category + +Valid Values: .true. or .false. +Default: Defined in namelist_defaults.xml + + + +if true, transit time distribution forcing source is applied to tracers in 'freshwaterTracers' category + +Valid Values: .true. or .false. +Default: Defined in namelist_defaults.xml + + + +if true, surface value is computed for 'freshwaterTracers' category + +Valid Values: .true. or .false. +Default: Defined in namelist_defaults.xml + From bc6c774bae331c8199ad48a648e10dc219fb5910 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 29 Jul 2024 16:42:26 -0500 Subject: [PATCH 45/48] Fixup build-namelist --- components/mpas-ocean/bld/build-namelist | 1 + 1 file changed, 1 insertion(+) diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index 4e5193ad4a34..910e3a71d15e 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -1839,6 +1839,7 @@ my @groups = qw(run_modes tracer_forcing_dmstracers tracer_forcing_macromoleculestracers tracer_forcing_idealagetracers + tracer_forcing_freshwatertracers tracer_forcing_cfctracers am_globalstats am_surfaceareaweightedaverages From bc95733a5217a8a456c0bb50d9b5c45b16cc1d19 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 29 Jul 2024 18:16:34 -0500 Subject: [PATCH 46/48] Fixup namelist defaults --- .../mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index 5888bea24714..61787b391d4c 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -726,7 +726,7 @@ .false. -.false. +.true. .false. .false. .false. From d1febf3ef84a3caf3a379b2214d03fd6d9ed041d Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Tue, 24 Sep 2024 16:32:01 -0500 Subject: [PATCH 47/48] Add subglacial runoff as a freshwater tracer --- .../shared/mpas_ocn_surface_bulk_forcing.F | 77 +++++++++++++++++++ .../mpas-ocean/src/shared/mpas_ocn_tendency.F | 19 ++--- .../Registry_freshwaterTracers.xml | 41 ++++++++++ 3 files changed, 124 insertions(+), 13 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index a35a76d3ea81..e21d94cb9ba9 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -186,6 +186,9 @@ subroutine ocn_surface_bulk_forcing_tracers_subglacial_runoff(meshPool, groupNam if ( trim(groupName) == 'activeTracers' ) then call ocn_surface_bulk_forcing_active_tracers_subglacial_runoff(meshPool, forcingPool, & tracersSurfaceFluxSubglacialRunoff, err) + elseif ( trim(groupName) == 'freshwaterTracers' ) then + call ocn_surface_bulk_forcing_freshwater_tracers_subglacial_runoff(meshPool, forcingPool, & + tracersSurfaceFluxSubglacialRunoff, err)!{{{ end if call mpas_timer_stop("bulk_" // trim(groupName)) @@ -983,6 +986,80 @@ subroutine ocn_surface_bulk_forcing_freshwater_tracers(meshPool, forcingPool, tr end subroutine ocn_surface_bulk_forcing_freshwater_tracers!}}} +!*********************************************************************** +! +! routine ocn_surface_bulk_forcing_freshwater_tracers_subglacial_runoff +! +!> \brief computes a tracer tendency due to CFC surface fluxes +!> \author Irena Vankova +!> \date July 2024 +!> \details +!> This routine computes a tracer tendency due to CFC surface fluxes +! +!----------------------------------------------------------------------- + + subroutine ocn_surface_bulk_forcing_freshwater_tracers_subglacial_runoff(meshPool, forcingPool, & + tracersSurfaceFluxSubglacialRunoff, err)!{{{ + + !----------------------------------------------------------------- + ! + ! input variables + ! + !----------------------------------------------------------------- + type (mpas_pool_type), intent(in) :: meshPool !< Input: mesh information + + !----------------------------------------------------------------- + ! + ! input/output variables + ! + !----------------------------------------------------------------- + + type (mpas_pool_type), intent(inout) :: forcingPool + + ! two dimensional arrays + real (kind=RKIND), dimension(:,:), intent(inout) :: tracersSurfaceFluxSubglacialRunoff + + !----------------------------------------------------------------- + ! + ! output variables + ! + !----------------------------------------------------------------- + + integer, intent(out) :: err !< Output: Error flag + + !----------------------------------------------------------------- + ! + ! local variables + ! + !----------------------------------------------------------------- + integer :: iCell, nCells + integer, dimension(:), pointer :: nCellsArray + + type(mpas_pool_type),pointer :: tracersSurfaceFluxPool + integer, pointer :: indexSubglacialRunoff + real (kind=RKIND), dimension(:), pointer :: subglacialRunoffFlux + + err = 0 + + call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) + + call mpas_pool_get_subpool(forcingPool, 'tracersSurfaceFlux',tracersSurfaceFluxPool) + call mpas_pool_get_dimension(tracersSurfaceFluxPool, 'index_subglacialRunoffFreshWaterSurfaceFlux', indexSubglacialRunoff) + + call mpas_pool_get_array(forcingPool, 'subglacialRunoffFlux', subglacialRunoffFlux) + + nCells = nCellsArray( 3 ) + if (associated(indexSubglacialRunoff) .and. associated(subglacialRunoffFlux)) then + do iCell=1,nCells + tracersSurfaceFluxSubglacialRunoff(indexSubglacialRunoff, iCell) = subglacialRunoffFlux(iCell) / rho_sw + enddo ! iCell + else + call mpas_log_write('indexSubglacialRunoff or subglacialRunoffFlux is not associated') + tracersSurfaceFluxSubglacialRunoff(indexSubglacialRunoff, :) = 0.0_RKIND + endif + + end subroutine ocn_surface_bulk_forcing_freshwater_tracers_subglacial_runoff!}}} + end module ocn_surface_bulk_forcing diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index ece706bc2a8c..6e52ebf1d2df 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -844,21 +844,14 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & if (trim(config_subglacial_runoff_mode) == 'data') then ! Get surface flux due to subglacial runoff array - ! only active tracers have subglacial runoff flux for now, - ! but we still need to associate for ALL tracers modifiedGroupName = groupName // "SurfaceFluxSubglacialRunoff" call mpas_pool_get_array(tracersSurfaceFluxPool, & modifiedGroupName, & tracerGroupSurfaceFluxSubglacialRunoff) end if - if (associated(tracerGroupSurfaceFluxRunoff)) then - call mpas_log_write(modifiedGroupName // " is associated") - else - call mpas_log_write(modifiedGroupName // " is not associated") - endif + !call mpas_pool_print_summary(tracersSurfaceFluxPool, MPAS_POOL_FIELD, .true.) - call mpas_pool_print_summary(tracersSurfaceFluxPool, MPAS_POOL_FIELD, .true.) ! Get surface flux removed array to keep track of how much ! flux is ignored modifiedGroupName = groupName // "SurfaceFluxRemoved" @@ -1336,12 +1329,12 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & ! ! Compute tracer tendency due to non-local flux computed in KPP ! - if (config_use_cvmix_kpp .or. config_use_gotm) then - call ocn_compute_KPP_input_fields(statePool, forcingPool,& + if (config_use_cvmix_kpp) then + call mpas_timer_start("non-local flux from KPP") + if (.not. config_cvmix_kpp_nonlocal_with_implicit_mix) then + call ocn_compute_KPP_input_fields(statePool, forcingPool,& meshPool, timeLevel) - if (.not. config_cvmix_kpp_nonlocal_with_implicit_mix) then - call mpas_timer_start("non-local flux from KPP") if (computeBudgets) then !$omp parallel !$omp do schedule(runtime) private(k,n) @@ -1384,8 +1377,8 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & !$omp end do !$omp end parallel endif ! compute budgets - call mpas_timer_stop("non-local flux from KPP") end if ! not non-local with implicit mix + call mpas_timer_stop("non-local flux from KPP") end if ! KPP ! Compute tracer tendency due to production/destruction of diff --git a/components/mpas-ocean/src/tracer_groups/Registry_freshwaterTracers.xml b/components/mpas-ocean/src/tracer_groups/Registry_freshwaterTracers.xml index 9c0dc63f2272..657486133a75 100644 --- a/components/mpas-ocean/src/tracer_groups/Registry_freshwaterTracers.xml +++ b/components/mpas-ocean/src/tracer_groups/Registry_freshwaterTracers.xml @@ -67,6 +67,9 @@ + @@ -95,6 +98,9 @@ + @@ -123,6 +129,9 @@ + + + + + + + + + + + + + From 8690ea848a57f7dd2a1e61436f2b6cfdc4e0774e Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Wed, 25 Sep 2024 12:04:28 -0500 Subject: [PATCH 48/48] Check for association of all fw fluxes in bulk forcing --- .../shared/mpas_ocn_surface_bulk_forcing.F | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index e21d94cb9ba9..e7b57d1eeaff 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -953,12 +953,38 @@ subroutine ocn_surface_bulk_forcing_freshwater_tracers(meshPool, forcingPool, tr call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFlux) nCells = nCellsArray( 3 ) - do iCell=1,nCells + if (associated(indexRain)) then + do iCell=1,nCells freshwaterSurfaceFlux(indexRain, iCell) = rainFlux(iCell) / rho_sw + enddo ! iCell + else + call mpas_log_write('indexRain is not associated') + freshwaterSurfaceFlux(indexRain, :) = 0.0_RKIND + endif + if (associated(indexSnow)) then + do iCell=1,nCells freshwaterSurfaceFlux(indexSnow, iCell) = snowFlux(iCell) / rho_sw + enddo ! iCell + else + call mpas_log_write('indexSnow is not associated') + freshwaterSurfaceFlux(indexSnow, :) = 0.0_RKIND + endif + if (associated(indexIceRunoff)) then + do iCell=1,nCells freshwaterSurfaceFluxRunoff(indexIceRunoff, iCell) = iceRunoffFlux(iCell) / rho_sw + enddo ! iCell + else + call mpas_log_write('indexIceRunoff is not associated') + freshwaterSurfaceFlux(indexIceRunoff, :) = 0.0_RKIND + endif + if (associated(indexRiverRunoff)) then + do iCell=1,nCells freshwaterSurfaceFluxRunoff(indexRiverRunoff, iCell) = riverRunoffFlux(iCell) / rho_sw - enddo ! iCell + enddo ! iCell + else + call mpas_log_write('indexRiverRunoff is not associated') + freshwaterSurfaceFlux(indexRiverRunoff, :) = 0.0_RKIND + endif if (associated(indexSeaIceFreshWater)) then do iCell=1,nCells freshwaterSurfaceFlux(indexSeaIceFreshWater, iCell) = seaIceFreshWaterFlux(iCell) / rho_sw