Merge branch 'azamat/pgi/update-cmake-link-flags' (PR #6103)

Update pgi compiler flags: - add missing -lstdc++ to pgi and nvidia linker flags - fix mpas-ocean openacc nightly - enable mpas openacc on nvidiagpu + pm-gpu - fix other miscellaneous errors. [BFB]
E3SM-Project · Dec 18, 2023 · 524b754 · 524b754
2 parents 0fd6f76 + f7a3eec
commit 524b754
Show file tree

Hide file tree

Showing 13 changed files with 35 additions and 18 deletions.
diff --git a/cime_config/machines/Depends.pgigpu.cmake b/cime_config/machines/Depends.pgigpu.cmake
@@ -107,6 +107,7 @@ list(APPEND MPAS_ADD_ACC_FLAGS
   ${CMAKE_BINARY_DIR}/core_ocean/mode_forward/mpas_ocn_time_integration_rk4.f90
   ${CMAKE_BINARY_DIR}/core_ocean/mode_forward/mpas_ocn_time_integration_si.f90
   ${CMAKE_BINARY_DIR}/core_ocean/mode_forward/mpas_ocn_time_integration_split.f90
+  ${CMAKE_BINARY_DIR}/core_ocean/mode_forward/mpas_ocn_time_integration_split_ab2.f90
   ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_diagnostics.f90
   ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_effective_density_in_land_ice.f90
   ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_diagnostics_variables.f90

diff --git a/cime_config/machines/cmake_macros/intel_pm-cpu.cmake b/cime_config/machines/cmake_macros/intel_pm-cpu.cmake
@@ -27,6 +27,5 @@ string(APPEND CMAKE_CXX_FLAGS " -fp-model=precise") # and manually add precise
 string(APPEND CMAKE_Fortran_FLAGS " -fp-model=consistent -fimf-use-svml")
    #  string(APPEND FFLAGS " -qno-opt-dynamic-align")
  string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -g -traceback")
- string(APPEND CMAKE_CXX_FLAGS_RELEASE " -g -traceback")
 string(APPEND CMAKE_Fortran_FLAGS " -DHAVE_ERF_INTRINSICS")
 string(APPEND CMAKE_CXX_FLAGS " -fp-model=consistent")
diff --git a/cime_config/machines/cmake_macros/nvidia.cmake b/cime_config/machines/cmake_macros/nvidia.cmake
@@ -20,4 +20,6 @@ if (compile_threaded)
 endif()
 string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -Mfixed")
 string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -Mfree")
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -lstdc++")
+set(E3SM_LINK_WITH_FORTRAN "TRUE")
+
diff --git a/cime_config/machines/cmake_macros/nvidia_muller.cmake b/cime_config/machines/cmake_macros/nvidia_muller.cmake
@@ -2,7 +2,6 @@ string(APPEND CONFIG_ARGS " --host=cray")
 if (COMP_NAME STREQUAL gptl)
   string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
 endif()
-set(E3SM_LINK_WITH_FORTRAN "ON")
 string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
 string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
 string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")

diff --git a/cime_config/machines/cmake_macros/nvidiagpu.cmake b/cime_config/machines/cmake_macros/nvidiagpu.cmake
@@ -24,7 +24,9 @@ string(APPEND CMAKE_CXX_FLAGS_DEBUG " -g -O0 -Mnofma -Wall -traceback")
 string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
 string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -Mfixed")
 string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -Mfree")
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -lstdc++")
 if (compile_threaded)
   string(APPEND CMAKE_EXE_LINKER_FLAGS " -mp")
 endif()
+set(E3SM_LINK_WITH_FORTRAN "TRUE")
+
diff --git a/cime_config/machines/cmake_macros/nvidiagpu_muller.cmake b/cime_config/machines/cmake_macros/nvidiagpu_muller.cmake
@@ -6,7 +6,6 @@ if (COMP_NAME STREQUAL gptl)
 endif()
 string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK")
 string(APPEND CMAKE_CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math")
-set(E3SM_LINK_WITH_FORTRAN "ON")
 set(SCC "cc")
 set(SCXX "CC")
 set(SFC "ftn")
diff --git a/cime_config/machines/cmake_macros/nvidiagpu_pm-gpu.cmake b/cime_config/machines/cmake_macros/nvidiagpu_pm-gpu.cmake
@@ -1,11 +1,13 @@
 string(APPEND CONFIG_ARGS " --host=cray")
 set(USE_CUDA "TRUE")
-string(APPEND CPPDEFS " -DGPU")
+string(APPEND CPPDEFS " -DGPU -DMPAS_OPENACC")
 if (COMP_NAME STREQUAL gptl)
   string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
 endif()
 string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK")
 string(APPEND CMAKE_CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -acc -gpu=cc70,cc60 -Minfo=accel")
 set(SCC "cc")
 set(SCXX "CC")
 set(SFC "ftn")
+string(APPEND CMAKE_Fortran_FLAGS " -acc -gpu=cc70,cc60 -Minfo=accel")
diff --git a/cime_config/machines/cmake_macros/pgi.cmake b/cime_config/machines/cmake_macros/pgi.cmake
@@ -35,7 +35,7 @@ if (COMP_NAME STREQUAL gptl)
 endif()
 string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -Mfixed")
 string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -Mfree")
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -time -Wl,--allow-multiple-definition")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -time -Wl,--allow-multiple-definition -lstdc++")
 set(MPICC "mpicc")
 set(MPICXX "mpicxx")
 set(MPIFC "mpif90")

diff --git a/cime_config/machines/cmake_macros/pgigpu.cmake b/cime_config/machines/cmake_macros/pgigpu.cmake
@@ -33,7 +33,7 @@ if (COMP_NAME STREQUAL gptl)
 endif()
 string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -Mfixed")
 string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -Mfree")
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -time -Wl,--allow-multiple-definition -acc")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -time -Wl,--allow-multiple-definition -lstdc++ -acc")
 set(MPICC "mpicc")
 set(MPICXX "mpiCC")
 set(MPIFC "mpif90")

diff --git a/components/eam/cime_config/config_pes.xml b/components/eam/cime_config/config_pes.xml
@@ -491,6 +491,23 @@
       </pes>
     </mach>
   </grid>
+  <grid name="a%ne4np4_l%ne4np4_oi%oQU240_r%r05">
+    <mach name="summit|ascent">
+      <pes compset="any" pesize="any">
+        <comment>summit|ascent: any compset on ne4_oQU240 grid</comment>
+        <ntasks>
+          <ntasks_atm>-2</ntasks_atm>
+          <ntasks_lnd>-2</ntasks_lnd>
+          <ntasks_rof>-2</ntasks_rof>
+          <ntasks_ice>-2</ntasks_ice>
+          <ntasks_ocn>-2</ntasks_ocn>
+          <ntasks_cpl>-2</ntasks_cpl>
+          <ntasks_glc>-2</ntasks_glc>
+          <ntasks_wav>-2</ntasks_wav>
+        </ntasks>
+      </pes>
+    </mach>
+  </grid>
   <grid name="a%ne16np4_|a%360x720cru">
     <mach name="any">
       <pes compset="any" pesize="any">

diff --git a/components/eam/src/physics/crm/crm_physics.F90 b/components/eam/src/physics/crm/crm_physics.F90
@@ -1911,4 +1911,4 @@ end subroutine crm_surface_flux_bypass_tend
 !==================================================================================================
 
 end module crm_physics
- 
+
diff --git a/components/elm/cime_config/config_pes.xml b/components/elm/cime_config/config_pes.xml
@@ -394,7 +394,7 @@
         </ntasks>
       </pes>
     </mach>
-    <mach name="pm-cpu|alvarez">
+    <mach name="pm-cpu|alvarez|ascent">
       <pes compset="any" pesize="any">
         <comment>elm: grid l%4x5 on 1 full node</comment>
         <ntasks>

diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics_variables.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics_variables.F
@@ -864,10 +864,8 @@ subroutine ocn_diagnostics_variables_init(domain, jenkinsOn, hollandJenkinsOn, e
          !$acc                   )
       end if
       if ( trim(config_time_integrator) == 'split_explicit_ab2' ) then
-         !$acc enter data create(                       &
-         !$acc                   normalVelocityTendOld, &
-         !$acc                   CoriolisTermOld
-         !$acc                   )
+         !$acc enter data create(normalVelocityTendOld, &
+         !$acc                   CoriolisTermOld)
       end if
 
       !$acc enter data create(                                         &
@@ -1113,10 +1111,8 @@ subroutine ocn_diagnostics_variables_destroy(err) !{{{
          !$acc                   )
       end if
       if ( trim(config_time_integrator) == 'split_explicit_ab2' ) then
-         !$acc exit data delete(                       &
-         !$acc                  normalVelocityTendOld, &
-         !$acc                  CoriolisTermOld
-         !$acc                  )
+         !$acc exit data delete(normalVelocityTendOld, &
+         !$acc                  CoriolisTermOld)
       end if
 
       !$acc exit data delete(                                          &
Original file line number	Diff line number	Diff line change
Expand Up		@@ -1911,4 +1911,4 @@ end subroutine crm_surface_flux_bypass_tend
		!==================================================================================================

		end module crm_physics