Workaround for supporting GPU-aware MPI on Cray/AMD platforms

Co-authored-by: Paul Mullowney <[email protected]> Co-authored-by: Willem Deconinck <[email protected]>
ecmwf-ifs · Oct 11, 2024 · 6840754 · 6840754
1 parent 234c6a9
commit 6840754
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 3 deletions.
diff --git a/src/trans/gpu/internal/trgtol_mod.F90 b/src/trans/gpu/internal/trgtol_mod.F90
@@ -115,6 +115,7 @@ SUBROUTINE TRGTOL(ALLOCATOR,HTRGTOL,PREEL_REAL,KF_FS,KF_GP,KF_UV_G,KF_SCALARS_G,
     USE OML_MOD,                ONLY: OML_MY_THREAD
 #if ECTRANS_HAVE_MPI
     USE MPI_F08,                ONLY: MPI_COMM, MPI_REQUEST, MPI_FLOAT, MPI_DOUBLE
+    ! Missing: MPI_ISEND, MPI_IRECV on purpose due to cray-mpi bug (see https://github.com/ecmwf-ifs/ectrans/pull/157)
 #endif
     USE TPM_STATS,              ONLY: GSTATS => GSTATS_NVTX
     USE TPM_TRANS,              ONLY: NPROMA

diff --git a/src/trans/gpu/internal/trltog_mod.F90 b/src/trans/gpu/internal/trltog_mod.F90
@@ -116,6 +116,7 @@ SUBROUTINE TRLTOG(ALLOCATOR,HTRLTOG,PREEL_REAL,KF_FS,KF_GP,KF_UV_G,KF_SCALARS_G,
     USE ABORT_TRANS_MOD,        ONLY: ABORT_TRANS
 #if ECTRANS_HAVE_MPI
     USE MPI_F08,                ONLY: MPI_COMM, MPI_REQUEST, MPI_FLOAT, MPI_DOUBLE
+    ! Missing: MPI_ISEND, MPI_IRECV on purpose due to cray-mpi bug (see https://github.com/ecmwf-ifs/ectrans/pull/157)
 #endif
     USE TPM_STATS,              ONLY: GSTATS => GSTATS_NVTX
     USE TPM_TRANS,              ONLY: LDIVGP, LSCDERS, LUVDER, LVORGP, NPROMA

diff --git a/src/trans/gpu/internal/trltom_mod.F90 b/src/trans/gpu/internal/trltom_mod.F90
@@ -94,7 +94,8 @@ SUBROUTINE TRLTOM(ALLOCATOR,HTRLTOM,PFBUF_IN,PFBUF,KF_FS)
     USE TPM_DISTR,              ONLY: D, NPRTRW, NPROC, MYPROC, MYSETW
     USE TPM_GEN,                ONLY: LSYNC_TRANS, NERR
 #if ECTRANS_HAVE_MPI
-    USE MPI_F08,                ONLY: MPI_COMM, MPI_FLOAT, MPI_DOUBLE, MPI_ALLTOALLV
+    USE MPI_F08,                ONLY: MPI_COMM, MPI_FLOAT, MPI_DOUBLE
+    ! Missing: MPI_ALLTOALLV on purpose due to cray-mpi bug (see https://github.com/ecmwf-ifs/ectrans/pull/157)
 #endif
     USE TPM_STATS,              ONLY: GSTATS => GSTATS_NVTX
     USE BUFFERED_ALLOCATOR_MOD, ONLY: BUFFERED_ALLOCATOR, ASSIGN_PTR, GET_ALLOCATION

diff --git a/src/trans/gpu/internal/trmtol_mod.F90 b/src/trans/gpu/internal/trmtol_mod.F90
@@ -95,6 +95,7 @@ SUBROUTINE TRMTOL(ALLOCATOR,HTRMTOL,PFBUF_IN,PFBUF,KF_LEG)
     USE TPM_GEN,                ONLY: LSYNC_TRANS, NERR
 #if ECTRANS_HAVE_MPI
     USE MPI_F08,                ONLY: MPI_COMM, MPI_FLOAT, MPI_DOUBLE
+    ! Missing: MPI_ALLTOALLV on purpose due to cray-mpi bug (see https://github.com/ecmwf-ifs/ectrans/pull/157)
 #endif
     USE BUFFERED_ALLOCATOR_MOD, ONLY: BUFFERED_ALLOCATOR, ASSIGN_PTR, GET_ALLOCATION
     USE TPM_STATS,              ONLY: GSTATS => GSTATS_NVTX
@@ -158,7 +159,8 @@ SUBROUTINE TRMTOL(ALLOCATOR,HTRMTOL,PFBUF_IN,PFBUF,KF_LEG)
 #ifdef OMPGPU
 #endif
 #ifdef ACCGPU
-#ifdef __HIP_PLATFORM_AMD__  # Workaround for AMD GPUs - ASYNC execution of this kernel gives numerical errors
+#ifdef __HIP_PLATFORM_AMD__
+          ! Workaround for AMD GPUs - ASYNC execution of this kernel gives numerical errors
           !$ACC KERNELS DEFAULT(NONE) PRESENT(PFBUF,PFBUF_IN) COPYIN(FROM_RECV,TO_RECV,FROM_SEND,TO_SEND)
 #else
           !$ACC KERNELS ASYNC(1) DEFAULT(NONE) PRESENT(PFBUF,PFBUF_IN) COPYIN(FROM_RECV,TO_RECV,FROM_SEND,TO_SEND)
@@ -217,7 +219,8 @@ SUBROUTINE TRMTOL(ALLOCATOR,HTRMTOL,PFBUF_IN,PFBUF,KF_LEG)
       CALL GSTATS(421,1)
 
 #ifdef ACCGPU
-#ifndef __HIP_PLATFORM_AMD__  # Workaround for AMD GPUs - ASYNC execution of this kernel gives numerical errors
+#ifndef __HIP_PLATFORM_AMD__
+      ! Workaround for AMD GPUs - ASYNC execution of this kernel gives numerical errors
       !$ACC WAIT(1)
 #endif
 #endif