Merge branch 'bugfix.parallel.common.operations' into 'master.dev'

[bugfix.parallel.common.operations] Parallelize common MPI operations See merge request piclas/piclas!808
piclas-framework · Jul 10, 2023 · d7654da · d7654da
2 parents c3346b5 + 6350cc2
commit d7654da
Show file tree

Hide file tree

Showing 8 changed files with 255 additions and 127 deletions.
diff --git a/src/mesh/mesh.f90 b/src/mesh/mesh.f90
@@ -882,7 +882,7 @@ SUBROUTINE InitElemVolumes()
 #if USE_MPI
 #ifdef PARTICLES
 ! Compute-node mesh volume
-CNVolume = SUM(ElemVolume_Shared(:))
+CALL MPI_ALLREDUCE(LocalVolume,CNVolume,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_SHARED,IERROR)
 IF (myComputeNodeRank.EQ.0) THEN
   ! All-reduce between node leaders
   CALL MPI_ALLREDUCE(CNVolume,MeshVolume,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_LEADERS_SHARED,IERROR)

diff --git a/src/mesh/mesh_tools.f90 b/src/mesh/mesh_tools.f90
@@ -222,7 +222,7 @@ END FUNCTION GetGlobalSideID_iSide
 !==================================================================================================================================!
 PPURE INTEGER FUNCTION GetGlobalElemID_fromTotalElem(iElem)
 ! MODULES
-USE MOD_MPI_Shared_Vars, ONLY:CNTotalElem2GlobalElem
+USE MOD_Particle_Mesh_Vars, ONLY:CNTotalElem2GlobalElem
 ! INPUT / OUTPUT VARIABLES
 INTEGER,INTENT(IN)              :: iElem
 !-----------------------------------------------------------------------------------------------------------------------------------
@@ -242,7 +242,7 @@ END FUNCTION GetGlobalElemID_fromTotalElem
 !==================================================================================================================================!
 PPURE INTEGER FUNCTION GetGlobalSideID_fromTotalSide(iSide)
 ! MODULES
-USE MOD_MPI_Shared_Vars, ONLY:CNTotalSide2GlobalSide
+USE MOD_Particle_Mesh_Vars, ONLY:CNTotalSide2GlobalSide
 ! INPUT / OUTPUT VARIABLES
 INTEGER,INTENT(IN)              :: iSide
 !-----------------------------------------------------------------------------------------------------------------------------------
@@ -364,7 +364,7 @@ END FUNCTION CNSideID_is_iSide
 !==================================================================================================================================!
 PPURE INTEGER FUNCTION GetGlobalElem2CNTotalElem(iElem)
 ! MODULES
-USE MOD_MPI_Shared_Vars, ONLY:GlobalElem2CNTotalElem
+USE MOD_Particle_Mesh_Vars, ONLY:GlobalElem2CNTotalElem
 ! INPUT / OUTPUT VARIABLES
 INTEGER,INTENT(IN)              :: iElem ! Global element ID
 !-----------------------------------------------------------------------------------------------------------------------------------
@@ -384,7 +384,7 @@ END FUNCTION GetGlobalElem2CNTotalElem
 !==================================================================================================================================!
 PPURE INTEGER FUNCTION GetGlobalSide2CNTotalSide(iSide)
 ! MODULES
-USE MOD_MPI_Shared_Vars, ONLY:GlobalSide2CNTotalSide
+USE MOD_Particle_Mesh_Vars, ONLY:GlobalSide2CNTotalSide
 ! INPUT / OUTPUT VARIABLES
 INTEGER,INTENT(IN)              :: iSide ! Global element ID
 !-----------------------------------------------------------------------------------------------------------------------------------

diff --git a/src/mpi/mpi_shared_vars.f90 b/src/mpi/mpi_shared_vars.f90
@@ -70,10 +70,6 @@ MODULE MOD_MPI_Shared_Vars
                                                             !> 1 - sides from local leader to other leader
                                                             !> 2 - sides from other leader to local leader
 
-INTEGER, ALLOCATABLE :: CNTotalElem2GlobalElem(:)           !> Compute Nodes mapping 1:nTotal -> 1:nGlobal
-INTEGER, ALLOCATABLE :: GlobalElem2CNTotalElem(:)           !> Reverse Mapping
-INTEGER, ALLOCATABLE :: CNTotalSide2GlobalSide(:)           !> Compute Nodes mapping 1:nTotal -> 1:nGlobal
-INTEGER, ALLOCATABLE :: GlobalSide2CNTotalSide(:)           !> Reverse Mapping
 
 INTEGER            :: MPI_INFO_SHARED_LOOSE                 !> MPI_INFO object allowing for re-ordering of same origin atomic RMA operations
 

diff --git a/src/particles/dsmc/dsmc_symmetry.f90 b/src/particles/dsmc/dsmc_symmetry.f90
@@ -110,6 +110,7 @@ SUBROUTINE DSMC_2D_InitVolumes()
 REAL                            :: radius, triarea(2)
 #if USE_MPI
 REAL                            :: CNVolume
+INTEGER                         :: offsetElemCNProc
 #endif /*USE_MPI*/
 LOGICAL                         :: SymmetryBCExists
 INTEGER                         :: firstElem, lastElem, firstSide, lastSide
@@ -223,7 +224,9 @@ SUBROUTINE DSMC_2D_InitVolumes()
 CALL BARRIER_AND_SYNC(ElemVolume_Shared_Win    ,MPI_COMM_SHARED)
 CALL BARRIER_AND_SYNC(ElemCharLength_Shared_Win,MPI_COMM_SHARED)
 ! Compute-node mesh volume
-CNVolume = SUM(ElemVolume_Shared(:))
+offsetElemCNProc = offsetElem - offsetComputeNodeElem
+CNVolume = SUM(ElemVolume_Shared(offsetElemCNProc+1:offsetElemCNProc+nElems))
+CALL MPI_ALLREDUCE(MPI_IN_PLACE,CNVolume,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_SHARED,iError)
 IF (myComputeNodeRank.EQ.0) THEN
   ! All-reduce between node leaders
   CALL MPI_ALLREDUCE(CNVolume,MeshVolume,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_LEADERS_SHARED,IERROR)
@@ -630,7 +633,7 @@ SUBROUTINE DSMC_2D_SetInClones()
     PartStateIntEn(1:2,PositionNbr) = ClonedParticles(iPart,DelayCounter)%PartStateIntEn(1:2)
     IF(DSMC%ElectronicModel.GT.0) THEN
       PartStateIntEn(3,PositionNbr) = ClonedParticles(iPart,DelayCounter)%PartStateIntEn(3)
-      IF ((DSMC%ElectronicModel.EQ.2).AND.(.NOT.((SpecDSMC(ClonedParticles(iPart,DelayCounter)%Species)%InterID.EQ.4) & 
+      IF ((DSMC%ElectronicModel.EQ.2).AND.(.NOT.((SpecDSMC(ClonedParticles(iPart,DelayCounter)%Species)%InterID.EQ.4) &
           .OR.SpecDSMC(ClonedParticles(iPart,DelayCounter)%Species)%FullyIonized))) THEN
         IF(ALLOCATED(ElectronicDistriPart(PositionNbr)%DistriFunc)) DEALLOCATE(ElectronicDistriPart(PositionNbr)%DistriFunc)
         ALLOCATE(ElectronicDistriPart(PositionNbr)%DistriFunc(1:SpecDSMC(ClonedParticles(iPart,DelayCounter)%Species)%MaxElecQuant))
@@ -921,7 +924,7 @@ SUBROUTINE DSMC_2D_TreatIdenticalParticles(iPair, nPair, nPart, iElem, iPartIndx
                              + (PartState(5,iPart_p1) - PartState(5,iPart_p2))**2 &
                              + (PartState(6,iPart_p1) - PartState(6,iPart_p2))**2
   ELSE IF (iPair.LT.nPair) THEN
-    IF (.NOT.Coll_pData(iPair+1)%NeedForRec) THEN 
+    IF (.NOT.Coll_pData(iPair+1)%NeedForRec) THEN
     ! "Partner-Tausch": if there are pairs ahead in the pairing list, the next is pair is broken up and collision partners
     ! are swapped
       CollInf%Coll_CaseNum(Coll_pData(iPair)%PairType) = CollInf%Coll_CaseNum(Coll_pData(iPair)%PairType) - 1