Set guard cells for allocation using field solver stencil and particl…

…e shape factor (#1969)

Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H

@@ -108,6 +108,14 @@ struct CartesianCKCAlgorithm {
         return delta_t;
     }
 
+    /**
+     * \brief Returns maximum number of guard cells required by the field-solve
+     */
+    static amrex::IntVect GetMaxGuardCell () {
+        // The ckc solver requires one guard cell in each dimension
+        return (amrex::IntVect(AMREX_D_DECL(1,1,1)));
+    }
+
     /**
      * Perform derivative along x on a cell-centered grid, from a nodal field `F` */
     AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE

Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianNodalAlgorithm.H

@@ -55,6 +55,14 @@ struct CartesianNodalAlgorithm {
         return delta_t;
     }
 
+    /**
+     * \brief Returns maximum number of guard cells required by the field-solve
+     */
+    static amrex::IntVect GetMaxGuardCell () {
+        // The nodal solver requires one guard cell in each dimension
+        return (amrex::IntVect(AMREX_D_DECL(1,1,1)));
+    }
+
     /**
      * Perform derivative along x
      * (For a solver on a staggered grid, `UpwardDx` and `DownwardDx` take into

Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H

@@ -55,6 +55,14 @@ struct CartesianYeeAlgorithm {
         return delta_t;
     }
 
+    /**
+     * \brief Returns maximum number of guard cells required by the field-solve
+     */
+    static amrex::IntVect GetMaxGuardCell () {
+        // The yee solver requires one guard cell in each dimension
+        return (amrex::IntVect(AMREX_D_DECL(1,1,1)));
+    }
+
     /**
      * Perform derivative along x on a cell-centered grid, from a nodal field `F`*/
     AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE

Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H

@@ -65,6 +65,14 @@ struct CylindricalYeeAlgorithm {
         return delta_t;
     }
 
+    /**
+     * \brief Returns maximum number of guard cells required by the field-solve
+     */
+    static amrex::IntVect GetMaxGuardCell () {
+        // The cylindrical solver requires one guard cell in each dimension
+        return (amrex::IntVect(AMREX_D_DECL(1,1,1)));
+    }
+
     /** Applies the differential operator `1/r * d(rF)/dr`,
      * where `F` is on a *nodal* grid in `r`
      * and the differential operator is evaluated on a *cell-centered* grid.

Source/Parallelization/GuardCellManager.cpp

@@ -8,7 +8,13 @@
 #include "Filter/NCIGodfreyFilter.H"
 #include "Utils/WarpXAlgorithmSelection.H"
 #include "Utils/WarpXConst.H"
-
+#ifdef WARPX_DIM_RZ
+#    include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H"
+#else
+#    include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H"
+#    include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianNodalAlgorithm.H"
+#    include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H"
+#endif
 #include <AMReX_ParmParse.H>
 #include <AMReX.H>
 
@@ -184,11 +190,36 @@ guardCellManager::Init (
         ng_FieldSolver = ng_alloc_EB;
         ng_FieldSolverF = ng_alloc_EB;
         ng_FieldSolverG = ng_alloc_EB;
-    } else {
-        ng_FieldSolver = IntVect(AMREX_D_DECL(1, 1, 1));
-        ng_FieldSolverF = IntVect(AMREX_D_DECL(1, 1, 1));
-        ng_FieldSolverG = IntVect(AMREX_D_DECL(1, 1, 1));
     }
+#ifdef WARPX_DIM_RZ
+    else if (maxwell_solver_id == MaxwellSolverAlgo::Yee) {
+        ng_FieldSolver  = CylindricalYeeAlgorithm::GetMaxGuardCell();
+        ng_FieldSolverF = CylindricalYeeAlgorithm::GetMaxGuardCell();
+        ng_FieldSolverG = CylindricalYeeAlgorithm::GetMaxGuardCell();
+    }
+#else
+    else {
+        if (do_nodal) {
+            ng_FieldSolver  = CartesianNodalAlgorithm::GetMaxGuardCell();
+            ng_FieldSolverF = CartesianNodalAlgorithm::GetMaxGuardCell();
+            ng_FieldSolverG = CartesianNodalAlgorithm::GetMaxGuardCell();
+        } else if (maxwell_solver_id == MaxwellSolverAlgo::Yee) {
+            ng_FieldSolver  = CartesianYeeAlgorithm::GetMaxGuardCell();
+            ng_FieldSolverF = CartesianYeeAlgorithm::GetMaxGuardCell();
+            ng_FieldSolverG = CartesianYeeAlgorithm::GetMaxGuardCell();
+        } else if (maxwell_solver_id == MaxwellSolverAlgo::CKC) {
+            ng_FieldSolver  = CartesianCKCAlgorithm::GetMaxGuardCell();
+            ng_FieldSolverF = CartesianCKCAlgorithm::GetMaxGuardCell();
+            ng_FieldSolverG = CartesianCKCAlgorithm::GetMaxGuardCell();
+        }
+    }
+#endif
+
+    // Number of guard cells is the max of that determined by particle shape factor and
+    // the stencil used in the field solve
+    ng_alloc_EB.max( ng_FieldSolver );
+    ng_alloc_F.max( ng_FieldSolverF );
+    ng_alloc_G.max( ng_FieldSolverG );
 
     if (safe_guard_cells){
         // Run in safe mode: exchange all allocated guard cells at each
@@ -202,9 +233,6 @@ guardCellManager::Init (
             ng_MovingWindow = ng_alloc_EB;
         }
     } else {
-
-        ng_FieldSolver = ng_FieldSolver.min(ng_alloc_EB);
-
         // Compute number of cells required for Field Gather
         int FGcell[4] = {0,1,1,2}; // Index is nox
         IntVect ng_FieldGather_noNCI = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox]));
@@ -224,8 +252,6 @@ guardCellManager::Init (
         // Make sure we do not exchange more guard cells than allocated.
         ng_FieldGather = ng_FieldGather.min(ng_alloc_EB);
         ng_UpdateAux = ng_UpdateAux.min(ng_alloc_EB);
-        ng_FieldSolverF = ng_FieldSolverF.min(ng_alloc_F);
-        ng_FieldSolverG = ng_FieldSolverG.min(ng_alloc_G);
         // Only FillBoundary(ng_FieldGather) is called between consecutive
         // field solves. So ng_FieldGather must have enough cells
         // for the field solve too.