From ce92276bebc50942d0888e1651665bf5eac86878 Mon Sep 17 00:00:00 2001
From: atmyers <atmyers2@gmail.com>
Date: Fri, 25 Mar 2022 10:23:15 -0700
Subject: [PATCH 1/2] use the roundoff domain in enforcePeriodic

---
 Src/Base/AMReX_Geometry.H               | 8 ++++++++
 Src/Particle/AMReX_ParticleContainerI.H | 5 +++--
 Src/Particle/AMReX_ParticleUtil.H       | 9 ++++++---
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/Src/Base/AMReX_Geometry.H b/Src/Base/AMReX_Geometry.H
index 86a6d01e9c5..773e6c69519 100644
--- a/Src/Base/AMReX_Geometry.H
+++ b/Src/Base/AMReX_Geometry.H
@@ -193,6 +193,14 @@ public:
         return {{AMREX_D_DECL(prob_domain.hi(0),prob_domain.hi(1),prob_domain.hi(2))}};
     }
 
+    GpuArray<Real,AMREX_SPACEDIM> RoundoffLoArray () const noexcept {
+        return {{AMREX_D_DECL(roundoff_domain.lo(0),roundoff_domain.lo(1),roundoff_domain.lo(2))}};
+    }
+
+    GpuArray<Real,AMREX_SPACEDIM> RoundoffHiArray () const noexcept {
+        return {{AMREX_D_DECL(roundoff_domain.hi(0),roundoff_domain.hi(1),roundoff_domain.hi(2))}};
+    }
+
     //! Returns the overall size of the domain by multiplying the ProbLength's together
     Real ProbSize () const noexcept
     {
diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H
index bf29964a9d4..c924428b034 100644
--- a/Src/Particle/AMReX_ParticleContainerI.H
+++ b/Src/Particle/AMReX_ParticleContainerI.H
@@ -224,9 +224,10 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt, Allocator>
     const auto& geom = Geom(0);
     const auto plo = geom.ProbLoArray();
     const auto phi = geom.ProbHiArray();
+    const auto rhi = geom.RoundoffHiArray();
     const auto is_per = geom.isPeriodicArray();
 
-    return enforcePeriodic(p, plo, phi, is_per);
+    return enforcePeriodic(p, plo, phi, rhi, is_per);
 }
 
 template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt,
@@ -1231,7 +1232,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt, Allocator>
                 "perhaps particles have not been initialized correctly?");
 
             int num_stay = partitionParticlesByDest(src_tile, assign_grid, BufferMap(),
-                                                    plo, phi, is_per, lev, gid, tid,
+                                                    plo, phi, rhi, is_per, lev, gid, tid,
                                                     lev_min, lev_max, nGrow, remove_negative);
 
             int num_move = np - num_stay;
diff --git a/Src/Particle/AMReX_ParticleUtil.H b/Src/Particle/AMReX_ParticleUtil.H
index 40b31967c90..19fe6541dc0 100644
--- a/Src/Particle/AMReX_ParticleUtil.H
+++ b/Src/Particle/AMReX_ParticleUtil.H
@@ -481,6 +481,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
 bool enforcePeriodic (P& p,
                       amrex::GpuArray<amrex::Real,AMREX_SPACEDIM> const& plo,
                       amrex::GpuArray<amrex::Real,AMREX_SPACEDIM> const& phi,
+                      amrex::GpuArray<amrex::Real,AMREX_SPACEDIM> const& rhi,
                       amrex::GpuArray<int,AMREX_SPACEDIM> const& is_per) noexcept
 {
     bool shifted = false;
@@ -500,8 +501,9 @@ bool enforcePeriodic (P& p,
                 p.pos(idim) += static_cast<ParticleReal>(phi[idim] - plo[idim]);
             }
             // clamp to avoid precision issues;
-            if (p.pos(idim) == phi[idim]) p.pos(idim) = static_cast<ParticleReal>(plo[idim]);
-            if (p.pos(idim) > phi[idim]) p.pos(idim) = std::nextafter( (amrex::ParticleReal) phi[idim], (amrex::ParticleReal) plo[idim]);
+            if (p.pos(idim) >= rhi[idim]) {
+                p.pos(idim) = static_cast<ParticleReal>(plo[idim]);
+            }
             shifted = true;
         }
         AMREX_ASSERT( (p.pos(idim) >= plo[idim] ) && ( p.pos(idim) < phi[idim] ));
@@ -517,6 +519,7 @@ int
 partitionParticlesByDest (PTile& ptile, const PLocator& ploc, const ParticleBufferMap& pmap,
                           const GpuArray<Real,AMREX_SPACEDIM>& plo,
                           const GpuArray<Real,AMREX_SPACEDIM>& phi,
+                          const GpuArray<Real,AMREX_SPACEDIM>& rhi,
                           const GpuArray<int ,AMREX_SPACEDIM>& is_per,
                           int lev, int gid, int /*tid*/,
                           int lev_min, int lev_max, int nGrow, bool remove_negative)
@@ -563,7 +566,7 @@ partitionParticlesByDest (PTile& ptile, const PLocator& ploc, const ParticleBuff
                 else
                 {
                     auto p_prime = p;
-                    enforcePeriodic(p_prime, plo, phi, is_per);
+                    enforcePeriodic(p_prime, plo, phi, rhi, is_per);
                     auto tup_prime = ploc(p_prime, lev_min, lev_max, nGrow);
                     assigned_grid = amrex::get<0>(tup_prime);
                     assigned_lev  = amrex::get<1>(tup_prime);

From f04ddf386f5c2684ee2a1bf802ee0481776c3f87 Mon Sep 17 00:00:00 2001
From: atmyers <atmyers2@gmail.com>
Date: Fri, 25 Mar 2022 10:38:50 -0700
Subject: [PATCH 2/2] typo fix

---
 Src/Particle/AMReX_ParticleContainerI.H | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H
index c924428b034..269b94f9f6f 100644
--- a/Src/Particle/AMReX_ParticleContainerI.H
+++ b/Src/Particle/AMReX_ParticleContainerI.H
@@ -1212,6 +1212,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt, Allocator>
     Vector<std::map<int, int> > new_sizes(num_levels);
     const auto plo    = Geom(0).ProbLoArray();
     const auto phi    = Geom(0).ProbHiArray();
+    const auto rhi    = Geom(0).RoundoffHiArray();
     const auto is_per = Geom(0).isPeriodicArray();
     for (int lev = lev_min; lev <= finest_lev_particles; ++lev)
     {