mpi: Add safe_halo option

georgebisbas · Nov 2, 2023 · 56a7ddc · 56a7ddc
1 parent 8efd2fc
commit 56a7ddc
Show file tree

Hide file tree

Showing 6 changed files with 22 additions and 4 deletions.
diff --git a/devito/__init__.py b/devito/__init__.py
@@ -96,6 +96,15 @@ def reinit_compiler(val):
 # optimisations.
 configuration.add('safe-math', 0, [0, 1], preprocessor=bool, callback=reinit_compiler)
 
+# By default Devito allows some extra halo space to satisfy derivatives
+# which may result in OOB accesses. However, in most of cases these extra halo points
+# are redundant. In single-node simulations the space overhead is usually minimal
+# compared to the computational domain, however in multi-node simulations, this may
+# result in redundant ghost-cell communications, which may have high a communication
+# cost impact.
+preprocessor = lambda i: {0: 'None', 1: 'HALF', 2: 'CUSTOM'}.get(i, i)
+configuration.add('safe-halo', 0, [0, 1, 2], preprocessor=preprocessor)
+
 # Enable/disable automatic padding for allocated data
 configuration.add('autopadding', False, [False, True])
 

diff --git a/devito/ir/equations/algorithms.py b/devito/ir/equations/algorithms.py
@@ -121,8 +121,9 @@ def lower_exprs(expressions, **kwargs):
             f = i.function
 
             # Introduce shifting to align with the computational domain
-            indices = [(lower_exprs(a) + o) for a, o in
-                       zip(i.indices, f._size_nodomain.left)]
+            indices = []
+            for a, o in zip(i.indices, f._size_nodomain.left):
+                indices.append(lower_exprs(a) + o)
 
             # Substitute spacing (spacing only used in own dimension)
             indices = [i.xreplace({d.spacing: 1, -d.spacing: -1})

diff --git a/devito/parameters.py b/devito/parameters.py
@@ -152,7 +152,8 @@ def _signature_items(self):
     'DEVITO_FIRST_TOUCH': 'first-touch',
     'DEVITO_JIT_BACKDOOR': 'jit-backdoor',
     'DEVITO_IGNORE_UNKNOWN_PARAMS': 'ignore-unknowns',
-    'DEVITO_SAFE_MATH': 'safe-math'
+    'DEVITO_SAFE_MATH': 'safe-math',
+    'DEVITO_SAFE_HALO': 'safe-halo',
 }
 
 env_vars_deprecated = {

diff --git a/devito/types/dense.py b/devito/types/dense.py
@@ -1113,7 +1113,12 @@ def __halo_setup__(self, **kwargs):
         else:
             space_order = kwargs.get('space_order', 1)
             if isinstance(space_order, int):
-                halo = (space_order, space_order)
+                if configuration['safe-halo'] == 'HALF':
+                    halo = (int(space_order/2), int(space_order/2))
+                elif configuration['safe-halo'] == 'CUSTOM':
+                    halo = (int(space_order/2) + 1, int(space_order/2) + 1)
+                else:
+                    halo = (space_order, space_order)
             elif isinstance(space_order, tuple) and len(space_order) == 3:
                 _, left_points, right_points = space_order
                 halo = (left_points, right_points)

diff --git a/examples/seismic/acoustic/acoustic_example.py b/examples/seismic/acoustic/acoustic_example.py
@@ -38,6 +38,7 @@ def run(shape=(50, 50, 50), spacing=(20.0, 20.0, 20.0), tn=1000.0,
     save = full_run and not checkpointing
     # Define receiver geometry (spread across x, just below surface)
     rec, u, summary = solver.forward(save=save, autotune=autotune)
+    print("Norm u:", norm(u))
 
     if preset == 'constant-isotropic':
         # With a new m as Constant

diff --git a/examples/seismic/elastic/elastic_example.py b/examples/seismic/elastic/elastic_example.py
@@ -29,6 +29,7 @@ def run(shape=(50, 50), spacing=(20.0, 20.0), tn=1000.0,
     info("Applying Forward")
     # Define receiver geometry (spread across x, just below surface)
     rec1, rec2, v, tau, summary = solver.forward(autotune=autotune)
+    print("Norm v:", norm(v[0]))
     return (summary.gflopss, summary.oi, summary.timings,
             [rec1, rec2, v, tau])