Merge pull request #5924 from hjelmn/v3.1.x_osc_rdma_really_needs_to_…

…do_first_touch_before_registration_to_ensure_memory_locality v3.1.x: RDMA OSC: initialize segment memory before registering the segment
open-mpi · Oct 18, 2018 · f2f5bef · f2f5bef
2 parents 655c9c9 + 4fbe078
commit f2f5bef
Showing 1 changed file with 6 additions and 8 deletions.
diff --git a/ompi/mca/osc/rdma/osc_rdma_component.c b/ompi/mca/osc/rdma/osc_rdma_component.c
@@ -647,6 +647,7 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
 
         if (size && MPI_WIN_FLAVOR_ALLOCATE == module->flavor) {
             *base = (void *)((intptr_t) module->segment_base + my_base_offset);
+            memset (*base, 0, size);
         }
 
         module->rank_array = (ompi_osc_rdma_rank_data_t *) module->segment_base;
@@ -660,7 +661,12 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
         /* initialize my state */
         memset (module->state, 0, module->state_size);
 
+        /* barrier to make sure all ranks have attached and initialized */
+        shared_comm->c_coll->coll_barrier(shared_comm, shared_comm->c_coll->coll_barrier_module);
+
         if (0 == local_rank) {
+            /* unlink the shared memory backing file */
+            opal_shmem_unlink (&module->seg_ds);
             /* just go ahead and register the whole segment */
             ret = ompi_osc_rdma_register (module, MCA_BTL_ENDPOINT_ANY, module->segment_base, total_size, MCA_BTL_REG_FLAG_ACCESS_ANY,
                                           &module->state_handle);
@@ -681,14 +687,6 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
             }
         }
 
-        /* barrier to make sure all ranks have attached */
-        shared_comm->c_coll->coll_barrier(shared_comm, shared_comm->c_coll->coll_barrier_module);
-
-        /* unlink the shared memory backing file */
-        if (0 == local_rank) {
-            opal_shmem_unlink (&module->seg_ds);
-        }
-
         if (MPI_WIN_FLAVOR_ALLOCATE == module->flavor) {
             ompi_osc_rdma_region_t *region = (ompi_osc_rdma_region_t *) module->state->regions;
             module->state->disp_unit = module->disp_unit;