From 737311166281589de642dbcaa19ac8d4afca3caa Mon Sep 17 00:00:00 2001
From: Ralph Castain <rhc@open-mpi.org>
Date: Mon, 25 Apr 2016 08:04:35 -0700
Subject: [PATCH] Somehow, the logic for finalize got lost, so restore it here.
 If pmix.fence_nb is available, then call it and cycle opal_progress until
 complete. If pmix.fence_nb is not available, then do an MPI_Barrier and call
 pmix.fence.

Needs to go over to 2.x
---
 ompi/runtime/ompi_mpi_finalize.c | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/ompi/runtime/ompi_mpi_finalize.c b/ompi/runtime/ompi_mpi_finalize.c
index 029eef6b225..789a05d39d7 100644
--- a/ompi/runtime/ompi_mpi_finalize.c
+++ b/ompi/runtime/ompi_mpi_finalize.c
@@ -90,12 +90,19 @@
 extern bool ompi_enable_timing;
 extern bool ompi_enable_timing_ext;
 
+static void fence_cbfunc(int status, void *cbdata)
+{
+  volatile bool *active = (volatile bool*)cbdata;
+  *active = false;
+}
+
 int ompi_mpi_finalize(void)
 {
     int ret = MPI_SUCCESS;
     opal_list_item_t *item;
     ompi_proc_t** procs;
     size_t nprocs;
+    volatile bool active;
     OPAL_TIMING_DECLARE(tm);
     OPAL_TIMING_INIT_EXT(&tm, OPAL_TIMING_GET_TIME_OF_DAY);
 
@@ -239,7 +246,14 @@ int ompi_mpi_finalize(void)
        del_procs behavior around May of 2014 (see
        https://svn.open-mpi.org/trac/ompi/ticket/4669#comment:4 for
        more details). */
-    opal_pmix.fence(NULL, 0);
+    if (NULL != opal_pmix.fence_nb) {
+      active = true;
+      opal_pmix.fence_nb(NULL, 0, fence_cbfunc, (void*)&active);
+      OMPI_WAIT_FOR_COMPLETION(active);
+    } else {
+      MPI_Barrier(MPI_COMM_WORLD);
+      opal_pmix.fence(NULL, 0);
+    }
 
     /* check for timing request - get stop time and report elapsed
      time if so */