From a6bef90e3bb37e7773785d75ccca2b74cc911727 Mon Sep 17 00:00:00 2001
From: Jonathan Lifflander <jliffla@sandia.gov>
Date: Fri, 16 Oct 2020 12:33:36 -0700
Subject: [PATCH] #1111: rdma: fix missing allow MPI calls causing test failure

---
 src/vt/pmpi/generate_mpi_wrappers.pl |  1 +
 src/vt/rdmahandle/holder.impl.h      | 15 +++++++++++++++
 src/vt/rdmahandle/request_holder.cc  |  2 ++
 3 files changed, 18 insertions(+)

diff --git a/src/vt/pmpi/generate_mpi_wrappers.pl b/src/vt/pmpi/generate_mpi_wrappers.pl
index 50ace5c10d..f9c0e12b2c 100644
--- a/src/vt/pmpi/generate_mpi_wrappers.pl
+++ b/src/vt/pmpi/generate_mpi_wrappers.pl
@@ -127,6 +127,7 @@ sub extract_defs {
     MPI_Comm_rank
     MPI_Comm_size
     MPI_Get_.*(?<!accumulate)
+    MPI_Win_(lock|unlock)
 );
 
 sub should_guard_call {
diff --git a/src/vt/rdmahandle/holder.impl.h b/src/vt/rdmahandle/holder.impl.h
index 7514a8dc95..fa4599b111 100644
--- a/src/vt/rdmahandle/holder.impl.h
+++ b/src/vt/rdmahandle/holder.impl.h
@@ -46,6 +46,7 @@
 #define INCLUDED_VT_RDMAHANDLE_HOLDER_IMPL_H
 
 #include "vt/config.h"
+#include "vt/runtime/mpi_access.h"
 
 namespace vt { namespace rdma {
 
@@ -100,6 +101,7 @@ void Holder<T,E>::allocateDataWindow(std::size_t const in_len) {
 
 template <typename T, HandleEnum E>
 std::size_t Holder<T,E>::getCount(vt::NodeType node, Lock l) {
+  VT_ALLOW_MPI_CALLS;
   uint64_t result = 0;
   auto mpi_type = TypeMPI<uint64_t>::getType();
   {
@@ -116,6 +118,7 @@ std::size_t Holder<T,E>::getCount(vt::NodeType node, Lock l) {
 
 template <typename T, HandleEnum E>
 void Holder<T,E>::deallocate() {
+  VT_ALLOW_MPI_CALLS;
   if (E == HandleEnum::StaticSize and ready_) {
     MPI_Win_free(&data_window_);
     MPI_Free_mem(data_base_);
@@ -155,6 +158,7 @@ RequestHolder Holder<T,E>::rget(
         "MPI_Get({}, {}, {}, {}, {}, {}, {}, window);\n",
         print_ptr(ptr), len, mpi_type_str, node, offset, len, mpi_type_str
       );
+      VT_ALLOW_MPI_CALLS;
       MPI_Get(ptr, len, mpi_type, node, offset, len, mpi_type, data_window_);
     });
   } else {
@@ -164,6 +168,7 @@ RequestHolder Holder<T,E>::rget(
       "MPI_Rget({}, {}, {}, {}, {}, {}, {}, window);\n",
       print_ptr(ptr), len, mpi_type_str, node, offset, len, mpi_type_str
     );
+    VT_ALLOW_MPI_CALLS;
     MPI_Rget(ptr, len, mpi_type, node, offset, len, mpi_type, data_window_, r.add());
   }
   return r;
@@ -191,6 +196,7 @@ RequestHolder Holder<T,E>::rput(
         "MPI_Put({}, {}, {}, {}, {}, {}, {}, window);\n",
         print_ptr(ptr), len, mpi_type_str, node, offset, len, mpi_type_str
       );
+      VT_ALLOW_MPI_CALLS;
       MPI_Put(ptr, len, mpi_type, node, offset, len, mpi_type, data_window_);
     });
   } else {
@@ -200,6 +206,7 @@ RequestHolder Holder<T,E>::rput(
       "MPI_Rput({}, {}, {}, {}, {}, {}, {}, window);\n",
       print_ptr(ptr), len, mpi_type_str, node, offset, len, mpi_type_str
     );
+    VT_ALLOW_MPI_CALLS;
     MPI_Rput(ptr, len, mpi_type, node, offset, len, mpi_type, data_window_, r.add());
   }
   return r;
@@ -224,6 +231,7 @@ T Holder<T,E>::fetchOp(vt::NodeType node, Lock l, T in, int offset, MPI_Op op) {
       "MPI_Fetch_and_op({}, {}, {}, {}, {}, window);\n",
       in, print_ptr(&out), mpi_type_str, node, offset
     );
+    VT_ALLOW_MPI_CALLS;
     MPI_Fetch_and_op(&in, &out, mpi_type, node, offset, op, data_window_);
   }
   return out;
@@ -245,6 +253,7 @@ RequestHolder Holder<T,E>::raccum(
         "MPI_Accumulate({}, {}, {}, {}, {}, {}, {}, window);\n",
         print_ptr(ptr), len, mpi_type_str, node, offset, len, mpi_type_str
       );
+      VT_ALLOW_MPI_CALLS;
       MPI_Accumulate(
         ptr, len, mpi_type, node, offset, len, mpi_type, op, data_window_
       );
@@ -256,6 +265,7 @@ RequestHolder Holder<T,E>::raccum(
       "MPI_Raccumulate({}, {}, {}, {}, {}, {}, {}, window);\n",
       print_ptr(ptr), len, mpi_type_str, node, offset, len, mpi_type_str
     );
+    VT_ALLOW_MPI_CALLS;
     MPI_Raccumulate(
       ptr, len, mpi_type, node, offset, len, mpi_type, op, data_window_, r.add()
     );
@@ -273,26 +283,31 @@ void Holder<T,E>::accum(
 
 template <typename T, HandleEnum E>
 void Holder<T,E>::fence(int assert) {
+  VT_ALLOW_MPI_CALLS;
   MPI_Win_fence(assert, data_window_);
 }
 
 template <typename T, HandleEnum E>
 void Holder<T,E>::sync() {
+  VT_ALLOW_MPI_CALLS;
   MPI_Win_sync(data_window_);
 }
 
 template <typename T, HandleEnum E>
 void Holder<T,E>::flush(vt::NodeType node) {
+  VT_ALLOW_MPI_CALLS;
   MPI_Win_flush(node, data_window_);
 }
 
 template <typename T, HandleEnum E>
 void Holder<T,E>::flushLocal(vt::NodeType node) {
+  VT_ALLOW_MPI_CALLS;
   MPI_Win_flush_local(node, data_window_);
 }
 
 template <typename T, HandleEnum E>
 void Holder<T,E>::flushAll() {
+  VT_ALLOW_MPI_CALLS;
   MPI_Win_flush_all(data_window_);
 }
 
diff --git a/src/vt/rdmahandle/request_holder.cc b/src/vt/rdmahandle/request_holder.cc
index 4ebd8fa7be..b75b4f072f 100644
--- a/src/vt/rdmahandle/request_holder.cc
+++ b/src/vt/rdmahandle/request_holder.cc
@@ -47,6 +47,7 @@
 
 #include "vt/config.h"
 #include "vt/rdmahandle/request_holder.h"
+#include "vt/runtime/mpi_access.h"
 
 namespace vt { namespace rdma {
 
@@ -60,6 +61,7 @@ MPI_Request* RequestHolder::add() {
 }
 
 bool RequestHolder::test() {
+  VT_ALLOW_MPI_CALLS;
   std::vector<MPI_Request> new_reqs;
   std::vector<MPI_Status> stats;
   stats.resize(reqs_.size());