#900 trace- enqueue sending registration of user events

- This unwinds the stack to avoid failing attempting to re-grant MPI
  access while running under a PMPI wrapper..

src/vt/trace/trace.h

@@ -229,6 +229,10 @@ struct Trace : runtime::component::Component<Trace> {
   /**
    * \brief Collectively register a user event
    *
+   * \note All collectively registered events \b MUST be registered
+   * on all nodes in the same sequence. This guarantees valid unique IDs
+   * and avoids the necessity to send event registration messages.
+   *
    * \note For users, it is recommended that the free function be called
    * \c registerEventCollective
    *

src/vt/trace/trace_user.h

@@ -53,7 +53,11 @@
 namespace vt { namespace trace {
 
 /**
- * \brief Register a user event collectively
+ * \brief Register a user event collectively.
+ *
+ * \note All collectively registered events \b MUST be registered
+ * on all nodes in the same sequence. This guarantees valid unique IDs
+ * and avoids the necessity to send event registration messages.
  *
  * \param[in] name the name of the event
  *

src/vt/trace/trace_user_event.cc

@@ -108,8 +108,11 @@ UserEventIDType UserEventRegistry::newEventImpl(
 
   // Ensure newly inserted events are passed to rank 0 if requested.
   if (inserted and report_up and node not_eq 0) {
-    auto msg = makeMessage<NewUserEventMsg>(user, event, in_event);
-    theMsg()->sendMsg<NewUserEventMsg,newEventHan>(0, msg.get());
+    // Enqueue work, as this can be called from within PMPI wrappers.
+    theSched()->enqueue([user, event, in_event]{
+      auto msg = makeMessage<NewUserEventMsg>(user, event, in_event);
+      theMsg()->sendMsg<NewUserEventMsg,newEventHan>(0, msg.get());
+    });
   }
 
   return event;