diff --git a/src/vt/serialization/sizer.h b/src/vt/serialization/sizer.h index c8cd26b0e6..35ca714e00 100644 --- a/src/vt/serialization/sizer.h +++ b/src/vt/serialization/sizer.h @@ -62,9 +62,7 @@ struct MsgSizer< MsgT, typename std::enable_if_t::value - and (::vt::messaging::msg_serialization_mode::supported - or ::vt::messaging::msg_serialization_mode::required) - // ::vt::messaging::has_own_serialize + and ::vt::messaging::msg_serialization_mode::required > > { static std::size_t get(MsgT* msg) { diff --git a/src/vt/topos/location/location.h b/src/vt/topos/location/location.h index 761cb52701..fec0849067 100644 --- a/src/vt/topos/location/location.h +++ b/src/vt/topos/location/location.h @@ -229,24 +229,21 @@ struct EntityLocationCoord : LocationCoord { * * \param[in] id the entity ID * \param[in] home_node home node for entity - * \param[in] m pointer to the message + * \param[in] m message shared pointer */ template *f> void routeMsgHandler( - EntityID const& id, NodeType const& home_node, MessageT *m + EntityID const& id, NodeType const& home_node, + MsgSharedPtr const& msg ); /** - * \brief Route a serialized message with a custom handler + * \brief Route a message with a custom handler where the element is local * - * \param[in] id the entity ID - * \param[in] home_node home node for entity - * \param[in] msg pointer to the message + * \param[in] m message shared pointer */ - template *f> - void routeMsgSerializeHandler( - EntityID const& id, NodeType const& home_node, MsgSharedPtr msg - ); + template + void routeMsgHandlerLocal(MsgSharedPtr const& msg); /** * \brief Route a message to the default handler @@ -254,28 +251,15 @@ struct EntityLocationCoord : LocationCoord { * \param[in] id the entity ID * \param[in] home_node home node for the entity * \param[in] msg pointer to the message - * \param[in] serialize_msg whether it should be serialized (optional) * \param[in] from_node the sending node (optional) */ template void routeMsg( - EntityID const& id, NodeType const& home_node, MsgSharedPtr msg, - bool const serialize_msg = false, + EntityID const& id, NodeType const& home_node, + MsgSharedPtr const& msg, NodeType from_node = uninitialized_destination ); - /** - * \brief Route a message to the default handler - * - * \param[in] id the entity ID - * \param[in] home_node home node for the entity - * \param[in] msg pointer to the message - */ - template - void routeMsgSerialize( - EntityID const& id, NodeType const& home_node, MsgSharedPtr msg - ); - /** * \internal \brief Route a message with non-eager protocol * @@ -357,7 +341,7 @@ struct EntityLocationCoord : LocationCoord { * \return whether it is of eager size */ template - bool useEagerProtocol(MsgSharedPtr msg) const; + bool useEagerProtocol(MsgSharedPtr const& msg) const; private: /** @@ -392,21 +376,19 @@ struct EntityLocationCoord : LocationCoord { /** * \internal \brief Route a message to destination with eager protocol * - * \param[in] is_serialized whether it is serialized * \param[in] id the entity ID * \param[in] home_node the home node * \param[in] msg the message to route */ template void routeMsgEager( - bool const is_serialized, EntityID const& id, NodeType const& home_node, - MsgSharedPtr msg + EntityID const& id, NodeType const& home_node, + MsgSharedPtr const& msg ); /** * \internal \brief Route a message to destination with rendezvous protocol * - * \param[in] is_serialized whether it is serialized * \param[in] id the entity ID * \param[in] home_node the home node * \param[in] to_node destination node @@ -414,8 +396,8 @@ struct EntityLocationCoord : LocationCoord { */ template void routeMsgNode( - bool const is_serialized, EntityID const& id, NodeType const& home_node, - NodeType const& to_node, MsgSharedPtr msg + EntityID const& id, NodeType const& home_node, NodeType const& to_node, + MsgSharedPtr const& msg ); /** diff --git a/src/vt/topos/location/location.impl.h b/src/vt/topos/location/location.impl.h index bb3c4ad407..22fff5a6ab 100644 --- a/src/vt/topos/location/location.impl.h +++ b/src/vt/topos/location/location.impl.h @@ -277,14 +277,37 @@ void EntityLocationCoord::clearCache() { recs_.clearCache(); } +namespace detail { + +template +struct IsSerializable { + static constexpr bool const is_ser = false; +}; + +template +struct IsSerializable< + MsgT, + typename std::enable_if_t::value + and ::vt::messaging::msg_serialization_mode::required + > +> +{ + static constexpr bool const is_ser = true; +}; + +} /* end namespace detail */ + template template -bool EntityLocationCoord::useEagerProtocol(MsgSharedPtr msg) const { - - bool const is_small = sizeof(*msg) < small_msg_max_size; - bool const is_serialized = msg->getSerialize(); - // could change according to entity type or another criterion - return is_small and not is_serialized; +bool EntityLocationCoord::useEagerProtocol( + MsgSharedPtr const& msg +) const { + if (detail::IsSerializable::is_ser) { + return false; + } else { + return sizeof(*msg) < small_msg_max_size; + } } template @@ -313,8 +336,8 @@ void EntityLocationCoord::insertPendingEntityAction( template template void EntityLocationCoord::routeMsgEager( - bool const is_serialized, EntityID const& id, NodeType const& home_node, - MsgSharedPtr msg + EntityID const& id, NodeType const& home_node, + MsgSharedPtr const& msg ) { auto const& this_node = theContext()->getNode(); NodeType route_to_node = uninitialized_destination; @@ -325,8 +348,8 @@ void EntityLocationCoord::routeMsgEager( vt_debug_print( normal, location, "EntityLocationCoord: routeMsgEager: found={}, home_node={}, " - "route_to_node={}, is_serialized={}, id={}\n", - found, home_node, route_to_node, is_serialized, id + "route_to_node={}, id={}\n", + found, home_node, route_to_node, id ); if (found) { @@ -360,11 +383,11 @@ void EntityLocationCoord::routeMsgEager( vt_debug_print( normal, location, "EntityLocationCoord: routeMsgEager: home_node={}, route_node={}, " - "is_serialized={}, id={}\n", - home_node, route_to_node, is_serialized, id + "id={}\n", + home_node, route_to_node, id ); - return routeMsgNode(is_serialized,id,home_node,route_to_node,msg); + return routeMsgNode(id, home_node, route_to_node, msg); } template @@ -481,8 +504,8 @@ void EntityLocationCoord::sendEagerUpdate( template template void EntityLocationCoord::routeMsgNode( - bool const is_serialized, EntityID const& id, NodeType const& home_node, - NodeType const& to_node, MsgSharedPtr msg + EntityID const& id, NodeType const& home_node, NodeType const& to_node, + MsgSharedPtr const& msg ) { auto const& this_node = theContext()->getNode(); auto const epoch = theMsg()->getEpochContextMsg(msg); @@ -490,8 +513,8 @@ void EntityLocationCoord::routeMsgNode( vt_debug_print( normal, location, "EntityLocationCoord: routeMsgNode: to_node={}, this_node={}: inst={}, " - "is_serialized={}, home_node={}, id={}, ref={}, from={}, msg={}, epoch={:x}\n", - to_node, this_node, this_inst, is_serialized, home_node, id, + "home_node={}, id={}, ref={}, from={}, msg={}, epoch={:x}\n", + to_node, this_node, this_inst, home_node, id, envelopeGetRef(msg->env), msg->getLocFromNode(), print_ptr(msg.get()), epoch ); @@ -512,8 +535,9 @@ void EntityLocationCoord::routeMsgNode( // set the instance on the message to deliver to the correct manager msg->setLocInst(this_inst); + auto m = msg; // send to the node discovered by the location manager - theMsg()->sendMsg(to_node, msg); + theMsg()->sendMsg(to_node, m); } else { vt_debug_print( normal, location, @@ -609,7 +633,7 @@ void EntityLocationCoord::routeMsgNode( * typically when an non-migrated registration occurs off the home * node and messages are buffered, awaiting forwarding information. */ - routeMsgNode(is_serialized,id_,home_node,resolved,msg); + routeMsgNode(id_, home_node, resolved,msg); } theMsg()->popEpoch(epoch); theTerm()->consume(epoch); @@ -630,7 +654,8 @@ void EntityLocationCoord::routeNonEagerAction( template template *f> void EntityLocationCoord::routeMsgHandler( - EntityID const& id, NodeType const& home_node, MessageT *m + EntityID const& id, NodeType const& home_node, + MsgSharedPtr const& msg ) { using auto_registry::HandlerManagerType; @@ -638,47 +663,34 @@ void EntityLocationCoord::routeMsgHandler( # if vt_check_enabled(trace_enabled) HandlerManagerType::setHandlerTrace( - handler, envelopeGetTraceRuntimeEnabled(m->env) + handler, envelopeGetTraceRuntimeEnabled(msg->env) ); # endif - m->setHandler(handler); - auto msg = promoteMsg(m); - return routeMsg(id,home_node,msg); -} - -template -template *f> -void EntityLocationCoord::routeMsgSerializeHandler( - EntityID const& id, NodeType const& home_node, MsgSharedPtr m -) { - using auto_registry::HandlerManagerType; - - auto handler = auto_registry::makeAutoHandler(); + msg->setHandler(handler); -# if vt_check_enabled(trace_enabled) - HandlerManagerType::setHandlerTrace( - handler, envelopeGetTraceRuntimeEnabled(m->env) - ); -# endif - - m->setHandler(handler); - return routeMsg(id,home_node,m,true); + if (local_registered_.find(id) == local_registered_.end()) { + return routeMsg(id,home_node,msg); + } else { + return routeMsgHandlerLocal(msg); + } } template template -void EntityLocationCoord::routeMsgSerialize( - EntityID const& id, NodeType const& home_node, MsgSharedPtr m +void EntityLocationCoord::routeMsgHandlerLocal( + MsgSharedPtr const& msg ) { - return routeMsg(id,home_node,m,true); + runnable::makeRunnable(msg, true, msg->getHandler(), theContext()->getNode()) + .withTDEpochFromMsg() + .run(); } template template void EntityLocationCoord::routeMsg( - EntityID const& id, NodeType const& home_node, MsgSharedPtr msg, - bool const serialize_msg, NodeType from_node + EntityID const& id, NodeType const& home_node, + MsgSharedPtr const& msg, NodeType from_node ) { auto const from = from_node == uninitialized_destination ? theContext()->getNode() : @@ -688,7 +700,6 @@ void EntityLocationCoord::routeMsg( msg->setEntity(id); msg->setHomeNode(home_node); msg->setLocFromNode(from); - msg->setSerialize(serialize_msg); auto const msg_size = sizeof(*msg); bool const use_eager = useEagerProtocol(msg); @@ -697,9 +708,9 @@ void EntityLocationCoord::routeMsg( vt_debug_print( verbose, location, "routeMsg: inst={}, home={}, msg_size={}, is_large_msg={}, eager={}, " - "serialize_msg={}, in_from={}, from={}, msg{}, msg from={}, epoch={:x}\n", + "in_from={}, from={}, msg{}, msg from={}, epoch={:x}\n", this_inst, home_node, msg_size, msg_size > small_msg_max_size, use_eager, - serialize_msg, from_node, from, print_ptr(msg.get()), msg->getLocFromNode(), + from_node, from, print_ptr(msg.get()), msg->getLocFromNode(), epoch ); @@ -707,14 +718,14 @@ void EntityLocationCoord::routeMsg( if (use_eager) { theMsg()->pushEpoch(epoch); - routeMsgEager(serialize_msg, id, home_node, msg); + routeMsgEager(id, home_node, msg); theMsg()->popEpoch(epoch); } else { theTerm()->produce(epoch); // non-eager protocol: get location first then send message after resolution getLocation(id, home_node, [=](NodeType node) { theMsg()->pushEpoch(epoch); - routeMsgNode(serialize_msg, id, home_node, node, msg); + routeMsgNode(id, home_node, node, msg); theMsg()->popEpoch(epoch); theTerm()->consume(epoch); }); @@ -787,7 +798,6 @@ template auto const entity_id = msg->getEntity(); auto const home_node = msg->getHomeNode(); auto const inst = msg->getLocInst(); - auto const is_serialized = msg->getSerialize(); auto const from_node = msg->getLocFromNode(); auto const epoch = theMsg()->getEpochContextMsg(msg); @@ -795,9 +805,9 @@ template vt_debug_print( verbose, location, - "routedHandler: msg={}, ref={}, loc_inst={}, is_serialized={}, id={}, from={}, " + "routedHandler: msg={}, ref={}, loc_inst={}, id={}, from={}, " "epoch={:x}, hops={}, ask={}\n", - print_ptr(msg.get()), envelopeGetRef(msg->env), inst, is_serialized, entity_id, + print_ptr(msg.get()), envelopeGetRef(msg->env), inst, entity_id, from_node, epoch, msg->getHops(), msg->getAskNode() ); @@ -805,7 +815,7 @@ template LocationManager::applyInstance>( inst, [=](EntityLocationCoord* loc) { theMsg()->pushEpoch(epoch); - loc->routeMsg(entity_id, home_node, msg, is_serialized, from_node); + loc->routeMsg(entity_id, home_node, msg, from_node); theMsg()->popEpoch(epoch); theTerm()->consume(epoch); } diff --git a/src/vt/topos/location/message/msg.h b/src/vt/topos/location/message/msg.h index 8983ad3e07..d6e53825c8 100644 --- a/src/vt/topos/location/message/msg.h +++ b/src/vt/topos/location/message/msg.h @@ -104,8 +104,6 @@ struct EntityMsg : ActiveMessageT { bool hasHandler() const { return handler_ != uninitialized_handler; } void setHandler(HandlerType const han) { handler_ = han; } HandlerType getHandler() const { return handler_; } - void setSerialize(bool const is_serialize) { serialize_ = is_serialize; } - bool getSerialize() const { return serialize_; } void incHops() { hops_ += 1; } int16_t getHops() const { return hops_; } void setAskNode(NodeType const& node) { ask_node_ = node; } @@ -119,7 +117,6 @@ struct EntityMsg : ActiveMessageT { s | loc_from_node_; s | loc_man_inst_; s | handler_; - s | serialize_; s | hops_; s | ask_node_; } @@ -130,7 +127,6 @@ struct EntityMsg : ActiveMessageT { NodeType loc_from_node_ = uninitialized_destination; LocInstType loc_man_inst_ = no_loc_inst; HandlerType handler_ = uninitialized_handler; - bool serialize_ = false; int16_t hops_ = 0; NodeType ask_node_ = uninitialized_destination; }; diff --git a/src/vt/vrt/collection/manager.impl.h b/src/vt/vrt/collection/manager.impl.h index c43e92354c..342a848b32 100644 --- a/src/vt/vrt/collection/manager.impl.h +++ b/src/vt/vrt/collection/manager.impl.h @@ -1207,7 +1207,7 @@ messaging::PendingSend CollectionManager::sendMsgUntypedHandler( auto lm = theLocMan()->getCollectionLM(col_proxy); vtAssert(lm != nullptr, "LM must exist"); theMsg()->markAsCollectionMessage(msg); - lm->template routeMsgSerializeHandler< + lm->template routeMsgHandler< MsgT, collectionMsgTypedHandler >(idx, home_node, msg); theMsg()->popEpoch(cur_epoch); diff --git a/src/vt/vrt/context/context_vrtmanager.impl.h b/src/vt/vrt/context/context_vrtmanager.impl.h index 6a6bc8a437..381be9ad96 100644 --- a/src/vt/vrt/context/context_vrtmanager.impl.h +++ b/src/vt/vrt/context/context_vrtmanager.impl.h @@ -169,7 +169,7 @@ messaging::PendingSend VirtualContextManager::sendSerialMsg( innermsg->setProxy(toProxy); theLocMan()->vrtContextLoc->routeMsgHandler< SerialMsgT, SerializedMessenger::payloadMsgHandler - >(toProxy, home_node, innermsg.get()); + >(toProxy, home_node, innermsg); return messaging::PendingSend(nullptr); }, // custom data transfer lambda if above the eager threshold diff --git a/tests/unit/location/test_location_common.h b/tests/unit/location/test_location_common.h index d4acc5cdc1..8a7438f6ce 100644 --- a/tests/unit/location/test_location_common.h +++ b/tests/unit/location/test_location_common.h @@ -97,9 +97,7 @@ struct LongMsg : vt::LocationRoutedMsg { struct SerialMsg : ShortMsg { SerialMsg(int in_entity, vt::NodeType in_from) : ShortMsg(in_entity, in_from) - { - setSerialize(true); - } + { } }; template @@ -148,15 +146,13 @@ void verifyCacheConsistency( for (int iter = 0; iter < nb_rounds; ++iter) { // create an entity message to route auto msg = vt::makeMessage(entity, my_node); - // check if should be serialized or not - bool serialize = msg->getSerialize(); // perform the checks only after all entity messages have been // correctly delivered runInEpochCollective([&]{ if (my_node not_eq home) { // route entity message - vt::theLocMan()->virtual_loc->routeMsg(entity, home, msg, serialize); + vt::theLocMan()->virtual_loc->routeMsg(entity, home, msg); } }); @@ -170,8 +166,8 @@ void verifyCacheConsistency( vt_debug_print( normal, location, "verifyCacheConsistency: iter={}, entityID={}, home={}, bytes={}, " - "in cache={}, serialize={}\n", - iter, entity, msg->from_, sizeof(*msg), is_entity_cached, serialize + "in cache={}\n", + iter, entity, msg->from_, sizeof(*msg), is_entity_cached ); if (not is_eager) {