From 259e1bf59167f31e7e0573966922c56338db2c58 Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Tue, 13 Sep 2022 23:16:28 +0000 Subject: [PATCH 01/14] #1941: add EpochStack in termination.h --- src/vt/termination/termination.h | 40 +++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/vt/termination/termination.h b/src/vt/termination/termination.h index 42128bfdf1..4baf0ca292 100644 --- a/src/vt/termination/termination.h +++ b/src/vt/termination/termination.h @@ -71,6 +71,18 @@ namespace vt { namespace term { using DijkstraScholtenTerm = term::ds::StateDS; +struct EpochStack { + using DataType = epoch::detail::EpochImplType; + + void push(DataType in) { stack_[cur_++] = in; } + DataType top() const { return stack_[cur_-1]; } + void pop() { cur_--; } + int size() const { return cur_; } + + int cur_ = 0; + std::array stack_; +}; + /** * \struct TerminationDetector * @@ -102,6 +114,7 @@ struct TerminationDetector : using SuccessorBagType = EpochDependency::SuccessorBagType; using EpochGraph = termination::graph::EpochGraph; using EpochGraphMsg = termination::graph::EpochGraphMsg; + using EpochStackType = EpochStack; /** * \internal \brief Construct a termination detector @@ -407,11 +420,9 @@ struct TerminationDetector : * \param[in] state the epoch state * \param[in] num_units number of units * \param[in] produce whether its a produce or consume - * \param[in] node the node producing to or consuming from */ - void produceConsumeState( - TermStateType& state, TermCounterType const num_units, bool produce, - NodeType node + inline void produceConsumeState( + TermStateType& state, TermCounterType const num_units, bool produce ); /** @@ -422,7 +433,7 @@ struct TerminationDetector : * \param[in] produce whether its a produce or consume * \param[in] node the node producing to or consuming from */ - void produceConsume( + inline void produceConsume( EpochType epoch = any_epoch_sentinel, TermCounterType num_units = 1, bool produce = true, NodeType node = uninitialized_destination ); @@ -772,11 +783,26 @@ struct TerminationDetector : */ static void epochContinueHandler(TermMsg* msg); -private: +public: + inline EpochType getEpoch() const; + inline EpochType getGlobalEpoch() const; + inline void pushEpoch(EpochType const& epoch); + inline EpochType popEpoch(EpochType const& epoch = no_epoch); + + inline void pushEpochFast(EpochType epoch) { + epoch_stack_.push(epoch.get()); + } + inline void popEpochFast(EpochType epoch) { + epoch_stack_.pop(); + } + + inline EpochStackType& getEpochStack() { return epoch_stack_; } + // global termination state TermStateType any_epoch_state_; // hang detector termination state TermStateType hang_; +private: // epoch termination state EpochContainerType epoch_state_ = {}; // ready epoch list (misnomer: finishedEpoch was invoked) @@ -785,6 +811,8 @@ struct TerminationDetector : std::unordered_set epoch_wait_status_ = {}; // has printed epoch graph during abort bool has_printed_epoch_graph = false; + NodeType this_node_ = uninitialized_destination; + EpochStackType epoch_stack_; }; }} // end namespace vt::term From 684526d3fd94ce89553620cdbfb7c92f1aff246e Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Tue, 13 Sep 2022 23:19:18 +0000 Subject: [PATCH 02/14] #1941: add functionality to termination.impl.h --- src/vt/termination/termination.impl.h | 102 ++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/src/vt/termination/termination.impl.h b/src/vt/termination/termination.impl.h index a69ea91dd8..c07da587cb 100644 --- a/src/vt/termination/termination.impl.h +++ b/src/vt/termination/termination.impl.h @@ -83,6 +83,108 @@ inline bool TerminationDetector::isDS(EpochType epoch) { } } +inline void TerminationDetector::produceConsumeState( + TermStateType& state, TermCounterType const num_units, bool produce +) { + auto& counter = produce ? state.l_prod : state.l_cons; + counter += num_units; + + vt_debug_print( + verbose, term, + "produceConsumeState: epoch={:x}, event_count={}, l_prod={}, l_cons={}, " + "num_units={}, produce={}\n", + state.getEpoch(), state.getRecvChildCount(), state.l_prod, state.l_cons, num_units, + print_bool(produce) + ); + + if (state.readySubmitParent()) { + propagateEpoch(state); + } +} + +inline void TerminationDetector::produceConsume( + EpochType epoch, TermCounterType num_units, bool produce, NodeType node +) { + vt_debug_print( + normal, term, + "produceConsume: epoch={:x}, rooted={}, ds={}, count={}, produce={}, " + "node={}\n", + epoch, isRooted(epoch), isDS(epoch), num_units, produce, node + ); + + produceConsumeState(any_epoch_state_, num_units, produce); + + if (epoch != any_epoch_sentinel) { + if (isDS(epoch)) { + auto ds_term = getDSTerm(epoch); + + // If a node is not passed, use the current node (self-prod/cons) + if (node == uninitialized_destination) { + node = this_node_; + } + + if (produce) { + ds_term->msgSent(node,num_units); + } else { + ds_term->msgProcessed(node,num_units); + } + } else { + auto& state = findOrCreateState(epoch, false); + produceConsumeState(state, num_units, produce); + } + } +} + +inline EpochType TerminationDetector::getGlobalEpoch() const { + vtAssertInfo( + epoch_stack_.size() > 0, "Epoch stack size must be greater than zero", + epoch_stack_.size() + ); + return epoch_stack_.size() ? EpochType{epoch_stack_.top()} : term::any_epoch_sentinel; +} + + inline void TerminationDetector::pushEpoch(EpochType const& epoch) { + /* + * pushEpoch(epoch) pushes any epoch onto the local stack iff epoch != + * no_epoch; the epoch stack includes all locally pushed epochs and the + * current contexts pushed, transitively causally related active message + * handlers. + */ + vtAssertInfo( + epoch != no_epoch, "Do not push no_epoch onto the epoch stack", + epoch, no_epoch, epoch_stack_.size(), + epoch_stack_.size() > 0 ? EpochType{epoch_stack_.top()} : no_epoch + ); + if (epoch != no_epoch) { + epoch_stack_.push(epoch.get()); + } +} + +inline EpochType TerminationDetector::popEpoch(EpochType const& epoch) { + /* + * popEpoch(epoch) shall remove the top entry from epoch_size_, iif the size + * is non-zero and the `epoch' passed, if `epoch != no_epoch', is equal to the + * top of the `epoch_stack_.top()'; else, it shall remove any entry from the + * top of the stack. + */ + auto const& non_zero = epoch_stack_.size() > 0; + vtAssertExprInfo( + non_zero and (epoch_stack_.top() == epoch.get() or epoch == no_epoch), + epoch, non_zero, epoch_stack_.top() + ); + if (epoch == no_epoch) { + return non_zero ? epoch_stack_.pop(),EpochType{epoch_stack_.top()} : no_epoch; + } else { + return non_zero && epoch == EpochType{epoch_stack_.top()} ? + epoch_stack_.pop(),epoch : + no_epoch; + } +} + +inline EpochType TerminationDetector::getEpoch() const { + return getGlobalEpoch(); +} + }} /* end namespace vt::term */ #endif /*INCLUDED_VT_TERMINATION_TERMINATION_IMPL_H*/ From d61174f15a0061bf4e1f82bc2322b83c182563dd Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Tue, 13 Sep 2022 23:23:52 +0000 Subject: [PATCH 03/14] #1941: modify termination.cc --- src/vt/termination/termination.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/vt/termination/termination.cc b/src/vt/termination/termination.cc index 96c0a1cfa9..1695707270 100644 --- a/src/vt/termination/termination.cc +++ b/src/vt/termination/termination.cc @@ -63,8 +63,9 @@ namespace vt { namespace term { TerminationDetector::TerminationDetector() : collective::tree::Tree(collective::tree::tree_cons_tag_t), - any_epoch_state_(any_epoch_sentinel, false, true, getNumChildren()), - hang_(no_epoch, true, false, getNumChildren()) + any_epoch_state_(any_epoch_sentinel, false, true, getNumChildren()), + hang_(no_epoch, true, false, getNumChildren()), + this_node_(theContext()->getNode()) { } /*static*/ void TerminationDetector::makeRootedHandler(TermMsg* msg) { @@ -112,7 +113,7 @@ void TerminationDetector::setLocalTerminated( any_epoch_state_.notifyLocalTerminated(local_terminated); if (local_terminated && !no_propagate) { - theTerm()->maybePropagate(); + maybePropagate(); } } From 53f206d50212111216daf7a22adba37de220f5df Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Wed, 14 Sep 2022 00:29:10 +0000 Subject: [PATCH 04/14] #1941: remove repeated functions --- src/vt/termination/termination.cc | 52 ------------------------------- 1 file changed, 52 deletions(-) diff --git a/src/vt/termination/termination.cc b/src/vt/termination/termination.cc index 1695707270..979e1856aa 100644 --- a/src/vt/termination/termination.cc +++ b/src/vt/termination/termination.cc @@ -142,26 +142,6 @@ TerminationDetector::findOrCreateState(EpochType const& epoch, bool is_ready) { return epoch_iter->second; } -void TerminationDetector::produceConsumeState( - TermStateType& state, TermCounterType const num_units, bool produce, - NodeType node -) { - auto& counter = produce ? state.l_prod : state.l_cons; - counter += num_units; - - vt_debug_print( - verbose, term, - "produceConsumeState: epoch={:x}, event_count={}, l_prod={}, l_cons={}, " - "num_units={}, produce={}, node={}\n", - state.getEpoch(), state.getRecvChildCount(), state.l_prod, state.l_cons, num_units, - print_bool(produce), node - ); - - if (state.readySubmitParent()) { - propagateEpoch(state); - } -} - TerminationDetector::TermStateDSType* TerminationDetector::getDSTerm(EpochType epoch, bool is_root) { vt_debug_print( @@ -189,38 +169,6 @@ TerminationDetector::getDSTerm(EpochType epoch, bool is_root) { } } -void TerminationDetector::produceConsume( - EpochType epoch, TermCounterType num_units, bool produce, NodeType node -) { - vt_debug_print( - normal, term, - "produceConsume: epoch={:x}, rooted={}, ds={}, count={}, produce={}, " - "node={}\n", - epoch, isRooted(epoch), isDS(epoch), num_units, produce, node - ); - - // If a node is not passed, use the current node (self-prod/cons) - if (node == uninitialized_destination) { - node = theContext()->getNode(); - } - - produceConsumeState(any_epoch_state_, num_units, produce, node); - - if (epoch != any_epoch_sentinel) { - if (isDS(epoch)) { - auto ds_term = getDSTerm(epoch); - if (produce) { - ds_term->msgSent(node,num_units); - } else { - ds_term->msgProcessed(node,num_units); - } - } else { - auto& state = findOrCreateState(epoch, false); - produceConsumeState(state, num_units, produce, node); - } - } -} - void TerminationDetector::maybePropagate() { if (any_epoch_state_.readySubmitParent()) { propagateEpoch(any_epoch_state_); From acf44fcf33605b66abc4ae2bbeff1459e957e545 Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Mon, 19 Sep 2022 16:11:57 +0000 Subject: [PATCH 05/14] #1941: remove Epoch Stack from ActiveMessenger --- src/vt/context/runnable_context/td.cc | 48 +++++++-------------------- src/vt/messaging/active.cc | 17 ++-------- src/vt/messaging/active.h | 20 ----------- src/vt/messaging/active.impl.h | 44 ++---------------------- src/vt/termination/termination.h | 2 +- 5 files changed, 19 insertions(+), 112 deletions(-) diff --git a/src/vt/context/runnable_context/td.cc b/src/vt/context/runnable_context/td.cc index 00958539db..1337dc37d4 100644 --- a/src/vt/context/runnable_context/td.cc +++ b/src/vt/context/runnable_context/td.cc @@ -62,29 +62,17 @@ TD::TD(EpochType in_ep) } void TD::begin() { - theMsg()->pushEpoch(ep_); + theTerm()->pushEpoch(ep_); - auto& epoch_stack = theMsg()->getEpochStack(); + auto& epoch_stack = theTerm()->getEpochStack(); - vt_debug_print( - verbose, context, - "TD::begin: top={:x}, size={}\n", - epoch_stack.size() > 0 ? epoch_stack.top(): no_epoch, - epoch_stack.size() - ); base_epoch_stack_size_ = epoch_stack.size(); } void TD::end() { - auto& epoch_stack = theMsg()->getEpochStack(); + auto& epoch_stack = theTerm()->getEpochStack(); - vt_debug_print( - verbose, context, - "TD::end: top={:x}, size={}, base_size={}\n", - epoch_stack.size() > 0 ? epoch_stack.top(): no_epoch, - epoch_stack.size(), base_epoch_stack_size_ - ); vtAssert( base_epoch_stack_size_ <= epoch_stack.size(), @@ -92,47 +80,35 @@ void TD::end() { ); while (epoch_stack.size() > base_epoch_stack_size_) { - theMsg()->popEpoch(); + theTerm()->popEpoch(); } - theMsg()->popEpoch(ep_); + theTerm()->popEpoch(ep_); } void TD::suspend() { - auto& epoch_stack = theMsg()->getEpochStack(); + auto& epoch_stack = theTerm()->getEpochStack(); - vt_debug_print( - verbose, context, - "TD::suspend: top={:x}, size={}, base_size={}\n", - epoch_stack.size() > 0 ? epoch_stack.top(): no_epoch, - epoch_stack.size(), base_epoch_stack_size_ - ); while (epoch_stack.size() > base_epoch_stack_size_) { - suspended_epochs_.push_back(theMsg()->getEpoch()); - theMsg()->popEpoch(); + suspended_epochs_.push_back(theTerm()->getEpoch()); + theTerm()->popEpoch(); } - theMsg()->popEpoch(ep_); + theTerm()->popEpoch(ep_); } void TD::resume() { - theMsg()->pushEpoch(ep_); + theTerm()->pushEpoch(ep_); - auto& epoch_stack = theMsg()->getEpochStack(); + auto& epoch_stack = theTerm()->getEpochStack(); base_epoch_stack_size_ = epoch_stack.size(); - vt_debug_print( - verbose, context, - "TD::resume: top={:x}, size={}, base_size={}\n", - epoch_stack.size() > 0 ? epoch_stack.top(): no_epoch, - epoch_stack.size(), base_epoch_stack_size_ - ); for (auto it = suspended_epochs_.rbegin(); it != suspended_epochs_.rend(); ++it) { - theMsg()->pushEpoch(*it); + theTerm()->pushEpoch(*it); } suspended_epochs_.clear(); diff --git a/src/vt/messaging/active.cc b/src/vt/messaging/active.cc index 25db27fabf..ca9570e58c 100644 --- a/src/vt/messaging/active.cc +++ b/src/vt/messaging/active.cc @@ -157,6 +157,8 @@ void ActiveMessenger::initialize() { } void ActiveMessenger::startup() { + pushEpoch(term::any_epoch_sentinel); + auto const this_node = theContext()->getNode(); bare_handler_dummy_elm_id_for_lb_data_ = elm::ElmIDBits::createBareHandler(this_node); @@ -171,20 +173,7 @@ void ActiveMessenger::startup() { #endif } -/*virtual*/ ActiveMessenger::~ActiveMessenger() { - // Pop all extraneous epochs off the stack greater than 1 - auto stack_size = epoch_stack_.size(); - while (stack_size > 1) { - stack_size = (epoch_stack_.pop(), epoch_stack_.size()); - } - // Pop off the last epoch: term::any_epoch_sentinel - auto const ret_epoch = popEpoch(term::any_epoch_sentinel); - vtAssertInfo( - ret_epoch == term::any_epoch_sentinel, "Last pop must be any epoch", - ret_epoch, term::any_epoch_sentinel, epoch_stack_.size() - ); - vtAssertExpr(epoch_stack_.size() == 0); -} +/*virtual*/ ActiveMessenger::~ActiveMessenger() {}; trace::TraceEventIDType ActiveMessenger::makeTraceCreationSend( HandlerType const handler, ByteType serialized_msg_size, bool is_bcast diff --git a/src/vt/messaging/active.h b/src/vt/messaging/active.h index 3b50f71b47..2b44025004 100644 --- a/src/vt/messaging/active.h +++ b/src/vt/messaging/active.h @@ -325,7 +325,6 @@ struct ActiveMessenger : runtime::component::PollableComponent using ReadyHanTagType = std::tuple; using MaybeReadyType = std::vector; using HandlerManagerType = HandlerManager; - using EpochStackType = std::stack; using PendingSendType = PendingSend; /** @@ -1516,17 +1515,6 @@ struct ActiveMessenger : runtime::component::PollableComponent MsgSizeType const& msg_size, TagType const& send_tag ); - /** - * \internal - * \brief Get the current global epoch - * - * \c Returns the top epoch on the stack iff \c epoch_stack.size() > 0, else it - * returns \c vt::no_epoch - * - * \return the current global epoch - */ - inline EpochType getGlobalEpoch() const; - /** * \internal * \brief Push an epoch on the stack @@ -1563,12 +1551,6 @@ struct ActiveMessenger : runtime::component::PollableComponent */ inline EpochType getEpoch() const; - /** - * \internal - * \brief Access the epoch stack - */ - inline EpochStackType& getEpochStack() { return epoch_stack_; } - /** * \internal * \brief Get the epoch for a message based on the current context so an @@ -1644,7 +1626,6 @@ struct ActiveMessenger : runtime::component::PollableComponent | pending_handler_msgs_ | pending_recvs_ | cur_direct_buffer_tag_ - | epoch_stack_ | in_progress_active_msg_irecv | in_progress_data_irecv | in_progress_ops @@ -1755,7 +1736,6 @@ struct ActiveMessenger : runtime::component::PollableComponent ContWaitType pending_handler_msgs_ = {}; ContainerPendingType pending_recvs_ = {}; TagType cur_direct_buffer_tag_ = starting_direct_buffer_tag; - EpochStackType epoch_stack_; RequestHolder in_progress_active_msg_irecv; RequestHolder in_progress_data_irecv; RequestHolder in_progress_ops; diff --git a/src/vt/messaging/active.impl.h b/src/vt/messaging/active.impl.h index 6161b73fe5..705dc17a4e 100644 --- a/src/vt/messaging/active.impl.h +++ b/src/vt/messaging/active.impl.h @@ -445,54 +445,16 @@ ActiveMessenger::PendingSendType ActiveMessenger::broadcastMsgAuto( ); } -inline EpochType ActiveMessenger::getGlobalEpoch() const { - vtAssertInfo( - epoch_stack_.size() > 0, "Epoch stack size must be greater than zero", - epoch_stack_.size() - ); - return epoch_stack_.size() ? epoch_stack_.top() : term::any_epoch_sentinel; -} - inline void ActiveMessenger::pushEpoch(EpochType const& epoch) { - /* - * pushEpoch(epoch) pushes any epoch onto the local stack iff epoch != - * no_epoch; the epoch stack includes all locally pushed epochs and the - * current contexts pushed, transitively causally related active message - * handlers. - */ - vtAssertInfo( - epoch != no_epoch, "Do not push no_epoch onto the epoch stack", - epoch, no_epoch, epoch_stack_.size(), - epoch_stack_.size() > 0 ? epoch_stack_.top() : no_epoch - ); - if (epoch != no_epoch) { - epoch_stack_.push(epoch); - } + return theTerm()->pushEpoch(epoch); } inline EpochType ActiveMessenger::popEpoch(EpochType const& epoch) { - /* - * popEpoch(epoch) shall remove the top entry from epoch_size_, iif the size - * is non-zero and the `epoch' passed, if `epoch != no_epoch', is equal to the - * top of the `epoch_stack_.top()'; else, it shall remove any entry from the - * top of the stack. - */ - auto const& non_zero = epoch_stack_.size() > 0; - vtAssertExprInfo( - non_zero and (epoch_stack_.top() == epoch or epoch == no_epoch), - epoch, non_zero, epoch_stack_.top() - ); - if (epoch == no_epoch) { - return non_zero ? epoch_stack_.pop(),epoch_stack_.top() : no_epoch; - } else { - return non_zero && epoch == epoch_stack_.top() ? - epoch_stack_.pop(),epoch : - no_epoch; - } + return theTerm()->popEpoch(epoch); } inline EpochType ActiveMessenger::getEpoch() const { - return getGlobalEpoch(); + return theTerm()->getEpoch(); } template diff --git a/src/vt/termination/termination.h b/src/vt/termination/termination.h index 4baf0ca292..4c7774f349 100644 --- a/src/vt/termination/termination.h +++ b/src/vt/termination/termination.h @@ -77,7 +77,7 @@ struct EpochStack { void push(DataType in) { stack_[cur_++] = in; } DataType top() const { return stack_[cur_-1]; } void pop() { cur_--; } - int size() const { return cur_; } + unsigned int size() const { return cur_; } int cur_ = 0; std::array stack_; From 67ae800605cb555fa1522ff24d05b8aed56533c3 Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Mon, 19 Sep 2022 17:44:49 +0000 Subject: [PATCH 06/14] #1941: remove getGlobalEpoch --- src/vt/messaging/active.cc | 2 +- src/vt/termination/termination.h | 1 - src/vt/termination/termination.impl.h | 6 +----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/vt/messaging/active.cc b/src/vt/messaging/active.cc index ca9570e58c..ce3519d15a 100644 --- a/src/vt/messaging/active.cc +++ b/src/vt/messaging/active.cc @@ -173,7 +173,7 @@ void ActiveMessenger::startup() { #endif } -/*virtual*/ ActiveMessenger::~ActiveMessenger() {}; +/*virtual*/ ActiveMessenger::~ActiveMessenger() {} trace::TraceEventIDType ActiveMessenger::makeTraceCreationSend( HandlerType const handler, ByteType serialized_msg_size, bool is_bcast diff --git a/src/vt/termination/termination.h b/src/vt/termination/termination.h index 4c7774f349..7d1765357d 100644 --- a/src/vt/termination/termination.h +++ b/src/vt/termination/termination.h @@ -785,7 +785,6 @@ struct TerminationDetector : public: inline EpochType getEpoch() const; - inline EpochType getGlobalEpoch() const; inline void pushEpoch(EpochType const& epoch); inline EpochType popEpoch(EpochType const& epoch = no_epoch); diff --git a/src/vt/termination/termination.impl.h b/src/vt/termination/termination.impl.h index c07da587cb..2885c44c6d 100644 --- a/src/vt/termination/termination.impl.h +++ b/src/vt/termination/termination.impl.h @@ -135,14 +135,13 @@ inline void TerminationDetector::produceConsume( } } -inline EpochType TerminationDetector::getGlobalEpoch() const { +inline EpochType TerminationDetector::getEpoch() const { vtAssertInfo( epoch_stack_.size() > 0, "Epoch stack size must be greater than zero", epoch_stack_.size() ); return epoch_stack_.size() ? EpochType{epoch_stack_.top()} : term::any_epoch_sentinel; } - inline void TerminationDetector::pushEpoch(EpochType const& epoch) { /* * pushEpoch(epoch) pushes any epoch onto the local stack iff epoch != @@ -181,9 +180,6 @@ inline EpochType TerminationDetector::popEpoch(EpochType const& epoch) { } } -inline EpochType TerminationDetector::getEpoch() const { - return getGlobalEpoch(); -} }} /* end namespace vt::term */ From db0ffb1cc818633701158934e101e0f78a9677a8 Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Mon, 19 Sep 2022 20:25:38 +0000 Subject: [PATCH 07/14] #1941: do not pushEpoch in ActiveMessenger constructor --- src/vt/messaging/active.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/vt/messaging/active.cc b/src/vt/messaging/active.cc index ce3519d15a..ead645d317 100644 --- a/src/vt/messaging/active.cc +++ b/src/vt/messaging/active.cc @@ -75,12 +75,6 @@ ActiveMessenger::ActiveMessenger() # endif this_node_(theContext()->getNode()) { - /* - * Push the default epoch into the stack so it is always at the bottom of the - * stack during execution until the AM's destructor is invoked - */ - pushEpoch(term::any_epoch_sentinel); - // Register counters for AM/DM message sends and number of bytes amSentCounterGauge = diagnostic::CounterGauge{ registerCounter("AM_sent", "active messages sent"), From 80b564fd448c09ca60862568bdab381bff095003 Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Mon, 19 Sep 2022 22:44:30 +0000 Subject: [PATCH 08/14] #1941: use TerminationDetector for any_epoch_sentinel --- src/vt/messaging/active.cc | 2 -- src/vt/termination/termination.cc | 4 +++- src/vt/termination/termination.h | 15 ++++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/vt/messaging/active.cc b/src/vt/messaging/active.cc index ead645d317..b5c7de42dc 100644 --- a/src/vt/messaging/active.cc +++ b/src/vt/messaging/active.cc @@ -151,8 +151,6 @@ void ActiveMessenger::initialize() { } void ActiveMessenger::startup() { - pushEpoch(term::any_epoch_sentinel); - auto const this_node = theContext()->getNode(); bare_handler_dummy_elm_id_for_lb_data_ = elm::ElmIDBits::createBareHandler(this_node); diff --git a/src/vt/termination/termination.cc b/src/vt/termination/termination.cc index 979e1856aa..bcd1dd6912 100644 --- a/src/vt/termination/termination.cc +++ b/src/vt/termination/termination.cc @@ -66,7 +66,9 @@ TerminationDetector::TerminationDetector() any_epoch_state_(any_epoch_sentinel, false, true, getNumChildren()), hang_(no_epoch, true, false, getNumChildren()), this_node_(theContext()->getNode()) -{ } +{ + pushEpoch(term::any_epoch_sentinel); +} /*static*/ void TerminationDetector::makeRootedHandler(TermMsg* msg) { theTerm()->makeRootedHan(msg->new_epoch, false); diff --git a/src/vt/termination/termination.h b/src/vt/termination/termination.h index 7d1765357d..88a70588b9 100644 --- a/src/vt/termination/termination.h +++ b/src/vt/termination/termination.h @@ -121,7 +121,20 @@ struct TerminationDetector : */ TerminationDetector(); - virtual ~TerminationDetector() {} + virtual ~TerminationDetector() { + //Pop all extraneous epochs off the stack greater than 1 + auto stack_size = epoch_stack_.size(); + while (stack_size > 1) { + stack_size = (epoch_stack_.pop(), epoch_stack_.size()); + } + // Pop off the last epoch: term::any_epoch_sentinel + auto const ret_epoch = popEpoch(term::any_epoch_sentinel); + vtAssertInfo( + ret_epoch == term::any_epoch_sentinel, "Last pop must be any epoch", + ret_epoch, term::any_epoch_sentinel, epoch_stack_.size() + ); + vtAssertExpr(epoch_stack_.size() == 0); + } std::string name() override { return "TerminationDetector"; } From 6dcc46586f2ff554051cbcfe9ef733be794059f1 Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Mon, 19 Sep 2022 22:47:37 +0000 Subject: [PATCH 09/14] #1941: adjust getEpochStack in test --- tests/unit/active/test_async_op_threads.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/active/test_async_op_threads.cc b/tests/unit/active/test_async_op_threads.cc index 5fa85a3dde..7eb8770cad 100644 --- a/tests/unit/active/test_async_op_threads.cc +++ b/tests/unit/active/test_async_op_threads.cc @@ -70,7 +70,7 @@ struct MyObjGroup { } // get the epoch stack and store the original size - auto& epoch_stack = theMsg()->getEpochStack(); + auto& epoch_stack = theTerm()->getEpochStack(); std::size_t original_epoch_size = epoch_stack.size(); auto comm = theContext()->getComm(); @@ -95,7 +95,7 @@ struct MyObjGroup { done_ = true; // stack should be the size before running this method since we haven't // resumed the thread yet! - EXPECT_EQ(theMsg()->getEpochStack().size(), original_epoch_size - 2); + EXPECT_EQ(theTerm()->getEpochStack().size(), original_epoch_size - 2); } ); From ddd37f8f977a115a881d3b727524b4fe8f00dc4d Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Tue, 20 Sep 2022 19:44:36 +0000 Subject: [PATCH 10/14] #1941: modify assert --- src/vt/termination/termination.impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vt/termination/termination.impl.h b/src/vt/termination/termination.impl.h index 2885c44c6d..279bd0350b 100644 --- a/src/vt/termination/termination.impl.h +++ b/src/vt/termination/termination.impl.h @@ -169,7 +169,7 @@ inline EpochType TerminationDetector::popEpoch(EpochType const& epoch) { auto const& non_zero = epoch_stack_.size() > 0; vtAssertExprInfo( non_zero and (epoch_stack_.top() == epoch.get() or epoch == no_epoch), - epoch, non_zero, epoch_stack_.top() + epoch, non_zero, non_zero ? epoch_stack_.top() : no_epoch ); if (epoch == no_epoch) { return non_zero ? epoch_stack_.pop(),EpochType{epoch_stack_.top()} : no_epoch; From 9e6187ad47371795d9d09e0087c1f4de1535d679 Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Tue, 20 Sep 2022 23:29:17 +0000 Subject: [PATCH 11/14] #1941: make types commensurable --- src/vt/termination/termination.impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vt/termination/termination.impl.h b/src/vt/termination/termination.impl.h index 279bd0350b..45409699e6 100644 --- a/src/vt/termination/termination.impl.h +++ b/src/vt/termination/termination.impl.h @@ -169,7 +169,7 @@ inline EpochType TerminationDetector::popEpoch(EpochType const& epoch) { auto const& non_zero = epoch_stack_.size() > 0; vtAssertExprInfo( non_zero and (epoch_stack_.top() == epoch.get() or epoch == no_epoch), - epoch, non_zero, non_zero ? epoch_stack_.top() : no_epoch + epoch, non_zero, non_zero ? EpochType{epoch_stack_.top()} : no_epoch ); if (epoch == no_epoch) { return non_zero ? epoch_stack_.pop(),EpochType{epoch_stack_.top()} : no_epoch; From 13333c024b0b5b989da477da43f40b832f7da0ad Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Wed, 21 Sep 2022 19:22:36 +0000 Subject: [PATCH 12/14] #1941: simplify loop --- src/vt/termination/termination.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/vt/termination/termination.h b/src/vt/termination/termination.h index 88a70588b9..3f6c6bb692 100644 --- a/src/vt/termination/termination.h +++ b/src/vt/termination/termination.h @@ -123,9 +123,8 @@ struct TerminationDetector : virtual ~TerminationDetector() { //Pop all extraneous epochs off the stack greater than 1 - auto stack_size = epoch_stack_.size(); - while (stack_size > 1) { - stack_size = (epoch_stack_.pop(), epoch_stack_.size()); + while (epoch_stack_.size() > 1) { + epoch_stack_.pop(); } // Pop off the last epoch: term::any_epoch_sentinel auto const ret_epoch = popEpoch(term::any_epoch_sentinel); From 3bded51e4c2a936219c49ec3eea838103d1aa39d Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Wed, 21 Sep 2022 19:23:17 +0000 Subject: [PATCH 13/14] #1941: modify epoch argument passing --- src/vt/termination/termination.h | 6 +++--- src/vt/termination/termination.impl.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/vt/termination/termination.h b/src/vt/termination/termination.h index 3f6c6bb692..4b9b46a9ba 100644 --- a/src/vt/termination/termination.h +++ b/src/vt/termination/termination.h @@ -797,13 +797,13 @@ struct TerminationDetector : public: inline EpochType getEpoch() const; - inline void pushEpoch(EpochType const& epoch); - inline EpochType popEpoch(EpochType const& epoch = no_epoch); + inline void pushEpoch(EpochType epoch); + inline EpochType popEpoch(EpochType epoch = no_epoch); inline void pushEpochFast(EpochType epoch) { epoch_stack_.push(epoch.get()); } - inline void popEpochFast(EpochType epoch) { + inline void popEpochFast() { epoch_stack_.pop(); } diff --git a/src/vt/termination/termination.impl.h b/src/vt/termination/termination.impl.h index 45409699e6..6b61c87e6f 100644 --- a/src/vt/termination/termination.impl.h +++ b/src/vt/termination/termination.impl.h @@ -142,7 +142,7 @@ inline EpochType TerminationDetector::getEpoch() const { ); return epoch_stack_.size() ? EpochType{epoch_stack_.top()} : term::any_epoch_sentinel; } - inline void TerminationDetector::pushEpoch(EpochType const& epoch) { + inline void TerminationDetector::pushEpoch(EpochType epoch) { /* * pushEpoch(epoch) pushes any epoch onto the local stack iff epoch != * no_epoch; the epoch stack includes all locally pushed epochs and the @@ -159,7 +159,7 @@ inline EpochType TerminationDetector::getEpoch() const { } } -inline EpochType TerminationDetector::popEpoch(EpochType const& epoch) { +inline EpochType TerminationDetector::popEpoch(EpochType epoch) { /* * popEpoch(epoch) shall remove the top entry from epoch_size_, iif the size * is non-zero and the `epoch' passed, if `epoch != no_epoch', is equal to the From 6658d1769a09e1bc454d20dab02bcebd8a720a47 Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Wed, 21 Sep 2022 19:25:41 +0000 Subject: [PATCH 14/14] #1941: fix indentation --- src/vt/termination/termination.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/vt/termination/termination.h b/src/vt/termination/termination.h index 4b9b46a9ba..8e99a02b33 100644 --- a/src/vt/termination/termination.h +++ b/src/vt/termination/termination.h @@ -123,9 +123,9 @@ struct TerminationDetector : virtual ~TerminationDetector() { //Pop all extraneous epochs off the stack greater than 1 - while (epoch_stack_.size() > 1) { - epoch_stack_.pop(); - } + while (epoch_stack_.size() > 1) { + epoch_stack_.pop(); + } // Pop off the last epoch: term::any_epoch_sentinel auto const ret_epoch = popEpoch(term::any_epoch_sentinel); vtAssertInfo(