From 72fc80ccf1e4a3a8dc6c02e9773e7a41d0349c1b Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Wed, 20 Feb 2019 16:57:49 -0800 Subject: [PATCH 01/13] #280: lb: add new args for outputting stats for offline analysis --- src/vt/configs/arguments/args.cc | 3 +++ src/vt/configs/arguments/args.h | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/vt/configs/arguments/args.cc b/src/vt/configs/arguments/args.cc index b7268530cb..b7f16fb3b0 100644 --- a/src/vt/configs/arguments/args.cc +++ b/src/vt/configs/arguments/args.cc @@ -80,6 +80,9 @@ namespace vt { namespace arguments { /*static*/ std::string ArgConfig::vt_lb_file_name = "balance.in"; /*static*/ std::string ArgConfig::vt_lb_name = "NoLB"; /*static*/ int32_t ArgConfig::vt_lb_interval = 1; +/*static*/ bool ArgConfig::vt_lb_stats = false; +/*static*/ std::string ArgConfig::vt_lb_stats_dir = "vt_lb_stats"; +/*static*/ std::string ArgConfig::vt_lb_stats_file_name = "stats.out"; /*static*/ bool ArgConfig::vt_no_detect_hang = false; /*static*/ int64_t ArgConfig::vt_hang_freq = 1024; diff --git a/src/vt/configs/arguments/args.h b/src/vt/configs/arguments/args.h index ef8a7b0983..c040e720d1 100644 --- a/src/vt/configs/arguments/args.h +++ b/src/vt/configs/arguments/args.h @@ -83,6 +83,9 @@ struct ArgConfig { static std::string vt_lb_file_name; static std::string vt_lb_name; static int32_t vt_lb_interval; + static bool vt_lb_stats; + static std::string vt_lb_stats_dir; + static std::string vt_lb_stats_file_name; static bool vt_no_detect_hang; static int64_t vt_hang_freq; From a67292e9369c095421985c52b5035f38e701c960 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Wed, 20 Feb 2019 17:04:36 -0800 Subject: [PATCH 02/13] #280: runtime: add startup info about LB stats based on args --- src/vt/runtime/runtime.cc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/vt/runtime/runtime.cc b/src/vt/runtime/runtime.cc index 90cb3d3634..9535d84389 100644 --- a/src/vt/runtime/runtime.cc +++ b/src/vt/runtime/runtime.cc @@ -368,6 +368,10 @@ void Runtime::printStartupBanner() { auto f9 = warn_cr("--vt_lb", "lblite"); fmt::print("{}\t{}{}", vt_pre, f9, reset); } + if (ArgType::vt_lb_stats) { + auto f9 = warn_cr("--vt_lb_stats", "lblite"); + fmt::print("{}\t{}{}", vt_pre, f9, reset); + } #endif if (ArgType::vt_lb) { @@ -390,6 +394,26 @@ void Runtime::printStartupBanner() { } } + if (ArgType::vt_lb_stats) { + auto f9 = opt_on("--vt_lb_stats", "Load balancing statistics collection"); + fmt::print("{}\t{}{}", vt_pre, f9, reset); + + auto const fname = ArgType::vt_lb_stats_file_name; + if (fname != "") { + auto f11 = fmt::format("LB stats file name \"{}.0.out\"", fname); + auto f12 = opt_on("--vt_lb_stats_file_name", f11); + fmt::print("{}\t{}{}", vt_pre, f12, reset); + } + + auto const fdir = ArgType::vt_lb_stats_dir; + if (fdir != "") { + auto f11 = fmt::format("LB stats directory \"{}\"", fdir); + auto f12 = opt_on("--vt_lb_stats_dir", f11); + fmt::print("{}\t{}{}", vt_pre, f12, reset); + } + } + + #if !backend_check_enabled(trace_enabled) if (ArgType::vt_trace) { auto f9 = warn_cr("--vt_trace", "trace_enabled"); From 7274229e86252a7e66ac47f38ef113475edd0c60 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Wed, 20 Feb 2019 20:16:14 -0800 Subject: [PATCH 03/13] #280: vrt coll: add ready for by-element tracking --- src/vt/vrt/collection/holders/holder.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/vt/vrt/collection/holders/holder.h b/src/vt/vrt/collection/holders/holder.h index c1facfcdd4..c40e032377 100644 --- a/src/vt/vrt/collection/holders/holder.h +++ b/src/vt/vrt/collection/holders/holder.h @@ -91,6 +91,9 @@ struct Holder { void setGroupReady(bool const ready) { group_ready_ = ready; } NodeType groupRoot() const { return group_root_; } void setGroupRoot(NodeType const root) { group_root_ = root; } + CountType numReady() const { return elements_ready_; } + void addReady(CountType num = 1) { elements_ready_ += 1; } + void clearReady() { elements_ready_ = 0; } friend struct CollectionManager; @@ -106,6 +109,7 @@ struct Holder { bool group_ready_ = false; NodeType group_root_ = 0; CountType num_erased_not_removed_ = 0; + CountType elements_ready_ = 0; }; }}} /* end namespace vt::vrt::collection */ From 8f671599863e78585ec1be3cb13136a70fe735f0 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Wed, 20 Feb 2019 20:16:40 -0800 Subject: [PATCH 04/13] #280: vrt coll: add overload for findElmHolder --- src/vt/vrt/collection/manager.h | 8 ++++++-- src/vt/vrt/collection/manager.impl.h | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/vt/vrt/collection/manager.h b/src/vt/vrt/collection/manager.h index 85614393a9..b8210f6de3 100644 --- a/src/vt/vrt/collection/manager.h +++ b/src/vt/vrt/collection/manager.h @@ -643,11 +643,15 @@ struct CollectionManager { void checkReduceNoElements(); private: - template + template CollectionHolder* findColHolder(VirtualProxyType const& proxy); - template + + template Holder* findElmHolder(VirtualProxyType const& proxy); + template + Holder* findElmHolder(CollectionProxyWrapType proxy); + public: template void destroy(CollectionProxyWrapType const& proxy); diff --git a/src/vt/vrt/collection/manager.impl.h b/src/vt/vrt/collection/manager.impl.h index d8444cf8a7..a9a79a503b 100644 --- a/src/vt/vrt/collection/manager.impl.h +++ b/src/vt/vrt/collection/manager.impl.h @@ -2679,6 +2679,13 @@ Holder* CollectionManager::findElmHolder( } } +template +Holder* CollectionManager::findElmHolder( + CollectionProxyWrapType proxy +) { + return findElmHolder(proxy.getProxy()); +} + template std::size_t CollectionManager::numCollections() { return UniversalIndexHolder<>::getNumCollections(); From d647350ee236a24567759e6155699a568de0acd5 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Wed, 20 Feb 2019 20:16:56 -0800 Subject: [PATCH 05/13] #280: vrt coll: enhance comment --- src/vt/vrt/collection/manager.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/vt/vrt/collection/manager.h b/src/vt/vrt/collection/manager.h index b8210f6de3..f985bfdd78 100644 --- a/src/vt/vrt/collection/manager.h +++ b/src/vt/vrt/collection/manager.h @@ -604,7 +604,15 @@ struct CollectionManager { ); /* - * LB-related operations on the collection + * ====================================================================== + * LB-related operations on the collection + * ====================================================================== + */ + + /* + * The `nextPhase` function is called by a single node on the whole collection + * to indicate that LB is ready. This includes all collections and thus may + * require extra sync to invoke safely */ template void nextPhase( From 32fbff342c13cbee42ae3b265076308dc8e9ed1a Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Thu, 21 Feb 2019 14:09:35 -0800 Subject: [PATCH 06/13] #280: vrt coll: add LB statistics output --- src/vt/configs/arguments/args.cc | 33 ++++++++++----- src/vt/configs/arguments/args.h | 2 +- src/vt/runtime/runtime.cc | 4 +- src/vt/vrt/collection/balance/proc_stats.cc | 47 +++++++++++++++++++++ src/vt/vrt/collection/balance/proc_stats.h | 7 +++ src/vt/vrt/collection/manager.h | 11 ++++- 6 files changed, 89 insertions(+), 15 deletions(-) diff --git a/src/vt/configs/arguments/args.cc b/src/vt/configs/arguments/args.cc index b7f16fb3b0..180505e1ff 100644 --- a/src/vt/configs/arguments/args.cc +++ b/src/vt/configs/arguments/args.cc @@ -82,7 +82,7 @@ namespace vt { namespace arguments { /*static*/ int32_t ArgConfig::vt_lb_interval = 1; /*static*/ bool ArgConfig::vt_lb_stats = false; /*static*/ std::string ArgConfig::vt_lb_stats_dir = "vt_lb_stats"; -/*static*/ std::string ArgConfig::vt_lb_stats_file_name = "stats.out"; +/*static*/ std::string ArgConfig::vt_lb_stats_file = "stats"; /*static*/ bool ArgConfig::vt_no_detect_hang = false; /*static*/ int64_t ArgConfig::vt_hang_freq = 1024; @@ -327,25 +327,36 @@ namespace vt { namespace arguments { * Flags for enabling load balancing and configuring it */ - auto lb = "Enable load balancing"; - auto lb_file = "Enable reading LB configuration from file"; - auto lb_file_name = "LB configuration file to read"; - auto lb_name = "Name of the load balancer to use"; - auto lb_interval = "Load balancing interval"; + auto lb = "Enable load balancing"; + auto lb_file = "Enable reading LB configuration from file"; + auto lb_file_name = "LB configuration file to read"; + auto lb_name = "Name of the load balancer to use"; + auto lb_interval = "Load balancing interval"; + auto lb_stats = "Enable load balancing statistics"; + auto lb_stats_dir = "Load balancing statistics output directory"; + auto lb_stats_file = "Load balancing statistics output file name"; auto lbn = "NoLB"; auto lbi = 1; auto lbf = "balance.in"; - auto s = app.add_flag("--vt_lb", vt_lb, lb); - auto t = app.add_flag("--vt_lb_file", vt_lb_file, lb_file); - auto u = app.add_option("--vt_lb_file_name", vt_lb_file_name, lb_file_name, lbf); - auto v = app.add_option("--vt_lb_name", vt_lb_name, lb_name, lbn); - auto w = app.add_option("--vt_lb_interval", vt_lb_interval, lb_interval, lbi); + auto lbd = "vt_lb_stats"; + auto lbs = "stats"; + auto s = app.add_flag("--vt_lb", vt_lb, lb); + auto t = app.add_flag("--vt_lb_file", vt_lb_file, lb_file); + auto u = app.add_option("--vt_lb_file_name", vt_lb_file_name, lb_file_name, lbf); + auto v = app.add_option("--vt_lb_name", vt_lb_name, lb_name, lbn); + auto w = app.add_option("--vt_lb_interval", vt_lb_interval, lb_interval, lbi); + auto ww = app.add_flag("--vt_lb_stats", vt_lb_stats, lb_stats); + auto wx = app.add_option("--vt_lb_stats_dir", vt_lb_stats_dir, lb_stats_dir, lbd); + auto wy = app.add_option("--vt_lb_stats_file", vt_lb_stats_file, lb_stats_file,lbs); auto debugLB = "Load Balancing"; s->group(debugLB); t->group(debugLB); u->group(debugLB); v->group(debugLB); w->group(debugLB); + ww->group(debugLB); + wx->group(debugLB); + wy->group(debugLB); /* * Flags for controlling termination diff --git a/src/vt/configs/arguments/args.h b/src/vt/configs/arguments/args.h index c040e720d1..9bf6e6f4c8 100644 --- a/src/vt/configs/arguments/args.h +++ b/src/vt/configs/arguments/args.h @@ -85,7 +85,7 @@ struct ArgConfig { static int32_t vt_lb_interval; static bool vt_lb_stats; static std::string vt_lb_stats_dir; - static std::string vt_lb_stats_file_name; + static std::string vt_lb_stats_file; static bool vt_no_detect_hang; static int64_t vt_hang_freq; diff --git a/src/vt/runtime/runtime.cc b/src/vt/runtime/runtime.cc index 9535d84389..49fc932d73 100644 --- a/src/vt/runtime/runtime.cc +++ b/src/vt/runtime/runtime.cc @@ -398,10 +398,10 @@ void Runtime::printStartupBanner() { auto f9 = opt_on("--vt_lb_stats", "Load balancing statistics collection"); fmt::print("{}\t{}{}", vt_pre, f9, reset); - auto const fname = ArgType::vt_lb_stats_file_name; + auto const fname = ArgType::vt_lb_stats_file; if (fname != "") { auto f11 = fmt::format("LB stats file name \"{}.0.out\"", fname); - auto f12 = opt_on("--vt_lb_stats_file_name", f11); + auto f12 = opt_on("--vt_lb_stats_file", f11); fmt::print("{}\t{}{}", vt_pre, f12, reset); } diff --git a/src/vt/vrt/collection/balance/proc_stats.cc b/src/vt/vrt/collection/balance/proc_stats.cc index b2a6bdb366..05c60a1e43 100644 --- a/src/vt/vrt/collection/balance/proc_stats.cc +++ b/src/vt/vrt/collection/balance/proc_stats.cc @@ -46,10 +46,14 @@ #include "vt/vrt/collection/balance/proc_stats.h" #include "vt/vrt/collection/manager.h" #include "vt/timing/timing.h" +#include "vt/configs/arguments/args.h" #include #include +#include +#include +#include namespace vt { namespace vrt { namespace collection { namespace balance { @@ -64,6 +68,8 @@ std::unordered_map /*static*/ ProcStats::ElementIDType ProcStats::next_elm_ = 1; +/*static*/ FILE* ProcStats::stats_file_ = nullptr; + /*static*/ void ProcStats::clearStats() { ProcStats::proc_data_.clear(); ProcStats::proc_migrate_.clear(); @@ -83,4 +89,45 @@ std::unordered_map CollectionManager::releaseLBPhase(msg.get()); } +/*static*/ void ProcStats::createStatsFile() { + using ArgType = vt::arguments::ArgConfig; + auto const node = theContext()->getNode(); + auto const base_file = std::string(ArgType::vt_lb_stats_file); + auto const dir = std::string(ArgType::vt_lb_stats_dir); + auto const file = fmt::format("{}.{}.out", base_file, node); + auto const file_name = fmt::format("{}/{}", dir, file); + + debug_print( + vrt_coll, node, + "ProcStats: createStatsFile file={}\n", file_name + ); + + stats_file_ = fopen(file_name.c_str(), "w+"); +} + +/*static*/ void ProcStats::closeStatsFile() { + if (stats_file_) { + fclose(stats_file_); + } +} + +/*static*/ void ProcStats::outputStatsFile() { + if (stats_file_ == nullptr) { + createStatsFile(); + } + + vtAssertExpr(stats_file_ != nullptr); + + auto const num_iters = ProcStats::proc_data_.size(); + for (auto i = 0; i < num_iters; i++) { + for (auto&& elm : ProcStats::proc_data_.at(i)) { + auto obj_str = fmt::format("{},{},{}\n", i, elm.first, elm.second); + fprintf(stats_file_, "%s", obj_str.c_str()); + } + } + fflush(stats_file_); + + closeStatsFile(); +} + }}}} /* end namespace vt::vrt::collection::balance */ diff --git a/src/vt/vrt/collection/balance/proc_stats.h b/src/vt/vrt/collection/balance/proc_stats.h index 73b25c8872..41c603d1c3 100644 --- a/src/vt/vrt/collection/balance/proc_stats.h +++ b/src/vt/vrt/collection/balance/proc_stats.h @@ -54,6 +54,7 @@ #include #include #include +#include #include namespace vt { namespace vrt { namespace collection { namespace balance { @@ -72,6 +73,10 @@ struct ProcStats { static void clearStats(); static void releaseLB(); + static void createStatsFile(); + static void outputStatsFile(); + static void closeStatsFile(); + private: static ElementIDType getNextElm(); @@ -81,6 +86,8 @@ struct ProcStats { public: static std::vector> proc_data_; static std::unordered_map proc_migrate_; +private: + static FILE* stats_file_; }; }}}} /* end namespace vt::vrt::collection::balance */ diff --git a/src/vt/vrt/collection/manager.h b/src/vt/vrt/collection/manager.h index f985bfdd78..4ac528c582 100644 --- a/src/vt/vrt/collection/manager.h +++ b/src/vt/vrt/collection/manager.h @@ -71,6 +71,8 @@ #include "vt/collective/collective_alg.h" #include "vt/collective/reduce/reduce_msg.h" #include "vt/collective/reduce/reduce_hash.h" +#include "vt/configs/arguments/args.h" +#include "vt/vrt/collection/balance/proc_stats.h" #include #include @@ -113,6 +115,7 @@ struct CollectionManager { using CleanupListFnType = std::list; using DispatchHandlerType = auto_registry::AutoHandlerType; using ActionVecType = std::vector; + using ArgType = vt::arguments::ArgConfig; template using DistribConstructFn = std::function(IndexT idx)>; @@ -133,7 +136,13 @@ struct CollectionManager { CollectionManager() = default; - virtual ~CollectionManager() { cleanupAll<>(); } + virtual ~CollectionManager() { + cleanupAll<>(); + if (ArgType::vt_lb_stats) { + balance::ProcStats::outputStatsFile(); + balance::ProcStats::clearStats(); + } + } template void cleanupAll(); From 3b98b159ae40df77b33218ef0b0e24873a34e060 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Thu, 21 Feb 2019 15:23:52 -0800 Subject: [PATCH 07/13] #280: runtime: add "system" sync method for advanced uses --- src/vt/runtime/runtime.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/vt/runtime/runtime.h b/src/vt/runtime/runtime.h index d8c29d04bc..8db201e7f7 100644 --- a/src/vt/runtime/runtime.h +++ b/src/vt/runtime/runtime.h @@ -96,6 +96,8 @@ struct Runtime { RuntimeInstType getInstanceID() const { return instance_; } + void systemSync() { sync(); } + private: RuntimeInstType const instance_; From 1c8dcaac4b9680167b5df2fd3e81dc77d0aaeefe Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Thu, 21 Feb 2019 15:24:34 -0800 Subject: [PATCH 08/13] #280: lb: create directory for stats --- src/vt/vrt/collection/balance/proc_stats.cc | 19 +++++++++++++++++++ src/vt/vrt/collection/balance/proc_stats.h | 1 + 2 files changed, 20 insertions(+) diff --git a/src/vt/vrt/collection/balance/proc_stats.cc b/src/vt/vrt/collection/balance/proc_stats.cc index 05c60a1e43..063d96fea6 100644 --- a/src/vt/vrt/collection/balance/proc_stats.cc +++ b/src/vt/vrt/collection/balance/proc_stats.cc @@ -47,11 +47,13 @@ #include "vt/vrt/collection/manager.h" #include "vt/timing/timing.h" #include "vt/configs/arguments/args.h" +#include "vt/runtime/runtime.h" #include #include #include #include +#include #include @@ -70,6 +72,8 @@ std::unordered_map /*static*/ FILE* ProcStats::stats_file_ = nullptr; +/*static*/ bool ProcStats::created_dir_ = false; + /*static*/ void ProcStats::clearStats() { ProcStats::proc_data_.clear(); ProcStats::proc_migrate_.clear(); @@ -102,6 +106,21 @@ std::unordered_map "ProcStats: createStatsFile file={}\n", file_name ); + // Node 0 creates the directory + if (not created_dir_ and node == 0) { + mkdir(dir.c_str(), S_IRWXU); + created_dir_ = true; + } + + // Barrier: wait for node 0 to create directory before trying to put a file in + // the stats destination directory + if (curRT) { + curRT->systemSync(); + } else { + // Something is wrong + vtAssert(false, "Trying to dump stats when VT runtime is deallocated?"); + } + stats_file_ = fopen(file_name.c_str(), "w+"); } diff --git a/src/vt/vrt/collection/balance/proc_stats.h b/src/vt/vrt/collection/balance/proc_stats.h index 41c603d1c3..ee3884da81 100644 --- a/src/vt/vrt/collection/balance/proc_stats.h +++ b/src/vt/vrt/collection/balance/proc_stats.h @@ -88,6 +88,7 @@ struct ProcStats { static std::unordered_map proc_migrate_; private: static FILE* stats_file_; + static bool created_dir_; }; }}}} /* end namespace vt::vrt::collection::balance */ From c10329cb8d8dc7aba086a5dccbb4d5a78c11ac51 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Thu, 21 Feb 2019 15:25:04 -0800 Subject: [PATCH 09/13] #280: lb: make LB stat element IDs persist --- .../vrt/collection/balance/proc_stats.impl.h | 24 ++++++++++++------- src/vt/vrt/collection/types/migratable.h | 3 +++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/vt/vrt/collection/balance/proc_stats.impl.h b/src/vt/vrt/collection/balance/proc_stats.impl.h index b93eacf54e..74413585bb 100644 --- a/src/vt/vrt/collection/balance/proc_stats.impl.h +++ b/src/vt/vrt/collection/balance/proc_stats.impl.h @@ -60,7 +60,12 @@ template VirtualElmProxyType const& elm_proxy, ColT* col_elm, PhaseType const& phase, TimeType const& time ) { - auto const& next_elm = ProcStats::getNextElm(); + // Assign a new element ID if it's the first time this runs + if (col_elm->stats_elm_id_ == 0) { + col_elm->stats_elm_id_ = ProcStats::getNextElm(); + } + + auto const next_elm = col_elm->stats_elm_id_; debug_print( vrt_coll, node, @@ -77,14 +82,15 @@ template std::forward_as_tuple(time) ); auto migrate_iter = proc_migrate_.find(next_elm); - vtAssert(migrate_iter == proc_migrate_.end(), "Migrate func must not exist"); - proc_migrate_.emplace( - std::piecewise_construct, - std::forward_as_tuple(next_elm), - std::forward_as_tuple([elm_proxy,col_elm](NodeType node){ - col_elm->migrate(node); - }) - ); + if (migrate_iter == proc_migrate_.end()) { + proc_migrate_.emplace( + std::piecewise_construct, + std::forward_as_tuple(next_elm), + std::forward_as_tuple([elm_proxy,col_elm](NodeType node){ + col_elm->migrate(node); + }) + ); + } return next_elm; } diff --git a/src/vt/vrt/collection/types/migratable.h b/src/vt/vrt/collection/types/migratable.h index ffb4426845..35188540da 100644 --- a/src/vt/vrt/collection/types/migratable.h +++ b/src/vt/vrt/collection/types/migratable.h @@ -51,6 +51,7 @@ #include "vt/vrt/collection/types/migrate_hooks.h" #include "vt/vrt/collection/types/migratable.fwd.h" #include "vt/vrt/collection/balance/elm_stats.h" +#include "vt/vrt/collection/balance/proc_stats.h" namespace vt { namespace vrt { namespace collection { @@ -99,8 +100,10 @@ struct Migratable : MigrateHookBase { protected: friend struct balance::ElementStats; + friend struct balance::ProcStats; balance::ElementStats stats_; balance::ElementStats& getStats() { return stats_; } + balance::ProcStats::ElementIDType stats_elm_id_ = 0; }; }}} /* end namespace vt::vrt::collection */ From eb4bb3c4972e8c44eafabba70d097f97c2e2daf2 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Thu, 21 Feb 2019 15:29:00 -0800 Subject: [PATCH 10/13] #280: lb: conditionalize the stats output on LB feature --- src/vt/vrt/collection/manager.h | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/vt/vrt/collection/manager.h b/src/vt/vrt/collection/manager.h index 4ac528c582..a80a02c9bc 100644 --- a/src/vt/vrt/collection/manager.h +++ b/src/vt/vrt/collection/manager.h @@ -138,10 +138,16 @@ struct CollectionManager { virtual ~CollectionManager() { cleanupAll<>(); - if (ArgType::vt_lb_stats) { - balance::ProcStats::outputStatsFile(); - balance::ProcStats::clearStats(); - } + + // Statistics output when LB is enabled and appropriate flag is enabled + backend_enable_if( + lblite, { + if (ArgType::vt_lb_stats) { + balance::ProcStats::outputStatsFile(); + balance::ProcStats::clearStats(); + } + } + ); } template From 64ce7873127a4819c9f8259615c520182ed29138 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Tue, 26 Feb 2019 17:10:22 -0800 Subject: [PATCH 11/13] #280: lb: add missing serialization for element stats --- src/vt/vrt/collection/types/migratable.impl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vt/vrt/collection/types/migratable.impl.h b/src/vt/vrt/collection/types/migratable.impl.h index d79f2dd39b..35e5fa98af 100644 --- a/src/vt/vrt/collection/types/migratable.impl.h +++ b/src/vt/vrt/collection/types/migratable.impl.h @@ -55,6 +55,7 @@ template void Migratable::serialize(Serializer& s) { MigrateHookBase::serialize(s); s | stats_; + s | stats_elm_id_; } template From 31ad1d6dfb0b3c12f37d80d32d8c2d47cc0ec47d Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Tue, 26 Feb 2019 17:13:31 -0800 Subject: [PATCH 12/13] #280: lb: make {create,close}StatsFile private --- src/vt/vrt/collection/balance/proc_stats.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/vt/vrt/collection/balance/proc_stats.h b/src/vt/vrt/collection/balance/proc_stats.h index ee3884da81..f534070f20 100644 --- a/src/vt/vrt/collection/balance/proc_stats.h +++ b/src/vt/vrt/collection/balance/proc_stats.h @@ -73,8 +73,10 @@ struct ProcStats { static void clearStats(); static void releaseLB(); - static void createStatsFile(); static void outputStatsFile(); + +private: + static void createStatsFile(); static void closeStatsFile(); private: From ea1e42af1e3b27b2a3f5e0cbb0bd0a945a84e1b0 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Tue, 26 Feb 2019 17:15:15 -0800 Subject: [PATCH 13/13] #280: lb: set stats_file_ to nullptr after close --- src/vt/vrt/collection/balance/proc_stats.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vt/vrt/collection/balance/proc_stats.cc b/src/vt/vrt/collection/balance/proc_stats.cc index 063d96fea6..9d4d77a454 100644 --- a/src/vt/vrt/collection/balance/proc_stats.cc +++ b/src/vt/vrt/collection/balance/proc_stats.cc @@ -127,6 +127,7 @@ std::unordered_map /*static*/ void ProcStats::closeStatsFile() { if (stats_file_) { fclose(stats_file_); + stats_file_ = nullptr; } }