#2174: add initial integration of vt-tv

CMakeLists.txt

@@ -59,16 +59,17 @@ set(PROJECT_TOOLS_DIR   ${CMAKE_CURRENT_SOURCE_DIR}/tools)
 # Import the linking macros for VT-related targets
 include(cmake/link_vt.cmake)
 
-# Load packages that are required for core VT build
-include(cmake/load_packages.cmake)
-
 # Some options use TPL information (whether Perl was found etc.)
 include(cmake/configure_options.cmake)
 
-include(cmake/check_compiler.cmake)
 include(cmake/nvcc_no_deprecated_gpu_targets.cmake)
 include(cmake/load_bundled_libraries.cmake)
 
+# Load packages that are required for core VT build
+# We need to do this after bundled libraries so we avoid library duplication
+include(cmake/load_packages.cmake)
+include(cmake/check_compiler.cmake)
+
 
 # Primary VT build
 add_subdirectory(src)

ci/build_cpp.sh

@@ -96,6 +96,19 @@ else
     fi
 fi
 
+if test "${VT_TV_ENABLED}" -eq 1
+then
+    if test -d "${source_dir}/lib/vt-tv"
+    then
+        { echo "vt-tv already in lib... not downloading"; } 2>/dev/null
+    else
+        cd "${source_dir}/lib"
+        vt_tv_rev="master"
+        git clone -b "${vt_tv_rev}" --depth 1 https://github.com/DARMA-tasking/vt-tv.git
+        cd -
+    fi
+fi
+
 if test "${VT_ZOLTAN_ENABLED:-0}" -eq 1
 then
     export Zoltan_DIR=${ZOLTAN_DIR:-""}

ci/deps/vtk.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+if test $# -lt 2
+then
+    echo "usage: ./$0 <vtk-version> <installation-dir>"
+    exit 1
+fi
+
+vtk_version="$1"
+vtk_tar_name="v${vtk_version}.tar.gz"
+vtk_name="VTK-${vtk_version}"
+install_dir="${2-}"
+build_dir=/vtk-build/
+
+wget "https://github.com/Kitware/VTK/archive/refs/tags/$vtk_tar_name"
+tar xzf ${vtk_tar_name}
+rm ${vtk_tar_name}
+cd ${vtk_name}
+
+mkdir -p ${build_dir}
+mkdir -p ${install_dir}
+
+cd ${build_dir}
+cmake \
+  -DCMAKE_INSTALL_PREFIX:FILEPATH=${install_dir} \
+  ../${vtk_name}
+cmake --build ${build_dir} --target install

ci/docker/ubuntu-gnu-cpp.dockerfile

@@ -32,6 +32,11 @@ RUN apt-get update -y -q && \
     wget \
     zlib1g \
     zlib1g-dev \
+    libncurses5-dev \
+    m4 \
+    libgl1-mesa-dev \
+    libglu1-mesa-dev \
+    mesa-common-dev \
     brotli \
     python3 \
     python3-brotli \
@@ -64,6 +69,11 @@ RUN ./cmake.sh 3.23.4 ${arch}
 ENV PATH=/cmake/bin/:$PATH
 ENV LESSCHARSET=utf-8
 
+COPY ./ci/deps/vtk.sh vtk.sh
+RUN if [ "$VT_TV_ENABLED" -eq 1]; then \
+      ./vtk.sh 9.3.0 /vtk-install; \
+    fi
+
 COPY ./ci/deps/mpich.sh mpich.sh
 RUN if [ "$ubuntu" = "18.04" ]; then \
       ./mpich.sh 3.3.2 -j4; else \

cmake/configure_options.cmake

@@ -79,6 +79,14 @@ if(vt_trace_only)
     set(vt_mpi_guards ON)
 endif()
 
+if (TV_LIBRARY)
+    message(STATUS "vt-tv found and enabled")
+    set(vt_feature_cmake_tv "1")
+else()
+    message(STATUS "vt-tv not found")
+    set(vt_feature_cmake_tv "0")
+endif()
+
 define_option(vt_mpi_guards "user MPI prevention guards"
     "Build VT with poison MPI calls: code invoked from VT callbacks cannot invoke MPI functions"
     ON vt_feature_cmake_mpi_access_guards

cmake/link_vt.cmake

@@ -33,6 +33,7 @@ function(link_target_with_vt)
     LINK_FORT
     LINK_JSON
     LINK_BROTLI
+    LINK_VT_TV
   )
   set(
     multiValueArg
@@ -70,6 +71,14 @@ function(link_target_with_vt)
     endif()
   endif()
 
+  if (NOT DEFINED ARG_LINK_VT_TV AND ${ARG_DEFAULT_LINK_SET} OR ARG_LINK_VT_TV)
+    if (TV_LIBRARY)
+      target_link_libraries(
+        ${ARG_TARGET} PUBLIC ${ARG_BUILD_TYPE} ${TV_LIBRARY}
+      )
+    endif()
+  endif()
+
   if (NOT DEFINED ARG_LINK_ZOLTAN AND ${ARG_DEFAULT_LINK_SET} OR ARG_LINK_ZOLTAN)
     if (vt_zoltan_enabled)
       if (${ARG_DEBUG_LINK})

cmake/load_local_packages.cmake

@@ -13,3 +13,9 @@ else()
   endif()
 
 set(CHECKPOINT_LIBRARY vt::lib::checkpoint)
+
+if (EXISTS "${PROJECT_LIB_DIR}/vt-tv")
+  set(vt_tv_python_bindings_enabled OFF)
+  add_subdirectory(${PROJECT_LIB_DIR}/vt-tv)
+  set(TV_LIBRARY vt::lib::vt-tv)
+endif()

cmake_config.h.in

@@ -66,6 +66,7 @@
 #define vt_feature_cmake_debug_verbose       @vt_feature_cmake_debug_verbose@
 #define vt_feature_cmake_rdma_tests          @vt_feature_cmake_rdma_tests@
 #define vt_feature_cmake_external_fmt        @vt_feature_cmake_external_fmt@
+#define vt_feature_cmake_tv                  @vt_feature_cmake_tv@
 
 #define vt_detected_max_num_nodes @cmake_detected_max_num_nodes@
 

docker-compose.yml

@@ -120,6 +120,7 @@ x-vtopts: &vtopts
   CODECOV_TOKEN: ${CODECOV_TOKEN:-}
   TEST_LB_SCHEMA: ${TEST_LB_SCHEMA:-0}
   CMAKE_CXX_STANDARD: ${CMAKE_CXX_STANDARD:-17}
+  VT_TV_ENABLED: ${VT_TV:-0}
 
 services:
   ##############################################################################

lib/fmt/CMakeLists.txt

@@ -12,6 +12,8 @@ endfunction()
 add_headers(FMT_HEADERS core.h format.h format-inl.h ostream.h)
 set(FMT_SOURCES src/format.cc)
 
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
 add_library(fmt ${FMT_SOURCES} ${FMT_HEADERS})
 add_library(fmt::fmt ALIAS fmt)
 set_target_properties(fmt PROPERTIES OUTPUT_NAME fmt-vt)

src/vt/configs/arguments/app_config.h

@@ -229,6 +229,9 @@ struct AppConfig {
 
   bool vt_debug_print_flush = false;
 
+  bool vt_tv = false;
+  std::string vt_tv_config_file = "";
+
   bool vt_user_1 = false;
   bool vt_user_2 = false;
   bool vt_user_3 = false;
@@ -391,6 +394,9 @@ struct AppConfig {
 
       | vt_debug_print_flush
 
+      | vt_tv
+      | vt_tv_config_file
+
       | vt_user_1
       | vt_user_2
       | vt_user_3

src/vt/configs/arguments/args.cc

@@ -675,6 +675,20 @@ void addRuntimeArgs(CLI::App& app, AppConfig& appConfig) {
   a3->group(configRuntime);
 }
 
+void addTVArgs(CLI::App& app, AppConfig& appConfig) {
+  auto tv_enabled = "Enable vt-tv visualization/mesh streaming";
+  auto tv_file = "File name for YAML vt-tv configuraton file";
+
+  auto a1 = app.add_flag("--vt_tv", appConfig.vt_tv, tv_enabled);
+  auto a2 = app.add_option(
+          "--vt_tv_config_file", appConfig.vt_tv_config_file, tv_file
+  );
+
+  auto configTV = "vt-tv Configuration";
+  a1->group(configTV);
+  a2->group(configTV);
+}
+
 void addThreadingArgs(
   [[maybe_unused]] CLI::App& app,
   [[maybe_unused]] AppConfig& appConfig
@@ -777,6 +791,7 @@ std::tuple<int, std::string> ArgConfig::parseToConfig(
   addSchedulerArgs(app, appConfig);
   addConfigFileArgs(app, appConfig);
   addRuntimeArgs(app, appConfig);
+  addTVArgs(app, appConfig);
   addThreadingArgs(app, appConfig);
 
   std::tuple<int, std::string> result = parseArguments(app, argc, argv, appConfig);

src/vt/configs/features/features_defines.h

@@ -73,6 +73,7 @@
 #define vt_feature_production_build    0 || vt_feature_cmake_production_build
 #define vt_feature_debug_verbose       0 || vt_feature_cmake_debug_verbose
 #define vt_feature_fmt_external        0 || vt_feature_cmake_external_fmt
+#define vt_feature_tv                  1 || vt_feature_cmake_tv
 
 #define vt_check_enabled(test_option) (vt_feature_ ## test_option != 0)
 

src/vt/vrt/collection/balance/lb_data_holder.cc

@@ -44,6 +44,10 @@
 #include "vt/vrt/collection/balance/lb_data_holder.h"
 #include "vt/context/context.h"
 
+#if vt_check_enabled(tv)
+#  include <vt-tv/api/info.h>
+#endif
+
 #include <nlohmann/json.hpp>
 
 namespace vt { namespace vrt { namespace collection { namespace balance {
@@ -275,6 +279,100 @@ std::unique_ptr<nlohmann::json> LBDataHolder::toJson(PhaseType phase) const {
   return std::make_unique<json>(std::move(j));
 }
 
+#if vt_check_enabled(tv)
+std::unique_ptr<vt::tv::PhaseWork> LBDataHolder::toTV(PhaseType phase) const {
+  using vt::tv::PhaseWork;
+  using vt::tv::ObjectWork;
+  using vt::tv::ObjectCommunicator;
+
+  std::unordered_map<ElementIDType, ObjectWork> objects;
+
+  if (node_data_.find(phase) != node_data_.end()) {
+    for (auto&& elm : node_data_.at(phase)) {
+      ElementIDStruct id = elm.first;
+      TimeType whole_phase_load = elm.second.whole_phase_load;
+      auto const& subphase_loads = elm.second.subphase_loads;
+
+      typename DataMapType::mapped_type user_defined;
+      if (
+        user_defined_lb_info_.find(phase) != user_defined_lb_info_.end() and
+        user_defined_lb_info_.at(phase).find(id) !=
+        user_defined_lb_info_.at(phase).end()
+      ) {
+        user_defined = user_defined_lb_info_.at(phase).at(id);
+      }
+      std::unordered_map<SubphaseType, double> subphase_map;
+      for (std::size_t i = 0; i < subphase_loads.size(); i++) {
+        subphase_map[i] = subphase_loads[i];
+      }
+      objects.try_emplace(
+        id.id,
+        id.id, whole_phase_load, std::move(subphase_map), std::move(user_defined)
+      );
+    }
+  }
+
+  if (node_comm_.find(phase) != node_comm_.end()) {
+    for (auto&& elm : node_comm_.at(phase)) {
+      auto const& key = elm.first;
+      auto const& volume = elm.second;
+      auto const& bytes = volume.bytes;
+      switch(key.cat_) {
+      case elm::CommCategory::SendRecv: {
+        auto from_id = key.fromObj();
+        auto to_id = key.toObj();
+
+        if (objects.find(from_id.id) != objects.end()) {
+          objects.at(from_id.id).addSentCommunications(to_id.id, bytes);
+        } else if (objects.find(to_id.id) != objects.end()) {
+          objects.at(to_id.id).addReceivedCommunications(from_id.id, bytes);
+        }
+        break;
+      }
+      default:
+        // skip all other communications for now
+        break;
+      }
+    }
+  }
+
+  return std::make_unique<PhaseWork>(phase, objects);;
+}
+
+std::unordered_map<ElementIDType, tv::ObjectInfo> LBDataHolder::getObjInfo(
+  PhaseType phase
+) const {
+  std::unordered_map<ElementIDType, tv::ObjectInfo> map;
+  if (node_data_.find(phase) != node_data_.end()) {
+    for (auto&& elm : node_data_.at(phase)) {
+      ElementIDStruct id = elm.first;
+
+      bool is_collection = false;
+      bool is_objgroup = false;
+
+      std::vector<uint64_t> idx;
+      if (node_idx_.find(id) != node_idx_.end()) {
+        is_collection = true;
+        idx = std::get<1>(node_idx_.find(id)->second);
+      }
+
+      if (node_objgroup_.find(id) != node_objgroup_.end()) {
+        is_objgroup = true;
+      }
+
+      tv::ObjectInfo oi{
+        id.id, id.getHomeNode(), id.isMigratable(), std::move(idx)
+      };
+      oi.setIsCollection(is_collection);
+      oi.setIsObjGroup(is_objgroup);
+      map[id.id] = std::move(oi);
+    }
+  }
+  return map;
+}
+
+#endif
+
 LBDataHolder::LBDataHolder(nlohmann::json const& j)
 {
   auto this_node = theContext()->getNode();

src/vt/vrt/collection/balance/lb_data_holder.h

@@ -48,6 +48,10 @@
 #include "vt/vrt/collection/balance/lb_common.h"
 #include "vt/elm/elm_comm.h"
 
+#if vt_check_enabled(tv)
+#  include <vt-tv/api/info.h>
+#endif
+
 #include <unordered_map>
 #include <memory>
 #include <variant>
@@ -97,6 +101,26 @@ struct LBDataHolder {
    */
   std::unique_ptr<nlohmann::json> toJson(PhaseType phase) const;
 
+#if vt_check_enabled(tv)
+          /**
+   * \brief Generate vt-tv data structure for visualization
+   *
+   * \param[in] phase the phase to generate
+   *
+   * \return a \c vt::tv::PhaseWork data structure
+   */
+  std::unique_ptr<tv::PhaseWork> toTV(PhaseType phase) const;
+
+  /**
+   * \brief Get all object info mapped here for a given phase
+   *
+   * \return map with object info
+   */
+  std::unordered_map<ElementIDType, tv::ObjectInfo> getObjInfo(
+    PhaseType phase
+  ) const;
+#endif
+
   /**
    * \brief Output a LB phase's metadata to JSON
    *