#2302: Start elaborating cmake system to turn and off measurement opt…

…ions (wip)
DARMA-tasking · Aug 2, 2024 · 30b1083 · 30b1083
1 parent 5c993b5
commit 30b1083
Show file tree

Hide file tree

Showing 17 changed files with 145 additions and 32 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@ build
 transport
 lib/*-build
 lib/checkpoint
+lib/papi
 lib/googletest
 lib/kokkos
 .emacs.desktop

diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh
@@ -73,17 +73,26 @@ else
     fi
 fi
 
-if test -d "${source_dir}/lib/papi"
+if test "$VT_PAPI_ENABLED" -eq 1 && "$VT_PERF_ENABLED" -eq 1
 then
-    { echo "papi already in lib... not downloading, building, and installing"; } 2>/dev/null
-else
-    cd "${source_dir}/lib"
-    git clone https://github.com/icl-utk-edu/papi.git
-    cd papi/src
-    export PAPI_BUILD=${build_dir}/papi
-    mkdir -p "$PAPI_BUILD"
-    CC="${CC:-cc}" F77="${F77:-gfortran}" ./configure --prefix=${PAPI_BUILD}/install
-    make -j ${dashj} && make install
+    echo "Both PAPI and perf measurements are enabled; this isn't possible, please turn off one of these options. Exiting."
+    exit
+fi
+
+if test "${VT_PAPI_ENABLED:-0}" -eq 1
+then
+    if test -d "${source_dir}/lib/papi"
+    then
+        { echo "papi already in lib... not downloading, building, and installing"; } 2>/dev/null
+    else
+        cd "${source_dir}/lib"
+        git clone --depth 1 https://github.com/icl-utk-edu/papi.git
+        cd papi/src
+        export PAPI_BUILD=${build_dir}/papi
+        mkdir -p "$PAPI_BUILD"
+        CC="${CC:-cc}" F77="${F77:-gfortran}" ./configure --prefix=${PAPI_BUILD}/install
+        make -j ${dashj} && make install
+    fi
 fi
 
 if test "${VT_ZOLTAN_ENABLED:-0}" -eq 1
@@ -113,8 +122,11 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
       -Dvt_ubsan_enabled="${VT_UBSAN_ENABLED:-0}" \
       -Dvt_werror_enabled="${VT_WERROR_ENABLED:-0}" \
       -Dvt_pool_enabled="${VT_POOL_ENABLED:-1}" \
+      -Dvt_build_tests="${VT_BUILD_TESTS:-1}" \
       -Dvt_build_extended_tests="${VT_EXTENDED_TESTS_ENABLED:-1}" \
       -Dvt_zoltan_enabled="${VT_ZOLTAN_ENABLED:-0}" \
+      -Dvt_papi_enabled="${VT_PAPI_ENABLED:-0}" \
+      -Dvt_perf_enabled="${VT_PERF_ENABLED:-0}" \
       -Dvt_production_build_enabled="${VT_PRODUCTION_BUILD_ENABLED:-0}" \
       -Dvt_unity_build_enabled="${VT_UNITY_BUILD_ENABLED:-0}" \
       -Dvt_diagnostics_enabled="${VT_DIAGNOSTICS_ENABLED:-1}" \

diff --git a/cmake/configure_options.cmake b/cmake/configure_options.cmake
@@ -98,6 +98,8 @@ define_option(vt_rdma_tests_enabled "RDMA tests" "Build VT with RDMA tests enabl
     ON vt_feature_cmake_rdma_tests
 )
 
+define_option(vt_papi_enabled "PAPI task measurement" "Build VT with PAPI to measure low level metrics of interest" OFF vt_feature_cmake_papi)
+define_option(vt_papi_enabled "perf task measurement" "Build VT with perf to measure low level metrics of interest" OFF vt_feature_cmake_perf)
 
 #####################################################
 #################### DIAGNOSTICS ####################

diff --git a/cmake/link_vt.cmake b/cmake/link_vt.cmake
@@ -213,17 +213,19 @@ function(link_target_with_vt)
   endif()
 
   if (NOT DEFINED ARG_LINK_PAPI AND ${ARG_DEFAULT_LINK_SET} OR ARG_LINK_PAPI)
-    if (${ARG_DEBUG_LINK})
-      message(STATUS "link_target_with_vt: papi=${ARG_LINK_PAPI}")
+    if (vt_papi_enabled)
+      if (${ARG_DEBUG_LINK})
+        message(STATUS "link_target_with_vt: papi=${ARG_LINK_PAPI}")
+      endif()
+      # message(STATUS "target_link_libraries: ${ARG_TARGET} PUBLIC  ${ARG_BUILD_TYPE} ${PAPI_LIBRARY}")
+      # message(STATUS "target_include_directories: ${ARG_TARGET} PUBLIC  ${ARG_BUILD_TYPE} ${PAPI_INCLUDE_DIR}")
+      target_link_libraries(
+        ${ARG_TARGET} PUBLIC ${ARG_BUILD_TYPE} ${PAPI_LIBRARY}
+      )
+      target_include_directories(
+        ${ARG_TARGET} PUBLIC ${ARG_BUILD_TYPE} ${PAPI_INCLUDE_DIR}
+      )
     endif()
-    message(STATUS "target_link_libraries: ${ARG_TARGET} PUBLIC  ${ARG_BUILD_TYPE} ${PAPI_LIBRARY}")
-    message(STATUS "target_include_directories: ${ARG_TARGET} PUBLIC  ${ARG_BUILD_TYPE} ${PAPI_INCLUDE_DIR}")
-    target_link_libraries(
-      ${ARG_TARGET} PUBLIC ${ARG_BUILD_TYPE} ${PAPI_LIBRARY}
-    )
-    target_include_directories(
-      ${ARG_TARGET} PUBLIC ${ARG_BUILD_TYPE} ${PAPI_INCLUDE_DIR}
-    )
   endif()
 
   if (${vt_mimalloc_enabled})

diff --git a/cmake/load_packages.cmake b/cmake/load_packages.cmake
@@ -18,7 +18,10 @@ include(cmake/load_doxygen.cmake)
 # Optionally link with Zoltan
 include(cmake/load_zoltan_package.cmake)
 
-# Link with PAPI
+# If enabled, link with PAPI
+include(cmake/load_papi.cmake)
+
+# If enabled, test if perf is available and works
 include(cmake/load_papi.cmake)
 
 # Tests

diff --git a/cmake/load_papi.cmake b/cmake/load_papi.cmake
@@ -1,3 +1,12 @@
-find_package(PAPI REQUIRED)
-message(STATUS: "FOUND PAPI: PAPI LIBRARY: ${PAPI_LIBRARY}\n       PAPI INCLUDE DIR: ${PAPI_INCLUDE_DIR}")
-set(vt_papi_found "1")
+set(vt_papi_found "0")
+
+if (vt_papi_enabled)
+    if (vt_perf_enabled)
+        set(vt_papi_found "0")
+        set(vt_papi_enabled "0")
+        message(FATAL_ERROR "Both PAPI and perf measurements are enabled; this will cause errors, please turn off one of these options. Exiting.")
+    endif ()
+    find_package(PAPI REQUIRED)
+    message(STATUS: "FOUND PAPI: PAPI LIBRARY: ${PAPI_LIBRARY}\n       PAPI INCLUDE DIR: ${PAPI_INCLUDE_DIR}")
+    set(vt_papi_found "1")
+endif ()
diff --git a/cmake/load_perf.cmake b/cmake/load_perf.cmake
@@ -0,0 +1,47 @@
+set(vt_perf_found "0")
+
+if (vt_perf_enabled)
+    if (vt_papi_enabled)
+        message(FATAL_ERROR "Both PAPI and perf measurements are enabled; this will cause errors, please turn off one of these options. Exiting.")
+    endif ()
+    # check if linux
+    if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        # check if there's the perf header we need
+        INCLUDE(CheckIncludeFiles)
+        CHECK_INCLUDE_FILES("linux/perf_event.h" HAVE_PERF_EVENT_H)
+        if (HAVE_PERF_EVENT_H)
+            # check if the kernel is recent enough
+            string(REPLACE "." ";" VERSION_LIST ${CMAKE_SYSTEM_VERSION})
+            list(GET VERSION_LIST 0 KERNEL_MAJOR_VERSION)
+            if (KERNEL_MAJOR_VERSION GREATER_EQUAL 4)
+                # check if a simple perf stat runs without issues
+                execute_process(
+                    COMMAND "perf stat which cmake"
+                    RESULT_VARIABLE PERF_STAT_RESULT
+                    OUTPUT_QUIET
+                    ERROR_QUIET
+                )
+                if (PERF_STAT_RESULT EQUAL 0)
+                    set(vt_perf_enabled "1")
+                    set(vt_perf_found "1")
+                else ()
+                    message(WARNING "Perf measurements enabled but couldn't run perf stat successfully. Disabling perf measurements.")
+                    set(vt_perf_enabled "0")
+                    set(vt_perf_found "0")
+                endif ()
+            else ()
+                message(WARNING "Perf measurements enabled but Kernel major version is less than 4. Disabling perf measurements.")
+                set(vt_perf_enabled "0")
+                set(vt_perf_found "0")
+            endif ()
+        else ()
+            message(WARNING "Perf measurements enabled but couldn't find perf_event.h. Disabling perf measurements.")
+            set(vt_perf_enabled "0")
+            set(vt_perf_found "0")
+        endif ()
+    else ()
+        message(WARNING "Perf measurements enabled but system is not Linux. Disabling perf measurements.")
+        set(vt_perf_enabled "0")
+        set(vt_perf_found "0")
+    endif ()
+endif ()
diff --git a/cmake_config.h.in b/cmake_config.h.in
@@ -66,6 +66,8 @@
 #define vt_feature_cmake_debug_verbose       @vt_feature_cmake_debug_verbose@
 #define vt_feature_cmake_rdma_tests          @vt_feature_cmake_rdma_tests@
 #define vt_feature_cmake_external_fmt        @vt_feature_cmake_external_fmt@
+#define vt_feature_cmake_papi                @vt_feature_cmake_papi@
+#define vt_feature_cmake_perf                @vt_feature_cmake_perf@
 
 #define vt_detected_max_num_nodes @cmake_detected_max_num_nodes@
 

diff --git a/examples/collection/CMakeLists.txt b/examples/collection/CMakeLists.txt
@@ -10,8 +10,10 @@ set(
   insertable_collection
   reduce_integral
   transpose
-  do_flops
 )
+if ((vt_papi_enabled AND vt_papi_found) OR (vt_perf_enabled AND vt_perf_found))
+  list(APPEND COLLECTION_EXAMPLES do_flops)
+endif()
 
 foreach(EXAMPLE_NAME ${COLLECTION_EXAMPLES})
   # message("Example: building collection example >>>>> ${EXAMPLE_NAME}")

diff --git a/src/vt/configs/features/features_defines.h b/src/vt/configs/features/features_defines.h
@@ -73,6 +73,8 @@
 #define vt_feature_production_build    0 || vt_feature_cmake_production_build
 #define vt_feature_debug_verbose       0 || vt_feature_cmake_debug_verbose
 #define vt_feature_fmt_external        0 || vt_feature_cmake_external_fmt
+#define vt_feature_papi                0 || vt_feature_cmake_papi
+#define vt_feature_perf                0 || vt_feature_cmake_perf
 
 #define vt_check_enabled(test_option) (vt_feature_ ## test_option != 0)
 

diff --git a/src/vt/configs/features/features_featureswitch.h b/src/vt/configs/features/features_featureswitch.h
@@ -64,5 +64,7 @@
                                           "debug prints disabled)"
 #define vt_feature_str_trace_enabled      "Tracing Projections"
 #define vt_feature_str_zoltan             "Zoltan for load balancing"
+#define vt_feature_str_papi               "PAPI for event measurement"
+#define vt_feature_str_perf               "perf for event measurement"
 
 #endif /*INCLUDED_VT_CONFIGS_FEATURES_FEATURES_FEATURESWITCH_H*/
diff --git a/src/vt/context/context.cc b/src/vt/context/context.cc
@@ -91,6 +91,7 @@ Context::Context([[maybe_unused]] bool const is_interop, MPI_Comm comm) {
   numNodes_ = static_cast<NodeType>(numNodesLocal);
   thisNode_ = static_cast<NodeType>(thisNodeLocal);
 
+#if vt_check_enabled(papi)
   int retval;
 
   /* Initialize the PAPI library */
@@ -102,6 +103,7 @@ Context::Context([[maybe_unused]] bool const is_interop, MPI_Comm comm) {
   retval = PAPI_multiplex_init();
   if (retval != PAPI_OK)
       handle_papi_error(retval);
+#endif
 }
 
 Context::~Context() {

diff --git a/src/vt/context/context.h b/src/vt/context/context.h
@@ -46,7 +46,9 @@
 
 #include <memory>
 #include <mpi.h>
+#if vt_check_enabled(papi)
 #include <papi.h>
+#endif
 
 #include "vt/config.h"
 #include "vt/runtime/component/component_pack.h"
@@ -125,11 +127,13 @@ struct Context : runtime::component::Component<Context> {
 
   std::string name() override { return "Context"; }
 
+#if vt_check_enabled(papi)
   void handle_papi_error (int retval)
   {
     printf("PAPI error %d: %s\n", retval, PAPI_strerror(retval));
     exit(1);
   }
+#endif
 
   template <typename SerializerT>
   void serialize(SerializerT& s) {

diff --git a/src/vt/context/runnable_context/lb_data.cc b/src/vt/context/runnable_context/lb_data.cc
@@ -78,6 +78,7 @@ typename LBData::ElementIDStruct const& LBData::getCurrentElementID() const {
   return cur_elm_id_;
 }
 
+#if vt_check_enabled(papi)
 std::unordered_map<std::string, uint64_t> LBData::getPAPIMetrics() {
   std::unordered_map<std::string, uint64_t> papi_metrics = {};
   for (size_t i = 0; i < papiData_->native_events.size(); i++) {
@@ -89,5 +90,6 @@ std::unordered_map<std::string, uint64_t> LBData::getPAPIMetrics() {
   papi_metrics[std::string("virt_cycles")] = papiData_->end_virt_cycles - papiData_->start_virt_cycles;
   return papi_metrics;
 }
+#endif
 
 }} /* end namespace vt::ctx */
diff --git a/src/vt/context/runnable_context/lb_data.h b/src/vt/context/runnable_context/lb_data.h
@@ -46,9 +46,10 @@
 
 #include "vt/vrt/collection/balance/lb_common.h"
 #include "vt/elm/elm_lb_data.fwd.h"
-#include "vt/context/runnable_context/papi_data.h"
 
-#include <papi.h>
+#if vt_check_enabled(papi)
+#include "vt/context/runnable_context/papi_data.h"
+#endif
 
 namespace vt { namespace ctx {
 
@@ -81,9 +82,12 @@ struct LBData {
   LBData(ElementLBData* in_lb_data, ElementIDStruct const& in_elm_id)
     : lb_data_(in_lb_data),
       cur_elm_id_(in_elm_id),
-      should_instrument_(true),
-      papiData_(std::make_unique<PAPIData>())
-  { }
+      should_instrument_(true)
+  {
+#if vt_check_enabled(papi)
+    papiData_ = std::make_unique<PAPIData>();
+#endif
+  }
 
   /**
    * \brief Return whether time is required
@@ -119,32 +123,39 @@ struct LBData {
    */
   ElementIDStruct const& getCurrentElementID() const;
 
+#if vt_check_enabled(papi)
   /**
    * \brief Start PAPI metrics map for the running context
    */
   void startPAPIMetrics() { papiData_->start(); }
+#endif
 
+#if vt_check_enabled(papi)
   /**
    * \brief Stop PAPI metrics map for the running context
    *
    * \note has to be called after startPAPIMetrics
    *
    */
   void stopPAPIMetrics() { papiData_->stop(); }
+#endif
 
+#if vt_check_enabled(papi)
   /**
    * \brief Get the current PAPI metrics map for the running context
    *
    * \return the PAPI metrics map
    */
   std::unordered_map<std::string, uint64_t> getPAPIMetrics();
+#endif
 
 private:
   ElementLBData* lb_data_ = nullptr;     /**< Element LB data */
   ElementIDStruct cur_elm_id_ = {};   /**< Current element ID  */
   bool should_instrument_ = false;    /**< Whether we are instrumenting */
-  int EventSet_ = PAPI_NULL;
+#if vt_check_enabled(papi)
   std::unique_ptr<PAPIData> papiData_;
+#endif
 };
 
 }} /* end namespace vt::ctx */

diff --git a/src/vt/context/runnable_context/lb_data.impl.h b/src/vt/context/runnable_context/lb_data.impl.h
@@ -50,7 +50,9 @@
 #include "vt/vrt/collection/manager.h"
 
 #include <memory>
+#if vt_check_enabled(papi)
 #include <papi.h>
+#endif
 
 namespace vt { namespace ctx {
 
@@ -59,8 +61,10 @@ LBData::LBData(ElmT* in_elm, MsgT* msg)
   : lb_data_(&in_elm->getLBData()),
     cur_elm_id_(in_elm->getElmID()),
     should_instrument_(msg->lbLiteInstrument()),
-    papiData_(std::make_unique<PAPIData>())
 {
+#if vt_check_enabled(papi)
+  papiData_ = std::make_unique<PAPIData>();
+#endif
   // record the communication LB data right away!
   theCollection()->recordLBData(in_elm, msg);
 }

diff --git a/src/vt/runtime/runtime_banner.cc b/src/vt/runtime/runtime_banner.cc
@@ -139,6 +139,12 @@ void Runtime::printStartupBanner() {
 #if vt_check_enabled(diagnostics)
   features.push_back(vt_feature_str_diagnostics);
 #endif
+#if vt_check_enabled(papi)
+  features.push_back(vt_feature_str_papi);
+#endif
+#if vt_check_enabled(perf)
+  features.push_back(vt_feature_str_perf);
+#endif
 
   std::string dirty = "";
   if (vt_git_clean_status == "DIRTY") {