From cf97ddbe3e5088f11ee3f54f59a8ccfe43b57695 Mon Sep 17 00:00:00 2001 From: Kevin Huck Date: Tue, 20 Dec 2022 16:09:53 -0800 Subject: [PATCH] Debugging starpu support --- cmake/Modules/FindSTARPU.cmake | 74 ++++++++++++++--------- src/apex/apex_starpu.cpp | 58 ++++++++++++++---- src/wrappers/pthread_wrapper_internal.cpp | 21 ++++++- 3 files changed, 111 insertions(+), 42 deletions(-) diff --git a/cmake/Modules/FindSTARPU.cmake b/cmake/Modules/FindSTARPU.cmake index 60382fbb..80b2981f 100644 --- a/cmake/Modules/FindSTARPU.cmake +++ b/cmake/Modules/FindSTARPU.cmake @@ -5,6 +5,8 @@ # STARPU_LIBRARIES - The libraries needed to use StarPU # STARPU_DEFINITIONS - Compiler switches required for using StarPU +find_package(PkgConfig) + if(NOT DEFINED $STARPU_ROOT) if(DEFINED ENV{STARPU_ROOT}) # message(" env STARPU_ROOT is defined as $ENV{STARPU_ROOT}") @@ -12,34 +14,48 @@ if(NOT DEFINED $STARPU_ROOT) endif() endif() -# TODO problem because lib name and include path contain the version number, ie libstarpu-1.3.a - -find_path(STARPU_INCLUDE_DIR NAMES starpu.h - HINTS ${STARPU_ROOT}/include/starpu/1.3/ $ENV{STARPU_ROOT}/include/starpu/1.3/) - -if(APPLE) - find_library(STARPU_LIBRARY NAMES libstarpu-1.3.a starpu-1.3 - HINTS ${STARPU_ROOT}/* $ENV{STARPU_ROOT}/*) -else() - find_library(STARPU_LIBRARY NAMES starpu-1.3 - HINTS ${STARPU_ROOT}/* $ENV{STARPU_ROOT}/*) -endif(APPLE) - -include(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set STARPU_FOUND to TRUE -# if all listed variables are TRUE -find_package_handle_standard_args(STARPU DEFAULT_MSG - STARPU_LIBRARY STARPU_INCLUDE_DIR) -# STARPU_INCLUDE_DIR) - -#mark_as_advanced(STARPU_INCLUDE_DIR STARPU_LIBRARY) -mark_as_advanced(STARPU_INCLUDE_DIR) -message( "Using ${STARPU_INCLUDE_DIR} as StarPU include dir" ) - -if(STARPU_FOUND) - set(STARPU_LIBRARIES ${STARPU_LIBRARY} ) - set(STARPU_INCLUDE_DIRS ${STARPU_INCLUDE_DIR}) - set(STARPU_DIR ${STARPU_ROOT}) - add_definitions(-DAPEX_HAVE_STARPU) +if(NOT DEFINED $STARPU_ROOT) + if(DEFINED STARPU_DIR) + # message(" env STARPU_ROOT is defined as $ENV{STARPU_ROOT}") + set(STARPU_ROOT $STARPU_DIR) + endif() +endif() + +message(INFO " will check ${STARPU_ROOT} for STARPU") +set(CMAKE_PREFIX_PATH} "${CMAKE_PREFIX_PATH} ${STARPU_ROOT}") +set(ENV{PKG_CONFIG_PATH} "${STARPU_ROOT}/libs/pkgconfig") +pkg_check_modules(STARPU QUIET libstarpu) + +if(NOT STARPU_FOUND) + # TODO problem because lib name and include path contain the version number, ie libstarpu-1.3.a + + find_path(STARPU_INCLUDE_DIR NAMES starpu.h + HINTS ${STARPU_ROOT}/include/starpu/1.4 ${STARPU_ROOT}/include/starpu/1.3) + + if(APPLE) + find_library(STARPU_LIBRARY NAMES starpu-1.4 starpu-1.3 + HINTS ${STARPU_ROOT}) + else() + find_library(STARPU_LIBRARY NAMES starpu-1.4 starpu-1.3 + HINTS ${STARPU_ROOT}) + endif(APPLE) + + include(FindPackageHandleStandardArgs) + # handle the QUIETLY and REQUIRED arguments and set STARPU_FOUND to TRUE + # if all listed variables are TRUE + find_package_handle_standard_args(STARPU DEFAULT_MSG + STARPU_LIBRARY STARPU_INCLUDE_DIR) + # STARPU_INCLUDE_DIR) + + #mark_as_advanced(STARPU_INCLUDE_DIR STARPU_LIBRARY) + mark_as_advanced(STARPU_INCLUDE_DIR) + message( "Using ${STARPU_INCLUDE_DIR} as StarPU include dir" ) + + if(STARPU_FOUND) + set(STARPU_LIBRARIES ${STARPU_LIBRARY} ) + set(STARPU_INCLUDE_DIRS ${STARPU_INCLUDE_DIR}) + set(STARPU_DIR ${STARPU_ROOT}) + add_definitions(-DAPEX_HAVE_STARPU) + endif() endif() diff --git a/src/apex/apex_starpu.cpp b/src/apex/apex_starpu.cpp index 3d2a0ea8..58dffa7a 100644 --- a/src/apex/apex_starpu.cpp +++ b/src/apex/apex_starpu.cpp @@ -3,6 +3,8 @@ #include #include "apex.hpp" #include +#include +#include void starpu_perf_counter_collection_start( void ); void starpu_perf_counter_collection_stop( void ); @@ -230,6 +232,39 @@ extern "C" { } } + void xferfunction_cb( struct starpu_prof_tool_info* prof_info, + union starpu_prof_tool_event_info* event_info, + struct starpu_prof_tool_api_info* api_info ) { + + std::string event_name {event_types[prof_info->event_type]}; + std::string device_name {device_types[prof_info->driver_type]}; + std::stringstream info; + + info << "[{ memnode " << prof_info->memnode << " }]"; + event_name = event_name + info.str(); + static thread_local std::stack > my_stack; + if (prof_info->event_type == starpu_prof_tool_event_end_transfer) { + if (my_stack.size() == 0) { + /* + static std::mutex mtx; + std::unique_lock l(mtx); + std::cerr << "APEX Timer stack is empty, bug in StarPU support! " + << event_name << " " << device_name + << std::endl; + */ + return; + } + auto t = my_stack.top(); + apex::stop(t); + my_stack.pop(); + } else { + //std::cout << "Transfer start " << event_name << std::endl; + auto t = apex::new_task(event_name); + apex::start(t); + my_stack.push(t); + } + } + /******************************************************************************/ /* Initialize myself as an external library. */ /******************************************************************************/ @@ -238,6 +273,15 @@ extern "C" { void starpu_prof_tool_library_register(starpu_prof_tool_entry_register_func reg, starpu_prof_tool_entry_register_func unreg) { + enum starpu_prof_tool_command info; // = starpu_prof_tool_command_reg; + /* This one must be called at the *beginning* of the initialization + Otherwise the flag might be set too late */ + reg( starpu_prof_tool_event_init_begin, &enable_counters, info ); + /* This one must be called at the *end* of the initialization + Otherwise the counters might not be ready yet */ + reg( starpu_prof_tool_event_init_end, &init_counters, info ); + /* This one must be called at the end, but I don't know precisely when yet */ + reg( starpu_prof_tool_event_terminate, &finalize_counters, info ); device_types[starpu_prof_tool_driver_cpu] = "CPU"; device_types[starpu_prof_tool_driver_gpu] = "GPU"; @@ -260,16 +304,6 @@ void starpu_prof_tool_library_register(starpu_prof_tool_entry_register_func reg, event_types[starpu_prof_tool_event_user_start] = "StarPU user event "; event_types[starpu_prof_tool_event_user_end] = "StarPU user event "; - enum starpu_prof_tool_command info; // = starpu_prof_tool_command_reg; - /* This one must be called at the *beginning* of the initialization - Otherwise the flag might be set too late */ - reg( starpu_prof_tool_event_init_begin, &enable_counters, info ); - /* This one must be called at the *end* of the initialization - Otherwise the counters might not be ready yet */ - reg( starpu_prof_tool_event_init_end, &init_counters, info ); - /* This one must be called at the end, but I don't know precisely when yet */ - reg( starpu_prof_tool_event_terminate, &finalize_counters, info ); - reg( starpu_prof_tool_event_init_begin, &myfunction_cb, info ); reg( starpu_prof_tool_event_init_end, &myfunction_cb, info ); reg( starpu_prof_tool_event_init, &myfunction_cb, info ); @@ -282,8 +316,8 @@ void starpu_prof_tool_library_register(starpu_prof_tool_entry_register_func reg, reg( starpu_prof_tool_event_end_cpu_exec, &myfunction_cb, info ); reg( starpu_prof_tool_event_start_gpu_exec, &myfunction_cb, info ); reg( starpu_prof_tool_event_end_gpu_exec, &myfunction_cb, info ); - //reg( starpu_prof_tool_event_start_transfer, &myfunction_cb, info ); - //reg( starpu_prof_tool_event_end_transfer, &myfunction_cb, info ); + reg( starpu_prof_tool_event_start_transfer, &xferfunction_cb, info ); + reg( starpu_prof_tool_event_end_transfer, &xferfunction_cb, info ); } diff --git a/src/wrappers/pthread_wrapper_internal.cpp b/src/wrappers/pthread_wrapper_internal.cpp index d5d6867b..6c4246e3 100644 --- a/src/wrappers/pthread_wrapper_internal.cpp +++ b/src/wrappers/pthread_wrapper_internal.cpp @@ -19,6 +19,23 @@ #include #include #include +#include "apex_error_handling.hpp" + +#include "global_constructor_destructor.h" +#if defined(HAS_CONSTRUCTORS) +DEFINE_CONSTRUCTOR(apex_init_static_void) +DEFINE_DESTRUCTOR(apex_finalize_static_void) + +void apex_init_static_void() { + //printf("Here! %s\n",__func__); + apex::init("APEX Pthread Wrapper",0,1); +} +void apex_finalize_static_void() { + //printf("There! %s\n",__func__); + apex::finalize(); +} +#endif // HAS_CONSTRUCTORS + std::atomic task_id(-1); @@ -31,7 +48,7 @@ struct apex_system_wrapper_t { bool initialized; apex_system_wrapper_t() : initialized(true) { - apex::init("APEX Pthread Wrapper",0,1); + //apex::init("APEX Pthread Wrapper",0,1); /* * Here we are limiting the stack size to whatever the user requests. * Do it after we initialized APEX, because APEX spawns two other threads @@ -212,6 +229,8 @@ int apex_pthread_create_wrapper(pthread_create_p pthread_create_call, pthread_t * threadp, const pthread_attr_t * attr, start_routine_p start_routine, void * arg) { + //printf("Here!\n"); + //apex_print_backtrace(); // disable the memory wrapper apex::in_apex prevent_problems; std::shared_ptr parent_task = apex::new_task("pthread_create");