Debugging starpu support

UO-OACISS · Dec 21, 2022 · cf97ddb · cf97ddb
1 parent 43b57be
commit cf97ddb
Show file tree

Hide file tree

Showing 3 changed files with 111 additions and 42 deletions.
diff --git a/cmake/Modules/FindSTARPU.cmake b/cmake/Modules/FindSTARPU.cmake
@@ -5,41 +5,57 @@
 #  STARPU_LIBRARIES - The libraries needed to use StarPU
 #  STARPU_DEFINITIONS - Compiler switches required for using StarPU
 
+find_package(PkgConfig)
+
 if(NOT DEFINED $STARPU_ROOT)
 	if(DEFINED ENV{STARPU_ROOT})
 		# message("   env STARPU_ROOT is defined as $ENV{STARPU_ROOT}")
 		set(STARPU_ROOT $ENV{STARPU_ROOT})
 	endif()
 endif()
 
-# TODO problem because lib name and include path contain the version number, ie libstarpu-1.3.a
-
-find_path(STARPU_INCLUDE_DIR NAMES starpu.h
-	HINTS ${STARPU_ROOT}/include/starpu/1.3/ $ENV{STARPU_ROOT}/include/starpu/1.3/)
-
-if(APPLE)
-    find_library(STARPU_LIBRARY NAMES libstarpu-1.3.a starpu-1.3
-	    HINTS ${STARPU_ROOT}/* $ENV{STARPU_ROOT}/*)
-else()
-    find_library(STARPU_LIBRARY NAMES starpu-1.3
-	    HINTS ${STARPU_ROOT}/* $ENV{STARPU_ROOT}/*)
-endif(APPLE)
-
-include(FindPackageHandleStandardArgs)
-# handle the QUIETLY and REQUIRED arguments and set STARPU_FOUND to TRUE
-# if all listed variables are TRUE
-find_package_handle_standard_args(STARPU  DEFAULT_MSG
-                                  STARPU_LIBRARY STARPU_INCLUDE_DIR)
-#                                  STARPU_INCLUDE_DIR)
-
-#mark_as_advanced(STARPU_INCLUDE_DIR STARPU_LIBRARY)
-mark_as_advanced(STARPU_INCLUDE_DIR)
-message( "Using ${STARPU_INCLUDE_DIR} as StarPU include dir" )
-
-if(STARPU_FOUND)
-  set(STARPU_LIBRARIES ${STARPU_LIBRARY} )
-  set(STARPU_INCLUDE_DIRS ${STARPU_INCLUDE_DIR})
-  set(STARPU_DIR ${STARPU_ROOT})
-  add_definitions(-DAPEX_HAVE_STARPU)
+if(NOT DEFINED $STARPU_ROOT)
+	if(DEFINED STARPU_DIR)
+		# message("   env STARPU_ROOT is defined as $ENV{STARPU_ROOT}")
+		set(STARPU_ROOT $STARPU_DIR)
+	endif()
+endif()
+
+message(INFO " will check ${STARPU_ROOT} for STARPU")
+set(CMAKE_PREFIX_PATH} "${CMAKE_PREFIX_PATH} ${STARPU_ROOT}")
+set(ENV{PKG_CONFIG_PATH} "${STARPU_ROOT}/libs/pkgconfig")
+pkg_check_modules(STARPU QUIET libstarpu)
+
+if(NOT STARPU_FOUND)
+    # TODO problem because lib name and include path contain the version number, ie libstarpu-1.3.a
+
+    find_path(STARPU_INCLUDE_DIR NAMES starpu.h
+	    HINTS ${STARPU_ROOT}/include/starpu/1.4 ${STARPU_ROOT}/include/starpu/1.3)
+
+    if(APPLE)
+        find_library(STARPU_LIBRARY NAMES starpu-1.4 starpu-1.3
+	        HINTS ${STARPU_ROOT})
+    else()
+        find_library(STARPU_LIBRARY NAMES starpu-1.4 starpu-1.3
+	        HINTS ${STARPU_ROOT})
+    endif(APPLE)
+
+    include(FindPackageHandleStandardArgs)
+    # handle the QUIETLY and REQUIRED arguments and set STARPU_FOUND to TRUE
+    # if all listed variables are TRUE
+    find_package_handle_standard_args(STARPU  DEFAULT_MSG
+                                    STARPU_LIBRARY STARPU_INCLUDE_DIR)
+    #                                  STARPU_INCLUDE_DIR)
+
+    #mark_as_advanced(STARPU_INCLUDE_DIR STARPU_LIBRARY)
+    mark_as_advanced(STARPU_INCLUDE_DIR)
+    message( "Using ${STARPU_INCLUDE_DIR} as StarPU include dir" )
+
+    if(STARPU_FOUND)
+        set(STARPU_LIBRARIES ${STARPU_LIBRARY} )
+        set(STARPU_INCLUDE_DIRS ${STARPU_INCLUDE_DIR})
+        set(STARPU_DIR ${STARPU_ROOT})
+        add_definitions(-DAPEX_HAVE_STARPU)
+    endif()
 endif()
 
diff --git a/src/apex/apex_starpu.cpp b/src/apex/apex_starpu.cpp
@@ -3,6 +3,8 @@
 #include <inttypes.h>
 #include "apex.hpp"
 #include <starpu_profiling_tool.h>
+#include <mutex>
+#include <stack>
 
 void starpu_perf_counter_collection_start( void );
 void starpu_perf_counter_collection_stop( void );
@@ -230,6 +232,39 @@ extern "C" {
     }
 }
 
+    void xferfunction_cb( struct starpu_prof_tool_info* prof_info,
+        union starpu_prof_tool_event_info* event_info,
+        struct starpu_prof_tool_api_info* api_info ) {
+
+        std::string event_name {event_types[prof_info->event_type]};
+        std::string device_name {device_types[prof_info->driver_type]};
+        std::stringstream info;
+
+        info << "[{ memnode " << prof_info->memnode << " }]";
+        event_name = event_name + info.str();
+        static thread_local std::stack<std::shared_ptr<apex::task_wrapper> > my_stack;
+        if (prof_info->event_type == starpu_prof_tool_event_end_transfer) {
+            if (my_stack.size() == 0) {
+            /*
+                static std::mutex mtx;
+                std::unique_lock<std::mutex> l(mtx);
+                std::cerr << "APEX Timer stack is empty, bug in StarPU support! "
+                    << event_name << " " << device_name
+                    << std::endl;
+                    */
+                return;
+            }
+            auto t = my_stack.top();
+            apex::stop(t);
+            my_stack.pop();
+        } else {
+            //std::cout << "Transfer start " << event_name << std::endl;
+            auto t = apex::new_task(event_name);
+            apex::start(t);
+            my_stack.push(t);
+        }
+    }
+
     /******************************************************************************/
     /*           Initialize myself as an external library.                       */
     /******************************************************************************/
@@ -238,6 +273,15 @@ extern "C" {
 
 void starpu_prof_tool_library_register(starpu_prof_tool_entry_register_func reg,
     starpu_prof_tool_entry_register_func unreg) {
+    enum  starpu_prof_tool_command info; // = starpu_prof_tool_command_reg;
+    /* This one must be called at the *beginning* of the initialization
+       Otherwise the flag might be set too late */
+    reg( starpu_prof_tool_event_init_begin, &enable_counters, info );
+    /* This one must be called at the *end* of the initialization
+       Otherwise the counters might not be ready yet */
+    reg( starpu_prof_tool_event_init_end, &init_counters, info );
+    /* This one must be called at the end, but I don't know precisely when yet */
+    reg( starpu_prof_tool_event_terminate, &finalize_counters, info );
 
     device_types[starpu_prof_tool_driver_cpu] = "CPU";
     device_types[starpu_prof_tool_driver_gpu] = "GPU";
@@ -260,16 +304,6 @@ void starpu_prof_tool_library_register(starpu_prof_tool_entry_register_func reg,
     event_types[starpu_prof_tool_event_user_start] = "StarPU user event ";
     event_types[starpu_prof_tool_event_user_end] = "StarPU user event ";
 
-    enum  starpu_prof_tool_command info; // = starpu_prof_tool_command_reg;
-    /* This one must be called at the *beginning* of the initialization
-       Otherwise the flag might be set too late */
-    reg( starpu_prof_tool_event_init_begin, &enable_counters, info );
-    /* This one must be called at the *end* of the initialization
-       Otherwise the counters might not be ready yet */
-    reg( starpu_prof_tool_event_init_end, &init_counters, info );
-    /* This one must be called at the end, but I don't know precisely when yet */
-    reg( starpu_prof_tool_event_terminate, &finalize_counters, info );
-
     reg( starpu_prof_tool_event_init_begin, &myfunction_cb, info );
     reg( starpu_prof_tool_event_init_end, &myfunction_cb, info );
     reg( starpu_prof_tool_event_init, &myfunction_cb, info );
@@ -282,8 +316,8 @@ void starpu_prof_tool_library_register(starpu_prof_tool_entry_register_func reg,
     reg( starpu_prof_tool_event_end_cpu_exec, &myfunction_cb, info );
     reg( starpu_prof_tool_event_start_gpu_exec, &myfunction_cb, info );
     reg( starpu_prof_tool_event_end_gpu_exec, &myfunction_cb, info );
-    //reg( starpu_prof_tool_event_start_transfer, &myfunction_cb, info );
-    //reg( starpu_prof_tool_event_end_transfer, &myfunction_cb, info );
+    reg( starpu_prof_tool_event_start_transfer, &xferfunction_cb, info );
+    reg( starpu_prof_tool_event_end_transfer, &xferfunction_cb, info );
 
     }
 

diff --git a/src/wrappers/pthread_wrapper_internal.cpp b/src/wrappers/pthread_wrapper_internal.cpp
@@ -19,6 +19,23 @@
 #include <sys/resource.h>
 #include <memory>
 #include <atomic>
+#include "apex_error_handling.hpp"
+
+#include "global_constructor_destructor.h"
+#if defined(HAS_CONSTRUCTORS)
+DEFINE_CONSTRUCTOR(apex_init_static_void)
+DEFINE_DESTRUCTOR(apex_finalize_static_void)
+
+void apex_init_static_void() {
+    //printf("Here! %s\n",__func__);
+    apex::init("APEX Pthread Wrapper",0,1);
+}
+void apex_finalize_static_void() {
+    //printf("There! %s\n",__func__);
+    apex::finalize();
+}
+#endif // HAS_CONSTRUCTORS
+
 
 std::atomic<int64_t> task_id(-1);
 
@@ -31,7 +48,7 @@ struct apex_system_wrapper_t
 {
   bool initialized;
   apex_system_wrapper_t() : initialized(true) {
-    apex::init("APEX Pthread Wrapper",0,1);
+    //apex::init("APEX Pthread Wrapper",0,1);
     /*
      * Here we are limiting the stack size to whatever the user requests.
      * Do it after we initialized APEX, because APEX spawns two other threads
@@ -212,6 +229,8 @@ int apex_pthread_create_wrapper(pthread_create_p pthread_create_call,
     pthread_t * threadp, const pthread_attr_t * attr,
     start_routine_p start_routine, void * arg)
 {
+  //printf("Here!\n");
+  //apex_print_backtrace();
   // disable the memory wrapper
   apex::in_apex prevent_problems;
   std::shared_ptr<apex::task_wrapper> parent_task = apex::new_task("pthread_create");