Adding Kokkos unit test, but only enable it if APEX builds Kokkos as …

…a submodule. We can't guarantee that the installed Kokkos will only provide host support. This allows us to test Kokkos support on CI.

CMakeLists.txt

@@ -675,12 +675,17 @@ if(APEX_WITH_KOKKOS)
     find_package(Kokkos)
     if (Kokkos_FOUND)
         message(INFO " Using Kokkos include: ${Kokkos_INCLUDE_DIRS}/impl")
+        include_directories(${Kokkos_INCLUDE_DIRS})
         include_directories(${Kokkos_INCLUDE_DIRS}/impl)
+        # Can only build Kokkos test with an installed Kokkos.
     else()
         message(INFO " Kokkos not found, cloning submodule to get required headers.")
         include(cmake/Modules/AddGitSubmodule.cmake)
         add_git_submodule(kokkos FALSE)
         include_directories(${PROJECT_SOURCE_DIR}/kokkos/core/src/impl)
+        include_directories(${Kokkos_INCLUDE_DIRS_RET})
+        #include_directories(${Kokkos_INCLUDE_DIRS_RET}/impl)
+        add_subdirectory(kokkos)
     endif()
     add_definitions(-DAPEX_WITH_KOKKOS)
 endif()
@@ -1174,6 +1179,9 @@ if(APEX_BUILD_TESTS)
     if (APEX_HIP_TESTS)
         add_subdirectory (src/unit_tests/HIP)
     endif (APEX_HIP_TESTS)
+    if (APEX_WITH_KOKKOS AND NOT Kokkos_FOUND)
+        add_subdirectory (src/unit_tests/Kokkos)
+    endif (APEX_WITH_KOKKOS AND NOT Kokkos_FOUND)
 else()
     #add_subdirectory (src/unit_tests/C EXCLUDE_FROM_ALL)
     #add_subdirectory (src/unit_tests/C++ EXCLUDE_FROM_ALL)

src/apex/CMakeLists_standalone.cmake

@@ -151,6 +151,8 @@ endif (APEX_WITH_MPI)
 if (APEX_WITH_CUDA)
     add_definitions(-DAPEX_WITH_CUDA)
     include_directories (${CUDAToolkit_INCLUDE_DIR})
+    include_directories(${CUPTI_INCLUDE_DIRS})
+    include_directories(${NVML_INCLUDE_DIRS})
     if (CUPTI_FOUND)
         SET(CUPTI_SOURCE cupti_trace.cpp)
     endif(CUPTI_FOUND)

src/unit_tests/Kokkos/CMakeLists.txt

@@ -0,0 +1,32 @@
+# Make sure the compiler can find include files from our Apex library.
+include_directories (${APEX_SOURCE_DIR}/src/apex)
+#include_directories(${Kokkos_INCLUDE_DIRS_RET})
+
+# Make sure the linker can find the Apex library once it is built.
+link_directories (${APEX_BINARY_DIR}/src/apex)
+#link_directories (${APEX_BINARY_DIR}/src/apex_pthread_wrapper)
+
+set(example_programs
+    simple
+   )
+
+foreach(example_program ${example_programs})
+  set(sources ${example_program}.cpp)
+  source_group("Source Files" FILES ${sources})
+  add_executable("${example_program}_kokkos" ${sources})
+  target_link_libraries ("${example_program}_kokkos" apex kokkos ${LIBS})
+  if (OPENMP_FOUND)
+    set_target_properties("${example_program}_kokkos" PROPERTIES COMPILE_FLAGS ${OpenMP_CXX_FLAGS})
+  endif (OPENMP_FOUND)
+  add_dependencies ("${example_program}_kokkos" apex)
+  add_dependencies (tests "${example_program}_kokkos")
+  add_test ("test_${example_program}_kokkos" "${example_program}_kokkos")
+  set_tests_properties("test_${example_program}_kokkos" PROPERTIES TIMEOUT 30)
+  set_property(TEST "test_${example_program}_kokkos" APPEND PROPERTY ENVIRONMENT "KOKKOS_TOOLS_LIBS=${PROJECT_BINARY_DIR}/src/apex/libapex.so")
+  set_property(TEST "test_${example_program}_kokkos" APPEND PROPERTY ENVIRONMENT "APEX_SCREEN_OUTPUT=1")
+  set_property(TEST "test_${example_program}_kokkos" APPEND PROPERTY ENVIRONMENT "OMP_PROC_BIND=spread")
+  set_property(TEST "test_${example_program}_kokkos" APPEND PROPERTY ENVIRONMENT "OMP_PLACES=threads")
+  set_property(TEST "test_${example_program}_kokkos" APPEND PROPERTY ENVIRONMENT "OMP_NUM_THREADS=4")
+
+endforeach()
+

src/unit_tests/Kokkos/simple.cpp

@@ -0,0 +1,54 @@
+#include <cmath>
+#include <iostream>
+#include <vector>
+#include <thread>
+#include <Kokkos_Core.hpp>
+
+#ifndef EXECUTION_SPACE
+#define EXECUTION_SPACE DefaultHostExecutionSpace
+#endif
+
+void go(size_t i) {
+    int n = 512;
+    Kokkos::View<double*> a("a",n);
+    Kokkos::View<double*> b("b",n);
+    Kokkos::View<double*> c("c",n);
+
+    auto range = Kokkos::RangePolicy<Kokkos::EXECUTION_SPACE>(0,n);
+
+    Kokkos::parallel_for(
+        "initialize", range, KOKKOS_LAMBDA(size_t const i) {
+            auto x = static_cast<double>(i);
+            a(i) = sin(x) * sin(x);
+            b(i) = cos(x) * cos(x);
+        }
+    );
+
+    Kokkos::parallel_for(
+        "xpy", range, KOKKOS_LAMBDA(size_t const i) {
+            c(i) = a(i) + b(i);
+        }
+    );
+
+    double sum = 0.0;
+
+    Kokkos::parallel_reduce(
+        "sum", range, KOKKOS_LAMBDA(size_t const i, double &lsum) {
+            lsum += c(i);
+        }, sum
+    );
+
+    if (i % 10 == 0) {
+        std::cout << "sum = " << sum / n << std::endl;
+    }
+}
+
+int main(int argc, char *argv[]) {
+    Kokkos::initialize(argc, argv);
+    std::cout << "Kokkos execution space: "
+        << Kokkos::DefaultExecutionSpace::name() << std::endl;
+    for (size_t i = 0 ; i < 10 ; i++) {
+        go(i);
+    }
+    Kokkos::finalize();
+}