Merge branch 'ABHI/abagusetty/homme/aurora' (PR E3SM-Project#6720)

Add appropriate flags for MKL
-mkl for older Intel MKL modules
-qmkl for newer oneMKL modules

-mkl is deprecated with newer oneMKL library leading
to linking issues referring to the MKL functions

components/homme/CMakeLists.txt

@@ -256,9 +256,22 @@ elseif (NOT TARGET csm_share)
   message (FATAL_ERROR "Aborting.")
 endif()
 
-OPTION(HOMME_USE_MKL "Whether to use Intel's MKL instead of blas/lapack" FALSE)
+OPTION(HOMME_USE_MKL "Whether to use Intel's MKL/oneMKL instead of blas/lapack" FALSE)
 IF(HOMME_USE_MKL)
-  MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -mkl will be added to each executable/library.")
+  IF(DEFINED ENV{MKLROOT})
+    SET(MKL_ROOT_PATH "$ENV{MKLROOT}")
+    IF(MKL_ROOT_PATH MATCHES "oneapi/mkl")
+      SET(MKL_TYPE "oneMKL")
+      MESSAGE(STATUS "Detected oneMKL based on MKLROOT: ${MKL_ROOT_PATH}")
+      FIND_PACKAGE(MKL REQUIRED $ENV{MKLROOT}/lib/cmake/mkl)
+    ELSE()
+      SET(MKL_TYPE "Intel MKL")
+      MESSAGE(STATUS "Detected standalone Intel MKL based on MKLROOT: ${MKL_ROOT_PATH}")
+    ENDIF()
+  ELSE()
+    MESSAGE(FATAL_ERROR "MKLROOT environment variable is not set. Please set it to your MKL installation path.")
+  ENDIF()
+  MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -mkl/-qmkl will be added to each executable/library.")
 ELSE()
   OPTION(HOMME_FIND_BLASLAPACK "Whether to use system blas/lapack" FALSE)
   MESSAGE(STATUS "HOMME_FIND_BLASLAPACK=${HOMME_FIND_BLASLAPACK}")

components/homme/cmake/HommeMacros.cmake

@@ -162,17 +162,20 @@ macro(createTestExec execName execType macroNP macroNC
   ENDIF ()
 
   IF (HOMME_USE_KOKKOS)
-    target_link_libraries(${execName} Kokkos::kokkos)
+    TARGET_LINK_LIBRARIES(${execName} Kokkos::kokkos)
   ENDIF ()
 
   # Move the module files out of the way so the parallel build
   # doesn't have a race condition
   SET_TARGET_PROPERTIES(${execName}
-                        PROPERTIES Fortran_MODULE_DIRECTORY ${EXEC_MODULE_DIR})
+    PROPERTIES Fortran_MODULE_DIRECTORY ${EXEC_MODULE_DIR})
 
   IF (HOMME_USE_MKL)
-    TARGET_COMPILE_OPTIONS(${execName} PUBLIC -mkl)
-    TARGET_LINK_LIBRARIES(${execName} -mkl)
+    IF (MKL_TYPE STREQUAL "oneMKL")
+      TARGET_LINK_LIBRARIES(${execName} -qmkl)
+    ELSEIF (MKL_TYPE STREQUAL "Intel MKL")
+      TARGET_LINK_LIBRARIES(${execName} -mkl)
+    ENDIF ()
   ELSE()
     IF (NOT HOMME_FIND_BLASLAPACK)
       TARGET_LINK_LIBRARIES(${execName} lapack blas)
@@ -270,8 +273,11 @@ macro(createExecLib libName execType libSrcs inclDirs macroNP
   ENDIF ()
 
   IF (HOMME_USE_MKL)
-    TARGET_COMPILE_OPTIONS(${libName} PUBLIC -mkl)
-    TARGET_LINK_LIBRARIES(${libName} -mkl)
+    IF (MKL_TYPE STREQUAL "oneMKL")
+      TARGET_LINK_LIBRARIES(${libName} -qmkl)
+    ELSEIF (MKL_TYPE STREQUAL "Intel MKL")
+      TARGET_LINK_LIBRARIES(${libName} -mkl)
+    ENDIF ()
   ELSE()
     IF (NOT HOMME_FIND_BLASLAPACK)
       TARGET_LINK_LIBRARIES(${libName} lapack blas)

components/homme/test/unit_tests/CMakeLists.txt

@@ -22,14 +22,17 @@ macro(cxx_unit_test target_name target_f90_srcs target_cxx_srcs include_dirs con
   cxx_unit_test_add_test(${target_name}_test ${target_name} ${NUM_CPUS})
 
   TARGET_LINK_LIBRARIES(${target_name} timing ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
-  target_link_libraries(${target_name} Kokkos::kokkos)
+  TARGET_LINK_LIBRARIES(${target_name} Kokkos::kokkos)
   IF (HOMME_USE_MKL)
-    TARGET_COMPILE_OPTIONS (${target_name} PUBLIC -mkl)
-    TARGET_LINK_LIBRARIES (${target_name} -mkl)
+    IF (MKL_TYPE STREQUAL "oneMKL")
+      TARGET_LINK_LIBRARIES(${target_name} -qmkl)
+    ELSEIF (MKL_TYPE STREQUAL "Intel MKL")
+      TARGET_LINK_LIBRARIES(${target_name} -mkl)
+    ENDIF ()
   ENDIF()
 
   # Link csm_share lib
-  target_link_libraries(${target_name} csm_share)
+  TARGET_LINK_LIBRARIES(${target_name} csm_share)
 
   STRING(TOUPPER "${PERFORMANCE_PROFILE}" PERF_PROF_UPPER)
   IF ("${PERF_PROF_UPPER}" STREQUAL "VTUNE")

components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt

@@ -36,10 +36,14 @@ ADD_LIBRARY(preqx_kokkos_ut_lib
 TARGET_INCLUDE_DIRECTORIES(preqx_kokkos_ut_lib PUBLIC ${EXEC_INCLUDE_DIRS})
 TARGET_INCLUDE_DIRECTORIES(preqx_kokkos_ut_lib PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
 TARGET_COMPILE_DEFINITIONS(preqx_kokkos_ut_lib PUBLIC "HAVE_CONFIG_H")
-target_link_libraries(preqx_kokkos_ut_lib Kokkos::kokkos)
+TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib Kokkos::kokkos)
 TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib timing csm_share ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
 IF (HOMME_USE_MKL)
-  TARGET_LINK_LIBRARIES (preqx_kokkos_ut_lib -mkl)
+  IF (MKL_TYPE STREQUAL "oneMKL")
+    TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib -qmkl)
+  ELSEIF (MKL_TYPE STREQUAL "Intel MKL")
+    TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib -mkl)
+  ENDIF ()
 ENDIF()
 IF(NOT BUILD_HOMME_WITHOUT_PIOLIBRARY)
   IF(HOMME_USE_SCORPIO)

components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt

@@ -42,10 +42,14 @@ ADD_LIBRARY(thetal_kokkos_ut_lib
 TARGET_INCLUDE_DIRECTORIES(thetal_kokkos_ut_lib PUBLIC ${EXEC_INCLUDE_DIRS})
 TARGET_INCLUDE_DIRECTORIES(thetal_kokkos_ut_lib PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
 TARGET_COMPILE_DEFINITIONS(thetal_kokkos_ut_lib PUBLIC "HAVE_CONFIG_H")
-target_link_libraries(thetal_kokkos_ut_lib Kokkos::kokkos)
+TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib Kokkos::kokkos)
 TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib timing csm_share ${COMPOSE_LIBRARY_CPP} ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
 IF (HOMME_USE_MKL)
-  TARGET_LINK_LIBRARIES (thetal_kokkos_ut_lib -mkl)
+  IF (MKL_TYPE STREQUAL "oneMKL")
+    TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib -qmkl)
+  ELSEIF (MKL_TYPE STREQUAL "Intel MKL")
+    TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib -mkl)
+  ENDIF ()
 ENDIF()
 IF(BUILD_HOMME_WITHOUT_PIOLIBRARY)
   TARGET_COMPILE_DEFINITIONS(thetal_kokkos_ut_lib PUBLIC HOMME_WITHOUT_PIOLIBRARY)