Add CI container compilation tests for GPU code (#46)

.github/containers/x86_64-broadwell-cuda11.6.1/Dockerfile

@@ -0,0 +1,28 @@
+FROM nvidia/cuda:11.6.1-devel-ubi8 AS base
+MAINTAINER Giorgis Georgakoudis <[email protected]>
+RUN \
+    yum install -y dnf &&\
+    dnf install -y git xz autoconf automake unzip patch gcc-gfortran bzip2 file &&\
+    dnf upgrade -y &&\
+    dnf clean all
+COPY repo repo
+RUN \
+    mkdir -p ams-spack-env
+COPY spack.yaml ams-spack-env/spack.yaml
+
+FROM base AS setup-spack-env
+RUN \
+    git clone --depth 1 --branch releases/v0.20 https://github.com/spack/spack.git &&\
+    source spack/share/spack/setup-env.sh &&\
+    spack env activate -p ams-spack-env &&\
+    spack external find --all --not-buildable --exclude openssl --exclude openblas --exclude bzip2
+
+FROM setup-spack-env AS install-spack-env
+RUN \
+    source spack/share/spack/setup-env.sh &&\
+    spack env activate -p ams-spack-env &&\
+    spack install --fail-fast
+
+FROM install-spack-env AS clean-spack
+RUN \
+    spack clean --all

.github/containers/x86_64-broadwell-cuda11.6.1/repo/packages/AMS/package.py

@@ -0,0 +1,121 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# AMSLib Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from spack.package import *
+
+
+class Ams(CMakePackage, CudaPackage):
+
+    homepage = "https://lc.llnl.gov/gitlab/autonomous-multiscale-project/marbl-matprops-miniapp"
+    git = "ssh://[email protected]:7999/autonomous-multiscale-project/marbl-matprops-miniapp.git"
+
+    maintainers = ["parasyris1", "koparasy"]
+
+    version("develop", branch="develop", submodules=False)
+    version("release", branch="release", submodules=False)
+    version("main", branch="main", submodules=False)
+
+    variant("faiss", default=False, description="Build with FAISS index as uncertainty quantification module")
+    variant("caliper", default=False, description="Build with caliper for gather performance counters")
+    variant("torch", default=False, description="Use torch for surrogate models")
+    variant("mpi", default=False, description="Enable MPI support")
+    variant("examples", default=False, description="Enable MPI support")
+    variant("redis", default=False, description="Enable redis database")
+    variant("hdf5", default=False, description="Enable HDF5 data storage")
+    variant("rabbitmq", default=False, description="Enable RabbitMQ as data broker")
+    variant("verbose", default=False, description="Enable AMSLib verbose output (controlled by environment variable)")
+
+    depends_on("umpire")
+    depends_on("mpi", when="+mpi")
+
+    depends_on("caliper+cuda", when="+caliper +cuda")
+    depends_on("faiss+cuda", when="+faiss +cuda")
+    depends_on("mfem+cuda", when="+examples +cuda")
+    depends_on("py-torch+cuda", when="+torch +cuda")
+
+    depends_on("py-torch~cuda", when="+torch ~cuda")
+    depends_on("caliper ~cuda", when="+caliper ~cuda")
+    depends_on("faiss ~cuda", when="+faiss ~cuda")
+    depends_on("mfem ~cuda", when="+examples ~cuda")
+
+    depends_on("redis-plus-plus", when="+redis")
+    depends_on("hdf5", when="+hdf5")
+    depends_on("amqp-cpp +tcp", when="+rabbitmq")
+
+    with when("+cuda"):
+        cuda_archs=CudaPackage.cuda_arch_values
+        with when("+examples"):
+            depends_on("mfem+cuda")
+            for sm_ in cuda_archs:
+                depends_on("mfem +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_))
+
+        with when("+torch"):
+            depends_on("py-torch+cuda")
+            for sm_ in cuda_archs:
+                depends_on("py-torch +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_))
+
+        with when("+caliper"):
+            depends_on("caliper+cuda", when="+caliper")
+            for sm_ in cuda_archs:
+                depends_on("caliper +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_))
+
+        depends_on("umpire+cuda")
+        for sm_ in cuda_archs:
+            depends_on("umpire +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_))
+
+        with when("+faiss"):
+            depends_on("faiss+cuda", when="+faiss")
+            for sm_ in cuda_archs:
+                depends_on("umpire +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_))
+
+    def cmake_args(self):
+        spec = self.spec
+        args = []
+        args.append("-DUMPIRE_DIR={0}".format(spec["umpire"].prefix))
+        args.append("-DWITH_MPI={0}".format("On" if "+mpi" in spec else "Off"))
+
+        args.append("-DWITH_DB={0}".format("On" if ("+redis" in spec or "hdf5" in spec or "+rabbitmq" in spec) else "Off"))
+
+        if "+verbose" in spec:
+            args.append("-DWITH_AMS_DEBUG=On")
+
+        if "+hdf5" in spec:
+            args.append("-DWITH_HDF5=On")
+            args.append("-DHDF5_Dir={0}".format(spec["hdf5"].prefix))
+
+        if "+cuda" in spec:
+            args.append("-DWITH_CUDA=On")
+            cuda_arch = spec.variants["cuda_arch"].value[0]
+            args.append("-DAMS_CUDA_ARCH={0}".format(cuda_arch))
+
+        if "+caliper" in spec:
+            args.append("-DWITH_CALIPER=On")
+            args.append("-DCALIPER_DIR={0}/share/cmake/caliper".format(spec["caliper"].prefix))
+        else:
+            args.append("-DWITH_CALIPER=Off")
+
+        if "+faiss" in spec:
+            args.append("-DWITH_FAISS=On")
+            args.append("-DFAISS_DIR={0}".format(spec['faiss'].prefix))
+        else:
+            args.append("-DWITH_FAISS=Off")
+
+        if "+torch" in spec:
+            args.append("-DWITH_TORCH=On")
+            args.append("-DTorch_DIR={0}/lib/python{1}/site-packages/torch/share/cmake/Torch".format(spec['py-torch'].prefix, spec['python'].version.up_to(2)))
+
+        if "+redis" in spec:
+            args.append("-DWITH_REDIS=On")
+            args.append("-DREDIS_PLUS_PLUS_DIR={0}".format(spec["redis-plus-plus"].prefix))
+
+        if "+rabbitmq" in spec:
+            args.append("-DWITH_RMQ=On")
+            args.append("-Damqpcpp_DIR={0}/cmake".format(spec["amqp-cpp"].prefix))
+
+        if "+examples" in spec:
+            args.append("-DWITH_EXAMPLES=On")
+            args.append("-DMFEM_DIR={0}".format(spec['mfem'].prefix))
+
+        return args

.github/containers/x86_64-broadwell-cuda11.6.1/repo/packages/cmake/5882-enable-cce-fortran-preprocessing.patch

@@ -0,0 +1,31 @@
+From ef513fe3d1d864d865d7143699834228988a7cd7 Mon Sep 17 00:00:00 2001
+From: Brad King <[email protected]>
+Date: Fri, 5 Mar 2021 08:08:16 -0500
+Subject: [PATCH] Cray: Enable explicit Fortran preprocessing for Ninja
+ generator
+
+Cray 11.0 adds support for preprocessing with output written to a
+specified file (instead of always next to the source).  Use it to
+enable Cray Fortran with the Ninja generator.
+
+Patch-by: James Elliott
+Fixes: #20731
+---
+ Modules/Compiler/Cray-Fortran.cmake | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/Modules/Compiler/Cray-Fortran.cmake b/Modules/Compiler/Cray-Fortran.cmake
+index 696ae76074..0d5e1c7679 100644
+--- a/Modules/Compiler/Cray-Fortran.cmake
++++ b/Modules/Compiler/Cray-Fortran.cmake
+@@ -19,3 +19,7 @@ else()
+   set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-eZ")
+   set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-dZ")
+ endif()
++
++if (NOT CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 11.0)
++  set(CMAKE_Fortran_PREPROCESS_SOURCE "<CMAKE_Fortran_COMPILER> -o <PREPROCESSED_SOURCE> <DEFINES> <INCLUDES> <FLAGS> -eP <SOURCE>")
++endif()
+-- 
+GitLab
+

.github/containers/x86_64-broadwell-cuda11.6.1/repo/packages/cmake/cmake-macos-add-coreservices.patch

@@ -0,0 +1,28 @@
+commit 475e78d9071b34690617a85853433a9fc15da057
+Author: Chuck Atkins <[email protected]>
+Date:   Mon Jan 28 16:28:28 2019 -0500
+
+    macOS: Add missing explicit dependency on CoreServices framework
+
+    On Apple, the implementation of cmGlobalXCodeGenerator::Open uses
+    LSOpenCFURLRef from CoreServices.  This get's transitively pulled in
+    from CMake's libuv build but ends up generating a linker error when
+    using an external libuv.  This explicitly adds the appropriate
+    dependency.
+
+diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt
+index 311f3f4e56..8aff8f6b2f 100644
+--- a/Source/CMakeLists.txt
++++ b/Source/CMakeLists.txt
+@@ -791,9 +791,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "sparc"
+   endif()
+ endif()
+
+-# On Apple we need CoreFoundation
++# On Apple we need CoreFoundation and CoreServices
+ if(APPLE)
+   target_link_libraries(CMakeLib "-framework CoreFoundation")
++  target_link_libraries(CMakeLib "-framework CoreServices")
+ endif()
+
+ if(WIN32 AND NOT UNIX)

.github/containers/x86_64-broadwell-cuda11.6.1/repo/packages/cmake/cmake-revert-findmpi-link-flag-list.patch

@@ -0,0 +1,29 @@
+From 89fc3b1fd22f97f9380990b521dd79f306ac18fd Mon Sep 17 00:00:00 2001
+From: Chuck Atkins <[email protected]>
+Date: Thu, 25 Jul 2019 09:37:20 -0400
+Subject: [PATCH] Revert "FindMPI: Store imported target link flags as a list
+ instead of a string"
+
+This reverts commit f7eaa342de316707d99e6ae29c693a480861560d.
+---
+ Modules/FindMPI.cmake | 4 +---
+ 1 file changed, 1 insertion(+), 3 deletions(-)
+
+diff --git a/Modules/FindMPI.cmake b/Modules/FindMPI.cmake
+index a80f799..fe09764 100644
+--- a/Modules/FindMPI.cmake
++++ b/Modules/FindMPI.cmake
+@@ -1144,9 +1144,7 @@ macro(_MPI_create_imported_target LANG)
+
+   set_property(TARGET MPI::MPI_${LANG} PROPERTY INTERFACE_LINK_LIBRARIES "")
+   if(MPI_${LANG}_LINK_FLAGS)
+-    separate_arguments(_MPI_${LANG}_LINK_FLAGS NATIVE_COMMAND "${MPI_${LANG}_LINK_FLAGS}")
+-    set_property(TARGET MPI::MPI_${LANG} APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${_MPI_${LANG}_LINK_FLAGS}")
+-    unset(_MPI_${LANG}_LINK_FLAGS)
++    set_property(TARGET MPI::MPI_${LANG} APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${MPI_${LANG}_LINK_FLAGS}")
+   endif()
+   # If the compiler links MPI implicitly, no libraries will be found as they're contained within
+   # CMAKE_<LANG>_IMPLICIT_LINK_LIBRARIES already.
+-- 
+2.5.5
+

.github/containers/x86_64-broadwell-cuda11.6.1/repo/packages/cmake/fix-xlf-ninja-mr-4075.patch

@@ -0,0 +1,145 @@
+From 19f267c75e84b72c4de42570be0c4222bb93aaff Mon Sep 17 00:00:00 2001
+From: Brad King <[email protected]>
+Date: Thu, 21 Nov 2019 14:38:35 -0500
+Subject: [PATCH] XL: Add support for Ninja and XL Fortran
+
+The Ninja generator's support for Fortran requires that source files
+be preprocessed explicitly first.  However, the `xlf` compiler does
+not have a simple `-E` option or equivalent to do preprocessing.
+The only documented way to get preprocessed output is to use `-d`
+to leave it behind, but only at an inflexible location.
+
+Instead, create our own `cpp` wrapper script and substitute it for the
+real preprocessor using `-tF -B ...`.  Teach the wrapper to map the
+`cpp` output to the location we need and then invoke the real `cpp`
+underneath.
+
+Fixes: #19450
+---
+ Help/release/dev/xlf-ninja.rst              |  5 ++++
+ Modules/CMakeDetermineCompilerId.cmake      | 10 +++++++
+ Modules/CMakeDetermineFortranCompiler.cmake |  5 ++++
+ Modules/CMakeFortranCompiler.cmake.in       |  1 +
+ Modules/Compiler/XL-Fortran.cmake           |  4 +++
+ Modules/Compiler/XL-Fortran/cpp             | 29 +++++++++++++++++++++
+ 6 files changed, 54 insertions(+)
+ create mode 100644 Help/release/dev/xlf-ninja.rst
+ create mode 100755 Modules/Compiler/XL-Fortran/cpp
+
+diff --git a/Help/release/dev/xlf-ninja.rst b/Help/release/dev/xlf-ninja.rst
+new file mode 100644
+index 0000000000..916e713008
+--- /dev/null
++++ b/Help/release/dev/xlf-ninja.rst
+@@ -0,0 +1,5 @@
++xlf-ninja
++---------
++
++* The IBM XL Fortran compiler is now supported by the :generator:`Ninja`
++  generator.
+diff --git a/Modules/CMakeDetermineCompilerId.cmake b/Modules/CMakeDetermineCompilerId.cmake
+index f7ef755aeb..0b3664c5de 100644
+--- a/Modules/CMakeDetermineCompilerId.cmake
++++ b/Modules/CMakeDetermineCompilerId.cmake
+@@ -182,6 +182,10 @@ function(CMAKE_DETERMINE_COMPILER_ID lang flagvar src)
+     message(STATUS "The ${lang} compiler identification is unknown")
+   endif()
+
++  if(lang STREQUAL "Fortran" AND CMAKE_${lang}_COMPILER_ID STREQUAL "XL")
++    set(CMAKE_${lang}_XL_CPP "${CMAKE_${lang}_COMPILER_ID_CPP}" PARENT_SCOPE)
++  endif()
++
+   set(CMAKE_${lang}_COMPILER_ID "${CMAKE_${lang}_COMPILER_ID}" PARENT_SCOPE)
+   set(CMAKE_${lang}_PLATFORM_ID "${CMAKE_${lang}_PLATFORM_ID}" PARENT_SCOPE)
+   set(CMAKE_${lang}_COMPILER_ARCHITECTURE_ID "${CMAKE_${lang}_COMPILER_ARCHITECTURE_ID}" PARENT_SCOPE)
+@@ -542,6 +546,12 @@ Id flags: ${testflags} ${CMAKE_${lang}_COMPILER_ID_FLAGS_ALWAYS}
+       ERROR_VARIABLE CMAKE_${lang}_COMPILER_ID_OUTPUT
+       RESULT_VARIABLE CMAKE_${lang}_COMPILER_ID_RESULT
+       )
++    if("${CMAKE_${lang}_COMPILER_ID_OUTPUT}" MATCHES "exec: [^\n]*\\((/[^,\n]*/cpp),CMakeFortranCompilerId.F")
++      set(_cpp "${CMAKE_MATCH_1}")
++      if(EXISTS "${_cpp}")
++        set(CMAKE_${lang}_COMPILER_ID_CPP "${_cpp}" PARENT_SCOPE)
++      endif()
++    endif()
+   endif()
+
+   # Check the result of compilation.
+diff --git a/Modules/CMakeDetermineFortranCompiler.cmake b/Modules/CMakeDetermineFortranCompiler.cmake
+index 5ddd64fae8..e8505417d6 100644
+--- a/Modules/CMakeDetermineFortranCompiler.cmake
++++ b/Modules/CMakeDetermineFortranCompiler.cmake
+@@ -271,6 +271,11 @@ include(CMakeFindBinUtils)
+ include(Compiler/${CMAKE_Fortran_COMPILER_ID}-FindBinUtils OPTIONAL)
+ unset(_CMAKE_PROCESSING_LANGUAGE)
+
++if(CMAKE_Fortran_XL_CPP)
++  set(_SET_CMAKE_Fortran_XL_CPP
++    "set(CMAKE_Fortran_XL_CPP \"${CMAKE_Fortran_XL_CPP}\")")
++endif()
++
+ if(CMAKE_Fortran_COMPILER_ARCHITECTURE_ID)
+   set(_SET_CMAKE_Fortran_COMPILER_ARCHITECTURE_ID
+     "set(CMAKE_Fortran_COMPILER_ARCHITECTURE_ID ${CMAKE_Fortran_COMPILER_ARCHITECTURE_ID})")
+diff --git a/Modules/CMakeFortranCompiler.cmake.in b/Modules/CMakeFortranCompiler.cmake.in
+index ae7b73ac4a..34f44aa542 100644
+--- a/Modules/CMakeFortranCompiler.cmake.in
++++ b/Modules/CMakeFortranCompiler.cmake.in
+@@ -6,6 +6,7 @@ set(CMAKE_Fortran_COMPILER_WRAPPER "@CMAKE_Fortran_COMPILER_WRAPPER@")
+ set(CMAKE_Fortran_PLATFORM_ID "@CMAKE_Fortran_PLATFORM_ID@")
+ set(CMAKE_Fortran_SIMULATE_ID "@CMAKE_Fortran_SIMULATE_ID@")
+ set(CMAKE_Fortran_SIMULATE_VERSION "@CMAKE_Fortran_SIMULATE_VERSION@")
++@_SET_CMAKE_Fortran_XL_CPP@
+ @_SET_CMAKE_Fortran_COMPILER_ARCHITECTURE_ID@
+ @SET_MSVC_Fortran_ARCHITECTURE_ID@
+ set(CMAKE_AR "@CMAKE_AR@")
+diff --git a/Modules/Compiler/XL-Fortran.cmake b/Modules/Compiler/XL-Fortran.cmake
+index c4fb09712a..1683dff4f0 100644
+--- a/Modules/Compiler/XL-Fortran.cmake
++++ b/Modules/Compiler/XL-Fortran.cmake
+@@ -18,3 +18,7 @@ string(APPEND CMAKE_Fortran_FLAGS_INIT " -qthreaded -qhalt=e")
+ # xlf: 1501-214 (W) command option E reserved for future use - ignored
+ set(CMAKE_Fortran_CREATE_PREPROCESSED_SOURCE)
+ set(CMAKE_Fortran_CREATE_ASSEMBLY_SOURCE)
++
++set(CMAKE_Fortran_PREPROCESS_SOURCE
++  "<CMAKE_Fortran_COMPILER> <DEFINES> <INCLUDES> <FLAGS> -qpreprocess -qnoobject -qsuppress=1517-020 -tF -B \"${CMAKE_CURRENT_LIST_DIR}/XL-Fortran/\" -WF,--cpp,\"${CMAKE_Fortran_XL_CPP}\",--out,<PREPROCESSED_SOURCE> <SOURCE>"
++  )
+diff --git a/Modules/Compiler/XL-Fortran/cpp b/Modules/Compiler/XL-Fortran/cpp
+new file mode 100755
+index 0000000000..1fd62c26a0
+--- /dev/null
++++ b/Modules/Compiler/XL-Fortran/cpp
+@@ -0,0 +1,29 @@
++#!/usr/bin/env bash
++
++# Source file.
++src="$(printf %q "$1")"
++shift
++
++# Output file the compiler expects.
++out="$(printf %q "$1")"
++shift
++
++# Create the file the compiler expects.  It will check syntax.
++>"$out"
++
++cpp='cpp'
++opts=''
++while test "$#" != 0; do
++    case "$1" in
++    # Extract the option for the path to cpp.
++    --cpp) shift; cpp="$(printf %q "$1")" ;;
++    # Extract the option for our own output file.
++    --out) shift; out="$(printf %q "$1")" ;;
++    # Collect the rest of the command line.
++    *) opts="$opts $(printf %q "$1")" ;;
++    esac
++    shift
++done
++
++# Execute the real preprocessor tool.
++eval "exec $cpp $src $out $opts"
+-- 
+2.24.1
+

.github/containers/x86_64-broadwell-cuda11.6.1/repo/packages/cmake/fujitsu_add_linker_option.patch

@@ -0,0 +1,10 @@
+--- spack-src/Modules/FortranCInterface/Verify/CMakeLists.txt.org	2020-06-05 15:54:59.559043595 +0900
++++ spack-src/Modules/FortranCInterface/Verify/CMakeLists.txt	2020-06-05 15:58:28.150062948 +0900
+@@ -4,6 +4,7 @@
+ cmake_minimum_required(VERSION ${CMAKE_VERSION})
+ project(VerifyFortranC C Fortran)
+
++set (CMAKE_EXE_LINKER_FLAGS "--linkfortran")
+ option(VERIFY_CXX "Whether to verify C++ and Fortran" OFF)
+ if(VERIFY_CXX)
+   enable_language(CXX)