From c0cd436b58729e50c7d2b532f426df1419ab79d3 Mon Sep 17 00:00:00 2001
From: G-Ragghianti <ragghianti@icl.utk.edu>
Date: Fri, 26 Apr 2024 13:14:25 -0400
Subject: [PATCH 1/8] adding spack testing to ci

---
 .github/workflows/main.yml            |  2 +-
 .github/workflows/spack.sh            | 65 +++++++++++++++++++++++++++
 .github/workflows/spack.yml           | 33 ++++++++++++++
 .github/workflows/spack_packages_yaml | 39 ++++++++++++++++
 unit_test/test_internal_blas.cc       |  1 +
 5 files changed, 139 insertions(+), 1 deletion(-)
 create mode 100755 .github/workflows/spack.sh
 create mode 100644 .github/workflows/spack.yml
 create mode 100644 .github/workflows/spack_packages_yaml

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 98ff3d69c..598c4f243 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -24,7 +24,7 @@ jobs:
       fail-fast: false
     runs-on: ${{ matrix.device }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Configure
         run: .github/workflows/configure.sh ${{matrix.maker}} ${{matrix.device}}
       - name: Build
diff --git a/.github/workflows/spack.sh b/.github/workflows/spack.sh
new file mode 100755
index 000000000..6c7975ec9
--- /dev/null
+++ b/.github/workflows/spack.sh
@@ -0,0 +1,65 @@
+#!/bin/bash -e
+
+bstage=$1
+device=$2
+
+set +x
+trap 'echo "# $BASH_COMMAND"' DEBUG
+
+source /etc/profile
+
+COMPILER=gcc@11
+MPI=intel-oneapi-mpi
+BLAS=intel-oneapi-mkl
+
+# Get the system-install ROCM version
+ROCM_VER=$(ls /opt | grep rocm- | sed s/rocm-//)
+# Get the system-install CUDA version
+CUDA_VER=$(cd /usr/local && ls -d cuda-*.* | sed "s/cuda-//")
+
+if [ "${bstage}" = "deps" ]; then
+  git clone https://github.com/spack/spack
+  cp .github/workflows/spack_packages_yaml spack/etc/spack/packages.yaml
+  # Use $ROCM_VER in the spack package config
+  sed -i "s/ROCMVER/$ROCM_VER/" spack/etc/spack/packages.yaml
+  # Use $CUDA_VER in the spack package config
+  sed -i "s/CUDAVER/$CUDA_VER/" spack/etc/spack/packages.yaml
+fi
+
+export HOME=$(pwd)
+source spack/share/spack/setup-env.sh
+module load $COMPILER
+spack compiler find --scope=site
+
+if [ "${device}" = "cpu" ]; then
+  SPEC=""
+elif [ "${device}" = "gpu_nvidia" ]; then
+  ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | sed -e 's/\.//')
+  SPEC="+cuda cuda_arch=$ARCH"
+elif [ "${device}" = "gpu_amd" ]; then
+  TARGET=$(rocminfo | grep Name | grep gfx | head -1 | awk '{print $2}')
+  SPEC="+rocm amdgpu_target=$TARGET ^hip@$ROCM_VER"
+else
+  SPEC="+sycl"
+  COMPILER=oneapi
+  module load intel-oneapi-compilers
+  spack compiler find --scope=site
+fi
+SPEC="slate@master $SPEC %$COMPILER ^$MPI ^$BLAS"
+echo SPEC=$SPEC
+
+if [ "${bstage}" = "deps" ]; then
+  # Change the stage directory so we can find it later
+  spack config --scope=site add config:build_stage:`pwd`/spack-stage
+  spack spec $SPEC
+  spack install --only=dependencies --fail-fast $SPEC
+elif [ "${bstage}" = "build" ]; then
+  spack dev-build -i $SPEC
+elif [ "${bstage}" = "test" ]; then
+  spack uninstall -y slate
+  ls -l spack-stage
+  spack dev-build --test=root $SPEC 
+elif [ "${bstage}" = "smoke" ]; then
+  spack test run slate
+fi
+
diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml
new file mode 100644
index 000000000..c702cbc91
--- /dev/null
+++ b/.github/workflows/spack.yml
@@ -0,0 +1,33 @@
+name: Spack
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ master ]
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  slate_spack:
+    timeout-minutes: 360
+    strategy:
+      matrix:
+        device: [gpu_nvidia, gpu_amd, cpu]
+      fail-fast: false
+    runs-on: ${{ matrix.device }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build Dependencies
+        run: .github/workflows/spack.sh deps ${{matrix.device}}
+      - name: Build SLATE
+        run: .github/workflows/spack.sh build ${{matrix.device}}
+      - name: Pre-install Test
+        run: .github/workflows/spack.sh test ${{matrix.device}}
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: slate spack ${{matrix.device}} pre-install-test
+          path: spack-stage/install-time-test-log.txt
+      - name: Post-install Test
+        run: .github/workflows/spack.sh smoke ${{matrix.device}}
diff --git a/.github/workflows/spack_packages_yaml b/.github/workflows/spack_packages_yaml
new file mode 100644
index 000000000..e4d0c99fc
--- /dev/null
+++ b/.github/workflows/spack_packages_yaml
@@ -0,0 +1,39 @@
+packages:
+  all:
+    target: ['x86_64']
+  cuda:
+    buildable: false
+    externals:
+    - spec: cuda@CUDAVER
+      prefix: /usr/local/cuda-CUDAVER/
+  hip:
+    buildable: false
+    externals:
+    - spec: hip@ROCMVER
+      prefix: /opt/rocm-ROCMVER/
+  rocblas:
+    buildable: false
+    externals:
+    - spec: rocblas@ROCMVER
+      prefix: /opt/rocm-ROCMVER/rocblas/
+  llvm-amdgpu:
+    buildable: false
+    externals:
+    - spec: llvm-amdgpu@ROCMVER
+      prefix: /opt/rocm-ROCMVER
+  comgr:
+    buildable: false
+    externals:
+    - spec: comgr@ROCMVER
+      prefix: /opt/rocm-ROCMVER
+  rocsolver:
+    buildable: false
+    externals:
+    - spec: rocsolver@ROCMVER
+      prefix: /opt/rocm-ROCMVER/rocsolver
+  hsa-rocr-dev:
+    buildable: false
+    externals:
+    - spec: hsa-rocr-dev@ROCMVER
+      prefix: /opt/rocm-ROCMVER/hsa/
+
diff --git a/unit_test/test_internal_blas.cc b/unit_test/test_internal_blas.cc
index a2cbac79c..fed5b1c3e 100644
--- a/unit_test/test_internal_blas.cc
+++ b/unit_test/test_internal_blas.cc
@@ -53,6 +53,7 @@ const char* target_name(slate::Target target)
         case slate::Target::Devices:   return "Devices";
         default: assert(false);
     }
+    return "Error";
 }
 
 // -----------------------------------------------------------------------------

From 52f79fd2a47ed6af7cddc322b9d6508a58572513 Mon Sep 17 00:00:00 2001
From: G-Ragghianti <ragghianti@icl.utk.edu>
Date: Fri, 26 Apr 2024 13:18:45 -0400
Subject: [PATCH 2/8] setting location of test log

---
 .github/workflows/spack.sh  | 1 -
 .github/workflows/spack.yml | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/spack.sh b/.github/workflows/spack.sh
index 6c7975ec9..826aae1a9 100755
--- a/.github/workflows/spack.sh
+++ b/.github/workflows/spack.sh
@@ -57,7 +57,6 @@ elif [ "${bstage}" = "build" ]; then
   spack dev-build -i $SPEC
 elif [ "${bstage}" = "test" ]; then
   spack uninstall -y slate
-  ls -l spack-stage
   spack dev-build --test=root $SPEC 
 elif [ "${bstage}" = "smoke" ]; then
   spack test run slate
diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml
index c702cbc91..f8b3b5517 100644
--- a/.github/workflows/spack.yml
+++ b/.github/workflows/spack.yml
@@ -28,6 +28,6 @@ jobs:
         if: always()
         with:
           name: slate spack ${{matrix.device}} pre-install-test
-          path: spack-stage/install-time-test-log.txt
+          path: spack-stage/github/spack-stage-slate-master-*/install-time-test-log.txt
       - name: Post-install Test
         run: .github/workflows/spack.sh smoke ${{matrix.device}}

From 039fe3e1ba67d4d2533a93e16497568e84e364d2 Mon Sep 17 00:00:00 2001
From: G-Ragghianti <ragghianti@icl.utk.edu>
Date: Fri, 26 Apr 2024 13:59:23 -0400
Subject: [PATCH 3/8] integrating output of test results into github log

---
 .github/workflows/spack.sh  | 2 +-
 .github/workflows/spack.yml | 5 -----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/spack.sh b/.github/workflows/spack.sh
index 826aae1a9..8a9ca1794 100755
--- a/.github/workflows/spack.sh
+++ b/.github/workflows/spack.sh
@@ -57,7 +57,7 @@ elif [ "${bstage}" = "build" ]; then
   spack dev-build -i $SPEC
 elif [ "${bstage}" = "test" ]; then
   spack uninstall -y slate
-  spack dev-build --test=root $SPEC 
+  spack dev-build --test=root $SPEC || cat spack-stage/github/spack-stage-slate-master-*/install-time-test-log.txt
 elif [ "${bstage}" = "smoke" ]; then
   spack test run slate
 fi
diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml
index f8b3b5517..59887dd5a 100644
--- a/.github/workflows/spack.yml
+++ b/.github/workflows/spack.yml
@@ -24,10 +24,5 @@ jobs:
         run: .github/workflows/spack.sh build ${{matrix.device}}
       - name: Pre-install Test
         run: .github/workflows/spack.sh test ${{matrix.device}}
-      - uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: slate spack ${{matrix.device}} pre-install-test
-          path: spack-stage/github/spack-stage-slate-master-*/install-time-test-log.txt
       - name: Post-install Test
         run: .github/workflows/spack.sh smoke ${{matrix.device}}

From 276008474602fc7d117ef06bb963692846b5a7dc Mon Sep 17 00:00:00 2001
From: G-Ragghianti <ragghianti@icl.utk.edu>
Date: Fri, 26 Apr 2024 14:16:51 -0400
Subject: [PATCH 4/8] script logic fix

---
 .github/workflows/spack.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/spack.sh b/.github/workflows/spack.sh
index 8a9ca1794..1ee43f97e 100755
--- a/.github/workflows/spack.sh
+++ b/.github/workflows/spack.sh
@@ -57,7 +57,11 @@ elif [ "${bstage}" = "build" ]; then
   spack dev-build -i $SPEC
 elif [ "${bstage}" = "test" ]; then
   spack uninstall -y slate
-  spack dev-build --test=root $SPEC || cat spack-stage/github/spack-stage-slate-master-*/install-time-test-log.txt
+  spack dev-build --test=root $SPEC
+  if [ $? -gt 0 ]; then
+    cat spack-stage/github/spack-stage-slate-master-*/install-time-test-log.txt
+    exit 22
+  fi
 elif [ "${bstage}" = "smoke" ]; then
   spack test run slate
 fi

From 440ed15ebcc920684ad325c82083709a0e115538 Mon Sep 17 00:00:00 2001
From: G-Ragghianti <ragghianti@icl.utk.edu>
Date: Fri, 26 Apr 2024 14:42:16 -0400
Subject: [PATCH 5/8] typo

---
 .github/workflows/spack.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/spack.sh b/.github/workflows/spack.sh
index 1ee43f97e..5c05d27f1 100755
--- a/.github/workflows/spack.sh
+++ b/.github/workflows/spack.sh
@@ -57,11 +57,10 @@ elif [ "${bstage}" = "build" ]; then
   spack dev-build -i $SPEC
 elif [ "${bstage}" = "test" ]; then
   spack uninstall -y slate
-  spack dev-build --test=root $SPEC
-  if [ $? -gt 0 ]; then
+  spack dev-build --test=root $SPEC || (
     cat spack-stage/github/spack-stage-slate-master-*/install-time-test-log.txt
     exit 22
-  fi
+  )
 elif [ "${bstage}" = "smoke" ]; then
   spack test run slate
 fi

From 18bd29a6890e3c6782cf25490ee6740babc39368 Mon Sep 17 00:00:00 2001
From: G-Ragghianti <ragghianti@icl.utk.edu>
Date: Tue, 14 May 2024 12:06:19 -0400
Subject: [PATCH 6/8] removing tripper for pull requests

---
 .github/workflows/spack.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml
index 59887dd5a..912826435 100644
--- a/.github/workflows/spack.yml
+++ b/.github/workflows/spack.yml
@@ -2,8 +2,6 @@ name: Spack
 
 # Controls when the workflow will run
 on:
-  pull_request:
-    branches: [ master ]
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 

From 58ad24cdb7a27e8a4b589f0697de8167df2e6c3d Mon Sep 17 00:00:00 2001
From: G-Ragghianti <ragghianti@icl.utk.edu>
Date: Wed, 5 Jun 2024 08:23:43 -0400
Subject: [PATCH 7/8] temporarily enabling spack workflow run

---
 .github/workflows/spack.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml
index 912826435..f05ec582e 100644
--- a/.github/workflows/spack.yml
+++ b/.github/workflows/spack.yml
@@ -2,6 +2,10 @@ name: Spack
 
 # Controls when the workflow will run
 on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 

From dc3114cb5322a732a55f0a5913bc1d67d5625248 Mon Sep 17 00:00:00 2001
From: G-Ragghianti <ragghianti@icl.utk.edu>
Date: Wed, 5 Jun 2024 09:34:56 -0400
Subject: [PATCH 8/8] removing temporary trigger for PR testing

---
 .github/workflows/spack.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml
index f05ec582e..912826435 100644
--- a/.github/workflows/spack.yml
+++ b/.github/workflows/spack.yml
@@ -2,10 +2,6 @@ name: Spack
 
 # Controls when the workflow will run
 on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch: