Sayef/mds profiling dev

.gitignore

@@ -6,5 +6,6 @@ install/*
 *__pycache__*
 *nvblas.conf
 spack-*
+spack_*
 .vscode/
 coinhsl-*

.gitlab/ci/crusher.gitlab-ci.yml

@@ -0,0 +1,105 @@
+# Crusher Variables
+stages:
+  - build
+  - test
+
+workflow:
+  rules:
+    # Only run if we are running manually or through a schedule
+    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - when: never
+
+.crusher_build_variables:
+  variables:
+    WORKDIR: /gpfs/alpine/csc359/proj-shared/ci/${CI_PIPELINE_ID}
+
+.crusher_test_variables:
+  variables:
+    # Don't clone for test jobs
+    GIT_STRATEGY: none
+    # Only for slurm tagged jobs...
+    SCHEDULER_PARAMETERS: "-N 1 -A CSC359_crusher --time=60"
+    WORKDIR: /gpfs/alpine/csc359/proj-shared/ci/${CI_PIPELINE_ID}
+
+# Crusher Jobs
+Crusher Build:
+  stage: build
+  tags: [crusher, shell]
+  script:
+    - mkdir -p "$WORKDIR"
+    - cp -r . "$WORKDIR"
+    - cd "$WORKDIR"
+    - export srcdir=$WORKDIR builddir=$WORKDIR/build installdir=$WORKDIR/install
+    - MY_CLUSTER=crusher ./buildsystem/build.sh --build-only --job=clang-hip
+    - res=$?
+    - exit $res
+  extends: .crusher_build_variables
+
+Crusher Test:
+  stage: test
+  dependencies: ["Crusher Build"]
+  tags: [crusher, slurm]
+  script:
+    - cd "$WORKDIR"
+    - export srcdir=$WORKDIR builddir=$WORKDIR/build installdir=$WORKDIR/install
+      # Logger test failing on Crusher
+    - export CTESTARGS="--output-on-failure -E Python"
+    - MY_CLUSTER=crusher ./buildsystem/build.sh --test-only --job=clang-hip
+    - res=$?
+    - exit $res
+  after_script:
+    - cd "$WORKDIR/.."
+    - rm -rf "$WORKDIR"
+  extends: .crusher_test_variables
+
+Crusher Python Test:
+  stage: test
+  dependencies: ["Crusher Build"]
+  variables:
+    # Don't clone for test jobs
+    GIT_STRATEGY: none
+  allow_failure: true
+  tags: [crusher, slurm]
+  script:
+    - cd "$WORKDIR"
+    - export srcdir=$WORKDIR builddir=$WORKDIR/build installdir=$WORKDIR/install
+    - export CTESTARGS="--output-on-failure -R Python"
+    - MY_CLUSTER=crusher ./buildsystem/build.sh --test-only --job=clang-hip
+    - res=$?
+    - exit $res
+  extends: .crusher_test_variables
+# ---
+
+# Reporting Crusher Status to PNNL
+.report-status:
+  variables:
+    GIT_STRATEGY: none
+  tags: [crusher, shell]
+  script:
+    # For complete details on the GitLab API please see:
+    # https://docs.gitlab.com/ee/api/commits.html#post-the-build-status-to-a-commit
+    # Make sure to create the token with Developer level access and API scope
+    - curl -X POST -H @${GITLAB_CURL_HEADERS} https://gitlab.pnnl.gov/api/v4/projects/251/statuses/${CI_COMMIT_SHA}?state=${CI_JOB_NAME}\&name=Crusher\&target_url=${CI_PIPELINE_URL}
+  environment:
+    name: reporting-gitlab
+
+pending:
+  extends: .report-status
+  stage: .pre
+
+# Post running status to show builds at least passed, tests are running
+running:
+  extends: .report-status
+  stage: test
+
+success:
+  extends: .report-status
+  stage: .post
+
+failed:
+  stage: .post
+  extends: .report-status
+  rules:
+    - when: on_failure
+# ---

.gitmodules

@@ -9,4 +9,4 @@
 	url = https://github.com/pybind/pybind11.git
 [submodule "tpl/spack"]
 	path = tpl/spack
-	url = https://github.com/CameronRutherford/spack.git
+	url = https://github.com/spack/spack.git

CMakeLists.txt

@@ -99,7 +99,7 @@ set(EXAGO_CTEST_LAUNCH_COMMAND
     CACHE STRING "Command to use when launching tests"
 )
 
-option(EXAGO_ENABLE_LOGGING "Enable internal logging" OFF)
+option(EXAGO_ENABLE_LOGGING "Enable internal logging" ON)
 
 # When building with HIP support, OpenMP is not supported.
 if(EXAGO_ENABLE_RAJA AND EXAGO_ENABLE_HIP)

applications/sopflow_main.cpp

@@ -120,6 +120,7 @@ int main(int argc, char **argv) {
   ierr = SOPFLOWDestroy(&sopflow);
   CHKERRQ(ierr);
 
+  MPI_Barrier(PETSC_COMM_WORLD);
   ExaGOFinalize();
   // PetscFinalize();
   return 0;

buildsystem/README.md

@@ -18,6 +18,8 @@ Each folder which builds a configuration of ExaGO should have a following:
 Platforms:
 
 - Crusher
+- Frontier
+- Incline
 
 Description:
 

buildsystem/build.sh

@@ -197,29 +197,50 @@ then
 fi
 
 # Correctly identify clusters based on hostname
+# Valid cluster suggestions are spack based,
+# however you can have any job with a valid:
+# buildsystem/<job-name>/<platform>Variables.sh
 case $MY_CLUSTER in
   newell*)
     export MY_CLUSTER=newell
     ;;
+  incline*|dmi*)
+    export MY_CLUSTER=incline
+    ;;
   dl*|deception|*fat*)
     export MY_CLUSTER=deception
     ;;
+  crusher*)
+    export MY_CLUSTER=crusher
+    ;; 
+  ascent*)
+    export MY_CLUSTER=ascent
+    ;;
+  summit*)
+    export MY_CLUSTER=summit
+    ;;
+  frontier*)
+    export MY_CLUSTER=frontier
+    ;;
   *)
-    echo "Cluster $MY_CLUSTER not identified - you'll have to set relevant variables manually."
+    echo "${MY_CLUSTER} did not match any directories in /buildsystem/spack/"
+    echo "Try one of the following platforms: "
+    echo $(ls -d ./buildsystem/spack/*/ | tr '\n' '\0' | xargs -0 -n 1 basename )
+    exit 1 
     ;;
 esac
 
 ulimit -s unlimited || echo 'Could not set stack size to unlimited.'
 ulimit -l unlimited || echo 'Could not set max locked memory to unlimited.'
 
-. /etc/profile.d/modules.sh
-module purge
-
 varfile="$SRCDIR/buildsystem/$JOB/$(echo $MY_CLUSTER)Variables.sh"
 
 if [[ -f "$varfile" ]]; then
-  source "$varfile"
-  echo Sourced system-specific variables for $MY_CLUSTER
+  # source varfile without stderr or stout if it exists, error if failure
+  set -xv
+  source $varfile || { echo "Could not source $varfile"; exit 1; }
+  # source $varfile 2>/dev/null || { echo "Could not source $varfile"; exit 1; }
+  set +xv
 fi
 
 # module list

buildsystem/clang-hip/cache.cmake

@@ -1,4 +1,4 @@
-message(STATUS "Loading CMake cache for a GCC+CUDA+MPI build")
+message(STATUS "Loading CMake cache for a GCC+HIP+MPI build")
 
 set(prefix ${CMAKE_SOURCE_DIR}/install)
 message(STATUS "Setting initial installation prefix to ${prefix}")
@@ -7,14 +7,14 @@ set(CMAKE_INSTALL_PREFIX
     CACHE PATH ""
 )
 
-set(arch 60)
-message(STATUS "Using initial cuda architecture of ${arch}")
-set(CMAKE_CUDA_ARCHITECTURES
+set(arch gfx90a)
+message(STATUS "Using initial AMD GPU target architecture of ${arch}")
+set(AMDGPU_TARGETS
     ${arch}
     CACHE STRING ""
 )
 
-message(STATUS "Building static libraries only since HiOp is static.")
+message(STATUS "Just building static libraries for now.")
 set(EXAGO_BUILD_SHARED
     OFF
     CACHE BOOL ""
@@ -30,7 +30,7 @@ set(CMAKE_BUILD_TYPE
     CACHE STRING ""
 )
 
-message(STATUS "Enabling GPU, HiOp, MPI, PETSC, and RAJA")
+message(STATUS "Enabling GPU (HIP), HiOp, MPI, PETSC, and RAJA")
 set(EXAGO_ENABLE_GPU
     ON
     CACHE BOOL ""
@@ -72,10 +72,16 @@ set(EXAGO_RUN_TESTS
     CACHE BOOL ""
 )
 
-message(STATUS "Enabling Python when building without Ipopt")
+message(STATUS "Disabling Python when building with Ipopt on Crusher")
 set(EXAGO_ENABLE_PYTHON
     OFF
     CACHE BOOL ""
 )
 
+message(STATUS "Enabling Logging")
+set(EXAGO_ENABLE_LOGGING
+    ON
+    CACHE BOOL ""
+)
+
 message(STATUS "Done setting initial CMake cache")

buildsystem/clang-hip/crusher/base.sh

@@ -3,15 +3,21 @@
 export MY_CLUSTER=crusher
 export PROJ_DIR=/autofs/nccs-svm1_proj/csc359
 
-module purge
+module reset
 
 # System modules
-module load rocm/5.2.0
-module load libfabric/1.15.0.0
+module load PrgEnv-amd
+module load craype-x86-trento
+module load craype-accel-amd-gfx90a
+module load amd/5.2.0
+module load cray-mpich/8.1.25
+module load libfabric
 
 # Consider changing to $(which clang) as for deception
-export CC=/opt/rocm-5.2.0/llvm/bin/clang
-export CXX=/opt/rocm-5.2.0/llvm/bin/clang++
-export FC=/opt/rocm-5.2.0/llvm/bin/flang
+export CC=/opt/rocm-5.2.0/llvm/bin/amdclang
+export CXX=/opt/rocm-5.2.0/llvm/bin/amdclang++
+export FC=/opt/rocm-5.2.0/llvm/bin/amdflang
 
 export EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DEXAGO_CTEST_LAUNCH_COMMAND='srun'"
+export EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DAMDGPU_TARGETS='gfx90a'"
+

buildsystem/clang-hip/frontier/base.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+export MY_CLUSTER=frontier
+export PROJ_DIR=/autofs/nccs-svm1_proj/csc359
+
+module reset
+
+# System modules
+module load PrgEnv-amd
+module load craype-x86-trento
+module load craype-accel-amd-gfx90a
+module load amd/5.2.0
+module load cray-mpich/8.1.25
+module load libfabric
+
+# Consider changing to $(which clang) as for deception
+export CC=/opt/rocm-5.2.0/llvm/bin/amdclang
+export CXX=/opt/rocm-5.2.0/llvm/bin/amdclang++
+export FC=/opt/rocm-5.2.0/llvm/bin/amdflang
+
+export EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DEXAGO_CTEST_LAUNCH_COMMAND='srun'"

buildsystem/clang-hip/frontier/frontierExago.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+export SRCDIR=${SRCDIR:-$PWD}
+
+# Platform specific configuration
+source $SRCDIR/buildsystem/clang-hip/frontier/base.sh
+
+# Spack modules
+source $SRCDIR/buildsystem/spack/frontier/modules/exago.sh
+

buildsystem/clang-hip/frontier/frontierOptimizedExago.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+export SRCDIR=${SRCDIR:-$PWD}
+
+# Platform specific configuration
+source $SRCDIR/buildsystem/clang-hip/frontier/base.sh
+
+# Spack modules
+source $SRCDIR/buildsystem/spack/frontier/modules/exago-optimized.sh
+

buildsystem/clang-hip/frontier/frontierOptimizedVariables.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+export SRCDIR=${SRCDIR:-$PWD}
+
+# Platform specific configuration
+source $SRCDIR/buildsystem/clang-hip/frontier/base.sh
+
+# Spack modules
+source $SRCDIR/buildsystem/spack/frontier/modules/optimized-dependencies.sh 
+

buildsystem/clang-hip/frontierVariables.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+export SRCDIR=${SRCDIR:-$PWD}
+
+# Platform specific configuration
+source $SRCDIR/buildsystem/clang-hip/frontier/base.sh
+
+# Spack modules
+source $SRCDIR/buildsystem/spack/frontier/modules/dependencies.sh 
+
+

buildsystem/clang-hip/incline/base.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+. /etc/profile.d/modules.sh
+
+module purge
+
+# MPI module is finnicky on incline
+modules=$(module list 2>&1)
+if echo $modules | grep -q 'openmpi'; then
+  module load gcc/8.4.0
+  module rm openmpi
+fi
+
+# System modules
+module load gcc/8.4.0
+module load openmpi/4.1.4
+module load rocm/5.3.0
+module load cmake/3.21.4
+# Must load python module!
+module load python/3.7.0
+
+# Consider changing to $(which clang) as for deception
+export CC=$(which clang)
+export CXX=$(which clang++)
+export FC=$(which gfortran)
+
+export EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DAMDGPU_TARGETS='gfx908'"
+

buildsystem/clang-hip/incline/inclineExago.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+export SRCDIR=${SRCDIR:-$PWD}
+
+# Platform specific configuration
+source $SRCDIR/buildsystem/clang-hip/incline/base.sh
+
+# Spack modules
+source $SRCDIR/buildsystem/spack/incline/modules/exago.sh

buildsystem/clang-hip/incline/inclineOptimizedExago.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+export SRCDIR=${SRCDIR:-$PWD}
+
+# Platform specific configuration
+source $SRCDIR/buildsystem/clang-hip/incline/base.sh
+
+# Spack modules
+source $SRCDIR/buildsystem/spack/incline/modules/exago-optimized.sh