Merge pull request #1135 from berquist/smoke-test

Smoke test bamboo changes with GitHub Actions

.github/dependabot.yml

@@ -0,0 +1,7 @@
+---
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+---
+name: ci
+
+# yamllint disable-line rule:truthy
+on:
+  push:
+  pull_request:
+
+concurrency:
+  # yamllint disable-line rule:line-length
+  group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
+  cancel-in-progress: true
+
+jobs:
+  prechecks:
+    uses: ./.github/workflows/pre-commit.yml
+  all-prechecks:
+    needs: [prechecks]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Success
+        run: "true"
+  tests:
+    needs: [prechecks]
+    uses: ./.github/workflows/test.yml
+  all:
+    needs: [tests]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Success
+        run: "true"

.github/workflows/pre-commit.yml

@@ -1,16 +1,20 @@
 ---
-# In the absence of using the pre-commit.ci service...
 name: pre-commit
 
 # yamllint disable-line rule:truthy
 on:
-  pull_request:
-  push:
+  workflow_call:
+  workflow_dispatch:
+
+concurrency:
+  # yamllint disable-line rule:line-length
+  group: style-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
+  cancel-in-progress: true
 
 jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
-      - uses: pre-commit/[email protected].0
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+      - uses: pre-commit/[email protected].1

.github/workflows/test.yml

@@ -0,0 +1,111 @@
+---
+# yamllint disable rule:line-length
+name: run bamboo
+
+# yamllint disable-line rule:truthy
+on:
+  workflow_call:
+  workflow_dispatch:
+
+concurrency:
+  group: test-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
+  cancel-in-progress: true
+
+jobs:
+  run-bamboo:
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - macos-14
+          - ubuntu-24.04
+        build-type:
+          - sstmainline_config
+          - sstmainline_coreonly_config
+          - sstmainline_config_no_mpi
+          - sstmainline_coreonly_config_no_mpi
+          - sstmainline_config_make_dist_test
+          - sstmainline_config_core_make_dist_test
+        exclude:
+          - os: macos-14
+            build-type: sstmainline_config
+          - os: macos-14
+            build-type: sstmainline_coreonly_config
+          - os: macos-14
+            build-type: sstmainline_config_make_dist_test
+          - os: macos-14
+            build-type: sstmainline_config_core_make_dist_test
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: install system-level dependencies (macOS)
+        run: |
+          brew install \
+              autoconf \
+              automake \
+              coreutils \
+              doxygen \
+              libtool \
+              lmod \
+              ncurses \
+              pygments
+          python -m pip install blessings
+          # echo "source /opt/homebrew/opt/lmod/init/profile" >> ~/.bashrc
+        if: matrix.os == 'macos-14'
+      - name: install system-level dependencies (Ubuntu)
+        run: |
+          export DEBIAN_FRONTEND=noninteractive
+          sudo apt-get -y update
+          sudo apt-get -y install \
+              autoconf \
+              automake \
+              doxygen \
+              libopenmpi-dev \
+              libtool-bin \
+              lmod \
+              python3-blessings \
+              python3-pygments
+          # echo "source /usr/share/lmod/lmod/init/bash" >> ~/.bashrc
+        if: matrix.os == 'ubuntu-24.04'
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          create-symlink: true
+          key: ${{ matrix.os }}-${{ matrix.build-type }}
+          verbose: 2
+      - name: set env and prepare dir structure
+        run: |
+          # Keep this common in order to increase ccache effectiveness
+          # sst_deps_user_dir="${GITHUB_WORKSPACE}/../${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER}_${GITHUB_RUN_ATTEMPT}"
+          sst_deps_user_dir="${GITHUB_WORKSPACE}/../sst-deps-user-dir"
+          mkdir -p "${sst_deps_user_dir}"
+          SST_DEPS_USER_DIR="$(realpath "${sst_deps_user_dir}")"
+          echo "SST_DEPS_USER_DIR=${SST_DEPS_USER_DIR}" >> "${GITHUB_ENV}"
+      - name: fetch elements dependencies
+        run: |
+          pristine="${SST_DEPS_USER_DIR}"/sstDeps/src/pristine
+          mkdir -p "${pristine}"
+          cd "${pristine}"
+          wget https://github.com/umd-memsys/DRAMSim2/archive/refs/tags/v2.2.2.tar.gz
+      - name: run bamboo
+        run: |
+          # needed for ncurses part of interactive sst-info
+          export TERM=dumb
+          MAKEFLAGS="-j$(nproc)"
+          export MAKEFLAGS
+          if [[ "$(uname)" == "Darwin" ]]; then
+              export PMIX_MCA_gds=hash
+          fi
+          export SST_DEPS_USER_MODE=1
+          mkdir -p "${SST_DEPS_USER_DIR}/devel/trunk"
+          mv "${GITHUB_WORKSPACE}" "${SST_DEPS_USER_DIR}/devel/sqe"
+          cd "${SST_DEPS_USER_DIR}/devel/trunk"
+          ../sqe/buildsys/bamboo.sh ${{ matrix.build-type }} none none none none none
+          ccache -s -vv
+      - uses: actions/upload-artifact@v4
+        with:
+          name: test-output-${{ matrix.os }}-${{ matrix.build-type }}
+          path: sst_test_outputs
+          compression-level: 9
+        if: '!cancelled()'
+# yamllint enable rule:line-length

.pre-commit-config.yaml

@@ -16,8 +16,9 @@ repos:
       - id: fix-byte-order-marker
       - id: mixed-line-ending
         args: [--fix, "lf"]
-      - id: no-commit-to-branch
-        args: [--branch, main, --branch, master, --branch, devel]
+      # TODO disable until we can skip on merge commits to main/master/devel
+      # - id: no-commit-to-branch
+      #   args: [--branch, main, --branch, master, --branch, devel]
       - id: trailing-whitespace
         exclude: |
           (?x)(

buildsys/bamboo.sh

@@ -445,6 +445,28 @@ getconfig() {
             externalelementConfigStr="${NOBUILD}"
             junoConfigStr="${NOBUILD}"
             ;;
+
+        sstmainline_coreonly_config_no_mpi)
+            #-----------------------------------------------------------------
+            # sstmainline_coreonly_config
+            #     This option used for configuring SST with supported stabledevel deps
+            #-----------------------------------------------------------------
+            if [[ ${MPIHOME:+isSet} == isSet ]] ; then
+                echo ' ' ; echo " Test is flawed!  MPI module is loaded!" ; echo ' '
+                exit 1
+            fi
+            export | egrep SST_DEPS_
+            coreMiscEnv="${cc_environment}"
+            elementsMiscEnv="${cc_environment}"
+            depsStr="-r none" # Dependencies only needed for elements
+            setConvenienceVars "$depsStr"
+            coreConfigStr="$corebaseoptions $coreMiscEnv --disable-mpi"
+            elementsConfigStr="${NOBUILD}"
+            macroConfigStr="${NOBUILD}"
+            externalelementConfigStr="${NOBUILD}"
+            junoConfigStr="${NOBUILD}"
+            ;;
+
         sstmainline_config_linux_with_cuda)
             #-----------------------------------------------------------------
             # sstmainline_config_linux_with_cuda
@@ -675,9 +697,6 @@ getconfig() {
             ;;
 
         *)
-            #-----------------------------------------------------------------
-            #  Unrecognized Scenario,  This is an error in the bamboo code
-            #-----------------------------------------------------------------
             echo ' ' ; echo "Unrecognized Scenario,  This is an error in the bamboo code"
             echo " UNRECOGNIZED:   ${1}"
             exit 1
@@ -697,6 +716,31 @@ getconfig() {
    fi
 }
 
+set_up_environment_modules() {
+    # For some reason, .bashrc is not being run prior to
+    # this script. Kludge initialization of modules.
+
+    echo "Attempt to initialize the modules utility"
+
+    locations=(/etc/profile.modules /etc/profile.d/modules.sh /opt/homebrew/opt/lmod/init/profile /usr/share/lmod/lmod/init/profile)
+
+    for location in "${locations[@]}"; do
+        if [ -r "${location}" ]; then
+            # shellcheck disable=SC1090
+            source "${location}"
+            echo "bamboo.sh: loaded ${location}"
+        fi
+    done
+
+    echo "Testing modules utility via ModuleEx..."
+    echo "ModuleEx avail"
+    ModuleEx avail
+    if [ $? -ne 0 ] ; then
+        echo " ModuleEx Failed"
+        exit 1
+    fi
+}
+
 #-------------------------------------------------------------------------
 # Function: linuxSetMPI
 # Description:
@@ -710,28 +754,7 @@ getconfig() {
 #   Return value:
 linuxSetMPI() {
 
-    # For some reason, .bashrc is not being run prior to
-    # this script. Kludge initialization of modules.
-
-    echo "Attempt to initialize the modules utility.  Look for modules init file in 1 of 2 places"
-
-    echo "Location 1: ls -l /etc/profile.modules"
-    echo "Location 2: ls -l /etc/profile.d/modules.sh"
-    if [ -r /etc/profile.modules ] ; then
-        source /etc/profile.modules
-        echo "bamboo.sh: loaded /etc/profile.modules"
-    elif [ -r /etc/profile.d/modules.sh ] ; then
-        source /etc/profile.d/modules.sh
-        echo "bamboo.sh: loaded /etc/profile.d/modules"
-    fi
-
-   echo "Testing modules utility via ModuleEx..."
-   echo "ModuleEx avail"
-   ModuleEx avail
-   if [ $? -ne 0 ] ; then
-       echo " ModuleEx Failed"
-       exit 1
-   fi
+    set_up_environment_modules
 
    # build MPI selector
    if [[ "$2" =~ openmpi.* ]]
@@ -754,10 +777,7 @@ linuxSetMPI() {
        elif [[ "$3" =~ intel.* ]]
        then
            ModuleEx load intel/${4}
-           if [[ "$3" == *intel-15* ]] ; then
-               ModuleEx load gcc/gcc-4.8.1
-           fi
-
+           echo "LOADED intel/${4} compiler"
        fi
    fi
 
@@ -853,28 +873,19 @@ darwinSetMPI() {
     macosVersion=`echo ${macosVersionFull} | awk -F. '{print $1 "." $2 }'`
     echo "  ******************* macosVersion= $macosVersion "
 
-
     # macports or hybrid clang/macports
     PATH="/opt/local/bin:/usr/local/bin:$PATH"
     export PATH
 
     # Point to aclocal per instructions from sourceforge on MacOSX installation
     export ACLOCAL_FLAGS="-I/opt/local/share/aclocal $ACLOCAL_FLAGS"
-    echo $ACLOCAL_FLAGS
+    echo "ACLOCAL_FLAGS=${ACLOCAL_FLAGS}"
 
-    # Initialize modules for Jenkins (taken from $HOME/.bashrc on Mac)
-    if [ -f /etc/profile.modules ]
-    then
-        source /etc/profile.modules
-        echo "bamboo.sh: loaded /etc/profile.modules."
-        # put any module loads here
-        echo "bamboo.sh: Loading Modules for MacOSX"
-        ldModules_MacOS_Clang $compiler $2  # any Xcode
+    set_up_environment_modules
 
-    else
-        echo "ERROR: unable to locate /etc/profile.modules - cannot load modules"
-        exit
-    fi
+    # Initialize modules for Jenkins (taken from $HOME/.bashrc on Mac)
+    echo "bamboo.sh: Loading Modules for MacOSX"
+    ldModules_MacOS_Clang $compiler $2  # any Xcode
 
     echo "bamboo.sh: MacOS build."
     echo "bamboo.sh:   MPI = $2"
@@ -1772,6 +1783,10 @@ export SST_BUILD_TYPE=""
 # - If false, use each given branch as-is.
 export SST_TEST_MERGE=${SST_TEST_MERGE:-true}
 
+# Number of threads and (MPI) ranks to run tests with (not compile with)
+export SST_MULTI_THREAD_COUNT=${SST_MULTI_THREAD_COUNT:-1}
+export SST_MULTI_RANK_COUNT=${SST_MULTI_RANK_COUNT:-1}
+
 cloneOtherRepos
 
 # Load test definitions
@@ -1844,7 +1859,7 @@ else
     echo "bamboo.sh: KERNEL = $kernel"
 
     case ${build_type} in
-        sstmainline_config|sstmainline_coreonly_config|sstmainline_config_no_gem5|sstmainline_config_clang_core_only|sstmainline_config_macosx_no_gem5|sstmainline_config_no_mpi|sstmainline_config_make_dist_test|sstmainline_config_core_make_dist_test|documentation|sstmainline_config_all|sstmainline_config_linux_with_cuda|sstmainline_config_linux_with_cuda_no_mpi|sst-macro_withsstcore_mac|sst-macro_nosstcore_mac|sst-macro_withsstcore_linux|sst-macro_nosstcore_linux|sst_Macro_make_dist)
+        sstmainline_config|sstmainline_coreonly_config|sstmainline_coreonly_config_no_mpi|sstmainline_config_no_gem5|sstmainline_config_clang_core_only|sstmainline_config_macosx_no_gem5|sstmainline_config_no_mpi|sstmainline_config_make_dist_test|sstmainline_config_core_make_dist_test|documentation|sstmainline_config_all|sstmainline_config_linux_with_cuda|sstmainline_config_linux_with_cuda_no_mpi|sst-macro_withsstcore_mac|sst-macro_nosstcore_mac|sst-macro_withsstcore_linux|sst-macro_nosstcore_linux|sst_Macro_make_dist)
             #   Save Parameters in case they are needed later
             SST_DIST_MPI=${mpi_type}
             _UNUSED="none"

buildsys/deps/include/depsDefinitions.sh

@@ -64,6 +64,10 @@ export SST_DEPS_SRC=$SST_BASE/sstDeps/src
 # be tar.gz files, zipfiles, or other archive formats. This can be
 # thought of as the "pre-staging" area for SST dependencies.
 export SST_DEPS_SRC_PRISTINE=${SST_DEPS_SRC}/pristine
+if [[ ! -d "${SST_DEPS_SRC_PRISTINE}" ]]; then
+    echo "SST_DEPS_SRC_PRISTINE doesn't exist: ${SST_DEPS_SRC_PRISTINE}"
+    exit 1
+fi
 
 # Location where pristine source files are unarchived and patched as a
 # precursor to build and installation. This can be thought of as the