neuron-ci.yml

name: NEURON CI

concurrency:
  # Don't cancel on master, creating a PR when a push workflow is already going will cancel the push workflow in favour of the PR workflow
  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_id || github.event.number && github.head_ref || github.ref_name }}
  cancel-in-progress: true

on:
  merge_group:
  push:
    branches:
      # If nothing else, this is important for the ccache logic below...
      - master
      - release/**
  pull_request:
    branches:
      - master
      - release/**
# TODO : https://github.com/neuronsimulator/nrn/issues/1063
#    paths-ignore:
#      - '**.md'
#      - '**.rst'
#      - 'docs/**'


jobs:
  ci:
    runs-on: ${{ matrix.os }}

    name: ${{ matrix.os }} - ${{ matrix.config.build_mode }} (${{ matrix.config.cmake_option }}${{ matrix.config.config_options }}${{ matrix.config.matrix_eval }}${{ matrix.config.sanitizer }})

    timeout-minutes: 75

    env:
      INSTALL_DIR: install
      SDK_ROOT: $(xcrun --sdk macosx --show-sdk-path)
      SKIP_WHEELHOUSE_REPAIR: true
      BUILD_TYPE: Release
      DESIRED_CMAKE_VERSION: 3.17
      DYNAMIC_PYTHON_CMAKE_VERSION: 3.18
      PY_MIN_VERSION: ${{ matrix.config.python_min_version || '3.9' }}
      PY_MAX_VERSION: ${{ matrix.config.python_max_version || '3.12' }}
      MUSIC_INSTALL_DIR: /opt/MUSIC
      MUSIC_VERSION: 1.2.1

    strategy:
      matrix:
        os: [macOS-13, ubuntu-20.04]
        config:
          - { matrix_eval : "CC=gcc-9 CXX=g++-9",   build_mode: "setuptools"}
          - { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", music: ON}
          - { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", python_dynamic: ON}
          - { matrix_eval : "CC=gcc-9 CXX=g++-9" ,  build_mode: "cmake", cmake_option: "-DNRN_ENABLE_CORENEURON=ON"}
          - { matrix_eval : "CC=gcc-9 CXX=g++-9",   build_mode: "cmake", cmake_option: "-DNRN_ENABLE_MPI=OFF -DNRN_ENABLE_INTERVIEWS=OFF -DNRN_ENABLE_CORENEURON=ON"}
          - { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", cmake_option: "-DNRN_ENABLE_PYTHON=OFF -DNRN_ENABLE_RX3D=OFF -DNRN_ENABLE_CORENEURON=ON"}
        include:
          - os: ubuntu-22.04
            config:
              build_mode: cmake
              cmake_option: -DNRN_ENABLE_CORENEURON=ON
                -DNRN_ENABLE_INTERVIEWS=OFF -DNMODL_SANITIZERS=undefined
              flag_warnings: ON
              sanitizer: undefined
          - os: ubuntu-22.04
            config:
              build_mode: cmake
              # TODO: CoreNEURON is only LeakSanitizer-clean if we disable MPI
              cmake_option: -DNRN_ENABLE_CORENEURON=ON
                -DNRN_ENABLE_INTERVIEWS=OFF -DNMODL_SANITIZERS=address
              # TODO: address-leak is the dream, but there are many problems,
              # including external ones from the MPI implementations
              sanitizer: address
          - os: ubuntu-24.04
            config:
              build_mode: cmake
              # Cannot use a non-instrumented OpenMP with TSan, and we don't
              # have a TSan-instrumented OpenMP runtime available.
              # TODO: debug RX3D + TSan
              cmake_option: -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_MPI=OFF
                -DCORENRN_ENABLE_OPENMP=OFF -DNRN_ENABLE_RX3D=OFF
              sanitizer: thread
          - os: macOS-13
            config:
              build_mode: cmake
              # TODO: investigate rxd test timeouts in this build and re-enable them
              cmake_option: -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_INTERVIEWS=OFF
                -DNRN_ENABLE_RX3D=OFF -DNMODL_SANITIZERS=address
              sanitizer: address
          - os: macOS-14
            config:
              build_mode: cmake
              # TODO: investigate rxd test timeouts in this build and re-enable them
              cmake_option: -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_INTERVIEWS=OFF
                -DNRN_ENABLE_RX3D=OFF -DNMODL_SANITIZERS=address
              sanitizer: thread
      fail-fast: false

    steps:

      - name: Fix kernel mmap rnd bits
        # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with
        # high-entropy ASLR in much newer kernels that GitHub runners are
        # using leading to random crashes: https://reviews.llvm.org/D148280
        run: sudo sysctl vm.mmap_rnd_bits=28
        if: matrix.os == 'ubuntu-22.04'

      - name: Setup cmake
        uses: jwlawson/actions-setup-cmake@v2
        with:
          cmake-version : ${{(matrix.config.python_dynamic || matrix.config.build_mode == 'setuptools') && env.DYNAMIC_PYTHON_CMAKE_VERSION || env.DESIRED_CMAKE_VERSION}}

      - name: Install homebrew packages
        if: startsWith(matrix.os, 'macOS')
        run: |
          # Unlink and re-link to prevent errors when GitHub macOS runner images
          # install Python outside of brew; See actions/setup-python#577 and BlueBrain/libsonata/pull/317
          brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done
          brew install ccache coreutils doxygen flex bison mpich ninja xz autoconf automake libtool
          # We use both for dynamic mpi in nrn
          brew unlink mpich
          brew install openmpi
          brew install --cask xquartz
          if [[ "${{matrix.os}}" == "macOS-14" ]]; then
            brew install cmake
            echo "$(brew --prefix)/opt/cmake/bin" >> $GITHUB_PATH
          fi
          echo "$(brew --prefix)/opt/flex/bin:$(brew --prefix)/opt/bison/bin" >> $GITHUB_PATH
          # Core https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
          if [[ "${{matrix.os}}" == "macOS-13" ]]; then
            echo CMAKE_BUILD_PARALLEL_LEVEL=4 >> $GITHUB_ENV
            echo CTEST_PARALLEL_LEVEL=4 >> $GITHUB_ENV
          else
            echo CMAKE_BUILD_PARALLEL_LEVEL=3 >> $GITHUB_ENV
            echo CTEST_PARALLEL_LEVEL=3 >> $GITHUB_ENV
          fi
          echo CI_OS_NAME=osx >> $GITHUB_ENV
        shell: bash

      - name: Install apt packages
        if: startsWith(matrix.os, 'ubuntu')
        run: |
          sudo apt-get install build-essential ccache libopenmpi-dev \
            libmpich-dev libx11-dev libxcomposite-dev mpich ninja-build \
            openmpi-bin flex libfl-dev bison libreadline-dev
          # The sanitizer builds use ubuntu 22.04
          if [[ "${{matrix.os}}" == "ubuntu-20.04" ]]; then
            sudo apt-get install g++-7 g++-8
          fi
          # Core https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
          echo CMAKE_BUILD_PARALLEL_LEVEL=4 >> $GITHUB_ENV
          echo CTEST_PARALLEL_LEVEL=4 >> $GITHUB_ENV
          echo CI_OS_NAME=linux >> $GITHUB_ENV
        shell: bash

      - uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Clone nmodl
        working-directory: ${{runner.workspace}}/nrn
        run: |
            git submodule update --init --recursive --force --depth 1 -- external/nmodl

      - name: Set up Python@${{ env.PY_MIN_VERSION }}
        if: ${{matrix.config.python_dynamic == 'ON'}}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PY_MIN_VERSION }}

      - name: Install Python@${{ env.PY_MIN_VERSION }} dependencies
        if: ${{ matrix.config.python_dynamic == 'ON' }}
        working-directory: ${{runner.workspace}}/nrn
        run: |
          python -m pip install --upgrade -r external/nmodl/requirements.txt
          python -m pip install --upgrade -r ci_requirements.txt
          python -m pip install --upgrade pip -r nrn_requirements.txt

      - name: Set up Python@${{ env.PY_MAX_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PY_MAX_VERSION }}

      - name: Install Python@${{ env.PY_MAX_VERSION }} dependencies
        working-directory: ${{runner.workspace}}/nrn
        run: |
          python -m pip install --upgrade -r external/nmodl/requirements.txt
          python -m pip install --upgrade -r ci_requirements.txt
          python -m pip install --upgrade pip -r nrn_requirements.txt

      - name: Install a new automake
        # A automake >= 1.16.5 is needed for python 3.12 because it generates a python script
        # called py-compile and the original one is not supporting this version of python
        # Once ubuntu got a newer version of automake we can remove this part.
        if: matrix.config.music == 'ON' && startsWith(matrix.os, 'ubuntu')
        run: |
          curl -L -o automake.tar.xz https://ftpmirror.gnu.org/gnu/automake/automake-1.16.5.tar.xz
          tar -xf automake.tar.xz
          cd automake-1.16.5/
          ./configure --prefix=/usr/
          make -j
          sudo make -j install
          automake --version
        working-directory: ${{runner.temp}}

      - name: Setup MUSIC@${{ env.MUSIC_VERSION }}
        if: matrix.config.music == 'ON'
        run: |
          python3 -m venv music-venv
          source music-venv/bin/activate
          python3 -m pip install 'mpi4py<4' cython numpy setuptools
          sudo mkdir -p $MUSIC_INSTALL_DIR
          sudo chown -R $USER $MUSIC_INSTALL_DIR
          curl -L -o MUSIC.zip https://github.com/INCF/MUSIC/archive/refs/tags/${MUSIC_VERSION}.zip
          unzip MUSIC.zip && mv MUSIC-* MUSIC && cd MUSIC
          ./autogen.sh
          # on some systems MPI library detection fails, provide exact flags/compilers
          ./configure --with-python-sys-prefix --prefix=$MUSIC_INSTALL_DIR --disable-anysource MPI_CXXFLAGS="-g -O3" MPI_CFLAGS="-g -O3" MPI_LDFLAGS=" " CC=mpicc CXX=mpicxx
          make -j install
          deactivate
        working-directory: ${{runner.temp}}

      - name: Register gcc problem matcher
        if: ${{matrix.config.flag_warnings == 'ON'}}
        run: echo "::add-matcher::.github/problem-matchers/gcc.json"

      - name: Register sanitizer problem matcher
        if: ${{matrix.config.sanitizer}}
        run: echo "::add-matcher::.github/problem-matchers/${{matrix.config.sanitizer}}.json"

      - name: Hash config dictionary
        run: |
          cat << EOF > matrix.json
          ${{toJSON(matrix.config)}}
          EOF
          echo matrix.config JSON:
          cat matrix.json
          echo -----

      - name: Restore compiler cache
        uses: actions/cache/restore@v4
        id: restore-compiler-cache
        with:
          path: ${{runner.workspace}}/ccache
          key: ${{matrix.os}}-${{hashfiles('matrix.json')}}-${{github.ref}}-${{github.sha}}
          restore-keys: |
            ${{matrix.os}}-${{hashfiles('matrix.json')}}-${{github.ref}}-
            ${{matrix.os}}-${{hashfiles('matrix.json')}}-

      - name: Build and Test
        id: build-test
        shell: bash
        working-directory: ${{runner.workspace}}/nrn
        run:  |
          # OS related
          if [ "$RUNNER_OS" == "Linux" ]; then
            export ${MATRIX_EVAL};
            export SHELL="/bin/bash"
          else
            export CXX=${CXX:-g++};
            export CC=${CC:-gcc};
          fi
          if [ "$RUNNER_OS" == "macOS" ]; then
            # TODO - this is a workaround that was implemented for Azure being reported as getting stuck.
            # However it does not get stuck: neuron module not found and script goes to interpreter, seeming stuck.
            # This needs to be addressed and SKIP_EMBEDED_PYTHON_TEST logic removed everywhere.
            export SKIP_EMBEDED_PYTHON_TEST="true"
            # long TMPDIR path on MacOS can results into runtime failures with OpenMPI
            # Set shorter path as discussed in https://github.com/open-mpi/ompi/issues/8510
            export TMPDIR=/tmp/$GITHUB_JOB
            mkdir -p $TMPDIR
          fi

          # Python setup
          export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR/lib/python/
          # Python setup
          export PYTHON_MIN=$(command -v $PYTHON_MIN_NAME);
          export PYTHON_MAX=$(command -v $PYTHON_MAX_NAME);
          export PYTHON=$PYTHON_MAX
          if [ "$RUNNER_OS" == "macOS" ]; then
            # Python is not installed as a framework, so we need to writ 'backend: TkAgg' to `matplotlibrc`.
            # Since we are in a virtual environment, we cannot use `$HOME/matplotlibrc`
            # The following solution is generic and relies on `matplotlib.__file__` to know where to append backend setup.
            $PYTHON -c "import os,matplotlib; f =open(os.path.join(os.path.dirname(matplotlib.__file__), 'mpl-data/matplotlibrc'),'a'); f.write('backend: TkAgg');f.close();"
          fi;

          # Some logging
          echo $LANG
          echo $LC_ALL
          python3 -c 'import os,sys; os.set_blocking(sys.stdout.fileno(), True)'
          cmake --version

          # different builds with CMake
          if [[ "$BUILD_MODE" == "cmake" ]]; then
              cmake_args=(-G Ninja)
              # Sanitizer-specific setup
              if [[ -n "${{matrix.config.sanitizer}}" ]]; then
                if [ "$RUNNER_OS" == "Linux" ]; then
                  if [[ "${{matrix.config.sanitizer}}" == "thread" ]]; then
                    # GitHub/ubuntu-22.04 + clang-14 seems to have problems with TSan.
                    # Vanilla 22.04 + clang-16 from apt.llvm.org seemed to work.
                    # Use gcc-12 instead, as GitHub/ubuntu-22.04 already has it.
                    CC=$(command -v gcc-12)
                    CXX=$(command -v g++-12)
                  else
                    CC=$(command -v clang-14)
                    CXX=$(command -v clang++-14)
                    symbolizer_path="$(readlink -f "$(command -v llvm-symbolizer-14)")"
                    cmake_args+=(-DLLVM_SYMBOLIZER_PATH="${symbolizer_path}")
                  fi
                fi
                cmake_args+=(-DCMAKE_BUILD_TYPE=Custom \
                             -DCMAKE_C_FLAGS="-O1 -g" \
                             -DCMAKE_CXX_FLAGS="-O1 -g" \
                             -DNRN_SANITIZERS=$(echo ${{matrix.config.sanitizer}} | sed -e 's/-/,/g'))
              fi
              cmake_args+=(-DCMAKE_C_COMPILER="${CC}" \
                           -DCMAKE_C_COMPILER_LAUNCHER=ccache \
                           -DCMAKE_CXX_COMPILER="${CXX}" \
                           -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                           -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
                           -DNRN_ENABLE_TESTS=ON \
                           -DNRN_ENABLE_PERFORMANCE_TESTS=OFF \
                           ${{matrix.config.cmake_option}})
              if [[ "$NRN_ENABLE_PYTHON_DYNAMIC" == "ON" ]]; then
                cmake_args+=(-DNRN_ENABLE_PYTHON=ON \
                             -DNRN_ENABLE_PYTHON_DYNAMIC=ON \
                             -DNRN_PYTHON_DYNAMIC="${PYTHON_MIN};${PYTHON_MAX}" \
                             -DNRN_ENABLE_CORENEURON=ON)
              else
                cmake_args+=(-DPYTHON_EXECUTABLE="${PYTHON}")
              fi
              if [[ "$NRN_ENABLE_MUSIC" == "ON" ]]; then
                cmake_args+=(-DNRN_ENABLE_MUSIC=ON \
                             -DCMAKE_PREFIX_PATH=${MUSIC_INSTALL_DIR} \
                             -DMUSIC_ROOT=${MUSIC_INSTALL_DIR})
              fi
              # Enable more warnings in the builds whose compiler warnings we
              # highlight in the GitHub UI
              if [[ "${{matrix.config.flag_warnings}}" == "ON" ]]; then
                cmake_args+=(-DNRN_EXTRA_CXX_FLAGS="-Wall \
                                -Wno-char-subscripts \
                                -Wno-unknown-pragmas \
                                -Wno-unused-variable \
                                -Wno-unused-function \
                                -Wno-unused-but-set-variable \
                                -Wno-reorder \
                                -Wno-sign-compare" \
                             -DNRN_EXTRA_MECH_CXX_FLAGS="-Wno-sometimes-uninitialized \
                                -Wno-missing-braces")
              fi
              mkdir build && cd build
              echo "Building with: ${cmake_args[@]}"
              cmake .. "${cmake_args[@]}"
              if ccache --version | grep -E '^ccache version 4\.(4|4\.1)$'
              then
                echo "------- Disable ccache direct mode -------"
                # https://github.com/ccache/ccache/issues/935
                export CCACHE_NODIRECT=1
              fi
              ccache -z
              # Older versions don't support -v (verbose)
              ccache -vs 2>/dev/null || ccache -s
              cmake --build . --parallel
              ccache -vs 2>/dev/null || ccache -s
              if [ "$RUNNER_OS" == "macOS" ]
              then
                mkdir -p src/nrnpython
                echo $'[install]\nprefix='>src/nrnpython/setup.cfg;
              fi
              if [[ "$NRN_ENABLE_PYTHON_DYNAMIC" == "ON" ]]; then
                echo "--RUNNING BASIC TESTS FROM BUILD DIR--"
                for python in "${PYTHON_MIN}" "${PYTHON_MAX}"
                do
                  echo "Using ${python}"
                  NEURONHOME="${PWD}/share/nrn" \
                    PYTHONPATH="${PWD}/lib/python" \
                    PATH="${PWD}/bin" \
                    LD_LIBRARY_PATH="${PWD}/lib:${LD_LIBRARY_PATH}" \
                    DYLD_LIBRARY_PATH="${PWD}/lib:${DYLD_LIBRARY_PATH}" \
                    "${python}" -c "from neuron import h; import neuron; neuron.test()"
                done
              fi
              ctest --output-on-failure
              cmake --build . --target install
              export PATH="${INSTALL_DIR}/bin:${PATH}"
              if [[ -f "${INSTALL_DIR}/bin/nrn-enable-sanitizer" ]]; then
                echo --- bin/nrn-enable-sanitizer ---
                cat bin/nrn-enable-sanitizer
                echo ---
                nrn_enable_sanitizer=${INSTALL_DIR}/bin/nrn-enable-sanitizer
                nrn_enable_sanitizer_preload_python="${nrn_enable_sanitizer} --preload python"
              else
                echo nrn-enable-sanitizer not found, not using it
              fi
          elif [[ "$BUILD_MODE" == "setuptools" ]]; then
            ./packaging/python/build_wheels.bash CI;
          fi;
          if [[ -z "${nrn_enable_sanitizer_preload_python}" ]]; then
            nrn_enable_sanitizer_preload_python="${PYTHON}"
          fi

          # basic test for cmake when python is not disabled
          if [[ "$BUILD_MODE" == "cmake" && ! "${cmake_args[*]}" =~ "NRN_ENABLE_PYTHON=OFF" ]]; then
            ${nrn_enable_sanitizer_preload_python} --version && ${nrn_enable_sanitizer_preload_python} -c 'import neuron; neuron.test()'
          fi;

          # test neurondemo with cmake
          if [[ "$BUILD_MODE" != "setuptools" ]]; then
            ${nrn_enable_sanitizer} neurondemo -nogui -c 'demo(4)' -c 'run()' -c 'quit()'
          fi;

          # with cmake dynamic check python_min and python_max together
          if [[ "$BUILD_MODE" == "cmake" && "$NRN_ENABLE_PYTHON_DYNAMIC" == "ON" ]]; then
            ${nrn_enable_sanitizer_preload_python} -c 'import neuron; neuron.test()'
            $PYTHON_MIN -c 'import neuron; neuron.test()'
          fi;

          # run rxd tests manually if rxd is enabled *and CoreNEURON is
          # disabled -- otherwise hh-related tests fail
          if [[ "$BUILD_MODE" == "cmake" \
                && ! "${cmake_args[*]}" =~ "NRN_ENABLE_RX3D=OFF" \
                && ! "${cmake_args[*]}" =~ "NRN_ENABLE_CORENEURON=ON" ]]; then
            ${nrn_enable_sanitizer_preload_python} ../share/lib/python/neuron/rxdtests/run_all.py
          fi;

          if [ "$BUILD_MODE" == "setuptools" ]; then
            neuron_wheel=wheelhouse/NEURON*.whl;
            # test with virtual environment
            ./packaging/python/test_wheels.sh $PYTHON $neuron_wheel
            # test with global installation
            ./packaging/python/test_wheels.sh $PYTHON $neuron_wheel false
          fi;
        env:
          BUILD_MODE: ${{ matrix.config.build_mode }}
          CCACHE_BASEDIR: ${{runner.workspace}}/nrn
          CCACHE_DIR: ${{runner.workspace}}/ccache
          NRN_ENABLE_PYTHON_DYNAMIC : ${{ matrix.config.python_dynamic }}
          NRN_ENABLE_MUSIC: ${{ matrix.config.music }}
          PYTHON_MIN_NAME: "python${{ env.PY_MIN_VERSION }}"
          PYTHON_MAX_NAME: "python${{ env.PY_MAX_VERSION }}"
          INSTALL_DIR : ${{ runner.workspace }}/install
          MATRIX_EVAL: ${{ matrix.config.matrix_eval }}

      - name: Save compiler cache
        uses: actions/cache/save@v4
        if: always() && steps.restore-compiler-cache.outputs.cache-hit != 'true'
        with:
          path: ${{runner.workspace}}/ccache
          key: |
            ${{matrix.os}}-${{hashfiles('matrix.json')}}-${{github.ref}}-
            ${{matrix.os}}-${{hashfiles('matrix.json')}}-

      # This step will set up an SSH connection on tmate.io for live debugging.
      # To enable it, you have to:
      #   * add 'live-debug-ci' to your PR title
      #   * push something to your PR branch (note that just re-running the pipeline disregards the title update)
      - name: live debug session on failure (manual steps required, check `.github/neuron-ci.yml`)
        if: failure() && contains(github.event.pull_request.title, 'live-debug-ci')
        uses: mxschmitt/action-tmate@v3

  # see https://github.com/orgs/community/discussions/26822
  final:
    name: Final CI
    needs: [ci]
    if: ${{ always() }}
    runs-on: ubuntu-latest
    steps:
      - name: Check ci matrix all done
        if: >-
          ${{
               contains(needs.*.result, 'failure')
            || contains(needs.*.result, 'cancelled')
            || contains(needs.*.result, 'skipped')
          }}
        run: exit 1