fix missing variable and add explanations #7516
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: NEURON CI | |
| concurrency: | |
| # Don't cancel on master, creating a PR when a push workflow is already going will cancel the push workflow in favour of the PR workflow | |
| group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_id || github.event.number && github.head_ref || github.ref_name }} | |
| cancel-in-progress: true | |
| on: | |
| merge_group: | |
| push: | |
| branches: | |
| # If nothing else, this is important for the ccache logic below... | |
| - master | |
| - release/** | |
| pull_request: | |
| branches: | |
| - master | |
| - release/** | |
| # TODO : https://github.com/neuronsimulator/nrn/issues/1063 | |
| # paths-ignore: | |
| # - '**.md' | |
| # - '**.rst' | |
| # - 'docs/**' | |
| jobs: | |
| ci: | |
| runs-on: ${{ matrix.os }} | |
| name: ${{ matrix.os }} - ${{ matrix.config.build_mode }} (${{ matrix.config.cmake_option }}${{ matrix.config.config_options }}${{ matrix.config.matrix_eval }}${{ matrix.config.sanitizer }}) | |
| timeout-minutes: 75 | |
| env: | |
| INSTALL_DIR: install | |
| SDK_ROOT: $(xcrun --sdk macosx --show-sdk-path) | |
| SKIP_WHEELHOUSE_REPAIR: true | |
| BUILD_TYPE: Release | |
| DESIRED_CMAKE_VERSION: 3.17 | |
| DYNAMIC_PYTHON_CMAKE_VERSION: 3.18 | |
| PY_MIN_VERSION: ${{ matrix.config.python_min_version || '3.8' }} | |
| PY_MAX_VERSION: ${{ matrix.config.python_max_version || '3.11' }} | |
| MUSIC_INSTALL_DIR: /opt/MUSIC | |
| MUSIC_VERSION: 1.2.1 | |
| strategy: | |
| matrix: | |
| os: [ macOS-12, ubuntu-20.04] | |
| config: | |
| - { matrix_eval : "CC=gcc-9 CXX=g++-9", build_mode: "setuptools"} | |
| - { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", music: ON} | |
| - { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", python_dynamic: ON} | |
| - { matrix_eval : "CC=gcc-9 CXX=g++-9" , build_mode: "cmake", cmake_option: "-DNRN_ENABLE_CORENEURON=ON"} | |
| - { matrix_eval : "CC=gcc-9 CXX=g++-9", build_mode: "cmake", cmake_option: "-DNRN_ENABLE_MPI=OFF -DNRN_ENABLE_INTERVIEWS=OFF -DNRN_ENABLE_CORENEURON=ON"} | |
| - { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", cmake_option: "-DNRN_ENABLE_PYTHON=OFF -DNRN_ENABLE_RX3D=OFF -DNRN_ENABLE_CORENEURON=ON"} | |
| include: | |
| - os: ubuntu-22.04 | |
| config: | |
| build_mode: cmake | |
| cmake_option: -DNRN_ENABLE_CORENEURON=ON | |
| -DNRN_ENABLE_INTERVIEWS=OFF -DNMODL_SANITIZERS=undefined | |
| flag_warnings: ON | |
| sanitizer: undefined | |
| - os: ubuntu-22.04 | |
| config: | |
| build_mode: cmake | |
| # TODO: CoreNEURON is only LeakSanitizer-clean if we disable MPI | |
| cmake_option: -DNRN_ENABLE_CORENEURON=ON | |
| -DNRN_ENABLE_INTERVIEWS=OFF -DNMODL_SANITIZERS=address | |
| # TODO: address-leak is the dream, but there are many problems, | |
| # including external ones from the MPI implementations | |
| sanitizer: address | |
| - os: ubuntu-22.04 | |
| config: | |
| build_mode: cmake | |
| # Cannot use a non-instrumented OpenMP with TSan, and we don't | |
| # have a TSan-instrumented OpenMP runtime available. | |
| # TODO: debug RX3D + TSan | |
| cmake_option: -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_MPI=OFF | |
| -DCORENRN_ENABLE_OPENMP=OFF -DNRN_ENABLE_RX3D=OFF | |
| sanitizer: thread | |
| - os: macOS-13 | |
| config: | |
| build_mode: cmake | |
| # TODO: investigate rxd test timeouts in this build and re-enable them | |
| cmake_option: -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_INTERVIEWS=OFF | |
| -DNRN_ENABLE_RX3D=OFF -DNMODL_SANITIZERS=address | |
| sanitizer: address | |
| fail-fast: false | |
| steps: | |
| - name: Fix kernel mmap rnd bits | |
| # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with | |
| # high-entropy ASLR in much newer kernels that GitHub runners are | |
| # using leading to random crashes: https://reviews.llvm.org/D148280 | |
| run: sudo sysctl vm.mmap_rnd_bits=28 | |
| if: matrix.os == 'ubuntu-22.04' | |
| - name: Setup cmake | |
| uses: jwlawson/actions-setup-cmake@v2 | |
| with: | |
| cmake-version : ${{(matrix.config.python_dynamic || matrix.config.build_mode == 'setuptools') && env.DYNAMIC_PYTHON_CMAKE_VERSION || env.DESIRED_CMAKE_VERSION}} | |
| - name: Install homebrew packages | |
| if: startsWith(matrix.os, 'macOS') | |
| run: | | |
| # Unlink and re-link to prevent errors when GitHub macOS runner images | |
| # install Python outside of brew; See actions/setup-python#577 and BlueBrain/libsonata/pull/317 | |
| brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done | |
| brew install ccache coreutils doxygen flex bison mpich ninja xz autoconf autoconf automake libtool | |
| # We use both for dynamic mpi in nrn | |
| brew unlink mpich | |
| brew install openmpi | |
| brew install --cask xquartz | |
| echo /usr/local/opt/flex/bin:/usr/local/opt/bison/bin >> $GITHUB_PATH | |
| # Core https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources | |
| echo CMAKE_BUILD_PARALLEL_LEVEL=3 >> $GITHUB_ENV | |
| echo CTEST_PARALLEL_LEVEL=3 >> $GITHUB_ENV | |
| echo CI_OS_NAME=osx >> $GITHUB_ENV | |
| shell: bash | |
| - name: Install apt packages | |
| if: startsWith(matrix.os, 'ubuntu') | |
| run: | | |
| sudo apt-get install build-essential ccache libopenmpi-dev \ | |
| libmpich-dev libx11-dev libxcomposite-dev mpich ninja-build \ | |
| openmpi-bin flex libfl-dev bison | |
| # The sanitizer builds use ubuntu 22.04 | |
| if [[ "${{matrix.os}}" == "ubuntu-20.04" ]]; then | |
| sudo apt-get install g++-7 g++-8 | |
| fi | |
| # Core https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources | |
| echo CMAKE_BUILD_PARALLEL_LEVEL=2 >> $GITHUB_ENV | |
| echo CTEST_PARALLEL_LEVEL=2 >> $GITHUB_ENV | |
| echo CI_OS_NAME=linux >> $GITHUB_ENV | |
| shell: bash | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - name: Clone nmodl | |
| working-directory: ${{runner.workspace}}/nrn | |
| run: | | |
| git submodule update --init --recursive --force --depth 1 -- external/nmodl | |
| - name: Set up Python@${{ env.PY_MIN_VERSION }} | |
| if: ${{matrix.config.python_dynamic == 'ON'}} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PY_MIN_VERSION }} | |
| - name: Install Python@${{ env.PY_MIN_VERSION }} dependencies | |
| if: ${{ matrix.config.python_dynamic == 'ON' }} | |
| working-directory: ${{runner.workspace}}/nrn | |
| run: | | |
| python -m pip install --upgrade pip -r nrn_requirements.txt | |
| python -m pip install --upgrade -r external/nmodl/requirements.txt | |
| python -m pip install --upgrade -r ci_requirements.txt | |
| - name: Set up Python@${{ env.PY_MAX_VERSION }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PY_MAX_VERSION }} | |
| - name: Install Python@${{ env.PY_MAX_VERSION }} dependencies | |
| working-directory: ${{runner.workspace}}/nrn | |
| run: | | |
| python -m pip install --upgrade pip -r nrn_requirements.txt | |
| python -m pip install --upgrade -r external/nmodl/requirements.txt | |
| python -m pip install --upgrade -r ci_requirements.txt | |
| - name: Setup MUSIC@${{ env.MUSIC_VERSION }} | |
| if: matrix.config.music == 'ON' | |
| run: | | |
| python3 -m venv music-venv | |
| source music-venv/bin/activate | |
| python3 -m pip install mpi4py "cython<3" numpy | |
| sudo mkdir -p $MUSIC_INSTALL_DIR | |
| sudo chown -R $USER $MUSIC_INSTALL_DIR | |
| curl -L -o MUSIC.zip https://github.com/INCF/MUSIC/archive/refs/tags/${MUSIC_VERSION}.zip | |
| unzip MUSIC.zip && mv MUSIC-* MUSIC && cd MUSIC | |
| ./autogen.sh | |
| # on some systems MPI library detection fails, provide exact flags/compilers | |
| ./configure --with-python-sys-prefix --prefix=$MUSIC_INSTALL_DIR --disable-anysource MPI_CXXFLAGS="-g -O3" MPI_CFLAGS="-g -O3" MPI_LDFLAGS=" " CC=mpicc CXX=mpicxx | |
| make -j install | |
| deactivate | |
| working-directory: ${{runner.temp}} | |
| - name: Register gcc problem matcher | |
| if: ${{matrix.config.flag_warnings == 'ON'}} | |
| run: echo "::add-matcher::.github/problem-matchers/gcc.json" | |
| - name: Register sanitizer problem matcher | |
| if: ${{matrix.config.sanitizer}} | |
| run: echo "::add-matcher::.github/problem-matchers/${{matrix.config.sanitizer}}.json" | |
| - name: Hash config dictionary | |
| run: | | |
| cat << EOF > matrix.json | |
| ${{toJSON(matrix.config)}} | |
| EOF | |
| echo matrix.config JSON: | |
| cat matrix.json | |
| echo ----- | |
| - name: Restore compiler cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{runner.workspace}}/ccache | |
| save-always: true | |
| key: ${{matrix.os}}-${{hashfiles('matrix.json')}}-${{github.ref}}-${{github.sha}} | |
| restore-keys: | | |
| ${{matrix.os}}-${{hashfiles('matrix.json')}}-${{github.ref}}- | |
| ${{matrix.os}}-${{hashfiles('matrix.json')}}- | |
| - name: Build and Test | |
| id: build-test | |
| shell: bash | |
| working-directory: ${{runner.workspace}}/nrn | |
| run: | | |
| # OS related | |
| if [ "$RUNNER_OS" == "Linux" ]; then | |
| export ${MATRIX_EVAL}; | |
| export SHELL="/bin/bash" | |
| else | |
| export CXX=${CXX:-g++}; | |
| export CC=${CC:-gcc}; | |
| fi | |
| if [ "$RUNNER_OS" == "macOS" ]; then | |
| # TODO - this is a workaround that was implemented for Azure being reported as getting stuck. | |
| # However it does not get stuck: neuron module not found and script goes to interpreter, seeming stuck. | |
| # This needs to be addressed and SKIP_EMBEDED_PYTHON_TEST logic removed everywhere. | |
| export SKIP_EMBEDED_PYTHON_TEST="true" | |
| # long TMPDIR path on MacOS can results into runtime failures with OpenMPI | |
| # Set shorter path as discussed in https://github.com/open-mpi/ompi/issues/8510 | |
| export TMPDIR=/tmp/$GITHUB_JOB | |
| mkdir -p $TMPDIR | |
| fi | |
| # Python setup | |
| export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR/lib/python/ | |
| # Python setup | |
| export PYTHON_MIN=$(command -v $PYTHON_MIN_NAME); | |
| export PYTHON_MAX=$(command -v $PYTHON_MAX_NAME); | |
| export PYTHON=$PYTHON_MAX | |
| if [ "$RUNNER_OS" == "macOS" ]; then | |
| # Python is not installed as a framework, so we need to writ 'backend: TkAgg' to `matplotlibrc`. | |
| # Since we are in a virtual environment, we cannot use `$HOME/matplotlibrc` | |
| # The following solution is generic and relies on `matplotlib.__file__` to know where to append backend setup. | |
| $PYTHON -c "import os,matplotlib; f =open(os.path.join(os.path.dirname(matplotlib.__file__), 'mpl-data/matplotlibrc'),'a'); f.write('backend: TkAgg');f.close();" | |
| fi; | |
| # Some logging | |
| echo $LANG | |
| echo $LC_ALL | |
| python3 -c 'import os,sys; os.set_blocking(sys.stdout.fileno(), True)' | |
| cmake --version | |
| # different builds with CMake | |
| if [[ "$BUILD_MODE" == "cmake" ]]; then | |
| cmake_args=(-G Ninja) | |
| # Sanitizer-specific setup | |
| if [[ -n "${{matrix.config.sanitizer}}" ]]; then | |
| if [ "$RUNNER_OS" == "Linux" ]; then | |
| if [[ "${{matrix.config.sanitizer}}" == "thread" ]]; then | |
| # GitHub/ubuntu-22.04 + clang-14 seems to have problems with TSan. | |
| # Vanilla 22.04 + clang-16 from apt.llvm.org seemed to work. | |
| # Use gcc-12 instead, as GitHub/ubuntu-22.04 already has it. | |
| CC=$(command -v gcc-12) | |
| CXX=$(command -v g++-12) | |
| else | |
| CC=$(command -v clang-14) | |
| CXX=$(command -v clang++-14) | |
| symbolizer_path=$(realpath $(command -v llvm-symbolizer-14)) | |
| cmake_args+=(-DLLVM_SYMBOLIZER_PATH="${symbolizer_path}") | |
| fi | |
| fi | |
| cmake_args+=(-DCMAKE_BUILD_TYPE=Custom \ | |
| -DCMAKE_C_FLAGS="-O1 -g" \ | |
| -DCMAKE_CXX_FLAGS="-O1 -g" \ | |
| -DNRN_SANITIZERS=$(echo ${{matrix.config.sanitizer}} | sed -e 's/-/,/g')) | |
| fi | |
| cmake_args+=(-DCMAKE_C_COMPILER="${CC}" \ | |
| -DCMAKE_C_COMPILER_LAUNCHER=ccache \ | |
| -DCMAKE_CXX_COMPILER="${CXX}" \ | |
| -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
| -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ | |
| -DNRN_ENABLE_TESTS=ON \ | |
| -DNRN_ENABLE_PERFORMANCE_TESTS=OFF \ | |
| ${{matrix.config.cmake_option}}) | |
| if [[ "$NRN_ENABLE_PYTHON_DYNAMIC" == "ON" ]]; then | |
| cmake_args+=(-DNRN_ENABLE_PYTHON=ON \ | |
| -DNRN_ENABLE_PYTHON_DYNAMIC=ON \ | |
| -DNRN_PYTHON_DYNAMIC="${PYTHON_MIN};${PYTHON_MAX}" \ | |
| -DNRN_ENABLE_CORENEURON=ON) | |
| else | |
| cmake_args+=(-DPYTHON_EXECUTABLE="${PYTHON}") | |
| fi | |
| if [[ "$NRN_ENABLE_MUSIC" == "ON" ]]; then | |
| cmake_args+=(-DNRN_ENABLE_MUSIC=ON \ | |
| -DCMAKE_PREFIX_PATH=${MUSIC_INSTALL_DIR} \ | |
| -DMUSIC_ROOT=${MUSIC_INSTALL_DIR}) | |
| fi | |
| # Enable more warnings in the builds whose compiler warnings we | |
| # highlight in the GitHub UI | |
| if [[ "${{matrix.config.flag_warnings}}" == "ON" ]]; then | |
| cmake_args+=(-DNRN_EXTRA_CXX_FLAGS="-Wall \ | |
| -Wno-char-subscripts \ | |
| -Wno-unknown-pragmas \ | |
| -Wno-unused-variable \ | |
| -Wno-unused-function \ | |
| -Wno-unused-but-set-variable \ | |
| -Wno-reorder \ | |
| -Wno-sign-compare" \ | |
| -DNRN_EXTRA_MECH_CXX_FLAGS="-Wno-sometimes-uninitialized \ | |
| -Wno-missing-braces") | |
| fi | |
| mkdir build && cd build | |
| echo "Building with: ${cmake_args[@]}" | |
| cmake .. "${cmake_args[@]}" | |
| if ccache --version | grep -E '^ccache version 4\.(4|4\.1)$' | |
| then | |
| echo "------- Disable ccache direct mode -------" | |
| # https://github.com/ccache/ccache/issues/935 | |
| export CCACHE_NODIRECT=1 | |
| fi | |
| ccache -z | |
| # Older versions don't support -v (verbose) | |
| ccache -vs 2>/dev/null || ccache -s | |
| cmake --build . --parallel | |
| ccache -vs 2>/dev/null || ccache -s | |
| if [ "$RUNNER_OS" == "macOS" ] | |
| then | |
| mkdir -p src/nrnpython | |
| echo $'[install]\nprefix='>src/nrnpython/setup.cfg; | |
| fi | |
| if [[ "$NRN_ENABLE_PYTHON_DYNAMIC" == "ON" ]]; then | |
| echo "--RUNNING BASIC TESTS FROM BUILD DIR--" | |
| for python in "${PYTHON_MIN}" "${PYTHON_MAX}" | |
| do | |
| echo "Using ${python}" | |
| NEURONHOME="${PWD}/share/nrn" \ | |
| PYTHONPATH="${PWD}/lib/python" \ | |
| PATH="${PWD}/bin" \ | |
| LD_LIBRARY_PATH="${PWD}/lib:${LD_LIBRARY_PATH}" \ | |
| DYLD_LIBRARY_PATH="${PWD}/lib:${DYLD_LIBRARY_PATH}" \ | |
| "${python}" -c "from neuron import h; import neuron; neuron.test()" | |
| done | |
| fi | |
| ctest --output-on-failure | |
| cmake --build . --target install | |
| export PATH="${INSTALL_DIR}/bin:${PATH}" | |
| if [[ -f "${INSTALL_DIR}/bin/nrn-enable-sanitizer" ]]; then | |
| echo --- bin/nrn-enable-sanitizer --- | |
| cat bin/nrn-enable-sanitizer | |
| echo --- | |
| nrn_enable_sanitizer=${INSTALL_DIR}/bin/nrn-enable-sanitizer | |
| nrn_enable_sanitizer_preload_python="${nrn_enable_sanitizer} --preload python" | |
| else | |
| echo nrn-enable-sanitizer not found, not using it | |
| fi | |
| elif [[ "$BUILD_MODE" == "setuptools" ]]; then | |
| ./packaging/python/build_wheels.bash CI; | |
| fi; | |
| if [[ -z "${nrn_enable_sanitizer_preload_python}" ]]; then | |
| nrn_enable_sanitizer_preload_python="${PYTHON}" | |
| fi | |
| # basic test for cmake when python is not disabled | |
| if [[ "$BUILD_MODE" == "cmake" && ! "${cmake_args[*]}" =~ "NRN_ENABLE_PYTHON=OFF" ]]; then | |
| ${nrn_enable_sanitizer_preload_python} --version && ${nrn_enable_sanitizer_preload_python} -c 'import neuron; neuron.test()' | |
| fi; | |
| # test neurondemo with cmake | |
| if [[ "$BUILD_MODE" != "setuptools" ]]; then | |
| ${nrn_enable_sanitizer} neurondemo -nogui -c 'demo(4)' -c 'run()' -c 'quit()' | |
| fi; | |
| # with cmake dynamic check python_min and python_max together | |
| if [[ "$BUILD_MODE" == "cmake" && "$NRN_ENABLE_PYTHON_DYNAMIC" == "ON" ]]; then | |
| ${nrn_enable_sanitizer_preload_python} -c 'import neuron; neuron.test()' | |
| $PYTHON_MIN -c 'import neuron; neuron.test()' | |
| fi; | |
| # run rxd tests manually if rxd is enabled *and CoreNEURON is | |
| # disabled -- otherwise hh-related tests fail | |
| if [[ "$BUILD_MODE" == "cmake" \ | |
| && ! "${cmake_args[*]}" =~ "NRN_ENABLE_RX3D=OFF" \ | |
| && ! "${cmake_args[*]}" =~ "NRN_ENABLE_CORENEURON=ON" ]]; then | |
| ${nrn_enable_sanitizer_preload_python} ../share/lib/python/neuron/rxdtests/run_all.py | |
| fi; | |
| if [ "$BUILD_MODE" == "setuptools" ]; then | |
| neuron_wheel=wheelhouse/NEURON*.whl; | |
| # test with virtual environment | |
| ./packaging/python/test_wheels.sh $PYTHON $neuron_wheel | |
| # test with global installation | |
| ./packaging/python/test_wheels.sh $PYTHON $neuron_wheel false | |
| fi; | |
| env: | |
| BUILD_MODE: ${{ matrix.config.build_mode }} | |
| CCACHE_BASEDIR: ${{runner.workspace}}/nrn | |
| CCACHE_DIR: ${{runner.workspace}}/ccache | |
| NRN_ENABLE_PYTHON_DYNAMIC : ${{ matrix.config.python_dynamic }} | |
| NRN_ENABLE_MUSIC: ${{ matrix.config.music }} | |
| PYTHON_MIN_NAME: "python${{ env.PY_MIN_VERSION }}" | |
| PYTHON_MAX_NAME: "python${{ env.PY_MAX_VERSION }}" | |
| INSTALL_DIR : ${{ runner.workspace }}/install | |
| MATRIX_EVAL: ${{ matrix.config.matrix_eval }} | |
| # This step will set up an SSH connection on tmate.io for live debugging. | |
| # To enable it, you have to: | |
| # * add 'live-debug-ci' to your PR title | |
| # * push something to your PR branch (note that just re-running the pipeline disregards the title update) | |
| - name: live debug session on failure (manual steps required, check `.github/neuron-ci.yml`) | |
| if: failure() && contains(github.event.pull_request.title, 'live-debug-ci') | |
| uses: mxschmitt/action-tmate@v3 | |
| # see https://github.com/orgs/community/discussions/26822 | |
| final: | |
| name: Final CI | |
| needs: [ci] | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check ci matrix all done | |
| if: >- | |
| ${{ | |
| contains(needs.*.result, 'failure') | |
| || contains(needs.*.result, 'cancelled') | |
| || contains(needs.*.result, 'skipped') | |
| }} | |
| run: exit 1 |