Merge branch 'master' into arith_object_flux

.github/workflows/coverage.yml

@@ -24,7 +24,7 @@ on:
 env:
   PY_MIN_VERSION: '3.8'
   PY_MID_VERSION: '3.10'
-  PY_MAX_VERSION: '3.11'
+  PY_MAX_VERSION: '3.12'
 
 jobs:
   coverage:
@@ -58,7 +58,7 @@ jobs:
         run: |
           python3 -m venv music-venv
           source music-venv/bin/activate
-          python3 -m pip install mpi4py "cython<3" numpy
+          python3 -m pip install mpi4py "cython<3" numpy setuptools
           sudo mkdir -p $MUSIC_INSTALL_DIR
           sudo chown -R $USER $MUSIC_INSTALL_DIR
           curl -L -o MUSIC.zip https://github.com/INCF/MUSIC/archive/refs/tags/${MUSIC_VERSION}.zip

.github/workflows/docs.yml

@@ -17,7 +17,7 @@ on:
       - release/**
 
 env:
-  DEFAULT_PY_VERSION: '3.11'
+  DEFAULT_PY_VERSION: '3.12'
 
 jobs:
   documentation:

.github/workflows/neuron-ci.yml

@@ -39,7 +39,7 @@ jobs:
       DESIRED_CMAKE_VERSION: 3.17
       DYNAMIC_PYTHON_CMAKE_VERSION: 3.18
       PY_MIN_VERSION: ${{ matrix.config.python_min_version || '3.8' }}
-      PY_MAX_VERSION: ${{ matrix.config.python_max_version || '3.11' }}
+      PY_MAX_VERSION: ${{ matrix.config.python_max_version || '3.12' }}
       MUSIC_INSTALL_DIR: /opt/MUSIC
       MUSIC_VERSION: 1.2.1
 
@@ -49,7 +49,7 @@ jobs:
         config:
           - { matrix_eval : "CC=gcc-9 CXX=g++-9",   build_mode: "setuptools"}
           - { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", music: ON}
-          - { matrix_eval : "CC=gcc-10 CXX=g++-10",   build_mode: "cmake", python_dynamic: ON}
+          - { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", python_dynamic: ON}
           - { matrix_eval : "CC=gcc-9 CXX=g++-9" ,  build_mode: "cmake", cmake_option: "-DNRN_ENABLE_CORENEURON=ON"}
           - { matrix_eval : "CC=gcc-9 CXX=g++-9",   build_mode: "cmake", cmake_option: "-DNRN_ENABLE_MPI=OFF -DNRN_ENABLE_INTERVIEWS=OFF -DNRN_ENABLE_CORENEURON=ON"}
           - { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", cmake_option: "-DNRN_ENABLE_PYTHON=OFF -DNRN_ENABLE_RX3D=OFF -DNRN_ENABLE_CORENEURON=ON"}
@@ -89,6 +89,14 @@ jobs:
       fail-fast: false
 
     steps:
+
+      - name: Fix kernel mmap rnd bits
+        # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with
+        # high-entropy ASLR in much newer kernels that GitHub runners are
+        # using leading to random crashes: https://reviews.llvm.org/D148280
+        run: sudo sysctl vm.mmap_rnd_bits=28
+        if: matrix.os == 'ubuntu-22.04'
+
       - name: Setup cmake
         uses: jwlawson/actions-setup-cmake@v2
         with:
@@ -100,15 +108,20 @@ jobs:
           # Unlink and re-link to prevent errors when GitHub macOS runner images
           # install Python outside of brew; See actions/setup-python#577 and BlueBrain/libsonata/pull/317
           brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done
-          brew install ccache coreutils doxygen flex bison mpich ninja xz autoconf autoconf automake libtool
+          brew install ccache coreutils doxygen flex bison mpich ninja xz autoconf automake libtool
           # We use both for dynamic mpi in nrn
           brew unlink mpich
           brew install openmpi
           brew install --cask xquartz
-          echo /usr/local/opt/flex/bin:/usr/local/opt/bison/bin >> $GITHUB_PATH
-          # Core https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
-          echo CMAKE_BUILD_PARALLEL_LEVEL=3 >> $GITHUB_ENV
-          echo CTEST_PARALLEL_LEVEL=3 >> $GITHUB_ENV
+          echo "$(brew --prefix)/opt/flex/bin:$(brew --prefix)/opt/bison/bin" >> $GITHUB_PATH
+          # Core https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
+          if [[ "${{matrix.os}}" == "macOS-13" ]]; then
+              echo CMAKE_BUILD_PARALLEL_LEVEL=4 >> $GITHUB_ENV
+              echo CTEST_PARALLEL_LEVEL=4 >> $GITHUB_ENV
+          else
+              echo CMAKE_BUILD_PARALLEL_LEVEL=3 >> $GITHUB_ENV
+              echo CTEST_PARALLEL_LEVEL=3 >> $GITHUB_ENV
+          fi
           echo CI_OS_NAME=osx >> $GITHUB_ENV
         shell: bash
 
@@ -122,9 +135,9 @@ jobs:
           if [[ "${{matrix.os}}" == "ubuntu-20.04" ]]; then
             sudo apt-get install g++-7 g++-8
           fi
-          # Core https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
-          echo CMAKE_BUILD_PARALLEL_LEVEL=2 >> $GITHUB_ENV
-          echo CTEST_PARALLEL_LEVEL=2 >> $GITHUB_ENV
+          # Core https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
+          echo CMAKE_BUILD_PARALLEL_LEVEL=4 >> $GITHUB_ENV
+          echo CTEST_PARALLEL_LEVEL=4 >> $GITHUB_ENV
           echo CI_OS_NAME=linux >> $GITHUB_ENV
         shell: bash
 
@@ -163,12 +176,27 @@ jobs:
           python -m pip install --upgrade -r external/nmodl/requirements.txt
           python -m pip install --upgrade -r ci_requirements.txt
 
+      - name: Install a new automake
+        # A new automake is needed for python 3.12 because it generate a python script
+        # called py-compile and the original one is not supporting this version of python
+        # Once ubuntu got a newer version of automake we can remove this part.
+        if: matrix.config.music == 'ON' && startsWith(matrix.os, 'ubuntu')
+        run: |
+          curl -L -o automake.tar.xz https://ftp.gnu.org/gnu/automake/automake-1.16.5.tar.xz
+          tar -xf automake.tar.xz
+          cd automake-1.16.5/
+          ./configure --prefix=/usr/
+          make -j
+          sudo make -j install
+          automake --version
+        working-directory: ${{runner.temp}}
+
       - name: Setup MUSIC@${{ env.MUSIC_VERSION }}
         if: matrix.config.music == 'ON'
         run: |
           python3 -m venv music-venv
           source music-venv/bin/activate
-          python3 -m pip install mpi4py "cython<3" numpy
+          python3 -m pip install mpi4py "cython<3" numpy setuptools
           sudo mkdir -p $MUSIC_INSTALL_DIR
           sudo chown -R $USER $MUSIC_INSTALL_DIR
           curl -L -o MUSIC.zip https://github.com/INCF/MUSIC/archive/refs/tags/${MUSIC_VERSION}.zip
@@ -265,7 +293,7 @@ jobs:
                   else
                     CC=$(command -v clang-14)
                     CXX=$(command -v clang++-14)
-                    symbolizer_path=$(realpath $(command -v llvm-symbolizer-14))
+                    symbolizer_path="$(readlink -f "$(command -v llvm-symbolizer-14)")"
                     cmake_args+=(-DLLVM_SYMBOLIZER_PATH="${symbolizer_path}")
                   fi
                 fi

.github/workflows/windows.yml

@@ -49,7 +49,7 @@ jobs:
     - name: Set up Python3
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
 
     - name: Download Dependencies
       run: |

.gitlab-ci.yml

@@ -167,7 +167,7 @@ simulation_stack:
   variables:
     bb5_ntasks: 2   # so we block 16 cores
     bb5_cpus_per_task: 8 # ninja -j {this}
-    bb5_memory: 76G # ~16*384/80
+    bb5_memory: 160G # ~2*16*384/80 (~2x more as we have seen OOMs)
 
 .spack_intel:
   variables:
@@ -192,7 +192,7 @@ simulation_stack:
   extends: [.ctest]
   variables:
     bb5_ntasks: 16
-    bb5_memory: 76G # ~16*384/80
+    bb5_memory: 160G # ~2*16*384/80 (~2x more as we have seen OOMs)
 
 # Build NMODL once with GCC
 build:nmodl:

.sanitizers/undefined.supp

@@ -30,6 +30,7 @@ shift-base:nrnRan4int
 unsigned-integer-overflow:_philox4x32bumpkey(r123array2x32)
 unsigned-integer-overflow:coreneuron::nrnran123_deletestream(coreneuron::nrnran123_State*, bool)
 unsigned-integer-overflow:coreneuron::nrnran123_newstream3(unsigned int, unsigned int, unsigned int, bool)
+unsigned-integer-overflow:Py_INCREF(_object*)
 vptr:NetCon::disconnect(ivObservable*)
 vptr:ObjObservable::object()
 vptr:PreSyn::disconnect(ivObservable*)

CMakeLists.txt

@@ -444,6 +444,10 @@ if(NRN_ENABLE_THREADS)
   list(APPEND NRN_COMPILE_DEFS NRN_ENABLE_THREADS)
 endif()
 
+# Some files are shared by NEURON and CoreNEURON and use #if CORENRN_BUILD to distinguish between
+# which build is occurring.
+list(APPEND NRN_COMPILE_DEFS CORENRN_BUILD=0)
+
 # =============================================================================
 # Profiler/Instrumentation Options
 # =============================================================================
@@ -729,7 +733,7 @@ add_custom_target(
 # =============================================================================
 # ~~~
 # Update hh.mod for CoreNEURON compatibility
-# - Replace GLOBAL variable by RANHE
+# - Replace GLOBAL variable by RANGE
 # - Comment out TABLE
 # ~~~
 # =============================================================================
@@ -805,7 +809,7 @@ if(NRN_ENABLE_TESTS)
   if(NOT TARGET Catch2::Catch2)
     nrn_add_external_project(catch2)
     set(CATCH_DIR ${PROJECT_SOURCE_DIR}/external/catch2)
-    list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external/catch2/contrib)
+    list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external/catch2/extras)
     include(Catch)
   endif()
   include(CTest)
@@ -826,10 +830,12 @@ if(NRN_ENABLE_TESTS)
   set(neurondemo_files
       "${neurondemo_prefix}/special"
       "${neurondemo_prefix}/${CMAKE_SHARED_LIBRARY_PREFIX}nrnmech${CMAKE_SHARED_LIBRARY_SUFFIX}")
+
   add_custom_command(
     OUTPUT ${neurondemo_files}
     COMMAND ${CMAKE_COMMAND} -E env ${NRN_RUN_FROM_BUILD_DIR_ENV}
             ${NRN_SANITIZER_ENABLE_ENVIRONMENT} "${PROJECT_BINARY_DIR}/bin/nrnivmodl"
+    DEPENDS nrnivmodl_dependency
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/share/nrn/demo/release
     VERBATIM)
 

CONTRIBUTING.md

@@ -94,6 +94,10 @@ formatted, indented, documented, and commented.
 
 The [Neuron Development Topics](https://neuronsimulator.github.io/nrn/dev/index.html) section of the documentation provides a starting point for understanding NEURON's internals.
 
+### Building NEURON
+
+To build NEURON, please follow the instructions [here](docs/install/install_instructions.md#installing-source-distributions).
+
 ### Code Formatting
 
 Currently we have enabled CMake and Clang code formatting using [cmake-format](https://github.com/cheshirekow/cmake_format) and [clang-format](https://clang.llvm.org/docs/ClangFormat.html). Before submitting a PR, make sure to run cmake-format and clang-format as below:
@@ -142,9 +146,6 @@ Or,
 
 See [cmake-format](https://github.com/cheshirekow/cmake_format) documentation for details.
 
-make && make format-pr
-```
-
 ## Python Contributions
 
 The Python source code is located under `share/lib/python/neuron`. Python unit tests are

azure-pipelines.yml

@@ -90,8 +90,8 @@ stages:
         matrix:
           Python38:
             python.version: '3.8'
-            python.org.version: '3.8.10'
-            python.installer.name: 'macos11.pkg'
+            python.org.version: '3.8.9'
+            python.installer.name: 'macosx10.9.pkg'
           Python39:
             python.version: '3.9'
             python.org.version: '3.9.13'

bin/CMakeLists.txt

@@ -36,7 +36,7 @@ include(CMakeListsNrnMech)
 # =============================================================================
 # nrnmech_makefile (based on coreneuron Configure templates)
 # =============================================================================
-nrn_configure_file(nrngui bin)
+configure_file("nrngui.in" "nrngui" @ONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/sortspike ${CMAKE_CURRENT_BINARY_DIR}/sortspike COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nrnivmodl_makefile_cmake.in
                ${PROJECT_BINARY_DIR}/bin/nrnmech_makefile @ONLY)
@@ -49,10 +49,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nrnpyenv.sh.in ${PROJECT_BINARY_DIR}/
 # file(CHMOD ...) and v3.20+ support setting permissions directly in configure_file(...).
 set(NRN_CONFIG_EXE_FILES "nrnivmodl" "neurondemo")
 foreach(NRN_CONFIG_EXE_FILE ${NRN_CONFIG_EXE_FILES})
-  nrn_configure_dest_src(${NRN_CONFIG_EXE_FILE} bin/tmp ${NRN_CONFIG_EXE_FILE} bin)
+  configure_file("${NRN_CONFIG_EXE_FILE}.in" "tmp/${NRN_CONFIG_EXE_FILE}" @ONLY)
   file(
-    COPY ${PROJECT_BINARY_DIR}/bin/tmp/${NRN_CONFIG_EXE_FILE}
-    DESTINATION ${PROJECT_BINARY_DIR}/bin
+    COPY "${CMAKE_CURRENT_BINARY_DIR}/tmp/${NRN_CONFIG_EXE_FILE}"
+    DESTINATION "${CMAKE_CURRENT_BINARY_DIR}"
     FILE_PERMISSIONS
       OWNER_READ
       OWNER_WRITE
@@ -62,7 +62,7 @@ foreach(NRN_CONFIG_EXE_FILE ${NRN_CONFIG_EXE_FILES})
       WORLD_READ
       WORLD_EXECUTE)
 endforeach()
-file(REMOVE_RECURSE ${PROJECT_BINARY_DIR}/bin/tmp)
+file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/bin/tmp")
 
 # =============================================================================
 # Install targets

bin/nrnivmodl-core.in

@@ -109,7 +109,7 @@ shopt -s nullglob
 if [ -d "$params_MODS_PATH" ]; then
     files=( "$params_MODS_PATH"/*.mod )
     if [ ${#files} -eq 0 ]; then
-        echo "WARNING: No mod files found in '$(realpath ${params_MODS_PATH})', compiling default ones only!"
+        echo "WARNING: No mod files found in '$(readlink -f "${params_MODS_PATH}")', compiling default ones only!"
     fi
 else
     echo "FATAL: Invalid mods directory: '$params_MODS_PATH'"

bin/nrnivmodl.in

@@ -34,7 +34,7 @@ export prefix
 export bindir
 export libdir
 
-if command -v xcrun; then
+if command -v xcrun >/dev/null 2>&1; then
     @NRN_OSX_BUILD_TRUE@export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)
     @NRN_OSX_BUILD_TRUE@export MACOSX_DEPLOYMENT_TARGET="@CMAKE_OSX_DEPLOYMENT_TARGET@"
     if [ -z "${MACOSX_DEPLOYMENT_TARGET}" ]; then

bin/nrnivmodl_core_makefile.in

@@ -71,7 +71,7 @@ CXX_LINK_EXE_CMD = $(CXX) $(CXXFLAGS) @CMAKE_EXE_LINKER_FLAGS@
 CXX_SHARED_LIB_CMD = $(CXX) $(CXXFLAGS) @CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS@ @CMAKE_SHARED_LIBRARY_CXX_FLAGS@ @CMAKE_SHARED_LINKER_FLAGS@
 
 # env variables required for mod2c or nmodl
-NMODL_ENV_VAR = @CORENRN_SANITIZER_ENABLE_ENVIRONMENT_STRING@ PYTHONPATH=@CORENRN_NMODL_PYTHONPATH@:${CORENRN_LIB_DIR}/python MODLUNIT=$(CORENRN_SHARE_NMODL_DIR)/nrnunits.lib
+NMODL_ENV_VAR = @CORENRN_SANITIZER_ENABLE_ENVIRONMENT_STRING@ PYTHONPATH=${PYTHONPATH}:@CORENRN_NMODL_PYTHONPATH@:${CORENRN_LIB_DIR}/python MODLUNIT=$(CORENRN_SHARE_NMODL_DIR)/nrnunits.lib
 
 ifeq (@CORENRN_ENABLE_GPU@, ON)
     nmodl_arguments_c=@NMODL_ACC_BACKEND_ARGS@ @NMODL_COMMON_ARGS@
@@ -145,7 +145,7 @@ endif
 
 
 # main target to build binary
-$(SPECIAL_EXE): $(corenrnmech_lib_target)
+$(SPECIAL_EXE): $(corenrnmech_lib_target) $(CORENRN_SHARE_CORENRN_DIR)/coreneuron.cpp
 	@printf " => $(C_GREEN)Binary$(C_RESET) creating $(SPECIAL_EXE)\n"
 	$(CXX_LINK_EXE_CMD) -o $(SPECIAL_EXE) $(CORENRN_SHARE_CORENRN_DIR)/coreneuron.cpp \
 	  -I$(CORENRN_INC_DIR) $(INCFLAGS) \

bin/nrnivmodl_makefile_cmake.in

@@ -121,7 +121,7 @@ C_GREEN := \033[32m
 
 # Take the main and link with nrnmech.
 # RPATH is set for DESTDIR_RPATH and coreneuron lib
-special: $(mech_lib)
+special: $(mech_lib) $(datadir)/nrnmain.cpp
 	@printf " => $(C_GREEN)LINKING$(C_RESET) executable \"${PWD}/$(special)\" LDFLAGS are: $(LDFLAGS)\n"
 	$(CXX_LINK_EXE) -I $(incdir) -I $(incdir)/nrncvode -DAUTO_DLOPEN_NRNMECH=0 $(datadir)/nrnmain.cpp -o $(special) \
 	  -L$(OBJS_DIR) -l$(mech_libname) $(NRNLIB_FLAGS) -l$(mech_libname) $(extra_lib_link) -Wl,-rpath,'$(DESTDIR_RPATH)' -Wl,-rpath,$(libdir) $(LDFLAGS) $(EXTRA_LDFLAGS)