.gitlab-ci.yml

include:
  - project: hpc/gitlab-pipelines
    file:
      - spack-build-components.gitlab-ci.yml
      - github-project-pipelines.gitlab-ci.yml
    ref: '$GITLAB_PIPELINES_BRANCH'
  - project: hpc/gitlab-upload-logs
    file: enable-upload.yml

# see https://gitlab.com/gitlab-org/gitlab/-/issues/263401 for why we specify the flags like this now
# 130 characters seems to be the point at which jobs refuse to run
.matrix:
  parallel:
    matrix:
      - build_mode: cmake
        cmake_coreneuron: "ON"
        cmake_interviews: "OFF"
        cmake_rx3d: "OFF"
        sanitizer: address

mac_m1_cmake_build:
  stage: .pre
  extends: .matrix
  cache:
    # either CI_MERGE_REQUEST_SOURCE_BRANCH_NAME or CI_COMMIT_BRANCH will be available,
    # depending on whether this pipeline runs for a merge request or on a branch
    # either way, we get the active branch in the cache key
    key: ${CI_JOB_NAME}-${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}${CI_COMMIT_BRANCH}
    paths:
      - ./ccache
  variables:
    CCACHE_BASEDIR: ${CI_PROJECT_DIR}/nrn
    CCACHE_DIR: ${CI_PROJECT_DIR}/ccache
    CMAKE_BUILD_PARALLEL_LEVEL: 3
    CTEST_PARALLEL_LEVEL: 3
  tags:
    - macos-arm64
  script:
    # Using macOS + ASan + virtual environments requires that you remove shims
    # before creating the venv. CMake would tell us this if we got it wrong.
    # See sanitizers.cmake for more information. If we find this gets copied to
    # too many different places, put the .py script in the repository.
    - |
      cat > resolve_shim.py << END_SCRIPT
      import ctypes
      dyld = ctypes.cdll.LoadLibrary('/usr/lib/system/libdyld.dylib')
      namelen = ctypes.c_ulong(1024)
      name = ctypes.create_string_buffer(b'\\000', namelen.value)
      dyld._NSGetExecutablePath(ctypes.byref(name), ctypes.byref(namelen))
      print(name.value.decode())
      END_SCRIPT
    - real_python=$(python3 resolve_shim.py)
    - echo "python3=$(command -v python3) is really ${real_python}"
    - PYTHONEXECUTABLE=${real_python} ${real_python} -mvenv venv
    - venv/bin/python3 -m ensurepip --upgrade --default-pip
    - git submodule update --init --recursive --force --depth 1 -- external/nmodl
    - venv/bin/pip install --upgrade -r external/nmodl/requirements.txt
    - venv/bin/pip install --upgrade pip -r nrn_requirements.txt
    - source ./venv/bin/activate
    - export PYTHON=${PWD}/venv/bin/python
    - ${PYTHON} --version
    - '${PYTHON} -c "import os,matplotlib; f = open(os.path.join(os.path.dirname(matplotlib.__file__), \"mpl-data/matplotlibrc\"),\"a\"); f.write(\"backend: TkAgg\");f.close();"'
    - 'export CXX=${CXX:-g++}'
    - 'export CC=${CC:-gcc}'
    - brew install flex bison
    - export PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:$PATH"
    - export INSTALL_DIR=$(pwd)/install
    - echo $LANG
    - echo $LC_ALL
    - 'export PYTHONPATH=$(${PYTHON} -c "import site; print(\":\".join(site.getsitepackages()))")'
    - ${PYTHON} -c 'import os,sys; os.set_blocking(sys.stdout.fileno(), True)'
    - cmake_args=(-G Ninja)
    - if [[ -n "${sanitizer}" ]]; then
    -   cmake_args+=(-DCMAKE_BUILD_TYPE=Custom
                     -DCMAKE_C_FLAGS="-O1 -g"
                     -DCMAKE_CXX_FLAGS="-O1 -g"
                     -DNRN_SANITIZERS=$(echo ${sanitizer} | sed -e 's/-/,/g'))
    - fi
    - cmake_args+=(-DCMAKE_C_COMPILER="${CC}"
                   -DCMAKE_C_COMPILER_LAUNCHER=ccache
                   -DCMAKE_CXX_COMPILER="${CXX}"
                   -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
                   -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
                   -DNRN_ENABLE_TESTS=ON
                   -DNRN_ENABLE_PERFORMANCE_TESTS=OFF
                   -DNRN_ENABLE_CORENEURON=${cmake_coreneuron}
                   -DNRN_ENABLE_INTERVIEWS=${cmake_interviews}
                   -DNRN_ENABLE_RX3D=${cmake_rx3d})
    - cmake_args+=(-DPYTHON_EXECUTABLE="${PYTHON}")
    - mkdir build && cd build
    - echo "Building with ${cmake_args[@]}"
    - cmake .. "${cmake_args[@]}"
    - ccache -z
    - ccache -vs 2>/dev/null
    - cmake --build . --parallel
    - ccache -vs 2>/dev/null
    - echo $'[install]\nprefix='>src/nrnpython/setup.cfg
    - cat src/nrnpython/setup.cfg
    - ctest --output-on-failure
    - cmake --build . --target install
    - 'export PATH=${INSTALL_DIR}/bin:${PATH}'
    - 'export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR/lib/python/'
    - if [[ -f "${INSTALL_DIR}/bin/nrn-enable-sanitizer" ]]; then
    -   echo --- bin/nrn-enable-sanitizer ---
    -   cat bin/nrn-enable-sanitizer
    -   echo ---
    -   nrn_enable_sanitizer=${INSTALL_DIR}/bin/nrn-enable-sanitizer
    -   nrn_enable_sanitizer_preload="${nrn_enable_sanitizer} --preload"
    - else
    -   echo nrn-enable-sanitizer not found, not using it
    - fi
    - $PYTHON --version && ${nrn_enable_sanitizer_preload} python -c 'import neuron; neuron.test()'
    - if [[ ! "${cmake_args[*]}" =~ "NRN_ENABLE_RX3D=OFF"
            && ! "${cmake_args[*]}" =~ "NRN_ENABLE_CORENEURON=ON" ]]; then
    -   ${nrn_enable_sanitizer_preload} python ../share/lib/python/neuron/rxdtests/run_all.py
    - fi;

variables:
  BLUECONFIGS_BRANCH:
    description: Branch of blueconfigs to trigger the simulation stack pipeline from
    value: main
  LIBSONATA_REPORT_BRANCH:
    description: Branch of libsonata-report to build BlueBrain models against in the simulation stack pipeline (LIBSONATA_REPORT_COMMIT and LIBSONATA_REPORT_TAG also possible)
    value: master
  NMODL_BRANCH:
    description: Branch of NMODL to build CoreNEURON against (NMODL_COMMIT and NMODL_TAG also possible)
    value: master
  SPACK_BRANCH:
    description: Branch of BlueBrain Spack to use for the CI pipeline
    value: develop
  SPACK_DEPLOYMENT_SUFFIX:
    description: Extra path component used when finding deployed software. Set to something like `pulls/1497` use software built for https://github.com/BlueBrain/spack/pull/1497. You probably want to set SPACK_BRANCH to the branch used in the relevant PR if you set this.
    value: ''

# Set up Spack
spack_setup:
  extends: .spack_setup_ccache
  script:
    - !reference [.spack_setup_ccache, script]
    # try and make sure these are propagated to the simulation_stack bridge job and child pipeline
    - echo "BLUECONFIGS_BRANCH=${BLUECONFIGS_BRANCH}" >> commit-mapping.env
    - echo "BLUECONFIGS_BRANCH=${BLUECONFIGS_BRANCH}" >> spack_clone_variables.env
  variables:
    NEURON_COMMIT: ${CI_COMMIT_SHA}
    # Enable fetching GitHub PR descriptions and parsing them to find out what
    # branches to build of other projects.
    PARSE_GITHUB_PR_DESCRIPTIONS: "true"
    # BLUECONFIGS_BRANCH does not correspond to a Spack package called blueconfigs
    SPACK_SETUP_IGNORE_PACKAGE_VARIABLES: BLUECONFIGS

# Performance seems to be terrible when we get too many jobs on a single node.
.build:
  extends: [.spack_build]
  variables:
    bb5_ntasks: 2   # so we block 16 cores
    bb5_cpus_per_task: 8 # ninja -j {this}
    bb5_memory: 160G # ~2*16*384/80 (~2x more as we have seen OOMs)

.spack_intel:
  variables:
    SPACK_PACKAGE_COMPILER: oneapi
.spack_nvhpc:
  variables:
    SPACK_PACKAGE_COMPILER: nvhpc
.build_neuron:
  extends: [.build]
  timeout: two hours
  variables:
    bb5_duration: "2:00:00"
    SPACK_PACKAGE: neuron
    SPACK_PACKAGE_SPEC: +coreneuron+tests~legacy-unit~rx3d build_type=FastDebug model_tests=channel-benchmark,olfactory,tqperf-heavy
.gpu_node:
  variables:
    bb5_constraint: volta
    bb5_cpus_per_task: 2
    bb5_partition: prod # assume this is a good source of GPU nodes
    bb5_exclusive: user # allocate gpu node exclusively for the CI user (to avoid errors from oversubscription)
.test_neuron:
  extends: [.ctest]
  variables:
    bb5_ntasks: 16
    bb5_memory: 160G # ~2*16*384/80 (~2x more as we have seen OOMs)

# Build NMODL once with GCC
build:nmodl:
  extends: [.build]
  variables:
    SPACK_PACKAGE: nmodl
    SPACK_PACKAGE_SPEC: ~legacy-unit
    SPACK_PACKAGE_COMPILER: gcc

# Build NEURON
.build_neuron_nmodl:
  extends: [.build_neuron]
  needs: ["build:nmodl"]

build:neuron:nmodl:intel:legacy:
  extends: [.build_neuron_nmodl, .spack_intel]
  variables:
    SPACK_PACKAGE_SPEC: ~rx3d~caliper~gpu+coreneuron~legacy-unit~openmp~shared~sympy+tests build_type=FastDebug model_tests=channel-benchmark,olfactory,tqperf-heavy

build:neuron:nmodl:intel:shared:
  extends: [.build_neuron_nmodl, .spack_intel]
  variables:
    SPACK_PACKAGE_SPEC: ~rx3d~caliper~gpu+coreneuron~legacy-unit~openmp+shared+sympy+tests build_type=FastDebug model_tests=channel-benchmark,olfactory,tqperf-heavy

build:neuron:nmodl:nvhpc:acc:legacy:
  extends: [.build_neuron_nmodl, .spack_nvhpc]
  variables:
    SPACK_PACKAGE_SPEC: ~rx3d~caliper+gpu+coreneuron~legacy-unit~openmp~shared~sympy+tests~unified build_type=FastDebug model_tests=channel-benchmark,olfactory,tqperf-heavy

build:neuron:nmodl:nvhpc:acc:shared:
  extends: [.build_neuron_nmodl, .spack_nvhpc]
  variables:
    SPACK_PACKAGE_SPEC: ~rx3d~caliper+gpu+coreneuron~legacy-unit~openmp+shared+sympy+tests~unified build_type=FastDebug model_tests=channel-benchmark,olfactory,tqperf-heavy

build:neuron:nmodl:nvhpc:omp:legacy:
  extends: [.build_neuron_nmodl, .spack_nvhpc]
  variables:
    SPACK_PACKAGE_SPEC: ~rx3d+caliper+gpu+coreneuron~legacy-unit+openmp~shared~sympy+tests~unified build_type=FastDebug model_tests=channel-benchmark,olfactory,tqperf-heavy ^caliper+cuda cuda_arch=70

build:neuron:nmodl:nvhpc:omp:
  extends: [.build_neuron_nmodl, .spack_nvhpc]
  variables:
    SPACK_PACKAGE_SPEC: ~rx3d+caliper+gpu+coreneuron~legacy-unit+openmp~shared+sympy+tests~unified build_type=FastDebug model_tests=channel-benchmark,olfactory,tqperf-heavy ^caliper+cuda cuda_arch=70

# Test NEURON

test:neuron:nmodl:intel:legacy:
  extends: [.test_neuron]
  needs: ["build:neuron:nmodl:intel:legacy"]

test:neuron:nmodl:intel:shared:
  extends: [.test_neuron]
  needs: ["build:neuron:nmodl:intel:shared"]

test:neuron:nmodl:nvhpc:acc:legacy:
  extends: [.test_neuron, .gpu_node]
  needs: ["build:neuron:nmodl:nvhpc:acc:legacy"]

test:neuron:nmodl:nvhpc:acc:shared:
  extends: [.test_neuron, .gpu_node]
  needs: ["build:neuron:nmodl:nvhpc:acc:shared"]

test:neuron:nmodl:nvhpc:omp:legacy:
  extends: [.test_neuron, .gpu_node]
  needs: ["build:neuron:nmodl:nvhpc:omp:legacy"]

test:neuron:nmodl:nvhpc:omp:
  extends: [.test_neuron, .gpu_node]
  needs: ["build:neuron:nmodl:nvhpc:omp"]


# Container building
mac_m1_container_build:
  stage: .pre
  tags:
    - macos-arm64
  script:
    - if [ -z "${ARM64_IMAGE_TAG}" ]; then
    -   echo "Please set the ARM64_IMAGE_TAG variable"
    -   exit 1
    - fi
    - cd packaging/python
    - echo "Replacing symlinks with their targets to keep podman happy"
    - find . -type l -exec cp $(realpath {}) ./TEMP \; -exec rm {} \; -exec mv TEMP {} \;
    - ls -l
    - export BUILDAH_FORMAT=docker # enables ONBUILD instructions which are not OCI compatible
    - machine_status=$(podman machine inspect | awk '/State/ {print $2}' | tr -d '",')
      # If you start the machine yourself, make sure BUILDAH_FORMAT and the http proxy variables are set in the shell before doing so!
    - if [[ "${machine_status}" != "running" ]]; then
    -     echo "Machine is in ${machine_status} status - starting"
    -     podman machine start
    - fi
    - podman build -t neuronsimulator/neuron_wheel:${ARM64_IMAGE_TAG} --build-arg MANYLINUX_IMAGE=manylinux2014_aarch64 -f Dockerfile .
    - podman login -u ${DOCKER_HUB_USER} -p ${DOCKER_HUB_AUTH_TOKEN} docker.io
    - podman push neuronsimulator/neuron_wheel:${ARM64_IMAGE_TAG}
    - podman rmi localhost/neuronsimulator/neuron_wheel:${ARM64_IMAGE_TAG}
  rules:
    - if: $CI_PIPELINE_SOURCE == "web"
  when: manual

x86_64_container_build:
  stage: .pre
  image:
    name: quay.io/buildah/stable
    entrypoint: [""]
  variables:
    KUBERNETES_CPU_LIMIT: 4
    KUBERNETES_CPU_REQUEST: 2
    KUBERNETES_MEMORY_LIMIT: 8Gi
    KUBERNETES_MEMORY_REQUEST: 4Gi
  tags:
    - kubernetes
  rules:
    - if: $CI_PIPELINE_SOURCE == "web"
  when: manual
  script:
    - if [ -z "${X86_IMAGE_TAG}" ]; then
    -   echo "Please set the X86_IMAGE_TAG variable"
    -   exit 1
    - fi
    - export STORAGE_DRIVER=vfs    # allows to build inside containers without additional mounts
    - export BUILDAH_FORMAT=docker # enables ONBUILD instructions which are not OCI compatible
    - cd packaging/python
    - buildah bud --iidfile image_id -t neuronsimulator/neuron_wheel:${X86_IMAGE_TAG} -f Dockerfile .
    - buildah login -u ${DOCKER_HUB_USER} -p ${DOCKER_HUB_AUTH_TOKEN} docker.io
    - buildah push $(<image_id) docker://docker.io/neuronsimulator/neuron_wheel:${X86_IMAGE_TAG}