ci: improve parallel builds by basing on number of cores (AcademySoft…

…wareFoundation#4115) With GitHub runners with more cores becoming available, adjust our CI scripts to do parallel builds with nprocs+2 instead of hard-coding. We use slightly more than the number of available cores because these builds and tests tend to be I/O bound, so they can be a little over-threaded to get maximum utilization. Signed-off-by: Larry Gritz <[email protected]>
lgritz · Jan 27, 2024 · a474188 · a474188
1 parent c3fda6a
commit a474188
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 6 deletions.
diff --git a/.github/workflows/macarm.yml b/.github/workflows/macarm.yml
@@ -54,7 +54,7 @@ jobs:
             cxx_std: 17
             python_ver: "3.11"
             aclang: 14
-            setenvs: export CTEST_TEST_TIMEOUT=600 USE_OPENCV=0 USE_QT=0 PARALLEL=8
+            setenvs: export CTEST_TEST_TIMEOUT=600 USE_OPENCV=0 USE_QT=0
     runs-on: ${{ matrix.os }}
     env:
       CC: clang

diff --git a/src/build-scripts/ci-startup.bash b/src/build-scripts/ci-startup.bash
@@ -39,11 +39,6 @@ export CMAKE_GENERATOR=${CMAKE_GENERATOR:=Ninja}
 export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:=Release}
 export CMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD:=11}
 
-export PARALLEL=${PARALLEL:=4}
-export PAR_MAKEFLAGS=-j${PARALLEL}
-export CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:=${PARALLEL}}
-export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:=${PARALLEL}}
-
 export LOCAL_DEPS_DIR=${LOCAL_DEPS_DIR:=$HOME/ext}
 export PATH=${LOCAL_DEPS_DIR}/dist/bin:$PATH
 export LD_LIBRARY_PATH=${LOCAL_DEPS_DIR}/dist/lib:$LD_LIBRARY_PATH
@@ -57,6 +52,25 @@ export TESTSUITE_CLEANUP_ON_SUCCESS=${TESTSUITE_CLEANUP_ON_SUCCESS:=1}
 export BUILD_WRAPPER_OUT_DIR="${PWD}/bw_output"
 export BW_OUTPUT_DIR="${PWD}/bw_output"
 
+# Parallel builds
+if [[ `uname -s` == "Linux" ]] ; then
+    echo "procs: " `nproc`
+    head -40 /proc/cpuinfo
+    export PARALLEL=${PARALLEL:=$((2 + `nproc`))}
+elif [[ "${RUNNER_OS}" == "macOS" ]] ; then
+    echo "procs: " `sysctl -n hw.ncpu`
+    sysctl machdep.cpu.features
+    export PARALLEL=${PARALLEL:=$((2 + `sysctl -n hw.ncpu`))}
+elif [[ "${RUNNER_OS}" == "Windows" ]] ; then
+    # Presumably Windows
+    export PARALLEL=${PARALLEL:=$((2 + ${NUMBER_OF_PROCESSORS}))}
+else
+    export PARALLEL=${PARALLEL:=6}
+fi
+export PAR_MAKEFLAGS=-j${PARALLEL}
+export CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:=${PARALLEL}}
+export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:=${PARALLEL}}
+
 
 mkdir -p build dist
 
@@ -72,8 +86,10 @@ ls
 env | sort
 
 if [[ `uname -s` == "Linux" ]] ; then
+    echo "nprocs: " `nproc`
     head -40 /proc/cpuinfo
 elif [[ "${RUNNER_OS}" == "macOS" ]] ; then
+    echo "nprocs: " `sysctl -n hw.ncpu`
     sysctl machdep.cpu.features
 fi