From 40e3f6684f2b5516766d39a7eb9da0f0c2d0f5e9 Mon Sep 17 00:00:00 2001 From: Teal Dulcet Date: Fri, 20 Oct 2023 06:40:05 -0700 Subject: [PATCH] Added CI and updated makemake.sh script. --- .github/workflows/ci.yml | 128 +++++++++++++++++++ makemake.sh | 265 +++++++++++++++++++++++++++++---------- 2 files changed, 327 insertions(+), 66 deletions(-) create mode 100644 .github/workflows/ci.yml mode change 100755 => 100644 makemake.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..7eba935f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,128 @@ +name: CI +on: + push: + pull_request: + schedule: + - cron: '0 0 1 * *' + +jobs: + Linux: + name: Mlucas Linux + + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.cc == 'clang' }} + strategy: + matrix: + os: [ubuntu-20.04, ubuntu-22.04] + cc: [gcc, clang] + fail-fast: false + env: + CC: ${{ matrix.cc }} + steps: + - uses: actions/checkout@v4 + - name: Before script + run: | + sed -i 's/-O3/-Og -fsanitize=address,undefined/' makemake.sh + $CC --version + - name: Script + run: | + set -x + bash -e -o pipefail -- makemake.sh + (cd obj; make clean) + for arg in '' '1word' '2word' '3word' '4word' 'nword'; do echo -e "\nMfactor $arg\n"; bash -e -o pipefail -- makemake.sh mfac $arg; (cd obj_mfac; make clean); done + echo -e '## Warnings\n```' >> $GITHUB_STEP_SUMMARY + grep 'warning:' obj/build.log | sed 's/\x1B\[\([0-9]\+\(;[0-9]\+\)*\)\?m//g' | awk '{ print $NF }' | sort | uniq -c | sort -nr >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + for s in t s m l; do time ./Mlucas -s $s -cpu "0:$(nproc --all)" |& tee -a test.log | grep -i 'error\|warn\|info'; done + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ matrix.os }}_${{ matrix.cc }}_mlucas + path: ${{ github.workspace }} + + GCC-analyzer: + name: GCC analyzer + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: GCC analyzer + run: | + set -x + gcc -c -fdiagnostics-color -g -O3 -march=native -DUSE_THREADS -fanalyzer src/*.c |& tee analyzer.log + rm -- *.o + echo -e '## GCC analyzer\n```' >> $GITHUB_STEP_SUMMARY + grep 'warning:' analyzer.log | sed 's/\x1B\[\([0-9]\+\(;[0-9]\+\)*\)\?m//g' | awk '{ print $NF }' | sort | uniq -c | sort -nr >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + Clang-Tidy: + name: Clang-Tidy + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Clang-Tidy + run: | + set -x + clang-tidy --use-color -checks='bugprone-*,cert-*,clang-analyzer-*,concurrency-*,misc-const-correctness,misc-redundant-expression,misc-unused-*,modernize-*,performance-*,portability-*,readability-const-return-type,readability-container-*,readability-duplicate-include,readability-else-after-return,readability-make-member-function-cons,readability-non-const-parameter,readability-redundant-*,readability-simplify-*,readability-string-compare,readability-use-anyofallof' -header-filter='.*' src/*.c -- -Wall -O3 -march=native -DUSE_THREADS |& tee clang-tidy.log + echo -e '## Clang-Tidy\n```' >> $GITHUB_STEP_SUMMARY + grep 'warning:' clang-tidy.log | sed 's/\x1B\[\([0-9]\+\(;[0-9]\+\)*\)\?m//g' | awk '{ print $NF }' | sort | uniq -c | sort -nr >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + Cppcheck: + name: Cppcheck + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install + run: | + sudo apt-get -yqq update + sudo apt-get -yqq install cppcheck + - name: Cppcheck + run: | + mkdir build + cppcheck --enable=all -DUSE_THREADS --force --cppcheck-build-dir=build --clang . + cppcheck --enable=all -DUSE_THREADS --force --cppcheck-build-dir=build --clang . &> cppcheck.log + echo -e '## Cppcheck\n```' >> $GITHUB_STEP_SUMMARY + grep '\(error\|warning\|style\|performance\|portability\|information\):' cppcheck.log | awk '{ print $2, $NF }' | sort | uniq -c | sort -nr >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + ShellCheck: + name: ShellCheck + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: ShellCheck + run: bash -c 'shopt -s globstar; shellcheck -o avoid-nullary-conditions,check-extra-masked-returns,check-set-e-suppressed,deprecate-which,quote-safe-variables,require-double-brackets -s bash **/*.sh' + continue-on-error: true + + macOS: + name: Mlucas macOS + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-11, macos-12, macos-13] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: Before script + run: | + sed -i '' 's/-O3/-Og -fsanitize=address,undefined/' makemake.sh + - name: Script + run: | + set -x + bash -e -o pipefail -- makemake.sh + (cd obj; make clean) + for arg in '' '1word' '2word' '3word' '4word' 'nword'; do echo -e "\nMfactor $arg\n"; bash -e -o pipefail -- makemake.sh mfac $arg; (cd obj_mfac; make clean); done + echo -e '## Warnings\n```' >> $GITHUB_STEP_SUMMARY + grep 'warning:' obj/build.log | sed 's/\x1B\[\([0-9]\+\(;[0-9]\+\)*\)\?m//g' | awk '{ print $NF }' | sort | uniq -c | sort -nr >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + for s in t s m l; do time ./Mlucas -s $s -cpu "0:$(sysctl -n hw.ncpu)" |& tee -a test.log | grep -i 'error\|warn\|info'; done + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ matrix.os }}_mlucas + path: ${{ github.workspace }} diff --git a/makemake.sh b/makemake.sh old mode 100755 new mode 100644 index 067664c4..56b46984 --- a/makemake.sh +++ b/makemake.sh @@ -6,7 +6,7 @@ ################################################################################ # # -# (C) 2021 by Ernst W. Mayer. # +# (C) 2021 by Ernst W. Mayer and Teal Dulcet. # # # # This program is free software; you can redistribute it and/or modify it # # under the terms of the GNU General Public License as published by the # @@ -26,7 +26,7 @@ # # ################################################################################ -# Exit if any of the sommands fail: +# Exit if any of the commands fail: set -e # for mode in avx512 avx2 avx sse2; do @@ -37,21 +37,18 @@ set -e # fi # done -# $0 contains script-name, but $@ starts with first ensuing cmd-line arg, if it exists: -echo "Total number of arguments : $#" -for ((i=1; i<=$#; ++i)); do - echo "Arg[$i] = ${!i}" -done -if [[ $# -gt 1 ]]; then - echo "Usage: $0 [SIMD build mode]" >&2 - echo "Only 1 optional argument supported, it must be one of the supported SIMD-arithmetic types:" >&2 - echo -e "\t[x86_64: avx512_skylake avx512_knl avx2 avx sse2]; [Armv8: asimd]; or 'nosimd' for scalar-double build.\n" >&2 - exit 1 -fi - DIR=obj -EXE=Mlucas -ARGS=() +Mlucas=Mlucas +Mfactor=Mfactor +TARGET=$Mlucas +ARGS=(-DUSE_THREADS) # Optional compile args +WORDS='' +# Optional link args +LARG=() + +MODES=() +GMP=1 +HWLOC=0 if echo "$OSTYPE" | grep -iq 'darwin'; then echo -e "MacOS detected for build host.\n" @@ -61,111 +58,247 @@ else # echo "$OSTYPE" | grep -iq 'linux' CPU_THREADS=$(nproc --all) fi -# Thx to tdulcet for streamlined case-based syntax here, but ugh - non-matching ')', really?: -if [[ $# -eq 1 ]]; then - - if [ "$1" = 'avx512_skylake' ]; then - echo "Building for avx512_skylake SIMD in directory obj_$1; the executable will be named Mlucas_$1" - ARGS+=( "-DUSE_AVX512" -march=skylake-avx512 ) - elif [ "$1" = 'avx512_knl' ]; then - echo "Building for avx2 SIMD in directory obj_$1; the executable will be named Mlucas_$1" - ARGS+=( "-DUSE_AVX512" -march=knl ) - elif [ "$1" = 'avx2' ]; then - echo "Building for avx2 SIMD in directory obj_$1; the executable will be named Mlucas_$1" - ARGS+=( "-DUSE_AVX2" -mavx2 ) - elif [ "$1" = 'avx' ]; then - echo "Building for avx SIMD in directory obj_$1; the executable will be named Mlucas_$1" - ARGS+=( "-DUSE_AVX" -mavx ) - elif [ "$1" = 'sse2' ]; then - echo "Building for sse2 SIMD in directory obj_$1; the executable will be named Mlucas_$1" - ARGS+=( "-DUSE_SSE2" ) - elif [ "$1" = 'asimd' ]; then - echo "Building for avx2 SIMD in directory obj_$1; the executable will be named Mlucas_$1" - ARGS+=( "-DUSE_ARM_V8_SIMD" ) - elif [ "$1" = 'nosimd' ]; then - echo "Building in scalar-double (no-SIMD) mode in directory obj_$1; the executable will be named Mlucas_$1" +if ! command -v make >/dev/null; then + echo "Error: This script requires Make" >&2 + echo "On Ubuntu and Debian run: 'sudo apt-get update' and 'sudo apt-get install build-essential -y'" >&2 + exit 1 +fi +if [[ -n $CC ]]; then + if ! command -v "$CC" >/dev/null; then + echo "Error: $CC is not installed." >&2 + exit 1 + fi +elif ! command -v gcc >/dev/null; then + echo "Error: This script requires the GNU C compiler" >&2 + echo "On Ubuntu and Debian run: 'sudo apt-get update' and 'sudo apt-get install build-essential -y'" >&2 + exit 1 +fi + +# $0 contains script-name, but $@ starts with first ensuing cmd-line arg, if it exists: +echo "Total number of input parameters = $#" + +# v21: Keep the cross-platform-build arch-specifying command-line flag, but now also need to +# support several added ones for 3rd-party-library usage. This needs to be in arbitrary argument +# order fashion, [details snipped] +arglist=("$@") # Local array into which we copy cmd-line args in order to be able to manipulate them +for i in "${!arglist[@]}"; do + echo "Arg[$i] = ${arglist[i]}" +done +# Now loop over the optional args and execute the above-described preprocessing step: +for arg in "$@"; do + + case ${arg,,} in + 'no_gmp') + GMP=0 + ;; + 'use_hwloc') + HWLOC=1 + ;; + 'avx512_skylake' | 'avx512_knl' | 'k1om' | 'avx2' | 'avx' | 'sse2' | 'asimd' | 'nosimd') + MODES+=("$arg") + ;; + 'mfac') + TARGET=$Mfactor + ;; + '1word' | '2word' | '3word' | '4word' | 'nword') + WORDS=$arg + ;; + *) + echo "Usage: $0 [SIMD build mode]" >&2 + echo "Optional arguments must be 'no_gmp', 'use_hwloc' or one and only one of the supported SIMD-arithmetic types:" >&2 + echo -e "\t[x86_64: avx512_skylake avx512_knl k1om avx2 avx sse2]; [Armv8: asimd]; or 'nosimd' for scalar-double build.\n" >&2 + exit 1 + ;; + esac + +done + +if ((GMP)); then + LARG+=(-lgmp) +else + echo "Building sans Gnu-MP ... this means no GCDs will be taken in p-1 work." + ARGS+=(-DINCLUDE_GMP=0) +fi + +if ((HWLOC)); then + echo "Building with HWLOC hardware-topology support." + ARGS+=(-DINCLUDE_HWLOC=1) + LARG+=(-lhwloc) +fi + +if [[ $TARGET == "$Mfactor" ]]; then + DIR+=_mfac + trap "rm $PWD/src/factor.c" EXIT + cp -vf src/factor.c{.txt,} +fi + +if [[ -n $WORDS ]]; then + if [[ $TARGET == "$Mfactor" ]]; then + arg=$WORDS + if [[ ${arg,,} == 'nword' ]]; then + WORDS=-D"${arg^^}" + else + WORDS=-DP"${arg^^}" + fi + Mfactor+="_$arg" + TARGET=$Mfactor + else + echo "Error: The argument '$WORDS' requires 'mfac'." >&2 + exit 1 + fi +fi + +# First if/elif clause handles cross-platform builds and non-default values for "Use GMP?" and "Use HWLOC?": +# o "Use GMP" = TRUE is default in link step, 'no_gmp' overrides; +# o "Use HWLOC" = FALSE is default, 'use_hwloc' overrides. +# Thx to tdulcet for offering a streamlined case-based syntax here, but ugh - non-matching ')', really?: +if [[ ${#MODES[*]} -gt 1 ]]; then + echo -e "Only one arch-specifying optional argument is allowed ... aborting." >&2 + exit 1 +fi + +if [[ ${#MODES[*]} -eq 1 ]]; then + + arg=${MODES[0],,} + + if [[ $arg == 'avx512_skylake' ]]; then + echo "Building for avx512_skylake SIMD in directory '${DIR}_${arg}'; the executable will be named '${TARGET}'" + ARGS+=(-DUSE_AVX512 -march=skylake-avx512) + elif [[ $arg == 'avx512_knl' ]]; then + echo "Building for avx512_knl SIMD in directory '${DIR}_${arg}'; the executable will be named '${TARGET}'" + ARGS+=(-DUSE_AVX512 -march=knl) + elif [[ $arg == 'k1om' ]]; then + echo "Building for 1st-gen Xeon Phi 512-bit SIMD in directory '${DIR}_${arg}'; the executable will be named '${TARGET}'" + ARGS+=(-DUSE_IMCI512) + elif [[ $arg == 'avx2' ]]; then + echo "Building for avx2 SIMD in directory '${DIR}_${arg}'; the executable will be named '${TARGET}'" + ARGS+=(-DUSE_AVX2 -mavx2) + elif [[ $arg == 'avx' ]]; then + echo "Building for avx SIMD in directory '${DIR}_${arg}'; the executable will be named '${TARGET}'" + ARGS+=(-DUSE_AVX -mavx) + elif [[ $arg == 'sse2' ]]; then + echo "Building for sse2 SIMD in directory '${DIR}_${arg}'; the executable will be named '${TARGET}'" + ARGS+=(-DUSE_SSE2) + elif [[ $arg == 'asimd' ]]; then + echo "Building for asimd SIMD in directory '${DIR}_${arg}'; the executable will be named '${TARGET}'" + ARGS+=(-DUSE_ARM_V8_SIMD) + elif [[ $arg == 'nosimd' ]]; then + echo "Building in scalar-double (no-SIMD) mode in directory '${DIR}_${arg}'; the executable will be named '${TARGET}'" # This one's a no-op else - echo "Unrecognized SIMD-build flag ... aborting." + echo "Unrecognized SIMD-build flag ... aborting." >&2 exit 1 fi - DIR+="_$1" - EXE+="_$1" + DIR+="_$arg" -elif uname -a | grep -iq 'Mac'; then +elif echo "$OSTYPE" | grep -iq 'darwin'; then - echo -e "MacOS detected.\n" + # MacOS: if sysctl -a | grep machdep.cpu.features | grep -iq 'avx512'; then echo -e "The CPU supports the AVX512 SIMD build mode.\n" - ARGS+=( "-DUSE_AVX512" -march=native ) + ARGS+=(-DUSE_AVX512 -march=native) elif sysctl -a | grep machdep.cpu.features | grep -iq 'avx2'; then echo -e "The CPU supports the AVX2 SIMD build mode.\n" - ARGS+=( "-DUSE_AVX2" -march=native -mavx2 ) + ARGS+=(-DUSE_AVX2 -march=native -mavx2) elif sysctl -a | grep machdep.cpu.features | grep -iq 'avx'; then echo -e "The CPU supports the AVX SIMD build mode.\n" - ARGS+=( "-DUSE_AVX" -march=native -mavx ) + ARGS+=(-DUSE_AVX -march=native -mavx) elif sysctl -a | grep machdep.cpu.features | grep -iq 'sse2'; then echo -e "The CPU supports the SSE2 SIMD build mode.\n" - ARGS+=( "-DUSE_SSE2" -march=native ) + # On my Core2Duo Mac, 'native' gives "error: bad value for -march= switch": + ARGS+=(-DUSE_SSE2 -march=core2) elif sysctl -a | grep machdep.cpu.features | grep -iq 'asimd'; then echo -e "The CPU supports the ASIMD build mode.\n" - ARGS+=( "-DUSE_ARM_V8_SIMD" -march=native ) + ARGS+=(-DUSE_ARM_V8_SIMD -march=native) else echo -e "The CPU supports no Mlucas-recognized ASIMD build mode ... building in scalar-double mode.\n" - ARGS+=( -march=native ) + ARGS+=(-march=native) fi else - echo -e "Assuming OS = Linux.\n" + # Linux: if grep -iq 'avx512' /proc/cpuinfo; then echo -e "The CPU supports the AVX512 SIMD build mode.\n" - ARGS+=( "-DUSE_AVX512" -march=native ) + ARGS+=(-DUSE_AVX512 -march=native) elif grep -iq 'avx2' /proc/cpuinfo; then echo -e "The CPU supports the AVX2 SIMD build mode.\n" - ARGS+=( "-DUSE_AVX2" -march=native -mavx2 ) + ARGS+=(-DUSE_AVX2 -march=native -mavx2) elif grep -iq 'avx' /proc/cpuinfo; then echo -e "The CPU supports the AVX SIMD build mode.\n" - ARGS+=( "-DUSE_AVX" -march=native -mavx ) + ARGS+=(-DUSE_AVX -march=native -mavx) elif grep -iq 'sse2' /proc/cpuinfo; then echo -e "The CPU supports the SSE2 SIMD build mode.\n" - ARGS+=( "-DUSE_SSE2" -march=native ) + ARGS+=(-DUSE_SSE2 -march=native) elif grep -iq 'asimd' /proc/cpuinfo; then echo -e "The CPU supports the ASIMD build mode.\n" - ARGS+=( "-DUSE_ARM_V8_SIMD" -march=native ) + ARGS+=(-DUSE_ARM_V8_SIMD -march=native) else echo -e "The CPU supports no Mlucas-recognized ASIMD build mode ... building in scalar-double mode.\n" - ARGS+=( -march=native ) + ARGS+=(-march=native) fi fi +if [[ -d $DIR ]]; then + echo "Warning: '$DIR' already exists" +fi + # -p prevents "File exists" warning if obj-dir already exists: mkdir -p "$DIR" cd "$DIR" +if [[ -x $TARGET ]]; then + echo "Error: '$DIR/$TARGET' already exists." >&2 + exit 1 +fi + # Clang-under-MacOS linker barfs if one tries to explicitly invoke standard libs - h/t tdulcet for the # conditional-inline syntax. Some OSes put the GMP headers in /usr/local/include, so -I that path in the # compile command. If said path does not exist, make silently ignores it. # Re. the -g flag to include the debugging symbols, they bloat executable size but if someone's Mlucas # crashes/segfaults, one can rerun with GDB (gdb -ex=r ./Mlucas) to see the filename, line number and # stack trace of the issue. If one wishes, one can run 'strip -g Mlucas' to remove the debugging symbols: -cat << EOF > Makefile +cat <Makefile CC?=gcc +CFLAGS=-fdiagnostics-color -Wall -g -O3 # -flto=auto +CPPFLAGS=-I/usr/local/include +LDLIBS=$(echo "$OSTYPE" | grep -iq 'darwin' || echo "-lm -lpthread -lrt") ${LARG[@]} + OBJS=\$(patsubst ../src/%.c, %.o, \$(wildcard ../src/*.c)) +OBJS_MFAC=getRealTime.o get_cpuid.o get_fft_radices.o get_fp_rnd_const.o imul_macro.o mi64.o qfloat.o rng_isaac.o \$(patsubst ../src/%.c, %.o, \$(wildcard ../src/two*.c)) types.o util.o threadpool.o factor.o -$EXE: \$(OBJS) - \$(CC) -Wall -g -o \$@ \$(OBJS) $(echo "$OSTYPE" | grep -iq 'darwin' || echo "-lm -lpthread -lrt") -lgmp +$Mlucas: \$(OBJS) + \$(CC) \$(LDFLAGS) \$(CFLAGS) -o \$@ \$^ \$(LDLIBS) +$Mfactor: \$(OBJS_MFAC) + \$(CC) \$(LDFLAGS) \$(CFLAGS) -o \$@ \$^ \$(LDLIBS) +factor.o: ../src/factor.c + \$(CC) \$(CFLAGS) \$(CPPFLAGS) -c ${ARGS[@]} -DFACTOR_STANDALONE $WORDS -DTRYQ=4 \$< %.o: ../src/%.c - \$(CC) -Wall -g -c -I/usr/local/include -O3 ${ARGS[@]} -DUSE_THREADS \$< + \$(CC) \$(CFLAGS) \$(CPPFLAGS) -c ${ARGS[@]} \$< clean: rm -f *.o + +.phony: clean EOF -echo -e "\nBuilding Mlucas" -printf "%s CPU cores detected ... parallel-building using that number of make threads.\n" "$CPU_THREADS" -if ! make -j "$CPU_THREADS" > build.log 2>&1; then - echo -e "There were build errors - see build.log for details.\n" - exit 1 +if [[ -e build.log ]]; then + cp -vf --backup=t build.log{,} +fi + +echo -e "Building $TARGET" +printf "%'d CPU cores detected ... parallel-building using that number of make threads.\n" "$CPU_THREADS" +if ! time make -O -j "$CPU_THREADS" "$TARGET" &>build.log; then + echo -e "\n*** There were build errors - see '${DIR}/build.log' for details. ***\n" >&2 + grep -A 2 'error:' build.log || tail build.log + # exit 1 fi + +echo -e "\nWarnings:\n" +grep 'warning:' build.log | awk '{ print $NF }' | sort | uniq -c | sort -nr + +echo -e "\nErrors:\n" +grep -A 2 'error:' build.log || echo "None" + +echo