GHI #37 Add Logic Tests #259
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run All Tests | |
| on: | |
| pull_request: | |
| branches: | |
| - '**' | |
| schedule: | |
| - cron: "0 0 * * *" | |
| jobs: | |
| test-native: | |
| if: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # verbose labels make things easier to read in GitHub Actions | |
| # platform gets converted to os, preset, compiler, architecture | |
| platform: | |
| - windows-gcc-x86_64 | |
| - windows-msvc-x86_32 | |
| - windows-msvc-x86_64 | |
| - macos-clang-x86_64 | |
| - linux-clang-x86_64 | |
| - linux-gcc-x86_64 | |
| # convert verbose labels to usable values (which don't appear in GitHub Actions GUI) | |
| include: | |
| # platform -> os, preset, compiler, architecture | |
| - platform: windows-gcc-x86_64 | |
| os: windows | |
| preset: patomic-ci-native-unix-gcc | |
| architecture: x86_64 | |
| - platform: windows-msvc-x86_32 | |
| os: windows | |
| preset: patomic-ci-native-win32-msvc | |
| architecture: x86_32 | |
| - platform: windows-msvc-x86_64 | |
| os: windows | |
| preset: patomic-ci-native-win64-msvc | |
| architecture: x86_64 | |
| - platform: macos-clang-x86_64 | |
| os: macos | |
| preset: patomic-ci-native-unix-clang | |
| architecture: x86_64 | |
| - platform: linux-clang-x86_64 | |
| os: ubuntu | |
| preset: patomic-ci-native-unix-clang | |
| architecture: x86_64 | |
| - platform: linux-gcc-x86_64 | |
| os: ubuntu | |
| preset: patomic-ci-native-unix-gcc | |
| architecture: x86_64 | |
| uses: ./.github/workflows/_reusable-test-native.yml | |
| with: | |
| os: ${{ matrix.os }} | |
| preset: ${{ matrix.preset }} | |
| architecture: ${{ matrix.architecture }} | |
| test-qemu: | |
| if: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # architecture gets converted to triple | |
| # short form here so that it doesn't take up the whole GitHub Action name | |
| architecture: | |
| - aarch64 | |
| - alpha | |
| - arm | |
| - armhf | |
| - hppa | |
| - m68k | |
| - mips | |
| - mips64 | |
| - mips64el | |
| - mipsel | |
| - ppc | |
| - ppc64 | |
| - ppc64le | |
| - riscv64 | |
| - s390x | |
| - sh4 | |
| - sparc64 | |
| - x86_32 | |
| # convert architectures to triples | |
| include: | |
| - architecture: aarch64 | |
| triple: aarch64-linux-gnu | |
| - architecture: alpha | |
| triple: alpha-linux-gnu | |
| skip_llvm: true # unsupported | |
| - architecture: arm | |
| triple: arm-linux-gnueabi | |
| - architecture: armhf | |
| triple: arm-linux-gnueabihf | |
| - architecture: hppa | |
| triple: hppa-linux-gnu | |
| skip_llvm: true # unsupported | |
| - architecture: m68k | |
| triple: m68k-linux-gnu | |
| skip_gcc: true # fixme: segfaults qemu (GHI #25) | |
| skip_llvm: true # fixme: ICEs clang (GHI #25) | |
| - architecture: mips | |
| triple: mips-linux-gnu | |
| - architecture: mips64 | |
| triple: mips64-linux-gnuabi64 | |
| - architecture: mips64el | |
| triple: mips64el-linux-gnuabi64 | |
| - architecture: mipsel | |
| triple: mipsel-linux-gnu | |
| - architecture: ppc | |
| triple: powerpc-linux-gnu | |
| - architecture: ppc64 | |
| triple: powerpc64-linux-gnu | |
| - architecture: ppc64le | |
| triple: powerpc64le-linux-gnu | |
| - architecture: riscv64 | |
| triple: riscv64-linux-gnu | |
| skip_gcc: true # fixme: errors on system header (GHI #25) | |
| - architecture: s390x | |
| triple: s390x-linux-gnu | |
| - architecture: sh4 | |
| triple: sh4-linux-gnu | |
| skip_gcc: true # fixme: segfaults qemu (GHI #25) | |
| skip_llvm: true # unsupported | |
| - architecture: sparc64 | |
| triple: sparc64-linux-gnu | |
| skip_gcc: true # fixme: segfaults qemu (GHI #25) | |
| - architecture: x86_32 | |
| triple: i686-linux-gnu | |
| skip_llvm: true # fixme: supported but not sure how (GHI #25) | |
| uses: ./.github/workflows/_reusable-test-qemu.yml | |
| with: | |
| triple: ${{ matrix.triple }} | |
| architecture: ${{ matrix.architecture }} | |
| skip_gcc: ${{ matrix.skip_gcc == true }} | |
| skip_llvm: ${{ matrix.skip_llvm == true }} | |
| publish-results: | |
| if: false | |
| runs-on: ubuntu-latest | |
| needs: | |
| - test-native | |
| - test-qemu | |
| steps: | |
| - name: Merge Test Result Artifacts | |
| uses: actions/upload-artifact/merge@v4 | |
| with: | |
| name: test-results | |
| pattern: test-results-* | |
| delete-merged: true | |
| - name: Download Test Result Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: test-results | |
| path: test-results/ | |
| - name: Publish Test Results | |
| uses: EnricoMi/publish-unit-test-result-action@v2 | |
| with: | |
| action_fail: true | |
| action_fail_on_inconclusive: true | |
| check_name: "Test Results" | |
| files: test-results/**/*.xml | |
| publish-coverage: | |
| if: false | |
| runs-on: ubuntu-latest | |
| needs: | |
| - test-native | |
| - test-qemu | |
| env: | |
| HI_LIMIT: 100 | |
| MED_LIMIT: 90 | |
| steps: | |
| - name: Install Dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install binutils # for c++filt | |
| sudo apt install lcov | |
| - name: Merge Test Coverage Artifacts | |
| uses: actions/upload-artifact/merge@v4 | |
| with: | |
| name: test-coverage-internal | |
| pattern: test-coverage-internal-* | |
| delete-merged: true | |
| - name: Download Test Coverage Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: test-coverage-internal | |
| path: test-coverage-internal/ | |
| - name: Checkout patomic | |
| uses: actions/checkout@v4 | |
| with: | |
| path: patomic | |
| - name: Copy patomic To All Original Mapping Paths | |
| run: | | |
| # go through all patomic.rootpath files | |
| find ./test-coverage-internal -type f -name '*patomic.rootpath' -print0 | while IFS= read -r -d '' rp_file; do | |
| root_path=$(cat "${rp_file}") | |
| # create root_path if it doesn't exist | |
| if [ ! -d "${root_path}" ]; then | |
| sudo mkdir -p "${root_path}" | |
| fi | |
| # ensure full permissions even if it exists | |
| sudo chmod -R a+rwx "${root_path}" | |
| # copy patomic to root_path if not already there | |
| if [ ! -d "${root_path}/patomic" ]; then | |
| cp -R ./patomic/ "${root_path}/patomic" | |
| fi | |
| done | |
| - name: Create Lcov Config File | |
| run: | | |
| touch ./lcovrc | |
| echo "genhtml_hi_limit = ${{ env.HI_LIMIT }}" >> ./lcovrc | |
| echo "genhtml_med_limit = ${{ env.MED_LIMIT }}" >> ./lcovrc | |
| echo "genhtml_function_hi_limit = ${{ env.HI_LIMIT }}" >> ./lcovrc | |
| echo "genhtml_function_med_limit = ${{ env.MED_LIMIT }}" >> ./lcovrc | |
| echo "lcov_branch_coverage = 1" >> ./lcovrc | |
| echo "Contents of lcovrc:" | |
| cat ./lcovrc | |
| sudo cp ./lcovrc /etc/lcovrc | |
| - name: Generate HTML Files | |
| run: | | |
| # directory into which to put user artifacts | |
| mkdir test-coverage | |
| # keep only coverage data relating to patomic source files | |
| # the include tracefile may be empty with clang, hence ignoring the errors | |
| find ./test-coverage-internal -mindepth 1 -maxdepth 1 -type d -not -path '*/.*' -print0 | while IFS= read -r -d '' dir; do | |
| mv "${dir}/patomic.lcov" "${dir}/patomic.lcov.old" | |
| root_path=$(cat "${dir}/patomic.rootpath") | |
| lcov --output-file "${dir}/patomic.lcov.include" --extract "${dir}/patomic.lcov.old" "${root_path}/patomic/include/*" --ignore-errors empty,unused | |
| lcov --output-file "${dir}/patomic.lcov.src" --extract "${dir}/patomic.lcov.old" "${root_path}/patomic/src/*" | |
| lcov --output-file "${dir}/patomic.lcov" --add-tracefile "${dir}/patomic.lcov.include" --add-tracefile "${dir}/patomic.lcov.src" --ignore-errors empty | |
| done | |
| # generate html files for each separate compilation | |
| find ./test-coverage-internal -mindepth 1 -maxdepth 1 -type d -not -path '*/.*' -print0 | while IFS= read -r -d '' dir; do | |
| arch=$(basename "${dir}") | |
| mkdir "./test-coverage/${arch}" | |
| genhtml --output-directory "./test-coverage/${arch}" --title "patomic-${arch}" --show-details --num-spaces 4 --legend --demangle-cpp --precision 2 "./test-coverage-internal/${arch}/patomic.lcov" | |
| done | |
| # generate html files for combined coverage | |
| mkdir ./test-coverage-internal/universal | |
| mkdir ./test-coverage/universal | |
| lcov_combine_command=("lcov") | |
| while IFS= read -r -d '' lcov_file; do | |
| lcov_combine_command+=("-a" "${lcov_file}") | |
| done < <(find ./test-coverage-internal -type f -name '*patomic.lcov' -print0) | |
| lcov_combine_command+=("-o" "./test-coverage-internal/universal/patomic.lcov") | |
| "${lcov_combine_command[@]}" | |
| genhtml --output-directory "./test-coverage/universal" --title "patomic-universal" --show-details --num-spaces 4 --legend --demangle-cpp --precision 2 ./test-coverage-internal/universal/patomic.lcov | |
| - name: Upload Coverage HTML Results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-coverage | |
| path: test-coverage/ | |
| - name: Parse Lcov Files Into Json | |
| shell: python | |
| run: | | |
| import json, os, pathlib, re, subprocess | |
| # constants | |
| BASE_DIR = pathlib.Path("./test-coverage-internal") | |
| OUT_FILE = pathlib.Path("./patomic.json") | |
| # iterate through all non-hidden directories | |
| archs = [d for d in next(os.walk(BASE_DIR))[1] if d[0] != '.'] | |
| json_out = {} | |
| for arch in archs: | |
| json_out[arch] = {} | |
| # get lcov summary | |
| cmd = f"lcov --summary {BASE_DIR / arch / 'patomic.lcov'}" | |
| lcov_summary = subprocess.run(cmd, capture_output=True, shell=True, check=True).stdout.decode() | |
| # go through lcov summary | |
| for line in lcov_summary.split('\n'): | |
| # skip non-info lines | |
| if not line.lstrip().startswith(("lines", "functions", "branches")): | |
| continue | |
| attr_name = line.lstrip().split('.', maxsplit=1)[0] | |
| # parse coverage stats | |
| if "no data found" in line: | |
| json_out[arch][attr_name] = { "percentage": 100.0, "count": 0, "total": 0 } | |
| else: | |
| pattern = r"(\d+\.\d+)% \((\d+) of (\d+)" | |
| matches = re.findall(pattern, line)[0] | |
| json_out[arch][attr_name] = { "percentage": float(matches[0]), "count": int(matches[1]), "total": int(matches[2]) } | |
| # write results to json file | |
| with open(OUT_FILE, 'w') as f: | |
| json.dump(json_out, f, indent=4) | |
| - name: Create Coverage Summary File | |
| id: coverage-summary | |
| shell: python | |
| run: | | |
| import json, os, pathlib | |
| # constants | |
| HI_LIMIT, MED_LIMIT = ${{ env.HI_LIMIT }}, ${{ env.MED_LIMIT }} | |
| IN_FILE = pathlib.Path("./patomic.json") | |
| OUT_FILE = pathlib.Path("./patomic.md") | |
| # markdown prologue | |
| md_string = "## Test Coverage\n" | |
| md_string += "| Target | Lines | Functions | Branches |\n" | |
| md_string += "| :----- | ----: | --------: | -------: |\n" | |
| # keep track of issues | |
| any_fail = False | |
| universal_warn = False | |
| # go through json file | |
| with open(IN_FILE, 'r') as fp: | |
| j = json.load(fp) | |
| # sort archs except for 'universal' first | |
| archs = sorted(list(j.keys()), key=lambda a: a if a != 'universal' else '\0') | |
| # helper lambdas | |
| make_emoji = lambda p: "✅" if p >= HI_LIMIT else ("⚠️" if p >= MED_LIMIT else "🛑") | |
| field_2str = lambda f: f"{float(f['percentage']):.2f}% ({f['count']}/{f['total']}) {make_emoji(float(f['percentage']))}" | |
| # write summary of each arch to markdown file | |
| for arch in archs: | |
| line = f"| `{arch}` | `{field_2str(j[arch]['lines'])}` | `{field_2str(j[arch]['functions'])}` | `{field_2str(j[arch]['branches'])}` |\n" | |
| md_string += line | |
| # check for warnings | |
| any_fail = any_fail or "🛑" in line | |
| universal_warn = universal_warn or (arch == 'universal' and "⚠️" in line) | |
| # markdown epilogue | |
| sha = "${{ github.event.pull_request.head.sha }}" | |
| md_string += f"\n`✅ >= {HI_LIMIT}`, `⚠️ >= {MED_LIMIT}`, `🛑 < {MED_LIMIT}`\n" | |
| md_string += f"\nResults for commit [`{sha[:8]}`](${{ github.server_url }}/${{ github.repository }}/commit/{sha}).\n" | |
| # write to file | |
| with open(OUT_FILE, 'wb') as fp: | |
| fp.write(md_string.encode()) | |
| # write checks to output | |
| with open(os.environ["GITHUB_OUTPUT"], 'a') as fp: | |
| print(f"any_fail={int(any_fail)}", file=fp) | |
| print(f"universal_warn={int(universal_warn)}", file=fp) | |
| - name: Publish Coverage in Job Summary | |
| run: | | |
| echo "$(cat ./patomic.md)" >> $GITHUB_STEP_SUMMARY | |
| - name: Publish Coverage in Pull Request Comment | |
| uses: thollander/actions-comment-pull-request@v2 | |
| with: | |
| comment_tag: coverage-summary | |
| filePath: patomic.md | |
| - name: Fail if Any Coverage Fails or Universal Coverage Warns | |
| run: | | |
| if [[ '${{ steps.coverage-summary.outputs.any_fail }}' != '0' ]]; then | |
| msg="At least one coverage result was below the threshold of ${{ env.MED_LIMIT }}% (a.k.a. 🛑)" | |
| echo "::error title=Any Coverage Failure::${msg}" | |
| exit 1 | |
| elif [[ '${{ steps.coverage-summary.outputs.universal_warn }}' != '0' ]]; then | |
| msg="Universal coverage result was below the threshold of ${{ env.HI_LIMIT }}% (a.k.a. ⚠️)" | |
| echo "::error title=Universal Coverage Warning::${msg}" | |
| exit 1 | |
| fi | |
| # keep this last so that we have it for debugging if something fails | |
| - name: Delete Test Coverage Artifacts | |
| uses: geekyeggo/delete-artifact@v2 | |
| with: | |
| name: test-coverage-internal |