From a7b0865888162af41d1128525975dde5bcc9ac61 Mon Sep 17 00:00:00 2001 From: James Netherton Date: Wed, 8 May 2024 07:34:46 +0100 Subject: [PATCH] Add GitHub Action to report test failures in the workflow summary --- .../actions/test-summary-report/action.yaml | 91 +++++++++++++++++++ .github/workflows/camel-master-cron.yaml | 50 +++++++++- .github/workflows/ci-build.yaml | 50 +++++++++- .github/workflows/quarkus-master-cron.yaml | 50 +++++++++- 4 files changed, 238 insertions(+), 3 deletions(-) create mode 100644 .github/actions/test-summary-report/action.yaml diff --git a/.github/actions/test-summary-report/action.yaml b/.github/actions/test-summary-report/action.yaml new file mode 100644 index 000000000000..2b76bf99b0fe --- /dev/null +++ b/.github/actions/test-summary-report/action.yaml @@ -0,0 +1,91 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 'Test Summary Report' +description: 'Parses JUnit test reports and publishes them to the GitHub workflow summary' +inputs: + test-report-xml-base-dir: + description: 'The base directory from which to search for JUnit XML reports' + required: true + default: '.' + test-report-xml-includes: + description: 'The path glob for files to examine when generating the test report summary' + required: true + default: '**/target/*-reports/TEST-*.xml' + +runs: + using: "composite" + steps: + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 20 + - name: Install script dependencies + shell: bash + run: | + npm install junit2json + - name: Generate test summary + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const junit = require('junit2json'); + const testReportGlobPattern = "${{ inputs.test-report-xml-base-dir }}/${{ inputs.test-report-xml-includes }}".replace(/\.+\/+/g, "") + const summaryData = []; + + // Configure test summary table headers + summaryData.push([ + {data: 'Test Class', header: true}, + {data: 'Test Name', header: true}, + {data: 'Failure', header: true}, + {data: 'Details', header: true} + ]); + + // Iterate and parse surefire / failsafe reports and use the info to build the summary + const globber = await glob.create(testReportGlobPattern); + for await (const reportFile of globber.globGenerator()) { + const file = fs.readFileSync(reportFile); + const report = junit.parse(file).then((report) => { + if (report.errors > 0 || report.failures > 0) { + report.testcase.forEach((testCase) => { + if (testCase.failure !== undefined) { + const shortClassName = `${testCase.classname.substring(testCase.classname.lastIndexOf('.') + 1)}`; + const className = `
${shortClassName}\n${testCase.classname}
`; + const details = `
View\n
${testCase.failure[0].inner}
`; + let testName = `${testCase.name}`; + if (testCase.length > 25) { + testName = `
View\n${testCase.name}
`; + } + + let message = `
${testCase.failure[0].message}
`; + if (message.length > 50) { + message = `
View\n${message}
`; + } + summaryData.push([className, testName, message, details]); + } + }); + } + }); + } + + // Write the summary data if there were test failures + if (summaryData.length > 1) { + await core.summary + .addHeading("Test Failures", "3") + .addTable(summaryData) + .write(); + } diff --git a/.github/workflows/camel-master-cron.yaml b/.github/workflows/camel-master-cron.yaml index 0d4852a28f98..7a57fcd44ee7 100644 --- a/.github/workflows/camel-master-cron.yaml +++ b/.github/workflows/camel-master-cron.yaml @@ -208,6 +208,11 @@ jobs: --fail-at-end \ -pl "${NATIVE_MODULES[*]}" fi + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: integration-tests - name: Fail if there are uncommitted changes shell: bash run: | @@ -258,6 +263,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip -Dcamel-quarkus.update-extension-doc-page.skip \ --fail-at-end \ test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: extensions-core - name: cd extensions && mvn test run: | cd extensions @@ -265,6 +275,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip -Dcamel-quarkus.update-extension-doc-page.skip \ --fail-at-end \ test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: extensions - name: cd test-framework && mvn test run: | cd test-framework @@ -272,6 +287,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip -Dcamel-quarkus.update-extension-doc-page.skip \ --fail-at-end \ test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: test-framework - name: cd tooling && mvn verify run: | cd tooling @@ -279,12 +299,22 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ --fail-at-end \ verify + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: tooling - name: cd catalog && mvn test run: | cd catalog ../mvnw ${CQ_MAVEN_ARGS} \ -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: catalog - name: Report Build Failure if: failure() || cancelled() run: | @@ -335,6 +365,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ --fail-at-end \ clean test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: integration-tests-jvm - name: Report Build Failure if: failure() || cancelled() run: | @@ -390,6 +425,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ --fail-at-end \ clean verify + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: integration-tests integration-tests-alternative-platform: runs-on: ${{ matrix.os }} @@ -435,6 +475,11 @@ jobs: -Dskip-testcontainers-tests -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ --fail-at-end \ clean verify + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: integration-tests - name: Report Build Failure if: failure() || cancelled() shell: bash @@ -515,13 +560,16 @@ jobs: done if [[ ${#BUILD_FAILURES[@]} -gt 0 ]]; then - echo -e "\nBuild errors were encountred in the following projects:\n" + echo -e "\nBuild errors were encountered in the following projects:\n" for FAILURE in ${BUILD_FAILURES[@]}; do echo "* ${FAILURE}" done echo -e "\nCheck build logs for further information." exit 1 fi + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} - name: Report Build Failure if: failure() || cancelled() run: | diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index 67fdb3b9e039..250f94b68ed0 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -280,6 +280,11 @@ jobs: --fail-at-end \ -pl "${NATIVE_MODULES[*]}" fi + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: integration-tests - name: Fail if there are uncommitted changes shell: bash run: | @@ -321,6 +326,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip -Dcamel-quarkus.update-extension-doc-page.skip \ --fail-at-end \ test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: extensions-core - name: cd extensions && mvn test run: | cd extensions @@ -328,6 +338,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip -Dcamel-quarkus.update-extension-doc-page.skip \ --fail-at-end \ test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: extensions - name: cd test-framework && mvn test run: | cd test-framework @@ -335,6 +350,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip -Dcamel-quarkus.update-extension-doc-page.skip \ --fail-at-end \ test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: test-framework - name: cd tooling && mvn verify run: | cd tooling @@ -342,12 +362,22 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ --fail-at-end \ verify + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: tooling - name: cd catalog && mvn test run: | cd catalog ../mvnw ${CQ_MAVEN_ARGS} ${BRANCH_OPTIONS} \ -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: catalog extensions-jvm-tests: runs-on: ubuntu-latest @@ -388,6 +418,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ --fail-at-end \ clean test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: integration-tests-jvm integration-tests-alternative-jdk: name: Integration Tests Alternative JDK 21 ${{matrix.name}} @@ -435,6 +470,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ --fail-at-end \ clean verify + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: integration-tests integration-tests-alternative-platform: runs-on: ${{ matrix.os }} @@ -481,6 +521,11 @@ jobs: -Dskip-testcontainers-tests -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ --fail-at-end \ clean verify + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: integration-tests examples-tests: name: Examples Tests - ${{matrix.name}} @@ -548,10 +593,13 @@ jobs: done if [[ ${#BUILD_FAILURES[@]} -gt 0 ]]; then - echo -e "\nBuild errors were encountred in the following projects:\n" + echo -e "\nBuild errors were encountered in the following projects:\n" for FAILURE in ${BUILD_FAILURES[@]}; do echo "* ${FAILURE}" done echo -e "\nCheck build logs for further information." exit 1 fi + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} diff --git a/.github/workflows/quarkus-master-cron.yaml b/.github/workflows/quarkus-master-cron.yaml index 2b517137d621..343682dce564 100644 --- a/.github/workflows/quarkus-master-cron.yaml +++ b/.github/workflows/quarkus-master-cron.yaml @@ -209,6 +209,11 @@ jobs: --fail-at-end \ -pl "${NATIVE_MODULES[*]}" fi + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: integration-tests - name: Fail if there are uncommitted changes shell: bash run: | @@ -259,6 +264,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip -Dcamel-quarkus.update-extension-doc-page.skip \ --fail-at-end \ test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: extensions-core - name: cd extensions && mvn test run: | cd extensions @@ -266,6 +276,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip -Dcamel-quarkus.update-extension-doc-page.skip \ --fail-at-end \ test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: extensions - name: cd test-framework && mvn test run: | cd test-framework @@ -273,6 +288,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip -Dcamel-quarkus.update-extension-doc-page.skip \ --fail-at-end \ test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: test-framework - name: cd tooling && mvn verify run: | cd tooling @@ -280,12 +300,22 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ --fail-at-end \ verify + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: tooling - name: cd catalog && mvn test run: | cd catalog ../mvnw ${CQ_MAVEN_ARGS} ${BRANCH_OPTIONS} \ -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: catalog - name: Report Build Failure if: failure() || cancelled() run: | @@ -336,6 +366,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ --fail-at-end \ clean test + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: integration-tests-jvm - name: Report Build Failure if: failure() || cancelled() run: | @@ -391,6 +426,11 @@ jobs: -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ --fail-at-end \ clean verify + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: integration-tests integration-tests-alternative-platform: runs-on: ${{ matrix.os }} @@ -436,6 +476,11 @@ jobs: -Dskip-testcontainers-tests -Dformatter.skip -Dimpsort.skip -Denforcer.skip \ --fail-at-end \ clean verify + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} + with: + test-report-xml-base-dir: integration-tests - name: Report Build Failure if: failure() || cancelled() shell: bash @@ -516,13 +561,16 @@ jobs: done if [[ ${#BUILD_FAILURES[@]} -gt 0 ]]; then - echo -e "\nBuild errors were encountred in the following projects:\n" + echo -e "\nBuild errors were encountered in the following projects:\n" for FAILURE in ${BUILD_FAILURES[@]}; do echo "* ${FAILURE}" done echo -e "\nCheck build logs for further information." exit 1 fi + - name: Report test failures + uses: ./.github/actions/test-summary-report + if: ${{ failure() }} - name: Report Build Failure if: failure() || cancelled() run: |