From f5b5dfa1a27736f767b69cb05a13188cade7eeb3 Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Sat, 8 Oct 2022 14:57:01 +0200 Subject: [PATCH 1/2] gnutests: save comparison in a comment log --- .github/workflows/GnuTests.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index d7e6c136de3..899fc42da91 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -4,7 +4,7 @@ name: GnuTests # * note: to run a single test => `REPO/util/run-gnu-test.sh PATH/TO/TEST/SCRIPT` -on: [push, pull_request_target] +on: [push, pull_request] permissions: contents: read @@ -187,6 +187,9 @@ jobs: REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log' REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}' + COMMENT_LOG='${{ steps.vars.outputs.path_reference }}/comment.txt' + mkdir -p ${{ steps.vars.outputs.path_reference }} + touch ${COMMENT_LOG} if test -f "${REF_LOG_FILE}"; then echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort) @@ -196,28 +199,36 @@ jobs: for LINE in ${REF_FAILING} do if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}"; then - echo "::warning ::Congrats! The gnu test ${LINE} is no longer failing!" + MSG="Congrats! The gnu test ${LINE} is no longer failing!" + echo "::warning ::$MSG" + echo $MSG >> ${COMMENT_LOG} fi done for LINE in ${NEW_FAILING} do if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" then - echo "::error ::GNU test failed: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" + MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" + echo "::error ::$MSG" + echo $MSG >> ${COMMENT_LOG} have_new_failures="true" fi done for LINE in ${REF_ERROR} do if ! grep -Fxq ${LINE}<<<"${NEW_ERROR}"; then - echo "::warning ::Congrats! The gnu test ${LINE} is no longer ERROR!" + MSG="Congrats! The gnu test ${LINE} is no longer ERROR!" + echo "::warning ::$MSG" + echo $MSG >> ${COMMENT_LOG} fi done for LINE in ${NEW_ERROR} do if ! grep -Fxq ${LINE}<<<"${REF_ERROR}" then - echo "::error ::GNU test error: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" + MSG="GNU test error: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" + echo "::error ::$MSG" + echo $MSG >> ${COMMENT_LOG} have_new_failures="true" fi done @@ -226,6 +237,11 @@ jobs: echo "::warning ::Skipping test failure comparison; no prior reference test logs are available." fi if test -n "${have_new_failures}" ; then exit -1 ; fi + - name: Upload comparison log for comment on PR + uses: actions/upload-artifact@v3 + with: + name: comment.txt + path: ${{ steps.vars.outputs.path_reference }}/comment.txt' - name: Compare test summary VS reference if: success() || failure() # run regardless of prior step success/failure shell: bash From b9ce759a4bf083cced67062508ff23157ae1265c Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Sat, 8 Oct 2022 15:08:44 +0200 Subject: [PATCH 2/2] gnutests: also store PR number in comment artifact --- .github/workflows/GnuComment.yml | 55 ++++++++++++++++++++++++++++++++ .github/workflows/GnuTests.yml | 14 +++++--- 2 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/GnuComment.yml diff --git a/.github/workflows/GnuComment.yml b/.github/workflows/GnuComment.yml new file mode 100644 index 00000000000..fd73520e6ee --- /dev/null +++ b/.github/workflows/GnuComment.yml @@ -0,0 +1,55 @@ +name: Comment on GNU test results on the PR + +# read-write repo token +# access to secrets +on: + workflow_run: + workflows: ["GnuTests"] + types: + - completed + +jobs: + upload: + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + steps: + - name: 'Download artifact' + uses: actions/github-script@v3.1.0 + with: + script: | + # List all artifacts from GnuTests + var artifacts = await github.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{github.event.workflow_run.id }}, + }); + + # Download the "comment" artifact, which contains a PR number (NR) and result.txt + var matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "comment" + })[0]; + var download = await github.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var fs = require('fs'); + fs.writeFileSync('${{github.workspace}}/comment.zip', Buffer.from(download.data)); + - run: unzip comment.zip + + - name: 'Comment on PR' + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + var fs = require('fs'); + var issue_number = Number(fs.readFileSync('./NR')); + await github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + body: fs.readFileSync('./result.txt') + }); diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 899fc42da91..4db4220c9ad 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -187,9 +187,15 @@ jobs: REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log' REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}' - COMMENT_LOG='${{ steps.vars.outputs.path_reference }}/comment.txt' + mkdir -p ${{ steps.vars.outputs.path_reference }} + + COMMENT_DIR="${{ steps.vars.outputs.path_reference }}/comment" + mkdir -p ${COMMENT_DIR} + echo ${{ github.event.number }} > ${COMMENT_DIR}/NR + COMMENT_LOG="${COMMENT_DIR}/result.txt" touch ${COMMENT_LOG} + if test -f "${REF_LOG_FILE}"; then echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort) @@ -237,11 +243,11 @@ jobs: echo "::warning ::Skipping test failure comparison; no prior reference test logs are available." fi if test -n "${have_new_failures}" ; then exit -1 ; fi - - name: Upload comparison log for comment on PR + - name: Upload comparison log (for GnuComment workflow) uses: actions/upload-artifact@v3 with: - name: comment.txt - path: ${{ steps.vars.outputs.path_reference }}/comment.txt' + name: comment + path: ${{ steps.vars.outputs.path_reference }}/comment/ - name: Compare test summary VS reference if: success() || failure() # run regardless of prior step success/failure shell: bash