Skip to content

Backstop Smoke Tests #51

Backstop Smoke Tests

Backstop Smoke Tests #51

name: Smoke Test Checks
on:
workflow_dispatch:
workflow_call:
permissions:
actions: write
contents: write
pull-requests: write
env:
NODE_VERSION: 20
jobs:
smoke-puppeteer:
name: πŸ’¨ Puppeteer
runs-on: ubuntu-latest
steps:
- name: Checkout actions
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }}
sparse-checkout: |
.github/actions
path: actions
- uses: ./actions/.github/actions/setup-base
id: base
- uses: ./actions/.github/actions/setup-node
id: node
with:
WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }}
- name: "𓋏 Run `npm run smoke-test`"
continue-on-error: true
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
npm run smoke-test
- name: "Validate Puppeteer Report"
id: validate
continue-on-error: true
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
set +e
PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/__fixtures__/smoke-test.json) \
<(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV
if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
echo "# ❎ Puppeteer Report Different" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "$PUPPETEER_TEST_RESULT" >> $GITHUB_STEP_SUMMARY
echo '```'
else
echo "# βœ… Puppeteer Report Validated" >> $GITHUB_STEP_SUMMARY
fi
- name: "Full Smoke Test Diff"
id: diff
continue-on-error: true
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
set +e
FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
echo "<details>" >> $GITHUB_STEP_SUMMARY
echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "</details>" >> $GITHUB_STEP_SUMMARY
- name: "Report Validation Outcome"
run: |
if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
exit 1
else
exit 0
fi
smoke-playwright:
name: πŸ’¨ Playwright
runs-on: ubuntu-latest
steps:
- name: Checkout actions
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }}
sparse-checkout: |
.github/actions
path: actions
- uses: ./actions/.github/actions/setup-base
id: base
- uses: ./actions/.github/actions/setup-node
id: node
with:
WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }}
- name: "🎭 Run `npm run smoke-test-playwright`"
continue-on-error: true
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
npx playwright install --with-deps
npm run smoke-test-playwright
- name: "Validate Playwright Report"
id: validate
continue-on-error: true
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
set +e
PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq -S 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/__fixtures__/smoke-test-playwright.json) \
<(jq -S 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV
if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
echo "# ❎ Playwright Report Different" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_STEP_SUMMARY
echo '```'
else
echo "# βœ… Playwright Report Validated" >> $GITHUB_STEP_SUMMARY
fi
- name: "Full Smoke Test Diff"
id: diff
continue-on-error: true
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
set +e
FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
echo "<details>" >> $GITHUB_STEP_SUMMARY
echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "</details>" >> $GITHUB_STEP_SUMMARY
- name: "Report Validation Outcome"
run: |
if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
exit 1
else
exit 0
fi