Skip to content

Backstop Sanity Tests #34

Backstop Sanity Tests

Backstop Sanity Tests #34

name: Sanity Test Checks
on:
workflow_dispatch:
inputs:
DEBUG_ENABLED:
description: "Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)"
type: boolean
required: false
default: false
DEBUG_DETACHED:
description: "whether to launch tmate in detached mode (if the tmate action is run)"
type: boolean
required: false
default: true
workflow_call:
permissions:
actions: write
contents: write
pull-requests: write
env:
NODE_VERSION: 20
jobs:
sanity-puppeteer:
name: 🤪 Puppeteer
runs-on: ubuntu-latest
steps:
- name: Checkout actions
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }}
sparse-checkout: |
.github/actions
path: actions
- uses: ./actions/.github/actions/setup-base
id: base
with:
DEBUG_ENABLED: ${{ inputs.DEBUG_ENABLED }}
DEBUG_DETACHED: ${{ inputs.DEBUG_DETACHED }}
- uses: ./.github/actions/setup-node
id: node
- name: "𓋏 Run `npm run sanity-test`"
continue-on-error: true
shell: bash
run: |
npm run sanity-test
- name: "Validate Puppeteer Report"
id: validate-puppeteer
continue-on-error: true
shell: bash
run: |
set +e
PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test.json) \
<(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV
if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
echo "# ❎ Puppeteer Sanity Different" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
else
echo "# ✅ Puppeteer Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
fi
- name: "Full Sanity Report Diff"
id: diff
continue-on-error: true
shell: bash
run: |
set +e
FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
echo "<details>" >> $GITHUB_STEP_SUMMARY
echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "</details>" >> $GITHUB_STEP_SUMMARY
- name: "Report Validation Outcome"
shell: bash
run: |
if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
exit 1
else
exit 0
fi
sanity-playwright:
name: 🤪 Playwright
runs-on: ubuntu-latest
steps:
- name: Checkout actions
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }}
sparse-checkout: |
.github/actions
path: actions
- uses: ./actions/.github/actions/setup-base
id: base
with:
DEBUG_ENABLED: ${{ inputs.DEBUG_ENABLED }}
DEBUG_DETACHED: ${{ inputs.DEBUG_DETACHED }}
- uses: ./.github/actions/setup-node
id: node
- name: "🎭 Run `npm run sanity-test-playwright`"
continue-on-error: true
shell: bash
run: |
npx playwright install --with-deps
npm run sanity-test-playwright
- name: "Validate Playwright Report"
id: validate-playwright
continue-on-error: true
shell: bash
run: |
set +e
PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test-playwright.json) \
<(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV
if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
echo "# ❎ Playwright Sanity Report Different" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
else
echo "# ✅ Playwright Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
fi
- name: "Full Sanity Report Diff"
id: diff
continue-on-error: true
shell: bash
run: |
set +e
FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
echo "<details>" >> $GITHUB_STEP_SUMMARY
echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "</details>" >> $GITHUB_STEP_SUMMARY
- name: "Report Validation Outcome"
shell: bash
run: |
if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
exit 1
else
exit 0
fi