backstop-sanity-test.yml

name: Sanity Test Checks

on:
  workflow_dispatch:
  workflow_call:

permissions:
  actions: write
  contents: write
  pull-requests: write

env:
  NODE_VERSION: 20

jobs:
  sanity-puppeteer:
    name: 🤪 Puppeteer
    runs-on: ubuntu-latest
    steps:
      # - uses: actions/checkout@v4

      - name: "Setup Node, npm Cache, and Install"
        uses: ./.github/actions/node-cache-install

      - name: "𓋏 Run `npm run sanity-test`"
        continue-on-error: true
        run: |
          npm run sanity-test

      - name: "Validate Puppeteer Report"
        id: validate-puppeteer
        continue-on-error: true
        run: |
          set +e
          PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test.json) \
            <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json))
          echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV
          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
            echo "# ❎ Puppeteer Sanity Different" >> $GITHUB_STEP_SUMMARY
            echo '```diff' >> $GITHUB_STEP_SUMMARY
            echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY
          else
            echo "# ✅ Puppeteer Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
          fi

      - name: "Full Sanity Report Diff"
        id: diff
        continue-on-error: true
        run: |
          set +e
          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
          echo "<details>" >> $GITHUB_STEP_SUMMARY
          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo '```diff' >> $GITHUB_STEP_SUMMARY
          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "</details>" >> $GITHUB_STEP_SUMMARY

      - name: "Report Validation Outcome"
        run: |
          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
            exit 1
          else
            exit 0
          fi

  sanity-playwright:
    name: 🤪 Playwright
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          ref: ${{ github.event.pull_request.head.sha || github.ref }}

      - name: "Setup Node, npm Cache, and Install"
        uses: ./.github/actions/node-cache-install

      - name: "🎭 Run `npm run sanity-test-playwright`"
        continue-on-error: true
        run: |
          npx playwright install --with-deps
          npm run sanity-test-playwright

      - name: "Validate Playwright Report"
        id: validate-playwright
        continue-on-error: true
        run: |
          set +e
          PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test-playwright.json) \
            <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json))
          echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV
          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
            echo "# ❎ Playwright Sanity Report Different" >> $GITHUB_STEP_SUMMARY
            echo '```diff' >> $GITHUB_STEP_SUMMARY
            echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY
          else
            echo "# ✅ Playwright Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
          fi

      - name: "Full Sanity Report Diff"
        id: diff
        continue-on-error: true
        run: |
          set +e
          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
          echo "<details>" >> $GITHUB_STEP_SUMMARY
          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo '```diff' >> $GITHUB_STEP_SUMMARY
          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "</details>" >> $GITHUB_STEP_SUMMARY

      - name: "Report Validation Outcome"
        run: |
          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
            exit 1
          else
            exit 0
          fi