ci(tests): adds fixtures and sanity-test validation (garris#1533)

.github/workflows/backstop-integration-test.yml

@@ -1,4 +1,4 @@
-name: Backstop Integration Tests
+name: Integration Test Check
 
 on:
   workflow_dispatch:
@@ -14,7 +14,7 @@ env:
 
 jobs:
   integration:
-    name: 🧩 Integration
+    name: 🧩 Validate Integration Test Results
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -35,3 +35,46 @@ jobs:
       - name: "𓋏 Run `npm run integration-test`"
         run: |
           npm run integration-test
+
+      - name: "Validate Integration Test Results"
+        id: validate
+        continue-on-error: true
+        run: |
+          set +e
+          TEST_REPORT_DIR=$(ls -ltd integrationTestDir/backstop_data/bitmaps_test/*/ | grep '^d' | head -n 1 | awk '{print $NF}')
+          echo "TEST_REPORT_DIR=$TEST_REPORT_DIR" >> $GITHUB_ENV
+          TEST_RESULT=$(diff <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/integration-test.json) <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' ${TEST_REPORT_DIR}report.json))
+          echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV
+          if [[ "$TEST_RESULT" != "" ]]; then
+            echo "# ❎ Integration Test Report Shapes Differ:" >> $GITHUB_STEP_SUMMARY
+            echo "## Failing Diff" >> $GITHUB_STEP_SUMMARY
+            echo '```diff' >> $GITHUB_STEP_SUMMARY
+            echo "${TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
+            echo '```' >> $GITHUB_STEP_SUMMARY
+          else
+            echo "# ✅ Integration Test Report Shapes Match" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: "Full Integration Test Diff"
+        id: diff
+        continue-on-error: true
+        run: |
+          set +e
+          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/integration-test.json) <(jq -S '.tests[]' ${TEST_REPORT_DIR}report.json))
+          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+          echo "<details>" >> $GITHUB_STEP_SUMMARY
+          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo '```diff' >> $GITHUB_STEP_SUMMARY
+          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+      - name: "Test Results Validation Outcome"
+        run: |
+          if [[ "$TEST_RESULT" != "" ]]; then
+            exit 1
+          else
+            exit 0
+          fi

.github/workflows/backstop-sanity-test.yml

@@ -1,4 +1,4 @@
-name: Backstop Sanity Tests
+name: Sanity Test Checks
 
 on:
   workflow_dispatch:
@@ -33,9 +33,51 @@ jobs:
         run: npm ci
 
       - name: "𓋏 Run `npm run sanity-test`"
+        continue-on-error: true
         run: |
           npm run sanity-test
 
+      - name: "Validate Puppeteer Report"
+        id: validate-puppeteer
+        continue-on-error: true
+        run: |
+          set +e
+          PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test.json) \
+            <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json))
+          echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV
+          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
+            echo "# ❎ Puppeteer Sanity Different" >> $GITHUB_STEP_SUMMARY
+            echo '```diff' >> $GITHUB_STEP_SUMMARY
+            echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
+            echo '```' >> $GITHUB_STEP_SUMMARY
+          else
+            echo "# ✅ Puppeteer Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: "Full Sanity Report Diff"
+        id: diff
+        continue-on-error: true
+        run: |
+          set +e
+          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
+          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+          echo "<details>" >> $GITHUB_STEP_SUMMARY
+          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo '```diff' >> $GITHUB_STEP_SUMMARY
+          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+      - name: "Report Validation Outcome"
+        run: |
+          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
+            exit 1
+          else
+            exit 0
+          fi
+
   sanity-playwright:
     name: 🤪 Playwright
     runs-on: ubuntu-latest
@@ -56,6 +98,48 @@ jobs:
         run: npm ci
 
       - name: "🎭 Run `npm run sanity-test-playwright`"
+        continue-on-error: true
         run: |
           npx playwright install --with-deps
           npm run sanity-test-playwright
+
+      - name: "Validate Playwright Report"
+        id: validate-playwright
+        continue-on-error: true
+        run: |
+          set +e
+          PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test-playwright.json) \
+            <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json))
+          echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV
+          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
+            echo "# ❎ Playwright Sanity Report Different" >> $GITHUB_STEP_SUMMARY
+            echo '```diff' >> $GITHUB_STEP_SUMMARY
+            echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
+            echo '```' >> $GITHUB_STEP_SUMMARY
+          else
+            echo "# ✅ Playwright Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: "Full Sanity Report Diff"
+        id: diff
+        continue-on-error: true
+        run: |
+          set +e
+          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
+          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+          echo "<details>" >> $GITHUB_STEP_SUMMARY
+          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo '```diff' >> $GITHUB_STEP_SUMMARY
+          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+      - name: "Report Validation Outcome"
+        run: |
+          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
+            exit 1
+          else
+            exit 0
+          fi

.github/workflows/backstop-smoke-test.yml

@@ -1,4 +1,4 @@
-name: Backstop Smoke Tests
+name: Smoke Test Checks
 
 on:
   workflow_dispatch:
@@ -37,6 +37,47 @@ jobs:
         run: |
           npm run smoke-test
 
+      - name: "Validate Puppeteer Report"
+        id: validate
+        continue-on-error: true
+        run: |
+          set +e
+          PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/__fixtures__/smoke-test.json) \
+            <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json))
+          echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV
+          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
+            echo "# ❎ Puppeteer Report Different" >> $GITHUB_STEP_SUMMARY
+            echo '```diff' >> $GITHUB_STEP_SUMMARY
+            echo "$PUPPETEER_TEST_RESULT" >> $GITHUB_STEP_SUMMARY
+            echo '```'
+          else
+            echo "# ✅ Puppeteer Report Validated" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: "Full Smoke Test Diff"
+        id: diff
+        continue-on-error: true
+        run: |
+          set +e
+          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
+          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+          echo "<details>" >> $GITHUB_STEP_SUMMARY
+          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo '```diff' >> $GITHUB_STEP_SUMMARY
+          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+      - name: "Report Validation Outcome"
+        run: |
+          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
+            exit 1
+          else
+            exit 0
+          fi
+
   smoke-playwright:
     name: 💨 Playwright
     runs-on: ubuntu-latest
@@ -61,3 +102,44 @@ jobs:
         run: |
           npx playwright install --with-deps
           npm run smoke-test-playwright
+
+      - name: "Validate Playwright Report"
+        id: validate
+        continue-on-error: true
+        run: |
+          set +e
+          PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq -S 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/__fixtures__/smoke-test-playwright.json) \
+            <(jq -S 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json))
+          echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV
+          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
+            echo "# ❎ Playwright Report Different" >> $GITHUB_STEP_SUMMARY
+            echo '```diff' >> $GITHUB_STEP_SUMMARY
+            echo "$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_STEP_SUMMARY
+            echo '```'
+          else
+            echo "# ✅ Playwright Report Validated" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: "Full Smoke Test Diff"
+        id: diff
+        continue-on-error: true
+        run: |
+          set +e
+          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
+          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+          echo "<details>" >> $GITHUB_STEP_SUMMARY
+          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo '```diff' >> $GITHUB_STEP_SUMMARY
+          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+      - name: "Report Validation Outcome"
+        run: |
+          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
+            exit 1
+          else
+            exit 0
+          fi

.github/workflows/docker-sanity-test.yml

@@ -1,4 +1,4 @@
-name: Docker Sanity Tests
+name: Docker Sanity Test Checks
 
 on:
   workflow_dispatch:
@@ -63,9 +63,51 @@ jobs:
           docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG
 
       - name: "𓋏 Run `backstop test` in Docker"
+        continue-on-error: true
         run: |
+          set +e
           cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test
 
+      - name: "Validate Puppeteer Docker Test Results"
+        continue-on-error: true
+        run: |
+          set +e
+          PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/__fixtures__/sanity-test-docker.json) \
+            <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/configs/backstop_data/bitmaps_test/**/report.json))
+          echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV
+          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
+            echo "# ❎ Puppeteer Smoke Different" >> $GITHUB_STEP_SUMMARY
+            echo '```diff' >> $GITHUB_STEP_SUMMARY
+            echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
+            echo '```' >> $GITHUB_STEP_SUMMARY
+          else
+            echo "# ✅ Puppeteer Smoke Report Valid" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: "Full Smoke Report Diff"
+        id: diff
+        continue-on-error: true
+        run: |
+          set +e
+          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-docker.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
+          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+          echo "<details>" >> $GITHUB_STEP_SUMMARY
+          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo '```diff' >> $GITHUB_STEP_SUMMARY
+          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+      - name: "Report Validation Outcome"
+        run: |
+          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
+            exit 1
+          else
+            exit 0
+          fi
+
   sanity-test-playwright:
     name: 🤪 Playwright
     runs-on: ubuntu-latest
@@ -111,5 +153,47 @@ jobs:
           docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG
 
       - name: "🎭 Run `backstop test --confg=playwright` in Docker"
+        continue-on-error: true
         run: |
+          set +e
           cd test/configs/ && docker run --rm -t --entrypoint='' --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG sh -c "chmod -R 777 /root && chmod -R 777 /opt/pw-browsers && npm --verbose --foreground-scripts i -D playwright && npx --verbose --foreground-scripts --yes playwright@$PLAYWRIGHT_VERSION install && backstop test --config=playwright"
+
+      - name: "Validate Playwright Docker Test Results"
+        continue-on-error: true
+        run: |
+          set +e
+          diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/__fixtures__/sanity-test-playwright-docker.json) \
+            <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/configs/backstop_data/bitmaps_test/**/report.json)
+          echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV
+          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
+            echo "# ❎ Playwright Sanity Report Different" >> $GITHUB_STEP_SUMMARY
+            echo '```diff' >> $GITHUB_STEP_SUMMARY
+            echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
+            echo '```' >> $GITHUB_STEP_SUMMARY
+          else
+            echo "# ✅ Playwright Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: "Full Sanity Report Diff"
+        id: diff
+        continue-on-error: true
+        run: |
+          set +e
+          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright-docker.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
+          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+          echo "<details>" >> $GITHUB_STEP_SUMMARY
+          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo '```diff' >> $GITHUB_STEP_SUMMARY
+          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+      - name: "Report Validation Outcome"
+        run: |
+          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
+            exit 1
+          else
+            exit 0
+          fi