style: log reports.json diffs in GitHub Actions summary

dgrebb · Jan 1, 2024 · b1426b0 · b1426b0
1 parent 2d1d41c
commit b1426b0
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 4 deletions.
diff --git a/.github/workflows/backstop-sanity-test.yml b/.github/workflows/backstop-sanity-test.yml
@@ -42,8 +42,13 @@ jobs:
           diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test.json) \
             <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null
           if [[ $? == "0" ]]; then
+            echo "# ✅ Puppeteer Sanity Test Valid" >> $GITHUB_STEP_SUMMARY
             exit 0
           else
+            RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test.json) \
+            <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json))
+            echo "# Puppeteer Sanity Validation Diff" >> $GITHUB_STEP_SUMMARY
+            echo "${RESULT}" >> $GITHUB_STEP_SUMMARY
             exit 1
           fi
 
@@ -77,7 +82,11 @@ jobs:
           diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test-playwright.json) \
             <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null
           if [[ $? == "0" ]]; then
+            echo "# ✅ Playwright Sanity Test Valid" >> $GITHUB_STEP_SUMMARY
             exit 0
           else
+            RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test-playwright.json) <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json))
+            echo "# Playwright Sanity Validation Diff" >> $GITHUB_STEP_SUMMARY
+            echo "${RESULT}" >> $GITHUB_STEP_SUMMARY
             exit 1
           fi
diff --git a/.github/workflows/backstop-smoke-test.yml b/.github/workflows/backstop-smoke-test.yml
@@ -42,8 +42,13 @@ jobs:
           diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test.json) \
             <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null
           if [[ $? == "0" ]]; then
+            echo "# ✅ Puppeteer Smoke Test Valid" >> $GITHUB_STEP_SUMMARY
             exit 0
           else
+            RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-playwright.json) \
+            <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json))
+            echo "# Puppeteer Smoke Validation Results" >> $GITHUB_STEP_SUMMARY
+            echo "${RESULT}" >> $GITHUB_STEP_SUMMARY
             exit 1
           fi
 
@@ -77,7 +82,11 @@ jobs:
           diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-playwright.json) \
             <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null
           if [[ $? == "0" ]]; then
+            echo "# ✅ Playwright Smoke Test Valid" >> $GITHUB_STEP_SUMMARY
             exit 0
           else
+            RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-playwright.json) <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json))
+            echo "# Playwright Smoke Validation Diff" >> $GITHUB_STEP_SUMMARY
+            echo "${RESULT}" >> $GITHUB_STEP_SUMMARY
             exit 1
           fi
diff --git a/.github/workflows/docker-sanity-test.yml b/.github/workflows/docker-sanity-test.yml
@@ -67,13 +67,17 @@ jobs:
         run: |
           cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test
 
-      - name: "Validate Puppeteer Test Results"
+      - name: "Validate Puppeteer Docker Test Results"
         run: |
           diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test-docker.json) \
             <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null
           if [[ $? == "0" ]]; then
+            echo "# ✅ Puppeteer Docker Sanity Test Valid" >> $GITHUB_STEP_SUMMARY
             exit 0
           else
+            RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test-docker.json) <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json))
+            echo "# Puppeteer Docker Sanity Validation Diff" >> $GITHUB_STEP_SUMMARY
+            echo "${RESULT}" >> $GITHUB_STEP_SUMMARY
             exit 1
           fi
 
@@ -126,12 +130,16 @@ jobs:
         run: |
           cd test/configs/ && docker run --rm -t --entrypoint='' --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG sh -c "chmod -R 777 /root && chmod -R 777 /opt/pw-browsers && npm --verbose --foreground-scripts i -D playwright && npx --verbose --foreground-scripts --yes playwright@$PLAYWRIGHT_VERSION install && backstop test --config=playwright"
 
-      - name: "Validate Playwright Test Results"
+      - name: "Validate Playwright Docker Test Results"
         run: |
           diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test-playwright-docker.json) \
             <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null
           if [[ $? == "0" ]]; then
+            echo "# ✅ Playwright Docker Sanity Test Valid" >> $GITHUB_STEP_SUMMARY
             exit 0
           else
+            RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test-playwright-docker.json) <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json))
+            echo "# Playwright Docker Sanity Validation Diff" >> $GITHUB_STEP_SUMMARY
+            echo "${RESULT}" >> $GITHUB_STEP_SUMMARY
             exit 1
           fi
diff --git a/.github/workflows/docker-smoke-test.yml b/.github/workflows/docker-smoke-test.yml
@@ -66,13 +66,17 @@ jobs:
         run: |
           cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test --config=backstop_features
 
-      - name: "Validate Puppeteer Test Results"
+      - name: "Validate Puppeteer Docker Test Results"
         run: |
           diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-docker.json) \
             <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null
           if [[ $? == "0" ]]; then
+            echo "# ✅ Puppeteer Docker Smoke Test Valid" >> $GITHUB_STEP_SUMMARY
             exit 0
           else
+            RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-docker.json) <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json))
+            echo "# Puppeteer Docker Smoke Validation Diff" >> $GITHUB_STEP_SUMMARY
+            echo "${RESULT}" >> $GITHUB_STEP_SUMMARY
             exit 1
           fi
 
@@ -124,12 +128,16 @@ jobs:
         run: |
           cd test/configs/ && docker run --rm -t --entrypoint='' --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG sh -c "chmod -R 777 /root && chmod -R 777 /opt/pw-browsers && npm --verbose i -D playwright && npx --verbose --foreground-scripts --yes playwright@$PLAYWRIGHT_VERSION install && backstop test --config=backstop_features_pw"
 
-      - name: "Validate Playwright Test Results"
+      - name: "Validate Playwright Docker Test Results"
         run: |
           diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-playwright-docker.json) \
             <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null
           if [[ $? == "0" ]]; then
+            echo "# ✅ Playwright Docker Smoke Test Valid" >> $GITHUB_STEP_SUMMARY
             exit 0
           else
+            RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-playwright-docker.json) <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json))
+            echo "# Playwright Docker Smoke Validation Diff" >> $GITHUB_STEP_SUMMARY
+            echo "${RESULT}" >> $GITHUB_STEP_SUMMARY
             exit 1
           fi