diff --git a/.github/workflows/backstop-sanity-test.yml b/.github/workflows/backstop-sanity-test.yml index 505c3cad0..fe3c04860 100644 --- a/.github/workflows/backstop-sanity-test.yml +++ b/.github/workflows/backstop-sanity-test.yml @@ -42,8 +42,13 @@ jobs: diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test.json) \ <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null if [[ $? == "0" ]]; then + echo "# ✅ Puppeteer Sanity Test Valid" >> $GITHUB_STEP_SUMMARY exit 0 else + RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test.json) \ + <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json)) + echo "# Puppeteer Sanity Validation Diff" >> $GITHUB_STEP_SUMMARY + echo "${RESULT}" >> $GITHUB_STEP_SUMMARY exit 1 fi @@ -77,7 +82,11 @@ jobs: diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test-playwright.json) \ <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null if [[ $? == "0" ]]; then + echo "# ✅ Playwright Sanity Test Valid" >> $GITHUB_STEP_SUMMARY exit 0 else + RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test-playwright.json) <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json)) + echo "# Playwright Sanity Validation Diff" >> $GITHUB_STEP_SUMMARY + echo "${RESULT}" >> $GITHUB_STEP_SUMMARY exit 1 fi diff --git a/.github/workflows/backstop-smoke-test.yml b/.github/workflows/backstop-smoke-test.yml index ec37257a6..8f0ad01ec 100644 --- a/.github/workflows/backstop-smoke-test.yml +++ b/.github/workflows/backstop-smoke-test.yml @@ -42,8 +42,13 @@ jobs: diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test.json) \ <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null if [[ $? == "0" ]]; then + echo "# ✅ Puppeteer Smoke Test Valid" >> $GITHUB_STEP_SUMMARY exit 0 else + RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-playwright.json) \ + <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json)) + echo "# Puppeteer Smoke Validation Results" >> $GITHUB_STEP_SUMMARY + echo "${RESULT}" >> $GITHUB_STEP_SUMMARY exit 1 fi @@ -77,7 +82,11 @@ jobs: diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-playwright.json) \ <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null if [[ $? == "0" ]]; then + echo "# ✅ Playwright Smoke Test Valid" >> $GITHUB_STEP_SUMMARY exit 0 else + RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-playwright.json) <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json)) + echo "# Playwright Smoke Validation Diff" >> $GITHUB_STEP_SUMMARY + echo "${RESULT}" >> $GITHUB_STEP_SUMMARY exit 1 fi diff --git a/.github/workflows/docker-sanity-test.yml b/.github/workflows/docker-sanity-test.yml index 40165815c..efd71304d 100644 --- a/.github/workflows/docker-sanity-test.yml +++ b/.github/workflows/docker-sanity-test.yml @@ -67,13 +67,17 @@ jobs: run: | cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test - - name: "Validate Puppeteer Test Results" + - name: "Validate Puppeteer Docker Test Results" run: | diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test-docker.json) \ <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null if [[ $? == "0" ]]; then + echo "# ✅ Puppeteer Docker Sanity Test Valid" >> $GITHUB_STEP_SUMMARY exit 0 else + RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test-docker.json) <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json)) + echo "# Puppeteer Docker Sanity Validation Diff" >> $GITHUB_STEP_SUMMARY + echo "${RESULT}" >> $GITHUB_STEP_SUMMARY exit 1 fi @@ -126,12 +130,16 @@ jobs: run: | cd test/configs/ && docker run --rm -t --entrypoint='' --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG sh -c "chmod -R 777 /root && chmod -R 777 /opt/pw-browsers && npm --verbose --foreground-scripts i -D playwright && npx --verbose --foreground-scripts --yes playwright@$PLAYWRIGHT_VERSION install && backstop test --config=playwright" - - name: "Validate Playwright Test Results" + - name: "Validate Playwright Docker Test Results" run: | diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test-playwright-docker.json) \ <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null if [[ $? == "0" ]]; then + echo "# ✅ Playwright Docker Sanity Test Valid" >> $GITHUB_STEP_SUMMARY exit 0 else + RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/sanity-test-playwright-docker.json) <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json)) + echo "# Playwright Docker Sanity Validation Diff" >> $GITHUB_STEP_SUMMARY + echo "${RESULT}" >> $GITHUB_STEP_SUMMARY exit 1 fi diff --git a/.github/workflows/docker-smoke-test.yml b/.github/workflows/docker-smoke-test.yml index ad7179db4..758f779ab 100644 --- a/.github/workflows/docker-smoke-test.yml +++ b/.github/workflows/docker-smoke-test.yml @@ -66,13 +66,17 @@ jobs: run: | cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test --config=backstop_features - - name: "Validate Puppeteer Test Results" + - name: "Validate Puppeteer Docker Test Results" run: | diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-docker.json) \ <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null if [[ $? == "0" ]]; then + echo "# ✅ Puppeteer Docker Smoke Test Valid" >> $GITHUB_STEP_SUMMARY exit 0 else + RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-docker.json) <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json)) + echo "# Puppeteer Docker Smoke Validation Diff" >> $GITHUB_STEP_SUMMARY + echo "${RESULT}" >> $GITHUB_STEP_SUMMARY exit 1 fi @@ -124,12 +128,16 @@ jobs: run: | cd test/configs/ && docker run --rm -t --entrypoint='' --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG sh -c "chmod -R 777 /root && chmod -R 777 /opt/pw-browsers && npm --verbose i -D playwright && npx --verbose --foreground-scripts --yes playwright@$PLAYWRIGHT_VERSION install && backstop test --config=backstop_features_pw" - - name: "Validate Playwright Test Results" + - name: "Validate Playwright Docker Test Results" run: | diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-playwright-docker.json) \ <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json) 1>/dev/null if [[ $? == "0" ]]; then + echo "# ✅ Playwright Docker Smoke Test Valid" >> $GITHUB_STEP_SUMMARY exit 0 else + RESULT=$(diff -c <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/__fixtures__/smoke-test-playwright-docker.json) <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage, .tests[].pair.diff.analysisTime, .tests[].pair.diff.misMatchPercentage, .tests[].pair.diff.rawMisMatchPercentage)" test/configs/backstop_data/bitmaps_test/**/report.json)) + echo "# Playwright Docker Smoke Validation Diff" >> $GITHUB_STEP_SUMMARY + echo "${RESULT}" >> $GITHUB_STEP_SUMMARY exit 1 fi