Skip to content

Commit

Permalink
wip: add smoke test composite
Browse files Browse the repository at this point in the history
  • Loading branch information
@dgrebb
dgrebb committed Jan 16, 2024
1 parent 7e12f74 commit 424d314
Show file tree
Hide file tree
Showing 4 changed files with 163 additions and 197 deletions.
39 changes: 26 additions & 13 deletions .github/actions/sanity-test-checks/action.yml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: "Validate Sanity Report"
description: "Diff the sanity-test report with a fixture and validate expected shape."
description: "Diff the sanity-test reports with a fixture and validate expected shape."

inputs:
WORKSPACE_ROOT:
Expand All @@ -22,18 +22,31 @@ runs:
working-directory: ${{ inputs.WORKSPACE_ROOT }}
run: |
set +e
TEST_RESULT=$(diff -c <(jq \
'walk(if type == "object" then with_entries(.value |= \
if type == "object" or type == "array" then . \
else "" \
end) \
else . end)' \
test/__fixtures__/${{ inputs.FIXTURE }}) \
<(jq 'walk(if type == "object" then with_entries(.value |= \
if type == "object" or type == "array" then . \
else "" end) else . end)' \
test/configs/backstop_data/bitmaps_test/**/report.json)\
FIXTURE="test/__fixtures__/${{ inputs.FIXTURE }}"
TEST_RESULT=$(\
diff -c \
<(jq \
'walk(if type == "object" then with_entries(.value |= \
if type == "object" or type == "array" \
then . \
else \
"" \
end) \
else . \
end)' \
$FIXTURE) \
<(jq \
'walk(if type == "object" then with_entries(.value |= \
if type == "object" or type == "array" \
then . \
else \
"" \
end) \
else . \
end)' \
test/configs/backstop_data/bitmaps_test/**/report.json) \
)
echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV
if [[ "$TEST_RESULT" != "" ]]; then
echo "# ❎ ${{ inputs.RUNNER }} Sanity Different" >> $GITHUB_STEP_SUMMARY
Expand All @@ -44,7 +57,7 @@ runs:
echo "# ✅ ${{ inputs.RUNNER }} Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
fi
- name: "Full Smoke Test Diff"
- name: "Full Sanity Report Diff"
id: diff
continue-on-error: true
shell: bash
Expand Down
93 changes: 93 additions & 0 deletions .github/actions/smoke-test-checks/action.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
name: "Validate Smoke Report"
description: "Diff the smoke-test reports with a fixture and validate expected shape."

inputs:
WORKSPACE_ROOT:
description: "Working Directory"
required: true
RUNNER:
description: "Test Runner"
required: true
FIXTURE:
description: "Test Report Fixture"
required: true

runs:
using: "composite"
steps:
- name: "Validate ${{ inputs.RUNNER }} Report"
id: validate
continue-on-error: true
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
set +e
FIXTURE="test/__fixtures__/${{ inputs.FIXTURE }}"
TEST_RESULT=$(\
diff -c \
<(jq \
'walk(if type == "object" then with_entries(.value |= \
if type == "object" or type == "array"
then . \
else \
"" \
end) \
else . \
end) | \
del(\
.tests[].pair.diff, \
.tests[].pair.diffImage \
)'\
$FIXTURE) \
<(jq \
'walk(if type == "object" then with_entries(.value |= \
if type == "object" or type == "array" \
then . \
else \
"" \
end) \
else . \
end) | \
del(\
.tests[].pair.diff, \
.tests[].pair.diffImage\
)' \
test/configs/backstop_data/bitmaps_test/**/report.json))
echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV
if [[ "$TEST_RESULT" != "" ]]; then
echo "# ❎ ${{ inputs.RUNNER }} Report Different" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "$TEST_RESULT" >> $GITHUB_STEP_SUMMARY
echo '```'
else
echo "# ✅ ${{ inputs.RUNNER }} Report Validated" >> $GITHUB_STEP_SUMMARY
fi
- name: "Full Smoke Report Diff"
id: diff
continue-on-error: true
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
set +e
FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/${{ inputs.FIXTURE }}) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
echo "<details>" >> $GITHUB_STEP_SUMMARY
echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "</details>" >> $GITHUB_STEP_SUMMARY
- name: "Report Validation Outcome"
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
if [[ "$TEST_RESULT" != "" ]]; then
exit 1
else
exit 0
fi
72 changes: 10 additions & 62 deletions .github/workflows/backstop-smoke-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,50 +40,11 @@ jobs:
run: |
npm run smoke-test
- name: "Validate Puppeteer Report"
id: validate
continue-on-error: true
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
set +e
PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/__fixtures__/smoke-test.json) \
<(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV
if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
echo "# ❎ Puppeteer Report Different" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "$PUPPETEER_TEST_RESULT" >> $GITHUB_STEP_SUMMARY
echo '```'
else
echo "# ✅ Puppeteer Report Validated" >> $GITHUB_STEP_SUMMARY
fi
- name: "Full Smoke Test Diff"
id: diff
continue-on-error: true
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
set +e
FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
echo "<details>" >> $GITHUB_STEP_SUMMARY
echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "</details>" >> $GITHUB_STEP_SUMMARY
- name: "Report Validation Outcome"
run: |
if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
exit 1
else
exit 0
fi
- uses: ./actions/.github/actions/smoke-test-checks
with:
WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }}
RUNNER: "Puppeteer"
FIXTURE: "smoke-test.json"

smoke-playwright:
name: 💨 Playwright
Expand Down Expand Up @@ -113,21 +74,8 @@ jobs:
npx playwright install --with-deps
npm run smoke-test-playwright
- name: "Validate Playwright Report"
id: validate
continue-on-error: true
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
set +e
PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq -S 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/__fixtures__/smoke-test-playwright.json) \
<(jq -S 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV
if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
echo "# ❎ Playwright Report Different" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_STEP_SUMMARY
echo '```'
else
echo "# ✅ Playwright Report Validated" >> $GITHUB_STEP_SUMMARY
fi
- uses: ./actions/.github/actions/smoke-test-checks
with:
WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }}
RUNNER: "Playwright"
FIXTURE: "smoke-test-playwright.json"
Loading

0 comments on commit 424d314

Please sign in to comment.