wip: add sanity-test composite

dgrebb · Jan 16, 2024 · 6151dc3 · 6151dc3
1 parent c6c9e1a
commit 6151dc3
Show file tree

Hide file tree

Showing 2 changed files with 71 additions and 92 deletions.
diff --git a/.github/actions/sanity-test-checks/action.yml b/.github/actions/sanity-test-checks/action.yml
@@ -0,0 +1,61 @@
+name: "Validate Sanity Report"
+description: "Diff the sanity-test report with a fixture and validate expected shape."
+
+inputs:
+  WORKSPACE_ROOT:
+    description: "Working Directory"
+    required: true
+  RUNNER:
+    description: "Test Runner"
+    required: true
+  FIXTURE:
+    description: "Test Report Fixture"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: "Validate ${{ inputs.RUNNER }} Report"
+      id: validate-report
+      continue-on-error: true
+      shell: bash
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      run: |
+        set +e
+        TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/${{ inputs.FIXTURE }}) \
+          <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json))
+        echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV
+        if [[ "$TEST_RESULT" != "" ]]; then
+          echo "# ❎ ${{ inputs.RUNNER }} Sanity Different" >> $GITHUB_STEP_SUMMARY
+          echo '```diff' >> $GITHUB_STEP_SUMMARY
+          echo "${TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+        else
+          echo "# ✅ ${{ inputs.RUNNER }} Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
+        fi
+
+    - name: "Full Smoke Test Diff"
+      id: diff
+      continue-on-error: true
+      shell: bash
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      run: |
+        set +e
+        FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/${{ inputs.FIXTURE }}.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
+        echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+        echo "<details>" >> $GITHUB_STEP_SUMMARY
+        echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo '```diff' >> $GITHUB_STEP_SUMMARY
+        echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+    - name: "Report Validation Outcome"
+      run: |
+        if [[ "$TEST_RESULT" != "" ]]; then
+          exit 1
+        else
+          exit 0
+        fi
diff --git a/.github/workflows/backstop-sanity-test.yml b/.github/workflows/backstop-sanity-test.yml
@@ -41,52 +41,11 @@ jobs:
         run: |
           npm run sanity-test
 
-      - name: "Validate Puppeteer Report"
-        id: validate-puppeteer
-        continue-on-error: true
-        shell: bash
-        working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
-        run: |
-          set +e
-          PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test.json) \
-            <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json))
-          echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV
-          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
-            echo "# ❎ Puppeteer Sanity Different" >> $GITHUB_STEP_SUMMARY
-            echo '```diff' >> $GITHUB_STEP_SUMMARY
-            echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
-            echo '```' >> $GITHUB_STEP_SUMMARY
-          else
-            echo "# ✅ Puppeteer Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
-          fi
-
-      - name: "Full Sanity Report Diff"
-        id: diff
-        continue-on-error: true
-        shell: bash
-        working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
-        run: |
-          set +e
-          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
-          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
-          echo "<details>" >> $GITHUB_STEP_SUMMARY
-          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo '```diff' >> $GITHUB_STEP_SUMMARY
-          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
-          echo '```' >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "</details>" >> $GITHUB_STEP_SUMMARY
-
-      - name: "Report Validation Outcome"
-        shell: bash
-        working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
-        run: |
-          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
-            exit 1
-          else
-            exit 0
-          fi
+      - uses: ./actions/.github/actions/sanity-test-checks
+        with:
+          WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }}
+          RUNNER: "Puppeteer"
+          FIXTURE: "sanity-test.json"
 
   sanity-playwright:
     name: 🤪 Playwright
@@ -119,49 +78,8 @@ jobs:
         run: |
           npm run sanity-test-playwright
 
-      - name: "Validate Playwright Report"
-        id: validate-playwright
-        continue-on-error: true
-        shell: bash
-        working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
-        run: |
-          set +e
-          PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test-playwright.json) \
-            <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json))
-          echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV
-          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
-            echo "# ❎ Playwright Sanity Report Different" >> $GITHUB_STEP_SUMMARY
-            echo '```diff' >> $GITHUB_STEP_SUMMARY
-            echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
-            echo '```' >> $GITHUB_STEP_SUMMARY
-          else
-            echo "# ✅ Playwright Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
-          fi
-
-      - name: "Full Sanity Report Diff"
-        id: diff
-        continue-on-error: true
-        shell: bash
-        working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
-        run: |
-          set +e
-          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
-          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
-          echo "<details>" >> $GITHUB_STEP_SUMMARY
-          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo '```diff' >> $GITHUB_STEP_SUMMARY
-          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
-          echo '```' >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "</details>" >> $GITHUB_STEP_SUMMARY
-
-      - name: "Report Validation Outcome"
-        shell: bash
-        working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
-        run: |
-          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
-            exit 1
-          else
-            exit 0
-          fi
+      - uses: ./actions/.github/actions/sanity-test-checks
+        with:
+          WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }}
+          RUNNER: "Playwright"
+          FIXTURE: "sanity-test-playwright.json"