Composite Actions Tests

.github/actions/cache-playwright/action.yml

@@ -0,0 +1,38 @@
+name: "Install and Cache Playwright"
+description: "Sets up a cache and/or installs Playwright and its binaries."
+
+inputs:
+  WORKSPACE_ROOT:
+    description: "The workspace root."
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Get installed Playwright version
+      shell: bash
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      id: playwright-version
+      run: echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright' || 'latest')" >> $GITHUB_ENV
+
+    - name: Cache playwright binaries
+      uses: actions/cache@v3
+      id: playwright-cache
+      with:
+        path: |
+          ~/.cache/ms-playwright
+        key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}
+
+    - shell: bash
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      run: npm ci
+
+    - shell: bash
+      if: steps.playwright-cache.outputs.cache-hit != 'true'
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      run: npx playwright install --with-deps
+
+    - shell: bash
+      if: steps.playwright-cache.outputs.cache-hit != 'true'
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      run: npx playwright install-deps

.github/actions/prepare-docker/action.yml

@@ -0,0 +1,40 @@
+name: "Pull & Prepare Docker Workflow"
+description: "Sets name and tag, logs in to GHCR, and pulls Docker image."
+
+inputs:
+  REGISTRY:
+    description: "GitHub Registry"
+    required: true
+  ACTOR:
+    description: "GitHub Username"
+    required: true
+  GITHUB_TOKEN:
+    description: "GitHub Token"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set Name and Tag Vars
+      shell: bash
+      working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
+      env:
+        name: "${{ env.BRANCH_NAME }}"
+      run: |
+        echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV}
+        echo "TAG=${name/\//-}" >> $GITHUB_ENV
+        echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV
+        echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright')"  >> $GITHUB_ENV
+
+    - name: Log in to the Container registry
+      uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+      with:
+        registry: ${{ inputs.REGISTRY }}
+        username: ${{ inputs.ACTOR }}
+        password: ${{ inputs.GITHUB_TOKEN }}
+
+    - name: Pull Image
+      shell: bash
+      working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
+      run: |
+        docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG

.github/actions/sanity-test-checks/action.yml

@@ -0,0 +1,86 @@
+name: "Validate Sanity Report"
+description: "Diff the sanity-test reports with a fixture and validate expected shape."
+
+inputs:
+  WORKSPACE_ROOT:
+    description: "Working Directory"
+    required: true
+  RUNNER:
+    description: "Test Runner"
+    required: true
+  FIXTURE:
+    description: "Test Report Fixture"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: "Validate ${{ inputs.RUNNER }} Report"
+      id: validate-report
+      continue-on-error: true
+      shell: bash
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      run: |
+        set +e
+        FIXTURE="test/__fixtures__/${{ inputs.FIXTURE }}"
+        TEST_RESULT=$(\
+          diff -c \
+            <(jq \
+              'walk(if type == "object" then with_entries(.value |= \
+                if type == "object" or type == "array" \
+                  then . \
+                else \
+                  "" \
+                end) \
+              else . \
+            end)' \
+          $FIXTURE) \
+            <(jq \
+              'walk(if type == "object" then with_entries(.value |= \
+                if type == "object" or type == "array" \
+                  then . \
+                else \
+                  "" \
+                end) \
+              else . \
+            end)' \
+          test/configs/backstop_data/bitmaps_test/**/report.json) \
+        )
+
+        echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV
+        if [[ "$TEST_RESULT" != "" ]]; then
+          echo "# ❎ ${{ inputs.RUNNER }} Sanity Different" >> $GITHUB_STEP_SUMMARY
+          echo '```diff' >> $GITHUB_STEP_SUMMARY
+          echo "${TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+        else
+          echo "# ✅ ${{ inputs.RUNNER }} Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
+        fi
+
+    - name: "Full Sanity Report Diff"
+      id: diff
+      continue-on-error: true
+      shell: bash
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      run: |
+        set +e
+        FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' $FIXTURE) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
+        echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+        echo "<details>" >> $GITHUB_STEP_SUMMARY
+        echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo '```diff' >> $GITHUB_STEP_SUMMARY
+        echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+    - name: "Report Validation Outcome"
+      shell: bash
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      run: |
+        if [[ "$TEST_RESULT" != "" ]]; then
+          exit 1
+        else
+          exit 0
+        fi

.github/actions/setup-base/action.yml

@@ -0,0 +1,23 @@
+name: "Setup Base Workflow"
+description: "Checks out the BackstopJS repo at the calling or current branch with shallow history. Pulls Docker image by branch tag. Installs dependencies. Optionally sets up interactive remote debug shell."
+
+outputs:
+  WORKSPACE_ROOT:
+    description: "Export the root workspace for all workflows."
+    value: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set workspace root
+      id: workspace_root
+      shell: bash
+      run: |
+        WORKSPACE_ROOT=$GITHUB_WORKSPACE/repo
+        echo "WORKSPACE_ROOT=$WORKSPACE_ROOT" | tee -a $GITHUB_OUTPUT
+
+    - name: Checkout to workspace
+      shell: bash
+      run: |
+        git clone -b ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} https://github.com/${{ github.repository }}.git ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}
+        ls ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}

.github/actions/setup-node/action.yml

@@ -0,0 +1,22 @@
+name: "Setup Node and Dependencies"
+description: "Sets up Node, npm caching, and installs dependencies"
+
+inputs:
+  WORKSPACE_ROOT:
+    description: "The workspace root."
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: ⬢ Setup Node & Cache
+      uses: actions/setup-node@v4
+      with:
+        node-version: ${{ env.NODE_VERSION }}
+        cache: "npm"
+        cache-dependency-path: ${{ inputs.WORKSPACE_ROOT }}/package-lock.json
+
+    - name: ↧ Install
+      shell: bash
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      run: npm ci

.github/actions/smoke-test-checks/action.yml

@@ -0,0 +1,93 @@
+name: "Validate Smoke Report"
+description: "Diff the smoke-test reports with a fixture and validate expected shape."
+
+inputs:
+  WORKSPACE_ROOT:
+    description: "Working Directory"
+    required: true
+  RUNNER:
+    description: "Test Runner"
+    required: true
+  FIXTURE:
+    description: "Test Report Fixture"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: "Validate ${{ inputs.RUNNER }} Report"
+      id: validate
+      continue-on-error: true
+      shell: bash
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      run: |
+        set +e
+        FIXTURE="test/__fixtures__/${{ inputs.FIXTURE }}"
+        TEST_RESULT=$(\
+          diff -c \
+            <(jq \
+              'walk(if type == "object" then with_entries(.value |= \
+                if type == "object" or type == "array"
+                  then . \
+                else \
+                  "" \
+                end) \
+              else . \
+            end) | \
+            del(\
+              .tests[].pair.diff, \
+              .tests[].pair.diffImage \
+            )'\
+          $FIXTURE) \
+            <(jq \
+              'walk(if type == "object" then with_entries(.value |= \
+                if type == "object" or type == "array" \
+                  then . \
+                else \
+                  "" \
+                end) \
+              else . \
+            end) | \
+            del(\
+              .tests[].pair.diff, \
+              .tests[].pair.diffImage\
+            )' \
+          test/configs/backstop_data/bitmaps_test/**/report.json))
+
+        echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV
+        if [[ "$TEST_RESULT" != "" ]]; then
+          echo "# ❎ ${{ inputs.RUNNER }} Report Different" >> $GITHUB_STEP_SUMMARY
+          echo '```diff' >> $GITHUB_STEP_SUMMARY
+          echo "$TEST_RESULT" >> $GITHUB_STEP_SUMMARY
+          echo '```'
+        else
+          echo "# ✅ ${{ inputs.RUNNER }} Report Validated" >> $GITHUB_STEP_SUMMARY
+        fi
+
+    - name: "Full Smoke Report Diff"
+      id: diff
+      continue-on-error: true
+      shell: bash
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      run: |
+        set +e
+        FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' $FIXTURE) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
+        echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+        echo "<details>" >> $GITHUB_STEP_SUMMARY
+        echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo '```diff' >> $GITHUB_STEP_SUMMARY
+        echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+    - name: "Report Validation Outcome"
+      shell: bash
+      working-directory: ${{ inputs.WORKSPACE_ROOT }}
+      run: |
+        if [[ "$TEST_RESULT" != "" ]]; then
+          exit 1
+        else
+          exit 0
+        fi

.github/workflows/backstop-integration-test.yml

@@ -1,4 +1,4 @@
-name: Backstop Integration Tests
+name: Integration Test Check
 
 on:
   workflow_dispatch:
@@ -14,7 +14,7 @@ env:
 
 jobs:
   integration:
-    name: 🧩 Integration
+    name: 🧩 Validate Integration Test Results
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -23,15 +23,52 @@ jobs:
           fetch-depth: 1
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
 
-      - name: ⬢ Setup Node & Cache
-        uses: actions/setup-node@v4
-        with:
-          cache: "npm"
-          cache-dependency-path: package-lock.json
-
-      - name: ↧ Install
-        run: npm ci
+      - name: "Setup Node, npm Cache, and Install"
+        uses: ./.github/actions/setup-base
 
       - name: "𓋏 Run `npm run integration-test`"
         run: |
           npm run integration-test
+
+      - name: "Validate Integration Test Results"
+        id: validate
+        continue-on-error: true
+        run: |
+          set +e
+          TEST_REPORT_DIR=$(ls -ltd integrationTestDir/backstop_data/bitmaps_test/*/ | grep '^d' | head -n 1 | awk '{print $NF}')
+          echo "TEST_REPORT_DIR=$TEST_REPORT_DIR" >> $GITHUB_ENV
+          TEST_RESULT=$(diff <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/integration-test.json) <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' ${TEST_REPORT_DIR}report.json))
+          echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV
+          if [[ "$TEST_RESULT" != "" ]]; then
+            echo "# ❎ Integration Test Report Shapes Differ:" >> $GITHUB_STEP_SUMMARY
+            echo "## Failing Diff" >> $GITHUB_STEP_SUMMARY
+            echo '```diff' >> $GITHUB_STEP_SUMMARY
+            echo "${TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
+            echo '```' >> $GITHUB_STEP_SUMMARY
+          else
+            echo "# ✅ Integration Test Report Shapes Match" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: "Full Integration Test Diff"
+        id: diff
+        continue-on-error: true
+        run: |
+          set +e
+          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/integration-test.json) <(jq -S '.tests[]' ${TEST_REPORT_DIR}report.json))
+          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
+          echo "<details>" >> $GITHUB_STEP_SUMMARY
+          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo '```diff' >> $GITHUB_STEP_SUMMARY
+          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+      - name: "Test Results Validation Outcome"
+        run: |
+          if [[ "$TEST_RESULT" != "" ]]; then
+            exit 1
+          else
+            exit 0
+          fi