From 424d314ee5be483208ed2e61c6b4587023f3b30c Mon Sep 17 00:00:00 2001 From: Dan Grebb Date: Tue, 16 Jan 2024 01:04:54 -0500 Subject: [PATCH] wip: add smoke test composite --- .github/actions/sanity-test-checks/action.yml | 39 +++-- .github/actions/smoke-test-checks/action.yml | 93 +++++++++++ .github/workflows/backstop-smoke-test.yml | 72 ++------ .github/workflows/docker-smoke-test.yml | 156 ++++-------------- 4 files changed, 163 insertions(+), 197 deletions(-) create mode 100644 .github/actions/smoke-test-checks/action.yml diff --git a/.github/actions/sanity-test-checks/action.yml b/.github/actions/sanity-test-checks/action.yml index 22b0420c5..9a40b6e0f 100644 --- a/.github/actions/sanity-test-checks/action.yml +++ b/.github/actions/sanity-test-checks/action.yml @@ -1,5 +1,5 @@ name: "Validate Sanity Report" -description: "Diff the sanity-test report with a fixture and validate expected shape." +description: "Diff the sanity-test reports with a fixture and validate expected shape." inputs: WORKSPACE_ROOT: @@ -22,18 +22,31 @@ runs: working-directory: ${{ inputs.WORKSPACE_ROOT }} run: | set +e - TEST_RESULT=$(diff -c <(jq \ - 'walk(if type == "object" then with_entries(.value |= \ - if type == "object" or type == "array" then . \ - else "" \ - end) \ - else . end)' \ - test/__fixtures__/${{ inputs.FIXTURE }}) \ - <(jq 'walk(if type == "object" then with_entries(.value |= \ - if type == "object" or type == "array" then . \ - else "" end) else . end)' \ - test/configs/backstop_data/bitmaps_test/**/report.json)\ + FIXTURE="test/__fixtures__/${{ inputs.FIXTURE }}" + TEST_RESULT=$(\ + diff -c \ + <(jq \ + 'walk(if type == "object" then with_entries(.value |= \ + if type == "object" or type == "array" \ + then . \ + else \ + "" \ + end) \ + else . \ + end)' \ + $FIXTURE) \ + <(jq \ + 'walk(if type == "object" then with_entries(.value |= \ + if type == "object" or type == "array" \ + then . \ + else \ + "" \ + end) \ + else . \ + end)' \ + test/configs/backstop_data/bitmaps_test/**/report.json) \ ) + echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV if [[ "$TEST_RESULT" != "" ]]; then echo "# ❎ ${{ inputs.RUNNER }} Sanity Different" >> $GITHUB_STEP_SUMMARY @@ -44,7 +57,7 @@ runs: echo "# ✅ ${{ inputs.RUNNER }} Sanity Report Valid" >> $GITHUB_STEP_SUMMARY fi - - name: "Full Smoke Test Diff" + - name: "Full Sanity Report Diff" id: diff continue-on-error: true shell: bash diff --git a/.github/actions/smoke-test-checks/action.yml b/.github/actions/smoke-test-checks/action.yml new file mode 100644 index 000000000..03b14f087 --- /dev/null +++ b/.github/actions/smoke-test-checks/action.yml @@ -0,0 +1,93 @@ +name: "Validate Smoke Report" +description: "Diff the smoke-test reports with a fixture and validate expected shape." + +inputs: + WORKSPACE_ROOT: + description: "Working Directory" + required: true + RUNNER: + description: "Test Runner" + required: true + FIXTURE: + description: "Test Report Fixture" + required: true + +runs: + using: "composite" + steps: + - name: "Validate ${{ inputs.RUNNER }} Report" + id: validate + continue-on-error: true + shell: bash + working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} + run: | + set +e + FIXTURE="test/__fixtures__/${{ inputs.FIXTURE }}" + TEST_RESULT=$(\ + diff -c \ + <(jq \ + 'walk(if type == "object" then with_entries(.value |= \ + if type == "object" or type == "array" + then . \ + else \ + "" \ + end) \ + else . \ + end) | \ + del(\ + .tests[].pair.diff, \ + .tests[].pair.diffImage \ + )'\ + $FIXTURE) \ + <(jq \ + 'walk(if type == "object" then with_entries(.value |= \ + if type == "object" or type == "array" \ + then . \ + else \ + "" \ + end) \ + else . \ + end) | \ + del(\ + .tests[].pair.diff, \ + .tests[].pair.diffImage\ + )' \ + test/configs/backstop_data/bitmaps_test/**/report.json)) + + echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV + if [[ "$TEST_RESULT" != "" ]]; then + echo "# ❎ ${{ inputs.RUNNER }} Report Different" >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + echo "$TEST_RESULT" >> $GITHUB_STEP_SUMMARY + echo '```' + else + echo "# ✅ ${{ inputs.RUNNER }} Report Validated" >> $GITHUB_STEP_SUMMARY + fi + + - name: "Full Smoke Report Diff" + id: diff + continue-on-error: true + shell: bash + working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} + run: | + set +e + FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/${{ inputs.FIXTURE }}) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) + echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + echo "Expand Diff" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + + - name: "Report Validation Outcome" + shell: bash + working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} + run: | + if [[ "$TEST_RESULT" != "" ]]; then + exit 1 + else + exit 0 + fi diff --git a/.github/workflows/backstop-smoke-test.yml b/.github/workflows/backstop-smoke-test.yml index da4f7e904..c9d66d62d 100644 --- a/.github/workflows/backstop-smoke-test.yml +++ b/.github/workflows/backstop-smoke-test.yml @@ -40,50 +40,11 @@ jobs: run: | npm run smoke-test - - name: "Validate Puppeteer Report" - id: validate - continue-on-error: true - shell: bash - working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} - run: | - set +e - PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/__fixtures__/smoke-test.json) \ - <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV - if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then - echo "# ❎ Puppeteer Report Different" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "$PUPPETEER_TEST_RESULT" >> $GITHUB_STEP_SUMMARY - echo '```' - else - echo "# ✅ Puppeteer Report Validated" >> $GITHUB_STEP_SUMMARY - fi - - - name: "Full Smoke Test Diff" - id: diff - continue-on-error: true - shell: bash - working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} - run: | - set +e - FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Expand Diff" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - - name: "Report Validation Outcome" - run: | - if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then - exit 1 - else - exit 0 - fi + - uses: ./actions/.github/actions/smoke-test-checks + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} + RUNNER: "Puppeteer" + FIXTURE: "smoke-test.json" smoke-playwright: name: 💨 Playwright @@ -113,21 +74,8 @@ jobs: npx playwright install --with-deps npm run smoke-test-playwright - - name: "Validate Playwright Report" - id: validate - continue-on-error: true - shell: bash - working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} - run: | - set +e - PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq -S 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/__fixtures__/smoke-test-playwright.json) \ - <(jq -S 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV - if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then - echo "# ❎ Playwright Report Different" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_STEP_SUMMARY - echo '```' - else - echo "# ✅ Playwright Report Validated" >> $GITHUB_STEP_SUMMARY - fi + - uses: ./actions/.github/actions/smoke-test-checks + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} + RUNNER: "Playwright" + FIXTURE: "smoke-test-playwright.json" diff --git a/.github/workflows/docker-smoke-test.yml b/.github/workflows/docker-smoke-test.yml index 5fc46e5ee..a0b3998a7 100644 --- a/.github/workflows/docker-smoke-test.yml +++ b/.github/workflows/docker-smoke-test.yml @@ -27,33 +27,22 @@ jobs: packages: write steps: - - name: Checkout + - name: Checkout actions uses: actions/checkout@v4 with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha || github.ref }} + ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} + sparse-checkout: | + .github/actions + path: actions - - name: "Setup Node, npm Cache, and Install" - uses: ./.github/actions/setup-base + - uses: ./actions/.github/actions/setup-base + id: base - - name: Set Name and Tag Vars - env: - name: "${{ env.BRANCH_NAME }}" - run: | - echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV} - echo "TAG=${name/\//-}" >> $GITHUB_ENV - echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV - - - name: Log in to the Container Registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + - uses: ./actions/.github/actions/prepare-docker with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Pull Image - run: | - docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG + REGISTRY: ${{ env.REGISTRY }} + ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: "𓋏 Run `backstop test --confg=backstop_features` in Docker" continue-on-error: true @@ -61,45 +50,12 @@ jobs: set +e cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test --config=backstop_features - - name: "Validate Puppeteer Report" - continue-on-error: true - run: | - set +e - PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/smoke-test-docker.json) \ - <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV - if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then - echo "# ❎ Puppeteer Smoke Different" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - else - echo "# ✅ Puppeteer Smoke Report Valid" >> $GITHUB_STEP_SUMMARY - fi - - - name: "Full Smoke Report Diff" - id: diff - continue-on-error: true - run: | - set +e - FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/smoke-test-docker.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Expand Diff" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - - name: "Report Validation Outcome" - run: | - if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then - exit 1 - else - exit 0 - fi + - name: "Validate Puppeteer Docker Test Results" + uses: ./actions/.github/actions/smoke-test-checks + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} + RUNNER: "Puppeteer" + FIXTURE: "smoke-test-docker.json" smoke-test-playwright: name: 💨 Playwright @@ -110,33 +66,22 @@ jobs: packages: write steps: - - name: Checkout + - name: Checkout actions uses: actions/checkout@v4 with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha || github.ref }} + ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} + sparse-checkout: | + .github/actions + path: actions - - name: "Setup Node, npm Cache, and Install" - uses: ./.github/actions/setup-base + - uses: ./actions/.github/actions/setup-base + id: base - - name: Set Name and Tag Vars - env: - name: "${{ env.BRANCH_NAME }}" - run: | - echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV} - echo "TAG=${name/\//-}" >> $GITHUB_ENV - echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + - uses: ./actions/.github/actions/prepare-docker with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Pull Image - run: | - docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG + REGISTRY: ${{ env.REGISTRY }} + ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: "🎭 Run `backstop test --confg=backstop_features_pw` in Docker" continue-on-error: true @@ -144,42 +89,9 @@ jobs: set +e cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test --config=backstop_features_pw - - name: "Validate Playwright Report" - continue-on-error: true - run: | - set +e - PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/smoke-test-playwright-docker.json) \ - <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV - if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then - echo "# ❎ Playwright Sanity Report Different" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - else - echo "# ✅ Playwright Sanity Report Valid" >> $GITHUB_STEP_SUMMARY - fi - - - name: "Full Sanity Report Diff" - id: diff - continue-on-error: true - run: | - set +e - FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/smoke-test-playwright-docker.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Expand Diff" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - - name: "Report Validation Outcome" - run: | - if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then - exit 1 - else - exit 0 - fi + - name: "Validate Playwright Docker Test Results" + uses: ./actions/.github/actions/smoke-test-checks + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} + RUNNER: "Playwright" + FIXTURE: "smoke-test-playwright-docker.json"