ci(tests): adds fixtures and sanity-test validation (garris#1533)

garris#1533
dgrebb · Dec 31, 2023 · f575965 · f575965
1 parent 141938c
commit f575965
Show file tree

Hide file tree

Showing 5 changed files with 2,268 additions and 0 deletions.
diff --git a/.github/workflows/backstop-sanity-test.yml b/.github/workflows/backstop-sanity-test.yml
@@ -33,9 +33,20 @@ jobs:
         run: npm ci
 
       - name: "𓋏 Run `npm run sanity-test`"
+        continue-on-error: true
         run: |
           npm run sanity-test
 
+      - name: "Validate Test Results"
+        run: |
+          diff -q <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage)" test/__fixtures__/sanity-test.json) \
+          <(jq -S ". | del(.tests[].pair.test, .tests[].pair.testLog, .tests[].pair.diffImage)" test/configs/backstop_data/bitmaps_test/**/report.json)
+          if [[ $? == "0" ]]; then
+            exit 0
+          else
+            exit 1
+          fi
+
   sanity-playwright:
     name: 🤪 Playwright
     runs-on: ubuntu-latest

diff --git a/test/__fixtures__/sanity-test-playwright.json b/test/__fixtures__/sanity-test-playwright.json
@@ -0,0 +1,40 @@
+{
+  "testSuite": "BackstopJS",
+  "tests": [
+    {
+      "pair": {
+        "reference": "../bitmaps_reference/backstop_playwright_BackstopJS_Homepage_0_document_0_phone.png",
+        "test": "../bitmaps_test/20231230-012035/backstop_playwright_BackstopJS_Homepage_0_document_0_phone.png",
+        "selector": "document",
+        "fileName": "backstop_playwright_BackstopJS_Homepage_0_document_0_phone.png",
+        "label": "BackstopJS Homepage",
+        "requireSameDimensions": true,
+        "misMatchThreshold": 0.1,
+        "url": "https://garris.github.io/BackstopJS/",
+        "referenceUrl": "",
+        "expect": 0,
+        "viewportLabel": "phone",
+        "error": "Reference file not found /Users/dgrebb/Projects/BackstopJS/test/configs/backstop_data/bitmaps_reference/backstop_playwright_BackstopJS_Homepage_0_document_0_phone.png"
+      },
+      "status": "fail"
+    },
+    {
+      "pair": {
+        "reference": "../bitmaps_reference/backstop_playwright_BackstopJS_Homepage_0_document_1_tablet.png",
+        "test": "../bitmaps_test/20231230-012035/backstop_playwright_BackstopJS_Homepage_0_document_1_tablet.png",
+        "selector": "document",
+        "fileName": "backstop_playwright_BackstopJS_Homepage_0_document_1_tablet.png",
+        "label": "BackstopJS Homepage",
+        "requireSameDimensions": true,
+        "misMatchThreshold": 0.1,
+        "url": "https://garris.github.io/BackstopJS/",
+        "referenceUrl": "",
+        "expect": 0,
+        "viewportLabel": "tablet",
+        "error": "Reference file not found /Users/dgrebb/Projects/BackstopJS/test/configs/backstop_data/bitmaps_reference/backstop_playwright_BackstopJS_Homepage_0_document_1_tablet.png"
+      },
+      "status": "fail"
+    }
+  ],
+  "id": "backstop_playwright"
+}
diff --git a/test/__fixtures__/sanity-test.json b/test/__fixtures__/sanity-test.json
@@ -0,0 +1,62 @@
+{
+  "testSuite": "BackstopJS",
+  "tests": [
+    {
+      "pair": {
+        "reference": "../bitmaps_reference/backstop_default_BackstopJS_Homepage_0_document_0_phone.png",
+        "test": "../bitmaps_test/20231231-005924/backstop_default_BackstopJS_Homepage_0_document_0_phone.png",
+        "testLog": "../bitmaps_test/20231231-005924/backstop_default_BackstopJS_Homepage_0_document_0_phone.log.json",
+        "selector": "document",
+        "fileName": "backstop_default_BackstopJS_Homepage_0_document_0_phone.png",
+        "label": "BackstopJS Homepage",
+        "requireSameDimensions": true,
+        "misMatchThreshold": 0.1,
+        "url": "https://garris.github.io/BackstopJS/",
+        "referenceUrl": "",
+        "expect": 0,
+        "viewportLabel": "phone",
+        "diff": {
+          "isSameDimensions": true,
+          "dimensionDifference": {
+            "width": 0,
+            "height": 0
+          },
+          "rawMisMatchPercentage": 3.750635162601626,
+          "misMatchPercentage": "3.75",
+          "analysisTime": 13
+        },
+        "diffImage": "../bitmaps_test/20231231-005924/failed_diff_backstop_default_BackstopJS_Homepage_0_document_0_phone.png"
+      },
+      "status": "fail"
+    },
+    {
+      "pair": {
+        "reference": "../bitmaps_reference/backstop_default_BackstopJS_Homepage_0_document_1_tablet.png",
+        "test": "../bitmaps_test/20231231-005924/backstop_default_BackstopJS_Homepage_0_document_1_tablet.png",
+        "testLog": "../bitmaps_test/20231231-005924/backstop_default_BackstopJS_Homepage_0_document_1_tablet.log.json",
+        "selector": "document",
+        "fileName": "backstop_default_BackstopJS_Homepage_0_document_1_tablet.png",
+        "label": "BackstopJS Homepage",
+        "requireSameDimensions": true,
+        "misMatchThreshold": 0.1,
+        "url": "https://garris.github.io/BackstopJS/",
+        "referenceUrl": "",
+        "expect": 0,
+        "viewportLabel": "tablet",
+        "diff": {
+          "isSameDimensions": true,
+          "dimensionDifference": {
+            "width": 0,
+            "height": 0
+          },
+          "rawMisMatchPercentage": 1.9487233813980136,
+          "misMatchPercentage": "1.95",
+          "analysisTime": 34
+        },
+        "diffImage": "../bitmaps_test/20231231-005924/failed_diff_backstop_default_BackstopJS_Homepage_0_document_1_tablet.png"
+      },
+      "status": "fail"
+    }
+  ],
+  "id": "backstop_default"
+}