Run e2e tests in both Chrome and Electron (#5359)

* Update circle.yml to run e2e tests in chrome + electron

* WIP: create multiple tests per browser, but utilizing the same base snapshot

- create a dynamic test title with browser appended
- use a base named snapshot without the browser name appended
- patch snap-shot-it not to rethrow twice unnecessarily which ends up
throwing the wrong error

* Add e2e.coffee to stop-only ignore

* Convert existing x-browser e2e tests to use e2e.it

* update snapshots

* really add e2e to stop-only skip

* Replace screenshot dimensions in all tests, run only in process.env.BROWSER

* add e2e.it to more tests that should be x-browser

* update snapshots

* PEMDAS

* default e2e browser -> electron

Run afterEaches when skipping testo

Recurse upwards with runAfterEach

stop promise chains

* Revert "default e2e browser -> electron"

This reverts commit 3104f99.

* Undo

* [email protected]

* Remove snap-shot-it patch

* Change e2e snapshot to use allowSharedSnapshot

* Add keepScreenshotDimensions to screenshot specs

* update snapshots

* fix snapshotit

* keepScreenshotDimensions: true -> sanitizeScreenshotDimensions: false

* Fix screenshot snapshots

* reduce snapshot noise

* rm 0_simple_spec

* deabstract circle.yml command

* fix circle.yml

* reduce snapshot noise

* clean up 6_visit_spec snapshot

* use allowSharedSnapshot: true in 5_spec_isolation

* Fix 5_screenshots, 2_form_submissions

* Normalize screenshot sizes in 5_spec_isolation snapshot

* Add sanitizeScreenshotDimensions only to non-e2e-project tests
Add --window-size=1280,720 --kiosk to chrome args

* update snapshots

* Use e2e.it for chrome/electron-specific tests

* handle logic for skipping tests when default browsers do not match specified browser

- implement e2e.it.skip to match mocha’s it.skip
- allow turning off dynamic test generation


Co-authored-by: Brian Mann <[email protected]>

circle.yml

@@ -37,6 +37,24 @@ executors:
       PLATFORM: mac
 
 commands:
+  run-e2e-tests:
+    parameters:
+      browser:
+        description: browser shortname to target
+        type: string
+      chunk:
+        description: e2e test chunk number
+        type: integer
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run:
+          command: npm run test-e2e -- --chunk << parameters.chunk >> --browser << parameters.browser >>
+          working_directory: packages/server
+      - store_test_results:
+          path: /tmp/cypress
+      - store-npm-logs
+
   store-npm-logs:
     description: Saves any NPM debug logs as artifacts in case there is a problem
     steps:
@@ -331,101 +349,117 @@ jobs:
           path: /tmp/artifacts
       - store-npm-logs
 
-  "server-e2e-tests-1":
+  "server-e2e-tests-chrome-1":
     <<: *defaults
     steps:
-      - attach_workspace:
-          at: ~/
-      - run:
-          command: npm run test-e2e -- --chunk 1
-          working_directory: packages/server
-      - store_test_results:
-          path: /tmp/cypress
-      - store-npm-logs
+      - run-e2e-tests:
+          browser: chrome
+          chunk: 1
 
-  "server-e2e-tests-2":
+  "server-e2e-tests-chrome-2":
     <<: *defaults
     steps:
-      - attach_workspace:
-          at: ~/
-      - run:
-          command: npm run test-e2e -- --chunk 2
-          working_directory: packages/server
-      - store_test_results:
-          path: /tmp/cypress
-      - store-npm-logs
+      - run-e2e-tests:
+          browser: chrome
+          chunk: 2
 
-  "server-e2e-tests-3":
+  "server-e2e-tests-chrome-3":
     <<: *defaults
     steps:
-      - attach_workspace:
-          at: ~/
-      - run:
-          command: npm run test-e2e -- --chunk 3
-          working_directory: packages/server
-      - store_test_results:
-          path: /tmp/cypress
+      - run-e2e-tests:
+          browser: chrome
+          chunk: 3
 
+  "server-e2e-tests-chrome-4":
+    <<: *defaults
+    steps:
+      - run-e2e-tests:
+          browser: chrome
+          chunk: 4
 
-  "server-e2e-tests-4":
+  "server-e2e-tests-chrome-5":
     <<: *defaults
     steps:
-      - attach_workspace:
-          at: ~/
-      - run:
-          command: npm run test-e2e -- --chunk 4
-          working_directory: packages/server
-      - store_test_results:
-          path: /tmp/cypress
-      - store-npm-logs
+      - run-e2e-tests:
+          browser: chrome
+          chunk: 5
 
-  "server-e2e-tests-5":
+  "server-e2e-tests-chrome-6":
     <<: *defaults
     steps:
-      - attach_workspace:
-          at: ~/
-      - run:
-          command: npm run test-e2e -- --chunk 5
-          working_directory: packages/server
-      - store_test_results:
-          path: /tmp/cypress
-      - store-npm-logs
+      - run-e2e-tests:
+          browser: chrome
+          chunk: 6
 
-  "server-e2e-tests-6":
+  "server-e2e-tests-chrome-7":
     <<: *defaults
     steps:
-      - attach_workspace:
-          at: ~/
-      - run:
-          command: npm run test-e2e -- --chunk 6
-          working_directory: packages/server
-      - store_test_results:
-          path: /tmp/cypress
-      - store-npm-logs
+      - run-e2e-tests:
+          browser: chrome
+          chunk: 7
 
-  "server-e2e-tests-7":
+  "server-e2e-tests-chrome-8":
     <<: *defaults
     steps:
-      - attach_workspace:
-          at: ~/
-      - run:
-          command: npm run test-e2e -- --chunk 7
-          working_directory: packages/server
-      - store_test_results:
-          path: /tmp/cypress
-      - store-npm-logs
+      - run-e2e-tests:
+          browser: chrome
+          chunk: 8
 
-  "server-e2e-tests-8":
-      <<: *defaults
-      steps:
-      - attach_workspace:
-          at: ~/
-      - run:
-          command: npm run test-e2e -- --chunk 8
-          working_directory: packages/server
-      - store_test_results:
-          path: /tmp/cypress
-      - store-npm-logs
+  "server-e2e-tests-electron-1":
+    <<: *defaults
+    steps:
+      - run-e2e-tests:
+          browser: electron
+          chunk: 1
+
+  "server-e2e-tests-electron-2":
+    <<: *defaults
+    steps:
+      - run-e2e-tests:
+          browser: electron
+          chunk: 2
+
+  "server-e2e-tests-electron-3":
+    <<: *defaults
+    steps:
+      - run-e2e-tests:
+          browser: electron
+          chunk: 3
+
+  "server-e2e-tests-electron-4":
+    <<: *defaults
+    steps:
+      - run-e2e-tests:
+          browser: electron
+          chunk: 4
+
+  "server-e2e-tests-electron-5":
+    <<: *defaults
+    steps:
+      - run-e2e-tests:
+          browser: electron
+          chunk: 5
+
+  "server-e2e-tests-electron-6":
+    <<: *defaults
+    steps:
+      - run-e2e-tests:
+          browser: electron
+          chunk: 6
+
+  "server-e2e-tests-electron-7":
+    <<: *defaults
+    steps:
+      - run-e2e-tests:
+          browser: electron
+          chunk: 7
+
+  "server-e2e-tests-electron-8":
+    <<: *defaults
+    steps:
+      - run-e2e-tests:
+          browser: electron
+          chunk: 8
 
   "driver-integration-tests-chrome":
     <<: *defaults
@@ -829,28 +863,52 @@ linux-workflow: &linux-workflow
     - server-performance-tests:
         requires:
           - build
-    - server-e2e-tests-1:
+    - server-e2e-tests-chrome-1:
+        requires:
+          - build
+    - server-e2e-tests-chrome-2:
+        requires:
+          - build
+    - server-e2e-tests-chrome-3:
+        requires:
+          - build
+    - server-e2e-tests-chrome-4:
+        requires:
+          - build
+    - server-e2e-tests-chrome-5:
+        requires:
+          - build
+    - server-e2e-tests-chrome-6:
+        requires:
+          - build
+    - server-e2e-tests-chrome-7:
+        requires:
+          - build
+    - server-e2e-tests-chrome-8:
+        requires:
+          - build
+    - server-e2e-tests-electron-1:
         requires:
           - build
-    - server-e2e-tests-2:
+    - server-e2e-tests-electron-2:
         requires:
           - build
-    - server-e2e-tests-3:
+    - server-e2e-tests-electron-3:
         requires:
           - build
-    - server-e2e-tests-4:
+    - server-e2e-tests-electron-4:
         requires:
           - build
-    - server-e2e-tests-5:
+    - server-e2e-tests-electron-5:
         requires:
           - build
-    - server-e2e-tests-6:
+    - server-e2e-tests-electron-6:
         requires:
           - build
-    - server-e2e-tests-7:
+    - server-e2e-tests-electron-7:
         requires:
           - build
-    - server-e2e-tests-8:
+    - server-e2e-tests-electron-8:
         requires:
           - build
     - driver-integration-tests-chrome:

cli/package.json

@@ -86,7 +86,7 @@
     "proxyquire": "2.1.0",
     "shelljs": "0.8.3",
     "sinon": "7.2.2",
-    "snap-shot-it": "7.8.0",
+    "snap-shot-it": "7.9.0",
     "spawn-mock": "1.0.0",
     "strip-ansi": "4.0.0"
   },

package.json

@@ -47,7 +47,7 @@
     "set-next-ci-version": "node ./scripts/binary.js setNextVersion",
     "prestart": "npm run check-deps-pre",
     "start": "node ./cli/bin/cypress open --dev --global",
-    "stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,spec_helper.coffee",
+    "stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components --exclude e2e.coffee",
     "stop-only-all": "npm run stop-only -- --folder packages",
     "test": "echo '⚠️ This root monorepo is only for local development and new contributions. There are no tests.'",
     "test-debug-package": "node ./scripts/test-debug-package.js",
@@ -152,7 +152,7 @@
     "shelljs": "0.8.3",
     "shx": "0.3.2",
     "sinon": "7.3.2",
-    "snap-shot-it": "7.8.0",
+    "snap-shot-it": "7.9.0",
     "stop-only": "3.0.1",
     "strip-ansi": "4.0.0",
     "terminal-banner": "1.1.0",

packages/server/__snapshots__/1_base_url_spec.coffee.js

@@ -1,4 +1,4 @@
-exports['e2e baseUrl https passes 1'] = `
+exports['e2e baseUrl / https / passes'] = `
 
 ====================================================================================================
 
@@ -59,7 +59,7 @@ exports['e2e baseUrl https passes 1'] = `
 
 `
 
-exports['e2e baseUrl http passes 1'] = `
+exports['e2e baseUrl / http / passes'] = `
 
 ====================================================================================================