diff --git a/circle.yml b/circle.yml index b8d7ab603050..ab6bae4b14cb 100644 --- a/circle.yml +++ b/circle.yml @@ -37,6 +37,24 @@ executors: PLATFORM: mac commands: + run-e2e-tests: + parameters: + browser: + description: browser shortname to target + type: string + chunk: + description: e2e test chunk number + type: integer + steps: + - attach_workspace: + at: ~/ + - run: + command: npm run test-e2e -- --chunk << parameters.chunk >> --browser << parameters.browser >> + working_directory: packages/server + - store_test_results: + path: /tmp/cypress + - store-npm-logs + store-npm-logs: description: Saves any NPM debug logs as artifacts in case there is a problem steps: @@ -331,101 +349,117 @@ jobs: path: /tmp/artifacts - store-npm-logs - "server-e2e-tests-1": + "server-e2e-tests-chrome-1": <<: *defaults steps: - - attach_workspace: - at: ~/ - - run: - command: npm run test-e2e -- --chunk 1 - working_directory: packages/server - - store_test_results: - path: /tmp/cypress - - store-npm-logs + - run-e2e-tests: + browser: chrome + chunk: 1 - "server-e2e-tests-2": + "server-e2e-tests-chrome-2": <<: *defaults steps: - - attach_workspace: - at: ~/ - - run: - command: npm run test-e2e -- --chunk 2 - working_directory: packages/server - - store_test_results: - path: /tmp/cypress - - store-npm-logs + - run-e2e-tests: + browser: chrome + chunk: 2 - "server-e2e-tests-3": + "server-e2e-tests-chrome-3": <<: *defaults steps: - - attach_workspace: - at: ~/ - - run: - command: npm run test-e2e -- --chunk 3 - working_directory: packages/server - - store_test_results: - path: /tmp/cypress + - run-e2e-tests: + browser: chrome + chunk: 3 + "server-e2e-tests-chrome-4": + <<: *defaults + steps: + - run-e2e-tests: + browser: chrome + chunk: 4 - "server-e2e-tests-4": + "server-e2e-tests-chrome-5": <<: *defaults steps: - - attach_workspace: - at: ~/ - - run: - command: npm run test-e2e -- --chunk 4 - working_directory: packages/server - - store_test_results: - path: /tmp/cypress - - store-npm-logs + - run-e2e-tests: + browser: chrome + chunk: 5 - "server-e2e-tests-5": + "server-e2e-tests-chrome-6": <<: *defaults steps: - - attach_workspace: - at: ~/ - - run: - command: npm run test-e2e -- --chunk 5 - working_directory: packages/server - - store_test_results: - path: /tmp/cypress - - store-npm-logs + - run-e2e-tests: + browser: chrome + chunk: 6 - "server-e2e-tests-6": + "server-e2e-tests-chrome-7": <<: *defaults steps: - - attach_workspace: - at: ~/ - - run: - command: npm run test-e2e -- --chunk 6 - working_directory: packages/server - - store_test_results: - path: /tmp/cypress - - store-npm-logs + - run-e2e-tests: + browser: chrome + chunk: 7 - "server-e2e-tests-7": + "server-e2e-tests-chrome-8": <<: *defaults steps: - - attach_workspace: - at: ~/ - - run: - command: npm run test-e2e -- --chunk 7 - working_directory: packages/server - - store_test_results: - path: /tmp/cypress - - store-npm-logs + - run-e2e-tests: + browser: chrome + chunk: 8 - "server-e2e-tests-8": - <<: *defaults - steps: - - attach_workspace: - at: ~/ - - run: - command: npm run test-e2e -- --chunk 8 - working_directory: packages/server - - store_test_results: - path: /tmp/cypress - - store-npm-logs + "server-e2e-tests-electron-1": + <<: *defaults + steps: + - run-e2e-tests: + browser: electron + chunk: 1 + + "server-e2e-tests-electron-2": + <<: *defaults + steps: + - run-e2e-tests: + browser: electron + chunk: 2 + + "server-e2e-tests-electron-3": + <<: *defaults + steps: + - run-e2e-tests: + browser: electron + chunk: 3 + + "server-e2e-tests-electron-4": + <<: *defaults + steps: + - run-e2e-tests: + browser: electron + chunk: 4 + + "server-e2e-tests-electron-5": + <<: *defaults + steps: + - run-e2e-tests: + browser: electron + chunk: 5 + + "server-e2e-tests-electron-6": + <<: *defaults + steps: + - run-e2e-tests: + browser: electron + chunk: 6 + + "server-e2e-tests-electron-7": + <<: *defaults + steps: + - run-e2e-tests: + browser: electron + chunk: 7 + + "server-e2e-tests-electron-8": + <<: *defaults + steps: + - run-e2e-tests: + browser: electron + chunk: 8 "driver-integration-tests-chrome": <<: *defaults @@ -829,28 +863,52 @@ linux-workflow: &linux-workflow - server-performance-tests: requires: - build - - server-e2e-tests-1: + - server-e2e-tests-chrome-1: + requires: + - build + - server-e2e-tests-chrome-2: + requires: + - build + - server-e2e-tests-chrome-3: + requires: + - build + - server-e2e-tests-chrome-4: + requires: + - build + - server-e2e-tests-chrome-5: + requires: + - build + - server-e2e-tests-chrome-6: + requires: + - build + - server-e2e-tests-chrome-7: + requires: + - build + - server-e2e-tests-chrome-8: + requires: + - build + - server-e2e-tests-electron-1: requires: - build - - server-e2e-tests-2: + - server-e2e-tests-electron-2: requires: - build - - server-e2e-tests-3: + - server-e2e-tests-electron-3: requires: - build - - server-e2e-tests-4: + - server-e2e-tests-electron-4: requires: - build - - server-e2e-tests-5: + - server-e2e-tests-electron-5: requires: - build - - server-e2e-tests-6: + - server-e2e-tests-electron-6: requires: - build - - server-e2e-tests-7: + - server-e2e-tests-electron-7: requires: - build - - server-e2e-tests-8: + - server-e2e-tests-electron-8: requires: - build - driver-integration-tests-chrome: diff --git a/cli/package.json b/cli/package.json index e39ca410e074..c8c3ec3900e5 100644 --- a/cli/package.json +++ b/cli/package.json @@ -86,7 +86,7 @@ "proxyquire": "2.1.0", "shelljs": "0.8.3", "sinon": "7.2.2", - "snap-shot-it": "7.8.0", + "snap-shot-it": "7.9.0", "spawn-mock": "1.0.0", "strip-ansi": "4.0.0" }, diff --git a/package.json b/package.json index b774bc5ad511..71de8a6e23c7 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "set-next-ci-version": "node ./scripts/binary.js setNextVersion", "prestart": "npm run check-deps-pre", "start": "node ./cli/bin/cypress open --dev --global", - "stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,spec_helper.coffee", + "stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components --exclude e2e.coffee", "stop-only-all": "npm run stop-only -- --folder packages", "test": "echo '⚠️ This root monorepo is only for local development and new contributions. There are no tests.'", "test-debug-package": "node ./scripts/test-debug-package.js", @@ -152,7 +152,7 @@ "shelljs": "0.8.3", "shx": "0.3.2", "sinon": "7.3.2", - "snap-shot-it": "7.8.0", + "snap-shot-it": "7.9.0", "stop-only": "3.0.1", "strip-ansi": "4.0.0", "terminal-banner": "1.1.0", diff --git a/packages/server/__snapshots__/1_base_url_spec.coffee.js b/packages/server/__snapshots__/1_base_url_spec.coffee.js index 4a6ffdac121e..ddb86519fed0 100644 --- a/packages/server/__snapshots__/1_base_url_spec.coffee.js +++ b/packages/server/__snapshots__/1_base_url_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e baseUrl https passes 1'] = ` +exports['e2e baseUrl / https / passes'] = ` ==================================================================================================== @@ -59,7 +59,7 @@ exports['e2e baseUrl https passes 1'] = ` ` -exports['e2e baseUrl http passes 1'] = ` +exports['e2e baseUrl / http / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/1_commands_outside_of_test_spec.coffee.js b/packages/server/__snapshots__/1_commands_outside_of_test_spec.coffee.js index 605b1a5e23e4..f79031aa7bcf 100644 --- a/packages/server/__snapshots__/1_commands_outside_of_test_spec.coffee.js +++ b/packages/server/__snapshots__/1_commands_outside_of_test_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e commands outside of test passes on passing assertions 1'] = ` +exports['e2e commands outside of test / passes on passing assertions'] = ` ==================================================================================================== @@ -55,179 +55,7 @@ exports['e2e commands outside of test passes on passing assertions 1'] = ` ` -exports['e2e commands outside of test [chrome] fails on cy commands 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (commands_outside_of_test_spec.coffee) │ - │ Searched: cypress/integration/commands_outside_of_test_spec.coffee │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: commands_outside_of_test_spec.coffee... (1 of 1) - - - 1) An uncaught error was detected outside of a test - - 0 passing - 1 failing - - 1) An uncaught error was detected outside of a test: - Uncaught CypressError: Cannot call "cy.viewport()" outside a running test. - -This usually happens when you accidentally write commands outside an it(...) test. - -If that is the case, just move these commands inside an it(...) test. - -Check your test file for errors. - -https://on.cypress.io/cannot-execute-commands-outside-test - -This error originated from your test code, not from Cypress. - -When Cypress detects uncaught errors originating from your test code it will automatically fail the current test. - -Cypress could not associate this error to any specific test. - -We dynamically generated a new test to display this failure. - at stack trace line - at stack trace line - at stack trace line - at stack trace line - at stack trace line - at stack trace line - at stack trace line - - - - - (Results) - - ┌────────────────────────────────────────────────────┐ - │ Tests: 1 │ - │ Passing: 0 │ - │ Failing: 1 │ - │ Pending: 0 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: true │ - │ Duration: X seconds │ - │ Spec Ran: commands_outside_of_test_spec.coffee │ - └────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /foo/bar/.projects/e2e/cypress/screenshots/commands_outside_of_test_spec.coffee/An uncaught error was detected outside of a test (failed).png (YYYYxZZZZ) - - - (Video) - - - Started processing: Compressing to 32 CRF - - Finished processing: /foo/bar/.projects/e2e/cypress/videos/abc123.mp4 (X seconds) - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ commands_outside_of_test_spec.coffee XX:XX 1 - 1 - - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - 1 of 1 failed (100%) XX:XX 1 - 1 - - - - -` - -exports['e2e commands outside of test [chrome] fails on failing assertions 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (assertions_failing_outside_of_test_spec.coffee) │ - │ Searched: cypress/integration/assertions_failing_outside_of_test_spec.coffee │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: assertions_failing_outside_of_test_spec.coffee... (1 of 1) - - - 1) An uncaught error was detected outside of a test - - 0 passing - 1 failing - - 1) An uncaught error was detected outside of a test: - expected true to be false - -This error originated from your test code, not from Cypress. - -When Cypress detects uncaught errors originating from your test code it will automatically fail the current test. - -Cypress could not associate this error to any specific test. - -We dynamically generated a new test to display this failure. - AssertionError: expected true to be false - - - - - (Results) - - ┌──────────────────────────────────────────────────────────────┐ - │ Tests: 1 │ - │ Passing: 0 │ - │ Failing: 1 │ - │ Pending: 0 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: true │ - │ Duration: X seconds │ - │ Spec Ran: assertions_failing_outside_of_test_spec.coffee │ - └──────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /foo/bar/.projects/e2e/cypress/screenshots/assertions_failing_outside_of_test_spec.coffee/An uncaught error was detected outside of a test (failed).png (YYYYxZZZZ) - - - (Video) - - - Started processing: Compressing to 32 CRF - - Finished processing: /foo/bar/.projects/e2e/cypress/videos/abc123.mp4 (X seconds) - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ assertions_failing_outside_of_test_s… XX:XX 1 - 1 - - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - 1 of 1 failed (100%) XX:XX 1 - 1 - - - - -` - -exports['e2e commands outside of test [electron] fails on cy commands 1'] = ` +exports['e2e commands outside of test / fails on cy commands'] = ` ==================================================================================================== @@ -320,7 +148,7 @@ We dynamically generated a new test to display this failure. ` -exports['e2e commands outside of test [electron] fails on failing assertions 1'] = ` +exports['e2e commands outside of test / fails on failing assertions'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/2_cdp_spec.ts.js b/packages/server/__snapshots__/2_cdp_spec.ts.js index 9d4d4ccb30c1..81daa1fa80fe 100644 --- a/packages/server/__snapshots__/2_cdp_spec.ts.js +++ b/packages/server/__snapshots__/2_cdp_spec.ts.js @@ -1,4 +1,4 @@ -exports['e2e cdp fails when remote debugging port cannot be connected to 1'] = ` +exports['e2e cdp / fails when remote debugging port cannot be connected to'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/2_cookies_spec.coffee.js b/packages/server/__snapshots__/2_cookies_spec.coffee.js index 6a4546431d17..61fbdc72ea6a 100644 --- a/packages/server/__snapshots__/2_cookies_spec.coffee.js +++ b/packages/server/__snapshots__/2_cookies_spec.coffee.js @@ -1,75 +1,4 @@ -exports['e2e cookies passes in chrome 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (cookies_spec.coffee) │ - │ Searched: cypress/integration/cookies_spec.coffee │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: cookies_spec.coffee... (1 of 1) - - - cookies - with whitelist - ✓ can get all cookies - ✓ resets cookies between tests correctly - ✓ should be only two left now - ✓ handles undefined cookies - without whitelist - ✓ sends cookies to localhost:2121 - ✓ handles expired cookies secure - ✓ issue: #224 sets expired cookies between redirects - ✓ issue: #1321 failing to set or parse cookie - ✓ issue: #2724 does not fail on invalid cookies - - - 9 passing - - - (Results) - - ┌───────────────────────────────────┐ - │ Tests: 9 │ - │ Passing: 9 │ - │ Failing: 0 │ - │ Pending: 0 │ - │ Skipped: 0 │ - │ Screenshots: 0 │ - │ Video: true │ - │ Duration: X seconds │ - │ Spec Ran: cookies_spec.coffee │ - └───────────────────────────────────┘ - - - (Video) - - - Started processing: Compressing to 32 CRF - - Finished processing: /foo/bar/.projects/e2e/cypress/videos/abc123.mp4 (X seconds) - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ cookies_spec.coffee XX:XX 9 9 - - - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - All specs passed! XX:XX 9 9 - - - - - -` - -exports['e2e cookies passes in electron 1'] = ` +exports['e2e cookies / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/2_domain_spec.coffee.js b/packages/server/__snapshots__/2_domain_spec.coffee.js index 788b96e306b8..96293c3a2c75 100644 --- a/packages/server/__snapshots__/2_domain_spec.coffee.js +++ b/packages/server/__snapshots__/2_domain_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e domain passing 1'] = ` +exports['e2e domain / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/2_form_submissions_spec.coffee.js b/packages/server/__snapshots__/2_form_submissions_spec.coffee.js index bffcf83f9916..17b61a841c90 100644 --- a/packages/server/__snapshots__/2_form_submissions_spec.coffee.js +++ b/packages/server/__snapshots__/2_form_submissions_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e forms
submissions passes with http on localhost 1'] = ` +exports['e2e forms / submissions / passes with https on localhost'] = ` ==================================================================================================== @@ -66,7 +66,7 @@ exports['e2e forms submissions passes with http on localhost 1'] = ` ` -exports['e2e forms submissions passes with https on localhost 1'] = ` +exports['e2e forms / submissions / passes with http on localhost'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/2_go_spec.coffee.js b/packages/server/__snapshots__/2_go_spec.coffee.js index 595011135e39..133ef96385ee 100644 --- a/packages/server/__snapshots__/2_go_spec.coffee.js +++ b/packages/server/__snapshots__/2_go_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e go passes 1'] = ` +exports['e2e go / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/2_iframe_spec.coffee.js b/packages/server/__snapshots__/2_iframe_spec.coffee.js index 16af86e6f99b..a86174ff48f2 100644 --- a/packages/server/__snapshots__/2_iframe_spec.coffee.js +++ b/packages/server/__snapshots__/2_iframe_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e iframes passes 1'] = ` +exports['e2e iframes / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/2_images_spec.coffee.js b/packages/server/__snapshots__/2_images_spec.coffee.js index 459db9893363..e3b468b74d56 100644 --- a/packages/server/__snapshots__/2_images_spec.coffee.js +++ b/packages/server/__snapshots__/2_images_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e images passes 1'] = ` +exports['e2e images / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/3_js_error_handling_spec.coffee.js b/packages/server/__snapshots__/3_js_error_handling_spec.coffee.js index ec3c6aa219f1..fc747cc889d4 100644 --- a/packages/server/__snapshots__/3_js_error_handling_spec.coffee.js +++ b/packages/server/__snapshots__/3_js_error_handling_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e js error handling fails 1'] = ` +exports['e2e js error handling / fails'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/3_page_loading_spec.coffee.js b/packages/server/__snapshots__/3_page_loading_spec.coffee.js index c21c8d993b82..96ee36c99a3e 100644 --- a/packages/server/__snapshots__/3_page_loading_spec.coffee.js +++ b/packages/server/__snapshots__/3_page_loading_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e page_loading passes 1'] = ` +exports['e2e page_loading / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/3_plugins_spec.coffee.js b/packages/server/__snapshots__/3_plugins_spec.coffee.js index dc967c4f1bb1..4a1732c8fb2c 100644 --- a/packages/server/__snapshots__/3_plugins_spec.coffee.js +++ b/packages/server/__snapshots__/3_plugins_spec.coffee.js @@ -179,7 +179,7 @@ exports['e2e plugins can modify config from plugins 1'] = ` ` -exports['e2e plugins works with user extensions 1'] = ` +exports['e2e plugins / works with user extensions'] = ` ==================================================================================================== @@ -349,8 +349,8 @@ exports['e2e plugins calls after:screenshot for cy.screenshot() and failure scre (Screenshots) - /foo/bar/.projects/plugin-after-screenshot/screenshot-replacement.png (2x2) - - /foo/bar/.projects/plugin-after-screenshot/cypress/screenshots/after_screenshot_spec.coffee/ignored-values.png (1280x720) - - /foo/bar/.projects/plugin-after-screenshot/cypress/screenshots/after_screenshot_spec.coffee/invalid-return.png (1280x720) + - /foo/bar/.projects/plugin-after-screenshot/cypress/screenshots/after_screenshot_spec.coffee/ignored-values.png (YYYYxZZZZ) + - /foo/bar/.projects/plugin-after-screenshot/cypress/screenshots/after_screenshot_spec.coffee/invalid-return.png (YYYYxZZZZ) - /foo/bar/.projects/plugin-after-screenshot/screenshot-replacement.png (1x1) diff --git a/packages/server/__snapshots__/3_user_agent_spec.coffee.js b/packages/server/__snapshots__/3_user_agent_spec.coffee.js index 683c1c390102..3bb2547765ba 100644 --- a/packages/server/__snapshots__/3_user_agent_spec.coffee.js +++ b/packages/server/__snapshots__/3_user_agent_spec.coffee.js @@ -1,66 +1,4 @@ -exports['e2e user agent passes on chrome 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (user_agent_spec.coffee) │ - │ Searched: cypress/integration/user_agent_spec.coffee │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: user_agent_spec.coffee... (1 of 1) - - - user agent - ✓ is set on visits - ✓ is set on requests - - - 2 passing - - - (Results) - - ┌──────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 2 │ - │ Failing: 0 │ - │ Pending: 0 │ - │ Skipped: 0 │ - │ Screenshots: 0 │ - │ Video: true │ - │ Duration: X seconds │ - │ Spec Ran: user_agent_spec.coffee │ - └──────────────────────────────────────┘ - - - (Video) - - - Started processing: Compressing to 32 CRF - - Finished processing: /foo/bar/.projects/e2e/cypress/videos/abc123.mp4 (X seconds) - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ user_agent_spec.coffee XX:XX 2 2 - - - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - All specs passed! XX:XX 2 2 - - - - - -` - -exports['e2e user agent passes on electron 1'] = ` +exports['e2e user agent / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/4_request_spec.coffee.js b/packages/server/__snapshots__/4_request_spec.coffee.js index 1c85b3a3854b..2cc8de7c008a 100644 --- a/packages/server/__snapshots__/4_request_spec.coffee.js +++ b/packages/server/__snapshots__/4_request_spec.coffee.js @@ -1,76 +1,4 @@ -exports['e2e requests passes in electron 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (request_spec.coffee) │ - │ Searched: cypress/integration/request_spec.coffee │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: request_spec.coffee... (1 of 1) - - - redirects + requests - ✓ gets and sets cookies from cy.request - ✓ visits idempotant - ✓ automatically follows redirects - ✓ can turn off automatically following redirects - ✓ follows all redirects even when they change methods - ✓ can submit json body - ✓ can submit form url encoded body - ✓ can send qs query params - ✓ passes even on non 2xx or 3xx status code - ✓ sets Accept header to */* by default - ✓ can override the accept header - ✓ issue #375: does not duplicate request cookies on 302 redirect - - - 12 passing - - - (Results) - - ┌───────────────────────────────────┐ - │ Tests: 12 │ - │ Passing: 12 │ - │ Failing: 0 │ - │ Pending: 0 │ - │ Skipped: 0 │ - │ Screenshots: 0 │ - │ Video: true │ - │ Duration: X seconds │ - │ Spec Ran: request_spec.coffee │ - └───────────────────────────────────┘ - - - (Video) - - - Started processing: Compressing to 32 CRF - - Finished processing: /foo/bar/.projects/e2e/cypress/videos/abc123.mp4 (X seconds) - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ request_spec.coffee XX:XX 12 12 - - - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - All specs passed! XX:XX 12 12 - - - - - -` - -exports['e2e requests passes in chrome 1'] = ` +exports['e2e requests / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/4_screenshot_element_capture_spec.coffee.js b/packages/server/__snapshots__/4_screenshot_element_capture_spec.coffee.js index 5256495e618a..2902b3336631 100644 --- a/packages/server/__snapshots__/4_screenshot_element_capture_spec.coffee.js +++ b/packages/server/__snapshots__/4_screenshot_element_capture_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e screenshot element capture passes 1'] = ` +exports['e2e screenshot element capture / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/4_screenshot_fullpage_capture_spec.coffee.js b/packages/server/__snapshots__/4_screenshot_fullpage_capture_spec.coffee.js index b976fbeca076..93cff6b6ecdc 100644 --- a/packages/server/__snapshots__/4_screenshot_fullpage_capture_spec.coffee.js +++ b/packages/server/__snapshots__/4_screenshot_fullpage_capture_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e screenshot fullPage capture passes 1'] = ` +exports['e2e screenshot fullPage capture / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/4_screenshot_nested_file_spec.coffee.js b/packages/server/__snapshots__/4_screenshot_nested_file_spec.coffee.js index a9a715397d76..7f8b95fa33b7 100644 --- a/packages/server/__snapshots__/4_screenshot_nested_file_spec.coffee.js +++ b/packages/server/__snapshots__/4_screenshot_nested_file_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e screenshot in nested spec passes 1'] = ` +exports['e2e screenshot in nested spec / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/4_websockets_spec.coffee.js b/packages/server/__snapshots__/4_websockets_spec.coffee.js index 5d5148f78fb3..aaa71200edf7 100644 --- a/packages/server/__snapshots__/4_websockets_spec.coffee.js +++ b/packages/server/__snapshots__/4_websockets_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e websockets passes 1'] = ` +exports['e2e websockets / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/4_xhr_spec.coffee.js b/packages/server/__snapshots__/4_xhr_spec.coffee.js index 2e8b55f6c938..7ace67d4e0ee 100644 --- a/packages/server/__snapshots__/4_xhr_spec.coffee.js +++ b/packages/server/__snapshots__/4_xhr_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e xhr passes 1'] = ` +exports['e2e xhr / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/5_screenshot_viewport_capture_spec.coffee.js b/packages/server/__snapshots__/5_screenshot_viewport_capture_spec.coffee.js index 3fc8ecc72080..98a9d94b034b 100644 --- a/packages/server/__snapshots__/5_screenshot_viewport_capture_spec.coffee.js +++ b/packages/server/__snapshots__/5_screenshot_viewport_capture_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e screenshot viewport capture passes 1'] = ` +exports['e2e screenshot viewport capture / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/5_screenshots_spec.coffee.js b/packages/server/__snapshots__/5_screenshots_spec.coffee.js index f94fe4816fb5..2f89870855e6 100644 --- a/packages/server/__snapshots__/5_screenshots_spec.coffee.js +++ b/packages/server/__snapshots__/5_screenshots_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e screenshots passes 1'] = ` +exports['e2e screenshots / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/5_server_sent_events_spec.coffee.js b/packages/server/__snapshots__/5_server_sent_events_spec.coffee.js index ea749b1ee3df..1e8d97175351 100644 --- a/packages/server/__snapshots__/5_server_sent_events_spec.coffee.js +++ b/packages/server/__snapshots__/5_server_sent_events_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e server sent events passes 1'] = ` +exports['e2e server sent events / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/5_spec_isolation_spec.coffee.js b/packages/server/__snapshots__/5_spec_isolation_spec.coffee.js index 0ece8d0d9714..ccc555d82bdd 100644 --- a/packages/server/__snapshots__/5_spec_isolation_spec.coffee.js +++ b/packages/server/__snapshots__/5_spec_isolation_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e spec_isolation failing 1'] = { +exports['e2e spec isolation fails'] = { "startedTestsAt": "2018-02-01T20:14:19.323Z", "endedTestsAt": "2018-02-01T20:14:19.323Z", "totalDuration": 5555, diff --git a/packages/server/__snapshots__/5_stdout_spec.coffee.js b/packages/server/__snapshots__/5_stdout_spec.coffee.js index 5e47b0976cd7..de0de4f7d078 100644 --- a/packages/server/__snapshots__/5_stdout_spec.coffee.js +++ b/packages/server/__snapshots__/5_stdout_spec.coffee.js @@ -337,7 +337,7 @@ A video will not be recorded when using this mode. ` -exports['e2e stdout logs that chrome cannot be recorded 1'] = ` +exports['e2e stdout / logs that chrome cannot be recorded'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/5_subdomain_spec.coffee.js b/packages/server/__snapshots__/5_subdomain_spec.coffee.js index 5818ed1b3a39..eadf0362bdb6 100644 --- a/packages/server/__snapshots__/5_subdomain_spec.coffee.js +++ b/packages/server/__snapshots__/5_subdomain_spec.coffee.js @@ -1,74 +1,4 @@ -exports['e2e subdomain passes in electron 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (subdomain_spec.coffee) │ - │ Searched: cypress/integration/subdomain_spec.coffee │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: subdomain_spec.coffee... (1 of 1) - - - subdomains - ✓ can swap to help.foobar.com:2292 - ✓ can directly visit a subdomain in another test - ✓ issue: #207: does not duplicate or hostOnly cookies as a domain cookie - ✓ correctly sets domain based cookies - - issue #362: do not set domain based (non hostOnly) cookies by default - - sets a hostOnly cookie by default - ✓ issue #361: incorrect cookie synchronization between cy.request redirects - ✓ issue #362: incorrect cookie synchronization between cy.visit redirects - ✓ issue #600 can visit between nested subdomains - - - 7 passing - 2 pending - - - (Results) - - ┌─────────────────────────────────────┐ - │ Tests: 9 │ - │ Passing: 7 │ - │ Failing: 0 │ - │ Pending: 2 │ - │ Skipped: 0 │ - │ Screenshots: 0 │ - │ Video: true │ - │ Duration: X seconds │ - │ Spec Ran: subdomain_spec.coffee │ - └─────────────────────────────────────┘ - - - (Video) - - - Started processing: Compressing to 32 CRF - - Finished processing: /foo/bar/.projects/e2e/cypress/videos/abc123.mp4 (X seconds) - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ subdomain_spec.coffee XX:XX 9 7 - 2 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - All specs passed! XX:XX 9 7 - 2 - - - -` - -exports['e2e subdomain passes in chrome 1'] = ` +exports['e2e subdomain / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/5_task_not_registered_spec.coffee.js b/packages/server/__snapshots__/5_task_not_registered_spec.coffee.js index aec838fb7c37..e1c2bfb1a08e 100644 --- a/packages/server/__snapshots__/5_task_not_registered_spec.coffee.js +++ b/packages/server/__snapshots__/5_task_not_registered_spec.coffee.js @@ -64,7 +64,7 @@ https://on.cypress.io/api/task (Screenshots) - - /foo/bar/.projects/task-not-registered/cypress/screenshots/task_not_registered_spec.coffee/fails because the 'task' event is not registered in plugins file (failed).png (1280x720) + - /foo/bar/.projects/task-not-registered/cypress/screenshots/task_not_registered_spec.coffee/fails because the 'task' event is not registered in plugins file (failed).png (YYYYxZZZZ) (Video) diff --git a/packages/server/__snapshots__/6_uncaught_spec_errors_spec.coffee.js b/packages/server/__snapshots__/6_uncaught_spec_errors_spec.coffee.js index df91163c4c41..12c8e1cbce00 100644 --- a/packages/server/__snapshots__/6_uncaught_spec_errors_spec.coffee.js +++ b/packages/server/__snapshots__/6_uncaught_spec_errors_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e uncaught errors failing1 1'] = ` +exports['e2e uncaught errors / failing1'] = ` ==================================================================================================== @@ -80,7 +80,7 @@ We dynamically generated a new test to display this failure. ` -exports['e2e uncaught errors failing2 1'] = ` +exports['e2e uncaught errors / failing2'] = ` ==================================================================================================== @@ -163,7 +163,7 @@ We dynamically generated a new test to display this failure. ` -exports['e2e uncaught errors failing3 1'] = ` +exports['e2e uncaught errors / failing3'] = ` ==================================================================================================== @@ -240,7 +240,7 @@ When Cypress detects uncaught errors originating from your test code it will aut ` -exports['e2e uncaught errors failing4 1'] = ` +exports['e2e uncaught errors / failing4'] = ` ==================================================================================================== @@ -322,7 +322,7 @@ Because this error occurred during a 'before all' hook we are skipping the remai ` -exports['e2e uncaught errors failing5 1'] = ` +exports['e2e uncaught errors / failing5'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/6_uncaught_support_file_spec.coffee.js b/packages/server/__snapshots__/6_uncaught_support_file_spec.coffee.js index 64daeb1b103f..1adcc18b96a5 100644 --- a/packages/server/__snapshots__/6_uncaught_support_file_spec.coffee.js +++ b/packages/server/__snapshots__/6_uncaught_support_file_spec.coffee.js @@ -56,7 +56,7 @@ We dynamically generated a new test to display this failure. (Screenshots) - - /foo/bar/.projects/uncaught-support-file/cypress/screenshots/spec.coffee/An uncaught error was detected outside of a test (failed).png (1280x720) + - /foo/bar/.projects/uncaught-support-file/cypress/screenshots/spec.coffee/An uncaught error was detected outside of a test (failed).png (YYYYxZZZZ) (Video) diff --git a/packages/server/__snapshots__/6_viewport_spec.coffee.js b/packages/server/__snapshots__/6_viewport_spec.coffee.js index 817ade5c2cb5..80bbc57afb69 100644 --- a/packages/server/__snapshots__/6_viewport_spec.coffee.js +++ b/packages/server/__snapshots__/6_viewport_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e viewport passes 1'] = ` +exports['e2e viewport / passes'] = ` ==================================================================================================== diff --git a/packages/server/__snapshots__/6_visit_spec.coffee.js b/packages/server/__snapshots__/6_visit_spec.coffee.js index 5a3c52e5927d..46fb46db62c9 100644 --- a/packages/server/__snapshots__/6_visit_spec.coffee.js +++ b/packages/server/__snapshots__/6_visit_spec.coffee.js @@ -1,4 +1,4 @@ -exports['e2e visit low response timeout passes 1'] = ` +exports['e2e visit / low response timeout / passes'] = ` ==================================================================================================== @@ -73,7 +73,7 @@ exports['e2e visit low response timeout passes 1'] = ` ` -exports['e2e visit low response timeout fails when network connection immediately fails 1'] = ` +exports['e2e visit / low response timeout / fails when network connection immediately fails'] = ` ==================================================================================================== @@ -179,7 +179,7 @@ Error: connect ECONNREFUSED 127.0.0.1:16795 ` -exports['e2e visit low response timeout fails when server responds with 500 1'] = ` +exports['e2e visit / low response timeout / fails when server responds with 500'] = ` ==================================================================================================== @@ -275,7 +275,7 @@ If you do not want status codes to cause failures pass the option: 'failOnStatus ` -exports['e2e visit low response timeout fails when file server responds with 404 1'] = ` +exports['e2e visit / low response timeout / fails when file server responds with 404'] = ` ==================================================================================================== @@ -371,7 +371,7 @@ The internal Cypress web server responded with: ` -exports['e2e visit low response timeout fails when content type isnt html 1'] = ` +exports['e2e visit / low response timeout / fails when content type isnt html'] = ` ==================================================================================================== @@ -469,7 +469,7 @@ cy.request() will automatically get and set cookies and enable you to parse resp ` -exports['e2e visit normal response timeouts fails when visit times out 1'] = ` +exports['e2e visit / normal response timeouts / fails when visit times out'] = ` ==================================================================================================== @@ -592,7 +592,7 @@ When this 'load' event occurs, Cypress will continue running commands. ` -exports['e2e visit low responseTimeout, normal pageLoadTimeout fails when response never ends 1'] = ` +exports['e2e visit / low responseTimeout, normal pageLoadTimeout / fails when response never ends'] = ` ==================================================================================================== @@ -798,4 +798,4 @@ Error: ESOCKETTIMEDOUT 1 of 1 failed (100%) XX:XX 3 - 3 - - -` \ No newline at end of file +` diff --git a/packages/server/__snapshots__/6_web_security_spec.coffee.js b/packages/server/__snapshots__/6_web_security_spec.coffee.js index 5ae3d6225304..a5ae54dc2baf 100644 --- a/packages/server/__snapshots__/6_web_security_spec.coffee.js +++ b/packages/server/__snapshots__/6_web_security_spec.coffee.js @@ -167,7 +167,7 @@ https://on.cypress.io/cross-origin-violation ` -exports['e2e web security when disabled passes 1'] = ` +exports['e2e web security / when disabled / passes'] = ` ==================================================================================================== diff --git a/packages/server/package.json b/packages/server/package.json index e65a68a97f15..eff8183020b3 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -177,7 +177,7 @@ "proxyquire": "2.1.0", "react": "15.6.2", "repl.history": "0.1.4", - "snap-shot-it": "7.8.0", + "snap-shot-it": "7.9.0", "ssestream": "1.0.1", "stream-to-promise": "1.1.1", "supertest": "4.0.2", diff --git a/packages/server/test/e2e/1_base_url_spec.coffee b/packages/server/test/e2e/1_base_url_spec.coffee index d99c3e172449..9fb0fb68a1f3 100644 --- a/packages/server/test/e2e/1_base_url_spec.coffee +++ b/packages/server/test/e2e/1_base_url_spec.coffee @@ -12,12 +12,11 @@ describe "e2e baseUrl", -> } }) - it "passes", -> - e2e.exec(@, { - spec: "base_url_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) + e2e.it "passes", { + spec: "base_url_spec.coffee" + snapshot: true + expectedExitCode: 0 + } context "http", -> e2e.setup({ @@ -30,9 +29,8 @@ describe "e2e baseUrl", -> } }) - it "passes", -> - e2e.exec(@, { - spec: "base_url_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) + e2e.it "passes", { + spec: "base_url_spec.coffee" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/1_busted_support_file_spec.coffee b/packages/server/test/e2e/1_busted_support_file_spec.coffee index c86006bbf3c6..68b76dc690ce 100644 --- a/packages/server/test/e2e/1_busted_support_file_spec.coffee +++ b/packages/server/test/e2e/1_busted_support_file_spec.coffee @@ -9,6 +9,7 @@ describe "e2e busted support file", -> it "passes", -> e2e.exec(@, { project: bustedSupportFile + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 1 }) diff --git a/packages/server/test/e2e/1_commands_outside_of_test_spec.coffee b/packages/server/test/e2e/1_commands_outside_of_test_spec.coffee index e7eb4c16f8a9..7c9ef2b054c6 100644 --- a/packages/server/test/e2e/1_commands_outside_of_test_spec.coffee +++ b/packages/server/test/e2e/1_commands_outside_of_test_spec.coffee @@ -1,33 +1,22 @@ e2e = require("../support/helpers/e2e") -_ = require('lodash') describe "e2e commands outside of test", -> e2e.setup() - _.each [ - 'chrome', - 'electron' - ], (browser) -> + e2e.it "fails on cy commands", { + spec: "commands_outside_of_test_spec.coffee" + snapshot: true + expectedExitCode: 1 + } - it "[#{browser}] fails on cy commands", -> - e2e.exec(@, { - spec: "commands_outside_of_test_spec.coffee" - snapshot: true - expectedExitCode: 1 - browser - }) + e2e.it "fails on failing assertions", { + spec: "assertions_failing_outside_of_test_spec.coffee" + snapshot: true + expectedExitCode: 1 + } - it "[#{browser}] fails on failing assertions", -> - e2e.exec(@, { - spec: "assertions_failing_outside_of_test_spec.coffee" - snapshot: true - expectedExitCode: 1 - browser - }) - - it "passes on passing assertions", -> - e2e.exec(@, { - spec: "assertions_passing_outside_of_test_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) + e2e.it "passes on passing assertions", { + spec: "assertions_passing_outside_of_test_spec.coffee" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/1_system_node_spec.coffee b/packages/server/test/e2e/1_system_node_spec.coffee index 67e61669aa65..7c8f42f64447 100644 --- a/packages/server/test/e2e/1_system_node_spec.coffee +++ b/packages/server/test/e2e/1_system_node_spec.coffee @@ -23,6 +23,7 @@ describe "e2e system node", -> } } spec: "spec.js" + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 0 }) diff --git a/packages/server/test/e2e/2_cdp_spec.ts b/packages/server/test/e2e/2_cdp_spec.ts index 2c5c5f0a54fd..546b2214c14c 100644 --- a/packages/server/test/e2e/2_cdp_spec.ts +++ b/packages/server/test/e2e/2_cdp_spec.ts @@ -17,13 +17,11 @@ describe('e2e cdp', function () { restoreEnv() }) - it('fails when remote debugging port cannot be connected to', function () { - return e2e.exec(this, { - project: Fixtures.projectPath('remote-debugging-port-removed'), - spec: 'spec.ts', - browser: 'chrome', - expectedExitCode: 1, - snapshot: true, - }) + e2e.it('fails when remote debugging port cannot be connected to', { + project: Fixtures.projectPath('remote-debugging-port-removed'), + spec: 'spec.ts', + browser: 'chrome', + expectedExitCode: 1, + snapshot: true, }) }) diff --git a/packages/server/test/e2e/2_controllers_spec.coffee b/packages/server/test/e2e/2_controllers_spec.coffee index a1c89b7e5c5c..01769a639f8c 100644 --- a/packages/server/test/e2e/2_controllers_spec.coffee +++ b/packages/server/test/e2e/2_controllers_spec.coffee @@ -10,6 +10,7 @@ describe "e2e plugins", -> e2e.exec(@, { spec: "spec.js" project: nonExistentSpec + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 1 }) diff --git a/packages/server/test/e2e/2_cookies_spec.coffee b/packages/server/test/e2e/2_cookies_spec.coffee index 09f448c34923..dcdb2511e178 100644 --- a/packages/server/test/e2e/2_cookies_spec.coffee +++ b/packages/server/test/e2e/2_cookies_spec.coffee @@ -80,14 +80,8 @@ describe "e2e cookies", -> } }) - [ - "electron", - "chrome" - ].forEach (browser) -> - it "passes in #{browser}", -> - e2e.exec(@, { - spec: "cookies_spec.coffee" - snapshot: true - expectedExitCode: 0 - browser - }) + e2e.it "passes", { + spec: "cookies_spec.coffee" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/2_domain_spec.coffee b/packages/server/test/e2e/2_domain_spec.coffee index 677e91f58010..d40c565764e0 100644 --- a/packages/server/test/e2e/2_domain_spec.coffee +++ b/packages/server/test/e2e/2_domain_spec.coffee @@ -13,15 +13,11 @@ describe "e2e domain", -> } }) - it "passing", -> - ## run both domain specs back to back to ensure - ## that the internal server + project state is - ## reset each time we spawn the browser - e2e.exec(@, { - spec: "domain*" - snapshot: true - expectedExitCode: 0 - config: { - hosts - } - }) + e2e.it "passes", { + spec: "domain*" + snapshot: true + expectedExitCode: 0 + config: { + hosts + } + } diff --git a/packages/server/test/e2e/2_form_submissions_spec.coffee b/packages/server/test/e2e/2_form_submissions_spec.coffee index 6510000c9e4f..19a8e687f1c2 100644 --- a/packages/server/test/e2e/2_form_submissions_spec.coffee +++ b/packages/server/test/e2e/2_form_submissions_spec.coffee @@ -126,22 +126,20 @@ describe "e2e forms", -> .then (resp) -> fs.outputFileAsync(pathToLargeImage, resp) - it "passes with https on localhost", -> - e2e.exec(@, { - config: { - baseUrl: "https://localhost:#{HTTPS_PORT}" - } - spec: "form_submission_multipart_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) - - it "passes with http on localhost", -> - e2e.exec(@, { - config: { - baseUrl: "http://localhost:#{HTTP_PORT}" - } - spec: "form_submission_multipart_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) + e2e.it "passes with https on localhost", { + config: { + baseUrl: "https://localhost:#{HTTPS_PORT}" + } + spec: "form_submission_multipart_spec.coffee" + snapshot: true + expectedExitCode: 0 + } + + e2e.it "passes with http on localhost", { + config: { + baseUrl: "http://localhost:#{HTTP_PORT}" + } + spec: "form_submission_multipart_spec.coffee" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/2_go_spec.coffee b/packages/server/test/e2e/2_go_spec.coffee index bb1fb1ac096b..ce41d44d83e3 100644 --- a/packages/server/test/e2e/2_go_spec.coffee +++ b/packages/server/test/e2e/2_go_spec.coffee @@ -15,13 +15,11 @@ describe "e2e go", -> } }) - it "passes", -> - ## this tests that history changes work as intended - ## there have been regressions in electron which would - ## otherwise cause these tests to fail - - e2e.exec(@, { - spec: "go_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) + ## this tests that history changes work as intended + ## there have been regressions in electron which would + ## otherwise cause these tests to fail + e2e.it "passes", { + spec: "go_spec.coffee" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/2_iframe_spec.coffee b/packages/server/test/e2e/2_iframe_spec.coffee index c45009d78739..d697b5c35b94 100644 --- a/packages/server/test/e2e/2_iframe_spec.coffee +++ b/packages/server/test/e2e/2_iframe_spec.coffee @@ -66,15 +66,14 @@ describe "e2e iframes", -> } }) - it "passes", -> - e2e.exec(@, { - spec: "iframe_spec.coffee" - snapshot: true - expectedExitCode: 0 - config: { - hosts: { - "*.foo.com": "127.0.0.1" - "*.bar.com": "127.0.0.1" - } + e2e.it "passes", { + spec: "iframe_spec.coffee" + snapshot: true + expectedExitCode: 0 + config: { + hosts: { + "*.foo.com": "127.0.0.1" + "*.bar.com": "127.0.0.1" } - }) + } + } diff --git a/packages/server/test/e2e/2_images_spec.coffee b/packages/server/test/e2e/2_images_spec.coffee index e27dbc712ffe..a2e0c30d4b64 100644 --- a/packages/server/test/e2e/2_images_spec.coffee +++ b/packages/server/test/e2e/2_images_spec.coffee @@ -8,12 +8,11 @@ describe "e2e images", -> } }) - it "passes", -> - ## this tests that images are correctly proxied and that we are not - ## accidentally modifying their bytes in the stream + ## this tests that images are correctly proxied and that we are not + ## accidentally modifying their bytes in the stream - e2e.exec(@, { - spec: "images_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) + e2e.it "passes", { + spec: "images_spec.coffee" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/3_issue_2891_spec.coffee b/packages/server/test/e2e/3_issue_2891_spec.coffee index 8697c3fea644..126e57f3dd84 100644 --- a/packages/server/test/e2e/3_issue_2891_spec.coffee +++ b/packages/server/test/e2e/3_issue_2891_spec.coffee @@ -10,6 +10,7 @@ describe "e2e issue 2891", -> e2e.exec(@, { project: Fixtures.projectPath("default-layout") spec: "default_layout_spec.js" + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 0 }) diff --git a/packages/server/test/e2e/3_js_error_handling_spec.coffee b/packages/server/test/e2e/3_js_error_handling_spec.coffee index 9b2e2a13c75e..c08814e5d258 100644 --- a/packages/server/test/e2e/3_js_error_handling_spec.coffee +++ b/packages/server/test/e2e/3_js_error_handling_spec.coffee @@ -41,9 +41,8 @@ describe "e2e js error handling", -> }] }) - it "fails", -> - e2e.exec(@, { - spec: "js_error_handling_failing_spec.coffee" - snapshot: true - expectedExitCode: 5 - }) + e2e.it "fails", { + spec: "js_error_handling_failing_spec.coffee" + snapshot: true + expectedExitCode: 5 + } diff --git a/packages/server/test/e2e/3_new_project_spec.coffee b/packages/server/test/e2e/3_new_project_spec.coffee index 433fdb51afaa..100993b7d783 100644 --- a/packages/server/test/e2e/3_new_project_spec.coffee +++ b/packages/server/test/e2e/3_new_project_spec.coffee @@ -20,6 +20,7 @@ describe "e2e new project", -> .catch => e2e.exec(@, { project: noScaffoldingPath + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 0 }) diff --git a/packages/server/test/e2e/3_page_loading_spec.coffee b/packages/server/test/e2e/3_page_loading_spec.coffee index d50a595ce58a..36fb4a652a5b 100644 --- a/packages/server/test/e2e/3_page_loading_spec.coffee +++ b/packages/server/test/e2e/3_page_loading_spec.coffee @@ -61,15 +61,13 @@ describe "e2e page_loading", -> }] }) - it "passes", -> - ## this tests that __cypress.initial is set correctly whilst navigating - ## between pages, or during cy.reload - ## additionally this creates an edge case where after __cypress.initial is - ## set we send an XHR which should not inject because its requested for JSON - ## but that another XHR which is requested for html should inject - - e2e.exec(@, { - spec: "page_loading_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) + ## this tests that __cypress.initial is set correctly whilst navigating + ## between pages, or during cy.reload + ## additionally this creates an edge case where after __cypress.initial is + ## set we send an XHR which should not inject because its requested for JSON + ## but that another XHR which is requested for html should inject + e2e.it "passes", { + spec: "page_loading_spec.coffee" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/3_plugins_spec.coffee b/packages/server/test/e2e/3_plugins_spec.coffee index 3c820a6df501..cab9d1cfe08b 100644 --- a/packages/server/test/e2e/3_plugins_spec.coffee +++ b/packages/server/test/e2e/3_plugins_spec.coffee @@ -17,6 +17,7 @@ describe "e2e plugins", -> e2e.exec(@, { spec: "app_spec.coffee" project: workingPreprocessor + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 0 }) @@ -25,6 +26,7 @@ describe "e2e plugins", -> e2e.exec(@, { spec: "app_spec.coffee" project: pluginsAsyncError + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 1 }) @@ -35,18 +37,19 @@ describe "e2e plugins", -> env: "foo=foo,bar=bar" config: { pageLoadTimeout: 10000 } project: pluginConfig + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 0 }) - it "works with user extensions", -> - e2e.exec(@, { - browser: "chrome" - spec: "app_spec.coffee" - project: pluginExtension - snapshot: true - expectedExitCode: 0 - }) + e2e.it "works with user extensions", { + browser: "chrome" + spec: "app_spec.coffee" + project: pluginExtension + sanitizeScreenshotDimensions: true + snapshot: true + expectedExitCode: 0 + } it "handles absolute path to pluginsFile", -> e2e.exec(@, { @@ -58,6 +61,7 @@ describe "e2e plugins", -> ) } project: pluginsAbsolutePath + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 0 }) @@ -66,6 +70,7 @@ describe "e2e plugins", -> e2e.exec(@, { spec: "after_screenshot_spec.coffee" project: pluginAfterScreenshot + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 1 }) diff --git a/packages/server/test/e2e/3_user_agent_spec.coffee b/packages/server/test/e2e/3_user_agent_spec.coffee index f92804e4bcb3..d173c25fec9a 100644 --- a/packages/server/test/e2e/3_user_agent_spec.coffee +++ b/packages/server/test/e2e/3_user_agent_spec.coffee @@ -23,18 +23,8 @@ describe "e2e user agent", -> } }) - it "passes on chrome", -> - e2e.exec(@, { - browser: "chrome" - spec: "user_agent_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) - - it "passes on electron", -> - e2e.exec(@, { - browser: "electron" - spec: "user_agent_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) + e2e.it "passes", { + spec: "user_agent_spec.coffee" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/4_request_spec.coffee b/packages/server/test/e2e/4_request_spec.coffee index aef588dac536..44abadc5f953 100644 --- a/packages/server/test/e2e/4_request_spec.coffee +++ b/packages/server/test/e2e/4_request_spec.coffee @@ -134,17 +134,11 @@ describe "e2e requests", -> "localhost:2293": 0 } - [ - "electron", - "chrome" - ].forEach (browser) -> - it "passes in #{browser}", -> - e2e.exec(@, { - spec: "request_spec.coffee" - snapshot: true - expectedExitCode: 0 - browser - }) + e2e.it "passes", { + spec: "request_spec.coffee" + snapshot: true + expectedExitCode: 0 + } it "fails when network immediately fails", -> e2e.exec(@, { diff --git a/packages/server/test/e2e/4_screenshot_element_capture_spec.coffee b/packages/server/test/e2e/4_screenshot_element_capture_spec.coffee index 0619f0d034a1..2095d2b1ee3b 100644 --- a/packages/server/test/e2e/4_screenshot_element_capture_spec.coffee +++ b/packages/server/test/e2e/4_screenshot_element_capture_spec.coffee @@ -16,12 +16,10 @@ describe "e2e screenshot element capture", -> } }) - it "passes", -> - ## this tests that consistent screenshots are taken for element captures, - ## that the runner UI is hidden and that the page is scrolled properly - - e2e.exec(@, { - spec: "screenshot_element_capture_spec.coffee" - expectedExitCode: 0 - snapshot: true - }) + ## this tests that consistent screenshots are taken for element captures, + ## that the runner UI is hidden and that the page is scrolled properly + e2e.it "passes", { + spec: "screenshot_element_capture_spec.coffee" + expectedExitCode: 0 + snapshot: true + } diff --git a/packages/server/test/e2e/4_screenshot_fullpage_capture_spec.coffee b/packages/server/test/e2e/4_screenshot_fullpage_capture_spec.coffee index 65bfbef74cc3..9e5fa22983aa 100644 --- a/packages/server/test/e2e/4_screenshot_fullpage_capture_spec.coffee +++ b/packages/server/test/e2e/4_screenshot_fullpage_capture_spec.coffee @@ -18,12 +18,10 @@ describe "e2e screenshot fullPage capture", -> } }) - it "passes", -> - ## this tests that consistent screenshots are taken for fullPage captures, - ## that the runner UI is hidden and that the page is scrolled properly - - e2e.exec(@, { - spec: "screenshot_fullpage_capture_spec.coffee" - expectedExitCode: 0 - snapshot: true - }) + ## this tests that consistent screenshots are taken for fullPage captures, + ## that the runner UI is hidden and that the page is scrolled properly + e2e.it "passes", { + spec: "screenshot_fullpage_capture_spec.coffee" + expectedExitCode: 0 + snapshot: true + } diff --git a/packages/server/test/e2e/4_screenshot_nested_file_spec.coffee b/packages/server/test/e2e/4_screenshot_nested_file_spec.coffee index 0f1b010dfe06..79990aec46be 100644 --- a/packages/server/test/e2e/4_screenshot_nested_file_spec.coffee +++ b/packages/server/test/e2e/4_screenshot_nested_file_spec.coffee @@ -3,9 +3,8 @@ e2e = require("../support/helpers/e2e") describe "e2e screenshot in nested spec", -> e2e.setup() - it "passes", -> - e2e.exec(@, { - spec: "nested-1/nested-2/screenshot_nested_file_spec.coffee" - expectedExitCode: 0 - snapshot: true - }) + e2e.it "passes", { + spec: "nested-1/nested-2/screenshot_nested_file_spec.coffee" + expectedExitCode: 0 + snapshot: true + } diff --git a/packages/server/test/e2e/4_websockets_spec.coffee b/packages/server/test/e2e/4_websockets_spec.coffee index 0e2f5fd4c67a..386b10ccc85a 100644 --- a/packages/server/test/e2e/4_websockets_spec.coffee +++ b/packages/server/test/e2e/4_websockets_spec.coffee @@ -30,9 +30,8 @@ describe "e2e websockets", -> }) ## https://github.com/cypress-io/cypress/issues/556 - it "passes", -> - e2e.exec(@, { - spec: "websockets_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) + e2e.it "passes", { + spec: "websockets_spec.coffee" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/4_xhr_spec.coffee b/packages/server/test/e2e/4_xhr_spec.coffee index d013a23dd7e1..5d2a67b43308 100644 --- a/packages/server/test/e2e/4_xhr_spec.coffee +++ b/packages/server/test/e2e/4_xhr_spec.coffee @@ -26,9 +26,8 @@ describe "e2e xhr", -> } }) - it "passes", -> - e2e.exec(@, { - spec: "xhr_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) + e2e.it "passes", { + spec: "xhr_spec.coffee" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/5_screenshot_viewport_capture_spec.coffee b/packages/server/test/e2e/5_screenshot_viewport_capture_spec.coffee index 74e06497ecf9..9969452a4297 100644 --- a/packages/server/test/e2e/5_screenshot_viewport_capture_spec.coffee +++ b/packages/server/test/e2e/5_screenshot_viewport_capture_spec.coffee @@ -14,12 +14,10 @@ describe "e2e screenshot viewport capture", -> } }) - it "passes", -> - ## this tests that consistent screenshots are taken for app - ## captures (namely that the runner UI is hidden) - - e2e.exec(@, { - spec: "screenshot_viewport_capture_spec.coffee" - expectedExitCode: 0 - snapshot: true - }) + ## this tests that consistent screenshots are taken for app + ## captures (namely that the runner UI is hidden) + e2e.it "passes", { + spec: "screenshot_viewport_capture_spec.coffee" + expectedExitCode: 0 + snapshot: true + } diff --git a/packages/server/test/e2e/5_screenshots_spec.coffee b/packages/server/test/e2e/5_screenshots_spec.coffee index 08c7798dc9a2..21dc17946677 100644 --- a/packages/server/test/e2e/5_screenshots_spec.coffee +++ b/packages/server/test/e2e/5_screenshots_spec.coffee @@ -56,67 +56,68 @@ describe "e2e screenshots", -> } }) - it "passes", -> - ## this tests that screenshots can be manually generated - ## and are also generated automatically on failure with - ## the test title as the file name - - e2e.exec(@, { - spec: "screenshots_spec.js" - expectedExitCode: 4 - snapshot: true - timeout: 180000 - }) - .then -> - screenshot = (paths...) -> - path.join(e2ePath, "cypress", "screenshots", "screenshots_spec.js", paths...) - - screenshot1 = screenshot("black.png") - screenshot2 = screenshot("red.png") - screenshot3 = screenshot("foo", "bar", "baz.png") - screenshot4 = screenshot("taking screenshots -- generates pngs on failure (failed).png") - screenshot5 = screenshot("taking screenshots -- before hooks -- empty test 1 -- before all hook (failed).png") - screenshot6 = screenshot("taking screenshots -- each hooks -- empty test 2 -- before each hook (failed).png") - screenshot7 = screenshot("taking screenshots -- each hooks -- empty test 2 -- after each hook (failed).png") - screenshot8 = screenshot("taking screenshots -- ensures unique paths when there's a non-named screenshot and a failure.png") - screenshot9 = screenshot("taking screenshots -- ensures unique paths when there's a non-named screenshot and a failure (failed).png") - - Promise.all([ - fs.statAsync(screenshot1).get("size") - fs.statAsync(screenshot2).get("size") - fs.statAsync(screenshot3).get("size") - fs.statAsync(screenshot4).get("size") - fs.statAsync(screenshot5).get("size") - fs.statAsync(screenshot6).get("size") - fs.statAsync(screenshot7).get("size") - fs.statAsync(screenshot8).get("size") - fs.statAsync(screenshot9).get("size") - ]) - .then (sizes) -> - ## make sure all of the values are unique - expect(sizes).to.deep.eq(_.uniq(sizes)) + ## this tests that screenshots can be manually generated + ## and are also generated automatically on failure with + ## the test title as the file name + e2e.it "passes", { + spec: "screenshots_spec.js" + expectedExitCode: 4 + snapshot: true + timeout: 180000 + onRun: (exec, browser) -> + exec() + .then -> + screenshot = (paths...) -> + path.join(e2ePath, "cypress", "screenshots", "screenshots_spec.js", paths...) - ## png1 should not be within 5k of png2 - expect(sizes[0]).not.to.be.closeTo(sizes[1], 5000) + screenshot1 = screenshot("black.png") + screenshot2 = screenshot("red.png") + screenshot3 = screenshot("foo", "bar", "baz.png") + screenshot4 = screenshot("taking screenshots -- generates pngs on failure (failed).png") + screenshot5 = screenshot("taking screenshots -- before hooks -- empty test 1 -- before all hook (failed).png") + screenshot6 = screenshot("taking screenshots -- each hooks -- empty test 2 -- before each hook (failed).png") + screenshot7 = screenshot("taking screenshots -- each hooks -- empty test 2 -- after each hook (failed).png") + screenshot8 = screenshot("taking screenshots -- ensures unique paths when there's a non-named screenshot and a failure.png") + screenshot9 = screenshot("taking screenshots -- ensures unique paths when there's a non-named screenshot and a failure (failed).png") - ## TODO: this assertion is flaky, sometimes the sizes - ## are as close as 200 bytes, commenting out for now - ## until it can be further investigated - ## - ## png3 should not be within 1.5k of png4 - # expect(sizes[2]).not.to.be.closeTo(sizes[3], 1500) - .then -> Promise.all([ - sizeOf(screenshot1) - sizeOf(screenshot2) - sizeOf(screenshot3) - sizeOf(screenshot4) - sizeOf(screenshot5) - sizeOf(screenshot6) - sizeOf(screenshot7) + fs.statAsync(screenshot1).get("size") + fs.statAsync(screenshot2).get("size") + fs.statAsync(screenshot3).get("size") + fs.statAsync(screenshot4).get("size") + fs.statAsync(screenshot5).get("size") + fs.statAsync(screenshot6).get("size") + fs.statAsync(screenshot7).get("size") + fs.statAsync(screenshot8).get("size") + fs.statAsync(screenshot9).get("size") ]) - .then (dimensions = []) -> - ## all of the images should be 1280x720 - ## since thats what we run headlessly - _.each dimensions, (dimension) -> - expect(dimension).to.deep.eq({width: 1280, height: 720, type: "png"}) + .then (sizes) -> + ## make sure all of the values are unique + expect(sizes).to.deep.eq(_.uniq(sizes)) + + ## png1 should not be within 5k of png2 + expect(sizes[0]).not.to.be.closeTo(sizes[1], 5000) + + ## TODO: this assertion is flaky, sometimes the sizes + ## are as close as 200 bytes, commenting out for now + ## until it can be further investigated + ## + ## png3 should not be within 1.5k of png4 + # expect(sizes[2]).not.to.be.closeTo(sizes[3], 1500) + .then -> + Promise.all([ + sizeOf(screenshot1) + sizeOf(screenshot2) + sizeOf(screenshot3) + sizeOf(screenshot4) + sizeOf(screenshot5) + sizeOf(screenshot6) + sizeOf(screenshot7) + ]) + .then (dimensions = []) -> + if browser is "electron" + ## all of the images should be 1280x720 + ## since thats what we run headlessly + _.each dimensions, (dimension) -> + expect(dimension).to.deep.eq({width: 1280, height: 720, type: "png"}) + } diff --git a/packages/server/test/e2e/5_server_sent_events_spec.coffee b/packages/server/test/e2e/5_server_sent_events_spec.coffee index da1fee9d4834..30d797bf5f9d 100644 --- a/packages/server/test/e2e/5_server_sent_events_spec.coffee +++ b/packages/server/test/e2e/5_server_sent_events_spec.coffee @@ -56,9 +56,8 @@ describe "e2e server sent events", -> }) ## https://github.com/cypress-io/cypress/issues/1440 - it "passes", -> - e2e.exec(@, { - spec: "server_sent_events_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) + e2e.it "passes", { + spec: "server_sent_events_spec.coffee" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/5_spec_isolation_spec.coffee b/packages/server/test/e2e/5_spec_isolation_spec.coffee index e429c39ce35e..dc1f5cd17afe 100644 --- a/packages/server/test/e2e/5_spec_isolation_spec.coffee +++ b/packages/server/test/e2e/5_spec_isolation_spec.coffee @@ -158,71 +158,73 @@ expectRunsToHaveCorrectStats = (runs = []) -> screenshot.screenshotId = "some-random-id" screenshot.path = e2e.normalizeStdout(screenshot.path) + screenshot describe "e2e spec_isolation", -> e2e.setup() - it "failing", -> - e2e.exec(@, { - spec: specs - outputPath: outputPath - snapshot: false - expectedExitCode: 5 - }) - .then -> - ## now what we want to do is read in the outputPath - ## and snapshot it so its what we expect after normalizing it - fs.readJsonAsync(outputPath) - .then (json) -> - ## ensure that config has been set - expect(json.config).to.be.an('object') - expect(json.config.projectName).to.eq("e2e") - expect(json.config.projectRoot).to.eq(e2ePath) - - ## but zero out config because it's too volatile - json.config = {} - - expect(json.browserPath).to.be.a('string') - expect(json.browserName).to.be.a('string') - expect(json.browserVersion).to.be.a('string') - expect(json.osName).to.be.a('string') - expect(json.osVersion).to.be.a('string') - expect(json.cypressVersion).to.be.a('string') - - _.extend(json, { - browserPath: 'path/to/browser' - browserName: 'FooBrowser' - browserVersion: '88' - osName: 'FooOS' - osVersion: '1234' - cypressVersion: '9.9.9' - }) - - ## ensure the totals are accurate - expect(json.totalTests).to.eq( - _.sum([ - json.totalFailed, - json.totalPassed, - json.totalPending, - json.totalSkipped - ]) - ) - - expectStartToBeBeforeEnd(json, "startedTestsAt", "endedTestsAt") - - ## ensure totalDuration matches all of the stats durations - expectDurationWithin( - json, - "totalDuration", - _.sumBy(json.runs, "stats.wallClockDuration"), - _.sumBy(json.runs, "stats.wallClockDuration"), - 5555 - ) - - ## should be 4 total runs - expect(json.runs).to.have.length(4) - - expectRunsToHaveCorrectStats(json.runs) - - snapshot(json) + e2e.it "fails", { + spec: specs + outputPath: outputPath + snapshot: false + expectedExitCode: 5 + onRun: (exec) -> + exec() + .then -> + ## now what we want to do is read in the outputPath + ## and snapshot it so its what we expect after normalizing it + fs.readJsonAsync(outputPath) + .then (json) -> + ## ensure that config has been set + expect(json.config).to.be.an('object') + expect(json.config.projectName).to.eq("e2e") + expect(json.config.projectRoot).to.eq(e2ePath) + + ## but zero out config because it's too volatile + json.config = {} + + expect(json.browserPath).to.be.a('string') + expect(json.browserName).to.be.a('string') + expect(json.browserVersion).to.be.a('string') + expect(json.osName).to.be.a('string') + expect(json.osVersion).to.be.a('string') + expect(json.cypressVersion).to.be.a('string') + + _.extend(json, { + browserPath: 'path/to/browser' + browserName: 'FooBrowser' + browserVersion: '88' + osName: 'FooOS' + osVersion: '1234' + cypressVersion: '9.9.9' + }) + + ## ensure the totals are accurate + expect(json.totalTests).to.eq( + _.sum([ + json.totalFailed, + json.totalPassed, + json.totalPending, + json.totalSkipped + ]) + ) + + expectStartToBeBeforeEnd(json, "startedTestsAt", "endedTestsAt") + + ## ensure totalDuration matches all of the stats durations + expectDurationWithin( + json, + "totalDuration", + _.sumBy(json.runs, "stats.wallClockDuration"), + _.sumBy(json.runs, "stats.wallClockDuration"), + 5555 + ) + + ## should be 4 total runs + expect(json.runs).to.have.length(4) + + expectRunsToHaveCorrectStats(json.runs) + + snapshot('e2e spec isolation fails', json, { allowSharedSnapshot: true }) + } diff --git a/packages/server/test/e2e/5_stdout_spec.coffee b/packages/server/test/e2e/5_stdout_spec.coffee index ea967a294cf4..3fcd1a2f1415 100644 --- a/packages/server/test/e2e/5_stdout_spec.coffee +++ b/packages/server/test/e2e/5_stdout_spec.coffee @@ -35,10 +35,9 @@ describe "e2e stdout", -> expectedExitCode: 0 }) - it "logs that chrome cannot be recorded", -> - e2e.exec(@, { - spec: "simple_spec.coffee" - browser: "chrome" - snapshot: true - expectedExitCode: 0 - }) + e2e.it "logs that chrome cannot be recorded", { + spec: "simple_spec.coffee" + browser: "chrome" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/5_subdomain_spec.coffee b/packages/server/test/e2e/5_subdomain_spec.coffee index e6504d266e90..7ae1a241532a 100644 --- a/packages/server/test/e2e/5_subdomain_spec.coffee +++ b/packages/server/test/e2e/5_subdomain_spec.coffee @@ -102,19 +102,13 @@ describe "e2e subdomain", -> } }) - [ - "electron", - "chrome" - ].forEach (browser) -> - it "passes in #{browser}", -> - e2e.exec(@, { - spec: "subdomain_spec.coffee" - snapshot: true - expectedExitCode: 0 - config: { - hosts: { - "*.foobar.com": "127.0.0.1" - } - } - browser - }) + e2e.it "passes", { + spec: "subdomain_spec.coffee" + snapshot: true + expectedExitCode: 0 + config: { + hosts: { + "*.foobar.com": "127.0.0.1" + } + } + } diff --git a/packages/server/test/e2e/5_task_not_registered_spec.coffee b/packages/server/test/e2e/5_task_not_registered_spec.coffee index b90d058aadd8..776652e74f2b 100644 --- a/packages/server/test/e2e/5_task_not_registered_spec.coffee +++ b/packages/server/test/e2e/5_task_not_registered_spec.coffee @@ -8,6 +8,7 @@ describe "e2e task", -> e2e.exec(@, { project: Fixtures.projectPath("task-not-registered") spec: "task_not_registered_spec.coffee" + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 1 }) diff --git a/packages/server/test/e2e/6_task_spec.coffee b/packages/server/test/e2e/6_task_spec.coffee index 52061f4e9f43..916750f369da 100644 --- a/packages/server/test/e2e/6_task_spec.coffee +++ b/packages/server/test/e2e/6_task_spec.coffee @@ -21,6 +21,7 @@ describe "e2e task", -> e2e.exec(@, { project: Fixtures.projectPath("multiple-task-registrations") spec: "multiple_task_registrations_spec.coffee" + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 0 }) diff --git a/packages/server/test/e2e/6_uncaught_spec_errors_spec.coffee b/packages/server/test/e2e/6_uncaught_spec_errors_spec.coffee index 0cf98aa24999..c2677c847237 100644 --- a/packages/server/test/e2e/6_uncaught_spec_errors_spec.coffee +++ b/packages/server/test/e2e/6_uncaught_spec_errors_spec.coffee @@ -3,37 +3,32 @@ e2e = require("../support/helpers/e2e") describe "e2e uncaught errors", -> e2e.setup() - it "failing1", -> - e2e.exec(@, { - spec: "uncaught_synchronous_before_tests_parsed.coffee" - snapshot: true - expectedExitCode: 1 - }) + e2e.it "failing1", { + spec: "uncaught_synchronous_before_tests_parsed.coffee" + snapshot: true + expectedExitCode: 1 + } - it "failing2", -> - e2e.exec(@, { - spec: "uncaught_synchronous_during_hook_spec.coffee" - snapshot: true - expectedExitCode: 1 - }) + e2e.it "failing2", { + spec: "uncaught_synchronous_during_hook_spec.coffee" + snapshot: true + expectedExitCode: 1 + } - it "failing3", -> - e2e.exec(@, { - spec: "uncaught_during_test_spec.coffee" - snapshot: true - expectedExitCode: 1 - }) + e2e.it "failing3", { + spec: "uncaught_during_test_spec.coffee" + snapshot: true + expectedExitCode: 1 + } - it "failing4", -> - e2e.exec(@, { - spec: "uncaught_during_hook_spec.coffee" - snapshot: true - expectedExitCode: 1 - }) + e2e.it "failing4", { + spec: "uncaught_during_hook_spec.coffee" + snapshot: true + expectedExitCode: 1 + } - it "failing5", -> - e2e.exec(@, { - spec: "caught_async_sync_test_spec.coffee" - snapshot: true - expectedExitCode: 4 - }) + e2e.it "failing5", { + spec: "caught_async_sync_test_spec.coffee" + snapshot: true + expectedExitCode: 4 + } diff --git a/packages/server/test/e2e/6_uncaught_support_file_spec.coffee b/packages/server/test/e2e/6_uncaught_support_file_spec.coffee index 7a39170cb17b..c773758150c7 100644 --- a/packages/server/test/e2e/6_uncaught_support_file_spec.coffee +++ b/packages/server/test/e2e/6_uncaught_support_file_spec.coffee @@ -9,6 +9,7 @@ describe "e2e uncaught support file errors", -> it "failing", -> e2e.exec(@, { project: uncaughtSupportFile + sanitizeScreenshotDimensions: true snapshot: true expectedExitCode: 1 }) diff --git a/packages/server/test/e2e/6_video_compression_spec.coffee b/packages/server/test/e2e/6_video_compression_spec.coffee index 55ae8d2e3243..7bd5cceb2429 100644 --- a/packages/server/test/e2e/6_video_compression_spec.coffee +++ b/packages/server/test/e2e/6_video_compression_spec.coffee @@ -12,24 +12,20 @@ EXPECTED_DURATION_MS = NUM_TESTS * MS_PER_TEST describe "e2e video compression", -> e2e.setup() - [ - 'chrome', - 'electron' - ].map (browser) -> - it "passes in #{browser}", -> + e2e.it "passes", { + spec: "video_compression_spec.coffee" + snapshot: false + config: { + env: { + NUM_TESTS + MS_PER_TEST + } + } + expectedExitCode: 0 + onRun: (exec) -> process.env.VIDEO_COMPRESSION_THROTTLE = 10 - e2e.exec(@, { - spec: "video_compression_spec.coffee" - snapshot: false - config: { - env: { - NUM_TESTS - MS_PER_TEST - } - } - expectedExitCode: 0 - }) + exec() .tap -> videosPath = Fixtures.projectPath("e2e/cypress/videos/*") @@ -46,3 +42,5 @@ describe "e2e video compression", -> .get("stdout") .then (stdout) -> expect(stdout).to.match(/Compression progress:\s+\d{1,3}%/) + + }, diff --git a/packages/server/test/e2e/6_viewport_spec.coffee b/packages/server/test/e2e/6_viewport_spec.coffee index 363ad1dca289..6f4ec612b0a6 100644 --- a/packages/server/test/e2e/6_viewport_spec.coffee +++ b/packages/server/test/e2e/6_viewport_spec.coffee @@ -8,9 +8,8 @@ describe "e2e viewport", -> } }) - it "passes", -> - e2e.exec(@, { - spec: "viewport_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) + e2e.it "passes", { + spec: "viewport_spec.coffee" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/e2e/6_visit_spec.coffee b/packages/server/test/e2e/6_visit_spec.coffee index 33830c105848..5e94f6e214db 100644 --- a/packages/server/test/e2e/6_visit_spec.coffee +++ b/packages/server/test/e2e/6_visit_spec.coffee @@ -62,55 +62,48 @@ describe "e2e visit", -> } }) - it "passes", -> - ## this tests that hashes are applied during a visit - ## which forces the browser to scroll to the div - ## additionally this tests that jquery.js is not truncated - ## due to __cypress.initial cookies not being cleared by - ## the hash.html response - - ## additionally this tests that xhr request headers + body - ## can reach the backend without being modified or changed - ## by the cypress proxy in any way - - e2e.exec(@, { - spec: "visit_spec.coffee" - snapshot: true - expectedExitCode: 0 - }) - - it "fails when network connection immediately fails", -> - e2e.exec(@, { - spec: "visit_http_network_error_failing_spec.coffee" - snapshot: true - expectedExitCode: 1 - }) - - it "fails when server responds with 500", -> - e2e.exec(@, { - spec: "visit_http_500_response_failing_spec.coffee" - snapshot: true - expectedExitCode: 1 - }) - - it "fails when file server responds with 404", -> - e2e.exec(@, { - spec: "visit_file_404_response_failing_spec.coffee" - snapshot: true - expectedExitCode: 1 - }) - - it "fails when content type isnt html", -> - e2e.exec(@, { - spec: "visit_non_html_content_type_failing_spec.coffee" - snapshot: true - expectedExitCode: 1 - }) - - it "calls onBeforeLoad when overwriting cy.visit", -> - e2e.exec(@, { - spec: "issue_2196_spec.coffee" - }) + ## this tests that hashes are applied during a visit + ## which forces the browser to scroll to the div + ## additionally this tests that jquery.js is not truncated + ## due to __cypress.initial cookies not being cleared by + ## the hash.html response + + ## additionally this tests that xhr request headers + body + ## can reach the backend without being modified or changed + ## by the cypress proxy in any way + e2e.it "passes", { + spec: "visit_spec.coffee" + snapshot: true + expectedExitCode: 0 + } + + e2e.it "fails when network connection immediately fails", { + spec: "visit_http_network_error_failing_spec.coffee" + snapshot: true + expectedExitCode: 1 + } + + e2e.it "fails when server responds with 500", { + spec: "visit_http_500_response_failing_spec.coffee" + snapshot: true + expectedExitCode: 1 + } + + e2e.it "fails when file server responds with 404", { + spec: "visit_file_404_response_failing_spec.coffee" + snapshot: true + expectedExitCode: 1 + } + + e2e.it "fails when content type isnt html", { + spec: "visit_non_html_content_type_failing_spec.coffee" + snapshot: true + expectedExitCode: 1 + } + + e2e.it "calls onBeforeLoad when overwriting cy.visit", { + spec: "issue_2196_spec.coffee" + } context "low responseTimeout, normal pageLoadTimeout", -> e2e.setup({ @@ -124,12 +117,11 @@ describe "e2e visit", -> } }) - it "fails when response never ends", -> - e2e.exec(@, { - spec: "visit_response_never_ends_failing_spec.js", - snapshot: true, - expectedExitCode: 3 - }) + e2e.it "fails when response never ends", { + spec: "visit_response_never_ends_failing_spec.js", + snapshot: true, + expectedExitCode: 3 + } context "normal response timeouts", -> e2e.setup({ @@ -143,9 +135,8 @@ describe "e2e visit", -> } }) - it "fails when visit times out", -> - e2e.exec(@, { - spec: "visit_http_timeout_failing_spec.coffee" - snapshot: true - expectedExitCode: 2 - }) + e2e.it "fails when visit times out", { + spec: "visit_http_timeout_failing_spec.coffee" + snapshot: true + expectedExitCode: 2 + } diff --git a/packages/server/test/e2e/6_web_security_spec.coffee b/packages/server/test/e2e/6_web_security_spec.coffee index b8f79496237b..1c2e22d01593 100644 --- a/packages/server/test/e2e/6_web_security_spec.coffee +++ b/packages/server/test/e2e/6_web_security_spec.coffee @@ -80,10 +80,9 @@ describe "e2e web security", -> } }) - it "passes", -> - e2e.exec(@, { - spec: "web_security_spec.coffee" - browser: "chrome" - snapshot: true - expectedExitCode: 0 - }) + e2e.it "passes", { + spec: "web_security_spec.coffee" + browser: "chrome" + snapshot: true + expectedExitCode: 0 + } diff --git a/packages/server/test/support/fixtures/projects/e2e/cypress/plugins/index.js b/packages/server/test/support/fixtures/projects/e2e/cypress/plugins/index.js index 19b4dfa77fd9..fb262f2bbaab 100644 --- a/packages/server/test/support/fixtures/projects/e2e/cypress/plugins/index.js +++ b/packages/server/test/support/fixtures/projects/e2e/cypress/plugins/index.js @@ -23,6 +23,17 @@ module.exports = (on) => { }) } + on('before:browser:launch', (browser, args) => { + if (browser.family === 'chrome') { + // so that screenshot sizes match those of Electron + args.push('--window-size=1280,720') + // removes stuff from the top of the window to make `window-size` equal the viewport size + args.push('--kiosk') + } + + return args + }) + on('task', { 'returns:undefined' () {}, diff --git a/packages/server/test/support/helpers/e2e.coffee b/packages/server/test/support/helpers/e2e.coffee index 4e2495c157ee..a25407d98644 100644 --- a/packages/server/test/support/helpers/e2e.coffee +++ b/packages/server/test/support/helpers/e2e.coffee @@ -33,6 +33,8 @@ Promise.config({ e2ePath = Fixtures.projectPath("e2e") pathUpToProjectName = Fixtures.projectPath("") +DEFAULT_BROWSERS = ['electron', 'chrome'] + stackTraceLinesRe = /^(\s+)at\s(.+)/gm browserNameVersionRe = /(Browser\:\s+)(Custom |)(Electron|Chrome|Canary|Chromium|Firefox)(\s\d+)(\s\(\w+\))?(\s+)/ availableBrowsersRe = /(Available browsers found are: )(.+)/g @@ -99,7 +101,7 @@ normalizeStdout = (str, options = {}) -> .replace(/(Uploading Results.*?\n\n)((.*-.*[\s\S\r]){2,}?)(\n\n)/g, replaceUploadingResults) ## replaces multiple lines of uploading results (since order not guaranteed) .replace(/^(\- )(\/.*\/packages\/server\/)(.*)$/gm, "$1$3") ## fix "Require stacks" for CI - if options.browser isnt undefined and options.browser isnt 'electron' + if options.sanitizeScreenshotDimensions str = str.replace(/\(\d{2,4}x\d{2,4}\)/g, "(YYYYxZZZZ)") ## screenshot dimensions return str.split("\n") @@ -158,8 +160,96 @@ copy = -> ) ) -module.exports = { - normalizeStdout +getMochaItFn = (only, skip, browser, specifiedBrowser) -> + ## if we've been told to skip this test + ## or if we specified a particular browser and this + ## doesn't match the one we're currently trying to run... + if skip or (specifiedBrowser and specifiedBrowser isnt browser) + ## then skip this test + return it.skip + + if only + return it.only + + return it + +getBrowsers = (generateTestsForDefaultBrowsers, browser, defaultBrowsers) -> + ## if we're generating tests for default browsers + if generateTestsForDefaultBrowsers + ## then return an array of default browsers + return defaultBrowsers + + ## but if we haven't been told to generate tests for default browsers + ## and weren't provided a specified browser then throw + if not browser + throw new Error('A browser must be specified when { generateTestsForDefaultBrowsers: false }.') + + ## otherwise return the specified browser + return [browser] + +localItFn = (title, options = {}) -> + options = _ + .chain(options) + .clone() + .defaults({ + only: false, + skip: false, + browser: process.env.BROWSER + generateTestsForDefaultBrowsers: true + onRun: (execFn, browser, ctx) -> + execFn() + }) + .value() + + { only, skip, browser, generateTestsForDefaultBrowsers, onRun, spec, expectedExitCode } = options + + if not title + throw new Error('e2e.it(...) must be passed a title as the first argument') + + ## LOGIC FOR AUTOGENERATING DYNAMIC TESTS + ## - if generateTestsForDefaultBrowsers + ## - create multiple tests for each default browser + ## - if browser is specified in options: + ## ...skip the tests for each default browser if that browser + ## ...does not match the specified one (used in CI) + ## - else only generate a single test with the specified browser + + ## run the tests for all the default browsers, or if a browser + ## has been specified, only run it for that + specifiedBrowser = browser + browsersToTest = getBrowsers(generateTestsForDefaultBrowsers, browser, DEFAULT_BROWSERS) + + browserToTest = (browser) -> + mochaItFn = getMochaItFn(only, skip, browser, specifiedBrowser) + + testTitle = "#{title} [#{browser}]" + + mochaItFn testTitle, -> + originalTitle = @test.parent.titlePath().concat(title).join(" / ") + + ctx = @ + + execFn = (overrides = {}) -> + e2e.exec(ctx, _.extend({}, options, overrides, { + browser, originalTitle + })) + + onRun(execFn, browser, ctx) + + return _.each(browsersToTest, browserToTest) + +localItFn.only = (title, options) -> + options.only = true + localItFn(title, options) + +localItFn.skip = (title, options) -> + options.skip = true + localItFn(title, options) + +module.exports = e2e = { + normalizeStdout, + + it: localItFn snapshot: (args...) -> args = _.compact(args) @@ -239,9 +329,11 @@ module.exports = { options: (ctx, options = {}) -> _.defaults(options, { - browser: process.env.BROWSER + browser: 'electron' project: e2ePath timeout: if options.exit is false then 3000000 else 120000 + originalTitle: null + sanitizeScreenshotDimensions: false }) ctx.timeout(options.timeout) @@ -363,7 +455,11 @@ module.exports = { expect(headless).to.include("(headless)") str = normalizeStdout(stdout, options) - snapshot(str) + + if options.originalTitle + snapshot(options.originalTitle, str, { allowSharedSnapshot: true }) + else + snapshot(str) return { code: code