From aee8978377faed128cb12b8c01a65627f5896906 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Wed, 19 Oct 2022 12:18:49 -0500 Subject: [PATCH 01/13] fix(testIsolation): improve the behavior, clarify config options and sync with session command --- cli/types/cypress.d.ts | 38 +- packages/config/src/browser.ts | 8 +- packages/config/src/options.ts | 25 +- packages/config/src/project/utils.ts | 8 +- packages/config/test/index.spec.ts | 2 +- packages/config/test/project/utils.spec.ts | 8 +- .../e2e/commands/sessions/sessions.cy.js | 1494 ++++++++++++----- packages/driver/cypress/e2e/util/config.cy.js | 4 +- packages/driver/src/cy/commands/navigation.ts | 9 +- .../driver/src/cy/commands/sessions/index.ts | 7 +- .../driver/src/cy/commands/sessions/utils.ts | 4 + packages/driver/src/util/config.ts | 7 + 12 files changed, 1173 insertions(+), 441 deletions(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index 561259cdeaa7..f17031d5fcd0 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2808,12 +2808,38 @@ declare namespace Cypress { */ supportFile: string | false /** - * The test isolation level applied to ensure a clean slate between tests. - * - legacy - resets/clears aliases, intercepts, clock, viewport, cookies, and local storage before each test. - * - strict - applies all resets/clears from legacy, plus clears the page by visiting 'about:blank' to ensure clean app state before each test. - * @default "legacy", however, when experimentalSessionAndOrigin=true, the default is "strict" - */ - testIsolation: 'legacy' | 'strict' + * The test isolation ensures a clean browser context between tests for end-to-end tests. + * + * Cypress will always resets/clears aliases, intercepts, clock, and viewport before each test + * to ensure a clean test slate; i.e. this configuration only impacts the browser context. + * + * Note: the `cy.session()` command will inherent this value to determine where or not the page is + * cleared when the command executes. + * + * Options: + * When experimentalSessionAndOrigin=false + * - legacy - "Pass through" to document the default Cypress test isolation behavior. The page is not cleared + * in end-to-end tests and cookies, local storage and session storage in the current domains are + * cleared before each test. The `cy.session()` command is not available. + * NOTE: this behavior will be changed in the next major release and the new default will be 'on'. + * + * When experimentalSessionAndOrigin=true + * - on - The page is cleared and cookies, local storage and session storage in all domains are cleared before each + * test. The `cy.session()` command will also clear the page and current browser context when establishing + * the browser session. + * - off - The current browser state will persist between tests. The page does not clear and cookies, local storage and + * session storage will be available in the next test. The `cy.session()` command will only clear the current + * browser context when establishing the browser session - the current page will not clear. + * + * NOTES: + * - Turning test isolation off may improve performance of end-to-end tests, however, previous tests could impact the + * browser state of the next test and cause inconsistency when using .only(). + * - The page is always cleared in component testing to ensure a clean page for rendering isolated components. + * + * @default 'legacy', when running end-to-end tests. When running component tests or running end-to-end tests with + * experimentalSessionAndOrigin=true, the default is 'on'. + */ + testIsolation: 'legacy' | 'on' | 'off' /** * Path to folder where videos will be saved after a headless or CI run * @default "cypress/videos" diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index 9a67529c084c..7d1677d4e381 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -163,7 +163,11 @@ export const validate = (cfg: any, onErr: (property: ErrResult | string) => void // key has a validation rule & value different from the default if (validationFn && value !== defaultValues[key]) { - const result = validationFn(key, value) + const result = validationFn(key, value, { + testingType: cfg.testingType, + // TODO: remove with experimentalSessionAndOrigin. Fixed with: https://github.com/cypress-io/cypress/issues/21471 + experimentalSessionAndOrigin: cfg.experimentalSessionAndOrigin, + }) if (result !== true) { return onErr(result) @@ -199,6 +203,8 @@ export const validateOverridableAtRunTime = (config: any, isSuiteLevelOverride: return } + // TODO: add a hook to ensure valid testing-type configuration is being set at runtime for all configuration values. + if (overrideLevel === 'never' || (overrideLevel === 'suite' && !isSuiteLevelOverride)) { onErr({ invalidConfigKey: configKey, diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 94fb4383c233..2d88dfd5fe1c 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -200,6 +200,7 @@ const driverConfigOptions: Array = [ isExperimental: true, requireRestartOnChange: 'server', }, { + // TODO: remove with experimentalSessionAndOrigin. Fixed with: https://github.com/cypress-io/cypress/issues/21471 name: 'experimentalSessionAndOrigin', defaultValue: false, validation: validate.isBoolean, @@ -373,11 +374,27 @@ const driverConfigOptions: Array = [ name: 'testIsolation', // TODO: https://github.com/cypress-io/cypress/issues/23093 // When experimentalSessionAndOrigin is removed and released as GA, - // update the defaultValue from 'legacy' to 'strict' and + // update the defaultValue from 'legacy' to 'on' and // update this code to remove the check/override specific to enable - // strict by default when experimentalSessionAndOrigin=true - defaultValue: 'legacy', - validation: validate.isOneOf('legacy', 'strict'), + // 'on' by default when experimentalSessionAndOrigin=true + defaultValue: (options: Record = {}) => { + if (options.testingType === 'component') { + return 'legacy' + } + + return options.config?.e2e?.experimentalSessionAndOrigin ? 'on' : 'legacy' + }, + validation: (key: string, value: any, { testingType, experimentalSessionAndOrigin }: { testingType: TestingType, experimentalSessionAndOrigin: boolean }) => { + if (testingType === 'component') { + return new Error('This is not a validate component-testing configuration option.') + } + + if (experimentalSessionAndOrigin) { + return validate.isOneOf('on', 'off')(key, value) + } + + return validate.isOneOf('legacy')(key, value) + }, overrideLevel: 'suite', }, { name: 'trashAssetsBeforeRuns', diff --git a/packages/config/src/project/utils.ts b/packages/config/src/project/utils.ts index be927d357086..29dc13815650 100644 --- a/packages/config/src/project/utils.ts +++ b/packages/config/src/project/utils.ts @@ -477,15 +477,15 @@ export function mergeDefaults ( }, testingType) // TODO: https://github.com/cypress-io/cypress/issues/23093 - // testIsolation should equal 'strict' by default when experimentalSessionAndOrigin=true + // testIsolation should equal 'on' by default when experimentalSessionAndOrigin=true // Once experimentalSessionAndOrigin is made GA, remove this logic and update the defaultValue - // to be be 'strict' + // to be be 'on' if (testingType === 'e2e' && config.experimentalSessionAndOrigin) { if (config.rawJson.testIsolation) { config.resolved.testIsolation.from = 'config' } else { - config.testIsolation = 'strict' - config.resolved.testIsolation.value = 'strict' + config.testIsolation = 'on' + config.resolved.testIsolation.value = 'on' config.resolved.testIsolation.from === 'default' } } diff --git a/packages/config/test/index.spec.ts b/packages/config/test/index.spec.ts index 22de97ebb98e..2cef53c3d862 100644 --- a/packages/config/test/index.spec.ts +++ b/packages/config/test/index.spec.ts @@ -195,7 +195,7 @@ describe('config/src/index', () => { const isSuiteOverride = true - configUtil.validateOverridableAtRunTime({ testIsolation: 'strict' }, isSuiteOverride, errorFn) + configUtil.validateOverridableAtRunTime({ testIsolation: 'on' }, isSuiteOverride, errorFn) expect(errorFn).to.have.callCount(0) }) diff --git a/packages/config/test/project/utils.spec.ts b/packages/config/test/project/utils.spec.ts index 843b5069d358..49d42736b97c 100644 --- a/packages/config/test/project/utils.spec.ts +++ b/packages/config/test/project/utils.spec.ts @@ -1206,7 +1206,7 @@ describe('config/src/project/utils', () => { }) }) - it('sets testIsolation=strict by default when experimentalSessionAndOrigin=true and e2e testing', () => { + it('sets testIsolation=on by default when experimentalSessionAndOrigin=true and e2e testing', () => { sinon.stub(utils, 'getProcessEnvVars').returns({}) const obj = { @@ -1227,7 +1227,7 @@ describe('config/src/project/utils', () => { expect(cfg.resolved).to.have.property('experimentalSessionAndOrigin') expect(cfg.resolved.experimentalSessionAndOrigin).to.deep.eq({ value: true, from: 'config' }) expect(cfg.resolved).to.have.property('testIsolation') - expect(cfg.resolved.testIsolation).to.deep.eq({ value: 'strict', from: 'default' }) + expect(cfg.resolved.testIsolation).to.deep.eq({ value: 'on', from: 'default' }) }) }) @@ -1239,7 +1239,7 @@ describe('config/src/project/utils', () => { supportFile: false, baseUrl: 'http://localhost:8080', experimentalSessionAndOrigin: true, - testIsolation: 'legacy', + testIsolation: 'on', } const options = { @@ -1253,7 +1253,7 @@ describe('config/src/project/utils', () => { expect(cfg.resolved).to.have.property('experimentalSessionAndOrigin') expect(cfg.resolved.experimentalSessionAndOrigin).to.deep.eq({ value: true, from: 'config' }) expect(cfg.resolved).to.have.property('testIsolation') - expect(cfg.resolved.testIsolation).to.deep.eq({ value: 'legacy', from: 'config' }) + expect(cfg.resolved.testIsolation).to.deep.eq({ value: 'on', from: 'config' }) }) }) }) diff --git a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js index a77ca3303190..660c17c633c2 100644 --- a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js +++ b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js @@ -74,25 +74,9 @@ describe('cy.session', { retries: 0 }, () => { }) }) - describe('test:before:run:async', () => { - it('clears session data before each run', async () => { - const clearCurrentSessionData = cy.spy(Cypress.session, 'clearCurrentSessionData') - - await Cypress.action('runner:test:before:run:async', {}) - - expect(clearCurrentSessionData).to.be.called - }) - - it('resets rendered html origins before each run', async () => { - const backendSpy = cy.spy(Cypress, 'backend') - - await Cypress.action('runner:test:before:run:async', {}) - - expect(backendSpy).to.be.calledWith('reset:rendered:html:origins') - }) - - describe('testIsolation=strict', { testIsolation: 'strict' }, () => { - it('clears page before each run when testIsolation=strict', () => { + describe('testIsolation=on', { testIsolation: 'on' }, () => { + describe('test:before:run:async', () => { + it('clears page before each run', () => { cy.visit('/fixtures/form.html') .then(async () => { cy.spy(Cypress, 'action').log(false) @@ -104,248 +88,215 @@ describe('cy.session', { retries: 0 }, () => { }) .url('about:blank') }) - }) - describe('testIsolation=legacy', { testIsolation: 'legacy' }, () => { - it('does not clear page', () => { - cy.visit('/fixtures/form.html') - .then(async () => { - cy.spy(Cypress, 'action').log(false) + it('clears session data before each run', async () => { + const clearCurrentSessionData = cy.spy(Cypress.session, 'clearCurrentSessionData') - await Cypress.action('runner:test:before:run:async', {}) + await Cypress.action('runner:test:before:run:async', {}) - expect(Cypress.action).not.to.be.calledWith('cy:url:changed') - expect(Cypress.action).not.to.be.calledWith('cy:visit:blank') - }) - .url('/fixtures/form.html') + expect(clearCurrentSessionData).to.be.called }) - }) - }) - describe('session flows', () => { - let logs = [] - let clearPageCount = 0 - let sessionGroupId - let setup - let validate + it('resets rendered html origins before each run', async () => { + const backendSpy = cy.spy(Cypress, 'backend') - const handleSetup = () => { - // create session clears page before running - cy.contains('Default blank page') - cy.contains('This page was cleared by navigating to about:blank.') + await Cypress.action('runner:test:before:run:async', {}) - cy.visit('/fixtures/auth/index.html') - cy.contains('You are not logged in') - cy.window().then((win) => { - win.sessionStorage.setItem('cypressAuthToken', JSON.stringify({ body: { username: 'tester' } })) + expect(backendSpy).to.be.calledWith('reset:rendered:html:origins') }) - } - const handleValidate = () => { - // both create & restore session clears page after running - cy.contains('Default blank page') - cy.contains('This page was cleared by navigating to about:blank.') + it('clears the browser context before each run', () => { + cy.window() + .then((win) => { + win.cookie = 'key=value; SameSite=Strict; Secure; Path=/fixtures' + win.localStorage.setItem('animal', 'bear') + win.sessionStorage.setItem('food', 'burgers') + }) + .then(async () => { + cy.spy(Cypress, 'action').log(false) - cy.visit('/fixtures/auth/index.html') - cy.contains('Welcome tester') - } + await Cypress.action('runner:test:before:run:async', {}) + + expect(Cypress.action).to.be.calledWith('cy:url:changed', '') + expect(Cypress.action).to.be.calledWith('cy:visit:blank', { type: 'session-lifecycle' }) + }) - before(() => { - setup = cy.stub().callsFake(handleSetup).as('setupSession') - validate = cy.stub().callsFake(handleValidate).as('validateSession') + cy.window().its('cookie').should('be.undefined') + cy.window().its('localStorage').should('have.length', 0) + cy.window().its('sessionStorage').should('have.length', 0) + }) }) - const resetMocks = () => { - logs = [] - clearPageCount = 0 - sessionGroupId = undefined - setup.reset() - setup.callsFake(handleSetup) - validate.reset() - validate.callsFake(handleValidate) - } + describe('session flows', () => { + let logs = [] + let clearPageCount = 0 + let sessionGroupId + let setup + let validate - const setupTestContext = () => { - resetMocks() - clearAllSavedSessions() - cy.on('log:added', (attrs, log) => { - if (attrs.name === 'session' || attrs.name === 'sessions_manager' || attrs.name === 'page load' || attrs.alias?.includes('setupSession') || attrs.alias?.includes('validateSession')) { - logs.push(log) - if (!sessionGroupId) { - sessionGroupId = attrs.id - } - } - }) + const handleSetup = () => { + // create session clears page before running + cy.contains('Default blank page') + cy.contains('This page was cleared by navigating to about:blank.') - cy.on('log:changed', (attrs, log) => { - const index = logs.findIndex((l) => l.id === attrs.id) + cy.visit('/fixtures/auth/index.html') + cy.contains('You are not logged in') + cy.window().then((win) => { + win.sessionStorage.setItem('cypressAuthToken', JSON.stringify({ body: { username: 'tester' } })) + }) + } - if (index) { - logs[index] = log - } - }) + const handleValidate = () => { + // both create & restore session clears page after running + cy.contains('Default blank page') + cy.contains('This page was cleared by navigating to about:blank.') - cy.on('internal:window:load', (args) => { - if (args.window.location.href === 'about:blank') { - clearPageCount++ - } - }) - } + cy.visit('/fixtures/auth/index.html') + cy.contains('Welcome tester') + } - describe('create session flow', () => { before(() => { - setupTestContext() - cy.log('Creating new session to test against') - expect(clearPageCount, 'total times session cleared the page').to.eq(0) - cy.session('session-1', setup) - }) - - // test must be first to run before blank page visit between each test - it('clears page after setup runs', () => { - cy.url().should('eq', 'about:blank') + setup = cy.stub().callsFake(handleSetup).as('setupSession') + validate = cy.stub().callsFake(handleValidate).as('validateSession') }) - it('successfully creates new session', () => { - expect(setup).to.be.calledOnce - expect(clearPageCount, 'total times session cleared the page').to.eq(2) - }) + const resetMocks = () => { + logs = [] + clearPageCount = 0 + sessionGroupId = undefined + setup.reset() + setup.callsFake(handleSetup) + validate.reset() + validate.callsFake(handleValidate) + } - it('groups session logs correctly', () => { - expect(logs[0].get()).to.deep.contain({ - name: 'session', - id: sessionGroupId, - sessionInfo: { - id: 'session-1', - isGlobalSession: false, - status: 'created', - }, + const setupTestContext = () => { + resetMocks() + clearAllSavedSessions() + cy.on('log:added', (attrs, log) => { + if (attrs.name === 'session' || attrs.name === 'sessions_manager' || attrs.name === 'page load' || attrs.alias?.includes('setupSession') || attrs.alias?.includes('validateSession')) { + logs.push(log) + if (!sessionGroupId) { + sessionGroupId = attrs.id + } + } }) - validateClearLogs([logs[1], logs[2]], sessionGroupId) + cy.on('log:changed', (attrs, log) => { + const index = logs.findIndex((l) => l.id === attrs.id) - const createNewSessionGroup = logs[3].get() - - expect(createNewSessionGroup).to.contain({ - displayName: 'Create new session', - groupStart: true, - group: sessionGroupId, + if (index) { + logs[index] = log + } }) - expect(logs[4].get()).to.deep.contain({ - alias: ['setupSession'], - group: createNewSessionGroup.id, + cy.on('internal:window:load', (args) => { + if (args.window.location.href === 'about:blank') { + clearPageCount++ + } }) + } - expect(logs[5].get()).to.contain({ - name: 'Clear page', - group: createNewSessionGroup.id, + describe('create session flow', () => { + before(() => { + setupTestContext() + cy.log('Creating new session to test against') + expect(clearPageCount, 'total times session cleared the page').to.eq(0) + cy.session('session-1', setup) }) - }) - - it('creates new session instrument with session details', () => { - const sessionInfo = logs[0].get('sessionInfo') - expect(sessionInfo).to.deep.eq({ - id: 'session-1', - isGlobalSession: false, - status: 'created', + // test must be first to run before blank page visit between each test + it('clears page after setup runs', () => { + cy.url().should('eq', 'about:blank') }) - }) - - it('has session details in the consoleProps', () => { - const consoleProps = logs[0].get('consoleProps')() - expect(consoleProps).to.deep.eq({ - Command: 'session', - id: 'session-1', - table: [], + it('successfully creates new session', () => { + expect(setup).to.be.calledOnce + expect(clearPageCount, 'total times session cleared the page').to.eq(2) }) - }) - }) - describe('create session with validation flow', () => { - let sessionId + it('groups session logs correctly', () => { + expect(logs[0].get()).to.deep.contain({ + name: 'session', + id: sessionGroupId, + sessionInfo: { + id: 'session-1', + isGlobalSession: false, + status: 'created', + }, + }) - before(() => { - setupTestContext() - cy.log('Creating new session with validation to test against') - sessionId = `session-${Cypress.state('test').id}` - cy.session(sessionId, setup, { validate }) - }) + validateClearLogs([logs[1], logs[2]], sessionGroupId) - // test must be first to run before blank page visit between each test - it('does not clear page visit from validate function', () => { - cy.url().should('contain', '/fixtures/auth/index.html') - }) + const createNewSessionGroup = logs[3].get() - it('successfully creates new session and validates it', () => { - expect(setup).to.be.calledOnce - expect(validate).to.be.calledOnce - expect(clearPageCount, 'total times session cleared the page').to.eq(2) - }) + expect(createNewSessionGroup).to.contain({ + displayName: 'Create new session', + groupStart: true, + group: sessionGroupId, + }) - it('groups session logs correctly', () => { - expect(logs[0].get()).to.deep.contain({ - name: 'session', - id: sessionGroupId, - sessionInfo: { - id: sessionId, - isGlobalSession: false, - status: 'created', - }, - }) + expect(logs[4].get()).to.deep.contain({ + alias: ['setupSession'], + group: createNewSessionGroup.id, + }) - validateClearLogs([logs[1], logs[2]], sessionGroupId) + expect(logs[5].get()).to.contain({ + name: 'Clear page', + group: createNewSessionGroup.id, + }) + }) - const createNewSessionGroup = logs[3].get() + it('creates new session instrument with session details', () => { + const sessionInfo = logs[0].get('sessionInfo') - expect(createNewSessionGroup).to.contain({ - displayName: 'Create new session', - groupStart: true, - group: sessionGroupId, + expect(sessionInfo).to.deep.eq({ + id: 'session-1', + isGlobalSession: false, + status: 'created', + }) }) - expect(logs[4].get()).to.deep.contain({ - alias: ['setupSession'], - group: createNewSessionGroup.id, - }) + it('has session details in the consoleProps', () => { + const consoleProps = logs[0].get('consoleProps')() - expect(logs[5].get()).to.contain({ - name: 'Clear page', - group: createNewSessionGroup.id, + expect(consoleProps).to.deep.eq({ + Command: 'session', + id: 'session-1', + table: [], + }) }) + }) - const validateSessionGroup = logs[6].get() + describe('create session with validation flow', () => { + let sessionId - expect(validateSessionGroup).to.contain({ - displayName: 'Validate session', - group: sessionGroupId, + before(() => { + setupTestContext() + cy.log('Creating new session with validation to test against') + sessionId = `session-${Cypress.state('test').id}` + cy.session(sessionId, setup, { validate }) }) - expect(logs[7].get()).to.deep.contain({ - alias: ['validateSession'], - group: validateSessionGroup.id, + // test must be first to run before blank page visit between each test + it('does not clear page visit from validate function', () => { + cy.url().should('contain', '/fixtures/auth/index.html') }) - }) - }) - - describe('create session with failed validation flow', () => { - it('fails validation and logs correctly', function (done) { - setupTestContext() - cy.log('Creating new session with validation to test against') - cy.once('fail', (err) => { + it('successfully creates new session and validates it', () => { expect(setup).to.be.calledOnce expect(validate).to.be.calledOnce expect(clearPageCount, 'total times session cleared the page').to.eq(2) - expect(err.message).to.contain('Your `cy.session` **validate** callback returned false') + }) + + it('groups session logs correctly', () => { expect(logs[0].get()).to.deep.contain({ name: 'session', id: sessionGroupId, sessionInfo: { - id: `session-${Cypress.state('test').id}`, + id: sessionId, isGlobalSession: false, - status: 'failed', + status: 'created', }, }) @@ -380,267 +331,228 @@ describe('cy.session', { retries: 0 }, () => { alias: ['validateSession'], group: validateSessionGroup.id, }) - - done() - }) - - validate.callsFake(() => false) - - cy.session(`session-${Cypress.state('test').id}`, setup, { validate }) - }) - }) - - describe('restores saved session flow', () => { - let sessionId - - before(() => { - setupTestContext() - cy.log('Creating new session for test') - sessionId = `session-${Cypress.state('test').id}` - cy.session(sessionId, setup) - .then(() => { - // reset and only test restored session - resetMocks() }) - - cy.log('restore session to test against') - cy.session(sessionId, setup) - }) - - // test must be first to run before blank page visit between each test - it('clears page after setup runs', () => { - cy.url().should('eq', 'about:blank') - }) - - it('successfully restores saved session', () => { - expect(setup).to.not.be.called - expect(validate).to.not.be.called - expect(clearPageCount, 'total times session cleared the page').to.eq(1) }) - it('groups session logs correctly', () => { - expect(logs[0].get()).to.contain({ - name: 'session', - id: sessionGroupId, - }) - - expect(logs[0].get()).to.deep.contain({ - name: 'session', - id: sessionGroupId, - sessionInfo: { - id: sessionId, - isGlobalSession: false, - status: 'restored', - }, - }) - - validateClearLogs([logs[1], logs[2]], sessionGroupId) + describe('create session with failed validation flow', () => { + it('fails validation and logs correctly', function (done) { + setupTestContext() + cy.log('Creating new session with validation to test against') + + cy.once('fail', (err) => { + expect(setup).to.be.calledOnce + expect(validate).to.be.calledOnce + expect(clearPageCount, 'total times session cleared the page').to.eq(2) + expect(err.message).to.contain('Your `cy.session` **validate** callback returned false') + expect(logs[0].get()).to.deep.contain({ + name: 'session', + id: sessionGroupId, + sessionInfo: { + id: `session-${Cypress.state('test').id}`, + isGlobalSession: false, + status: 'failed', + }, + }) + + validateClearLogs([logs[1], logs[2]], sessionGroupId) + + const createNewSessionGroup = logs[3].get() + + expect(createNewSessionGroup).to.contain({ + displayName: 'Create new session', + groupStart: true, + group: sessionGroupId, + }) + + expect(logs[4].get()).to.deep.contain({ + alias: ['setupSession'], + group: createNewSessionGroup.id, + }) + + expect(logs[5].get()).to.contain({ + name: 'Clear page', + group: createNewSessionGroup.id, + }) + + const validateSessionGroup = logs[6].get() + + expect(validateSessionGroup).to.contain({ + displayName: 'Validate session', + group: sessionGroupId, + }) + + expect(logs[7].get()).to.deep.contain({ + alias: ['validateSession'], + group: validateSessionGroup.id, + }) + + done() + }) - const restoreSavedSessionGroup = logs[3].get() + validate.callsFake(() => false) - expect(restoreSavedSessionGroup).to.contain({ - displayName: 'Restore saved session', - group: sessionGroupId, + cy.session(`session-${Cypress.state('test').id}`, setup, { validate }) }) }) - }) - describe('restores saved session with validation flow', () => { - let sessionId + describe('restores saved session flow', () => { + let sessionId - before(() => { - setupTestContext() - cy.log('Creating new session for test') - sessionId = `session-${Cypress.state('test').id}` - cy.session(sessionId, setup, { validate }) - .then(() => { + before(() => { + setupTestContext() + cy.log('Creating new session for test') + sessionId = `session-${Cypress.state('test').id}` + cy.session(sessionId, setup) + .then(() => { // reset and only test restored session - resetMocks() - }) - - cy.log('restore session to test against') - cy.session(sessionId, setup, { validate }) - }) - - // test must be first to run before blank page visit between each test - it('does not clear page visit from validate function', () => { - cy.url().should('contain', '/fixtures/auth/index.html') - }) - - it('successfully restores saved session', () => { - expect(setup).to.not.be.called - expect(validate).to.be.calledOnce - expect(clearPageCount, 'total times session cleared the page').to.eq(1) - }) + resetMocks() + }) - it('groups session logs correctly', () => { - expect(logs[0].get()).to.contain({ - name: 'session', - id: sessionGroupId, + cy.log('restore session to test against') + cy.session(sessionId, setup) }) - expect(logs[0].get()).to.deep.contain({ - name: 'session', - id: sessionGroupId, - sessionInfo: { - id: sessionId, - isGlobalSession: false, - status: 'restored', - }, + // test must be first to run before blank page visit between each test + it('clears page after setup runs', () => { + cy.url().should('eq', 'about:blank') }) - validateClearLogs([logs[1], logs[2]], sessionGroupId) - - const restoreSavedSessionGroup = logs[3].get() - - expect(restoreSavedSessionGroup).to.contain({ - displayName: 'Restore saved session', - group: sessionGroupId, + it('successfully restores saved session', () => { + expect(setup).to.not.be.called + expect(validate).to.not.be.called + expect(clearPageCount, 'total times session cleared the page').to.eq(1) }) - const validateSessionGroup = logs[4].get() - - expect(validateSessionGroup).to.contain({ - displayName: 'Validate session', - group: sessionGroupId, - }) + it('groups session logs correctly', () => { + expect(logs[0].get()).to.contain({ + name: 'session', + id: sessionGroupId, + }) - expect(logs[5].get()).to.deep.contain({ - alias: ['validateSession'], - group: validateSessionGroup.id, - }) - }) - }) + expect(logs[0].get()).to.deep.contain({ + name: 'session', + id: sessionGroupId, + sessionInfo: { + id: sessionId, + isGlobalSession: false, + status: 'restored', + }, + }) - describe('recreates existing session flow', () => { - let sessionId + validateClearLogs([logs[1], logs[2]], sessionGroupId) - before(() => { - setupTestContext() - cy.log('Creating new session for test') - sessionId = `session-${Cypress.state('test').id}` - cy.session(sessionId, setup, { validate }) - .then(() => { - // reset and only test restored session - resetMocks() - validate.callsFake(() => { - if (validate.callCount === 1) { - return false - } + const restoreSavedSessionGroup = logs[3].get() - handleValidate() + expect(restoreSavedSessionGroup).to.contain({ + displayName: 'Restore saved session', + group: sessionGroupId, }) }) - - cy.log('restore session to test against') - cy.session(sessionId, setup, { validate }) - }) - - // test must be first to run before blank page visit between each test - it('does not clear page visit from validate function', () => { - cy.url().should('contain', '/fixtures/auth/index.html') }) - it('successfully recreates session', () => { - expect(setup).to.be.calledOnce - expect(validate).to.be.calledTwice - expect(clearPageCount, 'total times session cleared the page').to.eq(3) - }) + describe('restores saved session with validation flow', () => { + let sessionId - it('groups session logs correctly', () => { - expect(logs[0].get()).to.contain({ - name: 'session', - id: sessionGroupId, - }) + before(() => { + setupTestContext() + cy.log('Creating new session for test') + sessionId = `session-${Cypress.state('test').id}` + cy.session(sessionId, setup, { validate }) + .then(() => { + // reset and only test restored session + resetMocks() + }) - expect(logs[0].get()).to.deep.contain({ - name: 'session', - id: sessionGroupId, - sessionInfo: { - id: sessionId, - isGlobalSession: false, - status: 'recreated', - }, + cy.log('restore session to test against') + cy.session(sessionId, setup, { validate }) }) - validateClearLogs([logs[1], logs[2]], sessionGroupId) - - const restoreSavedSessionGroup = logs[3].get() - - expect(restoreSavedSessionGroup).to.contain({ - displayName: 'Restore saved session', - group: sessionGroupId, + // test must be first to run before blank page visit between each test + it('does not clear page visit from validate function', () => { + cy.url().should('contain', '/fixtures/auth/index.html') }) - const validateSessionGroup = logs[4].get() - - expect(validateSessionGroup).to.contain({ - displayName: 'Validate session', - group: sessionGroupId, + it('successfully restores saved session', () => { + expect(setup).to.not.be.called + expect(validate).to.be.calledOnce + expect(clearPageCount, 'total times session cleared the page').to.eq(1) }) - expect(logs[5].get()).to.deep.contain({ - alias: ['validateSession'], - group: validateSessionGroup.id, - }) + it('groups session logs correctly', () => { + expect(logs[0].get()).to.contain({ + name: 'session', + id: sessionGroupId, + }) - expect(logs[6].get()).to.deep.contain({ - group: validateSessionGroup.id, - }) + expect(logs[0].get()).to.deep.contain({ + name: 'session', + id: sessionGroupId, + sessionInfo: { + id: sessionId, + isGlobalSession: false, + status: 'restored', + }, + }) - expect(logs[6].get('error').message).to.eq('Your `cy.session` **validate** callback returned false.') + validateClearLogs([logs[1], logs[2]], sessionGroupId) - validateClearLogs([logs[7], logs[8]], sessionGroupId) + const restoreSavedSessionGroup = logs[3].get() - const createNewSessionGroup = logs[9].get() + expect(restoreSavedSessionGroup).to.contain({ + displayName: 'Restore saved session', + group: sessionGroupId, + }) - expect(createNewSessionGroup).to.contain({ - displayName: 'Recreate session', - groupStart: true, - group: sessionGroupId, - }) + const validateSessionGroup = logs[4].get() - expect(logs[10].get()).to.deep.contain({ - alias: ['setupSession'], - group: createNewSessionGroup.id, - }) + expect(validateSessionGroup).to.contain({ + displayName: 'Validate session', + group: sessionGroupId, + }) - expect(logs[11].get()).to.contain({ - name: 'Clear page', - group: createNewSessionGroup.id, + expect(logs[5].get()).to.deep.contain({ + alias: ['validateSession'], + group: validateSessionGroup.id, + }) }) + }) - const secondValidateSessionGroup = logs[12].get() + describe('recreates existing session flow', () => { + let sessionId - expect(secondValidateSessionGroup).to.contain({ - displayName: 'Validate session', - group: sessionGroupId, - }) + before(() => { + setupTestContext() + cy.log('Creating new session for test') + sessionId = `session-${Cypress.state('test').id}` + cy.session(sessionId, setup, { validate }) + .then(() => { + // reset and only test restored session + resetMocks() + validate.callsFake(() => { + if (validate.callCount === 1) { + return false + } + + handleValidate() + }) + }) - expect(logs[13].get()).to.deep.contain({ - alias: ['validateSession'], - group: secondValidateSessionGroup.id, + cy.log('restore session to test against') + cy.session(sessionId, setup, { validate }) }) - }) - }) - describe('recreates existing session with failed validation flow', () => { - it('fails to recreate session and logs correctly', function (done) { - setupTestContext() - cy.log('Creating new session for test') - cy.session(`session-${Cypress.state('test').id}`, setup, { validate }) - .then(() => { - // reset and only test restored session - resetMocks() - validate.callsFake(() => false) + // test must be first to run before blank page visit between each test + it('does not clear page visit from validate function', () => { + cy.url().should('contain', '/fixtures/auth/index.html') }) - cy.once('fail', (err) => { - expect(err.message).to.contain('Your `cy.session` **validate** callback returned false') + it('successfully recreates session', () => { expect(setup).to.be.calledOnce expect(validate).to.be.calledTwice expect(clearPageCount, 'total times session cleared the page').to.eq(3) + }) + it('groups session logs correctly', () => { expect(logs[0].get()).to.contain({ name: 'session', id: sessionGroupId, @@ -650,9 +562,9 @@ describe('cy.session', { retries: 0 }, () => { name: 'session', id: sessionGroupId, sessionInfo: { - id: `session-${Cypress.state('test').id}`, + id: sessionId, isGlobalSession: false, - status: 'failed', + status: 'recreated', }, }) @@ -714,12 +626,762 @@ describe('cy.session', { retries: 0 }, () => { alias: ['validateSession'], group: secondValidateSessionGroup.id, }) - - done() }) + }) + + describe('recreates existing session with failed validation flow', () => { + it('fails to recreate session and logs correctly', function (done) { + setupTestContext() + cy.log('Creating new session for test') + cy.session(`session-${Cypress.state('test').id}`, setup, { validate }) + .then(() => { + // reset and only test restored session + resetMocks() + validate.callsFake(() => false) + }) + + cy.once('fail', (err) => { + expect(err.message).to.contain('Your `cy.session` **validate** callback returned false') + expect(setup).to.be.calledOnce + expect(validate).to.be.calledTwice + expect(clearPageCount, 'total times session cleared the page').to.eq(3) + + expect(logs[0].get()).to.contain({ + name: 'session', + id: sessionGroupId, + }) + + expect(logs[0].get()).to.deep.contain({ + name: 'session', + id: sessionGroupId, + sessionInfo: { + id: `session-${Cypress.state('test').id}`, + isGlobalSession: false, + status: 'failed', + }, + }) + + validateClearLogs([logs[1], logs[2]], sessionGroupId) + + const restoreSavedSessionGroup = logs[3].get() + + expect(restoreSavedSessionGroup).to.contain({ + displayName: 'Restore saved session', + group: sessionGroupId, + }) + + const validateSessionGroup = logs[4].get() + + expect(validateSessionGroup).to.contain({ + displayName: 'Validate session', + group: sessionGroupId, + }) + + expect(logs[5].get()).to.deep.contain({ + alias: ['validateSession'], + group: validateSessionGroup.id, + }) + + expect(logs[6].get()).to.deep.contain({ + group: validateSessionGroup.id, + }) + + expect(logs[6].get('error').message).to.eq('Your `cy.session` **validate** callback returned false.') + + validateClearLogs([logs[7], logs[8]], sessionGroupId) + + const createNewSessionGroup = logs[9].get() + + expect(createNewSessionGroup).to.contain({ + displayName: 'Recreate session', + groupStart: true, + group: sessionGroupId, + }) + + expect(logs[10].get()).to.deep.contain({ + alias: ['setupSession'], + group: createNewSessionGroup.id, + }) + + expect(logs[11].get()).to.contain({ + name: 'Clear page', + group: createNewSessionGroup.id, + }) - cy.log('restore session to test against') - cy.session(`session-${Cypress.state('test').id}`, setup, { validate }) + const secondValidateSessionGroup = logs[12].get() + + expect(secondValidateSessionGroup).to.contain({ + displayName: 'Validate session', + group: sessionGroupId, + }) + + expect(logs[13].get()).to.deep.contain({ + alias: ['validateSession'], + group: secondValidateSessionGroup.id, + }) + + done() + }) + + cy.log('restore session to test against') + cy.session(`session-${Cypress.state('test').id}`, setup, { validate }) + }) + }) + }) + }) + + describe('testIsolation=off', { testIsolation: 'off' }, () => { + before(async () => { + // manually ensure clear browser state! since we turned testIsolation off + await Cypress.session.clearCurrentSessionData() + }) + + describe('test:before:run:async', () => { + it('does not clear page before each run', () => { + cy.visit('/fixtures/form.html') + .then(async () => { + cy.spy(Cypress, 'action').log(false) + + await Cypress.action('runner:test:before:run:async', {}) + + expect(Cypress.action).not.to.be.calledWith('cy:url:changed') + expect(Cypress.action).not.to.be.calledWith('cy:visit:blank') + }) + .url('/fixtures/form.html') + }) + + it('does not clear session data before each run', async () => { + const clearCurrentSessionData = cy.spy(Cypress.session, 'clearCurrentSessionData') + + await Cypress.action('runner:test:before:run:async', {}) + + expect(clearCurrentSessionData).not.to.be.called + }) + + it('does not reset rendered html origins before each run', async () => { + const backendSpy = cy.spy(Cypress, 'backend') + + await Cypress.action('runner:test:before:run:async', {}) + + expect(backendSpy).not.to.be.calledWith('reset:rendered:html:origins') + }) + + it('does not clear the browser context before each run', () => { + cy.window() + .then((win) => { + win.cookie = 'key=value; SameSite=Strict; Secure; Path=/fixtures' + win.localStorage.setItem('animal', 'bear') + win.sessionStorage.setItem('food', 'burgers') + }) + .then(async () => { + cy.spy(Cypress, 'action').log(false) + + await Cypress.action('runner:test:before:run:async', {}) + + expect(Cypress.action).not.to.be.calledWith('cy:url:changed') + expect(Cypress.action).not.to.be.calledWith('cy:visit:blank') + }) + + cy.window().its('cookie').should('equal', 'key=value; SameSite=Strict; Secure; Path=/fixtures') + cy.window().its('localStorage').should('have.length', 1).should('deep.contain', { animal: 'bear' }) + cy.window().its('sessionStorage').should('have.length', 1).should('deep.contain', { food: 'burgers' }) + }) + }) + + describe('session flows', () => { + let logs = [] + let clearPageCount = 0 + let sessionGroupId + let setup + let validate + + const handleSetup = () => { + // create session clears page before running + cy.contains('Default blank page').should('not.exist') + + cy.visit('/fixtures/auth/index.html') + cy.contains('You are not logged in') + cy.window().then((win) => { + win.sessionStorage.setItem('cypressAuthToken', JSON.stringify({ body: { username: 'tester' } })) + }) + } + + const handleValidate = () => { + // both create & restore session clears page after running + cy.contains('Default blank page').should('not.exist') + + cy.visit('/fixtures/auth/index.html') + cy.contains('Welcome tester') + } + + before(() => { + setup = cy.stub().callsFake(handleSetup).as('setupSession') + validate = cy.stub().callsFake(handleValidate).as('validateSession') + }) + + const resetMocks = () => { + logs = [] + clearPageCount = 0 + sessionGroupId = undefined + setup.reset() + setup.callsFake(handleSetup) + validate.reset() + validate.callsFake(handleValidate) + } + + const setupTestContext = () => { + resetMocks() + clearAllSavedSessions() + cy.on('log:added', (attrs, log) => { + if (attrs.name === 'session' || attrs.name === 'sessions_manager' || attrs.name === 'page load' || attrs.alias?.includes('setupSession') || attrs.alias?.includes('validateSession')) { + logs.push(log) + if (!sessionGroupId) { + sessionGroupId = attrs.id + } + } + }) + + cy.on('log:changed', (attrs, log) => { + const index = logs.findIndex((l) => l.id === attrs.id) + + if (index) { + logs[index] = log + } + }) + + cy.on('internal:window:load', (args) => { + if (args.window.location.href === 'about:blank') { + clearPageCount++ + } + }) + } + + describe('create session flow', () => { + before(() => { + setupTestContext() + cy.log('Creating new session to test against') + expect(clearPageCount, 'total times session cleared the page').to.eq(0) + cy.session('session-1', setup) + }) + + it('does not clear the page after command', () => { + cy.url().should('contain', '/fixtures/auth/index.html') + }) + + it('successfully creates new session', () => { + expect(setup).to.be.calledOnce + expect(clearPageCount, 'total times session cleared the page').to.eq(0) + }) + + it('groups session logs correctly', () => { + expect(logs[0].get()).to.deep.contain({ + name: 'session', + id: sessionGroupId, + sessionInfo: { + id: 'session-1', + isGlobalSession: false, + status: 'created', + }, + }) + + expect(logs[1].get()).to.contain({ + displayName: 'Clear cookies, localStorage and sessionStorage', + group: sessionGroupId, + }) + + const createNewSessionGroup = logs[2].get() + + expect(createNewSessionGroup).to.contain({ + displayName: 'Create new session', + groupStart: true, + group: sessionGroupId, + }) + + expect(logs[3].get()).to.deep.contain({ + alias: ['setupSession'], + group: createNewSessionGroup.id, + }) + }) + + it('creates new session instrument with session details', () => { + const sessionInfo = logs[0].get('sessionInfo') + + expect(sessionInfo).to.deep.eq({ + id: 'session-1', + isGlobalSession: false, + status: 'created', + }) + }) + + it('has session details in the consoleProps', () => { + const consoleProps = logs[0].get('consoleProps')() + + expect(consoleProps).to.deep.eq({ + Command: 'session', + id: 'session-1', + table: [], + }) + }) + }) + + describe('create session with validation flow', () => { + let sessionId + + before(() => { + setupTestContext() + cy.log('Creating new session with validation to test against') + sessionId = `session-${Cypress.state('test').id}` + cy.session(sessionId, setup, { validate }) + }) + + it('does not clear the page after command', () => { + cy.url().should('contain', '/fixtures/auth/index.html') + }) + + it('successfully creates new session and validates it', () => { + expect(setup).to.be.calledOnce + expect(validate).to.be.calledOnce + expect(clearPageCount, 'total times session cleared the page').to.eq(0) + }) + + it('groups session logs correctly', () => { + expect(logs[0].get()).to.deep.contain({ + name: 'session', + id: sessionGroupId, + sessionInfo: { + id: sessionId, + isGlobalSession: false, + status: 'created', + }, + }) + + expect(logs[1].get()).to.contain({ + displayName: 'Clear cookies, localStorage and sessionStorage', + group: sessionGroupId, + }) + + const createNewSessionGroup = logs[2].get() + + expect(createNewSessionGroup).to.contain({ + displayName: 'Create new session', + groupStart: true, + group: sessionGroupId, + }) + + expect(logs[3].get()).to.deep.contain({ + alias: ['setupSession'], + group: createNewSessionGroup.id, + }) + + const validateSessionGroup = logs[4].get() + + expect(validateSessionGroup).to.contain({ + displayName: 'Validate session', + group: sessionGroupId, + }) + + expect(logs[5].get()).to.deep.contain({ + alias: ['validateSession'], + group: validateSessionGroup.id, + }) + }) + }) + + describe('create session with failed validation flow', () => { + it('fails validation and logs correctly', function (done) { + setupTestContext() + cy.log('Creating new session with validation to test against') + + cy.once('fail', (err) => { + expect(setup).to.be.calledOnce + expect(validate).to.be.calledOnce + expect(clearPageCount, 'total times session cleared the page').to.eq(0) + expect(err.message).to.contain('Your `cy.session` **validate** callback returned false') + expect(logs[0].get()).to.deep.contain({ + name: 'session', + id: sessionGroupId, + sessionInfo: { + id: `session-${Cypress.state('test').id}`, + isGlobalSession: false, + status: 'failed', + }, + }) + + expect(logs[1].get()).to.contain({ + displayName: 'Clear cookies, localStorage and sessionStorage', + group: sessionGroupId, + }) + + const createNewSessionGroup = logs[2].get() + + expect(createNewSessionGroup).to.contain({ + displayName: 'Create new session', + groupStart: true, + group: sessionGroupId, + }) + + expect(logs[3].get()).to.deep.contain({ + alias: ['setupSession'], + group: createNewSessionGroup.id, + }) + + const validateSessionGroup = logs[4].get() + + expect(validateSessionGroup).to.contain({ + displayName: 'Validate session', + group: sessionGroupId, + }) + + expect(logs[5].get()).to.deep.contain({ + alias: ['validateSession'], + group: validateSessionGroup.id, + }) + + done() + }) + + validate.callsFake(() => false) + + cy.session(`session-${Cypress.state('test').id}`, setup, { validate }) + }) + }) + + describe('restores saved session flow', () => { + let sessionId + + before(() => { + setupTestContext() + cy.log('Creating new session for test') + sessionId = `session-${Cypress.state('test').id}` + cy.session(sessionId, setup) + .then(() => { + // reset and only test restored session + resetMocks() + }) + + cy.log('restore session to test against') + cy.session(sessionId, setup) + }) + + it('does not clear the page after command', () => { + cy.url().should('contain', '/fixtures/auth/index.html') + }) + + it('successfully restores saved session', () => { + expect(setup).to.not.be.called + expect(validate).to.not.be.called + expect(clearPageCount, 'total times session cleared the page').to.eq(0) + }) + + it('groups session logs correctly', () => { + expect(logs[0].get()).to.contain({ + name: 'session', + id: sessionGroupId, + }) + + expect(logs[0].get()).to.deep.contain({ + name: 'session', + id: sessionGroupId, + sessionInfo: { + id: sessionId, + isGlobalSession: false, + status: 'restored', + }, + }) + + expect(logs[1].get()).to.contain({ + displayName: 'Clear cookies, localStorage and sessionStorage', + group: sessionGroupId, + }) + + const restoreSavedSessionGroup = logs[2].get() + + expect(restoreSavedSessionGroup).to.contain({ + displayName: 'Restore saved session', + group: sessionGroupId, + }) + }) + }) + + describe('restores saved session with validation flow', () => { + let sessionId + + before(() => { + setupTestContext() + cy.log('Creating new session for test') + sessionId = `session-${Cypress.state('test').id}` + cy.session(sessionId, setup, { validate }) + .then(() => { + // reset and only test restored session + resetMocks() + }) + + cy.log('restore session to test against') + cy.session(sessionId, setup, { validate }) + }) + + it('does not clear page after command', () => { + cy.url().should('contain', '/fixtures/auth/index.html') + }) + + it('successfully restores saved session', () => { + expect(setup).to.not.be.called + expect(validate).to.be.calledOnce + expect(clearPageCount, 'total times session cleared the page').to.eq(0) + }) + + it('groups session logs correctly', () => { + expect(logs[0].get()).to.contain({ + name: 'session', + id: sessionGroupId, + }) + + expect(logs[0].get()).to.deep.contain({ + name: 'session', + id: sessionGroupId, + sessionInfo: { + id: sessionId, + isGlobalSession: false, + status: 'restored', + }, + }) + + expect(logs[1].get()).to.contain({ + displayName: 'Clear cookies, localStorage and sessionStorage', + group: sessionGroupId, + }) + + const restoreSavedSessionGroup = logs[2].get() + + expect(restoreSavedSessionGroup).to.contain({ + displayName: 'Restore saved session', + group: sessionGroupId, + }) + + const validateSessionGroup = logs[3].get() + + expect(validateSessionGroup).to.contain({ + displayName: 'Validate session', + group: sessionGroupId, + }) + + expect(logs[4].get()).to.deep.contain({ + alias: ['validateSession'], + group: validateSessionGroup.id, + }) + }) + }) + + describe('recreates existing session flow', () => { + let sessionId + + before(() => { + setupTestContext() + cy.log('Creating new session for test') + sessionId = `session-${Cypress.state('test').id}` + cy.session(sessionId, setup, { validate }) + .then(() => { + // reset and only test restored session + resetMocks() + validate.callsFake(() => { + if (validate.callCount === 1) { + return false + } + + handleValidate() + }) + }) + + cy.log('restore session to test against') + cy.session(sessionId, setup, { validate }) + }) + + it('does not clear page after command', () => { + cy.url().should('contain', '/fixtures/auth/index.html') + }) + + it('successfully recreates session', () => { + expect(setup).to.be.calledOnce + expect(validate).to.be.calledTwice + expect(clearPageCount, 'total times session cleared the page').to.eq(0) + }) + + it('groups session logs correctly', () => { + expect(logs[0].get()).to.contain({ + name: 'session', + id: sessionGroupId, + }) + + expect(logs[0].get()).to.deep.contain({ + name: 'session', + id: sessionGroupId, + sessionInfo: { + id: sessionId, + isGlobalSession: false, + status: 'recreated', + }, + }) + + expect(logs[1].get()).to.contain({ + displayName: 'Clear cookies, localStorage and sessionStorage', + group: sessionGroupId, + }) + + const restoreSavedSessionGroup = logs[2].get() + + expect(restoreSavedSessionGroup).to.contain({ + displayName: 'Restore saved session', + group: sessionGroupId, + }) + + const validateSessionGroup = logs[3].get() + + expect(validateSessionGroup).to.contain({ + displayName: 'Validate session', + group: sessionGroupId, + }) + + expect(logs[4].get()).to.deep.contain({ + alias: ['validateSession'], + group: validateSessionGroup.id, + }) + + expect(logs[5].get()).to.deep.contain({ + group: validateSessionGroup.id, + }) + + expect(logs[5].get('error').message).to.eq('Your `cy.session` **validate** callback returned false.') + + expect(logs[6].get()).to.contain({ + displayName: 'Clear cookies, localStorage and sessionStorage', + group: sessionGroupId, + }) + + const createNewSessionGroup = logs[7].get() + + expect(createNewSessionGroup).to.contain({ + displayName: 'Recreate session', + groupStart: true, + group: sessionGroupId, + }) + + expect(logs[8].get()).to.deep.contain({ + alias: ['setupSession'], + group: createNewSessionGroup.id, + }) + + const secondValidateSessionGroup = logs[9].get() + + expect(secondValidateSessionGroup).to.contain({ + displayName: 'Validate session', + group: sessionGroupId, + }) + + expect(logs[10].get()).to.deep.contain({ + alias: ['validateSession'], + group: secondValidateSessionGroup.id, + }) + }) + }) + + describe('recreates existing session with failed validation flow', () => { + it('fails to recreate session and logs correctly', function (done) { + setupTestContext() + cy.log('Creating new session for test') + cy.session(`session-${Cypress.state('test').id}`, setup, { validate }) + .then(() => { + // reset and only test restored session + resetMocks() + validate.callsFake(() => false) + }) + + cy.once('fail', (err) => { + expect(err.message).to.contain('Your `cy.session` **validate** callback returned false') + expect(setup).to.be.calledOnce + expect(validate).to.be.calledTwice + expect(clearPageCount, 'total times session cleared the page').to.eq(0) + + expect(logs[0].get()).to.contain({ + name: 'session', + id: sessionGroupId, + }) + + expect(logs[0].get()).to.deep.contain({ + name: 'session', + id: sessionGroupId, + sessionInfo: { + id: `session-${Cypress.state('test').id}`, + isGlobalSession: false, + status: 'failed', + }, + }) + + expect(logs[1].get()).to.contain({ + displayName: 'Clear cookies, localStorage and sessionStorage', + group: sessionGroupId, + }) + + const restoreSavedSessionGroup = logs[2].get() + + expect(restoreSavedSessionGroup).to.contain({ + displayName: 'Restore saved session', + group: sessionGroupId, + }) + + const validateSessionGroup = logs[3].get() + + expect(validateSessionGroup).to.contain({ + displayName: 'Validate session', + group: sessionGroupId, + }) + + expect(logs[4].get()).to.deep.contain({ + alias: ['validateSession'], + group: validateSessionGroup.id, + }) + + expect(logs[5].get()).to.deep.contain({ + group: validateSessionGroup.id, + }) + + expect(logs[5].get('error').message).to.eq('Your `cy.session` **validate** callback returned false.') + + expect(logs[6].get()).to.contain({ + displayName: 'Clear cookies, localStorage and sessionStorage', + group: sessionGroupId, + }) + + const createNewSessionGroup = logs[7].get() + + expect(createNewSessionGroup).to.contain({ + displayName: 'Recreate session', + groupStart: true, + group: sessionGroupId, + }) + + expect(logs[8].get()).to.deep.contain({ + alias: ['setupSession'], + group: createNewSessionGroup.id, + }) + + const secondValidateSessionGroup = logs[9].get() + + expect(secondValidateSessionGroup).to.contain({ + displayName: 'Validate session', + group: sessionGroupId, + }) + + expect(logs[10].get()).to.deep.contain({ + alias: ['validateSession'], + group: secondValidateSessionGroup.id, + }) + + done() + }) + + cy.log('restore session to test against') + cy.session(`session-${Cypress.state('test').id}`, setup, { validate }) + }) }) }) }) diff --git a/packages/driver/cypress/e2e/util/config.cy.js b/packages/driver/cypress/e2e/util/config.cy.js index 47d4af4e1e9d..ec5107ee5e8f 100644 --- a/packages/driver/cypress/e2e/util/config.cy.js +++ b/packages/driver/cypress/e2e/util/config.cy.js @@ -110,7 +110,7 @@ describe('driver/src/cypress/validate_config', () => { expect(overrideLevel).to.eq('suite') expect(() => { - validateConfig(state, { testIsolation: 'strict' }) + validateConfig(state, { testIsolation: 'on' }) }).not.to.throw() }) @@ -192,7 +192,7 @@ describe('driver/src/cypress/validate_config', () => { }) expect(() => { - validateConfig(state, { testIsolation: 'legacy' }) + validateConfig(state, { testIsolation: 'off' }) }).not.to.throw() }) diff --git a/packages/driver/src/cy/commands/navigation.ts b/packages/driver/src/cy/commands/navigation.ts index 5e1aec95db4a..feaaa6aa2660 100644 --- a/packages/driver/src/cy/commands/navigation.ts +++ b/packages/driver/src/cy/commands/navigation.ts @@ -35,11 +35,12 @@ const reset = (test: any = {}) => { // before each test run! previouslyVisitedLocation = undefined - const { experimentalSessionAndOrigin, testIsolation } = Cypress.config() + // TODO: remove with experimentalSessionAndOrigin. Fixed with: https://github.com/cypress-io/cypress/issues/21471 + const { experimentalSessionAndOrigin } = Cypress.config() // make sure we reset that we haven't visited about blank again // strict test isolation resets the navigation history for us. - hasVisitedAboutBlank = experimentalSessionAndOrigin && testIsolation === 'strict' + hasVisitedAboutBlank = experimentalSessionAndOrigin currentlyVisitingAboutBlank = false @@ -1218,6 +1219,10 @@ export default (Commands, Cypress, cy, state, config) => { } const visit = () => { + // REMOVE THIS ONCE GA HITS. Sessions will handle visiting + // about blank. + // Fixed with: https://github.com/cypress-io/cypress/issues/21471 + // // if we've visiting for the first time during // a test then we want to first visit about:blank // so that we nuke the previous state. subsequent diff --git a/packages/driver/src/cy/commands/sessions/index.ts b/packages/driver/src/cy/commands/sessions/index.ts index 6e4c011e7c60..a28cf4ea7910 100644 --- a/packages/driver/src/cy/commands/sessions/index.ts +++ b/packages/driver/src/cy/commands/sessions/index.ts @@ -49,7 +49,12 @@ export default function (Commands, Cypress, cy) { Cypress.on('test:before:run:async', () => { if (Cypress.config('experimentalSessionAndOrigin')) { - const clearPage = Cypress.config('testIsolation') === 'strict' ? navigateAboutBlank(false) : new Cypress.Promise.resolve() + if (Cypress.config('testIsolation') === 'off') { + return + } + + // Component testing does not support navigation and handles clearing the page via mount utils + const clearPage = Cypress.testingType === 'e2e' ? navigateAboutBlank(false) : new Cypress.Promise.resolve() return clearPage .then(() => sessions.clearCurrentSessionData()) diff --git a/packages/driver/src/cy/commands/sessions/utils.ts b/packages/driver/src/cy/commands/sessions/utils.ts index f9153629e5f7..3932cd1c439d 100644 --- a/packages/driver/src/cy/commands/sessions/utils.ts +++ b/packages/driver/src/cy/commands/sessions/utils.ts @@ -184,6 +184,10 @@ const getPostMessageLocalStorage = (specWindow, origins): Promise => { } function navigateAboutBlank (session: boolean = true) { + if (Cypress.config('testIsolation') === 'off') { + return + } + Cypress.action('cy:url:changed', '') return Cypress.action('cy:visit:blank', { type: session ? 'session' : 'session-lifecycle' }) as unknown as Promise diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index 2a34d4c11cdb..63b0dba52193 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -113,6 +113,13 @@ export const validateConfig = (state: State, config: Record, skipCo }) } + config = { + testingType: Cypress.testingType, + // TODO: remove with experimentalSessionAndOrigin. Fixed with: https://github.com/cypress-io/cypress/issues/21471 + experimentalSessionAndOrigin: Cypress.originalConfig.experimentalSessionAndOrigin, + ...config, + } + validateConfigValues(config, (errResult: ErrResult | string) => { const stringify = (str) => format(JSON.stringify(str)) From 9585bfef6e6ad8e06aea907d07cb036214a79efe Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Wed, 19 Oct 2022 13:56:41 -0500 Subject: [PATCH 02/13] fix tests and how test isolation is determined and validated --- cli/types/cypress.d.ts | 14 +- packages/config/src/browser.ts | 4 +- packages/config/src/options.ts | 6 +- packages/config/src/project/utils.ts | 4 +- packages/config/test/index.spec.ts | 4 +- .../src/data/ProjectConfigManager.ts | 2 +- .../cypress/e2e/commands/navigation.cy.js | 254 +++++++++--------- packages/driver/cypress/e2e/util/config.cy.js | 8 +- packages/driver/src/util/config.ts | 3 +- 9 files changed, 152 insertions(+), 147 deletions(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index f17031d5fcd0..fbd2a2048f73 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2809,20 +2809,20 @@ declare namespace Cypress { supportFile: string | false /** * The test isolation ensures a clean browser context between tests for end-to-end tests. - * + * * Cypress will always resets/clears aliases, intercepts, clock, and viewport before each test * to ensure a clean test slate; i.e. this configuration only impacts the browser context. - * + * * Note: the `cy.session()` command will inherent this value to determine where or not the page is * cleared when the command executes. - * + * * Options: - * When experimentalSessionAndOrigin=false + * When experimentalSessionAndOrigin=false * - legacy - "Pass through" to document the default Cypress test isolation behavior. The page is not cleared * in end-to-end tests and cookies, local storage and session storage in the current domains are * cleared before each test. The `cy.session()` command is not available. * NOTE: this behavior will be changed in the next major release and the new default will be 'on'. - * + * * When experimentalSessionAndOrigin=true * - on - The page is cleared and cookies, local storage and session storage in all domains are cleared before each * test. The `cy.session()` command will also clear the page and current browser context when establishing @@ -2830,12 +2830,12 @@ declare namespace Cypress { * - off - The current browser state will persist between tests. The page does not clear and cookies, local storage and * session storage will be available in the next test. The `cy.session()` command will only clear the current * browser context when establishing the browser session - the current page will not clear. - * + * * NOTES: * - Turning test isolation off may improve performance of end-to-end tests, however, previous tests could impact the * browser state of the next test and cause inconsistency when using .only(). * - The page is always cleared in component testing to ensure a clean page for rendering isolated components. - * + * * @default 'legacy', when running end-to-end tests. When running component tests or running end-to-end tests with * experimentalSessionAndOrigin=true, the default is 'on'. */ diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index 7d1677d4e381..99f62876db41 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -155,7 +155,7 @@ export const matchesConfigKey = (key: string) => { return } -export const validate = (cfg: any, onErr: (property: ErrResult | string) => void) => { +export const validate = (testingType: TestingType, cfg: any, onErr: (property: ErrResult | string) => void) => { debug('validating configuration') return _.each(cfg, (value, key) => { @@ -164,7 +164,7 @@ export const validate = (cfg: any, onErr: (property: ErrResult | string) => void // key has a validation rule & value different from the default if (validationFn && value !== defaultValues[key]) { const result = validationFn(key, value, { - testingType: cfg.testingType, + testingType, // TODO: remove with experimentalSessionAndOrigin. Fixed with: https://github.com/cypress-io/cypress/issues/21471 experimentalSessionAndOrigin: cfg.experimentalSessionAndOrigin, }) diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 2d88dfd5fe1c..5cdebe444876 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -379,14 +379,14 @@ const driverConfigOptions: Array = [ // 'on' by default when experimentalSessionAndOrigin=true defaultValue: (options: Record = {}) => { if (options.testingType === 'component') { - return 'legacy' + return 'on' } - return options.config?.e2e?.experimentalSessionAndOrigin ? 'on' : 'legacy' + return options?.e2e?.experimentalSessionAndOrigin ? 'on' : 'legacy' }, validation: (key: string, value: any, { testingType, experimentalSessionAndOrigin }: { testingType: TestingType, experimentalSessionAndOrigin: boolean }) => { if (testingType === 'component') { - return new Error('This is not a validate component-testing configuration option.') + return validate.isOneOf('on')(key, value) } if (experimentalSessionAndOrigin) { diff --git a/packages/config/src/project/utils.ts b/packages/config/src/project/utils.ts index 29dc13815650..c68d93b871a8 100644 --- a/packages/config/src/project/utils.ts +++ b/packages/config/src/project/utils.ts @@ -401,7 +401,7 @@ export function mergeDefaults ( config.baseUrl = url.replace(/\/\/+$/, '/') } - const defaultsForRuntime = getDefaultValues(options) + const defaultsForRuntime = getDefaultValues(config) _.defaultsDeep(config, defaultsForRuntime) @@ -447,7 +447,7 @@ export function mergeDefaults ( // validate config again here so that we catch configuration errors coming // from the CLI overrides or env var overrides - validate(_.omit(config, 'browsers'), (validationResult: ConfigValidationFailureInfo | string) => { + validate(testingType, _.omit(config, 'browsers'), (validationResult: ConfigValidationFailureInfo | string) => { // return errors.throwErr('CONFIG_VALIDATION_ERROR', errMsg) if (_.isString(validationResult)) { return errors.throwErr('CONFIG_VALIDATION_MSG_ERROR', null, null, validationResult) diff --git a/packages/config/test/index.spec.ts b/packages/config/test/index.spec.ts index 2cef53c3d862..c1db20fe29a9 100644 --- a/packages/config/test/index.spec.ts +++ b/packages/config/test/index.spec.ts @@ -116,7 +116,7 @@ describe('config/src/index', () => { it('validates config', () => { const errorFn = sinon.spy() - configUtil.validate({ + configUtil.validate('e2e', { 'baseUrl': 'https://', }, errorFn) @@ -126,7 +126,7 @@ describe('config/src/index', () => { it('calls error callback if config is invalid', () => { const errorFn = sinon.spy() - configUtil.validate({ + configUtil.validate('e2e', { 'baseUrl': ' ', }, errorFn) diff --git a/packages/data-context/src/data/ProjectConfigManager.ts b/packages/data-context/src/data/ProjectConfigManager.ts index f9d16e47ddd8..f526e0f6d016 100644 --- a/packages/data-context/src/data/ProjectConfigManager.ts +++ b/packages/data-context/src/data/ProjectConfigManager.ts @@ -361,7 +361,7 @@ export class ProjectConfigManager { } private validateConfigFile (file: string | false, config: Cypress.ConfigOptions) { - validateConfig(config, (errMsg) => { + validateConfig(this._testingType, config, (errMsg) => { if (_.isString(errMsg)) { throw getError('CONFIG_VALIDATION_MSG_ERROR', 'configFile', file || null, errMsg) } diff --git a/packages/driver/cypress/e2e/commands/navigation.cy.js b/packages/driver/cypress/e2e/commands/navigation.cy.js index cccfc66cda22..19c29bfbb50e 100644 --- a/packages/driver/cypress/e2e/commands/navigation.cy.js +++ b/packages/driver/cypress/e2e/commands/navigation.cy.js @@ -581,8 +581,8 @@ describe('src/cy/commands/navigation', () => { }) }) - describe('removes window:load listeners when testIsolation=legacy', { testIsolation: 'legacy' }, () => { - it('removes 2x for about:blank and first url visit', () => { + if (!Cypress.config('experimentalSessionAndOrigin')) { + it('removes window:load listeners 2x for about:blank and first url visit when experimentalSessionAndOrigin=false', () => { const listeners = cy.listeners('window:load') const winLoad = cy.spy(cy, 'once').withArgs('window:load') @@ -593,17 +593,17 @@ describe('src/cy/commands/navigation', () => { expect(cy.listeners('window:load')).to.deep.eq(listeners) }) }) - }) + } if (Cypress.config('experimentalSessionAndOrigin')) { - describe('removes window:load listeners when testIsolation=strict', () => { + describe('removes window:load listeners for first url visit when experimentalSessionAndOrigin=true', () => { it('removes for first url visit', () => { const listeners = cy.listeners('window:load') const winLoad = cy.spy(cy, 'once').withArgs('window:load') cy.visit('/fixtures/generic.html').then(() => { - expect(winLoad).to.be.calledOnce + expect(winLoad).to.be.calledOnce // once for $iframe src expect(cy.listeners('window:load')).to.deep.eq(listeners) }) }) @@ -768,46 +768,48 @@ describe('src/cy/commands/navigation', () => { cy.get('div').should('contain', 'this should fail?') }) - describe('when only hashes are changing when testIsolation=legacy', { testIsolation: 'legacy' }, () => { - it('short circuits the visit if the page will not refresh', () => { - let count = 0 - const urls = [] + if (!Cypress.config('experimentalSessionAndOrigin')) { + describe('when only hashes are changing when experimentalSessionAndOrigin=false', () => { + it('short circuits the visit if the page will not refresh', () => { + let count = 0 + const urls = [] - cy.on('window:load', () => { - urls.push(cy.state('window').location.href) + cy.on('window:load', () => { + urls.push(cy.state('window').location.href) - count += 1 - }) + count += 1 + }) - cy - // about:blank yes (1) - .visit('/fixtures/generic.html?foo#bar') // yes (2) - .visit('/fixtures/generic.html?foo#foo') // no (2) - .visit('/fixtures/generic.html?bar#bar') // yes (3) - .visit('/fixtures/dimensions.html?bar#bar') // yes (4) - .visit('/fixtures/dimensions.html?baz#bar') // yes (5) - .visit('/fixtures/dimensions.html#bar') // yes (6) - .visit('/fixtures/dimensions.html') // yes (7) - .visit('/fixtures/dimensions.html#baz') // no (7) - .visit('/fixtures/dimensions.html#') // no (7) - .then(() => { - expect(count).to.eq(7) - - expect(urls).to.deep.eq([ - 'about:blank', - 'http://localhost:3500/fixtures/generic.html?foo#bar', - 'http://localhost:3500/fixtures/generic.html?bar#bar', - 'http://localhost:3500/fixtures/dimensions.html?bar#bar', - 'http://localhost:3500/fixtures/dimensions.html?baz#bar', - 'http://localhost:3500/fixtures/dimensions.html#bar', - 'http://localhost:3500/fixtures/dimensions.html', - ]) + cy + // about:blank yes (1) + .visit('/fixtures/generic.html?foo#bar') // yes (2) + .visit('/fixtures/generic.html?foo#foo') // no (2) + .visit('/fixtures/generic.html?bar#bar') // yes (3) + .visit('/fixtures/dimensions.html?bar#bar') // yes (4) + .visit('/fixtures/dimensions.html?baz#bar') // yes (5) + .visit('/fixtures/dimensions.html#bar') // yes (6) + .visit('/fixtures/dimensions.html') // yes (7) + .visit('/fixtures/dimensions.html#baz') // no (7) + .visit('/fixtures/dimensions.html#') // no (7) + .then(() => { + expect(count).to.eq(7) + + expect(urls).to.deep.eq([ + 'about:blank', + 'http://localhost:3500/fixtures/generic.html?foo#bar', + 'http://localhost:3500/fixtures/generic.html?bar#bar', + 'http://localhost:3500/fixtures/dimensions.html?bar#bar', + 'http://localhost:3500/fixtures/dimensions.html?baz#bar', + 'http://localhost:3500/fixtures/dimensions.html#bar', + 'http://localhost:3500/fixtures/dimensions.html', + ]) + }) }) }) - }) + } if (Cypress.config('experimentalSessionAndOrigin')) { - describe('when only hashes are changing when testIsolation=strict', () => { + describe('when only hashes are changing when experimentalSessionAndOrigin=true', () => { it('short circuits the visit if the page will not refresh', () => { let count = 0 const urls = [] @@ -2588,109 +2590,111 @@ describe('src/cy/commands/navigation', () => { }) }) - describe('filters page load events when going back with window navigation when testIsolation=legacy', { testIsolation: 'legacy' }, () => { + if (!Cypress.config('experimentalSessionAndOrigin')) { + describe('filters page load events when going back with window navigation when experimentalSessionAndOrigin=false', () => { // https://github.com/cypress-io/cypress/issues/19230 - it('when going back with window navigation', () => { - const emit = cy.spy(Cypress, 'emit').log(false).withArgs('navigation:changed') - - cy - .visit('/fixtures/generic.html') - .get('#hashchange').click() - .window().then((win) => { - return new Promise((resolve) => { - cy.once('navigation:changed', resolve) + it('when going back with window navigation', () => { + const emit = cy.spy(Cypress, 'emit').log(false).withArgs('navigation:changed') - win.history.back() - }).then(() => { + cy + .visit('/fixtures/generic.html') + .get('#hashchange').click() + .window().then((win) => { return new Promise((resolve) => { cy.once('navigation:changed', resolve) - win.history.forward() - }) - }) - }) + win.history.back() + }).then(() => { + return new Promise((resolve) => { + cy.once('navigation:changed', resolve) - cy.get('#dimensions').click() - .window().then((win) => { - return new Promise((resolve) => { - cy.on('navigation:changed', (event) => { - if (event.includes('(load)')) { - resolve() - } + win.history.forward() + }) }) - - win.history.back() }) - .then(() => { + + cy.get('#dimensions').click() + .window().then((win) => { return new Promise((resolve) => { - cy.on('navigation:changed', resolve) + cy.on('navigation:changed', (event) => { + if (event.includes('(load)')) { + resolve() + } + }) + win.history.back() }) - }) - .then(() => { - expect(emit.getCall(0)).to.be.calledWith( - 'navigation:changed', - 'page navigation event (load)', - ) - - expect(emit.getCall(1)).to.be.calledWith( - 'navigation:changed', - 'page navigation event (before:load)', - ) - - expect(emit.getCall(2)).to.be.calledWith( - 'navigation:changed', - 'page navigation event (load)', - ) - - expect(emit.getCall(3)).to.be.calledWithMatch( - 'navigation:changed', - 'hashchange', - ) - - expect(emit.getCall(4)).to.be.calledWithMatch( - 'navigation:changed', - 'hashchange', - ) - - expect(emit.getCall(5)).to.be.calledWithMatch( - 'navigation:changed', - 'hashchange', - ) - - expect(emit.getCall(6)).to.be.calledWith( - 'navigation:changed', - 'page navigation event (before:load)', - ) - - expect(emit.getCall(7)).to.be.calledWith( - 'navigation:changed', - 'page navigation event (load)', - ) - - expect(emit.getCall(8)).to.be.calledWith( - 'navigation:changed', - 'page navigation event (before:load)', - ) - - expect(emit.getCall(9)).to.be.calledWith( - 'navigation:changed', - 'page navigation event (load)', - ) - - expect(emit.getCall(10)).to.be.calledWithMatch( - 'navigation:changed', - 'hashchange', - ) - - expect(emit.callCount).to.eq(11) + .then(() => { + return new Promise((resolve) => { + cy.on('navigation:changed', resolve) + win.history.back() + }) + }) + .then(() => { + expect(emit.getCall(0)).to.be.calledWith( + 'navigation:changed', + 'page navigation event (load)', + ) + + expect(emit.getCall(1)).to.be.calledWith( + 'navigation:changed', + 'page navigation event (before:load)', + ) + + expect(emit.getCall(2)).to.be.calledWith( + 'navigation:changed', + 'page navigation event (load)', + ) + + expect(emit.getCall(3)).to.be.calledWithMatch( + 'navigation:changed', + 'hashchange', + ) + + expect(emit.getCall(4)).to.be.calledWithMatch( + 'navigation:changed', + 'hashchange', + ) + + expect(emit.getCall(5)).to.be.calledWithMatch( + 'navigation:changed', + 'hashchange', + ) + + expect(emit.getCall(6)).to.be.calledWith( + 'navigation:changed', + 'page navigation event (before:load)', + ) + + expect(emit.getCall(7)).to.be.calledWith( + 'navigation:changed', + 'page navigation event (load)', + ) + + expect(emit.getCall(8)).to.be.calledWith( + 'navigation:changed', + 'page navigation event (before:load)', + ) + + expect(emit.getCall(9)).to.be.calledWith( + 'navigation:changed', + 'page navigation event (load)', + ) + + expect(emit.getCall(10)).to.be.calledWithMatch( + 'navigation:changed', + 'hashchange', + ) + + expect(emit.callCount).to.eq(11) + }) }) }) }) - }) + } if (Cypress.config('experimentalSessionAndOrigin')) { - describe('filters page load events when going back with window navigation when testIsolation=strict', () => { + describe('filters page load events when going back with window navigation when experimentalSessionAndOrigin=true', () => { // https://github.com/cypress-io/cypress/issues/19230 it('when going back with window navigation', () => { const emit = cy.spy(Cypress, 'emit').log(false).withArgs('navigation:changed') diff --git a/packages/driver/cypress/e2e/util/config.cy.js b/packages/driver/cypress/e2e/util/config.cy.js index ec5107ee5e8f..84995ab7d5d2 100644 --- a/packages/driver/cypress/e2e/util/config.cy.js +++ b/packages/driver/cypress/e2e/util/config.cy.js @@ -110,7 +110,8 @@ describe('driver/src/cypress/validate_config', () => { expect(overrideLevel).to.eq('suite') expect(() => { - validateConfig(state, { testIsolation: 'on' }) + // TODO: remove with experimentalSessionAndOrigin. Fixed with: https://github.com/cypress-io/cypress/issues/21471 + validateConfig(state, { testIsolation: Cypress.config('experimentalSessionAndOrigin') ? 'on' : 'legacy' }) }).not.to.throw() }) @@ -127,7 +128,7 @@ describe('driver/src/cypress/validate_config', () => { expect(overrideLevel).to.eq('test') expect(() => { - validateConfig(state, { testIsolation: 'strict' }) + validateConfig(state, { testIsolation: 'on' }) }).to.throw(`The \`testIsolation\` configuration can only be overridden from a suite-level override.`) }) }) @@ -192,7 +193,8 @@ describe('driver/src/cypress/validate_config', () => { }) expect(() => { - validateConfig(state, { testIsolation: 'off' }) + // TODO: remove with experimentalSessionAndOrigin. Fixed with: https://github.com/cypress-io/cypress/issues/21471 + validateConfig(state, { testIsolation: Cypress.config('experimentalSessionAndOrigin') ? 'off' : 'legacy' }) }).not.to.throw() }) diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index 63b0dba52193..b90a31bc4e8a 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -114,13 +114,12 @@ export const validateConfig = (state: State, config: Record, skipCo } config = { - testingType: Cypress.testingType, // TODO: remove with experimentalSessionAndOrigin. Fixed with: https://github.com/cypress-io/cypress/issues/21471 experimentalSessionAndOrigin: Cypress.originalConfig.experimentalSessionAndOrigin, ...config, } - validateConfigValues(config, (errResult: ErrResult | string) => { + validateConfigValues(Cypress.testingType, config, (errResult: ErrResult | string) => { const stringify = (str) => format(JSON.stringify(str)) const format = (str) => `\`${str}\`` From e494f09ed70a5ecb2b0a255e23a99f300c5899bc Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Wed, 19 Oct 2022 13:58:23 -0500 Subject: [PATCH 03/13] align with new values --- .../e2e/cypress/e2e/testConfigOverrides/invalid.js | 12 ++++++------ .../e2e/testConfigOverrides/valid-suite-only.js | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js index 8d71a9c07787..00048f9916d5 100644 --- a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js +++ b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js @@ -54,18 +54,18 @@ describe('throws error correctly when beforeEach hook', () => { }) }) -it('throws error when invalid test-level override', { testIsolation: 'legacy' }, () => { +it('throws error when invalid test-level override', { testIsolation: 'off' }, () => { shouldNotExecute() }) it('throws error when invalid config opt in Cypress.config() in test', () => { - Cypress.config({ testIsolation: 'legacy' }) + Cypress.config({ testIsolation: 'off' }) shouldNotExecute() }) describe('throws error when invalid config opt in Cypress.config() in before hook', () => { before(() => { - Cypress.config({ testIsolation: 'legacy' }) + Cypress.config({ testIsolation: 'off' }) }) it('4', () => { @@ -75,7 +75,7 @@ describe('throws error when invalid config opt in Cypress.config() in before hoo describe('throws error when invalid config opt in Cypress.config() in beforeEach hook', () => { beforeEach(() => { - Cypress.config({ testIsolation: 'legacy' }) + Cypress.config({ testIsolation: 'off' }) }) it('5', () => { @@ -85,7 +85,7 @@ describe('throws error when invalid config opt in Cypress.config() in beforeEach describe('throws error when invalid config opt in Cypress.config() in after hook', () => { after(() => { - Cypress.config({ testIsolation: 'legacy' }) + Cypress.config({ testIsolation: 'off' }) }) it('5', () => { @@ -95,7 +95,7 @@ describe('throws error when invalid config opt in Cypress.config() in after hook describe('throws error when invalid config opt in Cypress.config() in afterEach hook', () => { afterEach(() => { - Cypress.config({ testIsolation: 'legacy' }) + Cypress.config({ testIsolation: 'off' }) }) it('5', () => { diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/valid-suite-only.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/valid-suite-only.js index 48f3310abe60..8c01f5880935 100644 --- a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/valid-suite-only.js +++ b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/valid-suite-only.js @@ -1,4 +1,4 @@ -describe('suite-level-only overrides run as expected', { testIsolation: 'legacy' }, () => { +describe('suite-level-only overrides run as expected', { testIsolation: 'off' }, () => { it('1st test passes', () => { cy.visit('https://example.cypress.io') }) @@ -13,7 +13,7 @@ describe('suite-level-only overrides run as expected', { testIsolation: 'legacy' }) describe('nested contexts ', () => { - describe('nested suite-level-only overrides run as expected', { testIsolation: 'legacy' }, () => { + describe('nested suite-level-only overrides run as expected', { testIsolation: 'off' }, () => { it('1st test passes', () => { cy.visit('https://example.cypress.io') }) From 9b4ad09ecdf4274ba4d97be822838867e0521e91 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Wed, 19 Oct 2022 15:36:42 -0500 Subject: [PATCH 04/13] fix --- cli/types/cypress.d.ts | 2 +- packages/config/src/browser.ts | 2 +- packages/config/src/options.ts | 2 +- packages/config/src/project/index.ts | 2 +- packages/config/src/project/utils.ts | 10 +++++++--- packages/config/test/index.spec.ts | 8 ++++---- packages/data-context/src/data/ProjectConfigManager.ts | 6 +++--- packages/driver/src/util/config.ts | 4 ++-- 8 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index fbd2a2048f73..09490cefdb1f 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2808,7 +2808,7 @@ declare namespace Cypress { */ supportFile: string | false /** - * The test isolation ensures a clean browser context between tests for end-to-end tests. + * The test isolation ensures a clean browser context between tests. * * Cypress will always resets/clears aliases, intercepts, clock, and viewport before each test * to ensure a clean test slate; i.e. this configuration only impacts the browser context. diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index 99f62876db41..7ad5069b9cc9 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -155,7 +155,7 @@ export const matchesConfigKey = (key: string) => { return } -export const validate = (testingType: TestingType, cfg: any, onErr: (property: ErrResult | string) => void) => { +export const validate = (cfg: any, onErr: (property: ErrResult | string) => void, testingType?: TestingType) => { debug('validating configuration') return _.each(cfg, (value, key) => { diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 5cdebe444876..8083c58ac240 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -382,7 +382,7 @@ const driverConfigOptions: Array = [ return 'on' } - return options?.e2e?.experimentalSessionAndOrigin ? 'on' : 'legacy' + return options?.experimentalSessionAndOrigin || options?.config?.e2e?.experimentalSessionAndOrigin ? 'on' : 'legacy' }, validation: (key: string, value: any, { testingType, experimentalSessionAndOrigin }: { testingType: TestingType, experimentalSessionAndOrigin: boolean }) => { if (testingType === 'component') { diff --git a/packages/config/src/project/index.ts b/packages/config/src/project/index.ts index d582ad856bd2..00e8984651e1 100644 --- a/packages/config/src/project/index.ts +++ b/packages/config/src/project/index.ts @@ -60,7 +60,7 @@ export function updateWithPluginValues (cfg: FullConfig, modifiedConfig: any, te } return errors.throwErr('CONFIG_VALIDATION_ERROR', 'configFile', configFile, validationResult) - }) + }, testingType) debug('validate that there is no breaking config options added by setupNodeEvents') diff --git a/packages/config/src/project/utils.ts b/packages/config/src/project/utils.ts index c68d93b871a8..b4e36634f664 100644 --- a/packages/config/src/project/utils.ts +++ b/packages/config/src/project/utils.ts @@ -401,7 +401,11 @@ export function mergeDefaults ( config.baseUrl = url.replace(/\/\/+$/, '/') } - const defaultsForRuntime = getDefaultValues(config) + const defaultsForRuntime = getDefaultValues({ + ...options, + // TODO: clean this up + experimentalSessionAndOrigin: config.experimentalSessionAndOrigin, + }) _.defaultsDeep(config, defaultsForRuntime) @@ -447,14 +451,14 @@ export function mergeDefaults ( // validate config again here so that we catch configuration errors coming // from the CLI overrides or env var overrides - validate(testingType, _.omit(config, 'browsers'), (validationResult: ConfigValidationFailureInfo | string) => { + validate(_.omit(config, 'browsers'), (validationResult: ConfigValidationFailureInfo | string) => { // return errors.throwErr('CONFIG_VALIDATION_ERROR', errMsg) if (_.isString(validationResult)) { return errors.throwErr('CONFIG_VALIDATION_MSG_ERROR', null, null, validationResult) } return errors.throwErr('CONFIG_VALIDATION_ERROR', null, null, validationResult) - }) + }, testingType) config = setAbsolutePaths(config) diff --git a/packages/config/test/index.spec.ts b/packages/config/test/index.spec.ts index c1db20fe29a9..8f9cd0ebf3eb 100644 --- a/packages/config/test/index.spec.ts +++ b/packages/config/test/index.spec.ts @@ -116,9 +116,9 @@ describe('config/src/index', () => { it('validates config', () => { const errorFn = sinon.spy() - configUtil.validate('e2e', { + configUtil.validate({ 'baseUrl': 'https://', - }, errorFn) + }, errorFn, 'e2e') expect(errorFn).to.have.callCount(0) }) @@ -126,9 +126,9 @@ describe('config/src/index', () => { it('calls error callback if config is invalid', () => { const errorFn = sinon.spy() - configUtil.validate('e2e', { + configUtil.validate({ 'baseUrl': ' ', - }, errorFn) + }, errorFn, 'e2e') expect(errorFn).to.have.been.calledWithMatch({ key: 'baseUrl' }) expect(errorFn).to.have.been.calledWithMatch({ type: 'a fully qualified URL (starting with `http://` or `https://`)' }) diff --git a/packages/data-context/src/data/ProjectConfigManager.ts b/packages/data-context/src/data/ProjectConfigManager.ts index f526e0f6d016..0a0179b3e003 100644 --- a/packages/data-context/src/data/ProjectConfigManager.ts +++ b/packages/data-context/src/data/ProjectConfigManager.ts @@ -50,7 +50,7 @@ export class ProjectConfigManager { private _pathToWatcherRecord: Record = {} private _watchers = new Set() private _registeredEventsTarget: TestingType | undefined - private _testingType: TestingType | null = null + private _testingType?: TestingType private _state: ConfigManagerState = 'pending' private _loadConfigPromise: Promise | undefined private _cachedLoadConfig: LoadConfigReply | undefined @@ -361,13 +361,13 @@ export class ProjectConfigManager { } private validateConfigFile (file: string | false, config: Cypress.ConfigOptions) { - validateConfig(this._testingType, config, (errMsg) => { + validateConfig(config, (errMsg) => { if (_.isString(errMsg)) { throw getError('CONFIG_VALIDATION_MSG_ERROR', 'configFile', file || null, errMsg) } throw getError('CONFIG_VALIDATION_ERROR', 'configFile', file || null, errMsg) - }) + }, this._testingType) return validateNoBreakingConfigLaunchpad( config, diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index b90a31bc4e8a..6aac7cf207ad 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -119,7 +119,7 @@ export const validateConfig = (state: State, config: Record, skipCo ...config, } - validateConfigValues(Cypress.testingType, config, (errResult: ErrResult | string) => { + validateConfigValues(config, (errResult: ErrResult | string) => { const stringify = (str) => format(JSON.stringify(str)) const format = (str) => `\`${str}\`` @@ -132,5 +132,5 @@ export const validateConfig = (state: State, config: Record, skipCo : `Expected ${format(errResult.key)} to be ${errResult.type}.\n\nInstead the value was: ${stringify(errResult.value)}` throw new (state('specWindow').Error)(errMsg) - }) + }, Cypress.testingType) } From ce40bc17eb232feffe1e524c220fa0202cbbd54e Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Wed, 19 Oct 2022 17:00:57 -0500 Subject: [PATCH 05/13] fix types --- packages/config/src/browser.ts | 2 +- packages/data-context/src/data/ProjectConfigManager.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index 7ad5069b9cc9..7f20d1adfc28 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -155,7 +155,7 @@ export const matchesConfigKey = (key: string) => { return } -export const validate = (cfg: any, onErr: (property: ErrResult | string) => void, testingType?: TestingType) => { +export const validate = (cfg: any, onErr: (property: ErrResult | string) => void, testingType: TestingType | null) => { debug('validating configuration') return _.each(cfg, (value, key) => { diff --git a/packages/data-context/src/data/ProjectConfigManager.ts b/packages/data-context/src/data/ProjectConfigManager.ts index 0a0179b3e003..a6eca63afd06 100644 --- a/packages/data-context/src/data/ProjectConfigManager.ts +++ b/packages/data-context/src/data/ProjectConfigManager.ts @@ -50,7 +50,7 @@ export class ProjectConfigManager { private _pathToWatcherRecord: Record = {} private _watchers = new Set() private _registeredEventsTarget: TestingType | undefined - private _testingType?: TestingType + private _testingType: TestingType | null = null private _state: ConfigManagerState = 'pending' private _loadConfigPromise: Promise | undefined private _cachedLoadConfig: LoadConfigReply | undefined From 7cf87423cecc560599be05d747d7cd273ba3dc4a Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Wed, 19 Oct 2022 17:04:45 -0500 Subject: [PATCH 06/13] fix system test --- system-tests/test/testConfigOverrides_spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/system-tests/test/testConfigOverrides_spec.ts b/system-tests/test/testConfigOverrides_spec.ts index f5a1f08876b1..6bab3c9e506d 100644 --- a/system-tests/test/testConfigOverrides_spec.ts +++ b/system-tests/test/testConfigOverrides_spec.ts @@ -16,6 +16,7 @@ describe('testConfigOverrides', () => { expectedExitCode: 0, browser: 'electron', config: { + experimentalSessionAndOrigin: true, video: false, }, }) @@ -25,6 +26,7 @@ describe('testConfigOverrides', () => { snapshot: true, expectedExitCode: 1, config: { + experimentalSessionAndOrigin: true, video: false, }, }) From 7cf70d7ce8ae01eafedb692a59473857a4d64a2e Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Thu, 20 Oct 2022 10:33:33 -0500 Subject: [PATCH 07/13] Update cli/types/cypress.d.ts Co-authored-by: Matt Henkes --- cli/types/cypress.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index 09490cefdb1f..42e14c4d739a 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2813,7 +2813,7 @@ declare namespace Cypress { * Cypress will always resets/clears aliases, intercepts, clock, and viewport before each test * to ensure a clean test slate; i.e. this configuration only impacts the browser context. * - * Note: the `cy.session()` command will inherent this value to determine where or not the page is + * Note: the `cy.session()` command will inherent this value to determine whether or not the page is * cleared when the command executes. * * Options: From b595ea3b3321036a4e3a238d432639f3d5c65951 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Fri, 21 Oct 2022 12:16:35 -0500 Subject: [PATCH 08/13] I think this was what this should be --- .../testConfigOverrides_spec.ts.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index 0468a0556b3d..a1e7e6387a84 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -5,10 +5,11 @@ exports['testConfigOverrides / fails when passing invalid config value browser'] (Run Starting) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (invalid-browser.js) │ - │ Searched: cypress/e2e/testConfigOverrides/invalid-browser.js │ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (invalid-browser.js) │ + │ Searched: cypress/e2e/testConfigOverrides/invalid-browser.js │ + │ Experiments: experimentalSessionAndOrigin=true │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -1195,10 +1196,11 @@ exports['testConfigOverrides / successfully runs valid suite-level-only override (Run Starting) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (valid-suite-only.js) │ - │ Searched: cypress/e2e/testConfigOverrides/valid-suite-only.js │ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (valid-suite-only.js) │ + │ Searched: cypress/e2e/testConfigOverrides/valid-suite-only.js │ + │ Experiments: experimentalSessionAndOrigin=true │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ From d8a949ad2598306da46ae2a3db806917aad11b71 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 24 Oct 2022 09:14:48 -0500 Subject: [PATCH 09/13] PR feedback --- cli/types/cypress.d.ts | 34 ++++++++++--------- packages/config/src/browser.ts | 1 + packages/config/src/project/utils.ts | 2 +- .../testConfigOverrides_spec.ts.js | 9 +++-- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index 42e14c4d739a..5809c31b5f96 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2813,29 +2813,31 @@ declare namespace Cypress { * Cypress will always resets/clears aliases, intercepts, clock, and viewport before each test * to ensure a clean test slate; i.e. this configuration only impacts the browser context. * - * Note: the `cy.session()` command will inherent this value to determine whether or not the page is - * cleared when the command executes. + * Note: the [`cy.session()`](https://on.cypress.io/session) command will inherent this value to determine whether + * or not the page is cleared when the command executes. This command is only available in end-to-end testing. * * Options: * When experimentalSessionAndOrigin=false * - legacy - "Pass through" to document the default Cypress test isolation behavior. The page is not cleared - * in end-to-end tests and cookies, local storage and session storage in the current domains are - * cleared before each test. The `cy.session()` command is not available. + * before each end-to-end test. Cookies, local storage and session storage in the current domains are + * cleared before each test. The `cy.session()` command is not available for end-to-end without this experiment. * NOTE: this behavior will be changed in the next major release and the new default will be 'on'. * * When experimentalSessionAndOrigin=true - * - on - The page is cleared and cookies, local storage and session storage in all domains are cleared before each - * test. The `cy.session()` command will also clear the page and current browser context when establishing - * the browser session. - * - off - The current browser state will persist between tests. The page does not clear and cookies, local storage and - * session storage will be available in the next test. The `cy.session()` command will only clear the current - * browser context when establishing the browser session - the current page will not clear. - * - * NOTES: - * - Turning test isolation off may improve performance of end-to-end tests, however, previous tests could impact the - * browser state of the next test and cause inconsistency when using .only(). - * - The page is always cleared in component testing to ensure a clean page for rendering isolated components. - * + * - on - The page is cleared before each test. Cookies, local storage and session storage in all domains are cleared + * before each test. The `cy.session()` command will also clear the page and current browser context when creating + * or restoring the browser session. + * - off - The current browser state will persist between tests. The page does not clear before the test and cookies, local + * storage and session storage will be available in the next test. The `cy.session()` command will only clear the + * current browser context when creating or restoring the browser session - the current page will not clear. + * + * Tradeoffs: + * Turning test isolation off may improve performance of end-to-end tests, however, previous tests could impact the + * browser state of the next test and cause inconsistency when using .only(). Be mindful to write isolated tests when + * test isolation is off. If a test in the suite impacts the state of other tests and it were to fail, you could see + * misleading errors in later tests which makes debugging clunky. See the [documentation](https://on.cypress.io/test-isolation) + * for more information. + * * @default 'legacy', when running end-to-end tests. When running component tests or running end-to-end tests with * experimentalSessionAndOrigin=true, the default is 'on'. */ diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index 7f20d1adfc28..d9b85e52a1e5 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -204,6 +204,7 @@ export const validateOverridableAtRunTime = (config: any, isSuiteLevelOverride: } // TODO: add a hook to ensure valid testing-type configuration is being set at runtime for all configuration values. + // https://github.com/cypress-io/cypress/issues/24365 if (overrideLevel === 'never' || (overrideLevel === 'suite' && !isSuiteLevelOverride)) { onErr({ diff --git a/packages/config/src/project/utils.ts b/packages/config/src/project/utils.ts index b4e36634f664..67418a2bd975 100644 --- a/packages/config/src/project/utils.ts +++ b/packages/config/src/project/utils.ts @@ -403,7 +403,7 @@ export function mergeDefaults ( const defaultsForRuntime = getDefaultValues({ ...options, - // TODO: clean this up + // TODO: clean this up. Fixed with: https://github.com/cypress-io/cypress/issues/21471 experimentalSessionAndOrigin: config.experimentalSessionAndOrigin, }) diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index a1e7e6387a84..f1a830a41cac 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -5,11 +5,10 @@ exports['testConfigOverrides / fails when passing invalid config value browser'] (Run Starting) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (invalid-browser.js) │ - │ Searched: cypress/e2e/testConfigOverrides/invalid-browser.js │ - │ Experiments: experimentalSessionAndOrigin=true │ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (invalid-browser.js) │ + │ Searched: cypress/e2e/testConfigOverrides/invalid-browser.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ From eba45db8683edbb5128d8c60a0cae5cada7ae62d Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 24 Oct 2022 09:54:20 -0500 Subject: [PATCH 10/13] correct link error & system test snapshot --- cli/types/cypress.d.ts | 2 +- .../__snapshots__/testConfigOverrides_spec.ts.js | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index 5809c31b5f96..f72633b938a0 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2837,7 +2837,7 @@ declare namespace Cypress { * test isolation is off. If a test in the suite impacts the state of other tests and it were to fail, you could see * misleading errors in later tests which makes debugging clunky. See the [documentation](https://on.cypress.io/test-isolation) * for more information. - * + * * @default 'legacy', when running end-to-end tests. When running component tests or running end-to-end tests with * experimentalSessionAndOrigin=true, the default is 'on'. */ diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index f1a830a41cac..a1e7e6387a84 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -5,10 +5,11 @@ exports['testConfigOverrides / fails when passing invalid config value browser'] (Run Starting) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (invalid-browser.js) │ - │ Searched: cypress/e2e/testConfigOverrides/invalid-browser.js │ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (invalid-browser.js) │ + │ Searched: cypress/e2e/testConfigOverrides/invalid-browser.js │ + │ Experiments: experimentalSessionAndOrigin=true │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ From 8aadb3ad780278e5535c59af1315bf4a43d59e14 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 24 Oct 2022 11:38:00 -0500 Subject: [PATCH 11/13] remove legacy mode and default to null --- cli/types/cypress.d.ts | 28 ++++++--------- .../config/__snapshots__/index.spec.ts.js | 4 +-- packages/config/src/options.ts | 35 +++++++++++++------ packages/config/test/project/utils.spec.ts | 4 +-- packages/driver/cypress/e2e/util/config.cy.js | 4 +-- 5 files changed, 41 insertions(+), 34 deletions(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index f72633b938a0..4df2ef8c16d4 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2808,7 +2808,8 @@ declare namespace Cypress { */ supportFile: string | false /** - * The test isolation ensures a clean browser context between tests. + * The test isolation ensures a clean browser context between tests. This option is only available when + * `experimentalSessionAndOrigin=true`. * * Cypress will always resets/clears aliases, intercepts, clock, and viewport before each test * to ensure a clean test slate; i.e. this configuration only impacts the browser context. @@ -2816,20 +2817,12 @@ declare namespace Cypress { * Note: the [`cy.session()`](https://on.cypress.io/session) command will inherent this value to determine whether * or not the page is cleared when the command executes. This command is only available in end-to-end testing. * - * Options: - * When experimentalSessionAndOrigin=false - * - legacy - "Pass through" to document the default Cypress test isolation behavior. The page is not cleared - * before each end-to-end test. Cookies, local storage and session storage in the current domains are - * cleared before each test. The `cy.session()` command is not available for end-to-end without this experiment. - * NOTE: this behavior will be changed in the next major release and the new default will be 'on'. - * - * When experimentalSessionAndOrigin=true - * - on - The page is cleared before each test. Cookies, local storage and session storage in all domains are cleared - * before each test. The `cy.session()` command will also clear the page and current browser context when creating - * or restoring the browser session. - * - off - The current browser state will persist between tests. The page does not clear before the test and cookies, local - * storage and session storage will be available in the next test. The `cy.session()` command will only clear the - * current browser context when creating or restoring the browser session - the current page will not clear. + * - on - The page is cleared before each test. Cookies, local storage and session storage in all domains are cleared + * before each test. The `cy.session()` command will also clear the page and current browser context when creating + * or restoring the browser session. + * - off - The current browser state will persist between tests. The page does not clear before the test and cookies, local + * storage and session storage will be available in the next test. The `cy.session()` command will only clear the + * current browser context when creating or restoring the browser session - the current page will not clear. * * Tradeoffs: * Turning test isolation off may improve performance of end-to-end tests, however, previous tests could impact the @@ -2838,10 +2831,9 @@ declare namespace Cypress { * misleading errors in later tests which makes debugging clunky. See the [documentation](https://on.cypress.io/test-isolation) * for more information. * - * @default 'legacy', when running end-to-end tests. When running component tests or running end-to-end tests with - * experimentalSessionAndOrigin=true, the default is 'on'. + * @default null, when experimentalSessionAndOrigin=false. The default is 'on' when experimentalSessionAndOrigin=true. */ - testIsolation: 'legacy' | 'on' | 'off' + testIsolation: null | 'on' | 'off' /** * Path to folder where videos will be saved after a headless or CI run * @default "cypress/videos" diff --git a/packages/config/__snapshots__/index.spec.ts.js b/packages/config/__snapshots__/index.spec.ts.js index 286c8a73ad0a..e5a96d011446 100644 --- a/packages/config/__snapshots__/index.spec.ts.js +++ b/packages/config/__snapshots__/index.spec.ts.js @@ -70,7 +70,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys 1 "supportFile": "cypress/support/e2e.{js,jsx,ts,tsx}", "supportFolder": false, "taskTimeout": 60000, - "testIsolation": "legacy", + "testIsolation": null, "trashAssetsBeforeRuns": true, "userAgent": null, "video": true, @@ -155,7 +155,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f "supportFile": "cypress/support/e2e.{js,jsx,ts,tsx}", "supportFolder": false, "taskTimeout": 60000, - "testIsolation": "legacy", + "testIsolation": null, "trashAssetsBeforeRuns": true, "userAgent": null, "video": true, diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 8083c58ac240..c3683c923614 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -31,6 +31,11 @@ const BREAKING_OPTION_ERROR_KEY: Readonly = [ 'TEST_FILES_RENAMED', ] as const +type ValidationOptions = { + testingType: TestingType | null + experimentalSessionAndOrigin: boolean +} + export type BreakingOptionErrorKey = typeof BREAKING_OPTION_ERROR_KEY[number] export type OverrideLevel = 'any' | 'suite' | 'never' @@ -90,7 +95,7 @@ export interface BreakingOption { showInLaunchpad?: boolean } -const isValidConfig = (testingType: string, config: any) => { +const isValidConfig = (testingType: string, config: any, opts: ValidationOptions) => { const status = validate.isPlainObject(testingType, config) if (status !== true) { @@ -99,7 +104,7 @@ const isValidConfig = (testingType: string, config: any) => { for (const rule of options) { if (rule.name in config && rule.validation) { - const status = rule.validation(`${testingType}.${rule.name}`, config[rule.name]) + const status = rule.validation(`${testingType}.${rule.name}`, config[rule.name], opts) if (status !== true) { return status @@ -374,26 +379,36 @@ const driverConfigOptions: Array = [ name: 'testIsolation', // TODO: https://github.com/cypress-io/cypress/issues/23093 // When experimentalSessionAndOrigin is removed and released as GA, - // update the defaultValue from 'legacy' to 'on' and + // update the defaultValue from undefined to 'on' and // update this code to remove the check/override specific to enable // 'on' by default when experimentalSessionAndOrigin=true defaultValue: (options: Record = {}) => { if (options.testingType === 'component') { - return 'on' + return null } - return options?.experimentalSessionAndOrigin || options?.config?.e2e?.experimentalSessionAndOrigin ? 'on' : 'legacy' + return options?.experimentalSessionAndOrigin || options?.config?.e2e?.experimentalSessionAndOrigin ? 'on' : null }, - validation: (key: string, value: any, { testingType, experimentalSessionAndOrigin }: { testingType: TestingType, experimentalSessionAndOrigin: boolean }) => { - if (testingType === 'component') { - return validate.isOneOf('on')(key, value) + validation: (key: string, value: any, opts: ValidationOptions) => { + const { testingType, experimentalSessionAndOrigin } = opts + + if (testingType == null || testingType === 'component') { + return true } - if (experimentalSessionAndOrigin) { + if (experimentalSessionAndOrigin && testingType === 'e2e') { return validate.isOneOf('on', 'off')(key, value) } - return validate.isOneOf('legacy')(key, value) + if (value == null) { + return true + } + + return { + key, + value, + type: 'not set unless the experimentalSessionAndOrigin flag is turned on', + } }, overrideLevel: 'suite', }, { diff --git a/packages/config/test/project/utils.spec.ts b/packages/config/test/project/utils.spec.ts index 49d42736b97c..75d01a011331 100644 --- a/packages/config/test/project/utils.spec.ts +++ b/packages/config/test/project/utils.spec.ts @@ -1074,7 +1074,7 @@ describe('config/src/project/utils', () => { supportFile: { value: false, from: 'config' }, supportFolder: { value: false, from: 'default' }, taskTimeout: { value: 60000, from: 'default' }, - testIsolation: { value: 'legacy', from: 'default' }, + testIsolation: { value: null, from: 'default' }, trashAssetsBeforeRuns: { value: true, from: 'default' }, userAgent: { value: null, from: 'default' }, video: { value: true, from: 'default' }, @@ -1190,7 +1190,7 @@ describe('config/src/project/utils', () => { supportFile: { value: false, from: 'config' }, supportFolder: { value: false, from: 'default' }, taskTimeout: { value: 60000, from: 'default' }, - testIsolation: { value: 'legacy', from: 'default' }, + testIsolation: { value: null, from: 'default' }, trashAssetsBeforeRuns: { value: true, from: 'default' }, userAgent: { value: null, from: 'default' }, video: { value: true, from: 'default' }, diff --git a/packages/driver/cypress/e2e/util/config.cy.js b/packages/driver/cypress/e2e/util/config.cy.js index 84995ab7d5d2..5c16a9a39bc4 100644 --- a/packages/driver/cypress/e2e/util/config.cy.js +++ b/packages/driver/cypress/e2e/util/config.cy.js @@ -111,7 +111,7 @@ describe('driver/src/cypress/validate_config', () => { expect(() => { // TODO: remove with experimentalSessionAndOrigin. Fixed with: https://github.com/cypress-io/cypress/issues/21471 - validateConfig(state, { testIsolation: Cypress.config('experimentalSessionAndOrigin') ? 'on' : 'legacy' }) + validateConfig(state, { testIsolation: Cypress.config('experimentalSessionAndOrigin') ? 'on' : null }) }).not.to.throw() }) @@ -194,7 +194,7 @@ describe('driver/src/cypress/validate_config', () => { expect(() => { // TODO: remove with experimentalSessionAndOrigin. Fixed with: https://github.com/cypress-io/cypress/issues/21471 - validateConfig(state, { testIsolation: Cypress.config('experimentalSessionAndOrigin') ? 'off' : 'legacy' }) + validateConfig(state, { testIsolation: Cypress.config('experimentalSessionAndOrigin') ? 'off' : null }) }).not.to.throw() }) From ece4eb89a88b4759c26bf3aa96003cd9826a2ee0 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 24 Oct 2022 11:39:23 -0500 Subject: [PATCH 12/13] Update cli/types/cypress.d.ts Co-authored-by: Bill Glesias --- cli/types/cypress.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index 4df2ef8c16d4..a6ed3a74d577 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2811,7 +2811,7 @@ declare namespace Cypress { * The test isolation ensures a clean browser context between tests. This option is only available when * `experimentalSessionAndOrigin=true`. * - * Cypress will always resets/clears aliases, intercepts, clock, and viewport before each test + * Cypress will always reset/clear aliases, intercepts, clock, and viewport before each test * to ensure a clean test slate; i.e. this configuration only impacts the browser context. * * Note: the [`cy.session()`](https://on.cypress.io/session) command will inherent this value to determine whether From c1b05b8d0adf46fe732499021a61ff4642f9fa8b Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 24 Oct 2022 16:32:27 -0500 Subject: [PATCH 13/13] okay webkit. --- .../__snapshots__/testConfigOverrides_spec.ts.js | 9 ++++----- system-tests/test/testConfigOverrides_spec.ts | 1 - 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index a1e7e6387a84..f1a830a41cac 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -5,11 +5,10 @@ exports['testConfigOverrides / fails when passing invalid config value browser'] (Run Starting) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (invalid-browser.js) │ - │ Searched: cypress/e2e/testConfigOverrides/invalid-browser.js │ - │ Experiments: experimentalSessionAndOrigin=true │ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (invalid-browser.js) │ + │ Searched: cypress/e2e/testConfigOverrides/invalid-browser.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ diff --git a/system-tests/test/testConfigOverrides_spec.ts b/system-tests/test/testConfigOverrides_spec.ts index 6bab3c9e506d..b5bafc733c5f 100644 --- a/system-tests/test/testConfigOverrides_spec.ts +++ b/system-tests/test/testConfigOverrides_spec.ts @@ -26,7 +26,6 @@ describe('testConfigOverrides', () => { snapshot: true, expectedExitCode: 1, config: { - experimentalSessionAndOrigin: true, video: false, }, })