From 28380a550a3e7ce358a0c3237d8a18eb43e37d45 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Fri, 29 Jul 2022 08:46:56 -0500 Subject: [PATCH 01/35] feat: add testIsolation config option and add support for varying config override levels during run-time --- .../config/__snapshots__/index.spec.ts.js | 15 +- packages/config/src/browser.ts | 42 ++- packages/config/src/options.ts | 184 ++++------ packages/config/test/index.spec.ts | 64 +++- .../cypress/e2e/commands/local_storage.cy.js | 7 +- .../e2e/commands/sessions/sessions.cy.js | 48 +++ .../cypress/e2e/e2e/testConfigOverrides.cy.js | 6 +- packages/driver/src/cy/commands/cookies.ts | 20 +- .../driver/src/cy/commands/local_storage.ts | 25 +- .../driver/src/cy/commands/sessions/index.ts | 5 +- packages/driver/src/cy/testConfigOverrides.ts | 37 +- packages/driver/src/cypress.ts | 27 +- packages/driver/src/cypress/cy.ts | 7 +- packages/driver/src/cypress/error_messages.ts | 10 +- packages/driver/src/util/config.ts | 4 +- packages/server/lib/config.ts | 41 ++- packages/server/lib/util/spec_writer.ts | 1 + packages/types/src/config.ts | 11 +- .../testConfigOverrides_spec.ts.js | 326 +++++++++--------- .../before-invalid.js} | 0 .../invalid-browser.js} | 0 .../invalid-suite-level-override.js} | 0 .../e2e/testConfigOverrides/invalid.js | 55 +++ .../only-invalid.js} | 0 .../skip-browser.js} | 0 system-tests/test/testConfigOverrides_spec.ts | 10 +- 26 files changed, 549 insertions(+), 396 deletions(-) rename system-tests/projects/e2e/cypress/e2e/{testConfigOverrides-before-invalid.js => testConfigOverrides/before-invalid.js} (100%) rename system-tests/projects/e2e/cypress/e2e/{testConfigOverrides-invalid-browser.js => testConfigOverrides/invalid-browser.js} (100%) rename system-tests/projects/e2e/cypress/e2e/{testConfigOverrides-invalid.js => testConfigOverrides/invalid-suite-level-override.js} (100%) create mode 100644 system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js rename system-tests/projects/e2e/cypress/e2e/{testConfigOverrides-only-invalid.js => testConfigOverrides/only-invalid.js} (100%) rename system-tests/projects/e2e/cypress/e2e/{testConfigOverrides-skip-browser.js => testConfigOverrides/skip-browser.js} (100%) diff --git a/packages/config/__snapshots__/index.spec.ts.js b/packages/config/__snapshots__/index.spec.ts.js index cf78cb0f86f2..507b2488df46 100644 --- a/packages/config/__snapshots__/index.spec.ts.js +++ b/packages/config/__snapshots__/index.spec.ts.js @@ -67,6 +67,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys 1 "supportFile": "cypress/support/e2e.{js,jsx,ts,tsx}", "supportFolder": false, "taskTimeout": 60000, + "testIsolation": "default", "trashAssetsBeforeRuns": true, "userAgent": null, "video": true, @@ -77,12 +78,13 @@ exports['config/src/index .getDefaultValues returns list of public config keys 1 "viewportWidth": 1000, "waitForAnimations": true, "watchForFileChanges": true, + "specPattern": "cypress/e2e/**/*.cy.{js,jsx,ts,tsx}", "additionalIgnorePattern": [], "autoOpen": false, "browsers": [], "clientRoute": "/__/", "configFile": "cypress.config.js", - "cypressBinaryRoot": "/root/cypress", + "cypressBinaryRoot": "/Users/emily/dev/cypress", "devServerPublicPathRoute": "/__cypress/src", "hosts": null, "isInteractive": true, @@ -146,6 +148,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f "supportFile": "cypress/support/e2e.{js,jsx,ts,tsx}", "supportFolder": false, "taskTimeout": 60000, + "testIsolation": "default", "trashAssetsBeforeRuns": true, "userAgent": null, "video": true, @@ -156,12 +159,13 @@ exports['config/src/index .getDefaultValues returns list of public config keys f "viewportWidth": 1000, "waitForAnimations": true, "watchForFileChanges": true, + "specPattern": "cypress/e2e/**/*.cy.{js,jsx,ts,tsx}", "additionalIgnorePattern": [], "autoOpen": false, "browsers": [], "clientRoute": "/__/", "configFile": "cypress.config.js", - "cypressBinaryRoot": "/root/cypress", + "cypressBinaryRoot": "/Users/emily/dev/cypress", "devServerPublicPathRoute": "/__cypress/src", "hosts": null, "isInteractive": true, @@ -172,8 +176,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f "socketId": null, "socketIoCookie": "__socket", "socketIoRoute": "/__socket", - "xhrRoute": "/xhrs/", - "specPattern": "cypress/e2e/**/*.cy.{js,jsx,ts,tsx}" + "xhrRoute": "/xhrs/" } exports['config/src/index .getPublicConfigKeys returns list of public config keys 1'] = [ @@ -221,6 +224,7 @@ exports['config/src/index .getPublicConfigKeys returns list of public config key "supportFile", "supportFolder", "taskTimeout", + "testIsolation", "trashAssetsBeforeRuns", "userAgent", "video", @@ -235,6 +239,5 @@ exports['config/src/index .getPublicConfigKeys returns list of public config key "browsers", "hosts", "isInteractive", - "modifyObstructiveCode", - "specPattern" + "modifyObstructiveCode" ] diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index f269f189352a..f0dd2909b528 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -1,7 +1,15 @@ import _ from 'lodash' import Debug from 'debug' -import { defaultSpecPattern, options, breakingOptions, breakingRootOptions, testingTypeBreakingOptions, additionalOptionsToResolveConfig } from './options' -import type { BreakingOption, BreakingOptionErrorKey } from './options' +import { + defaultSpecPattern, + options, + OVERRIDE_LEVELS, + breakingOptions, + breakingRootOptions, + testingTypeBreakingOptions, +} from './options' + +import type { BreakingOption, BreakingOptionErrorKey, OverrideLevel } from './options' import type { TestingType } from '@packages/types' // this export has to be done in 2 lines because of a bug in babel typescript @@ -21,10 +29,12 @@ const debug = Debug('cypress:config:browser') const dashesOrUnderscoresRe = /^(_-)+/ // takes an array and creates an index object of [keyKey]: [valueKey] -function createIndex> (arr: Array, keyKey: keyof T, valueKey: keyof T) { +function createIndex> (arr: Array, keyKey: keyof T, valueKey: keyof T, defaultValueFallback?: any) { return _.reduce(arr, (memo: Record, item) => { if (item[valueKey] !== undefined) { memo[item[keyKey] as string] = item[valueKey] + } else { + memo[item[keyKey] as string] = defaultValueFallback } return memo @@ -33,9 +43,9 @@ function createIndex> (arr: Array, keyKey: keyo const breakingKeys = _.map(breakingOptions, 'name') const defaultValues = createIndex(options, 'name', 'defaultValue') -const publicConfigKeys = _([...options, ...additionalOptionsToResolveConfig]).reject({ isInternal: true }).map('name').value() +const publicConfigKeys = _(options).reject({ isInternal: true }).map('name').value() const validationRules = createIndex(options, 'name', 'validation') -const testConfigOverrideOptions = createIndex(options, 'name', 'canUpdateDuringTestTime') +const testOverrideLevels = createIndex(options, 'name', 'overrideLevels', 'never') const restartOnChangeOptionsKeys = _.filter(options, 'requireRestartOnChange') const issuedWarnings = new Set() @@ -172,16 +182,22 @@ export const validateNoBreakingTestingTypeConfig = (cfg: any, testingType: keyof return validateNoBreakingOptions(options, cfg, onWarning, onErr, testingType) } -export const validateNoReadOnlyConfig = (config: any, onErr: (property: string) => void) => { - let errProperty +export const validateOverridableAtTestTest = (config: any, overrideLevel: OverrideLevel, onErr: (errorKey: string, invalidConfigKey: string) => void) => { + Object.keys(config).some((configKey) => { + const runTimeValidation = OVERRIDE_LEVELS.includes(overrideLevel) + const overrideLevels = testOverrideLevels[configKey] - Object.keys(config).some((option) => { - return errProperty = testConfigOverrideOptions[option] === false ? option : undefined - }) + if (!overrideLevels) { + // non-cypress configuration option. skip validation + return + } - if (errProperty) { - return onErr(errProperty) - } + if (runTimeValidation && (overrideLevels === 'never' || !overrideLevels.includes(overrideLevel))) { + const errPath = overrideLevel === 'runTime' ? 'config.invalid_cypress_config_api_override' : 'config.invalid_test_config_override' + + onErr(errPath, configKey) + } + }) } export const validateNeedToRestartOnChange = (cachedConfig: any, updatedConfig: any) => { diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index c05be50a02ea..9a463e0f8549 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -1,9 +1,11 @@ import os from 'os' import path from 'path' -import * as validate from './validation' // @ts-ignore import pkg from '@packages/root' +import type { TestingType } from '@packages/types' + +import * as validate from './validation' export type BreakingOptionErrorKey = | 'COMPONENT_FOLDER_REMOVED' @@ -27,32 +29,33 @@ export type BreakingOptionErrorKey = | 'RENAMED_CONFIG_OPTION' | 'TEST_FILES_RENAMED' -type TestingType = 'e2e' | 'component' +// The test-time override levels +export type OverrideLevel = 'supportFile' | 'suite' | 'test' | 'runTime' | 'never' + +export const OVERRIDE_LEVELS = ['supportFile', 'suite', 'test', 'runTime'] as OverrideLevel[] -interface ResolvedConfigOption { +interface ConfigOption { name: string defaultValue?: any validation: Function - isFolder?: boolean - isExperimental?: boolean + requireRestartOnChange?: 'server' | 'browser' /** - * Can be mutated with Cypress.config() or test-specific configuration overrides + * The list of test-time overrides levels supported by the configuration option. When undefined, + * it indicates the configuration value cannot be override via suite-/test-specific + * overrides or at run-time with Cypress.Config(). */ - canUpdateDuringTestTime?: boolean - specificTestingType?: TestingType - requireRestartOnChange?: 'server' | 'browser' + overrideLevels?: Exclude[] | 'never' } -interface RuntimeConfigOption { - name: string +interface DriverConfigOption extends ConfigOption { + isFolder?: boolean + isExperimental?: boolean +} + +// Cypress run-time options +interface RuntimeConfigOptions extends ConfigOption { defaultValue: any - validation: Function isInternal?: boolean - /** - * Can be mutated with Cypress.config() or test-specific configuration overrides - */ - canUpdateDuringTestTime?: boolean - requireRestartOnChange?: 'server' | 'browser' } export interface BreakingOption { @@ -86,8 +89,8 @@ export interface BreakingOption { showInLaunchpad?: boolean } -const isValidConfig = (key: string, config: any) => { - const status = validate.isPlainObject(key, config) +const isValidConfig = (testingType: string, config: any) => { + const status = validate.isPlainObject(testingType, config) if (status !== true) { return status @@ -95,7 +98,7 @@ const isValidConfig = (key: string, config: any) => { for (const rule of options) { if (rule.name in config && rule.validation) { - const status = rule.validation(`${key}.${rule.name}`, config[rule.name]) + const status = rule.validation(`${testingType}.${rule.name}`, config[rule.name]) if (status !== true) { return status @@ -116,42 +119,36 @@ export const defaultSpecPattern = { // - cli/types/index.d.ts (including allowed config options on TestOptions) // // Add options in alphabetical order for better readability - -// TODO - add boolean attribute to indicate read-only / static vs mutable options -// that can be updated during test executions -const resolvedOptions: Array = [ +const driverConfigOptions: Array = [ { name: 'animationDistanceThreshold', defaultValue: 5, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'arch', defaultValue: () => os.arch(), validation: validate.isString, - canUpdateDuringTestTime: false, }, { name: 'baseUrl', defaultValue: null, validation: validate.isFullyQualifiedUrl, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], requireRestartOnChange: 'server', }, { name: 'blockHosts', defaultValue: null, validation: validate.isStringOrArrayOfStrings, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'chromeWebSecurity', defaultValue: true, validation: validate.isBoolean, - canUpdateDuringTestTime: false, requireRestartOnChange: 'browser', }, { name: 'clientCertificates', defaultValue: [], validation: validate.isValidClientCertificatesSet, - canUpdateDuringTestTime: false, requireRestartOnChange: 'server', }, { name: 'component', @@ -161,18 +158,16 @@ const resolvedOptions: Array = [ indexHtmlFile: 'cypress/support/component-index.html', }, validation: isValidConfig, - canUpdateDuringTestTime: false, }, { name: 'defaultCommandTimeout', defaultValue: 4000, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'downloadsFolder', defaultValue: 'cypress/downloads', validation: validate.isString, isFolder: true, - canUpdateDuringTestTime: false, requireRestartOnChange: 'browser', }, { name: 'e2e', @@ -181,149 +176,136 @@ const resolvedOptions: Array = [ specPattern: defaultSpecPattern.e2e, }, validation: isValidConfig, - canUpdateDuringTestTime: false, }, { name: 'env', defaultValue: {}, validation: validate.isPlainObject, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'execTimeout', defaultValue: 60000, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'experimentalFetchPolyfill', defaultValue: false, validation: validate.isBoolean, isExperimental: true, - canUpdateDuringTestTime: false, }, { name: 'experimentalInteractiveRunEvents', defaultValue: false, validation: validate.isBoolean, isExperimental: true, - canUpdateDuringTestTime: false, requireRestartOnChange: 'server', }, { name: 'experimentalSessionAndOrigin', defaultValue: false, validation: validate.isBoolean, isExperimental: true, - canUpdateDuringTestTime: false, }, { name: 'experimentalModifyObstructiveThirdPartyCode', defaultValue: false, validation: validate.isBoolean, isExperimental: true, - canUpdateDuringTestTime: false, requireRestartOnChange: 'server', }, { name: 'experimentalSourceRewriting', defaultValue: false, validation: validate.isBoolean, isExperimental: true, - canUpdateDuringTestTime: false, requireRestartOnChange: 'server', }, { name: 'fileServerFolder', defaultValue: '', validation: validate.isString, isFolder: true, - canUpdateDuringTestTime: false, requireRestartOnChange: 'server', }, { name: 'fixturesFolder', defaultValue: 'cypress/fixtures', validation: validate.isStringOrFalse, isFolder: true, - canUpdateDuringTestTime: false, requireRestartOnChange: 'server', }, { name: 'excludeSpecPattern', defaultValue: (options: Record = {}) => options.testingType === 'component' ? ['**/__snapshots__/*', '**/__image_snapshots__/*'] : '*.hot-update.js', validation: validate.isStringOrArrayOfStrings, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'includeShadowDom', defaultValue: false, validation: validate.isBoolean, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'keystrokeDelay', defaultValue: 0, validation: validate.isNumberOrFalse, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'modifyObstructiveCode', defaultValue: true, validation: validate.isBoolean, - canUpdateDuringTestTime: false, requireRestartOnChange: 'server', }, { name: 'nodeVersion', validation: validate.isOneOf('bundled', 'system'), - canUpdateDuringTestTime: false, }, { name: 'numTestsKeptInMemory', defaultValue: 50, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'platform', defaultValue: () => os.platform(), validation: validate.isString, - canUpdateDuringTestTime: false, }, { name: 'pageLoadTimeout', defaultValue: 60000, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'port', defaultValue: null, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'projectId', defaultValue: null, validation: validate.isString, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'redirectionLimit', defaultValue: 20, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'reporter', defaultValue: 'spec', validation: validate.isString, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'reporterOptions', defaultValue: null, validation: validate.isPlainObject, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'requestTimeout', defaultValue: 5000, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'resolvedNodePath', defaultValue: null, validation: validate.isString, - canUpdateDuringTestTime: false, }, { name: 'resolvedNodeVersion', defaultValue: null, validation: validate.isString, - canUpdateDuringTestTime: false, }, { name: 'responseTimeout', defaultValue: 30000, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'retries', defaultValue: { @@ -331,110 +313,108 @@ const resolvedOptions: Array = [ openMode: 0, }, validation: validate.isValidRetriesConfig, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'screenshotOnRunFailure', defaultValue: true, validation: validate.isBoolean, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'screenshotsFolder', defaultValue: 'cypress/screenshots', validation: validate.isStringOrFalse, isFolder: true, - canUpdateDuringTestTime: false, requireRestartOnChange: 'server', }, { name: 'slowTestThreshold', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 250 : 10000, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'scrollBehavior', defaultValue: 'top', validation: validate.isOneOf('center', 'top', 'bottom', 'nearest', false), - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'supportFile', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 'cypress/support/component.{js,jsx,ts,tsx}' : 'cypress/support/e2e.{js,jsx,ts,tsx}', validation: validate.isStringOrFalse, - canUpdateDuringTestTime: false, requireRestartOnChange: 'server', }, { name: 'supportFolder', defaultValue: false, validation: validate.isStringOrFalse, isFolder: true, - canUpdateDuringTestTime: false, requireRestartOnChange: 'server', }, { name: 'taskTimeout', defaultValue: 60000, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + }, { + name: 'testIsolation', + // TODO: When experimentalSessionAndOrigin is removed and released as GA, + // update the defaultValue from 'default' to 'strict' and + // update this code to remove the check/override specific to enable + // strict by default when experimentalSessionAndOrigin=true + defaultValue: 'default', + validation: validate.isOneOf('default', 'strict'), + overrideLevels: ['supportFile', 'suite'], }, { name: 'trashAssetsBeforeRuns', defaultValue: true, validation: validate.isBoolean, - canUpdateDuringTestTime: false, }, { name: 'userAgent', defaultValue: null, validation: validate.isString, - canUpdateDuringTestTime: false, requireRestartOnChange: 'browser', }, { name: 'video', defaultValue: true, validation: validate.isBoolean, - canUpdateDuringTestTime: false, }, { name: 'videoCompression', defaultValue: 32, validation: validate.isNumberOrFalse, - canUpdateDuringTestTime: false, }, { name: 'videosFolder', defaultValue: 'cypress/videos', validation: validate.isString, isFolder: true, - canUpdateDuringTestTime: false, }, { name: 'videoUploadOnPasses', defaultValue: true, validation: validate.isBoolean, - canUpdateDuringTestTime: false, }, { name: 'viewportHeight', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 500 : 660, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'viewportWidth', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 500 : 1000, validation: validate.isNumber, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'waitForAnimations', defaultValue: true, validation: validate.isBoolean, - canUpdateDuringTestTime: true, + overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], }, { name: 'watchForFileChanges', defaultValue: true, validation: validate.isBoolean, - canUpdateDuringTestTime: false, requireRestartOnChange: 'server', }, - // Possibly add a defaultValue for specPattern https://github.com/cypress-io/cypress/issues/22507 { name: 'specPattern', + defaultValue: (options: Record = {}) => options.testingType === 'component' ? defaultSpecPattern.component : defaultSpecPattern.e2e, validation: validate.isStringOrArrayOfStrings, - canUpdateDuringTestTime: false, }, ] -const runtimeOptions: Array = [ +const runtimeOptions: Array = [ { // Internal config field, useful to ignore the e2e specPattern set by the user // or the default one when looking fot CT, it needs to be a config property because after @@ -444,24 +424,20 @@ const runtimeOptions: Array = [ defaultValue: (options: Record = {}) => options.testingType === 'component' ? defaultSpecPattern.e2e : [], validation: validate.isStringOrArrayOfStrings, isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'autoOpen', defaultValue: false, validation: validate.isBoolean, isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'browsers', defaultValue: [], validation: validate.isValidBrowserList, - canUpdateDuringTestTime: false, }, { name: 'clientRoute', defaultValue: '/__/', validation: validate.isString, isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'configFile', defaultValue: 'cypress.config.js', @@ -469,104 +445,82 @@ const runtimeOptions: Array = [ // not truly internal, but can only be set via cli, // so we don't consider it a "public" option isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'cypressBinaryRoot', defaultValue: path.join(__dirname, '..', '..', '..'), validation: validate.isString, isInternal: true, }, { + // ct-testing specific configuration name: 'devServerPublicPathRoute', defaultValue: '/__cypress/src', validation: validate.isString, isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'hosts', defaultValue: null, validation: validate.isPlainObject, - canUpdateDuringTestTime: false, }, { name: 'isInteractive', defaultValue: true, validation: validate.isBoolean, - canUpdateDuringTestTime: false, }, { name: 'isTextTerminal', defaultValue: false, validation: validate.isBoolean, isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'morgan', defaultValue: true, validation: validate.isBoolean, isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'modifyObstructiveCode', defaultValue: true, validation: validate.isBoolean, - canUpdateDuringTestTime: false, }, { name: 'namespace', defaultValue: '__cypress', validation: validate.isString, isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'reporterRoute', defaultValue: '/__cypress/reporter', validation: validate.isString, isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'socketId', defaultValue: null, validation: validate.isString, isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'socketIoCookie', defaultValue: '__socket', validation: validate.isString, isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'socketIoRoute', defaultValue: '/__socket', validation: validate.isString, isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'version', defaultValue: pkg.version, validation: validate.isString, isInternal: true, - canUpdateDuringTestTime: false, }, { name: 'xhrRoute', defaultValue: '/xhrs/', validation: validate.isString, isInternal: true, - canUpdateDuringTestTime: false, }, ] -export const options: Array = [ - ...resolvedOptions, +export const options: Array = [ + ...driverConfigOptions, ...runtimeOptions, ] -// These properties are going to be added to the resolved properties of the -// config, but do not mean that are valid config properties coming from the user. -export const additionalOptionsToResolveConfig = [ - { - name: 'specPattern', - isInternal: false, - }, -] - /** * Values not allowed in 10.X+ in the root, e2e and component config */ @@ -685,6 +639,11 @@ export const breakingRootOptions: Array = [ errorKey: 'CONFIG_FILE_INVALID_ROOT_CONFIG', isWarning: false, testingTypes: ['component', 'e2e'], + }, { + name: 'testIsolation', + errorKey: 'CONFIG_FILE_INVALID_ROOT_CONFIG', + isWarning: false, + testingTypes: ['e2e'], }, ] @@ -707,5 +666,10 @@ export const testingTypeBreakingOptions: { e2e: Array, component errorKey: 'CONFIG_FILE_INVALID_TESTING_TYPE_CONFIG_COMPONENT', isWarning: false, }, + { + name: 'testIsolation', + errorKey: 'CONFIG_FILE_INVALID_TESTING_TYPE_CONFIG_COMPONENT', + isWarning: false, + }, ], } diff --git a/packages/config/test/index.spec.ts b/packages/config/test/index.spec.ts index e349ced4b585..d861797406dc 100644 --- a/packages/config/test/index.spec.ts +++ b/packages/config/test/index.spec.ts @@ -4,6 +4,7 @@ import sinon from 'sinon' import sinonChai from 'sinon-chai' import * as configUtil from '../src/index' +import { OverrideLevel, OVERRIDE_LEVELS } from '../src/options' chai.use(sinonChai) const { expect } = chai @@ -189,28 +190,71 @@ describe('config/src/index', () => { }) }) - describe('.validateNoReadOnlyConfig', () => { - it('returns an error if validation fails', () => { + describe('.validateOverridableAtTestTest', () => { + ['supportFile', 'suite', 'test'].forEach((overrideLevel) => { + it(`calls onError handler if configuration cannot be overridden from ${overrideLevel} level`, () => { + const errorFn = sinon.spy() + + configUtil.validateOverridableAtTestTest({ chromeWebSecurity: false }, overrideLevel as OverrideLevel, errorFn) + + expect(errorFn).to.have.callCount(1) + expect(errorFn).to.have.been.calledWithMatch('config.invalid_test_config_override', 'chromeWebSecurity') + }) + }) + + it('calls onError handler if configuration cannot be overridden from runtime level', () => { const errorFn = sinon.spy() - configUtil.validateNoReadOnlyConfig({ chromeWebSecurity: false }, errorFn) + configUtil.validateOverridableAtTestTest({ chromeWebSecurity: false }, 'runTime', errorFn) expect(errorFn).to.have.callCount(1) - expect(errorFn).to.have.been.calledWithMatch(/chromeWebSecurity/) + expect(errorFn).to.have.been.calledWithMatch('config.invalid_cypress_config_api_override', 'chromeWebSecurity') }) - it('does not return an error if validation succeeds', () => { - const errorFn = sinon.spy() + describe('testIsolation', () => { + ['supportFile', 'suite'].forEach((overrideLevel) => { + it(`does not calls onError handler if configuration can be overridden from ${overrideLevel} level`, () => { + const errorFn = sinon.spy() - configUtil.validateNoReadOnlyConfig({ requestTimeout: 1000 }, errorFn) + configUtil.validateOverridableAtTestTest({ testIsolation: 'strict' }, overrideLevel as OverrideLevel, errorFn) - expect(errorFn).to.have.callCount(0) + expect(errorFn).to.have.callCount(0) + }) + }) + + it('calls onError handler if configuration cannot be overridden from test level', () => { + const errorFn = sinon.spy() + + configUtil.validateOverridableAtTestTest({ testIsolation: false }, 'test', errorFn) + + expect(errorFn).to.have.callCount(1) + expect(errorFn).to.have.been.calledWithMatch('config.invalid_test_config_override', 'testIsolation') + }) + + it('calls onError handler if configuration cannot be overridden from runTime level', () => { + const errorFn = sinon.spy() + + configUtil.validateOverridableAtTestTest({ testIsolation: false }, 'runTime', errorFn) + + expect(errorFn).to.have.callCount(1) + expect(errorFn).to.have.been.calledWithMatch('config.invalid_cypress_config_api_override', 'testIsolation') + }) + }) + + OVERRIDE_LEVELS.forEach((overrideLevel) => { + it(`does not call onErr if validation succeeds from ${overrideLevel} level`, () => { + const errorFn = sinon.spy() + + configUtil.validateOverridableAtTestTest({ requestTimeout: 1000 }, overrideLevel as OverrideLevel, errorFn) + + expect(errorFn).to.have.callCount(0) + }) }) - it('does not return an error if configuration is a non-Cypress config option', () => { + it('does not call onErr if configuration is a non-Cypress config option', () => { const errorFn = sinon.spy() - configUtil.validateNoReadOnlyConfig({ foo: 'bar' }, errorFn) + configUtil.validateOverridableAtTestTest({ foo: 'bar' }, 'runTime' as OverrideLevel, errorFn) expect(errorFn).to.have.callCount(0) }) diff --git a/packages/driver/cypress/e2e/commands/local_storage.cy.js b/packages/driver/cypress/e2e/commands/local_storage.cy.js index 5d7d5ab366da..f8337c03f23b 100644 --- a/packages/driver/cypress/e2e/commands/local_storage.cy.js +++ b/packages/driver/cypress/e2e/commands/local_storage.cy.js @@ -42,8 +42,11 @@ describe('src/cy/commands/local_storage', () => { const clear = cy.spy(Cypress.LocalStorage, 'clear') Cypress.emit('test:before:run', {}) - - expect(clear).to.be.calledWith([]) + if (Cypress.config('experimentalSessionAndOrigin')) { + expect(clear).not.to.be.called + } else { + expect(clear).to.be.calledWith([]) + } }) }) diff --git a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js index 7dc693fd107c..25c733ee19c3 100644 --- a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js +++ b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js @@ -43,6 +43,54 @@ describe('cy.session', { retries: 0 }, () => { }) }) + describe('test:before:run:async', () => { + it('clears session data before each run', async () => { + const clearCurrentSessionData = cy.spy(Cypress.session, 'clearCurrentSessionData') + + await Cypress.action('runner:test:before:run:async', {}) + + expect(clearCurrentSessionData).to.be.called + }) + + it('clears session data before each run', async () => { + const backendSpy = cy.spy(Cypress, 'backend') + + await Cypress.action('runner:test:before:run:async', {}) + + expect(backendSpy).to.be.calledWith('reset:rendered:html:origins') + }) + + describe('testIsolation=strict', { testIsolation: 'strict' }, () => { + it('clears page before each run when testIsolation=strict', () => { + cy.visit('/fixtures/form.html') + .then(async () => { + cy.spy(Cypress, 'action').log(false) + + await Cypress.action('runner:test:before:run:async', {}) + + expect(Cypress.action).to.be.calledWith('cy:url:changed', '') + expect(Cypress.action).to.be.calledWith('cy:visit:blank', { type: 'session-lifecycle' }) + }) + .url('about:blank') + }) + }) + + describe('testIsolation=default', { testIsolation: 'default' }, () => { + it('does not clear page', () => { + cy.visit('/fixtures/form.html') + .then(async () => { + cy.spy(Cypress, 'action').log(false) + + await Cypress.action('runner:test:before:run:async', {}) + + expect(Cypress.action).not.to.be.calledWith('cy:url:changed') + expect(Cypress.action).not.to.be.calledWith('cy:visit:blank') + }) + .url('/fixtures/form.html') + }) + }) + }) + describe('session flows', () => { let logs = [] let clearPageCount = 0 diff --git a/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js b/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js index c6c120d7e066..42386e13dc9e 100644 --- a/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js +++ b/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js @@ -377,15 +377,15 @@ describe('testConfigOverrides baseUrl @slow', () => { }) }) -describe('cannot set read-only properties', () => { +describe('cannot set override configuration options that', () => { afterEach(() => { window.top.__cySkipValidateConfig = true }) it('throws if mutating read-only config with Cypress.config()', (done) => { window.top.__cySkipValidateConfig = false - cy.on('fail', (err) => { - expect(err.message).to.include('`Cypress.config()` cannot mutate option `chromeWebSecurity` because it is a read-only property.') + cy.once('fail', (err) => { + expect(err.message).to.include('`Cypress.config()` cannot override `chromeWebSecurity` from the test-level.') done() }) diff --git a/packages/driver/src/cy/commands/cookies.ts b/packages/driver/src/cy/commands/cookies.ts index 176a8e359c45..5b1edcdfd01b 100644 --- a/packages/driver/src/cy/commands/cookies.ts +++ b/packages/driver/src/cy/commands/cookies.ts @@ -136,7 +136,7 @@ export default function (Commands, Cypress, cy, state, config) { const getAndClear = (log?, timeout?, options = {}) => { return automateCookies('get:cookies', options, log, timeout) .then((resp) => { - // bail early if we got no cookies! + // bail early if we got no cookies! if (resp && (resp.length === 0)) { return resp } @@ -178,15 +178,17 @@ export default function (Commands, Cypress, cy, state, config) { } } - // TODO: handle failure here somehow - // maybe by tapping into the Cypress reset - // stuff, or handling this in the runner itself? - // Cypress sessions will clear cookies on its own before each test - Cypress.on('test:before:run:async', () => { - if (!Cypress.config('experimentalSessionAndOrigin')) { + // TODO: Cypress sessions will clear cookies on its own before each test. + // Once experimentalSessionAndOrigin is made GA, remove this logic. Leave clearing + // session data (cookies / local storage / session storage) to reset functionality. + if (!Cypress.config('experimentalSessionAndOrigin')) { + // TODO: handle failure here somehow + // maybe by tapping into the Cypress reset + // stuff, or handling this in the runner itself? + Cypress.on('test:before:run:async', () => { return getAndClear() - } - }) + }) + } return Commands.addAll({ getCookie (name, userOptions: Partial = {}) { diff --git a/packages/driver/src/cy/commands/local_storage.ts b/packages/driver/src/cy/commands/local_storage.ts index 3c7b0ffe6055..31d5e9e08d92 100644 --- a/packages/driver/src/cy/commands/local_storage.ts +++ b/packages/driver/src/cy/commands/local_storage.ts @@ -21,16 +21,21 @@ const clearLocalStorage = (state, keys) => { } export default (Commands, Cypress, cy, state) => { - // this MUST be prepended before anything else - Cypress.prependListener('test:before:run', () => { - try { - // this may fail if the current - // window is bound to another origin - return clearLocalStorage(state, []) - } catch (error) { - return null - } - }) + // TODO: Cypress sessions will clear local storage on its own before each test. + // Once experimentalSessionAndOrigin is made GA, remove this logic. Leave clearing + // session data (cookies / local storage / session storage) to reset functionality + if (!Cypress.config('experimentalSessionAndOrigin')) { + // this MUST be prepended before anything else + Cypress.prependListener('test:before:run', () => { + try { + // this may fail if the current + // window is bound to another origin + return clearLocalStorage(state, []) + } catch (error) { + return null + } + }) + } Commands.addAll({ clearLocalStorage (keys, options: Partial = {}) { diff --git a/packages/driver/src/cy/commands/sessions/index.ts b/packages/driver/src/cy/commands/sessions/index.ts index eb6432f60560..61cd30470a3e 100644 --- a/packages/driver/src/cy/commands/sessions/index.ts +++ b/packages/driver/src/cy/commands/sessions/index.ts @@ -42,9 +42,12 @@ export default function (Commands, Cypress, cy) { if (Cypress.config('experimentalSessionAndOrigin')) { sessionsManager.currentTestRegisteredSessions.clear() - return navigateAboutBlank(false) + const clearPage = Cypress.config('testIsolation') === 'strict' ? navigateAboutBlank(false) : new Cypress.Promise.resolve() + + return clearPage .then(() => sessions.clearCurrentSessionData()) .then(() => Cypress.backend('reset:rendered:html:origins')) + .then(() => 'successfully cleared sessions data') } return diff --git a/packages/driver/src/cy/testConfigOverrides.ts b/packages/driver/src/cy/testConfigOverrides.ts index 4b93bb7caf71..a1d8c608e89e 100644 --- a/packages/driver/src/cy/testConfigOverrides.ts +++ b/packages/driver/src/cy/testConfigOverrides.ts @@ -8,17 +8,24 @@ type ResolvedTestConfigOverride = { * The list of test config overrides and the invocation details used to add helpful * error messaging to consumers if a test override fails validation. */ - testConfigList: Array + testConfigList: Array /** * The test config overrides that will apply to the test if it passes validation. - * */ + */ unverifiedTestConfig: Object + /** + * The current runnable level of test config overrides that are being applied. + * Used for accurate error reporting. + */ + applied?: string } type TestConfig = { - overrides: { - browser?: Object - } + // The level in which the configuration override was set. + overrideLevel: 'suite' | 'test' + // The configuration overrides. Browser is a valid configuration + // to indicate the suite or test should run for that browser(s). + overrides: Record invocationDetails: { stack: Object } @@ -28,18 +35,25 @@ type ConfigOverrides = { env: Object | undefined }; -function setConfig (testConfigList: Array, config, localConfigOverrides: ConfigOverrides = { env: undefined }) { - testConfigList.forEach(({ overrides: testConfigOverride, invocationDetails }) => { +function setConfig (testConfig: ResolvedTestConfigOverride, config, localConfigOverrides: ConfigOverrides = { env: undefined }) { + const { testConfigList = [] } = testConfig + + testConfigList.forEach((resolvedConfig) => { + const { overrides: testConfigOverride, overrideLevel, invocationDetails } = resolvedConfig as TestConfig + if (_.isArray(testConfigOverride)) { - setConfig(testConfigOverride, config, localConfigOverrides) - } else { + setConfig(resolvedConfig as ResolvedTestConfigOverride, config, localConfigOverrides) + } else if (Object.keys(testConfigOverride).length) { delete testConfigOverride.browser try { + testConfig.applied = overrideLevel + config(testConfigOverride) } catch (e: any) { let err = $errUtils.errByPath('config.invalid_test_override', { errMsg: e.message, + overrideLevel, }) err.stack = $errUtils.stackWithReplacedProps({ stack: invocationDetails.stack }, err) @@ -53,11 +67,9 @@ function setConfig (testConfigList: Array, config, localConfigOverri } function mutateConfiguration (testConfig: ResolvedTestConfigOverride, config, env) { - const { testConfigList } = testConfig || [] - let globalConfig = _.clone(config()) - const localConfigOverrides = setConfig(testConfigList, config) + const localConfigOverrides = setConfig(testConfig, config) // only store the global config values that updated globalConfig = _.pick(globalConfig, Object.keys(localConfigOverrides)) @@ -119,6 +131,7 @@ export function getResolvedTestConfigOverride (test): ResolvedTestConfigOverride testConfigList = testConfigList.concat(curr._testConfig.testConfigList) } else { testConfigList.unshift({ + overrideLevel: curr.type, overrides: curr._testConfig, invocationDetails: curr.invocationDetails, }) diff --git a/packages/driver/src/cypress.ts b/packages/driver/src/cypress.ts index ea1ba2ccfbf4..5ad618c37753 100644 --- a/packages/driver/src/cypress.ts +++ b/packages/driver/src/cypress.ts @@ -1,4 +1,4 @@ -import { validate, validateNoReadOnlyConfig } from '@packages/config' +import { validate as validateConfigValues, validateOverridableAtTestTest } from '@packages/config' import _ from 'lodash' import $ from 'jquery' import * as blobUtil from 'blob-util' @@ -238,21 +238,24 @@ class $Cypress { this.originalConfig = _.cloneDeep(config) this.config = $SetterGetter.create(config, (config) => { - if (this.isCrossOriginSpecBridge ? !window.__cySkipValidateConfig : !window.top!.__cySkipValidateConfig) { - validateNoReadOnlyConfig(config, (errProperty) => { - const errPath = this.state('runnable') - ? 'config.invalid_cypress_config_override' - : 'config.invalid_test_config_override' + const overrideLevel = this.state('runnable') ? 'runTime' : this.state('test')?._testConfig?.applied || 'supportFile' + const enforceConfigOverrideRules = this.isCrossOriginSpecBridge ? !window.__cySkipValidateConfig : !window.top!.__cySkipValidateConfig - const errMsg = $errUtils.errByPath(errPath, { - errProperty, - }) + validateOverridableAtTestTest(config, overrideLevel, (errPath, invalidConfigKey) => { + const errMsg = $errUtils.errByPath(errPath, { + invalidConfigKey, + overrideLevel: this.state('runnable')?.type || this.state('test')?._testConfig?.applied, + }) + if (enforceConfigOverrideRules) { throw new (this.state('specWindow').Error)(errMsg) - }) - } + } + + // eslint-disable-next-line no-console + console.warn(errMsg) + }) - validate(config, (errResult) => { + validateConfigValues(config, (errResult) => { const stringify = (str) => format(JSON.stringify(str)) const format = (str) => `\`${str}\`` diff --git a/packages/driver/src/cypress/cy.ts b/packages/driver/src/cypress/cy.ts index f2f5e8719f02..1ee7ae816be7 100644 --- a/packages/driver/src/cypress/cy.ts +++ b/packages/driver/src/cypress/cy.ts @@ -209,7 +209,6 @@ export class $Cy extends EventEmitter2 implements ITimeouts, IStability, IAssert overrides: IOverrides // Private methods - ensureSubjectByType: ReturnType['ensureSubjectByType'] ensureRunnable: ReturnType['ensureRunnable'] @@ -221,7 +220,6 @@ export class $Cy extends EventEmitter2 implements ITimeouts, IStability, IAssert private testConfigOverride: TestConfigOverride private commandFns: Record = {} - private selectorFns: Record = {} constructor (specWindow: SpecWindow, Cypress: ICypress, Cookies: ICookies, state: StateFunc, config: ICypress['config']) { super() @@ -643,6 +641,7 @@ export class $Cy extends EventEmitter2 implements ITimeouts, IStability, IAssert const s = this.state() const backup = { + test, window: s.window, document: s.document, $autIframe: s.$autIframe, @@ -659,9 +658,8 @@ export class $Cy extends EventEmitter2 implements ITimeouts, IStability, IAssert this.queue.reset() this.queue.clear() this.resetTimer() - this.testConfigOverride.restoreAndSetTestConfigOverrides(test, this.Cypress.config, this.Cypress.env) - this.removeAllListeners() + this.testConfigOverride.restoreAndSetTestConfigOverrides(test, this.Cypress.config, this.Cypress.env) } catch (err) { this.fail(err) } @@ -940,7 +938,6 @@ export class $Cy extends EventEmitter2 implements ITimeouts, IStability, IAssert this.state('promise', undefined) this.state('hookId', hookId) this.state('runnable', runnable) - this.state('test', $utils.getTestFromRunnable(runnable)) this.state('ctx', runnable.ctx) const { fn } = runnable diff --git a/packages/driver/src/cypress/error_messages.ts b/packages/driver/src/cypress/error_messages.ts index 380f835bf3a9..a98447eaf4fc 100644 --- a/packages/driver/src/cypress/error_messages.ts +++ b/packages/driver/src/cypress/error_messages.ts @@ -271,14 +271,16 @@ export default { docsUrl: 'https://on.cypress.io/config', }, invalid_test_override: { - message: `The config override passed to your test has the following validation error:\n\n{{errMsg}}`, + message: `The config override passed to your {{overrideLevel}}-level override has the following validation error:\n\n{{errMsg}}`, docsUrl: 'https://on.cypress.io/config', }, - invalid_cypress_config_override: { - message: `\`Cypress.config()\` cannot mutate option \`{{errProperty}}\` because it is a read-only property.`, + invalid_cypress_config_api_override: { + message: `\`Cypress.config()\` cannot override \`{{invalidConfigKey}}\` from the {{overrideLevel}}-level.`, + docsUrl: 'https://on.cypress.io/config', }, invalid_test_config_override: { - message: `Cypress test configuration cannot mutate option \`{{errProperty}}\` because it is a read-only property.`, + message: `\`{{invalidConfigKey}}\` configuration cannot been overridden from the {{overrideLevel}}-level.`, + docsUrl: 'https://on.cypress.io/config', }, }, diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index 00dc84e86b67..e8745f2ad55c 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -13,7 +13,9 @@ import { preprocessForSerialization } from './serialization' */ const omitConfigReadOnlyDifferences = (objectLikeConfig: Cypress.ObjectLike) => { Object.keys(objectLikeConfig).forEach((key) => { - if (options.find((option) => option.name === key)?.canUpdateDuringTestTime === false) { + const { overrideLevels = 'never' } = options.find((option) => option.name === key) || { } + + if (overrideLevels === 'never') { delete objectLikeConfig[key] } }) diff --git a/packages/server/lib/config.ts b/packages/server/lib/config.ts index 675fd5a80c7f..52a9edf2a383 100644 --- a/packages/server/lib/config.ts +++ b/packages/server/lib/config.ts @@ -104,6 +104,7 @@ export function mergeDefaults ( cliConfig: Record = {}, ) { const resolved = {} + const { testingType } = options config.rawJson = _.cloneDeep(config) @@ -144,13 +145,13 @@ export function mergeDefaults ( let additionalIgnorePattern = config.additionalIgnorePattern - if (options.testingType === 'component' && config.e2e && config.e2e.specPattern) { + if (testingType === 'component' && config.e2e && config.e2e.specPattern) { additionalIgnorePattern = config.e2e.specPattern } config = { ...config, - ...config[options.testingType], + ...config[testingType], additionalIgnorePattern, } @@ -199,8 +200,6 @@ export function mergeDefaults ( debug('validate that there is no breaking config options before setupNodeEvents') - const { testingType } = options - function makeConfigError (cyError: CypressError) { cyError.name = `Obsolete option used in config object` @@ -215,6 +214,14 @@ export function mergeDefaults ( throw makeConfigError(errors.get(err, ...args)) }, testingType) + // TODO: testIsolation should equal 'strict' by default when experimentalSessionAndOrigin=true + // Once experimentalSessionAndOrigin is made GA, remove this logic and update the defaultValue + // to be be 'strict' + if (testingType === 'e2e' && config.experimentalSessionAndOrigin && config.resolved.testIsolation.from === 'default') { + config.testIsolation = 'strict' + config.resolved.testIsolation.value = 'strict' + } + // We need to remove the nested propertied by testing type because it has been // flattened/compacted based on the current testing type that is selected // making the config only available with the properties that are valid, @@ -537,7 +544,7 @@ export function setUrls (obj) { return obj } -export function parseEnv (cfg: Record, envCLI: Record, resolved: Record = {}) { +export function parseEnv (cfg: Record, cliEnvs: Record, resolved: Record = {}) { const envVars = (resolved.env = {}) const resolveFrom = (from, obj = {}) => { @@ -549,13 +556,13 @@ export function parseEnv (cfg: Record, envCLI: Record, }) } - const envCfg = cfg.env != null ? cfg.env : {} + const configEnv = cfg.env != null ? cfg.env : {} const envFile = cfg.envFile != null ? cfg.envFile : {} - let envProc = getProcessEnvVars(process.env) || {} + let processEnvs = getProcessEnvVars(process.env) || {} - envCLI = envCLI != null ? envCLI : {} + cliEnvs = cliEnvs != null ? cliEnvs : {} - const configFromEnv = _.reduce(envProc, (memo: string[], val, key) => { + const configFromEnv = _.reduce(processEnvs, (memo: string[], val, key) => { const cfgKey = configUtils.matchesConfigKey(key) if (cfgKey) { @@ -576,21 +583,21 @@ export function parseEnv (cfg: Record, envCLI: Record, } , []) - envProc = _.chain(envProc) + processEnvs = _.chain(processEnvs) .omit(configFromEnv) .mapValues(hideSpecialVals) .value() - resolveFrom('config', envCfg) + resolveFrom('config', configEnv) resolveFrom('envFile', envFile) - resolveFrom('env', envProc) - resolveFrom('cli', envCLI) + resolveFrom('env', processEnvs) + resolveFrom('cli', cliEnvs) - // envCfg is from cypress.config.{js,ts,mjs,cjs} + // configEnvs is from cypress.config.{js,ts,mjs,cjs} // envFile is from cypress.env.json - // envProc is from process env vars - // envCLI is from CLI arguments - return _.extend(envCfg, envFile, envProc, envCLI) + // processEnvs is from process env vars + // cliEnvs is from CLI arguments + return _.extend(configEnv, envFile, processEnvs, cliEnvs) } export function getResolvedRuntimeConfig (config, runtimeConfig) { diff --git a/packages/server/lib/util/spec_writer.ts b/packages/server/lib/util/spec_writer.ts index e0cdebc0d783..7042e57f8941 100644 --- a/packages/server/lib/util/spec_writer.ts +++ b/packages/server/lib/util/spec_writer.ts @@ -385,6 +385,7 @@ export const createFile = (path: string) => { return fs.writeFile(path, newFileTemplate(path)) } +// current un-used with 10.x release export const countStudioUsage = (path: string) => { return fs.readFile(path) .then((specBuffer) => { diff --git a/packages/types/src/config.ts b/packages/types/src/config.ts index edc1850f92bd..652363856afe 100644 --- a/packages/types/src/config.ts +++ b/packages/types/src/config.ts @@ -2,7 +2,7 @@ import type { AllModeOptions } from '.' -export const RESOLVED_FROM = ['plugin', 'env', 'default', 'runtime', 'config'] as const +export const RESOLVED_FROM = ['default', 'config', 'plugin', 'envFile', 'env', 'cli', 'runTime'] as const export type ResolvedConfigurationOptionSource = typeof RESOLVED_FROM[number] @@ -32,15 +32,6 @@ export type ReceivedCypressOptions = Pick & Pick // TODO: Figure out how to type this better. -export interface SampleConfigFile{ - status: 'changes' | 'valid' | 'skipped' | 'error' - filePath: string - content: string - description?: string - warningText?: string - warningLink?: string -} - export interface SettingsOptions { testingType?: 'component' |'e2e' args?: AllModeOptions diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index 68ad9d2563e5..e9601d5aa6ec 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -7,14 +7,14 @@ exports['testConfigOverrides / fails when passing invalid config value browser'] ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 1 found (testConfigOverrides-invalid-browser.js) │ - │ Searched: cypress/e2e/testConfigOverrides-invalid-browser.js │ + │ Specs: 1 found (invalid-browser.js) │ + │ Searched: cypress/e2e/testConfigOverrides/invalid-browser.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: testConfigOverrides-invalid-browser.js (1 of 1) + Running: invalid-browser.js (1 of 1) 1) An uncaught error was detected outside of a test @@ -48,14 +48,14 @@ We dynamically generated a new test to display this failure. │ Screenshots: 1 │ │ Video: false │ │ Duration: X seconds │ - │ Spec Ran: testConfigOverrides-invalid-browser.js │ + │ Spec Ran: invalid-browser.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ (Screenshots) - - /XXX/XXX/XXX/cypress/screenshots/testConfigOverrides-invalid-browser.js/An uncau (1280x720) - ght error was detected outside of a test (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid-browser.js/An uncaught error was detect (2560x1328) + ed outside of a test (failed).png ==================================================================================================== @@ -65,78 +65,14 @@ We dynamically generated a new test to display this failure. Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ testConfigOverrides-invalid-browser XX:XX 1 - 1 - - │ - │ .js │ + │ ✖ invalid-browser.js XX:XX 1 - 1 - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - ` -exports['testConfigOverrides / has originalTitle when skip due to browser config'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (testConfigOverrides-skip-browser.js) │ - │ Searched: cypress/e2e/testConfigOverrides-skip-browser.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: testConfigOverrides-skip-browser.js (1 of 1) - - - suite - - has invalid testConfigOverrides (skipped due to browser) - - - 0 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 1 │ - │ Passing: 0 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 0 │ - │ Video: true │ - │ Duration: X seconds │ - │ Spec Ran: testConfigOverrides-skip-browser.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Video) - - - Started processing: Compressing to 32 CRF - - Finished processing: /XXX/XXX/XXX/cypress/videos/testConfigOverrides-skip-browse (X second) - r.js.mp4 - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ testConfigOverrides-skip-browser.js XX:XX 1 - - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 1 - - 1 - - - -` - -exports['testConfigOverrides / fails when passing invalid config values - [chrome,electron]'] = ` +exports['testConfigOverrides / fails when passing invalid config values - [firefox]'] = ` ==================================================================================================== @@ -145,14 +81,14 @@ exports['testConfigOverrides / fails when passing invalid config values - [chrom ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 1 found (testConfigOverrides-invalid.js) │ - │ Searched: cypress/e2e/testConfigOverrides-invalid.js │ + │ Specs: 1 found (invalid.js) │ + │ Searched: cypress/e2e/testConfigOverrides/invalid.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: testConfigOverrides-invalid.js (1 of 1) + Running: invalid.js (1 of 1) 1) inline test config override throws error @@ -193,56 +129,52 @@ Instead the value was: \`"null"\` 3) context config overrides throws error runs: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. Instead the value was: \`"1"\` https://on.cypress.io/config - Error [stack trace lines] 4) nested contexts overrides throws error at the correct line number 1st test fails on overrides: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`defaultCommandTimeout\` to be a number. Instead the value was: \`"500"\` https://on.cypress.io/config - Error [stack trace lines] 5) nested contexts overrides throws error at the correct line number 2nd test fails on overrides: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`defaultCommandTimeout\` to be a number. Instead the value was: \`"500"\` https://on.cypress.io/config - Error [stack trace lines] 6) nested contexts test override throws error at the correct line number: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your test-level override has the following validation error: Expected \`baseUrl\` to be a fully qualified URL (starting with \`http://\` or \`https://\`). Instead the value was: \`"not_an_http_url"\` https://on.cypress.io/config - Error [stack trace lines] 7) throws error correctly when before hook "before all" hook for "test config override throws error": - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your test-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -251,19 +183,17 @@ Instead the value was: \`"1"\` https://on.cypress.io/config Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`throws error correctly when...\` - Error [stack trace lines] 8) throws error correctly when beforeEach hook test config override throws error: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your test-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. Instead the value was: \`"1"\` https://on.cypress.io/config - Error [stack trace lines] @@ -280,16 +210,16 @@ https://on.cypress.io/config │ Screenshots: 2 │ │ Video: false │ │ Duration: X seconds │ - │ Spec Ran: testConfigOverrides-invalid.js │ + │ Spec Ran: invalid.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ (Screenshots) - - /XXX/XXX/XXX/cypress/screenshots/testConfigOverrides-invalid.js/inline test conf (1280x720) - ig override throws error (failed).png - - /XXX/XXX/XXX/cypress/screenshots/testConfigOverrides-invalid.js/inline test conf (1280x720) - ig override throws error when executed within cy cmd (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/inline test config override throws e (2560x1328) + rror (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/inline test config override throws e (2560x1328) + rror when executed within cy cmd (failed).png ==================================================================================================== @@ -299,14 +229,14 @@ https://on.cypress.io/config Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ testConfigOverrides-invalid.js XX:XX 9 1 8 - - │ + │ ✖ invalid.js XX:XX 9 1 8 - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ✖ 1 of 1 failed (100%) XX:XX 9 1 8 - - ` -exports['testConfigOverrides / fails when passing invalid config values with beforeEach - [chrome,electron]'] = ` +exports['testConfigOverrides / fails when passing invalid config values with beforeEach - [firefox]'] = ` ==================================================================================================== @@ -315,14 +245,14 @@ exports['testConfigOverrides / fails when passing invalid config values with bef ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 1 found (testConfigOverrides-before-invalid.js) │ - │ Searched: cypress/e2e/testConfigOverrides-before-invalid.js │ + │ Specs: 1 found (before-invalid.js) │ + │ Searched: cypress/e2e/testConfigOverrides/before-invalid.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: testConfigOverrides-before-invalid.js (1 of 1) + Running: before-invalid.js (1 of 1) runs all tests @@ -363,60 +293,56 @@ Instead the value was: \`"null"\` 3) runs all tests context config overrides throws error runs: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. Instead the value was: \`"1"\` https://on.cypress.io/config - Error [stack trace lines] 4) runs all tests nested contexts overrides throws error at the correct line number 1st test fails on overrides: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`defaultCommandTimeout\` to be a number. Instead the value was: \`"500"\` https://on.cypress.io/config - Error [stack trace lines] 5) runs all tests nested contexts overrides throws error at the correct line number 2nd test fails on overrides: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`defaultCommandTimeout\` to be a number. Instead the value was: \`"500"\` https://on.cypress.io/config - Error [stack trace lines] 6) runs all tests nested contexts test override throws error at the correct line number: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your test-level override has the following validation error: Expected \`baseUrl\` to be a fully qualified URL (starting with \`http://\` or \`https://\`). Instead the value was: \`"not_an_http_url"\` https://on.cypress.io/config - Error [stack trace lines] 7) runs all tests throws error correctly when before hook "before all" hook for "test config override throws error": - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your test-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -425,20 +351,18 @@ Instead the value was: \`"1"\` https://on.cypress.io/config Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`throws error correctly when...\` - Error [stack trace lines] 8) runs all tests throws error correctly when beforeEach hook test config override throws error: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your test-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. Instead the value was: \`"1"\` https://on.cypress.io/config - Error [stack trace lines] @@ -455,17 +379,16 @@ https://on.cypress.io/config │ Screenshots: 2 │ │ Video: false │ │ Duration: X seconds │ - │ Spec Ran: testConfigOverrides-before-invalid.js │ + │ Spec Ran: before-invalid.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ (Screenshots) - - /XXX/XXX/XXX/cypress/screenshots/testConfigOverrides-before-invalid.js/runs all (1280x720) - tests -- inline test config override throws error (failed).png - - /XXX/XXX/XXX/cypress/screenshots/testConfigOverrides-before-invalid.js/runs all (1280x720) - tests -- inline test config override throws error when executed within cy cmd (f - ailed).png + - /XXX/XXX/XXX/cypress/screenshots/before-invalid.js/runs all tests -- inline test (2560x1328) + config override throws error (failed).png + - /XXX/XXX/XXX/cypress/screenshots/before-invalid.js/runs all tests -- inline test (2560x1328) + config override throws error when executed within cy cmd (failed).png ==================================================================================================== @@ -475,15 +398,14 @@ https://on.cypress.io/config Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ testConfigOverrides-before-invalid. XX:XX 9 1 8 - - │ - │ js │ + │ ✖ before-invalid.js XX:XX 9 1 8 - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ✖ 1 of 1 failed (100%) XX:XX 9 1 8 - - ` -exports['testConfigOverrides / correctly fails when invalid config values for it.only [chrome,electron]'] = ` +exports['testConfigOverrides / correctly fails when invalid config values for it.only [firefox]'] = ` ==================================================================================================== @@ -492,14 +414,14 @@ exports['testConfigOverrides / correctly fails when invalid config values for it ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 1 found (testConfigOverrides-only-invalid.js) │ - │ Searched: cypress/e2e/testConfigOverrides-only-invalid.js │ + │ Specs: 1 found (only-invalid.js) │ + │ Searched: cypress/e2e/testConfigOverrides/only-invalid.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: testConfigOverrides-only-invalid.js (1 of 1) + Running: only-invalid.js (1 of 1) nested contexts @@ -513,14 +435,13 @@ exports['testConfigOverrides / correctly fails when invalid config values for it 1) nested contexts test override throws error at the correct line number: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. Instead the value was: \`"1"\` https://on.cypress.io/config - Error [stack trace lines] @@ -537,7 +458,7 @@ https://on.cypress.io/config │ Screenshots: 0 │ │ Video: false │ │ Duration: X seconds │ - │ Spec Ran: testConfigOverrides-only-invalid.js │ + │ Spec Ran: only-invalid.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -548,14 +469,14 @@ https://on.cypress.io/config Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ testConfigOverrides-only-invalid.js XX:XX 1 - 1 - - │ + │ ✖ only-invalid.js XX:XX 1 - 1 - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - ` -exports['testConfigOverrides / fails when passing invalid config values - [firefox]'] = ` +exports['testConfigOverrides / has originalTitle when skip due to browser config'] = ` ==================================================================================================== @@ -564,14 +485,76 @@ exports['testConfigOverrides / fails when passing invalid config values - [firef ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 1 found (testConfigOverrides-invalid.js) │ - │ Searched: cypress/e2e/testConfigOverrides-invalid.js │ + │ Specs: 1 found (skip-browser.js) │ + │ Searched: cypress/e2e/testConfigOverrides/skip-browser.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: testConfigOverrides-invalid.js (1 of 1) + Running: skip-browser.js (1 of 1) + + + suite + - has invalid testConfigOverrides (skipped due to browser) + + + 0 passing + 1 pending + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: true │ + │ Duration: X seconds │ + │ Spec Ran: skip-browser.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Video) + + - Started processing: Compressing to 32 CRF + - Finished processing: /XXX/XXX/XXX/cypress/videos/skip-browser.js.mp4 (X second) + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ skip-browser.js XX:XX 1 - - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 1 - - 1 - + + +` + +exports['testConfigOverrides / fails when passing invalid config values - [chrome,electron]'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (invalid.js) │ + │ Searched: cypress/e2e/testConfigOverrides/invalid.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: invalid.js (1 of 1) 1) inline test config override throws error @@ -612,52 +595,56 @@ Instead the value was: \`"null"\` 3) context config overrides throws error runs: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. Instead the value was: \`"1"\` https://on.cypress.io/config + Error [stack trace lines] 4) nested contexts overrides throws error at the correct line number 1st test fails on overrides: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`defaultCommandTimeout\` to be a number. Instead the value was: \`"500"\` https://on.cypress.io/config + Error [stack trace lines] 5) nested contexts overrides throws error at the correct line number 2nd test fails on overrides: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`defaultCommandTimeout\` to be a number. Instead the value was: \`"500"\` https://on.cypress.io/config + Error [stack trace lines] 6) nested contexts test override throws error at the correct line number: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your test-level override has the following validation error: Expected \`baseUrl\` to be a fully qualified URL (starting with \`http://\` or \`https://\`). Instead the value was: \`"not_an_http_url"\` https://on.cypress.io/config + Error [stack trace lines] 7) throws error correctly when before hook "before all" hook for "test config override throws error": - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your test-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -666,17 +653,19 @@ Instead the value was: \`"1"\` https://on.cypress.io/config Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`throws error correctly when...\` + Error [stack trace lines] 8) throws error correctly when beforeEach hook test config override throws error: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your test-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. Instead the value was: \`"1"\` https://on.cypress.io/config + Error [stack trace lines] @@ -693,16 +682,16 @@ https://on.cypress.io/config │ Screenshots: 2 │ │ Video: false │ │ Duration: X seconds │ - │ Spec Ran: testConfigOverrides-invalid.js │ + │ Spec Ran: invalid.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ (Screenshots) - - /XXX/XXX/XXX/cypress/screenshots/testConfigOverrides-invalid.js/inline test conf (1280x720) - ig override throws error (failed).png - - /XXX/XXX/XXX/cypress/screenshots/testConfigOverrides-invalid.js/inline test conf (1280x720) - ig override throws error when executed within cy cmd (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/inline test config override throws e (1280x720) + rror (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/inline test config override throws e (1280x720) + rror when executed within cy cmd (failed).png ==================================================================================================== @@ -712,14 +701,14 @@ https://on.cypress.io/config Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ testConfigOverrides-invalid.js XX:XX 9 1 8 - - │ + │ ✖ invalid.js XX:XX 9 1 8 - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ✖ 1 of 1 failed (100%) XX:XX 9 1 8 - - ` -exports['testConfigOverrides / fails when passing invalid config values with beforeEach - [firefox]'] = ` +exports['testConfigOverrides / fails when passing invalid config values with beforeEach - [chrome,electron]'] = ` ==================================================================================================== @@ -728,14 +717,14 @@ exports['testConfigOverrides / fails when passing invalid config values with bef ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 1 found (testConfigOverrides-before-invalid.js) │ - │ Searched: cypress/e2e/testConfigOverrides-before-invalid.js │ + │ Specs: 1 found (before-invalid.js) │ + │ Searched: cypress/e2e/testConfigOverrides/before-invalid.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: testConfigOverrides-before-invalid.js (1 of 1) + Running: before-invalid.js (1 of 1) runs all tests @@ -776,56 +765,60 @@ Instead the value was: \`"null"\` 3) runs all tests context config overrides throws error runs: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. Instead the value was: \`"1"\` https://on.cypress.io/config + Error [stack trace lines] 4) runs all tests nested contexts overrides throws error at the correct line number 1st test fails on overrides: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`defaultCommandTimeout\` to be a number. Instead the value was: \`"500"\` https://on.cypress.io/config + Error [stack trace lines] 5) runs all tests nested contexts overrides throws error at the correct line number 2nd test fails on overrides: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`defaultCommandTimeout\` to be a number. Instead the value was: \`"500"\` https://on.cypress.io/config + Error [stack trace lines] 6) runs all tests nested contexts test override throws error at the correct line number: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your test-level override has the following validation error: Expected \`baseUrl\` to be a fully qualified URL (starting with \`http://\` or \`https://\`). Instead the value was: \`"not_an_http_url"\` https://on.cypress.io/config + Error [stack trace lines] 7) runs all tests throws error correctly when before hook "before all" hook for "test config override throws error": - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your test-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -834,18 +827,20 @@ Instead the value was: \`"1"\` https://on.cypress.io/config Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`throws error correctly when...\` + Error [stack trace lines] 8) runs all tests throws error correctly when beforeEach hook test config override throws error: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your test-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. Instead the value was: \`"1"\` https://on.cypress.io/config + Error [stack trace lines] @@ -862,17 +857,16 @@ https://on.cypress.io/config │ Screenshots: 2 │ │ Video: false │ │ Duration: X seconds │ - │ Spec Ran: testConfigOverrides-before-invalid.js │ + │ Spec Ran: before-invalid.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ (Screenshots) - - /XXX/XXX/XXX/cypress/screenshots/testConfigOverrides-before-invalid.js/runs all (1280x720) - tests -- inline test config override throws error (failed).png - - /XXX/XXX/XXX/cypress/screenshots/testConfigOverrides-before-invalid.js/runs all (1280x720) - tests -- inline test config override throws error when executed within cy cmd (f - ailed).png + - /XXX/XXX/XXX/cypress/screenshots/before-invalid.js/runs all tests -- inline test (1280x720) + config override throws error (failed).png + - /XXX/XXX/XXX/cypress/screenshots/before-invalid.js/runs all tests -- inline test (1280x720) + config override throws error when executed within cy cmd (failed).png ==================================================================================================== @@ -882,15 +876,14 @@ https://on.cypress.io/config Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ testConfigOverrides-before-invalid. XX:XX 9 1 8 - - │ - │ js │ + │ ✖ before-invalid.js XX:XX 9 1 8 - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ✖ 1 of 1 failed (100%) XX:XX 9 1 8 - - ` -exports['testConfigOverrides / correctly fails when invalid config values for it.only [firefox]'] = ` +exports['testConfigOverrides / correctly fails when invalid config values for it.only [chrome,electron]'] = ` ==================================================================================================== @@ -899,14 +892,14 @@ exports['testConfigOverrides / correctly fails when invalid config values for it ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 1 found (testConfigOverrides-only-invalid.js) │ - │ Searched: cypress/e2e/testConfigOverrides-only-invalid.js │ + │ Specs: 1 found (only-invalid.js) │ + │ Searched: cypress/e2e/testConfigOverrides/only-invalid.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: testConfigOverrides-only-invalid.js (1 of 1) + Running: only-invalid.js (1 of 1) nested contexts @@ -920,13 +913,14 @@ exports['testConfigOverrides / correctly fails when invalid config values for it 1) nested contexts test override throws error at the correct line number: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config override passed to your suite-level override has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. Instead the value was: \`"1"\` https://on.cypress.io/config + Error [stack trace lines] @@ -943,7 +937,7 @@ https://on.cypress.io/config │ Screenshots: 0 │ │ Video: false │ │ Duration: X seconds │ - │ Spec Ran: testConfigOverrides-only-invalid.js │ + │ Spec Ran: only-invalid.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -954,7 +948,7 @@ https://on.cypress.io/config Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ testConfigOverrides-only-invalid.js XX:XX 1 - 1 - - │ + │ ✖ only-invalid.js XX:XX 1 - 1 - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides-before-invalid.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/before-invalid.js similarity index 100% rename from system-tests/projects/e2e/cypress/e2e/testConfigOverrides-before-invalid.js rename to system-tests/projects/e2e/cypress/e2e/testConfigOverrides/before-invalid.js diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides-invalid-browser.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid-browser.js similarity index 100% rename from system-tests/projects/e2e/cypress/e2e/testConfigOverrides-invalid-browser.js rename to system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid-browser.js diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides-invalid.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid-suite-level-override.js similarity index 100% rename from system-tests/projects/e2e/cypress/e2e/testConfigOverrides-invalid.js rename to system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid-suite-level-override.js diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js new file mode 100644 index 000000000000..a2d714023f67 --- /dev/null +++ b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js @@ -0,0 +1,55 @@ +const shouldNotExecute = () => { + throw new Error('Test Override validation should have failed & it block should not have executed.') +} + +it('inline test config override throws error', () => { + Cypress.config('baseUrl', '') +}) + +it('inline test config override throws error when executed within cy cmd', () => { + cy.then(() => { + Cypress.config('baseUrl', 'null') + }) +}) + +describe('context config overrides throws error', { retries: '1' }, () => { + it('runs', () => { + shouldNotExecute() + }) +}) + +describe('nested contexts overrides throws error at the correct line number', { defaultCommandTimeout: '500' }, () => { + it('1st test fails on overrides', () => { + shouldNotExecute() + }) + + it('2nd test fails on overrides', () => { + shouldNotExecute() + }) +}) + +describe('nested contexts ', () => { + describe('test override', () => { + it('throws error at the correct line number', { baseUrl: 'not_an_http_url' }, () => { + shouldNotExecute() + }) + + it('does execute 2nd test', () => { + expect(1).to.eq(1) + }) + }) +}) + +describe('throws error correctly when before hook', () => { + before(() => {}) + it('test config override throws error', { retries: '1' }, () => { + shouldNotExecute() + }) +}) + +describe('throws error correctly when beforeEach hook', () => { + beforeEach(() => {}) + it('test config override throws error', { retries: '1' }, () => { + shouldNotExecute() + }) +}) diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides-only-invalid.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/only-invalid.js similarity index 100% rename from system-tests/projects/e2e/cypress/e2e/testConfigOverrides-only-invalid.js rename to system-tests/projects/e2e/cypress/e2e/testConfigOverrides/only-invalid.js diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides-skip-browser.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/skip-browser.js similarity index 100% rename from system-tests/projects/e2e/cypress/e2e/testConfigOverrides-skip-browser.js rename to system-tests/projects/e2e/cypress/e2e/testConfigOverrides/skip-browser.js diff --git a/system-tests/test/testConfigOverrides_spec.ts b/system-tests/test/testConfigOverrides_spec.ts index 5c7652e4ef7f..d06c9671a8a0 100644 --- a/system-tests/test/testConfigOverrides_spec.ts +++ b/system-tests/test/testConfigOverrides_spec.ts @@ -11,7 +11,7 @@ describe('testConfigOverrides', () => { systemTests.setup() systemTests.it('fails when passing invalid config value browser', { - spec: 'testConfigOverrides-invalid-browser.js', + spec: 'testConfigOverrides/invalid-browser.js', snapshot: true, expectedExitCode: 1, config: { @@ -20,7 +20,7 @@ describe('testConfigOverrides', () => { }) systemTests.it('has originalTitle when skip due to browser config', { - spec: 'testConfigOverrides-skip-browser.js', + spec: 'testConfigOverrides/skip-browser.js', snapshot: true, outputPath, browser: 'electron', @@ -42,7 +42,7 @@ describe('testConfigOverrides', () => { permutations.forEach((browserList) => { systemTests.it(`fails when passing invalid config values - [${browserList}]`, { - spec: 'testConfigOverrides-invalid.js', + spec: 'testConfigOverrides/invalid.js', snapshot: true, browser: browserList, expectedExitCode: 8, @@ -52,7 +52,7 @@ describe('testConfigOverrides', () => { }) systemTests.it(`fails when passing invalid config values with beforeEach - [${browserList}]`, { - spec: 'testConfigOverrides-before-invalid.js', + spec: 'testConfigOverrides/before-invalid.js', snapshot: true, browser: browserList, expectedExitCode: 8, @@ -62,7 +62,7 @@ describe('testConfigOverrides', () => { }) systemTests.it(`correctly fails when invalid config values for it.only [${browserList}]`, { - spec: 'testConfigOverrides-only-invalid.js', + spec: 'testConfigOverrides/only-invalid.js', snapshot: true, browser: browserList, expectedExitCode: 1, From 8677ec774daf4227b4641a88f9e81d0119101d17 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Fri, 29 Jul 2022 14:28:26 -0500 Subject: [PATCH 02/35] clean up based on tests --- packages/config/src/browser.ts | 24 +- packages/config/src/options.ts | 62 ++-- .../cypress/e2e/e2e/testConfigOverrides.cy.js | 2 +- packages/driver/src/cy/testConfigOverrides.ts | 2 + packages/driver/src/cypress.ts | 54 +++- packages/driver/src/cypress/error_messages.ts | 42 ++- .../testConfigOverrides_spec.ts.js | 285 ++++++++++++++---- .../e2e/testConfigOverrides/invalid.js | 49 +++ .../invalid_before_test.js | 17 ++ .../invalid_before_test_async.js | 17 ++ system-tests/test/testConfigOverrides_spec.ts | 26 +- 11 files changed, 458 insertions(+), 122 deletions(-) create mode 100644 system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid_before_test.js create mode 100644 system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid_before_test_async.js diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index f0dd2909b528..143e41bfb4fa 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -3,13 +3,13 @@ import Debug from 'debug' import { defaultSpecPattern, options, - OVERRIDE_LEVELS, + ALL_OVERRIDE_LEVELS, breakingOptions, breakingRootOptions, testingTypeBreakingOptions, } from './options' -import type { BreakingOption, BreakingOptionErrorKey, OverrideLevel } from './options' +import type { BreakingOption, BreakingOptionErrorKey, OverrideLevel, OverrideLevels } from './options' import type { TestingType } from '@packages/types' // this export has to be done in 2 lines because of a bug in babel typescript @@ -50,6 +50,12 @@ const restartOnChangeOptionsKeys = _.filter(options, 'requireRestartOnChange') const issuedWarnings = new Set() +export type InvalidTestOverrideResult = { + invalidConfigKey: string + overrideLevel: string + supportedOverrideLevels: OverrideLevels +} + export type BreakingErrResult = { name: string newName?: string @@ -182,10 +188,10 @@ export const validateNoBreakingTestingTypeConfig = (cfg: any, testingType: keyof return validateNoBreakingOptions(options, cfg, onWarning, onErr, testingType) } -export const validateOverridableAtTestTest = (config: any, overrideLevel: OverrideLevel, onErr: (errorKey: string, invalidConfigKey: string) => void) => { +export const validateOverridableAtTestTest = (config: any, overrideLevel: Exclude, onErr: (result: InvalidTestOverrideResult) => void) => { Object.keys(config).some((configKey) => { - const runTimeValidation = OVERRIDE_LEVELS.includes(overrideLevel) - const overrideLevels = testOverrideLevels[configKey] + const runTimeValidation = ALL_OVERRIDE_LEVELS.includes(overrideLevel) + const overrideLevels = testOverrideLevels[configKey] as OverrideLevels if (!overrideLevels) { // non-cypress configuration option. skip validation @@ -193,9 +199,11 @@ export const validateOverridableAtTestTest = (config: any, overrideLevel: Overri } if (runTimeValidation && (overrideLevels === 'never' || !overrideLevels.includes(overrideLevel))) { - const errPath = overrideLevel === 'runTime' ? 'config.invalid_cypress_config_api_override' : 'config.invalid_test_config_override' - - onErr(errPath, configKey) + onErr({ + invalidConfigKey: configKey, + overrideLevel, + supportedOverrideLevels: overrideLevels, + }) } }) } diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 9a463e0f8549..04e4df98966e 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -30,9 +30,11 @@ export type BreakingOptionErrorKey = | 'TEST_FILES_RENAMED' // The test-time override levels -export type OverrideLevel = 'supportFile' | 'suite' | 'test' | 'runTime' | 'never' +export type OverrideLevel = 'code' | 'test:before:run' | 'test:before:run:async' | 'suite' | 'test' | 'runTime' | 'never' -export const OVERRIDE_LEVELS = ['supportFile', 'suite', 'test', 'runTime'] as OverrideLevel[] +export type OverrideLevels = Array<'code' | 'test:before:run' | 'test:before:run:async' | 'suite' | 'test' | 'runTime'> | 'never' + +export const ALL_OVERRIDE_LEVELS: OverrideLevels = ['code', 'test:before:run', 'test:before:run:async', 'suite', 'test', 'runTime'] interface ConfigOption { name: string @@ -44,7 +46,7 @@ interface ConfigOption { * it indicates the configuration value cannot be override via suite-/test-specific * overrides or at run-time with Cypress.Config(). */ - overrideLevels?: Exclude[] | 'never' + overrideLevels?: OverrideLevels } interface DriverConfigOption extends ConfigOption { @@ -124,7 +126,7 @@ const driverConfigOptions: Array = [ name: 'animationDistanceThreshold', defaultValue: 5, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'arch', defaultValue: () => os.arch(), @@ -133,13 +135,13 @@ const driverConfigOptions: Array = [ name: 'baseUrl', defaultValue: null, validation: validate.isFullyQualifiedUrl, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, requireRestartOnChange: 'server', }, { name: 'blockHosts', defaultValue: null, validation: validate.isStringOrArrayOfStrings, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'chromeWebSecurity', defaultValue: true, @@ -162,7 +164,7 @@ const driverConfigOptions: Array = [ name: 'defaultCommandTimeout', defaultValue: 4000, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'downloadsFolder', defaultValue: 'cypress/downloads', @@ -180,12 +182,12 @@ const driverConfigOptions: Array = [ name: 'env', defaultValue: {}, validation: validate.isPlainObject, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'execTimeout', defaultValue: 60000, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'experimentalFetchPolyfill', defaultValue: false, @@ -230,17 +232,17 @@ const driverConfigOptions: Array = [ name: 'excludeSpecPattern', defaultValue: (options: Record = {}) => options.testingType === 'component' ? ['**/__snapshots__/*', '**/__image_snapshots__/*'] : '*.hot-update.js', validation: validate.isStringOrArrayOfStrings, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'includeShadowDom', defaultValue: false, validation: validate.isBoolean, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'keystrokeDelay', defaultValue: 0, validation: validate.isNumberOrFalse, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'modifyObstructiveCode', defaultValue: true, @@ -253,7 +255,7 @@ const driverConfigOptions: Array = [ name: 'numTestsKeptInMemory', defaultValue: 50, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'platform', defaultValue: () => os.platform(), @@ -262,37 +264,37 @@ const driverConfigOptions: Array = [ name: 'pageLoadTimeout', defaultValue: 60000, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'port', defaultValue: null, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'projectId', defaultValue: null, validation: validate.isString, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'redirectionLimit', defaultValue: 20, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'reporter', defaultValue: 'spec', validation: validate.isString, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'reporterOptions', defaultValue: null, validation: validate.isPlainObject, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'requestTimeout', defaultValue: 5000, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'resolvedNodePath', defaultValue: null, @@ -305,7 +307,7 @@ const driverConfigOptions: Array = [ name: 'responseTimeout', defaultValue: 30000, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'retries', defaultValue: { @@ -313,12 +315,12 @@ const driverConfigOptions: Array = [ openMode: 0, }, validation: validate.isValidRetriesConfig, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'screenshotOnRunFailure', defaultValue: true, validation: validate.isBoolean, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'screenshotsFolder', defaultValue: 'cypress/screenshots', @@ -329,12 +331,12 @@ const driverConfigOptions: Array = [ name: 'slowTestThreshold', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 250 : 10000, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'scrollBehavior', defaultValue: 'top', validation: validate.isOneOf('center', 'top', 'bottom', 'nearest', false), - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'supportFile', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 'cypress/support/component.{js,jsx,ts,tsx}' : 'cypress/support/e2e.{js,jsx,ts,tsx}', @@ -350,7 +352,7 @@ const driverConfigOptions: Array = [ name: 'taskTimeout', defaultValue: 60000, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'testIsolation', // TODO: When experimentalSessionAndOrigin is removed and released as GA, @@ -359,7 +361,7 @@ const driverConfigOptions: Array = [ // strict by default when experimentalSessionAndOrigin=true defaultValue: 'default', validation: validate.isOneOf('default', 'strict'), - overrideLevels: ['supportFile', 'suite'], + overrideLevels: ['suite'], }, { name: 'trashAssetsBeforeRuns', defaultValue: true, @@ -390,17 +392,17 @@ const driverConfigOptions: Array = [ name: 'viewportHeight', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 500 : 660, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'viewportWidth', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 500 : 1000, validation: validate.isNumber, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'waitForAnimations', defaultValue: true, validation: validate.isBoolean, - overrideLevels: ['supportFile', 'suite', 'test', 'runTime'], + overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'watchForFileChanges', defaultValue: true, diff --git a/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js b/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js index 42386e13dc9e..77528a7c3387 100644 --- a/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js +++ b/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js @@ -385,7 +385,7 @@ describe('cannot set override configuration options that', () => { it('throws if mutating read-only config with Cypress.config()', (done) => { window.top.__cySkipValidateConfig = false cy.once('fail', (err) => { - expect(err.message).to.include('`Cypress.config()` cannot override `chromeWebSecurity` from the test-level.') + expect(err.message).to.include('`Cypress.config()` can never override `chromeWebSecurity` in a test at runTime') done() }) diff --git a/packages/driver/src/cy/testConfigOverrides.ts b/packages/driver/src/cy/testConfigOverrides.ts index a1d8c608e89e..26cab8f78d00 100644 --- a/packages/driver/src/cy/testConfigOverrides.ts +++ b/packages/driver/src/cy/testConfigOverrides.ts @@ -163,5 +163,7 @@ export class TestConfigOverride { if (Object.keys(resolvedTestConfig.unverifiedTestConfig).length > 0) { this.restoreTestConfigFn = mutateConfiguration(resolvedTestConfig, config, env) } + + resolvedTestConfig.applied = 'complete' } } diff --git a/packages/driver/src/cypress.ts b/packages/driver/src/cypress.ts index 5ad618c37753..45a806a3f109 100644 --- a/packages/driver/src/cypress.ts +++ b/packages/driver/src/cypress.ts @@ -238,22 +238,50 @@ class $Cypress { this.originalConfig = _.cloneDeep(config) this.config = $SetterGetter.create(config, (config) => { - const overrideLevel = this.state('runnable') ? 'runTime' : this.state('test')?._testConfig?.applied || 'supportFile' - const enforceConfigOverrideRules = this.isCrossOriginSpecBridge ? !window.__cySkipValidateConfig : !window.top!.__cySkipValidateConfig + let overrideLevel + const test = this.state('test') + + if (this.state('duringUserTestExecution')) { + overrideLevel = 'runTime' + } else if (test) { + if (test?._fired?.hasOwnProperty('runner:test:before:run:async')) { + overrideLevel = 'test:before:run:async' + } else if (test?._fired?.hasOwnProperty('runner:test:before:run')) { + overrideLevel = 'test:before:run' + } else { + overrideLevel = test._testConfig.applied // either suite or test + } + } else { + overrideLevel = 'code' + } - validateOverridableAtTestTest(config, overrideLevel, (errPath, invalidConfigKey) => { - const errMsg = $errUtils.errByPath(errPath, { - invalidConfigKey, - overrideLevel: this.state('runnable')?.type || this.state('test')?._testConfig?.applied, - }) + const skipValidation = this.isCrossOriginSpecBridge ? window.__cySkipValidateConfig : window.top!.__cySkipValidateConfig || false - if (enforceConfigOverrideRules) { - throw new (this.state('specWindow').Error)(errMsg) - } + if (!skipValidation) { + validateOverridableAtTestTest(config, overrideLevel, (validationResult) => { + const isReadOnly = validationResult.supportedOverrideLevels === 'never' + let errMsg - // eslint-disable-next-line no-console - console.warn(errMsg) - }) + if (overrideLevel === 'runTime') { + const errKey = isReadOnly ? 'config.cypress_config_api.cannot_override_readonly' : 'config.cypress_config_api.invalid_override' + + errMsg = $errUtils.errByPath(errKey, { + ...validationResult, + runnableType: this.state('runnable')?.type, + }) + } else if (overrideLevel.includes('test:before:run')) { + const errKey = isReadOnly ? 'config.event.cannot_override_readonly' : 'config.event.invalid_override' + + errMsg = $errUtils.errByPath(errKey, validationResult) + } else { + const errKey = isReadOnly ? 'config.test_config.cannot_override_readonly' : 'config.test_config.invalid_override' + + errMsg = $errUtils.errByPath(errKey, validationResult) + } + + throw new (this.state('specWindow').Error)(errMsg) + }) + } validateConfigValues(config, (errResult) => { const stringify = (str) => format(JSON.stringify(str)) diff --git a/packages/driver/src/cypress/error_messages.ts b/packages/driver/src/cypress/error_messages.ts index a98447eaf4fc..00288bfe16eb 100644 --- a/packages/driver/src/cypress/error_messages.ts +++ b/packages/driver/src/cypress/error_messages.ts @@ -266,21 +266,49 @@ export default { }, config: { + cypress_config_api: { + cannot_override_readonly: { + message: `\`Cypress.config()\` can never override \`{{invalidConfigKey}}\` in a {{runnableType}} at {{overrideLevel}} because it is a read-only configuration option.'`, + docsUrl: 'https://on.cypress.io/config', + }, + invalid_override (obj) { + return { + message: `\`Cypress.config()\` cannot override \`{{invalidConfigKey}}\` in a {{runnableType}} at {{overrideLevel}}. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.`, + docsUrl: 'https://on.cypress.io/config', + } + }, + }, invalid_argument: { message: `Setting the config via ${cmd('Cypress.config')} failed with the following validation error:\n\n{{errMsg}}`, docsUrl: 'https://on.cypress.io/config', }, invalid_test_override: { - message: `The config override passed to your {{overrideLevel}}-level override has the following validation error:\n\n{{errMsg}}`, + message: `The config passed to your {{overrideLevel}}-level overrides has the following validation error:\n\n{{errMsg}}`, docsUrl: 'https://on.cypress.io/config', }, - invalid_cypress_config_api_override: { - message: `\`Cypress.config()\` cannot override \`{{invalidConfigKey}}\` from the {{overrideLevel}}-level.`, - docsUrl: 'https://on.cypress.io/config', + event: { + cannot_override_readonly: { + message: `\`Cypress.config()\` can never override \`{{invalidConfigKey}}\` in a {{overrideLevel}} event handler because it is a read-only configuration option.'`, + docsUrl: 'https://on.cypress.io/config', + }, + invalid_override (obj) { + return { + message: `\`Cypress.config()\` cannot override \`{{invalidConfigKey}}\` in a {{overrideLevel}} event handler. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.`, + docsUrl: 'https://on.cypress.io/config', + } + }, }, - invalid_test_config_override: { - message: `\`{{invalidConfigKey}}\` configuration cannot been overridden from the {{overrideLevel}}-level.`, - docsUrl: 'https://on.cypress.io/config', + test_config: { + cannot_override_readonly: { + message: `The \`{{invalidConfigKey}}\` configuration cannot been overridden from a {{overrideLevel}}-level override because it is a read-only configuration option.`, + docsUrl: 'https://on.cypress.io/config', + }, + invalid_override (obj) { + return { + message: `The \`{{invalidConfigKey}}\` configuration cannot been overridden from a {{overrideLevel}}-level override. The \`{{invalidConfigKey}}\` option can only be set from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.`, + docsUrl: 'https://on.cypress.io/config', + } + }, }, }, diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index e9601d5aa6ec..1569a243fe3f 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -93,27 +93,43 @@ exports['testConfigOverrides / fails when passing invalid config values - [firef 1) inline test config override throws error 2) inline test config override throws error when executed within cy cmd + 3) throws error when invalid test-level override + 4) throws error when invalid config opt in Cypress.config() in test context config overrides throws error - 3) runs + 5) runs nested contexts overrides throws error at the correct line number - 4) 1st test fails on overrides - 5) 2nd test fails on overrides + 6) 1st test fails on overrides + 7) 2nd test fails on overrides nested contexts test override - 6) throws error at the correct line number + 8) throws error at the correct line number ✓ does execute 2nd test throws error correctly when before hook - 7) "before all" hook for "test config override throws error" + 9) "before all" hook for "test config override throws error" throws error correctly when beforeEach hook - 8) test config override throws error + 10) test config override throws error + + throws error when invalid config opt in Cypress.config() in before hook + 11) "before all" hook for "4" + + throws error when invalid config opt in Cypress.config() in beforeEach hook + 12) "before each" hook for "5" + + throws error when invalid config opt in Cypress.config() in after hook + 13) 5 + 14) "after all" hook for "5" + + throws error when invalid config opt in Cypress.config() in afterEach hook + 15) 5 + 16) "after each" hook for "5" 1 passing - 8 failing + 16 failing 1) inline test config override throws error: Error: Expected \`baseUrl\` to be a fully qualified URL (starting with \`http://\` or \`https://\`). @@ -127,9 +143,21 @@ Instead the value was: \`""\` Instead the value was: \`"null"\` [stack trace lines] - 3) context config overrides throws error + 3) throws error when invalid test-level override: + CypressError: The config passed to your test-level overrides has the following validation error: + +CypressError: The \`testIsolation\` configuration cannot been overridden from a test-level override. The \`testIsolation\` option can only be set from suite-level overrides. + +https://on.cypress.io/config + [stack trace lines] + + 4) throws error when invalid config opt in Cypress.config() in test: + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a test at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + [stack trace lines] + + 5) context config overrides throws error runs: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -138,9 +166,9 @@ Instead the value was: \`"1"\` https://on.cypress.io/config [stack trace lines] - 4) nested contexts overrides throws error at the correct line number + 6) nested contexts overrides throws error at the correct line number 1st test fails on overrides: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`defaultCommandTimeout\` to be a number. @@ -149,9 +177,9 @@ Instead the value was: \`"500"\` https://on.cypress.io/config [stack trace lines] - 5) nested contexts overrides throws error at the correct line number + 7) nested contexts overrides throws error at the correct line number 2nd test fails on overrides: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`defaultCommandTimeout\` to be a number. @@ -160,10 +188,10 @@ Instead the value was: \`"500"\` https://on.cypress.io/config [stack trace lines] - 6) nested contexts + 8) nested contexts test override throws error at the correct line number: - CypressError: The config override passed to your test-level override has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: Expected \`baseUrl\` to be a fully qualified URL (starting with \`http://\` or \`https://\`). @@ -172,9 +200,9 @@ Instead the value was: \`"not_an_http_url"\` https://on.cypress.io/config [stack trace lines] - 7) throws error correctly when before hook + 9) throws error correctly when before hook "before all" hook for "test config override throws error": - CypressError: The config override passed to your test-level override has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -185,9 +213,9 @@ https://on.cypress.io/config Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`throws error correctly when...\` [stack trace lines] - 8) throws error correctly when beforeEach hook + 10) throws error correctly when beforeEach hook test config override throws error: - CypressError: The config override passed to your test-level override has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -196,18 +224,56 @@ Instead the value was: \`"1"\` https://on.cypress.io/config [stack trace lines] + 11) throws error when invalid config opt in Cypress.config() in before hook + "before all" hook for "4": + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + +Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` + [stack trace lines] + + 12) throws error when invalid config opt in Cypress.config() in beforeEach hook + "before each" hook for "5": + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + +Because this error occurred during a \`before each\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` + [stack trace lines] + + 13) throws error when invalid config opt in Cypress.config() in after hook + 5: + Error: Test Override validation should have failed & it block should not have executed. + [stack trace lines] + + 14) throws error when invalid config opt in Cypress.config() in after hook + "after all" hook for "5": + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + +Because this error occurred during a \`after all\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` + [stack trace lines] + + 15) throws error when invalid config opt in Cypress.config() in afterEach hook + 5: + Error: Test Override validation should have failed & it block should not have executed. + [stack trace lines] + + 16) throws error when invalid config opt in Cypress.config() in afterEach hook + "after each" hook for "5": + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + +Because this error occurred during a \`after each\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` + [stack trace lines] + (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 9 │ + │ Tests: 15 │ │ Passing: 1 │ - │ Failing: 8 │ + │ Failing: 14 │ │ Pending: 0 │ │ Skipped: 0 │ - │ Screenshots: 2 │ + │ Screenshots: 9 │ │ Video: false │ │ Duration: X seconds │ │ Spec Ran: invalid.js │ @@ -220,6 +286,20 @@ https://on.cypress.io/config rror (failed).png - /XXX/XXX/XXX/cypress/screenshots/invalid.js/inline test config override throws e (2560x1328) rror when executed within cy cmd (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + in Cypress.config() in test (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + in Cypress.config() in before hook -- 4 -- before all hook (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + in Cypress.config() in beforeEach hook -- 5 -- before each hook (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + in Cypress.config() in after hook -- 5 (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + in Cypress.config() in after hook -- 5 -- after all hook (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + in Cypress.config() in afterEach hook -- 5 (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + in Cypress.config() in afterEach hook -- 5 -- after each hook (failed).png ==================================================================================================== @@ -229,9 +309,9 @@ https://on.cypress.io/config Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ invalid.js XX:XX 9 1 8 - - │ + │ ✖ invalid.js XX:XX 15 1 14 - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 1 failed (100%) XX:XX 9 1 8 - - + ✖ 1 of 1 failed (100%) XX:XX 15 1 14 - - ` @@ -293,7 +373,7 @@ Instead the value was: \`"null"\` 3) runs all tests context config overrides throws error runs: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -305,7 +385,7 @@ https://on.cypress.io/config 4) runs all tests nested contexts overrides throws error at the correct line number 1st test fails on overrides: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`defaultCommandTimeout\` to be a number. @@ -317,7 +397,7 @@ https://on.cypress.io/config 5) runs all tests nested contexts overrides throws error at the correct line number 2nd test fails on overrides: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`defaultCommandTimeout\` to be a number. @@ -330,7 +410,7 @@ https://on.cypress.io/config nested contexts test override throws error at the correct line number: - CypressError: The config override passed to your test-level override has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: Expected \`baseUrl\` to be a fully qualified URL (starting with \`http://\` or \`https://\`). @@ -342,7 +422,7 @@ https://on.cypress.io/config 7) runs all tests throws error correctly when before hook "before all" hook for "test config override throws error": - CypressError: The config override passed to your test-level override has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -356,7 +436,7 @@ Because this error occurred during a \`before all\` hook we are skipping the rem 8) runs all tests throws error correctly when beforeEach hook test config override throws error: - CypressError: The config override passed to your test-level override has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -435,7 +515,7 @@ exports['testConfigOverrides / correctly fails when invalid config values for it 1) nested contexts test override throws error at the correct line number: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -559,27 +639,43 @@ exports['testConfigOverrides / fails when passing invalid config values - [chrom 1) inline test config override throws error 2) inline test config override throws error when executed within cy cmd + 3) throws error when invalid test-level override + 4) throws error when invalid config opt in Cypress.config() in test context config overrides throws error - 3) runs + 5) runs nested contexts overrides throws error at the correct line number - 4) 1st test fails on overrides - 5) 2nd test fails on overrides + 6) 1st test fails on overrides + 7) 2nd test fails on overrides nested contexts test override - 6) throws error at the correct line number + 8) throws error at the correct line number ✓ does execute 2nd test throws error correctly when before hook - 7) "before all" hook for "test config override throws error" + 9) "before all" hook for "test config override throws error" throws error correctly when beforeEach hook - 8) test config override throws error + 10) test config override throws error + + throws error when invalid config opt in Cypress.config() in before hook + 11) "before all" hook for "4" + + throws error when invalid config opt in Cypress.config() in beforeEach hook + 12) "before each" hook for "5" + + throws error when invalid config opt in Cypress.config() in after hook + 13) 5 + 14) "after all" hook for "5" + + throws error when invalid config opt in Cypress.config() in afterEach hook + 15) 5 + 16) "after each" hook for "5" 1 passing - 8 failing + 16 failing 1) inline test config override throws error: Error: Expected \`baseUrl\` to be a fully qualified URL (starting with \`http://\` or \`https://\`). @@ -593,9 +689,22 @@ Instead the value was: \`""\` Instead the value was: \`"null"\` [stack trace lines] - 3) context config overrides throws error + 3) throws error when invalid test-level override: + CypressError: The config passed to your test-level overrides has the following validation error: + +CypressError: The \`testIsolation\` configuration cannot been overridden from a test-level override. The \`testIsolation\` option can only be set from suite-level overrides. + +https://on.cypress.io/config + Error + [stack trace lines] + + 4) throws error when invalid config opt in Cypress.config() in test: + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a test at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + [stack trace lines] + + 5) context config overrides throws error runs: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -605,9 +714,9 @@ https://on.cypress.io/config Error [stack trace lines] - 4) nested contexts overrides throws error at the correct line number + 6) nested contexts overrides throws error at the correct line number 1st test fails on overrides: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`defaultCommandTimeout\` to be a number. @@ -617,9 +726,9 @@ https://on.cypress.io/config Error [stack trace lines] - 5) nested contexts overrides throws error at the correct line number + 7) nested contexts overrides throws error at the correct line number 2nd test fails on overrides: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`defaultCommandTimeout\` to be a number. @@ -629,10 +738,10 @@ https://on.cypress.io/config Error [stack trace lines] - 6) nested contexts + 8) nested contexts test override throws error at the correct line number: - CypressError: The config override passed to your test-level override has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: Expected \`baseUrl\` to be a fully qualified URL (starting with \`http://\` or \`https://\`). @@ -642,9 +751,9 @@ https://on.cypress.io/config Error [stack trace lines] - 7) throws error correctly when before hook + 9) throws error correctly when before hook "before all" hook for "test config override throws error": - CypressError: The config override passed to your test-level override has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -656,9 +765,9 @@ Because this error occurred during a \`before all\` hook we are skipping the rem Error [stack trace lines] - 8) throws error correctly when beforeEach hook + 10) throws error correctly when beforeEach hook test config override throws error: - CypressError: The config override passed to your test-level override has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -668,18 +777,56 @@ https://on.cypress.io/config Error [stack trace lines] + 11) throws error when invalid config opt in Cypress.config() in before hook + "before all" hook for "4": + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + +Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` + [stack trace lines] + + 12) throws error when invalid config opt in Cypress.config() in beforeEach hook + "before each" hook for "5": + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + +Because this error occurred during a \`before each\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` + [stack trace lines] + + 13) throws error when invalid config opt in Cypress.config() in after hook + 5: + Error: Test Override validation should have failed & it block should not have executed. + [stack trace lines] + + 14) throws error when invalid config opt in Cypress.config() in after hook + "after all" hook for "5": + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + +Because this error occurred during a \`after all\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` + [stack trace lines] + + 15) throws error when invalid config opt in Cypress.config() in afterEach hook + 5: + Error: Test Override validation should have failed & it block should not have executed. + [stack trace lines] + + 16) throws error when invalid config opt in Cypress.config() in afterEach hook + "after each" hook for "5": + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + +Because this error occurred during a \`after each\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` + [stack trace lines] + (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 9 │ + │ Tests: 15 │ │ Passing: 1 │ - │ Failing: 8 │ + │ Failing: 14 │ │ Pending: 0 │ │ Skipped: 0 │ - │ Screenshots: 2 │ + │ Screenshots: 9 │ │ Video: false │ │ Duration: X seconds │ │ Spec Ran: invalid.js │ @@ -692,6 +839,20 @@ https://on.cypress.io/config rror (failed).png - /XXX/XXX/XXX/cypress/screenshots/invalid.js/inline test config override throws e (1280x720) rror when executed within cy cmd (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) + in Cypress.config() in test (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) + in Cypress.config() in before hook -- 4 -- before all hook (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) + in Cypress.config() in beforeEach hook -- 5 -- before each hook (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) + in Cypress.config() in after hook -- 5 (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) + in Cypress.config() in after hook -- 5 -- after all hook (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) + in Cypress.config() in afterEach hook -- 5 (failed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) + in Cypress.config() in afterEach hook -- 5 -- after each hook (failed).png ==================================================================================================== @@ -701,9 +862,9 @@ https://on.cypress.io/config Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ invalid.js XX:XX 9 1 8 - - │ + │ ✖ invalid.js XX:XX 15 1 14 - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 1 failed (100%) XX:XX 9 1 8 - - + ✖ 1 of 1 failed (100%) XX:XX 15 1 14 - - ` @@ -765,7 +926,7 @@ Instead the value was: \`"null"\` 3) runs all tests context config overrides throws error runs: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -778,7 +939,7 @@ https://on.cypress.io/config 4) runs all tests nested contexts overrides throws error at the correct line number 1st test fails on overrides: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`defaultCommandTimeout\` to be a number. @@ -791,7 +952,7 @@ https://on.cypress.io/config 5) runs all tests nested contexts overrides throws error at the correct line number 2nd test fails on overrides: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`defaultCommandTimeout\` to be a number. @@ -805,7 +966,7 @@ https://on.cypress.io/config nested contexts test override throws error at the correct line number: - CypressError: The config override passed to your test-level override has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: Expected \`baseUrl\` to be a fully qualified URL (starting with \`http://\` or \`https://\`). @@ -818,7 +979,7 @@ https://on.cypress.io/config 7) runs all tests throws error correctly when before hook "before all" hook for "test config override throws error": - CypressError: The config override passed to your test-level override has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -833,7 +994,7 @@ Because this error occurred during a \`before all\` hook we are skipping the rem 8) runs all tests throws error correctly when beforeEach hook test config override throws error: - CypressError: The config override passed to your test-level override has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. @@ -913,7 +1074,7 @@ exports['testConfigOverrides / correctly fails when invalid config values for it 1) nested contexts test override throws error at the correct line number: - CypressError: The config override passed to your suite-level override has the following validation error: + CypressError: The config passed to your suite-level overrides has the following validation error: Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js index a2d714023f67..22b9c2693aaa 100644 --- a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js +++ b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js @@ -53,3 +53,52 @@ describe('throws error correctly when beforeEach hook', () => { shouldNotExecute() }) }) + +it('throws error when invalid test-level override', { testIsolation: 'default' }, () => { + shouldNotExecute() +}) + +it('throws error when invalid config opt in Cypress.config() in test', () => { + Cypress.config({ testIsolation: 'default' }) + shouldNotExecute() +}) + +describe('throws error when invalid config opt in Cypress.config() in before hook', () => { + before(() => { + Cypress.config({ testIsolation: 'default' }) + }) + + it('4', () => { + shouldNotExecute() + }) +}) + +describe('throws error when invalid config opt in Cypress.config() in beforeEach hook', () => { + beforeEach(() => { + Cypress.config({ testIsolation: 'default' }) + }) + + it('5', () => { + shouldNotExecute() + }) +}) + +describe('throws error when invalid config opt in Cypress.config() in after hook', () => { + after(() => { + Cypress.config({ testIsolation: 'default' }) + }) + + it('5', () => { + shouldNotExecute() + }) +}) + +describe('throws error when invalid config opt in Cypress.config() in afterEach hook', () => { + afterEach(() => { + Cypress.config({ testIsolation: 'default' }) + }) + + it('5', () => { + shouldNotExecute() + }) +}) diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid_before_test.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid_before_test.js new file mode 100644 index 000000000000..d6df95fe6a86 --- /dev/null +++ b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid_before_test.js @@ -0,0 +1,17 @@ +const shouldNotExecute = () => { + throw new Error('Test Override validation should have failed & it block should not have executed.') +} + +Cypress.on('test:before:run', () => { + Cypress.config({ 'testIsolation': false }) +}) + +it('does not run', () => { + shouldNotExecute() +}) + +describe('nested', () => { + it('does not run 2', () => { + shouldNotExecute() + }) +}) diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid_before_test_async.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid_before_test_async.js new file mode 100644 index 000000000000..ef3e63644d5c --- /dev/null +++ b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid_before_test_async.js @@ -0,0 +1,17 @@ +const shouldNotExecute = () => { + throw new Error('Test Override validation should have failed & it block should not have executed.') +} + +Cypress.on('test:before:run:async', () => { + Cypress.config({ 'testIsolation': false }) +}) + +it('does not run', () => { + shouldNotExecute() +}) + +describe('nested', () => { + it('does not run 2', () => { + shouldNotExecute() + }) +}) diff --git a/system-tests/test/testConfigOverrides_spec.ts b/system-tests/test/testConfigOverrides_spec.ts index d06c9671a8a0..76290308f901 100644 --- a/system-tests/test/testConfigOverrides_spec.ts +++ b/system-tests/test/testConfigOverrides_spec.ts @@ -33,6 +33,30 @@ describe('testConfigOverrides', () => { }, }) + // systemTests.it.only('fails when setting invalid config opt with Cypress.config() in before:test:run', { + // spec: 'testConfigOverrides/invalid_before_test.js', + // snapshot: true, + // outputPath, + // browser: 'electron', + // expectedExitCode: 1, + // }) + + // systemTests.it('fails when setting invalid config opt with Cypress.config() in before:test:run:async', { + // spec: 'testConfigOverrides/invalid_before_test_async.js', + // snapshot: true, + // outputPath, + // browser: 'electron', + // expectedExitCode: 1, + // }) + + // systemTests.it('fails when setting invalid config opt with Cypress.config() in before:test:run:async', { + // spec: 'testConfigOverrides/invalid_code-level.js', + // snapshot: true, + // outputPath, + // browser: 'electron', + // expectedExitCode: 1, + // }) + // window.Error throws differently for firefox. break into // browser permutations for snapshot comparisons const permutations: BrowserName[][] = [ @@ -45,7 +69,7 @@ describe('testConfigOverrides', () => { spec: 'testConfigOverrides/invalid.js', snapshot: true, browser: browserList, - expectedExitCode: 8, + expectedExitCode: 14, config: { video: false, }, From 174b50cce86df0eb7deb0b7be3c714204cce220c Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Fri, 29 Jul 2022 15:38:17 -0500 Subject: [PATCH 03/35] fix unit tests & types --- .../src/data/ProjectConfigManager.ts | 2 +- .../cypress/e2e/e2e/origin/config_env.cy.ts | 2 +- packages/driver/src/util/config.ts | 9 +++- packages/server/lib/config.ts | 11 ++-- packages/server/test/unit/config_spec.js | 53 +++++++++++++++++++ 5 files changed, 71 insertions(+), 6 deletions(-) diff --git a/packages/data-context/src/data/ProjectConfigManager.ts b/packages/data-context/src/data/ProjectConfigManager.ts index 8e2e021c052b..fc2e5f5b85db 100644 --- a/packages/data-context/src/data/ProjectConfigManager.ts +++ b/packages/data-context/src/data/ProjectConfigManager.ts @@ -417,7 +417,7 @@ export class ProjectConfigManager { // @ts-ignore - we don't know if the browser is headed or headless at this point. // this is handled in open_project#launch. fullConfig.browsers = browsers - fullConfig.resolved.browsers = { 'value': fullConfig.browsers, 'from': 'runtime' } + fullConfig.resolved.browsers = { 'value': fullConfig.browsers, 'from': 'runTime' } } fullConfig.browsers = fullConfig.browsers?.map((browser) => { diff --git a/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts b/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts index 07cf55373869..7b45da4175d7 100644 --- a/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts @@ -21,7 +21,7 @@ // @ts-ignore window.top.__cySkipValidateConfig = false cy.on('fail', (err) => { - expect(err.message).to.include('`Cypress.config()` cannot mutate option `chromeWebSecurity` because it is a read-only property.') + expect(err.message).to.include('`Cypress.config()` can never override `chromeWebSecurity` in a test at runTime because it is a read-only configuration option.') resolve() }) diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index e8745f2ad55c..a7bb84a4ba1d 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -13,7 +13,14 @@ import { preprocessForSerialization } from './serialization' */ const omitConfigReadOnlyDifferences = (objectLikeConfig: Cypress.ObjectLike) => { Object.keys(objectLikeConfig).forEach((key) => { - const { overrideLevels = 'never' } = options.find((option) => option.name === key) || { } + const option = options.find((option) => option.name === key) + + // allow user defined config values + if (!option) { + return + } + + const overrideLevels = option.overrideLevels || 'never' if (overrideLevels === 'never') { delete objectLikeConfig[key] diff --git a/packages/server/lib/config.ts b/packages/server/lib/config.ts index 52a9edf2a383..26959a360c86 100644 --- a/packages/server/lib/config.ts +++ b/packages/server/lib/config.ts @@ -217,9 +217,14 @@ export function mergeDefaults ( // TODO: testIsolation should equal 'strict' by default when experimentalSessionAndOrigin=true // Once experimentalSessionAndOrigin is made GA, remove this logic and update the defaultValue // to be be 'strict' - if (testingType === 'e2e' && config.experimentalSessionAndOrigin && config.resolved.testIsolation.from === 'default') { - config.testIsolation = 'strict' - config.resolved.testIsolation.value = 'strict' + if (testingType === 'e2e' && config.experimentalSessionAndOrigin) { + if (config.rawJson.testIsolation) { + config.resolved.testIsolation.from = 'config' + } else { + config.testIsolation = 'strict' + config.resolved.testIsolation.value = 'strict' + config.resolved.testIsolation.from === 'default' + } } // We need to remove the nested propertied by testing type because it has been diff --git a/packages/server/test/unit/config_spec.js b/packages/server/test/unit/config_spec.js index 995eda3025ac..bdd613fc2fe4 100644 --- a/packages/server/test/unit/config_spec.js +++ b/packages/server/test/unit/config_spec.js @@ -1498,6 +1498,7 @@ describe('lib/config', () => { isInteractive: { value: true, from: 'default' }, keystrokeDelay: { value: 0, from: 'default' }, modifyObstructiveCode: { value: true, from: 'default' }, + nodeVersion: { value: undefined, from: 'default' }, numTestsKeptInMemory: { value: 50, from: 'default' }, pageLoadTimeout: { value: 60000, from: 'default' }, platform: { value: os.platform(), from: 'default' }, @@ -1513,10 +1514,12 @@ describe('lib/config', () => { retries: { value: { runMode: 0, openMode: 0 }, from: 'default' }, screenshotOnRunFailure: { value: true, from: 'default' }, screenshotsFolder: { value: 'cypress/screenshots', from: 'default' }, + specPattern: { value: 'cypress/e2e/**/*.cy.{js,jsx,ts,tsx}', from: 'default' }, slowTestThreshold: { value: 10000, from: 'default' }, supportFile: { value: false, from: 'config' }, supportFolder: { value: false, from: 'default' }, taskTimeout: { value: 60000, from: 'default' }, + testIsolation: { value: 'default', from: 'default' }, trashAssetsBeforeRuns: { value: true, from: 'default' }, userAgent: { value: null, from: 'default' }, video: { value: true, from: 'default' }, @@ -1606,6 +1609,7 @@ describe('lib/config', () => { isInteractive: { value: true, from: 'default' }, keystrokeDelay: { value: 0, from: 'default' }, modifyObstructiveCode: { value: true, from: 'default' }, + nodeVersion: { value: undefined, from: 'default' }, numTestsKeptInMemory: { value: 50, from: 'default' }, pageLoadTimeout: { value: 60000, from: 'default' }, platform: { value: os.platform(), from: 'default' }, @@ -1622,9 +1626,11 @@ describe('lib/config', () => { screenshotOnRunFailure: { value: true, from: 'default' }, screenshotsFolder: { value: 'cypress/screenshots', from: 'default' }, slowTestThreshold: { value: 10000, from: 'default' }, + specPattern: { value: 'cypress/e2e/**/*.cy.{js,jsx,ts,tsx}', from: 'default' }, supportFile: { value: false, from: 'config' }, supportFolder: { value: false, from: 'default' }, taskTimeout: { value: 60000, from: 'default' }, + testIsolation: { value: 'default', from: 'default' }, trashAssetsBeforeRuns: { value: true, from: 'default' }, userAgent: { value: null, from: 'default' }, video: { value: true, from: 'default' }, @@ -1639,6 +1645,53 @@ describe('lib/config', () => { }) }) }) + + it('sets testIsolation=strict by default when experimental=true and e2e testing', () => { + sinon.stub(configUtil, 'getProcessEnvVars').returns({}) + + const obj = { + projectRoot: '/foo/bar', + supportFile: false, + baseUrl: 'http://localhost:8080', + experimentalSessionAndOrigin: true, + } + + const options = { + testingType: 'e2e', + } + + return config.mergeDefaults(obj, options) + .then((cfg) => { + expect(cfg.resolved).to.have.property('experimentalSessionAndOrigin') + expect(cfg.resolved.experimentalSessionAndOrigin).to.deep.eq({ value: true, from: 'config' }) + expect(cfg.resolved).to.have.property('testIsolation') + expect(cfg.resolved.testIsolation).to.deep.eq({ value: 'strict', from: 'default' }) + }) + }) + + it('honors user config for testIsolation when experimental=true and e2e testing', () => { + sinon.stub(configUtil, 'getProcessEnvVars').returns({}) + + const obj = { + projectRoot: '/foo/bar', + supportFile: false, + baseUrl: 'http://localhost:8080', + experimentalSessionAndOrigin: true, + testIsolation: 'default', + } + + const options = { + testingType: 'e2e', + } + + return config.mergeDefaults(obj, options) + .then((cfg) => { + expect(cfg.resolved).to.have.property('experimentalSessionAndOrigin') + expect(cfg.resolved.experimentalSessionAndOrigin).to.deep.eq({ value: true, from: 'config' }) + expect(cfg.resolved).to.have.property('testIsolation') + expect(cfg.resolved.testIsolation).to.deep.eq({ value: 'default', from: 'config' }) + }) + }) }) }) From 3ca878d318c67e9900149cc8d4f0a93d628427a7 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Fri, 29 Jul 2022 18:29:16 -0500 Subject: [PATCH 04/35] system tests --- packages/driver/src/cypress.ts | 5 ++ .../testConfigOverrides_spec.ts.js | 83 +++++++++++++++++++ system-tests/test/testConfigOverrides_spec.ts | 25 ++---- 3 files changed, 97 insertions(+), 16 deletions(-) diff --git a/packages/driver/src/cypress.ts b/packages/driver/src/cypress.ts index 45a806a3f109..0f1bb64b5fdb 100644 --- a/packages/driver/src/cypress.ts +++ b/packages/driver/src/cypress.ts @@ -255,6 +255,11 @@ class $Cypress { overrideLevel = 'code' } + // FIXME: // bug in runner causes browser to hang in run mode when test:before:run throws an exception + if (overrideLevel === 'test:before:run' && this.config('isTextTerminal')) { + return + } + const skipValidation = this.isCrossOriginSpecBridge ? window.__cySkipValidateConfig : window.top!.__cySkipValidateConfig || false if (!skipValidation) { diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index 1569a243fe3f..864ad7cd8e05 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -1115,3 +1115,86 @@ https://on.cypress.io/config ` + +exports['testConfigOverrides / fails when setting invalid config opt with Cypress.config() in before:test:run:async'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (invalid_before_test_async.js) │ + │ Searched: cypress/e2e/testConfigOverrides/invalid_before_test_async.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: invalid_before_test_async.js (1 of 1) + + + 1) does not run + nested + 2) does not run 2 + + + 0 passing + 2 failing + + 1) does not run: + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a test:before:run:async event handler. The \`testIsolation\` option can only be overridden from suite-level overrides. + [stack trace lines] + + 2) nested + does not run 2: + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a test:before:run:async event handler. The \`testIsolation\` option can only be overridden from suite-level overrides. + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 0 │ + │ Failing: 2 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 2 │ + │ Video: true │ + │ Duration: X seconds │ + │ Spec Ran: invalid_before_test_async.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/invalid_before_test_async.js/does not run (fail (6400x2610) + ed).png + - /XXX/XXX/XXX/cypress/screenshots/invalid_before_test_async.js/nested -- does not (6400x2610) + run 2 (failed).png + + + (Video) + + - Started processing: Compressing to 32 CRF + - Finished processing: /XXX/XXX/XXX/cypress/videos/invalid_before_test_async.js.mp (X second) + 4 + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ invalid_before_test_async.js XX:XX 2 - 2 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 1 of 1 failed (100%) XX:XX 2 - 2 - - + + +` diff --git a/system-tests/test/testConfigOverrides_spec.ts b/system-tests/test/testConfigOverrides_spec.ts index 76290308f901..78dfd465d065 100644 --- a/system-tests/test/testConfigOverrides_spec.ts +++ b/system-tests/test/testConfigOverrides_spec.ts @@ -33,7 +33,8 @@ describe('testConfigOverrides', () => { }, }) - // systemTests.it.only('fails when setting invalid config opt with Cypress.config() in before:test:run', { + // FIXME: bug in runner causes browser to hang in run mode when test:before:run throws an exception + // systemTests.it.skip('fails when setting invalid config opt with Cypress.config() in before:test:run', { // spec: 'testConfigOverrides/invalid_before_test.js', // snapshot: true, // outputPath, @@ -41,21 +42,13 @@ describe('testConfigOverrides', () => { // expectedExitCode: 1, // }) - // systemTests.it('fails when setting invalid config opt with Cypress.config() in before:test:run:async', { - // spec: 'testConfigOverrides/invalid_before_test_async.js', - // snapshot: true, - // outputPath, - // browser: 'electron', - // expectedExitCode: 1, - // }) - - // systemTests.it('fails when setting invalid config opt with Cypress.config() in before:test:run:async', { - // spec: 'testConfigOverrides/invalid_code-level.js', - // snapshot: true, - // outputPath, - // browser: 'electron', - // expectedExitCode: 1, - // }) + systemTests.it('fails when setting invalid config opt with Cypress.config() in before:test:run:async', { + spec: 'testConfigOverrides/invalid_before_test_async.js', + snapshot: true, + outputPath, + browser: 'electron', + expectedExitCode: 2, + }) // window.Error throws differently for firefox. break into // browser permutations for snapshot comparisons From 819f71f12a0edd468d01c7271bebb3bbcbd4a186 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Fri, 29 Jul 2022 18:31:02 -0500 Subject: [PATCH 05/35] and fix --- packages/config/test/index.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/config/test/index.spec.ts b/packages/config/test/index.spec.ts index d861797406dc..5a19e2d8446b 100644 --- a/packages/config/test/index.spec.ts +++ b/packages/config/test/index.spec.ts @@ -4,7 +4,7 @@ import sinon from 'sinon' import sinonChai from 'sinon-chai' import * as configUtil from '../src/index' -import { OverrideLevel, OVERRIDE_LEVELS } from '../src/options' +import { OverrideLevel, ALL_OVERRIDE_LEVELS } from '../src/options' chai.use(sinonChai) const { expect } = chai @@ -241,7 +241,7 @@ describe('config/src/index', () => { }) }) - OVERRIDE_LEVELS.forEach((overrideLevel) => { + ALL_OVERRIDE_LEVELS.forEach((overrideLevel) => { it(`does not call onErr if validation succeeds from ${overrideLevel} level`, () => { const errorFn = sinon.spy() From 41fb01b8b488219bd8e0c8b886a0b75506e2d6c0 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Sun, 31 Jul 2022 16:20:07 -0500 Subject: [PATCH 06/35] fix tests & align runTime with runtime value passed to the dashboard --- packages/config/src/browser.ts | 4 +- packages/config/src/options.ts | 6 +-- packages/config/test/index.spec.ts | 54 ++++++++----------- .../src/data/ProjectConfigManager.ts | 2 +- .../cypress/e2e/e2e/origin/config_env.cy.ts | 2 +- .../cypress/e2e/e2e/testConfigOverrides.cy.js | 2 +- packages/driver/src/cypress.ts | 4 +- .../server/test/integration/cypress_spec.js | 4 +- packages/types/src/config.ts | 2 +- .../testConfigOverrides_spec.ts.js | 22 ++++---- 10 files changed, 46 insertions(+), 56 deletions(-) diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index 143e41bfb4fa..bff73073ba3b 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -190,7 +190,7 @@ export const validateNoBreakingTestingTypeConfig = (cfg: any, testingType: keyof export const validateOverridableAtTestTest = (config: any, overrideLevel: Exclude, onErr: (result: InvalidTestOverrideResult) => void) => { Object.keys(config).some((configKey) => { - const runTimeValidation = ALL_OVERRIDE_LEVELS.includes(overrideLevel) + const runtimeValidation = ALL_OVERRIDE_LEVELS.includes(overrideLevel) const overrideLevels = testOverrideLevels[configKey] as OverrideLevels if (!overrideLevels) { @@ -198,7 +198,7 @@ export const validateOverridableAtTestTest = (config: any, overrideLevel: Exclud return } - if (runTimeValidation && (overrideLevels === 'never' || !overrideLevels.includes(overrideLevel))) { + if (runtimeValidation && (overrideLevels === 'never' || !overrideLevels.includes(overrideLevel))) { onErr({ invalidConfigKey: configKey, overrideLevel, diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 04e4df98966e..cefc9a7bbc13 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -30,11 +30,11 @@ export type BreakingOptionErrorKey = | 'TEST_FILES_RENAMED' // The test-time override levels -export type OverrideLevel = 'code' | 'test:before:run' | 'test:before:run:async' | 'suite' | 'test' | 'runTime' | 'never' +export type OverrideLevel = 'code' | 'test:before:run' | 'test:before:run:async' | 'suite' | 'test' | 'runtime' | 'never' -export type OverrideLevels = Array<'code' | 'test:before:run' | 'test:before:run:async' | 'suite' | 'test' | 'runTime'> | 'never' +export type OverrideLevels = Array<'code' | 'test:before:run' | 'test:before:run:async' | 'suite' | 'test' | 'runtime'> | 'never' -export const ALL_OVERRIDE_LEVELS: OverrideLevels = ['code', 'test:before:run', 'test:before:run:async', 'suite', 'test', 'runTime'] +export const ALL_OVERRIDE_LEVELS: OverrideLevels = ['code', 'test:before:run', 'test:before:run:async', 'suite', 'test', 'runtime'] interface ConfigOption { name: string diff --git a/packages/config/test/index.spec.ts b/packages/config/test/index.spec.ts index 5a19e2d8446b..64ac2df995fa 100644 --- a/packages/config/test/index.spec.ts +++ b/packages/config/test/index.spec.ts @@ -191,57 +191,47 @@ describe('config/src/index', () => { }) describe('.validateOverridableAtTestTest', () => { - ['supportFile', 'suite', 'test'].forEach((overrideLevel) => { + ALL_OVERRIDE_LEVELS.forEach((overrideLevel) => { it(`calls onError handler if configuration cannot be overridden from ${overrideLevel} level`, () => { const errorFn = sinon.spy() configUtil.validateOverridableAtTestTest({ chromeWebSecurity: false }, overrideLevel as OverrideLevel, errorFn) expect(errorFn).to.have.callCount(1) - expect(errorFn).to.have.been.calledWithMatch('config.invalid_test_config_override', 'chromeWebSecurity') + expect(errorFn).to.have.been.calledWithMatch({ + invalidConfigKey: 'chromeWebSecurity', + overrideLevel, + supportedOverrideLevels: 'never', + }) }) }) - it('calls onError handler if configuration cannot be overridden from runtime level', () => { - const errorFn = sinon.spy() - - configUtil.validateOverridableAtTestTest({ chromeWebSecurity: false }, 'runTime', errorFn) - - expect(errorFn).to.have.callCount(1) - expect(errorFn).to.have.been.calledWithMatch('config.invalid_cypress_config_api_override', 'chromeWebSecurity') - }) - describe('testIsolation', () => { - ['supportFile', 'suite'].forEach((overrideLevel) => { - it(`does not calls onError handler if configuration can be overridden from ${overrideLevel} level`, () => { - const errorFn = sinon.spy() - - configUtil.validateOverridableAtTestTest({ testIsolation: 'strict' }, overrideLevel as OverrideLevel, errorFn) - - expect(errorFn).to.have.callCount(0) - }) - }) - - it('calls onError handler if configuration cannot be overridden from test level', () => { + it('does not calls onError handler if configuration can be overridden from suite level', () => { const errorFn = sinon.spy() - configUtil.validateOverridableAtTestTest({ testIsolation: false }, 'test', errorFn) + configUtil.validateOverridableAtTestTest({ testIsolation: 'strict' }, 'suite', errorFn) - expect(errorFn).to.have.callCount(1) - expect(errorFn).to.have.been.calledWithMatch('config.invalid_test_config_override', 'testIsolation') + expect(errorFn).to.have.callCount(0) }) - it('calls onError handler if configuration cannot be overridden from runTime level', () => { - const errorFn = sinon.spy() + ;['code', 'test:before:run', 'test:before:run:async', 'test', 'runtime'].forEach((overrideLevel) => { + it(`calls onError handler if configuration can be overridden from ${overrideLevel} level`, () => { + const errorFn = sinon.spy() - configUtil.validateOverridableAtTestTest({ testIsolation: false }, 'runTime', errorFn) + configUtil.validateOverridableAtTestTest({ testIsolation: false }, overrideLevel, errorFn) - expect(errorFn).to.have.callCount(1) - expect(errorFn).to.have.been.calledWithMatch('config.invalid_cypress_config_api_override', 'testIsolation') + expect(errorFn).to.have.callCount(1) + expect(errorFn).to.have.been.calledWithMatch({ + invalidConfigKey: 'testIsolation', + overrideLevel, + supportedOverrideLevels: ['suite'], + }) + }) }) }) - ALL_OVERRIDE_LEVELS.forEach((overrideLevel) => { + ALL_OVERRIDE_LEVELS.forEach((overrideLevel: OverrideLevel) => { it(`does not call onErr if validation succeeds from ${overrideLevel} level`, () => { const errorFn = sinon.spy() @@ -254,7 +244,7 @@ describe('config/src/index', () => { it('does not call onErr if configuration is a non-Cypress config option', () => { const errorFn = sinon.spy() - configUtil.validateOverridableAtTestTest({ foo: 'bar' }, 'runTime' as OverrideLevel, errorFn) + configUtil.validateOverridableAtTestTest({ foo: 'bar' }, 'runtime' as OverrideLevel, errorFn) expect(errorFn).to.have.callCount(0) }) diff --git a/packages/data-context/src/data/ProjectConfigManager.ts b/packages/data-context/src/data/ProjectConfigManager.ts index fc2e5f5b85db..8e2e021c052b 100644 --- a/packages/data-context/src/data/ProjectConfigManager.ts +++ b/packages/data-context/src/data/ProjectConfigManager.ts @@ -417,7 +417,7 @@ export class ProjectConfigManager { // @ts-ignore - we don't know if the browser is headed or headless at this point. // this is handled in open_project#launch. fullConfig.browsers = browsers - fullConfig.resolved.browsers = { 'value': fullConfig.browsers, 'from': 'runTime' } + fullConfig.resolved.browsers = { 'value': fullConfig.browsers, 'from': 'runtime' } } fullConfig.browsers = fullConfig.browsers?.map((browser) => { diff --git a/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts b/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts index 7b45da4175d7..ed2d97f70cc8 100644 --- a/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts @@ -21,7 +21,7 @@ // @ts-ignore window.top.__cySkipValidateConfig = false cy.on('fail', (err) => { - expect(err.message).to.include('`Cypress.config()` can never override `chromeWebSecurity` in a test at runTime because it is a read-only configuration option.') + expect(err.message).to.include('`Cypress.config()` can never override `chromeWebSecurity` in a test at runtime because it is a read-only configuration option.') resolve() }) diff --git a/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js b/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js index 77528a7c3387..a0462404f35d 100644 --- a/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js +++ b/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js @@ -385,7 +385,7 @@ describe('cannot set override configuration options that', () => { it('throws if mutating read-only config with Cypress.config()', (done) => { window.top.__cySkipValidateConfig = false cy.once('fail', (err) => { - expect(err.message).to.include('`Cypress.config()` can never override `chromeWebSecurity` in a test at runTime') + expect(err.message).to.include('`Cypress.config()` can never override `chromeWebSecurity` in a test at runtime') done() }) diff --git a/packages/driver/src/cypress.ts b/packages/driver/src/cypress.ts index 0f1bb64b5fdb..72631ea04422 100644 --- a/packages/driver/src/cypress.ts +++ b/packages/driver/src/cypress.ts @@ -242,7 +242,7 @@ class $Cypress { const test = this.state('test') if (this.state('duringUserTestExecution')) { - overrideLevel = 'runTime' + overrideLevel = 'runtime' } else if (test) { if (test?._fired?.hasOwnProperty('runner:test:before:run:async')) { overrideLevel = 'test:before:run:async' @@ -267,7 +267,7 @@ class $Cypress { const isReadOnly = validationResult.supportedOverrideLevels === 'never' let errMsg - if (overrideLevel === 'runTime') { + if (overrideLevel === 'runtime') { const errKey = isReadOnly ? 'config.cypress_config_api.cannot_override_readonly' : 'config.cypress_config_api.invalid_override' errMsg = $errUtils.errByPath(errKey, { diff --git a/packages/server/test/integration/cypress_spec.js b/packages/server/test/integration/cypress_spec.js index 68752f90fceb..2e78b58ad42d 100644 --- a/packages/server/test/integration/cypress_spec.js +++ b/packages/server/test/integration/cypress_spec.js @@ -1691,9 +1691,9 @@ describe('lib/cypress', () => { // this should be overriden by the env argument json.baseUrl = 'http://localhost:8080' - const { supportFile, specPattern, excludeSpecPattern, baseUrl, experimentalSessionAndOrigin, slowTestThreshold, ...rest } = json + const { supportFile, specPattern, excludeSpecPattern, baseUrl, experimentalSessionAndOrigin, slowTestThreshold, testIsolation, ...rest } = json - return settings.writeForTesting(this.todosPath, { ...rest, e2e: { baseUrl, experimentalSessionAndOrigin, supportFile, specPattern, excludeSpecPattern } }) + return settings.writeForTesting(this.todosPath, { ...rest, e2e: { baseUrl, experimentalSessionAndOrigin, supportFile, specPattern, testIsolation, excludeSpecPattern } }) }).then(async () => { await clearCtx() diff --git a/packages/types/src/config.ts b/packages/types/src/config.ts index 652363856afe..09005a94dcc5 100644 --- a/packages/types/src/config.ts +++ b/packages/types/src/config.ts @@ -2,7 +2,7 @@ import type { AllModeOptions } from '.' -export const RESOLVED_FROM = ['default', 'config', 'plugin', 'envFile', 'env', 'cli', 'runTime'] as const +export const RESOLVED_FROM = ['default', 'config', 'plugin', 'envFile', 'env', 'cli', 'runtime'] as const export type ResolvedConfigurationOptionSource = typeof RESOLVED_FROM[number] diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index 864ad7cd8e05..77622b03ac3a 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -54,7 +54,7 @@ We dynamically generated a new test to display this failure. (Screenshots) - - /XXX/XXX/XXX/cypress/screenshots/invalid-browser.js/An uncaught error was detect (2560x1328) + - /XXX/XXX/XXX/cypress/screenshots/invalid-browser.js/An uncaught error was detect (1280x720) ed outside of a test (failed).png @@ -152,7 +152,7 @@ https://on.cypress.io/config [stack trace lines] 4) throws error when invalid config opt in Cypress.config() in test: - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a test at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a test at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. [stack trace lines] 5) context config overrides throws error @@ -226,14 +226,14 @@ https://on.cypress.io/config 11) throws error when invalid config opt in Cypress.config() in before hook "before all" hook for "4": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] 12) throws error when invalid config opt in Cypress.config() in beforeEach hook "before each" hook for "5": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. Because this error occurred during a \`before each\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] @@ -245,7 +245,7 @@ Because this error occurred during a \`before each\` hook we are skipping the re 14) throws error when invalid config opt in Cypress.config() in after hook "after all" hook for "5": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. Because this error occurred during a \`after all\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] @@ -257,7 +257,7 @@ Because this error occurred during a \`after all\` hook we are skipping the rema 16) throws error when invalid config opt in Cypress.config() in afterEach hook "after each" hook for "5": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. Because this error occurred during a \`after each\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] @@ -699,7 +699,7 @@ https://on.cypress.io/config [stack trace lines] 4) throws error when invalid config opt in Cypress.config() in test: - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a test at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a test at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. [stack trace lines] 5) context config overrides throws error @@ -779,14 +779,14 @@ https://on.cypress.io/config 11) throws error when invalid config opt in Cypress.config() in before hook "before all" hook for "4": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] 12) throws error when invalid config opt in Cypress.config() in beforeEach hook "before each" hook for "5": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. Because this error occurred during a \`before each\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] @@ -798,7 +798,7 @@ Because this error occurred during a \`before each\` hook we are skipping the re 14) throws error when invalid config opt in Cypress.config() in after hook "after all" hook for "5": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. Because this error occurred during a \`after all\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] @@ -810,7 +810,7 @@ Because this error occurred during a \`after all\` hook we are skipping the rema 16) throws error when invalid config opt in Cypress.config() in afterEach hook "after each" hook for "5": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runTime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. Because this error occurred during a \`after each\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] From 6226550cb27201fa35d4b58b9456627406e2560a Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 1 Aug 2022 08:29:08 -0500 Subject: [PATCH 07/35] fix snapshots --- .../config/__snapshots__/index.spec.ts.js | 4 +-- .../testConfigOverrides_spec.ts.js | 28 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/config/__snapshots__/index.spec.ts.js b/packages/config/__snapshots__/index.spec.ts.js index 507b2488df46..e6ee21c95b12 100644 --- a/packages/config/__snapshots__/index.spec.ts.js +++ b/packages/config/__snapshots__/index.spec.ts.js @@ -84,7 +84,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys 1 "browsers": [], "clientRoute": "/__/", "configFile": "cypress.config.js", - "cypressBinaryRoot": "/Users/emily/dev/cypress", + "cypressBinaryRoot": "/root/cypress", "devServerPublicPathRoute": "/__cypress/src", "hosts": null, "isInteractive": true, @@ -165,7 +165,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f "browsers": [], "clientRoute": "/__/", "configFile": "cypress.config.js", - "cypressBinaryRoot": "/Users/emily/dev/cypress", + "cypressBinaryRoot": "/root/cypress", "devServerPublicPathRoute": "/__cypress/src", "hosts": null, "isInteractive": true, diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index 77622b03ac3a..a96838cf3320 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -54,7 +54,7 @@ We dynamically generated a new test to display this failure. (Screenshots) - - /XXX/XXX/XXX/cypress/screenshots/invalid-browser.js/An uncaught error was detect (1280x720) + - /XXX/XXX/XXX/cypress/screenshots/invalid-browser.js/An uncaught error was detect (1280x720) ed outside of a test (failed).png @@ -282,23 +282,23 @@ Because this error occurred during a \`after each\` hook we are skipping the rem (Screenshots) - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/inline test config override throws e (2560x1328) + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/inline test config override throws e (1280x720) rror (failed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/inline test config override throws e (2560x1328) + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/inline test config override throws e (1280x720) rror when executed within cy cmd (failed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in test (failed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in before hook -- 4 -- before all hook (failed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in beforeEach hook -- 5 -- before each hook (failed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in after hook -- 5 (failed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in after hook -- 5 -- after all hook (failed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in afterEach hook -- 5 (failed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (2560x1328) + - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in afterEach hook -- 5 -- after each hook (failed).png @@ -465,9 +465,9 @@ https://on.cypress.io/config (Screenshots) - - /XXX/XXX/XXX/cypress/screenshots/before-invalid.js/runs all tests -- inline test (2560x1328) + - /XXX/XXX/XXX/cypress/screenshots/before-invalid.js/runs all tests -- inline test (1280x720) config override throws error (failed).png - - /XXX/XXX/XXX/cypress/screenshots/before-invalid.js/runs all tests -- inline test (2560x1328) + - /XXX/XXX/XXX/cypress/screenshots/before-invalid.js/runs all tests -- inline test (1280x720) config override throws error when executed within cy cmd (failed).png @@ -1172,9 +1172,9 @@ exports['testConfigOverrides / fails when setting invalid config opt with Cypres (Screenshots) - - /XXX/XXX/XXX/cypress/screenshots/invalid_before_test_async.js/does not run (fail (6400x2610) + - /XXX/XXX/XXX/cypress/screenshots/invalid_before_test_async.js/does not run (fail (1280x720) ed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid_before_test_async.js/nested -- does not (6400x2610) + - /XXX/XXX/XXX/cypress/screenshots/invalid_before_test_async.js/nested -- does not (1280x720) run 2 (failed).png From 7b620339f040a27db2d654f105613bc55ebd8ef2 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 1 Aug 2022 08:53:58 -0500 Subject: [PATCH 08/35] add types --- cli/types/cypress.d.ts | 7 +++++++ system-tests/__snapshots__/issue_6407_spec.js | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index 07445dea6f01..83d5e2f87dd5 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2784,6 +2784,13 @@ declare namespace Cypress { * @default "cypress/support/{e2e|component}.js" */ supportFile: string | false + /** + * The test isolation level applied to ensure a clean slate between tests. + * - default - resets/clears aliases, intercepts, clock, viewport, cookies, and local storage before each test. + * - strict - all resets everything from default, plus clears the page by visiting 'about:blank' to ensure clean app state before each test. + * @default "default", however, when experimentalSessionAndOrigin=true, the default is "string" + */ + testIsolation: 'default' | 'strict' /** * Path to folder where videos will be saved after a headless or CI run * @default "cypress/videos" diff --git a/system-tests/__snapshots__/issue_6407_spec.js b/system-tests/__snapshots__/issue_6407_spec.js index ddc942ef9e17..539ec9d21e8f 100644 --- a/system-tests/__snapshots__/issue_6407_spec.js +++ b/system-tests/__snapshots__/issue_6407_spec.js @@ -23,9 +23,9 @@ exports['e2e issue 6407 throws if mutating read-only config with test configurat 1 failing 1) throws if mutating read-only config with test configuration: - CypressError: The config override passed to your test has the following validation error: + CypressError: The config passed to your test-level overrides has the following validation error: -CypressError: Cypress test configuration cannot mutate option \`chromeWebSecurity\` because it is a read-only property. +CypressError: The `chromeWebSecurity` configuration cannot been overridden from a test-level override because it is a read-only configuration option. https://on.cypress.io/config Error From 15ec8d84dfa8ec0b098af5e02c6fc49ce43ac63a Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 1 Aug 2022 11:02:27 -0500 Subject: [PATCH 09/35] fix --- system-tests/__snapshots__/issue_6407_spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-tests/__snapshots__/issue_6407_spec.js b/system-tests/__snapshots__/issue_6407_spec.js index 539ec9d21e8f..0109c576c5b4 100644 --- a/system-tests/__snapshots__/issue_6407_spec.js +++ b/system-tests/__snapshots__/issue_6407_spec.js @@ -25,7 +25,7 @@ exports['e2e issue 6407 throws if mutating read-only config with test configurat 1) throws if mutating read-only config with test configuration: CypressError: The config passed to your test-level overrides has the following validation error: -CypressError: The `chromeWebSecurity` configuration cannot been overridden from a test-level override because it is a read-only configuration option. +CypressError: The \`chromeWebSecurity\` configuration cannot been overridden from a code-level override because it is a read-only configuration option. https://on.cypress.io/config Error From dec1da8f1e15a20878c9dc1947b652112993a8b0 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Tue, 2 Aug 2022 09:05:42 -0500 Subject: [PATCH 10/35] Update cli/types/cypress.d.ts Co-authored-by: Lachlan Miller --- cli/types/cypress.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index 6cec001f3c90..024338ffeef6 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2788,7 +2788,7 @@ declare namespace Cypress { * The test isolation level applied to ensure a clean slate between tests. * - default - resets/clears aliases, intercepts, clock, viewport, cookies, and local storage before each test. * - strict - all resets everything from default, plus clears the page by visiting 'about:blank' to ensure clean app state before each test. - * @default "default", however, when experimentalSessionAndOrigin=true, the default is "string" + * @default "default", however, when experimentalSessionAndOrigin=true, the default is "strict" */ testIsolation: 'default' | 'strict' /** From 191105ba94ac63d4e1ed69e02bc83d3beab9ec12 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Tue, 2 Aug 2022 19:35:00 -0500 Subject: [PATCH 11/35] typescript changes per lachlan's review --- packages/config/src/browser.ts | 4 ++-- packages/config/src/options.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index bff73073ba3b..555ca8636cac 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -32,9 +32,9 @@ const dashesOrUnderscoresRe = /^(_-)+/ function createIndex> (arr: Array, keyKey: keyof T, valueKey: keyof T, defaultValueFallback?: any) { return _.reduce(arr, (memo: Record, item) => { if (item[valueKey] !== undefined) { - memo[item[keyKey] as string] = item[valueKey] + memo[item[keyKey]] = item[valueKey] } else { - memo[item[keyKey] as string] = defaultValueFallback + memo[item[keyKey]] = defaultValueFallback } return memo diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index cefc9a7bbc13..0f42da99d25d 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -30,11 +30,11 @@ export type BreakingOptionErrorKey = | 'TEST_FILES_RENAMED' // The test-time override levels -export type OverrideLevel = 'code' | 'test:before:run' | 'test:before:run:async' | 'suite' | 'test' | 'runtime' | 'never' +export const ALL_OVERRIDE_LEVELS = ['code', 'test:before:run', 'test:before:run:async', 'suite', 'test', 'runtime'] as const -export type OverrideLevels = Array<'code' | 'test:before:run' | 'test:before:run:async' | 'suite' | 'test' | 'runtime'> | 'never' +export type OverrideLevel = typeof ALL_OVERRIDE_LEVELS[number] | 'never' -export const ALL_OVERRIDE_LEVELS: OverrideLevels = ['code', 'test:before:run', 'test:before:run:async', 'suite', 'test', 'runtime'] +export type OverrideLevels = Array | 'never' interface ConfigOption { name: string From 92d5bc8ca269ff8ed7b97e8242ed3e288e7cdf2c Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Wed, 3 Aug 2022 14:17:20 -0500 Subject: [PATCH 12/35] change testIsolation's defaultValue from default to legacy --- cli/types/cypress.d.ts | 4 ++-- packages/config/__snapshots__/index.spec.ts.js | 4 ++-- packages/config/src/options.ts | 6 +++--- .../cypress/e2e/commands/sessions/sessions.cy.js | 2 +- packages/server/test/unit/config_spec.js | 8 ++++---- .../e2e/cypress/e2e/testConfigOverrides/invalid.js | 12 ++++++------ 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index 024338ffeef6..000839155195 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2786,11 +2786,11 @@ declare namespace Cypress { supportFile: string | false /** * The test isolation level applied to ensure a clean slate between tests. - * - default - resets/clears aliases, intercepts, clock, viewport, cookies, and local storage before each test. + * - legacy - resets/clears aliases, intercepts, clock, viewport, cookies, and local storage before each test. * - strict - all resets everything from default, plus clears the page by visiting 'about:blank' to ensure clean app state before each test. * @default "default", however, when experimentalSessionAndOrigin=true, the default is "strict" */ - testIsolation: 'default' | 'strict' + testIsolation: 'legacy' | 'strict' /** * Path to folder where videos will be saved after a headless or CI run * @default "cypress/videos" diff --git a/packages/config/__snapshots__/index.spec.ts.js b/packages/config/__snapshots__/index.spec.ts.js index e6ee21c95b12..61f61911bd9c 100644 --- a/packages/config/__snapshots__/index.spec.ts.js +++ b/packages/config/__snapshots__/index.spec.ts.js @@ -67,7 +67,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys 1 "supportFile": "cypress/support/e2e.{js,jsx,ts,tsx}", "supportFolder": false, "taskTimeout": 60000, - "testIsolation": "default", + "testIsolation": "legacy", "trashAssetsBeforeRuns": true, "userAgent": null, "video": true, @@ -148,7 +148,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f "supportFile": "cypress/support/e2e.{js,jsx,ts,tsx}", "supportFolder": false, "taskTimeout": 60000, - "testIsolation": "default", + "testIsolation": "legacy", "trashAssetsBeforeRuns": true, "userAgent": null, "video": true, diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 0f42da99d25d..72cff63e09e2 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -356,11 +356,11 @@ const driverConfigOptions: Array = [ }, { name: 'testIsolation', // TODO: When experimentalSessionAndOrigin is removed and released as GA, - // update the defaultValue from 'default' to 'strict' and + // update the defaultValue from 'legacy' to 'strict' and // update this code to remove the check/override specific to enable // strict by default when experimentalSessionAndOrigin=true - defaultValue: 'default', - validation: validate.isOneOf('default', 'strict'), + defaultValue: 'legacy', + validation: validate.isOneOf('legacy', 'strict'), overrideLevels: ['suite'], }, { name: 'trashAssetsBeforeRuns', diff --git a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js index 25c733ee19c3..8afefbda8e8c 100644 --- a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js +++ b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js @@ -75,7 +75,7 @@ describe('cy.session', { retries: 0 }, () => { }) }) - describe('testIsolation=default', { testIsolation: 'default' }, () => { + describe('testIsolation=legacy', { testIsolation: 'legacy' }, () => { it('does not clear page', () => { cy.visit('/fixtures/form.html') .then(async () => { diff --git a/packages/server/test/unit/config_spec.js b/packages/server/test/unit/config_spec.js index bdd613fc2fe4..d38502a62ccc 100644 --- a/packages/server/test/unit/config_spec.js +++ b/packages/server/test/unit/config_spec.js @@ -1519,7 +1519,7 @@ describe('lib/config', () => { supportFile: { value: false, from: 'config' }, supportFolder: { value: false, from: 'default' }, taskTimeout: { value: 60000, from: 'default' }, - testIsolation: { value: 'default', from: 'default' }, + testIsolation: { value: 'legacy', from: 'default' }, trashAssetsBeforeRuns: { value: true, from: 'default' }, userAgent: { value: null, from: 'default' }, video: { value: true, from: 'default' }, @@ -1630,7 +1630,7 @@ describe('lib/config', () => { supportFile: { value: false, from: 'config' }, supportFolder: { value: false, from: 'default' }, taskTimeout: { value: 60000, from: 'default' }, - testIsolation: { value: 'default', from: 'default' }, + testIsolation: { value: 'legacy', from: 'default' }, trashAssetsBeforeRuns: { value: true, from: 'default' }, userAgent: { value: null, from: 'default' }, video: { value: true, from: 'default' }, @@ -1677,7 +1677,7 @@ describe('lib/config', () => { supportFile: false, baseUrl: 'http://localhost:8080', experimentalSessionAndOrigin: true, - testIsolation: 'default', + testIsolation: 'legacy', } const options = { @@ -1689,7 +1689,7 @@ describe('lib/config', () => { expect(cfg.resolved).to.have.property('experimentalSessionAndOrigin') expect(cfg.resolved.experimentalSessionAndOrigin).to.deep.eq({ value: true, from: 'config' }) expect(cfg.resolved).to.have.property('testIsolation') - expect(cfg.resolved.testIsolation).to.deep.eq({ value: 'default', from: 'config' }) + expect(cfg.resolved.testIsolation).to.deep.eq({ value: 'legacy', from: 'config' }) }) }) }) diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js index 22b9c2693aaa..73bb69064896 100644 --- a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js +++ b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js @@ -54,18 +54,18 @@ describe('throws error correctly when beforeEach hook', () => { }) }) -it('throws error when invalid test-level override', { testIsolation: 'default' }, () => { +it('throws error when invalid test-level override', { testIsolation: 'legacy' }, () => { shouldNotExecute() }) it('throws error when invalid config opt in Cypress.config() in test', () => { - Cypress.config({ testIsolation: 'default' }) + Cypress.config({ testIsolation: 'legacy' }) shouldNotExecute() }) describe('throws error when invalid config opt in Cypress.config() in before hook', () => { before(() => { - Cypress.config({ testIsolation: 'default' }) + Cypress.config({ testIsolation: 'legacy' }) }) it('4', () => { @@ -75,7 +75,7 @@ describe('throws error when invalid config opt in Cypress.config() in before hoo describe('throws error when invalid config opt in Cypress.config() in beforeEach hook', () => { beforeEach(() => { - Cypress.config({ testIsolation: 'default' }) + Cypress.config({ testIsolation: 'legacy' }) }) it('5', () => { @@ -85,7 +85,7 @@ describe('throws error when invalid config opt in Cypress.config() in beforeEach describe('throws error when invalid config opt in Cypress.config() in after hook', () => { after(() => { - Cypress.config({ testIsolation: 'default' }) + Cypress.config({ testIsolation: 'legacy' }) }) it('5', () => { @@ -95,7 +95,7 @@ describe('throws error when invalid config opt in Cypress.config() in after hook describe('throws error when invalid config opt in Cypress.config() in afterEach hook', () => { afterEach(() => { - Cypress.config({ testIsolation: 'default' }) + Cypress.config({ testIsolation: 'legacy' }) }) it('5', () => { From fd510a91fc2795dc100cdd1d120910d342640e25 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Wed, 3 Aug 2022 14:23:35 -0500 Subject: [PATCH 13/35] add issues in comments and update missed default->legacy changes --- cli/types/cypress.d.ts | 4 ++-- packages/config/src/options.ts | 5 +++-- packages/driver/src/cypress.ts | 3 ++- packages/server/lib/config.ts | 3 ++- system-tests/test/testConfigOverrides_spec.ts | 3 ++- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index 000839155195..7da8ae2a46e2 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2787,8 +2787,8 @@ declare namespace Cypress { /** * The test isolation level applied to ensure a clean slate between tests. * - legacy - resets/clears aliases, intercepts, clock, viewport, cookies, and local storage before each test. - * - strict - all resets everything from default, plus clears the page by visiting 'about:blank' to ensure clean app state before each test. - * @default "default", however, when experimentalSessionAndOrigin=true, the default is "strict" + * - strict - all resets everything from legacy, plus clears the page by visiting 'about:blank' to ensure clean app state before each test. + * @default "legacy", however, when experimentalSessionAndOrigin=true, the default is "strict" */ testIsolation: 'legacy' | 'strict' /** diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 72cff63e09e2..7e37100bc5a4 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -34,7 +34,7 @@ export const ALL_OVERRIDE_LEVELS = ['code', 'test:before:run', 'test:before:run: export type OverrideLevel = typeof ALL_OVERRIDE_LEVELS[number] | 'never' -export type OverrideLevels = Array | 'never' +export type OverrideLevels = Readonly | 'never'> interface ConfigOption { name: string @@ -355,7 +355,8 @@ const driverConfigOptions: Array = [ overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'testIsolation', - // TODO: When experimentalSessionAndOrigin is removed and released as GA, + // TODO: https://github.com/cypress-io/cypress/issues/23093 + // When experimentalSessionAndOrigin is removed and released as GA, // update the defaultValue from 'legacy' to 'strict' and // update this code to remove the check/override specific to enable // strict by default when experimentalSessionAndOrigin=true diff --git a/packages/driver/src/cypress.ts b/packages/driver/src/cypress.ts index 31b531976df3..d5b862351c62 100644 --- a/packages/driver/src/cypress.ts +++ b/packages/driver/src/cypress.ts @@ -237,7 +237,8 @@ class $Cypress { overrideLevel = 'code' } - // FIXME: // bug in runner causes browser to hang in run mode when test:before:run throws an exception + // FIXME: https://github.com/cypress-io/cypress/issues/23039 + // bug in runner causes browser to hang in run mode when test:before:run throws an exception if (overrideLevel === 'test:before:run' && this.config('isTextTerminal')) { return } diff --git a/packages/server/lib/config.ts b/packages/server/lib/config.ts index 26959a360c86..34b32bc03d5e 100644 --- a/packages/server/lib/config.ts +++ b/packages/server/lib/config.ts @@ -214,7 +214,8 @@ export function mergeDefaults ( throw makeConfigError(errors.get(err, ...args)) }, testingType) - // TODO: testIsolation should equal 'strict' by default when experimentalSessionAndOrigin=true + // TODO: https://github.com/cypress-io/cypress/issues/23093 + // testIsolation should equal 'strict' by default when experimentalSessionAndOrigin=true // Once experimentalSessionAndOrigin is made GA, remove this logic and update the defaultValue // to be be 'strict' if (testingType === 'e2e' && config.experimentalSessionAndOrigin) { diff --git a/system-tests/test/testConfigOverrides_spec.ts b/system-tests/test/testConfigOverrides_spec.ts index 78dfd465d065..c9706598ef4b 100644 --- a/system-tests/test/testConfigOverrides_spec.ts +++ b/system-tests/test/testConfigOverrides_spec.ts @@ -33,7 +33,8 @@ describe('testConfigOverrides', () => { }, }) - // FIXME: bug in runner causes browser to hang in run mode when test:before:run throws an exception + // FIXME: https://github.com/cypress-io/cypress/issues/23039 + // bug in runner causes browser to hang in run mode when test:before:run throws an exception // systemTests.it.skip('fails when setting invalid config opt with Cypress.config() in before:test:run', { // spec: 'testConfigOverrides/invalid_before_test.js', // snapshot: true, From d0c1a0d6939cf8eda5c2afbf7bca71969ad720a7 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 8 Aug 2022 08:11:51 -0500 Subject: [PATCH 14/35] add missing todo --- packages/driver/src/cy/commands/cookies.ts | 3 ++- packages/driver/src/cy/commands/sessions/index.ts | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/driver/src/cy/commands/cookies.ts b/packages/driver/src/cy/commands/cookies.ts index 5b1edcdfd01b..5d60f9b5f7ad 100644 --- a/packages/driver/src/cy/commands/cookies.ts +++ b/packages/driver/src/cy/commands/cookies.ts @@ -178,7 +178,8 @@ export default function (Commands, Cypress, cy, state, config) { } } - // TODO: Cypress sessions will clear cookies on its own before each test. + // TODO: https://github.com/cypress-io/cypress/issues/23093 + // Cypress sessions will clear cookies on its own before each test. // Once experimentalSessionAndOrigin is made GA, remove this logic. Leave clearing // session data (cookies / local storage / session storage) to reset functionality. if (!Cypress.config('experimentalSessionAndOrigin')) { diff --git a/packages/driver/src/cy/commands/sessions/index.ts b/packages/driver/src/cy/commands/sessions/index.ts index 61cd30470a3e..31ee24a0f1af 100644 --- a/packages/driver/src/cy/commands/sessions/index.ts +++ b/packages/driver/src/cy/commands/sessions/index.ts @@ -47,7 +47,6 @@ export default function (Commands, Cypress, cy) { return clearPage .then(() => sessions.clearCurrentSessionData()) .then(() => Cypress.backend('reset:rendered:html:origins')) - .then(() => 'successfully cleared sessions data') } return From 9848dabdbe59a5c3544aa1522e2dd8d6af7bce1c Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 8 Aug 2022 11:07:31 -0500 Subject: [PATCH 15/35] break out driver config validation --- packages/config/src/browser.ts | 5 +- packages/config/src/validation.ts | 4 +- packages/driver/src/cypress.ts | 69 +------------- packages/driver/src/cypress/error_messages.ts | 70 +++++++------- packages/driver/src/cypress/setter_getter.ts | 10 +- packages/driver/src/util/config.ts | 95 +++++++++++++++++-- 6 files changed, 140 insertions(+), 113 deletions(-) diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index 555ca8636cac..771768092a45 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -9,8 +9,9 @@ import { testingTypeBreakingOptions, } from './options' -import type { BreakingOption, BreakingOptionErrorKey, OverrideLevel, OverrideLevels } from './options' import type { TestingType } from '@packages/types' +import type { BreakingOption, BreakingOptionErrorKey, OverrideLevel, OverrideLevels } from './options' +import type { ErrResult } from './validation' // this export has to be done in 2 lines because of a bug in babel typescript import * as validation from './validation' @@ -153,7 +154,7 @@ export const matchesConfigKey = (key: string) => { return } -export const validate = (cfg: any, onErr: (property: string) => void) => { +export const validate = (cfg: any, onErr: (property: ErrResult | string) => void) => { debug('validating configuration') return _.each(cfg, (value, key) => { diff --git a/packages/config/src/validation.ts b/packages/config/src/validation.ts index 87e8367006c3..f872a8a1afb4 100644 --- a/packages/config/src/validation.ts +++ b/packages/config/src/validation.ts @@ -13,7 +13,7 @@ const debug = Debug('cypress:server:validation') const str = JSON.stringify -type ErrResult = { +export type ErrResult = { key: string value: any type: string @@ -78,7 +78,7 @@ export const isValidBrowser = (browser: any): ErrResult | true => { /** * Validates the list of browsers. */ -export const isValidBrowserList = (key: string, browsers: any): ErrResult | true | string => { +export const isValidBrowserList = (_key: string, browsers: any): ErrResult | true | string => { debug('browsers %o', browsers) if (!browsers) { return 'Missing browsers list' diff --git a/packages/driver/src/cypress.ts b/packages/driver/src/cypress.ts index 25191fa42ca4..6e5032ac26a8 100644 --- a/packages/driver/src/cypress.ts +++ b/packages/driver/src/cypress.ts @@ -1,4 +1,3 @@ -import { validate as validateConfigValues, validateOverridableAtTestTest } from '@packages/config' import _ from 'lodash' import $ from 'jquery' import * as blobUtil from 'blob-util' @@ -26,6 +25,7 @@ import $Screenshot from './cypress/screenshot' import $SelectorPlayground from './cypress/selector_playground' import $Server from './cypress/server' import $SetterGetter from './cypress/setter_getter' +import { validateConfig } from './util/config' import $utils from './cypress/utils' import { $Chainer } from './cypress/chainer' @@ -238,71 +238,10 @@ class $Cypress { this.originalConfig = _.cloneDeep(config) this.config = $SetterGetter.create(config, (config) => { - let overrideLevel - const test = this.state('test') - - if (this.state('duringUserTestExecution')) { - overrideLevel = 'runtime' - } else if (test) { - if (test?._fired?.hasOwnProperty('runner:test:before:run:async')) { - overrideLevel = 'test:before:run:async' - } else if (test?._fired?.hasOwnProperty('runner:test:before:run')) { - overrideLevel = 'test:before:run' - } else { - overrideLevel = test._testConfig.applied // either suite or test - } - } else { - overrideLevel = 'code' - } - - // FIXME: https://github.com/cypress-io/cypress/issues/23039 - // bug in runner causes browser to hang in run mode when test:before:run throws an exception - if (overrideLevel === 'test:before:run' && this.config('isTextTerminal')) { - return - } - - const skipValidation = this.isCrossOriginSpecBridge ? window.__cySkipValidateConfig : window.top!.__cySkipValidateConfig || false - - if (!skipValidation) { - validateOverridableAtTestTest(config, overrideLevel, (validationResult) => { - const isReadOnly = validationResult.supportedOverrideLevels === 'never' - let errMsg - - if (overrideLevel === 'runtime') { - const errKey = isReadOnly ? 'config.cypress_config_api.cannot_override_readonly' : 'config.cypress_config_api.invalid_override' - - errMsg = $errUtils.errByPath(errKey, { - ...validationResult, - runnableType: this.state('runnable')?.type, - }) - } else if (overrideLevel.includes('test:before:run')) { - const errKey = isReadOnly ? 'config.event.cannot_override_readonly' : 'config.event.invalid_override' - - errMsg = $errUtils.errByPath(errKey, validationResult) - } else { - const errKey = isReadOnly ? 'config.test_config.cannot_override_readonly' : 'config.test_config.invalid_override' - - errMsg = $errUtils.errByPath(errKey, validationResult) - } - - throw new (this.state('specWindow').Error)(errMsg) - }) - } - - validateConfigValues(config, (errResult) => { - const stringify = (str) => format(JSON.stringify(str)) - - const format = (str) => `\`${str}\`` - - // TODO: this does not use the @packages/error rewriting rules - // for stdout vs markdown - it always inserts backticks for markdown - // and those leak out into the stdout formatting. - const errMsg = _.isString(errResult) - ? errResult - : `Expected ${format(errResult.key)} to be ${errResult.type}.\n\nInstead the value was: ${stringify(errResult.value)}` + const skipConfigOverrideValidation = this.isCrossOriginSpecBridge ? window.__cySkipValidateConfig : window.top!.__cySkipValidateConfig + const isRunMode = this.config('isTextTerminal') - throw new (this.state('specWindow').Error)(errMsg) - }) + return validateConfig(this.state, config, isRunMode, skipConfigOverrideValidation) }) this.env = $SetterGetter.create(env) diff --git a/packages/driver/src/cypress/error_messages.ts b/packages/driver/src/cypress/error_messages.ts index 00288bfe16eb..2a3f02421c87 100644 --- a/packages/driver/src/cypress/error_messages.ts +++ b/packages/driver/src/cypress/error_messages.ts @@ -266,49 +266,51 @@ export default { }, config: { - cypress_config_api: { - cannot_override_readonly: { - message: `\`Cypress.config()\` can never override \`{{invalidConfigKey}}\` in a {{runnableType}} at {{overrideLevel}} because it is a read-only configuration option.'`, + cypress_config_api (obj) { + let message = `\`Cypress.config()\` cannot override \`{{invalidConfigKey}}\`` + + if (obj.overrideLevel === 'runtime') { + message += ` in a {{runnableType}} at {{overrideLevel}}` + } + + if (obj.isReadOnly) { + message += ' because it is a read-only configuration option.' + } else { + message += `. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.` + } + + return { + message, docsUrl: 'https://on.cypress.io/config', - }, - invalid_override (obj) { - return { - message: `\`Cypress.config()\` cannot override \`{{invalidConfigKey}}\` in a {{runnableType}} at {{overrideLevel}}. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.`, - docsUrl: 'https://on.cypress.io/config', - } - }, - }, - invalid_argument: { - message: `Setting the config via ${cmd('Cypress.config')} failed with the following validation error:\n\n{{errMsg}}`, - docsUrl: 'https://on.cypress.io/config', + } }, invalid_test_override: { message: `The config passed to your {{overrideLevel}}-level overrides has the following validation error:\n\n{{errMsg}}`, docsUrl: 'https://on.cypress.io/config', }, - event: { - cannot_override_readonly: { - message: `\`Cypress.config()\` can never override \`{{invalidConfigKey}}\` in a {{overrideLevel}} event handler because it is a read-only configuration option.'`, + invalid_event_override (obj) { + let message = `\`Cypress.config()\` can never override \`{{invalidConfigKey}}\` in a {{event}} event handler because it is a read-only configuration option.'` + + if (!obj.isReadOnly) { + message = `\`Cypress.config()\` cannot override \`{{invalidConfigKey}}\` in a {{event}} event handler. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.` + } + + return { + message, docsUrl: 'https://on.cypress.io/config', - }, - invalid_override (obj) { - return { - message: `\`Cypress.config()\` cannot override \`{{invalidConfigKey}}\` in a {{overrideLevel}} event handler. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.`, - docsUrl: 'https://on.cypress.io/config', - } - }, + } }, - test_config: { - cannot_override_readonly: { - message: `The \`{{invalidConfigKey}}\` configuration cannot been overridden from a {{overrideLevel}}-level override because it is a read-only configuration option.`, + invalid_mocha_config_override (obj) { + let message = `The \`{{invalidConfigKey}}\` configuration cannot been overridden from a {{overrideLevel}}-level override because it is a read-only configuration option.` + + if (!obj.isReadOnly) { + message = `The \`{{invalidConfigKey}}\` configuration cannot been overridden from a {{overrideLevel}}-level override. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.` + } + + return { + message, docsUrl: 'https://on.cypress.io/config', - }, - invalid_override (obj) { - return { - message: `The \`{{invalidConfigKey}}\` configuration cannot been overridden from a {{overrideLevel}}-level override. The \`{{invalidConfigKey}}\` option can only be set from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.`, - docsUrl: 'https://on.cypress.io/config', - } - }, + } }, }, diff --git a/packages/driver/src/cypress/setter_getter.ts b/packages/driver/src/cypress/setter_getter.ts index d9ed920770d6..47dbf20247c9 100644 --- a/packages/driver/src/cypress/setter_getter.ts +++ b/packages/driver/src/cypress/setter_getter.ts @@ -33,11 +33,15 @@ export default { ret = value } - validate && validate(obj) + let shouldSet = validate ? validate(obj) : true - extend(state, obj) + if (shouldSet) { + extend(state, obj) - return ret + return ret + } + + return {} } // return the getter / setter function interface diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index a7bb84a4ba1d..62edfd62808b 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -1,6 +1,9 @@ import _ from 'lodash' -import { options } from '@packages/config' +import { testOverrideLevels, validateConfigValues, validateOverridableAtTestTest } from '@packages/config/src/driver' import { preprocessForSerialization } from './serialization' +import $errUtils from '../cypress/error_utils' + +import type { ErrResult, OverrideLevel } from '@packages/config/src/driver' /** * Mutates the config/env object serialized from the other origin to omit read-only values @@ -12,18 +15,16 @@ import { preprocessForSerialization } from './serialization' * @returns a reference to the config/env object passed in */ const omitConfigReadOnlyDifferences = (objectLikeConfig: Cypress.ObjectLike) => { - Object.keys(objectLikeConfig).forEach((key) => { - const option = options.find((option) => option.name === key) + Object.keys(objectLikeConfig).forEach((configKey) => { + const overrideLevels = testOverrideLevels[configKey] // allow user defined config values - if (!option) { + if (!overrideLevels) { return } - const overrideLevels = option.overrideLevels || 'never' - if (overrideLevels === 'never') { - delete objectLikeConfig[key] + delete objectLikeConfig[configKey] } }) @@ -69,3 +70,83 @@ export const preprocessConfig = (config: Cypress.Config) => { export const preprocessEnv = (env: Cypress.ObjectLike) => { return preprocessForSerialization(env) as Cypress.Config } + +export const getOverrideLevel = (state): OverrideLevel => { + const test = state('test') + let overrideLevel + + if (state('duringUserTestExecution')) { + overrideLevel = 'runtime' + } else if (test) { + if (test?._fired?.hasOwnProperty('runner:test:before:run:async')) { + overrideLevel = 'test:before:run:async' + } else if (test?._fired?.hasOwnProperty('runner:test:before:run')) { + overrideLevel = 'test:before:run' + } else { + overrideLevel = test._testConfig.applied // either suite or test + } + } else { + overrideLevel = 'code' + } + + return overrideLevel as OverrideLevel +} + +export const validateConfig = (state: Record, config: Record, isRunMode: boolean, skipConfigOverrideValidation: boolean = false): boolean => { + const overrideLevel = getOverrideLevel(state) + + // FIXME: https://github.com/cypress-io/cypress/issues/23039 + // bug in runner causes browser to hang in run mode when test:before:run throws an exception + if (overrideLevel === 'test:before:run' && isRunMode) { + return false + } + + if (!skipConfigOverrideValidation) { + validateOverridableAtTestTest(config, overrideLevel, (validationResult) => { + const isReadOnly = validationResult.supportedOverrideLevels === 'never' + let errMsg + + if (overrideLevel === 'runtime' || overrideLevel === 'code') { + errMsg = $errUtils.errByPath('config.cypress_config_api', { + ...validationResult, + isReadOnly, + runnableType: state('runnable')?.type, + }) + // } else if (overrideLevel === 'event') { + } else if (overrideLevel.includes('test:before:run')) { + const test = state('test') + const event = _.last(Object.keys(test._fired)) + + errMsg = $errUtils.errByPath('config.invalid_event_override', { + ...validationResult, + isReadOnly, + event: event?.replace('runner:', ''), + }) + } else { + errMsg = $errUtils.errByPath('config.invalid_mocha_config_override', { + ...validationResult, + isReadOnly, + }) + } + + throw new (state('specWindow').Error)(errMsg) + }) + } + + validateConfigValues(config, (errResult: ErrResult | string) => { + const stringify = (str) => format(JSON.stringify(str)) + + const format = (str) => `\`${str}\`` + + // TODO: this does not use the @packages/error rewriting rules + // for stdout vs markdown - it always inserts back ticks for markdown + // and those leak out into the stdout formatting. + const errMsg = _.isString(errResult) + ? errResult + : `Expected ${format(errResult.key)} to be ${errResult.type}.\n\nInstead the value was: ${stringify(errResult.value)}` + + throw new (state('specWindow').Error)(errMsg) + }) + + return true +} From 840d2bcc27f74e58e1f007b69ba0a0da550f2e88 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 8 Aug 2022 11:26:30 -0500 Subject: [PATCH 16/35] whoops. fix types --- packages/config/src/browser.ts | 8 ++++++-- packages/config/src/options.ts | 2 +- packages/driver/src/util/config.ts | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index 771768092a45..cc01f6096e77 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -18,11 +18,13 @@ import * as validation from './validation' export { defaultSpecPattern, - validation, options, breakingOptions, BreakingOption, BreakingOptionErrorKey, + ErrResult, + OverrideLevel, + validation, } const debug = Debug('cypress:config:browser') @@ -46,7 +48,9 @@ const breakingKeys = _.map(breakingOptions, 'name') const defaultValues = createIndex(options, 'name', 'defaultValue') const publicConfigKeys = _(options).reject({ isInternal: true }).map('name').value() const validationRules = createIndex(options, 'name', 'validation') -const testOverrideLevels = createIndex(options, 'name', 'overrideLevels', 'never') + +export const testOverrideLevels = createIndex(options, 'name', 'overrideLevels', 'never') + const restartOnChangeOptionsKeys = _.filter(options, 'requireRestartOnChange') const issuedWarnings = new Set() diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 7e37100bc5a4..481e5bba2706 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -43,7 +43,7 @@ interface ConfigOption { requireRestartOnChange?: 'server' | 'browser' /** * The list of test-time overrides levels supported by the configuration option. When undefined, - * it indicates the configuration value cannot be override via suite-/test-specific + * it indicates the configuration value cannot be overridden via suite-/test-specific * overrides or at run-time with Cypress.Config(). */ overrideLevels?: OverrideLevels diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index 62edfd62808b..a1c60606f474 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -1,9 +1,9 @@ import _ from 'lodash' -import { testOverrideLevels, validateConfigValues, validateOverridableAtTestTest } from '@packages/config/src/driver' +import { testOverrideLevels, validate as validateConfigValues, validateOverridableAtTestTest } from '@packages/config' import { preprocessForSerialization } from './serialization' import $errUtils from '../cypress/error_utils' -import type { ErrResult, OverrideLevel } from '@packages/config/src/driver' +import type { ErrResult, OverrideLevel } from '@packages/config' /** * Mutates the config/env object serialized from the other origin to omit read-only values From 88aa9e59abf776c416e5f1b1bf910ce7f657fa68 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 8 Aug 2022 12:05:36 -0500 Subject: [PATCH 17/35] reduce specificity of event override levels --- packages/config/src/options.ts | 4 +- packages/driver/cypress/e2e/util/config.cy.js | 268 ++++++++++++++++++ packages/driver/src/util/config.ts | 26 +- 3 files changed, 284 insertions(+), 14 deletions(-) create mode 100644 packages/driver/cypress/e2e/util/config.cy.js diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 481e5bba2706..7f92048fd704 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -30,9 +30,9 @@ export type BreakingOptionErrorKey = | 'TEST_FILES_RENAMED' // The test-time override levels -export const ALL_OVERRIDE_LEVELS = ['code', 'test:before:run', 'test:before:run:async', 'suite', 'test', 'runtime'] as const +export const ALL_OVERRIDE_LEVELS = ['code', 'event', 'suite', 'test', 'runtime'] as const -export type OverrideLevel = typeof ALL_OVERRIDE_LEVELS[number] | 'never' +export type OverrideLevel = typeof ALL_OVERRIDE_LEVELS[number] export type OverrideLevels = Readonly | 'never'> diff --git a/packages/driver/cypress/e2e/util/config.cy.js b/packages/driver/cypress/e2e/util/config.cy.js new file mode 100644 index 000000000000..31f46cc32350 --- /dev/null +++ b/packages/driver/cypress/e2e/util/config.cy.js @@ -0,0 +1,268 @@ +import $SetterGetter from '@packages/driver/src/cypress/setter_getter' +import { getOverrideLevel, validateConfig } from '../../../src/util/config' + +describe('driver/src/cypress/validate_config', () => { + describe('getOverrideLevel', () => { + it('returns override level of runtime', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: true, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('runtime') + }) + + it('returns override level of test:before:run:async', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: { + _fired: { 'runner:test:before:run': true, 'runner:test:before:run:async': true }, + }, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('event') + }) + + it('returns override level of test:before:run', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: { + _fired: { 'runner:test:before:run': true }, + }, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('event') + }) + + it('returns override level of suite', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: { + _testConfig: { applied: 'suite' }, + }, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('suite') + }) + + it('returns override level of test', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: { + _testConfig: { applied: 'test' }, + }, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('test') + }) + + it('returns override level of code', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: undefined, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('code') + }) + }) + + describe('validate config', () => { + it('does not throw for non-cypress configuration options', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + }) + const isRunMode = true + const shouldSet = validateConfig(state, { hello: 'world' }, isRunMode) + + expect(shouldSet).to.be.true + }) + + it('skips validation in run mode when override level is test:before:run', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: { + _fired: { 'runner:test:before:run': true }, + }, + }) + const isRunMode = true + const shouldSet = validateConfig(state, {}, isRunMode) + + expect(shouldSet).to.be.false + }) + + it('run validation in open mode when override level is test:before:run', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: { + _fired: { 'runner:test:before:run': true }, + }, + }) + const isRunMode = false + const shouldSet = validateConfig(state, {}, isRunMode) + + expect(shouldSet).to.be.true + }) + + describe('ensures override level', () => { + describe('throws when override level is runtime', () => { + it('and config override is read-only', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: true, + specWindow: { Error }, + runnable: { type: 'suite' }, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('runtime') + const isRunMode = true + + expect(() => { + validateConfig(state, { chromeWebSecurity: true }, isRunMode) + }).to.throw(`\`Cypress.config()\` cannot override \`chromeWebSecurity\` in a suite at runtime because it is a read-only configuration option.`) + }) + + it('and config override invalid at runtime', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: true, + specWindow: { Error }, + runnable: { type: 'test' }, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('runtime') + const isRunMode = true + + expect(() => { + validateConfig(state, { testIsolation: 'strict' }, isRunMode) + }).to.throw(`\`Cypress.config()\` cannot override \`testIsolation\` in a test at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides.`) + }) + }) + + describe('throws when override level is associated to an event config override', () => { + it('and config override is read-only', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: { + _fired: { 'runner:test:before:run': true }, + }, + specWindow: { Error }, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('event') + const isRunMode = false + + expect(() => { + validateConfig(state, { chromeWebSecurity: true }, isRunMode) + }).to.throw(`\`Cypress.config()\` can never override \`chromeWebSecurity\` in a test:before:run event handler because it is a read-only configuration option.`) + }) + + it('and config override invalid for override level', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: { + _fired: { 'runner:test:before:run': true, 'runner:test:before:run:async': true }, + }, + specWindow: { Error }, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('event') + const isRunMode = true + + expect(() => { + validateConfig(state, { testIsolation: 'strict' }, isRunMode) + }).to.throw(`\`Cypress.config()\` cannot override \`testIsolation\` in a test:before:run:async event handler. The \`testIsolation\` option can only be overridden from suite-level overrides.`) + }) + }) + + describe('throws when override level is associated to a mocha config override', () => { + it('and config override is read-only', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: { + _testConfig: { applied: 'suite' }, + }, + specWindow: { Error }, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('suite') + const isRunMode = true + + expect(() => { + validateConfig(state, { chromeWebSecurity: true }, isRunMode) + }).to.throw(`The \`chromeWebSecurity\` configuration cannot been overridden from a suite-level override because it is a read-only configuration option.`) + }) + + it('and config override invalid for override level', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: { + _testConfig: { applied: 'test' }, + }, + specWindow: { Error }, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('test') + const isRunMode = true + + expect(() => { + validateConfig(state, { testIsolation: 'strict' }, isRunMode) + }).to.throw(`The \`testIsolation\` configuration cannot been overridden from a test-level override. The \`testIsolation\` option can only be overridden from suite-level overrides.`) + }) + }) + + describe('throws when override level is code', () => { + it('and config override is read-only', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + specWindow: { Error }, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('code') + const isRunMode = true + + expect(() => { + validateConfig(state, { chromeWebSecurity: true }, isRunMode) + }).to.throw(`\`Cypress.config()\` cannot override \`chromeWebSecurity\` because it is a read-only configuration option.`) + }) + + it('and config override invalid at runtime', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + specWindow: { Error }, + }) + const overrideLevel = getOverrideLevel(state) + + expect(overrideLevel).to.eq('code') + const isRunMode = true + + expect(() => { + validateConfig(state, { testIsolation: 'strict' }, isRunMode) + }).to.throw(`\`Cypress.config()\` cannot override \`testIsolation\`. The \`testIsolation\` option can only be overridden from suite-level overrides.`) + }) + }) + }) + + it('throws when invalid configuration value', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: true, + specWindow: { Error }, + runnable: { type: 'test' }, + }) + const isRunMode = true + + expect(() => { + validateConfig(state, { viewportHeight: '300' }, isRunMode) + }).to.throw(`Expected \`viewportHeight\` to be a number.\n\nInstead the value was: \`"300"\``) + }) + }) +}) diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index a1c60606f474..3e50d2a41d7d 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -3,6 +3,8 @@ import { testOverrideLevels, validate as validateConfigValues, validateOverridab import { preprocessForSerialization } from './serialization' import $errUtils from '../cypress/error_utils' +import type { StateFunc as State } from '../cypress/state' + import type { ErrResult, OverrideLevel } from '@packages/config' /** @@ -78,27 +80,29 @@ export const getOverrideLevel = (state): OverrideLevel => { if (state('duringUserTestExecution')) { overrideLevel = 'runtime' } else if (test) { - if (test?._fired?.hasOwnProperty('runner:test:before:run:async')) { - overrideLevel = 'test:before:run:async' - } else if (test?._fired?.hasOwnProperty('runner:test:before:run')) { - overrideLevel = 'test:before:run' + if (Object.keys(test?._fired || {}).length) { + overrideLevel = 'event' } else { overrideLevel = test._testConfig.applied // either suite or test } } else { - overrideLevel = 'code' + overrideLevel = 'code' // supportFile or spec load execution } return overrideLevel as OverrideLevel } -export const validateConfig = (state: Record, config: Record, isRunMode: boolean, skipConfigOverrideValidation: boolean = false): boolean => { +export const validateConfig = (state: State, config: Record, isRunMode: boolean, skipConfigOverrideValidation: boolean = false): boolean => { const overrideLevel = getOverrideLevel(state) // FIXME: https://github.com/cypress-io/cypress/issues/23039 // bug in runner causes browser to hang in run mode when test:before:run throws an exception - if (overrideLevel === 'test:before:run' && isRunMode) { - return false + if (overrideLevel === 'event' && isRunMode) { + const event = _.last(Object.keys(state('test')._fired || {})) + + if (event === 'runner:test:before:run') { + return false + } } if (!skipConfigOverrideValidation) { @@ -112,10 +116,8 @@ export const validateConfig = (state: Record, config: Record Date: Mon, 8 Aug 2022 12:18:07 -0500 Subject: [PATCH 18/35] legacy -> lax --- cli/types/cypress.d.ts | 8 ++++---- packages/config/__snapshots__/index.spec.ts.js | 4 ++-- packages/config/src/options.ts | 8 +++----- .../cypress/e2e/commands/sessions/sessions.cy.js | 2 +- packages/server/test/unit/config_spec.js | 8 ++++---- .../e2e/cypress/e2e/testConfigOverrides/invalid.js | 12 ++++++------ 6 files changed, 20 insertions(+), 22 deletions(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index b0137e23e47a..edfcf7c478f8 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2786,11 +2786,11 @@ declare namespace Cypress { supportFile: string | false /** * The test isolation level applied to ensure a clean slate between tests. - * - legacy - resets/clears aliases, intercepts, clock, viewport, cookies, and local storage before each test. - * - strict - all resets everything from legacy, plus clears the page by visiting 'about:blank' to ensure clean app state before each test. - * @default "legacy", however, when experimentalSessionAndOrigin=true, the default is "strict" + * - lax - resets/clears aliases, intercepts, clock, viewport, cookies, and local storage before each test. + * - strict - all resets everything from lax, plus clears the page by visiting 'about:blank' to ensure clean app state before each test. + * @default "lax", however, when experimentalSessionAndOrigin=true, the default is "strict" */ - testIsolation: 'legacy' | 'strict' + testIsolation: 'lax' | 'strict' /** * Path to folder where videos will be saved after a headless or CI run * @default "cypress/videos" diff --git a/packages/config/__snapshots__/index.spec.ts.js b/packages/config/__snapshots__/index.spec.ts.js index 61f61911bd9c..6951ade8a338 100644 --- a/packages/config/__snapshots__/index.spec.ts.js +++ b/packages/config/__snapshots__/index.spec.ts.js @@ -67,7 +67,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys 1 "supportFile": "cypress/support/e2e.{js,jsx,ts,tsx}", "supportFolder": false, "taskTimeout": 60000, - "testIsolation": "legacy", + "testIsolation": "lax", "trashAssetsBeforeRuns": true, "userAgent": null, "video": true, @@ -148,7 +148,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f "supportFile": "cypress/support/e2e.{js,jsx,ts,tsx}", "supportFolder": false, "taskTimeout": 60000, - "testIsolation": "legacy", + "testIsolation": "lax", "trashAssetsBeforeRuns": true, "userAgent": null, "video": true, diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 7f92048fd704..bc402979776b 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -269,12 +269,10 @@ const driverConfigOptions: Array = [ name: 'port', defaultValue: null, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'projectId', defaultValue: null, validation: validate.isString, - overrideLevels: ALL_OVERRIDE_LEVELS, }, { name: 'redirectionLimit', defaultValue: 20, @@ -357,11 +355,11 @@ const driverConfigOptions: Array = [ name: 'testIsolation', // TODO: https://github.com/cypress-io/cypress/issues/23093 // When experimentalSessionAndOrigin is removed and released as GA, - // update the defaultValue from 'legacy' to 'strict' and + // update the defaultValue from 'lax' to 'strict' and // update this code to remove the check/override specific to enable // strict by default when experimentalSessionAndOrigin=true - defaultValue: 'legacy', - validation: validate.isOneOf('legacy', 'strict'), + defaultValue: 'lax', + validation: validate.isOneOf('lax', 'strict'), overrideLevels: ['suite'], }, { name: 'trashAssetsBeforeRuns', diff --git a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js index 8afefbda8e8c..65866f610955 100644 --- a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js +++ b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js @@ -75,7 +75,7 @@ describe('cy.session', { retries: 0 }, () => { }) }) - describe('testIsolation=legacy', { testIsolation: 'legacy' }, () => { + describe('testIsolation=lax', { testIsolation: 'lax' }, () => { it('does not clear page', () => { cy.visit('/fixtures/form.html') .then(async () => { diff --git a/packages/server/test/unit/config_spec.js b/packages/server/test/unit/config_spec.js index d38502a62ccc..0b56262488e5 100644 --- a/packages/server/test/unit/config_spec.js +++ b/packages/server/test/unit/config_spec.js @@ -1519,7 +1519,7 @@ describe('lib/config', () => { supportFile: { value: false, from: 'config' }, supportFolder: { value: false, from: 'default' }, taskTimeout: { value: 60000, from: 'default' }, - testIsolation: { value: 'legacy', from: 'default' }, + testIsolation: { value: 'lax', from: 'default' }, trashAssetsBeforeRuns: { value: true, from: 'default' }, userAgent: { value: null, from: 'default' }, video: { value: true, from: 'default' }, @@ -1630,7 +1630,7 @@ describe('lib/config', () => { supportFile: { value: false, from: 'config' }, supportFolder: { value: false, from: 'default' }, taskTimeout: { value: 60000, from: 'default' }, - testIsolation: { value: 'legacy', from: 'default' }, + testIsolation: { value: 'lax', from: 'default' }, trashAssetsBeforeRuns: { value: true, from: 'default' }, userAgent: { value: null, from: 'default' }, video: { value: true, from: 'default' }, @@ -1677,7 +1677,7 @@ describe('lib/config', () => { supportFile: false, baseUrl: 'http://localhost:8080', experimentalSessionAndOrigin: true, - testIsolation: 'legacy', + testIsolation: 'lax', } const options = { @@ -1689,7 +1689,7 @@ describe('lib/config', () => { expect(cfg.resolved).to.have.property('experimentalSessionAndOrigin') expect(cfg.resolved.experimentalSessionAndOrigin).to.deep.eq({ value: true, from: 'config' }) expect(cfg.resolved).to.have.property('testIsolation') - expect(cfg.resolved.testIsolation).to.deep.eq({ value: 'legacy', from: 'config' }) + expect(cfg.resolved.testIsolation).to.deep.eq({ value: 'lax', from: 'config' }) }) }) }) diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js index 73bb69064896..856d84f38db5 100644 --- a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js +++ b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js @@ -54,18 +54,18 @@ describe('throws error correctly when beforeEach hook', () => { }) }) -it('throws error when invalid test-level override', { testIsolation: 'legacy' }, () => { +it('throws error when invalid test-level override', { testIsolation: 'lax' }, () => { shouldNotExecute() }) it('throws error when invalid config opt in Cypress.config() in test', () => { - Cypress.config({ testIsolation: 'legacy' }) + Cypress.config({ testIsolation: 'lax' }) shouldNotExecute() }) describe('throws error when invalid config opt in Cypress.config() in before hook', () => { before(() => { - Cypress.config({ testIsolation: 'legacy' }) + Cypress.config({ testIsolation: 'lax' }) }) it('4', () => { @@ -75,7 +75,7 @@ describe('throws error when invalid config opt in Cypress.config() in before hoo describe('throws error when invalid config opt in Cypress.config() in beforeEach hook', () => { beforeEach(() => { - Cypress.config({ testIsolation: 'legacy' }) + Cypress.config({ testIsolation: 'lax' }) }) it('5', () => { @@ -85,7 +85,7 @@ describe('throws error when invalid config opt in Cypress.config() in beforeEach describe('throws error when invalid config opt in Cypress.config() in after hook', () => { after(() => { - Cypress.config({ testIsolation: 'legacy' }) + Cypress.config({ testIsolation: 'lax' }) }) it('5', () => { @@ -95,7 +95,7 @@ describe('throws error when invalid config opt in Cypress.config() in after hook describe('throws error when invalid config opt in Cypress.config() in afterEach hook', () => { afterEach(() => { - Cypress.config({ testIsolation: 'legacy' }) + Cypress.config({ testIsolation: 'lax' }) }) it('5', () => { From 16125cdfc23fabae1ef94d09b3623b3473937131 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 8 Aug 2022 12:19:09 -0500 Subject: [PATCH 19/35] nit --- packages/config/src/options.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index bc402979776b..87fc23bde87e 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -55,7 +55,7 @@ interface DriverConfigOption extends ConfigOption { } // Cypress run-time options -interface RuntimeConfigOptions extends ConfigOption { +interface RuntimeConfigOption extends ConfigOption { defaultValue: any isInternal?: boolean } @@ -415,7 +415,7 @@ const driverConfigOptions: Array = [ }, ] -const runtimeOptions: Array = [ +const runtimeOptions: Array = [ { // Internal config field, useful to ignore the e2e specPattern set by the user // or the default one when looking fot CT, it needs to be a config property because after @@ -517,7 +517,7 @@ const runtimeOptions: Array = [ }, ] -export const options: Array = [ +export const options: Array = [ ...driverConfigOptions, ...runtimeOptions, ] From c8cd6688bc0f36115277b15d68b70afd6a4d6bc2 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 8 Aug 2022 13:03:25 -0500 Subject: [PATCH 20/35] fix --- packages/config/test/index.spec.ts | 4 ++-- .../driver/cypress/e2e/e2e/origin/config_env.cy.ts | 2 +- packages/driver/cypress/e2e/util/config.cy.js | 8 ++++---- packages/driver/src/cypress/error_messages.ts | 14 ++++++++------ packages/driver/src/util/config.ts | 8 +------- .../__snapshots__/testConfigOverrides_spec.ts.js | 4 ++-- 6 files changed, 18 insertions(+), 22 deletions(-) diff --git a/packages/config/test/index.spec.ts b/packages/config/test/index.spec.ts index 64ac2df995fa..b9fc04572a39 100644 --- a/packages/config/test/index.spec.ts +++ b/packages/config/test/index.spec.ts @@ -215,11 +215,11 @@ describe('config/src/index', () => { expect(errorFn).to.have.callCount(0) }) - ;['code', 'test:before:run', 'test:before:run:async', 'test', 'runtime'].forEach((overrideLevel) => { + ;['code', 'event', 'test', 'runtime'].forEach((overrideLevel) => { it(`calls onError handler if configuration can be overridden from ${overrideLevel} level`, () => { const errorFn = sinon.spy() - configUtil.validateOverridableAtTestTest({ testIsolation: false }, overrideLevel, errorFn) + configUtil.validateOverridableAtTestTest({ testIsolation: false }, overrideLevel as OverrideLevel, errorFn) expect(errorFn).to.have.callCount(1) expect(errorFn).to.have.been.calledWithMatch({ diff --git a/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts b/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts index ed2d97f70cc8..4fd382f12e02 100644 --- a/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts @@ -21,7 +21,7 @@ // @ts-ignore window.top.__cySkipValidateConfig = false cy.on('fail', (err) => { - expect(err.message).to.include('`Cypress.config()` can never override `chromeWebSecurity` in a test at runtime because it is a read-only configuration option.') + expect(err.message).to.include('`Cypress.config()` cannot override `chromeWebSecurity` in a test at runtime because it is a read-only configuration option.') resolve() }) diff --git a/packages/driver/cypress/e2e/util/config.cy.js b/packages/driver/cypress/e2e/util/config.cy.js index 31f46cc32350..5fd22aa9e299 100644 --- a/packages/driver/cypress/e2e/util/config.cy.js +++ b/packages/driver/cypress/e2e/util/config.cy.js @@ -123,7 +123,7 @@ describe('driver/src/cypress/validate_config', () => { expect(() => { validateConfig(state, { chromeWebSecurity: true }, isRunMode) - }).to.throw(`\`Cypress.config()\` cannot override \`chromeWebSecurity\` in a suite at runtime because it is a read-only configuration option.`) + }).to.throw(`\`Cypress.config()\` can never override \`chromeWebSecurity\` in a suite at runtime because it is a read-only configuration option.`) }) it('and config override invalid at runtime', () => { @@ -197,7 +197,7 @@ describe('driver/src/cypress/validate_config', () => { expect(() => { validateConfig(state, { chromeWebSecurity: true }, isRunMode) - }).to.throw(`The \`chromeWebSecurity\` configuration cannot been overridden from a suite-level override because it is a read-only configuration option.`) + }).to.throw(`The \`chromeWebSecurity\` configuration can never be overridden from a suite-level override because it is a read-only configuration option.`) }) it('and config override invalid for override level', () => { @@ -215,7 +215,7 @@ describe('driver/src/cypress/validate_config', () => { expect(() => { validateConfig(state, { testIsolation: 'strict' }, isRunMode) - }).to.throw(`The \`testIsolation\` configuration cannot been overridden from a test-level override. The \`testIsolation\` option can only be overridden from suite-level overrides.`) + }).to.throw(`The \`testIsolation\` configuration cannot be overridden from a test-level override. The \`testIsolation\` option can only be overridden from suite-level overrides.`) }) }) @@ -232,7 +232,7 @@ describe('driver/src/cypress/validate_config', () => { expect(() => { validateConfig(state, { chromeWebSecurity: true }, isRunMode) - }).to.throw(`\`Cypress.config()\` cannot override \`chromeWebSecurity\` because it is a read-only configuration option.`) + }).to.throw(`\`Cypress.config()\` can never override \`chromeWebSecurity\` because it is a read-only configuration option.`) }) it('and config override invalid at runtime', () => { diff --git a/packages/driver/src/cypress/error_messages.ts b/packages/driver/src/cypress/error_messages.ts index 2a3f02421c87..a11ba5675336 100644 --- a/packages/driver/src/cypress/error_messages.ts +++ b/packages/driver/src/cypress/error_messages.ts @@ -267,13 +267,15 @@ export default { config: { cypress_config_api (obj) { - let message = `\`Cypress.config()\` cannot override \`{{invalidConfigKey}}\`` + const isReadOnly = obj.supportedOverrideLevels === 'never' + + let message = `\`Cypress.config()\` ${isReadOnly ? 'can never' : 'cannot'} override \`{{invalidConfigKey}}\`` if (obj.overrideLevel === 'runtime') { message += ` in a {{runnableType}} at {{overrideLevel}}` } - if (obj.isReadOnly) { + if (isReadOnly) { message += ' because it is a read-only configuration option.' } else { message += `. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.` @@ -291,7 +293,7 @@ export default { invalid_event_override (obj) { let message = `\`Cypress.config()\` can never override \`{{invalidConfigKey}}\` in a {{event}} event handler because it is a read-only configuration option.'` - if (!obj.isReadOnly) { + if (obj.supportedOverrideLevels !== 'never') { message = `\`Cypress.config()\` cannot override \`{{invalidConfigKey}}\` in a {{event}} event handler. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.` } @@ -301,10 +303,10 @@ export default { } }, invalid_mocha_config_override (obj) { - let message = `The \`{{invalidConfigKey}}\` configuration cannot been overridden from a {{overrideLevel}}-level override because it is a read-only configuration option.` + let message = `The \`{{invalidConfigKey}}\` configuration can never be overridden from a {{overrideLevel}}-level override because it is a read-only configuration option.` - if (!obj.isReadOnly) { - message = `The \`{{invalidConfigKey}}\` configuration cannot been overridden from a {{overrideLevel}}-level override. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.` + if (obj.supportedOverrideLevels !== 'never') { + message = `The \`{{invalidConfigKey}}\` configuration cannot be overridden from a {{overrideLevel}}-level override. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.` } return { diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index 3e50d2a41d7d..300c34972d52 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -107,13 +107,11 @@ export const validateConfig = (state: State, config: Record, isRunM if (!skipConfigOverrideValidation) { validateOverridableAtTestTest(config, overrideLevel, (validationResult) => { - const isReadOnly = validationResult.supportedOverrideLevels === 'never' let errMsg if (overrideLevel === 'runtime' || overrideLevel === 'code') { errMsg = $errUtils.errByPath('config.cypress_config_api', { ...validationResult, - isReadOnly, runnableType: state('runnable')?.type, }) } else if (overrideLevel === 'event') { @@ -121,14 +119,10 @@ export const validateConfig = (state: State, config: Record, isRunM errMsg = $errUtils.errByPath('config.invalid_event_override', { ...validationResult, - isReadOnly, event: event?.replace('runner:', ''), }) } else { - errMsg = $errUtils.errByPath('config.invalid_mocha_config_override', { - ...validationResult, - isReadOnly, - }) + errMsg = $errUtils.errByPath('config.invalid_mocha_config_override', validationResult) } throw new (state('specWindow').Error)(errMsg) diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index a96838cf3320..5f4e42808616 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -146,7 +146,7 @@ Instead the value was: \`"null"\` 3) throws error when invalid test-level override: CypressError: The config passed to your test-level overrides has the following validation error: -CypressError: The \`testIsolation\` configuration cannot been overridden from a test-level override. The \`testIsolation\` option can only be set from suite-level overrides. +CypressError: The \`testIsolation\` configuration cannot been overridden from a test-level override. The \`testIsolation\` option can only be overridden from suite-level overrides. https://on.cypress.io/config [stack trace lines] @@ -692,7 +692,7 @@ Instead the value was: \`"null"\` 3) throws error when invalid test-level override: CypressError: The config passed to your test-level overrides has the following validation error: -CypressError: The \`testIsolation\` configuration cannot been overridden from a test-level override. The \`testIsolation\` option can only be set from suite-level overrides. +CypressError: The \`testIsolation\` configuration cannot been overridden from a test-level override. The \`testIsolation\` option can only be overridden from suite-level overrides. https://on.cypress.io/config Error From 86f4e62573bfb5a1a9c988c07265204bfc8f053b Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Wed, 10 Aug 2022 14:53:20 -0500 Subject: [PATCH 21/35] Update packages/server/test/unit/config_spec.js Co-authored-by: Bill Glesias --- packages/server/test/unit/config_spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/test/unit/config_spec.js b/packages/server/test/unit/config_spec.js index 0b56262488e5..8960de58ace2 100644 --- a/packages/server/test/unit/config_spec.js +++ b/packages/server/test/unit/config_spec.js @@ -1669,7 +1669,7 @@ describe('lib/config', () => { }) }) - it('honors user config for testIsolation when experimental=true and e2e testing', () => { + it('honors user config for testIsolation when experimentalSessionAndOrigin=true and e2e testing', () => { sinon.stub(configUtil, 'getProcessEnvVars').returns({}) const obj = { From 06766abfda71c96d39434ccc2e2b2f7e288f6480 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Wed, 10 Aug 2022 14:53:34 -0500 Subject: [PATCH 22/35] Update packages/server/lib/config.ts Co-authored-by: Bill Glesias --- packages/server/lib/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/lib/config.ts b/packages/server/lib/config.ts index 34b32bc03d5e..238d51d5dcfd 100644 --- a/packages/server/lib/config.ts +++ b/packages/server/lib/config.ts @@ -216,7 +216,7 @@ export function mergeDefaults ( // TODO: https://github.com/cypress-io/cypress/issues/23093 // testIsolation should equal 'strict' by default when experimentalSessionAndOrigin=true - // Once experimentalSessionAndOrigin is made GA, remove this logic and update the defaultValue + // Once experimentalSessionAndOrigin is made GA, remove this logic and update the defaultValue // to be be 'strict' if (testingType === 'e2e' && config.experimentalSessionAndOrigin) { if (config.rawJson.testIsolation) { From 5c48123e8648ee9dfade768cd16d57d610169b7d Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Wed, 10 Aug 2022 14:53:45 -0500 Subject: [PATCH 23/35] Update packages/server/test/unit/config_spec.js Co-authored-by: Bill Glesias --- packages/server/test/unit/config_spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/test/unit/config_spec.js b/packages/server/test/unit/config_spec.js index 8960de58ace2..515b049027af 100644 --- a/packages/server/test/unit/config_spec.js +++ b/packages/server/test/unit/config_spec.js @@ -1646,7 +1646,7 @@ describe('lib/config', () => { }) }) - it('sets testIsolation=strict by default when experimental=true and e2e testing', () => { + it('sets testIsolation=strict by default when experimentalSessionAndOrigin=true and e2e testing', () => { sinon.stub(configUtil, 'getProcessEnvVars').returns({}) const obj = { From 7005a0ed61af344e3324acf29fbdf18f490ce8fb Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Thu, 11 Aug 2022 09:05:02 -0500 Subject: [PATCH 24/35] remove err workarounds that were fixed in https://github.com/cypress-io/cypress/issues/23039 --- packages/driver/cypress/e2e/util/config.cy.js | 71 +++++++------------ packages/driver/src/cypress.ts | 3 +- packages/driver/src/cypress/setter_getter.ts | 10 +-- packages/driver/src/util/config.ts | 14 +--- .../testConfigOverrides_spec.ts.js | 4 +- 5 files changed, 31 insertions(+), 71 deletions(-) diff --git a/packages/driver/cypress/e2e/util/config.cy.js b/packages/driver/cypress/e2e/util/config.cy.js index 5fd22aa9e299..3cadb9587531 100644 --- a/packages/driver/cypress/e2e/util/config.cy.js +++ b/packages/driver/cypress/e2e/util/config.cy.js @@ -76,36 +76,8 @@ describe('driver/src/cypress/validate_config', () => { const state = $SetterGetter.create({ duringUserTestExecution: false, }) - const isRunMode = true - const shouldSet = validateConfig(state, { hello: 'world' }, isRunMode) - expect(shouldSet).to.be.true - }) - - it('skips validation in run mode when override level is test:before:run', () => { - const state = $SetterGetter.create({ - duringUserTestExecution: false, - test: { - _fired: { 'runner:test:before:run': true }, - }, - }) - const isRunMode = true - const shouldSet = validateConfig(state, {}, isRunMode) - - expect(shouldSet).to.be.false - }) - - it('run validation in open mode when override level is test:before:run', () => { - const state = $SetterGetter.create({ - duringUserTestExecution: false, - test: { - _fired: { 'runner:test:before:run': true }, - }, - }) - const isRunMode = false - const shouldSet = validateConfig(state, {}, isRunMode) - - expect(shouldSet).to.be.true + expect(() => validateConfig(state, { hello: 'world' })).not.to.throw() }) describe('ensures override level', () => { @@ -119,10 +91,9 @@ describe('driver/src/cypress/validate_config', () => { const overrideLevel = getOverrideLevel(state) expect(overrideLevel).to.eq('runtime') - const isRunMode = true expect(() => { - validateConfig(state, { chromeWebSecurity: true }, isRunMode) + validateConfig(state, { chromeWebSecurity: true }) }).to.throw(`\`Cypress.config()\` can never override \`chromeWebSecurity\` in a suite at runtime because it is a read-only configuration option.`) }) @@ -135,10 +106,9 @@ describe('driver/src/cypress/validate_config', () => { const overrideLevel = getOverrideLevel(state) expect(overrideLevel).to.eq('runtime') - const isRunMode = true expect(() => { - validateConfig(state, { testIsolation: 'strict' }, isRunMode) + validateConfig(state, { testIsolation: 'strict' }) }).to.throw(`\`Cypress.config()\` cannot override \`testIsolation\` in a test at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides.`) }) }) @@ -155,10 +125,9 @@ describe('driver/src/cypress/validate_config', () => { const overrideLevel = getOverrideLevel(state) expect(overrideLevel).to.eq('event') - const isRunMode = false expect(() => { - validateConfig(state, { chromeWebSecurity: true }, isRunMode) + validateConfig(state, { chromeWebSecurity: true }) }).to.throw(`\`Cypress.config()\` can never override \`chromeWebSecurity\` in a test:before:run event handler because it is a read-only configuration option.`) }) @@ -173,10 +142,9 @@ describe('driver/src/cypress/validate_config', () => { const overrideLevel = getOverrideLevel(state) expect(overrideLevel).to.eq('event') - const isRunMode = true expect(() => { - validateConfig(state, { testIsolation: 'strict' }, isRunMode) + validateConfig(state, { testIsolation: 'strict' }) }).to.throw(`\`Cypress.config()\` cannot override \`testIsolation\` in a test:before:run:async event handler. The \`testIsolation\` option can only be overridden from suite-level overrides.`) }) }) @@ -193,10 +161,9 @@ describe('driver/src/cypress/validate_config', () => { const overrideLevel = getOverrideLevel(state) expect(overrideLevel).to.eq('suite') - const isRunMode = true expect(() => { - validateConfig(state, { chromeWebSecurity: true }, isRunMode) + validateConfig(state, { chromeWebSecurity: true }) }).to.throw(`The \`chromeWebSecurity\` configuration can never be overridden from a suite-level override because it is a read-only configuration option.`) }) @@ -211,10 +178,9 @@ describe('driver/src/cypress/validate_config', () => { const overrideLevel = getOverrideLevel(state) expect(overrideLevel).to.eq('test') - const isRunMode = true expect(() => { - validateConfig(state, { testIsolation: 'strict' }, isRunMode) + validateConfig(state, { testIsolation: 'strict' }) }).to.throw(`The \`testIsolation\` configuration cannot be overridden from a test-level override. The \`testIsolation\` option can only be overridden from suite-level overrides.`) }) }) @@ -228,10 +194,9 @@ describe('driver/src/cypress/validate_config', () => { const overrideLevel = getOverrideLevel(state) expect(overrideLevel).to.eq('code') - const isRunMode = true expect(() => { - validateConfig(state, { chromeWebSecurity: true }, isRunMode) + validateConfig(state, { chromeWebSecurity: true }) }).to.throw(`\`Cypress.config()\` can never override \`chromeWebSecurity\` because it is a read-only configuration option.`) }) @@ -243,25 +208,37 @@ describe('driver/src/cypress/validate_config', () => { const overrideLevel = getOverrideLevel(state) expect(overrideLevel).to.eq('code') - const isRunMode = true expect(() => { - validateConfig(state, { testIsolation: 'strict' }, isRunMode) + validateConfig(state, { testIsolation: 'strict' }) }).to.throw(`\`Cypress.config()\` cannot override \`testIsolation\`. The \`testIsolation\` option can only be overridden from suite-level overrides.`) }) }) }) + it('skips checking override level when opted-out', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: true, + specWindow: { Error }, + runnable: { type: 'test' }, + }) + + const skipOverrideCHeck = true + + expect(() => { + validateConfig(state, { chromeWebSecurity: true }, skipOverrideCHeck) + }).not.to.throw() + }) + it('throws when invalid configuration value', () => { const state = $SetterGetter.create({ duringUserTestExecution: true, specWindow: { Error }, runnable: { type: 'test' }, }) - const isRunMode = true expect(() => { - validateConfig(state, { viewportHeight: '300' }, isRunMode) + validateConfig(state, { viewportHeight: '300' }) }).to.throw(`Expected \`viewportHeight\` to be a number.\n\nInstead the value was: \`"300"\``) }) }) diff --git a/packages/driver/src/cypress.ts b/packages/driver/src/cypress.ts index 6e5032ac26a8..cfc7c7d2f435 100644 --- a/packages/driver/src/cypress.ts +++ b/packages/driver/src/cypress.ts @@ -239,9 +239,8 @@ class $Cypress { this.originalConfig = _.cloneDeep(config) this.config = $SetterGetter.create(config, (config) => { const skipConfigOverrideValidation = this.isCrossOriginSpecBridge ? window.__cySkipValidateConfig : window.top!.__cySkipValidateConfig - const isRunMode = this.config('isTextTerminal') - return validateConfig(this.state, config, isRunMode, skipConfigOverrideValidation) + return validateConfig(this.state, config, skipConfigOverrideValidation) }) this.env = $SetterGetter.create(env) diff --git a/packages/driver/src/cypress/setter_getter.ts b/packages/driver/src/cypress/setter_getter.ts index 47dbf20247c9..d9ed920770d6 100644 --- a/packages/driver/src/cypress/setter_getter.ts +++ b/packages/driver/src/cypress/setter_getter.ts @@ -33,15 +33,11 @@ export default { ret = value } - let shouldSet = validate ? validate(obj) : true + validate && validate(obj) - if (shouldSet) { - extend(state, obj) + extend(state, obj) - return ret - } - - return {} + return ret } // return the getter / setter function interface diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index 300c34972d52..8c706cc3a8f8 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -92,19 +92,9 @@ export const getOverrideLevel = (state): OverrideLevel => { return overrideLevel as OverrideLevel } -export const validateConfig = (state: State, config: Record, isRunMode: boolean, skipConfigOverrideValidation: boolean = false): boolean => { +export const validateConfig = (state: State, config: Record, skipConfigOverrideValidation: boolean = false) => { const overrideLevel = getOverrideLevel(state) - // FIXME: https://github.com/cypress-io/cypress/issues/23039 - // bug in runner causes browser to hang in run mode when test:before:run throws an exception - if (overrideLevel === 'event' && isRunMode) { - const event = _.last(Object.keys(state('test')._fired || {})) - - if (event === 'runner:test:before:run') { - return false - } - } - if (!skipConfigOverrideValidation) { validateOverridableAtTestTest(config, overrideLevel, (validationResult) => { let errMsg @@ -143,6 +133,4 @@ export const validateConfig = (state: State, config: Record, isRunM throw new (state('specWindow').Error)(errMsg) }) - - return true } diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index d0d71bb66867..c099d2ce0293 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -162,12 +162,12 @@ exports['testConfigOverrides / fails when setting invalid config opt with Cypres 2 failing 1) does not run: - Error: Test Override validation should have failed & it block should not have executed. + Error: CypressError: \`Cypress.config()\` can never override \`chromeWebSecurity\` in a test:before:run event handler because it is a read-only configuration option.' [stack trace lines] 2) nested does not run 2: - Error: Test Override validation should have failed & it block should not have executed. + Error: CypressError: \`Cypress.config()\` can never override \`chromeWebSecurity\` in a test:before:run event handler because it is a read-only configuration option.' [stack trace lines] From 9bfc380ccfc0638ed2920febb0d93f56f762f8d8 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Thu, 11 Aug 2022 09:12:45 -0500 Subject: [PATCH 25/35] Update packages/driver/cypress/e2e/util/config.cy.js Co-authored-by: Bill Glesias --- packages/driver/cypress/e2e/util/config.cy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/driver/cypress/e2e/util/config.cy.js b/packages/driver/cypress/e2e/util/config.cy.js index 3cadb9587531..79b2df5a23fb 100644 --- a/packages/driver/cypress/e2e/util/config.cy.js +++ b/packages/driver/cypress/e2e/util/config.cy.js @@ -1,4 +1,4 @@ -import $SetterGetter from '@packages/driver/src/cypress/setter_getter' +import $SetterGetter from '../../../src/cypress/setter_getter' import { getOverrideLevel, validateConfig } from '../../../src/util/config' describe('driver/src/cypress/validate_config', () => { From e3e5dded1c99e83537d8132b03c00093e7ce37d3 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Thu, 11 Aug 2022 09:14:04 -0500 Subject: [PATCH 26/35] Update packages/config/src/options.ts Co-authored-by: Bill Glesias --- packages/config/src/options.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 87fc23bde87e..1c751fbbf328 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -34,7 +34,7 @@ export const ALL_OVERRIDE_LEVELS = ['code', 'event', 'suite', 'test', 'runtime'] export type OverrideLevel = typeof ALL_OVERRIDE_LEVELS[number] -export type OverrideLevels = Readonly | 'never'> +export type OverrideLevels = Readonly | 'never'> interface ConfigOption { name: string From aa97c9af5d0552aaa4300b5c9fb00341fb4b2aa2 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Thu, 11 Aug 2022 09:15:07 -0500 Subject: [PATCH 27/35] Update packages/driver/cypress/e2e/commands/sessions/sessions.cy.js Co-authored-by: Bill Glesias --- packages/driver/cypress/e2e/commands/sessions/sessions.cy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js index 65866f610955..afdbd4c0f18f 100644 --- a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js +++ b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js @@ -52,7 +52,7 @@ describe('cy.session', { retries: 0 }, () => { expect(clearCurrentSessionData).to.be.called }) - it('clears session data before each run', async () => { + it('resets rendered html origins before each run', async () => { const backendSpy = cy.spy(Cypress, 'backend') await Cypress.action('runner:test:before:run:async', {}) From 84469840a997d4aadeeca627bb393cb8186b5b45 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Thu, 11 Aug 2022 09:51:29 -0500 Subject: [PATCH 28/35] fix failing test --- .../cypress/e2e/e2e/origin/config_env.cy.ts | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts b/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts index 4fd382f12e02..82c16fe2c887 100644 --- a/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts @@ -16,19 +16,17 @@ }) if (fnName === 'config') { - it(`throws if mutating read-only config values with Cypress.config()`, () => { - return new Promise((resolve) => { - // @ts-ignore - window.top.__cySkipValidateConfig = false - cy.on('fail', (err) => { - expect(err.message).to.include('`Cypress.config()` cannot override `chromeWebSecurity` in a test at runtime because it is a read-only configuration option.') - resolve() - }) + it(`throws if mutating read-only config values with Cypress.config()`, (done) => { + // @ts-ignore + window.top.__cySkipValidateConfig = false + cy.on('fail', (err) => { + expect(err.message).to.include('`Cypress.config()` can never override `chromeWebSecurity` in a test at runtime because it is a read-only configuration option.') + done() + }) - cy.origin('http://foobar.com:3500', () => { - // @ts-ignore - Cypress.config('chromeWebSecurity', false) - }) + cy.origin('http://foobar.com:3500', () => { + // @ts-ignore + Cypress.config('chromeWebSecurity', false) }) }) } From 32487975b16c97593850f888e317046f02ed2826 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Thu, 11 Aug 2022 09:52:58 -0500 Subject: [PATCH 29/35] fix snapshot --- system-tests/__snapshots__/issue_6407_spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-tests/__snapshots__/issue_6407_spec.js b/system-tests/__snapshots__/issue_6407_spec.js index 0109c576c5b4..bbf2e1e6fad8 100644 --- a/system-tests/__snapshots__/issue_6407_spec.js +++ b/system-tests/__snapshots__/issue_6407_spec.js @@ -25,7 +25,7 @@ exports['e2e issue 6407 throws if mutating read-only config with test configurat 1) throws if mutating read-only config with test configuration: CypressError: The config passed to your test-level overrides has the following validation error: -CypressError: The \`chromeWebSecurity\` configuration cannot been overridden from a code-level override because it is a read-only configuration option. +CypressError: The \`chromeWebSecurity\` configuration can never be overridden from a code-level override because it is a read-only configuration option. https://on.cypress.io/config Error From 073de5add45e839e725e0d2ae0e5fdfd14911c08 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Thu, 11 Aug 2022 10:35:39 -0500 Subject: [PATCH 30/35] overruled....changing lax back to legacy --- cli/types/cypress.d.ts | 8 ++++---- packages/config/__snapshots__/index.spec.ts.js | 4 ++-- packages/config/src/options.ts | 6 +++--- .../cypress/e2e/commands/sessions/sessions.cy.js | 2 +- packages/server/test/unit/config_spec.js | 8 ++++---- .../e2e/cypress/e2e/testConfigOverrides/invalid.js | 12 ++++++------ 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index edfcf7c478f8..b0137e23e47a 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2786,11 +2786,11 @@ declare namespace Cypress { supportFile: string | false /** * The test isolation level applied to ensure a clean slate between tests. - * - lax - resets/clears aliases, intercepts, clock, viewport, cookies, and local storage before each test. - * - strict - all resets everything from lax, plus clears the page by visiting 'about:blank' to ensure clean app state before each test. - * @default "lax", however, when experimentalSessionAndOrigin=true, the default is "strict" + * - legacy - resets/clears aliases, intercepts, clock, viewport, cookies, and local storage before each test. + * - strict - all resets everything from legacy, plus clears the page by visiting 'about:blank' to ensure clean app state before each test. + * @default "legacy", however, when experimentalSessionAndOrigin=true, the default is "strict" */ - testIsolation: 'lax' | 'strict' + testIsolation: 'legacy' | 'strict' /** * Path to folder where videos will be saved after a headless or CI run * @default "cypress/videos" diff --git a/packages/config/__snapshots__/index.spec.ts.js b/packages/config/__snapshots__/index.spec.ts.js index 6951ade8a338..61f61911bd9c 100644 --- a/packages/config/__snapshots__/index.spec.ts.js +++ b/packages/config/__snapshots__/index.spec.ts.js @@ -67,7 +67,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys 1 "supportFile": "cypress/support/e2e.{js,jsx,ts,tsx}", "supportFolder": false, "taskTimeout": 60000, - "testIsolation": "lax", + "testIsolation": "legacy", "trashAssetsBeforeRuns": true, "userAgent": null, "video": true, @@ -148,7 +148,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f "supportFile": "cypress/support/e2e.{js,jsx,ts,tsx}", "supportFolder": false, "taskTimeout": 60000, - "testIsolation": "lax", + "testIsolation": "legacy", "trashAssetsBeforeRuns": true, "userAgent": null, "video": true, diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 1c751fbbf328..d61491f7d16f 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -355,11 +355,11 @@ const driverConfigOptions: Array = [ name: 'testIsolation', // TODO: https://github.com/cypress-io/cypress/issues/23093 // When experimentalSessionAndOrigin is removed and released as GA, - // update the defaultValue from 'lax' to 'strict' and + // update the defaultValue from 'legacy' to 'strict' and // update this code to remove the check/override specific to enable // strict by default when experimentalSessionAndOrigin=true - defaultValue: 'lax', - validation: validate.isOneOf('lax', 'strict'), + defaultValue: 'legacy', + validation: validate.isOneOf('legacy', 'strict'), overrideLevels: ['suite'], }, { name: 'trashAssetsBeforeRuns', diff --git a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js index afdbd4c0f18f..639d439e2cc0 100644 --- a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js +++ b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js @@ -75,7 +75,7 @@ describe('cy.session', { retries: 0 }, () => { }) }) - describe('testIsolation=lax', { testIsolation: 'lax' }, () => { + describe('testIsolation=legacy', { testIsolation: 'legacy' }, () => { it('does not clear page', () => { cy.visit('/fixtures/form.html') .then(async () => { diff --git a/packages/server/test/unit/config_spec.js b/packages/server/test/unit/config_spec.js index 515b049027af..8823589a35d4 100644 --- a/packages/server/test/unit/config_spec.js +++ b/packages/server/test/unit/config_spec.js @@ -1519,7 +1519,7 @@ describe('lib/config', () => { supportFile: { value: false, from: 'config' }, supportFolder: { value: false, from: 'default' }, taskTimeout: { value: 60000, from: 'default' }, - testIsolation: { value: 'lax', from: 'default' }, + testIsolation: { value: 'legacy', from: 'default' }, trashAssetsBeforeRuns: { value: true, from: 'default' }, userAgent: { value: null, from: 'default' }, video: { value: true, from: 'default' }, @@ -1630,7 +1630,7 @@ describe('lib/config', () => { supportFile: { value: false, from: 'config' }, supportFolder: { value: false, from: 'default' }, taskTimeout: { value: 60000, from: 'default' }, - testIsolation: { value: 'lax', from: 'default' }, + testIsolation: { value: 'legacy', from: 'default' }, trashAssetsBeforeRuns: { value: true, from: 'default' }, userAgent: { value: null, from: 'default' }, video: { value: true, from: 'default' }, @@ -1677,7 +1677,7 @@ describe('lib/config', () => { supportFile: false, baseUrl: 'http://localhost:8080', experimentalSessionAndOrigin: true, - testIsolation: 'lax', + testIsolation: 'legacy', } const options = { @@ -1689,7 +1689,7 @@ describe('lib/config', () => { expect(cfg.resolved).to.have.property('experimentalSessionAndOrigin') expect(cfg.resolved.experimentalSessionAndOrigin).to.deep.eq({ value: true, from: 'config' }) expect(cfg.resolved).to.have.property('testIsolation') - expect(cfg.resolved.testIsolation).to.deep.eq({ value: 'lax', from: 'config' }) + expect(cfg.resolved.testIsolation).to.deep.eq({ value: 'legacy', from: 'config' }) }) }) }) diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js index 856d84f38db5..73bb69064896 100644 --- a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js +++ b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js @@ -54,18 +54,18 @@ describe('throws error correctly when beforeEach hook', () => { }) }) -it('throws error when invalid test-level override', { testIsolation: 'lax' }, () => { +it('throws error when invalid test-level override', { testIsolation: 'legacy' }, () => { shouldNotExecute() }) it('throws error when invalid config opt in Cypress.config() in test', () => { - Cypress.config({ testIsolation: 'lax' }) + Cypress.config({ testIsolation: 'legacy' }) shouldNotExecute() }) describe('throws error when invalid config opt in Cypress.config() in before hook', () => { before(() => { - Cypress.config({ testIsolation: 'lax' }) + Cypress.config({ testIsolation: 'legacy' }) }) it('4', () => { @@ -75,7 +75,7 @@ describe('throws error when invalid config opt in Cypress.config() in before hoo describe('throws error when invalid config opt in Cypress.config() in beforeEach hook', () => { beforeEach(() => { - Cypress.config({ testIsolation: 'lax' }) + Cypress.config({ testIsolation: 'legacy' }) }) it('5', () => { @@ -85,7 +85,7 @@ describe('throws error when invalid config opt in Cypress.config() in beforeEach describe('throws error when invalid config opt in Cypress.config() in after hook', () => { after(() => { - Cypress.config({ testIsolation: 'lax' }) + Cypress.config({ testIsolation: 'legacy' }) }) it('5', () => { @@ -95,7 +95,7 @@ describe('throws error when invalid config opt in Cypress.config() in after hook describe('throws error when invalid config opt in Cypress.config() in afterEach hook', () => { afterEach(() => { - Cypress.config({ testIsolation: 'lax' }) + Cypress.config({ testIsolation: 'legacy' }) }) it('5', () => { From 28a7140c17c0a0be0e5cbd44f28e054910088fd8 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Thu, 11 Aug 2022 10:37:55 -0500 Subject: [PATCH 31/35] fix --- system-tests/__snapshots__/issue_6407_spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-tests/__snapshots__/issue_6407_spec.js b/system-tests/__snapshots__/issue_6407_spec.js index bbf2e1e6fad8..2b94c877514c 100644 --- a/system-tests/__snapshots__/issue_6407_spec.js +++ b/system-tests/__snapshots__/issue_6407_spec.js @@ -25,7 +25,7 @@ exports['e2e issue 6407 throws if mutating read-only config with test configurat 1) throws if mutating read-only config with test configuration: CypressError: The config passed to your test-level overrides has the following validation error: -CypressError: The \`chromeWebSecurity\` configuration can never be overridden from a code-level override because it is a read-only configuration option. +CypressError: The \`chromeWebSecurity\` configuration can never be overridden from a test-level override because it is a read-only configuration option. https://on.cypress.io/config Error From 2b94621c0948a451bc02341ef549c1ea76ad2e5c Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Thu, 11 Aug 2022 14:25:32 -0500 Subject: [PATCH 32/35] change code -> fileLoad --- packages/config/src/options.ts | 10 ++++++++-- packages/driver/src/util/config.ts | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index d61491f7d16f..706d3bed734d 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -29,8 +29,14 @@ export type BreakingOptionErrorKey = | 'RENAMED_CONFIG_OPTION' | 'TEST_FILES_RENAMED' -// The test-time override levels -export const ALL_OVERRIDE_LEVELS = ['code', 'event', 'suite', 'test', 'runtime'] as const +// The test-time override levels (listed in order applied): +// fileLoad - config override via Cypress.config() when either loading the supportFile or specFile in the +// browser (this is before mocha as process the spec +// suite - config override via describe('', {...}, () => {}) +// test - config override via it('', {...}, () => {}) +// event - config override via Cypress.config() in test:before:runner or test:before:runner:async event +// runtime - config override via Cypress.config() when the test callback is executed +export const ALL_OVERRIDE_LEVELS = ['fileLoad', 'event', 'suite', 'test', 'runtime'] as const export type OverrideLevel = typeof ALL_OVERRIDE_LEVELS[number] diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index 8c706cc3a8f8..09ba7c677a7a 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -86,7 +86,7 @@ export const getOverrideLevel = (state): OverrideLevel => { overrideLevel = test._testConfig.applied // either suite or test } } else { - overrideLevel = 'code' // supportFile or spec load execution + overrideLevel = 'fileLoad' // supportFile or specFile load execution } return overrideLevel as OverrideLevel @@ -99,7 +99,7 @@ export const validateConfig = (state: State, config: Record, skipCo validateOverridableAtTestTest(config, overrideLevel, (validationResult) => { let errMsg - if (overrideLevel === 'runtime' || overrideLevel === 'code') { + if (overrideLevel === 'runtime' || overrideLevel === 'fileLoad') { errMsg = $errUtils.errByPath('config.cypress_config_api', { ...validationResult, runnableType: state('runnable')?.type, From 586ca74c338faada15e8dfaab6910396df56a492 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Thu, 11 Aug 2022 17:30:32 -0500 Subject: [PATCH 33/35] okayyyy another round of feedback applied --- packages/config/src/browser.ts | 21 +-- packages/config/src/options.ts | 65 +++---- packages/config/test/index.spec.ts | 68 ++++---- .../cypress/e2e/e2e/origin/config_env.cy.ts | 2 +- .../cypress/e2e/e2e/testConfigOverrides.cy.js | 2 +- packages/driver/cypress/e2e/util/config.cy.js | 158 ++++++------------ packages/driver/src/cy/testConfigOverrides.ts | 8 +- packages/driver/src/cypress/error_messages.ts | 56 ++----- packages/driver/src/util/config.ts | 66 ++++---- system-tests/__snapshots__/issue_6407_spec.js | 2 +- .../testConfigOverrides_spec.ts.js | 84 +++------- .../e2e/testConfigOverrides/invalid.js | 2 +- 12 files changed, 199 insertions(+), 335 deletions(-) diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index cc01f6096e77..eec8fab1ffa8 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -3,14 +3,13 @@ import Debug from 'debug' import { defaultSpecPattern, options, - ALL_OVERRIDE_LEVELS, breakingOptions, breakingRootOptions, testingTypeBreakingOptions, } from './options' import type { TestingType } from '@packages/types' -import type { BreakingOption, BreakingOptionErrorKey, OverrideLevel, OverrideLevels } from './options' +import type { BreakingOption, BreakingOptionErrorKey, OverrideLevel } from './options' import type { ErrResult } from './validation' // this export has to be done in 2 lines because of a bug in babel typescript @@ -23,7 +22,6 @@ export { BreakingOption, BreakingOptionErrorKey, ErrResult, - OverrideLevel, validation, } @@ -49,7 +47,7 @@ const defaultValues = createIndex(options, 'name', 'defaultValue') const publicConfigKeys = _(options).reject({ isInternal: true }).map('name').value() const validationRules = createIndex(options, 'name', 'validation') -export const testOverrideLevels = createIndex(options, 'name', 'overrideLevels', 'never') +export const testOverrideLevels = createIndex(options, 'name', 'overrideLevel', 'never') const restartOnChangeOptionsKeys = _.filter(options, 'requireRestartOnChange') @@ -57,8 +55,7 @@ const issuedWarnings = new Set() export type InvalidTestOverrideResult = { invalidConfigKey: string - overrideLevel: string - supportedOverrideLevels: OverrideLevels + supportedOverrideLevel: string } export type BreakingErrResult = { @@ -193,21 +190,19 @@ export const validateNoBreakingTestingTypeConfig = (cfg: any, testingType: keyof return validateNoBreakingOptions(options, cfg, onWarning, onErr, testingType) } -export const validateOverridableAtTestTest = (config: any, overrideLevel: Exclude, onErr: (result: InvalidTestOverrideResult) => void) => { +export const validateOverridableAtRunTime = (config: any, isSuiteLevelOverride: boolean, onErr: (result: InvalidTestOverrideResult) => void) => { Object.keys(config).some((configKey) => { - const runtimeValidation = ALL_OVERRIDE_LEVELS.includes(overrideLevel) - const overrideLevels = testOverrideLevels[configKey] as OverrideLevels + const overrideLevel: OverrideLevel = testOverrideLevels[configKey] - if (!overrideLevels) { + if (!overrideLevel) { // non-cypress configuration option. skip validation return } - if (runtimeValidation && (overrideLevels === 'never' || !overrideLevels.includes(overrideLevel))) { + if (overrideLevel === 'never' || (overrideLevel === 'suite' && !isSuiteLevelOverride)) { onErr({ invalidConfigKey: configKey, - overrideLevel, - supportedOverrideLevels: overrideLevels, + supportedOverrideLevel: overrideLevel, }) } }) diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 706d3bed734d..7bb3d9549a2b 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -29,18 +29,7 @@ export type BreakingOptionErrorKey = | 'RENAMED_CONFIG_OPTION' | 'TEST_FILES_RENAMED' -// The test-time override levels (listed in order applied): -// fileLoad - config override via Cypress.config() when either loading the supportFile or specFile in the -// browser (this is before mocha as process the spec -// suite - config override via describe('', {...}, () => {}) -// test - config override via it('', {...}, () => {}) -// event - config override via Cypress.config() in test:before:runner or test:before:runner:async event -// runtime - config override via Cypress.config() when the test callback is executed -export const ALL_OVERRIDE_LEVELS = ['fileLoad', 'event', 'suite', 'test', 'runtime'] as const - -export type OverrideLevel = typeof ALL_OVERRIDE_LEVELS[number] - -export type OverrideLevels = Readonly | 'never'> +export type OverrideLevel = 'any' | 'suite' | 'never' interface ConfigOption { name: string @@ -52,7 +41,7 @@ interface ConfigOption { * it indicates the configuration value cannot be overridden via suite-/test-specific * overrides or at run-time with Cypress.Config(). */ - overrideLevels?: OverrideLevels + overrideLevel?: OverrideLevel } interface DriverConfigOption extends ConfigOption { @@ -132,7 +121,7 @@ const driverConfigOptions: Array = [ name: 'animationDistanceThreshold', defaultValue: 5, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'arch', defaultValue: () => os.arch(), @@ -141,13 +130,13 @@ const driverConfigOptions: Array = [ name: 'baseUrl', defaultValue: null, validation: validate.isFullyQualifiedUrl, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', requireRestartOnChange: 'server', }, { name: 'blockHosts', defaultValue: null, validation: validate.isStringOrArrayOfStrings, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'chromeWebSecurity', defaultValue: true, @@ -170,7 +159,7 @@ const driverConfigOptions: Array = [ name: 'defaultCommandTimeout', defaultValue: 4000, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'downloadsFolder', defaultValue: 'cypress/downloads', @@ -188,12 +177,12 @@ const driverConfigOptions: Array = [ name: 'env', defaultValue: {}, validation: validate.isPlainObject, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'execTimeout', defaultValue: 60000, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'experimentalFetchPolyfill', defaultValue: false, @@ -238,17 +227,17 @@ const driverConfigOptions: Array = [ name: 'excludeSpecPattern', defaultValue: (options: Record = {}) => options.testingType === 'component' ? ['**/__snapshots__/*', '**/__image_snapshots__/*'] : '*.hot-update.js', validation: validate.isStringOrArrayOfStrings, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'includeShadowDom', defaultValue: false, validation: validate.isBoolean, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'keystrokeDelay', defaultValue: 0, validation: validate.isNumberOrFalse, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'modifyObstructiveCode', defaultValue: true, @@ -261,7 +250,7 @@ const driverConfigOptions: Array = [ name: 'numTestsKeptInMemory', defaultValue: 50, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'platform', defaultValue: () => os.platform(), @@ -270,7 +259,7 @@ const driverConfigOptions: Array = [ name: 'pageLoadTimeout', defaultValue: 60000, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'port', defaultValue: null, @@ -283,22 +272,22 @@ const driverConfigOptions: Array = [ name: 'redirectionLimit', defaultValue: 20, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'reporter', defaultValue: 'spec', validation: validate.isString, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'reporterOptions', defaultValue: null, validation: validate.isPlainObject, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'requestTimeout', defaultValue: 5000, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'resolvedNodePath', defaultValue: null, @@ -311,7 +300,7 @@ const driverConfigOptions: Array = [ name: 'responseTimeout', defaultValue: 30000, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'retries', defaultValue: { @@ -319,12 +308,12 @@ const driverConfigOptions: Array = [ openMode: 0, }, validation: validate.isValidRetriesConfig, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'screenshotOnRunFailure', defaultValue: true, validation: validate.isBoolean, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'screenshotsFolder', defaultValue: 'cypress/screenshots', @@ -335,12 +324,12 @@ const driverConfigOptions: Array = [ name: 'slowTestThreshold', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 250 : 10000, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'scrollBehavior', defaultValue: 'top', validation: validate.isOneOf('center', 'top', 'bottom', 'nearest', false), - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'supportFile', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 'cypress/support/component.{js,jsx,ts,tsx}' : 'cypress/support/e2e.{js,jsx,ts,tsx}', @@ -356,7 +345,7 @@ const driverConfigOptions: Array = [ name: 'taskTimeout', defaultValue: 60000, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'testIsolation', // TODO: https://github.com/cypress-io/cypress/issues/23093 @@ -366,7 +355,7 @@ const driverConfigOptions: Array = [ // strict by default when experimentalSessionAndOrigin=true defaultValue: 'legacy', validation: validate.isOneOf('legacy', 'strict'), - overrideLevels: ['suite'], + overrideLevel: 'suite', }, { name: 'trashAssetsBeforeRuns', defaultValue: true, @@ -397,17 +386,17 @@ const driverConfigOptions: Array = [ name: 'viewportHeight', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 500 : 660, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'viewportWidth', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 500 : 1000, validation: validate.isNumber, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'waitForAnimations', defaultValue: true, validation: validate.isBoolean, - overrideLevels: ALL_OVERRIDE_LEVELS, + overrideLevel: 'any', }, { name: 'watchForFileChanges', defaultValue: true, diff --git a/packages/config/test/index.spec.ts b/packages/config/test/index.spec.ts index b9fc04572a39..4007a4bdeb88 100644 --- a/packages/config/test/index.spec.ts +++ b/packages/config/test/index.spec.ts @@ -1,10 +1,10 @@ import chai from 'chai' +import path from 'path' import snapshot from 'snap-shot-it' import sinon from 'sinon' import sinonChai from 'sinon-chai' import * as configUtil from '../src/index' -import { OverrideLevel, ALL_OVERRIDE_LEVELS } from '../src/options' chai.use(sinonChai) const { expect } = chai @@ -46,6 +46,10 @@ describe('config/src/index', () => { }) expect(defaultValues.env).to.deep.eq({}) + const cypressBinaryRoot = defaultValues.cypressBinaryRoot.split(path.sep).pop() + + expect(cypressBinaryRoot).to.eq('cypress') + defaultValues.cypressBinaryRoot = `/root/cypress` // remove these since they are different depending on your machine ;['platform', 'arch', 'version'].forEach((x) => { @@ -66,6 +70,10 @@ describe('config/src/index', () => { }) expect(defaultValues.env).to.deep.eq({}) + const cypressBinaryRoot = defaultValues.cypressBinaryRoot.split(path.sep).pop() + + expect(cypressBinaryRoot).to.eq('cypress') + defaultValues.cypressBinaryRoot = `/root/cypress` // remove these since they are different depending on your machine ;['platform', 'arch', 'version'].forEach((x) => { @@ -190,61 +198,57 @@ describe('config/src/index', () => { }) }) - describe('.validateOverridableAtTestTest', () => { - ALL_OVERRIDE_LEVELS.forEach((overrideLevel) => { - it(`calls onError handler if configuration cannot be overridden from ${overrideLevel} level`, () => { - const errorFn = sinon.spy() + describe('.validateOverridableAtRunTime', () => { + it('calls onError handler if configuration override level=never', () => { + const errorFn = sinon.spy() - configUtil.validateOverridableAtTestTest({ chromeWebSecurity: false }, overrideLevel as OverrideLevel, errorFn) + configUtil.validateOverridableAtRunTime({ chromeWebSecurity: false }, false, errorFn) - expect(errorFn).to.have.callCount(1) - expect(errorFn).to.have.been.calledWithMatch({ - invalidConfigKey: 'chromeWebSecurity', - overrideLevel, - supportedOverrideLevels: 'never', - }) + expect(errorFn).to.have.callCount(1) + expect(errorFn).to.have.been.calledWithMatch({ + invalidConfigKey: 'chromeWebSecurity', + supportedOverrideLevel: 'never', }) }) - describe('testIsolation', () => { - it('does not calls onError handler if configuration can be overridden from suite level', () => { + describe('configuration override level=suite', () => { + it('does not calls onError handler if validating level is suite', () => { const errorFn = sinon.spy() - configUtil.validateOverridableAtTestTest({ testIsolation: 'strict' }, 'suite', errorFn) + const isSuiteOverride = true + + configUtil.validateOverridableAtRunTime({ testIsolation: 'strict' }, isSuiteOverride, errorFn) expect(errorFn).to.have.callCount(0) }) - ;['code', 'event', 'test', 'runtime'].forEach((overrideLevel) => { - it(`calls onError handler if configuration can be overridden from ${overrideLevel} level`, () => { - const errorFn = sinon.spy() + it('calls onError handler if validating level is not suite', () => { + const errorFn = sinon.spy() - configUtil.validateOverridableAtTestTest({ testIsolation: false }, overrideLevel as OverrideLevel, errorFn) + const isSuiteOverride = false - expect(errorFn).to.have.callCount(1) - expect(errorFn).to.have.been.calledWithMatch({ - invalidConfigKey: 'testIsolation', - overrideLevel, - supportedOverrideLevels: ['suite'], - }) + configUtil.validateOverridableAtRunTime({ testIsolation: false }, isSuiteOverride, errorFn) + + expect(errorFn).to.have.callCount(1) + expect(errorFn).to.have.been.calledWithMatch({ + invalidConfigKey: 'testIsolation', + supportedOverrideLevel: 'suite', }) }) }) - ALL_OVERRIDE_LEVELS.forEach((overrideLevel: OverrideLevel) => { - it(`does not call onErr if validation succeeds from ${overrideLevel} level`, () => { - const errorFn = sinon.spy() + it(`does not call onErr if config override level=any`, () => { + const errorFn = sinon.spy() - configUtil.validateOverridableAtTestTest({ requestTimeout: 1000 }, overrideLevel as OverrideLevel, errorFn) + configUtil.validateOverridableAtRunTime({ requestTimeout: 1000 }, false, errorFn) - expect(errorFn).to.have.callCount(0) - }) + expect(errorFn).to.have.callCount(0) }) it('does not call onErr if configuration is a non-Cypress config option', () => { const errorFn = sinon.spy() - configUtil.validateOverridableAtTestTest({ foo: 'bar' }, 'runtime' as OverrideLevel, errorFn) + configUtil.validateOverridableAtRunTime({ foo: 'bar' }, true, errorFn) expect(errorFn).to.have.callCount(0) }) diff --git a/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts b/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts index 82c16fe2c887..61fb510b38ac 100644 --- a/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/config_env.cy.ts @@ -20,7 +20,7 @@ // @ts-ignore window.top.__cySkipValidateConfig = false cy.on('fail', (err) => { - expect(err.message).to.include('`Cypress.config()` can never override `chromeWebSecurity` in a test at runtime because it is a read-only configuration option.') + expect(err.message).to.include('`Cypress.config()` can never override `chromeWebSecurity` because it is a read-only configuration option.') done() }) diff --git a/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js b/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js index a0462404f35d..ee774bd39fed 100644 --- a/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js +++ b/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js @@ -385,7 +385,7 @@ describe('cannot set override configuration options that', () => { it('throws if mutating read-only config with Cypress.config()', (done) => { window.top.__cySkipValidateConfig = false cy.once('fail', (err) => { - expect(err.message).to.include('`Cypress.config()` can never override `chromeWebSecurity` in a test at runtime') + expect(err.message).to.include('`Cypress.config()` can never override `chromeWebSecurity` because it is a read-only configuration option') done() }) diff --git a/packages/driver/cypress/e2e/util/config.cy.js b/packages/driver/cypress/e2e/util/config.cy.js index 79b2df5a23fb..c74840a9dd5c 100644 --- a/packages/driver/cypress/e2e/util/config.cy.js +++ b/packages/driver/cypress/e2e/util/config.cy.js @@ -1,15 +1,15 @@ import $SetterGetter from '../../../src/cypress/setter_getter' -import { getOverrideLevel, validateConfig } from '../../../src/util/config' +import { getMochaOverrideLevel, validateConfig } from '../../../src/util/config' describe('driver/src/cypress/validate_config', () => { - describe('getOverrideLevel', () => { - it('returns override level of runtime', () => { + describe('getMochaOverrideLevel', () => { + it('returns override level of undefined', () => { const state = $SetterGetter.create({ duringUserTestExecution: true, }) - const overrideLevel = getOverrideLevel(state) + const overrideLevel = getMochaOverrideLevel(state) - expect(overrideLevel).to.eq('runtime') + expect(overrideLevel).to.be.undefined }) it('returns override level of test:before:run:async', () => { @@ -19,21 +19,9 @@ describe('driver/src/cypress/validate_config', () => { _fired: { 'runner:test:before:run': true, 'runner:test:before:run:async': true }, }, }) - const overrideLevel = getOverrideLevel(state) + const overrideLevel = getMochaOverrideLevel(state) - expect(overrideLevel).to.eq('event') - }) - - it('returns override level of test:before:run', () => { - const state = $SetterGetter.create({ - duringUserTestExecution: false, - test: { - _fired: { 'runner:test:before:run': true }, - }, - }) - const overrideLevel = getOverrideLevel(state) - - expect(overrideLevel).to.eq('event') + expect(overrideLevel).to.be.undefined }) it('returns override level of suite', () => { @@ -43,7 +31,7 @@ describe('driver/src/cypress/validate_config', () => { _testConfig: { applied: 'suite' }, }, }) - const overrideLevel = getOverrideLevel(state) + const overrideLevel = getMochaOverrideLevel(state) expect(overrideLevel).to.eq('suite') }) @@ -55,19 +43,19 @@ describe('driver/src/cypress/validate_config', () => { _testConfig: { applied: 'test' }, }, }) - const overrideLevel = getOverrideLevel(state) + const overrideLevel = getMochaOverrideLevel(state) expect(overrideLevel).to.eq('test') }) - it('returns override level of code', () => { + it('returns override level of fileLoad', () => { const state = $SetterGetter.create({ duringUserTestExecution: false, test: undefined, }) - const overrideLevel = getOverrideLevel(state) + const overrideLevel = getMochaOverrideLevel(state) - expect(overrideLevel).to.eq('code') + expect(overrideLevel).to.be.undefined }) }) @@ -81,76 +69,23 @@ describe('driver/src/cypress/validate_config', () => { }) describe('ensures override level', () => { - describe('throws when override level is runtime', () => { - it('and config override is read-only', () => { - const state = $SetterGetter.create({ - duringUserTestExecution: true, - specWindow: { Error }, - runnable: { type: 'suite' }, - }) - const overrideLevel = getOverrideLevel(state) - - expect(overrideLevel).to.eq('runtime') - - expect(() => { - validateConfig(state, { chromeWebSecurity: true }) - }).to.throw(`\`Cypress.config()\` can never override \`chromeWebSecurity\` in a suite at runtime because it is a read-only configuration option.`) + it('throws when config override level is never', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: true, + specWindow: { Error }, + runnable: { type: 'suite' }, }) + const overrideLevel = getMochaOverrideLevel(state) - it('and config override invalid at runtime', () => { - const state = $SetterGetter.create({ - duringUserTestExecution: true, - specWindow: { Error }, - runnable: { type: 'test' }, - }) - const overrideLevel = getOverrideLevel(state) - - expect(overrideLevel).to.eq('runtime') + expect(overrideLevel).to.be.undefined - expect(() => { - validateConfig(state, { testIsolation: 'strict' }) - }).to.throw(`\`Cypress.config()\` cannot override \`testIsolation\` in a test at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides.`) - }) + expect(() => { + validateConfig(state, { chromeWebSecurity: true }) + }).to.throw(`\`Cypress.config()\` can never override \`chromeWebSecurity\` because it is a read-only configuration option.`) }) - describe('throws when override level is associated to an event config override', () => { - it('and config override is read-only', () => { - const state = $SetterGetter.create({ - duringUserTestExecution: false, - test: { - _fired: { 'runner:test:before:run': true }, - }, - specWindow: { Error }, - }) - const overrideLevel = getOverrideLevel(state) - - expect(overrideLevel).to.eq('event') - - expect(() => { - validateConfig(state, { chromeWebSecurity: true }) - }).to.throw(`\`Cypress.config()\` can never override \`chromeWebSecurity\` in a test:before:run event handler because it is a read-only configuration option.`) - }) - - it('and config override invalid for override level', () => { - const state = $SetterGetter.create({ - duringUserTestExecution: false, - test: { - _fired: { 'runner:test:before:run': true, 'runner:test:before:run:async': true }, - }, - specWindow: { Error }, - }) - const overrideLevel = getOverrideLevel(state) - - expect(overrideLevel).to.eq('event') - - expect(() => { - validateConfig(state, { testIsolation: 'strict' }) - }).to.throw(`\`Cypress.config()\` cannot override \`testIsolation\` in a test:before:run:async event handler. The \`testIsolation\` option can only be overridden from suite-level overrides.`) - }) - }) - - describe('throws when override level is associated to a mocha config override', () => { - it('and config override is read-only', () => { + describe('when config override level is suite', () => { + it('does not throw when runtime level is suite', () => { const state = $SetterGetter.create({ duringUserTestExecution: false, test: { @@ -158,16 +93,16 @@ describe('driver/src/cypress/validate_config', () => { }, specWindow: { Error }, }) - const overrideLevel = getOverrideLevel(state) + const overrideLevel = getMochaOverrideLevel(state) expect(overrideLevel).to.eq('suite') expect(() => { - validateConfig(state, { chromeWebSecurity: true }) - }).to.throw(`The \`chromeWebSecurity\` configuration can never be overridden from a suite-level override because it is a read-only configuration option.`) + validateConfig(state, { testIsolation: 'strict' }) + }).not.to.throw() }) - it('and config override invalid for override level', () => { + it('throws when runtime level is not suite', () => { const state = $SetterGetter.create({ duringUserTestExecution: false, test: { @@ -175,43 +110,48 @@ describe('driver/src/cypress/validate_config', () => { }, specWindow: { Error }, }) - const overrideLevel = getOverrideLevel(state) + const overrideLevel = getMochaOverrideLevel(state) expect(overrideLevel).to.eq('test') expect(() => { validateConfig(state, { testIsolation: 'strict' }) - }).to.throw(`The \`testIsolation\` configuration cannot be overridden from a test-level override. The \`testIsolation\` option can only be overridden from suite-level overrides.`) + }).to.throw(`The \`testIsolation\` configuration can only be overridden from a suite-level override.`) }) }) - describe('throws when override level is code', () => { + describe('when config override level is suite', () => { it('and config override is read-only', () => { const state = $SetterGetter.create({ duringUserTestExecution: false, specWindow: { Error }, }) - const overrideLevel = getOverrideLevel(state) + const overrideLevel = getMochaOverrideLevel(state) - expect(overrideLevel).to.eq('code') + expect(overrideLevel).to.be.undefined expect(() => { validateConfig(state, { chromeWebSecurity: true }) }).to.throw(`\`Cypress.config()\` can never override \`chromeWebSecurity\` because it is a read-only configuration option.`) }) - it('and config override invalid at runtime', () => { - const state = $SetterGetter.create({ - duringUserTestExecution: false, - specWindow: { Error }, + ;['test', 'suite'].forEach((mocha_runnable) => { + it(`and config override level is ${mocha_runnable}`, () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: { + _testConfig: { applied: mocha_runnable }, + }, + specWindow: { Error }, + }) + const overrideLevel = getMochaOverrideLevel(state) + + expect(overrideLevel).to.eq(mocha_runnable) + + expect(() => { + validateConfig(state, { chromeWebSecurity: false }) + }).to.throw(`The \`chromeWebSecurity\` configuration can never be overridden because it is a read-only configuration option.`) }) - const overrideLevel = getOverrideLevel(state) - - expect(overrideLevel).to.eq('code') - - expect(() => { - validateConfig(state, { testIsolation: 'strict' }) - }).to.throw(`\`Cypress.config()\` cannot override \`testIsolation\`. The \`testIsolation\` option can only be overridden from suite-level overrides.`) }) }) }) diff --git a/packages/driver/src/cy/testConfigOverrides.ts b/packages/driver/src/cy/testConfigOverrides.ts index 26cab8f78d00..021656cb5542 100644 --- a/packages/driver/src/cy/testConfigOverrides.ts +++ b/packages/driver/src/cy/testConfigOverrides.ts @@ -3,6 +3,10 @@ import $errUtils from '../cypress/error_utils' // See Test Config Overrides in ../../../../cli/types/cypress.d.ts +const mochaOverrideLevel = ['suite', 'test'] as const + +export type MochaOverrideLevel = typeof mochaOverrideLevel[number] + type ResolvedTestConfigOverride = { /** * The list of test config overrides and the invocation details used to add helpful @@ -17,12 +21,12 @@ type ResolvedTestConfigOverride = { * The current runnable level of test config overrides that are being applied. * Used for accurate error reporting. */ - applied?: string + applied?: MochaOverrideLevel | 'complete' } type TestConfig = { // The level in which the configuration override was set. - overrideLevel: 'suite' | 'test' + overrideLevel: MochaOverrideLevel // The configuration overrides. Browser is a valid configuration // to indicate the suite or test should run for that browser(s). overrides: Record diff --git a/packages/driver/src/cypress/error_messages.ts b/packages/driver/src/cypress/error_messages.ts index a11ba5675336..a8bb7269b7ef 100644 --- a/packages/driver/src/cypress/error_messages.ts +++ b/packages/driver/src/cypress/error_messages.ts @@ -266,54 +266,26 @@ export default { }, config: { - cypress_config_api (obj) { - const isReadOnly = obj.supportedOverrideLevels === 'never' - - let message = `\`Cypress.config()\` ${isReadOnly ? 'can never' : 'cannot'} override \`{{invalidConfigKey}}\`` - - if (obj.overrideLevel === 'runtime') { - message += ` in a {{runnableType}} at {{overrideLevel}}` - } - - if (isReadOnly) { - message += ' because it is a read-only configuration option.' - } else { - message += `. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.` - } - - return { - message, + cypress_config_api: { + read_only: { + message: `\`Cypress.config()\` can never override \`{{invalidConfigKey}}\` because it is a read-only configuration option.`, docsUrl: 'https://on.cypress.io/config', - } + }, + }, + invalid_mocha_config_override: { + read_only: { + message: `The \`{{invalidConfigKey}}\` configuration can never be overridden because it is a read-only configuration option.`, + docsUrl: 'https://on.cypress.io/config', + }, + suite_only: { + message: `The \`{{invalidConfigKey}}\` configuration can only be overridden from a suite-level override.`, + docsUrl: 'https://on.cypress.io/config', + }, }, invalid_test_override: { message: `The config passed to your {{overrideLevel}}-level overrides has the following validation error:\n\n{{errMsg}}`, docsUrl: 'https://on.cypress.io/config', }, - invalid_event_override (obj) { - let message = `\`Cypress.config()\` can never override \`{{invalidConfigKey}}\` in a {{event}} event handler because it is a read-only configuration option.'` - - if (obj.supportedOverrideLevels !== 'never') { - message = `\`Cypress.config()\` cannot override \`{{invalidConfigKey}}\` in a {{event}} event handler. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.` - } - - return { - message, - docsUrl: 'https://on.cypress.io/config', - } - }, - invalid_mocha_config_override (obj) { - let message = `The \`{{invalidConfigKey}}\` configuration can never be overridden from a {{overrideLevel}}-level override because it is a read-only configuration option.` - - if (obj.supportedOverrideLevels !== 'never') { - message = `The \`{{invalidConfigKey}}\` configuration cannot be overridden from a {{overrideLevel}}-level override. The \`{{invalidConfigKey}}\` option can only be overridden from ${obj.supportedOverrideLevels.join('-level and ')}-level overrides.` - } - - return { - message, - docsUrl: 'https://on.cypress.io/config', - } - }, }, contains: { diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index 09ba7c677a7a..59bd3a78c334 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -1,11 +1,11 @@ import _ from 'lodash' -import { testOverrideLevels, validate as validateConfigValues, validateOverridableAtTestTest } from '@packages/config' +import { testOverrideLevels, validate as validateConfigValues, validateOverridableAtRunTime } from '@packages/config' import { preprocessForSerialization } from './serialization' import $errUtils from '../cypress/error_utils' import type { StateFunc as State } from '../cypress/state' - -import type { ErrResult, OverrideLevel } from '@packages/config' +import type { MochaOverrideLevel } from '../cy/testConfigOverrides' +import type { ErrResult } from '@packages/config' /** * Mutates the config/env object serialized from the other origin to omit read-only values @@ -73,49 +73,41 @@ export const preprocessEnv = (env: Cypress.ObjectLike) => { return preprocessForSerialization(env) as Cypress.Config } -export const getOverrideLevel = (state): OverrideLevel => { +export const getMochaOverrideLevel = (state): MochaOverrideLevel | undefined => { const test = state('test') - let overrideLevel - - if (state('duringUserTestExecution')) { - overrideLevel = 'runtime' - } else if (test) { - if (Object.keys(test?._fired || {}).length) { - overrideLevel = 'event' - } else { - overrideLevel = test._testConfig.applied // either suite or test - } - } else { - overrideLevel = 'fileLoad' // supportFile or specFile load execution + + if (!state('duringUserTestExecution') && test && !Object.keys(test?._fired || {}).length) { + return test._testConfig.applied // either suite or test } - return overrideLevel as OverrideLevel + return undefined } +// Configuration can be override at multiple run-time levels. Ensure the configuration keys can +// be override and that the provided override values are the correct type. +// +// The run-time override levels (listed in order applied): +// fileLoad - config override via Cypress.config() when either loading the supportFile or specFile in the +// browser (this is before mocha as process the spec +// suite - config override via describe('', {...}, () => {}) +// test - config override via it('', {...}, () => {}) +// event - config override via Cypress.config() in test:before:runner or test:before:runner:async event +// runtime - config override via Cypress.config() when the test callback is executed export const validateConfig = (state: State, config: Record, skipConfigOverrideValidation: boolean = false) => { - const overrideLevel = getOverrideLevel(state) - if (!skipConfigOverrideValidation) { - validateOverridableAtTestTest(config, overrideLevel, (validationResult) => { - let errMsg - - if (overrideLevel === 'runtime' || overrideLevel === 'fileLoad') { - errMsg = $errUtils.errByPath('config.cypress_config_api', { - ...validationResult, - runnableType: state('runnable')?.type, - }) - } else if (overrideLevel === 'event') { - const event = _.last(Object.keys(state('test')._fired)) - - errMsg = $errUtils.errByPath('config.invalid_event_override', { - ...validationResult, - event: event?.replace('runner:', ''), - }) - } else { - errMsg = $errUtils.errByPath('config.invalid_mocha_config_override', validationResult) + const mochaOverrideLevel = getMochaOverrideLevel(state) + const isSuiteOverride = mochaOverrideLevel === 'suite' + + validateOverridableAtRunTime(config, isSuiteOverride, (validationResult) => { + let errKey = 'config.cypress_config_api.read_only' + + if (validationResult.supportedOverrideLevel === 'suite') { + errKey = 'config.invalid_mocha_config_override.suite_only' + } else if (mochaOverrideLevel) { + errKey = 'config.invalid_mocha_config_override.read_only' } - throw new (state('specWindow').Error)(errMsg) + throw new (state('specWindow').Error)($errUtils.errByPath(errKey, validationResult)) }) } diff --git a/system-tests/__snapshots__/issue_6407_spec.js b/system-tests/__snapshots__/issue_6407_spec.js index 2b94c877514c..d72fc7176231 100644 --- a/system-tests/__snapshots__/issue_6407_spec.js +++ b/system-tests/__snapshots__/issue_6407_spec.js @@ -25,7 +25,7 @@ exports['e2e issue 6407 throws if mutating read-only config with test configurat 1) throws if mutating read-only config with test configuration: CypressError: The config passed to your test-level overrides has the following validation error: -CypressError: The \`chromeWebSecurity\` configuration can never be overridden from a test-level override because it is a read-only configuration option. +CypressError: The \`chromeWebSecurity\` configuration can never be overridden because it is a read-only configuration option. https://on.cypress.io/config Error diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index c099d2ce0293..9beb5f2a694a 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -162,12 +162,12 @@ exports['testConfigOverrides / fails when setting invalid config opt with Cypres 2 failing 1) does not run: - Error: CypressError: \`Cypress.config()\` can never override \`chromeWebSecurity\` in a test:before:run event handler because it is a read-only configuration option.' + Error: CypressError: \`Cypress.config()\` can never override \`chromeWebSecurity\` because it is a read-only configuration option. [stack trace lines] 2) nested does not run 2: - Error: CypressError: \`Cypress.config()\` can never override \`chromeWebSecurity\` in a test:before:run event handler because it is a read-only configuration option.' + Error: CypressError: \`Cypress.config()\` can never override \`chromeWebSecurity\` because it is a read-only configuration option. [stack trace lines] @@ -245,12 +245,12 @@ exports['testConfigOverrides / fails when setting invalid config opt with Cypres 2 failing 1) does not run: - Error: CypressError: \`Cypress.config()\` can never override \`chromeWebSecurity\` in a test:before:run:async event handler because it is a read-only configuration option.' + Error: CypressError: \`Cypress.config()\` can never override \`chromeWebSecurity\` because it is a read-only configuration option. [stack trace lines] 2) nested does not run 2: - Error: CypressError: \`Cypress.config()\` can never override \`chromeWebSecurity\` in a test:before:run:async event handler because it is a read-only configuration option.' + Error: CypressError: \`Cypress.config()\` can never override \`chromeWebSecurity\` because it is a read-only configuration option. [stack trace lines] @@ -348,16 +348,14 @@ exports['testConfigOverrides / fails when passing invalid config values - [chrom 12) "before each" hook for "5" throws error when invalid config opt in Cypress.config() in after hook - 13) 5 - 14) "after all" hook for "5" + 13) "after all" hook for "5" throws error when invalid config opt in Cypress.config() in afterEach hook - 15) 5 - 16) "after each" hook for "5" + 14) "after each" hook for "5" 1 passing - 16 failing + 14 failing 1) inline test config override throws error: Error: Expected \`baseUrl\` to be a fully qualified URL (starting with \`http://\` or \`https://\`). @@ -374,14 +372,14 @@ Instead the value was: \`"null"\` 3) throws error when invalid test-level override: CypressError: The config passed to your test-level overrides has the following validation error: -CypressError: The \`testIsolation\` configuration cannot be overridden from a test-level override. The \`testIsolation\` option can only be overridden from suite-level overrides. +CypressError: The \`testIsolation\` configuration can only be overridden from a suite-level override. https://on.cypress.io/config Error [stack trace lines] 4) throws error when invalid config opt in Cypress.config() in test: - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a test at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: The \`testIsolation\` configuration can only be overridden from a suite-level override. [stack trace lines] 5) context config overrides throws error @@ -461,38 +459,28 @@ https://on.cypress.io/config 11) throws error when invalid config opt in Cypress.config() in before hook "before all" hook for "4": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: The \`testIsolation\` configuration can only be overridden from a suite-level override. Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] 12) throws error when invalid config opt in Cypress.config() in beforeEach hook "before each" hook for "5": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: The \`testIsolation\` configuration can only be overridden from a suite-level override. Because this error occurred during a \`before each\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] 13) throws error when invalid config opt in Cypress.config() in after hook - 5: - Error: Test Override validation should have failed & it block should not have executed. - [stack trace lines] - - 14) throws error when invalid config opt in Cypress.config() in after hook "after all" hook for "5": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: The \`testIsolation\` configuration can only be overridden from a suite-level override. Because this error occurred during a \`after all\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] - 15) throws error when invalid config opt in Cypress.config() in afterEach hook - 5: - Error: Test Override validation should have failed & it block should not have executed. - [stack trace lines] - - 16) throws error when invalid config opt in Cypress.config() in afterEach hook + 14) throws error when invalid config opt in Cypress.config() in afterEach hook "after each" hook for "5": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: The \`testIsolation\` configuration can only be overridden from a suite-level override. Because this error occurred during a \`after each\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] @@ -508,7 +496,7 @@ Because this error occurred during a \`after each\` hook we are skipping the rem │ Failing: 14 │ │ Pending: 0 │ │ Skipped: 0 │ - │ Screenshots: 9 │ + │ Screenshots: 7 │ │ Video: false │ │ Duration: X seconds │ │ Spec Ran: invalid.js │ @@ -527,12 +515,8 @@ Because this error occurred during a \`after each\` hook we are skipping the rem in Cypress.config() in before hook -- 4 -- before all hook (failed).png - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in beforeEach hook -- 5 -- before each hook (failed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) - in Cypress.config() in after hook -- 5 (failed).png - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in after hook -- 5 -- after all hook (failed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) - in Cypress.config() in afterEach hook -- 5 (failed).png - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in afterEach hook -- 5 -- after each hook (failed).png @@ -846,16 +830,14 @@ exports['testConfigOverrides / fails when passing invalid config values - [firef 12) "before each" hook for "5" throws error when invalid config opt in Cypress.config() in after hook - 13) 5 - 14) "after all" hook for "5" + 13) "after all" hook for "5" throws error when invalid config opt in Cypress.config() in afterEach hook - 15) 5 - 16) "after each" hook for "5" + 14) "after each" hook for "5" 1 passing - 16 failing + 14 failing 1) inline test config override throws error: Error: Expected \`baseUrl\` to be a fully qualified URL (starting with \`http://\` or \`https://\`). @@ -872,13 +854,13 @@ Instead the value was: \`"null"\` 3) throws error when invalid test-level override: CypressError: The config passed to your test-level overrides has the following validation error: -CypressError: The \`testIsolation\` configuration cannot be overridden from a test-level override. The \`testIsolation\` option can only be overridden from suite-level overrides. +CypressError: The \`testIsolation\` configuration can only be overridden from a suite-level override. https://on.cypress.io/config [stack trace lines] 4) throws error when invalid config opt in Cypress.config() in test: - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a test at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: The \`testIsolation\` configuration can only be overridden from a suite-level override. [stack trace lines] 5) context config overrides throws error @@ -952,38 +934,28 @@ https://on.cypress.io/config 11) throws error when invalid config opt in Cypress.config() in before hook "before all" hook for "4": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: The \`testIsolation\` configuration can only be overridden from a suite-level override. Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] 12) throws error when invalid config opt in Cypress.config() in beforeEach hook "before each" hook for "5": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: The \`testIsolation\` configuration can only be overridden from a suite-level override. Because this error occurred during a \`before each\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] 13) throws error when invalid config opt in Cypress.config() in after hook - 5: - Error: Test Override validation should have failed & it block should not have executed. - [stack trace lines] - - 14) throws error when invalid config opt in Cypress.config() in after hook "after all" hook for "5": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: The \`testIsolation\` configuration can only be overridden from a suite-level override. Because this error occurred during a \`after all\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] - 15) throws error when invalid config opt in Cypress.config() in afterEach hook - 5: - Error: Test Override validation should have failed & it block should not have executed. - [stack trace lines] - - 16) throws error when invalid config opt in Cypress.config() in afterEach hook + 14) throws error when invalid config opt in Cypress.config() in afterEach hook "after each" hook for "5": - Error: CypressError: \`Cypress.config()\` cannot override \`testIsolation\` in a hook at runtime. The \`testIsolation\` option can only be overridden from suite-level overrides. + Error: CypressError: The \`testIsolation\` configuration can only be overridden from a suite-level override. Because this error occurred during a \`after each\` hook we are skipping the remaining tests in the current suite: \`throws error when invalid c...\` [stack trace lines] @@ -999,7 +971,7 @@ Because this error occurred during a \`after each\` hook we are skipping the rem │ Failing: 14 │ │ Pending: 0 │ │ Skipped: 0 │ - │ Screenshots: 9 │ + │ Screenshots: 7 │ │ Video: false │ │ Duration: X seconds │ │ Spec Ran: invalid.js │ @@ -1018,12 +990,8 @@ Because this error occurred during a \`after each\` hook we are skipping the rem in Cypress.config() in before hook -- 4 -- before all hook (failed).png - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in beforeEach hook -- 5 -- before each hook (failed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) - in Cypress.config() in after hook -- 5 (failed).png - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in after hook -- 5 -- after all hook (failed).png - - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) - in Cypress.config() in afterEach hook -- 5 (failed).png - /XXX/XXX/XXX/cypress/screenshots/invalid.js/throws error when invalid config opt (1280x720) in Cypress.config() in afterEach hook -- 5 -- after each hook (failed).png diff --git a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js index 73bb69064896..8d71a9c07787 100644 --- a/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js +++ b/system-tests/projects/e2e/cypress/e2e/testConfigOverrides/invalid.js @@ -1,5 +1,5 @@ const shouldNotExecute = () => { - throw new Error('Test Override validation should have failed & it block should not have executed.') + expect(1).to.eq(1) // should not execute - use passing test to ensure correct fail count in system test } it('inline test config override throws error', () => { From 9449f40aba6442cd69c17475e4f57f6be28938a1 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Fri, 12 Aug 2022 09:05:03 -0500 Subject: [PATCH 34/35] Update cli/types/cypress.d.ts Co-authored-by: Matt Henkes --- cli/types/cypress.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index b0137e23e47a..b5b36be75206 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2787,7 +2787,7 @@ declare namespace Cypress { /** * The test isolation level applied to ensure a clean slate between tests. * - legacy - resets/clears aliases, intercepts, clock, viewport, cookies, and local storage before each test. - * - strict - all resets everything from legacy, plus clears the page by visiting 'about:blank' to ensure clean app state before each test. + * - strict - applies all resets/clears from legacy, plus clears the page by visiting 'about:blank' to ensure clean app state before each test. * @default "legacy", however, when experimentalSessionAndOrigin=true, the default is "strict" */ testIsolation: 'legacy' | 'strict' From f34fdb36e46c7e2817416d0b65029ade6a6f10c6 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Fri, 12 Aug 2022 10:23:14 -0500 Subject: [PATCH 35/35] Update packages/server/lib/util/spec_writer.ts Co-authored-by: Matt Henkes --- packages/server/lib/util/spec_writer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/lib/util/spec_writer.ts b/packages/server/lib/util/spec_writer.ts index 7042e57f8941..578924a02089 100644 --- a/packages/server/lib/util/spec_writer.ts +++ b/packages/server/lib/util/spec_writer.ts @@ -385,7 +385,7 @@ export const createFile = (path: string) => { return fs.writeFile(path, newFileTemplate(path)) } -// current un-used with 10.x release +// currently un-used post 10.x release export const countStudioUsage = (path: string) => { return fs.readFile(path) .then((specBuffer) => {