diff --git a/CHANGELOG.md b/CHANGELOG.md index 60e2445465f2..7651ae36c978 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - `[babel-plugin-jest-hoist]` Use `denylist` instead of the deprecated `blacklist` for Babel 8 support ([#14109](https://github.com/jestjs/jest/pull/14109)) - `[expect]` Check error instance type for `toThrow/toThrowError` ([#14576](https://github.com/jestjs/jest/pull/14576)) - `[jest-circus]` [**BREAKING**] Prevent false test failures caused by promise rejections handled asynchronously ([#14315](https://github.com/jestjs/jest/pull/14315)) +- `[jest-circus, jest-expect, jest-snapshot]` Pass `test.failing` tests when containing failing snapshot matchers ([#14313](https://github.com/jestjs/jest/pull/14313)) - `[jest-config]` Make sure to respect `runInBand` option ([#14578](https://github.com/facebook/jest/pull/14578)) - `[@jest/expect-utils]` Fix comparison of `DataView` ([#14408](https://github.com/jestjs/jest/pull/14408)) - `[jest-leak-detector]` Make leak-detector more aggressive when running GC ([#14526](https://github.com/jestjs/jest/pull/14526)) diff --git a/e2e/__tests__/__snapshots__/testFailingSnapshot.test.js.snap b/e2e/__tests__/__snapshots__/testFailingSnapshot.test.js.snap new file mode 100644 index 000000000000..bb6e51f728da --- /dev/null +++ b/e2e/__tests__/__snapshots__/testFailingSnapshot.test.js.snap @@ -0,0 +1,27 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`test.failing doesnt update or remove snapshots 1`] = ` +"// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[\`snapshots not updated nor removed 1\`] = \`"1"\`; + +exports[\`snapshots not updated nor removed 2\`] = \`"1"\`; + +exports[\`snapshots not updated nor removed 3\`] = \`"1"\`; +" +`; + +exports[`test.failing doesnt update or remove snapshots 2`] = ` +"/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +test.failing('inline snapshot not updated', () => { + // eslint-disable-next-line quotes + expect('0').toMatchInlineSnapshot(\`"1"\`); +}); +" +`; diff --git a/e2e/__tests__/testFailingSnapshot.test.js b/e2e/__tests__/testFailingSnapshot.test.js new file mode 100644 index 000000000000..70c61c191416 --- /dev/null +++ b/e2e/__tests__/testFailingSnapshot.test.js @@ -0,0 +1,64 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import * as path from 'path'; +import * as fs from 'graceful-fs'; +import {skipSuiteOnJasmine} from '@jest/test-utils'; +import runJest from '../runJest'; + +skipSuiteOnJasmine(); + +describe('test.failing', () => { + describe('should pass when', () => { + test.failing('snapshot matchers fails', () => { + expect('0').toMatchSnapshot(); + }); + + test.failing('snapshot doesnt exist', () => { + expect('0').toMatchSnapshot(); + }); + + test.failing('inline snapshot matchers fails', () => { + expect('0').toMatchInlineSnapshot('0'); + }); + + test.failing('at least one snapshot fails', () => { + expect('1').toMatchSnapshot(); + expect('0').toMatchSnapshot(); + }); + }); + + describe('should fail when', () => { + test.each([ + ['snapshot', 'snapshot'], + ['inline snapshot', 'inlineSnapshot'], + ])('%s matchers pass', (_, fileName) => { + const dir = path.resolve(__dirname, '../test-failing-snapshot-all-pass'); + const result = runJest(dir, [`./__tests__/${fileName}.test.js`]); + expect(result.exitCode).toBe(1); + }); + }); + + it('doesnt update or remove snapshots', () => { + const dir = path.resolve(__dirname, '../test-failing-snapshot'); + const result = runJest(dir, ['-u']); + expect(result.exitCode).toBe(0); + expect(result.stdout).not.toMatch(/snapshots? (written|removed|obsolete)/); + + const snapshot = fs + .readFileSync( + path.resolve(dir, './__tests__/__snapshots__/snapshot.test.js.snap'), + ) + .toString(); + expect(snapshot).toMatchSnapshot(); + + const inlineSnapshot = fs + .readFileSync(path.resolve(dir, './__tests__/inlineSnapshot.test.js')) + .toString(); + expect(inlineSnapshot).toMatchSnapshot(); + }); +}); diff --git a/e2e/test-failing-snapshot-all-pass/__tests__/__snapshots__/snapshot.test.js.snap b/e2e/test-failing-snapshot-all-pass/__tests__/__snapshots__/snapshot.test.js.snap new file mode 100644 index 000000000000..beec03f00559 --- /dev/null +++ b/e2e/test-failing-snapshot-all-pass/__tests__/__snapshots__/snapshot.test.js.snap @@ -0,0 +1,5 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`snapshots not updated 1`] = `"1"`; + +exports[`snapshots not updated 2`] = `"1"`; diff --git a/e2e/test-failing-snapshot-all-pass/__tests__/inlineSnapshot.test.js b/e2e/test-failing-snapshot-all-pass/__tests__/inlineSnapshot.test.js new file mode 100644 index 000000000000..9d621b6c3c14 --- /dev/null +++ b/e2e/test-failing-snapshot-all-pass/__tests__/inlineSnapshot.test.js @@ -0,0 +1,11 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +test.failing('inline snapshot not updated', () => { + // eslint-disable-next-line quotes + expect('1').toMatchInlineSnapshot(`"1"`); +}); diff --git a/e2e/test-failing-snapshot-all-pass/__tests__/snapshot.test.js b/e2e/test-failing-snapshot-all-pass/__tests__/snapshot.test.js new file mode 100644 index 000000000000..804477962a5c --- /dev/null +++ b/e2e/test-failing-snapshot-all-pass/__tests__/snapshot.test.js @@ -0,0 +1,11 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +test.failing('snapshots not updated', () => { + expect('1').toMatchSnapshot(); + expect('1').toMatchSnapshot(); +}); diff --git a/e2e/test-failing-snapshot-all-pass/package.json b/e2e/test-failing-snapshot-all-pass/package.json new file mode 100644 index 000000000000..148788b25446 --- /dev/null +++ b/e2e/test-failing-snapshot-all-pass/package.json @@ -0,0 +1,5 @@ +{ + "jest": { + "testEnvironment": "node" + } +} diff --git a/e2e/test-failing-snapshot/__tests__/__snapshots__/snapshot.test.js.snap b/e2e/test-failing-snapshot/__tests__/__snapshots__/snapshot.test.js.snap new file mode 100644 index 000000000000..f0502aba8f3b --- /dev/null +++ b/e2e/test-failing-snapshot/__tests__/__snapshots__/snapshot.test.js.snap @@ -0,0 +1,7 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`snapshots not updated nor removed 1`] = `"1"`; + +exports[`snapshots not updated nor removed 2`] = `"1"`; + +exports[`snapshots not updated nor removed 3`] = `"1"`; diff --git a/e2e/test-failing-snapshot/__tests__/inlineSnapshot.test.js b/e2e/test-failing-snapshot/__tests__/inlineSnapshot.test.js new file mode 100644 index 000000000000..1dc3873dd6b2 --- /dev/null +++ b/e2e/test-failing-snapshot/__tests__/inlineSnapshot.test.js @@ -0,0 +1,11 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +test.failing('inline snapshot not updated', () => { + // eslint-disable-next-line quotes + expect('0').toMatchInlineSnapshot(`"1"`); +}); diff --git a/e2e/test-failing-snapshot/__tests__/snapshot.test.js b/e2e/test-failing-snapshot/__tests__/snapshot.test.js new file mode 100644 index 000000000000..234b8f164212 --- /dev/null +++ b/e2e/test-failing-snapshot/__tests__/snapshot.test.js @@ -0,0 +1,12 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +test.failing('snapshots not updated nor removed', () => { + expect('1').toMatchSnapshot(); + expect('0').toMatchSnapshot(); + expect('0').toMatchSnapshot(); +}); diff --git a/e2e/test-failing-snapshot/package.json b/e2e/test-failing-snapshot/package.json new file mode 100644 index 000000000000..148788b25446 --- /dev/null +++ b/e2e/test-failing-snapshot/package.json @@ -0,0 +1,5 @@ +{ + "jest": { + "testEnvironment": "node" + } +} diff --git a/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapter.ts b/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapter.ts index aea3129f3ceb..682c7f3e4982 100644 --- a/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapter.ts +++ b/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapter.ts @@ -109,10 +109,16 @@ const _addSnapshotData = ( results: TestResult, snapshotState: SnapshotState, ) => { - for (const {fullName, status} of results.testResults) { - if (status === 'pending' || status === 'failed') { - // if test is skipped or failed, we don't want to mark + for (const {fullName, status, failing} of results.testResults) { + if ( + status === 'pending' || + status === 'failed' || + (failing && status === 'passed') + ) { + // If test is skipped or failed, we don't want to mark // its snapshots as obsolete. + // When tests called with test.failing pass, they've thrown an exception, + // so maintain any snapshots after the error. snapshotState.markSnapshotsAsCheckedForTest(fullName); } } diff --git a/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts b/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts index f5911893619c..75619d48a967 100644 --- a/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts +++ b/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts @@ -179,6 +179,7 @@ export const runAndTransformResultsToJestFormat = async ({ return { ancestorTitles, duration: testResult.duration, + failing: testResult.failing, failureDetails: testResult.errorsDetailed, failureMessages: testResult.errors, fullName: title @@ -242,7 +243,10 @@ const handleSnapshotStateAfterRetry = const eventHandler = async (event: Circus.Event) => { switch (event.name) { case 'test_start': { - jestExpect.setState({currentTestName: getTestID(event.test)}); + jestExpect.setState({ + currentTestName: getTestID(event.test), + testFailing: event.test.failing, + }); break; } case 'test_done': { diff --git a/packages/jest-circus/src/utils.ts b/packages/jest-circus/src/utils.ts index cdaac752b524..8510ed97e7f4 100644 --- a/packages/jest-circus/src/utils.ts +++ b/packages/jest-circus/src/utils.ts @@ -377,6 +377,7 @@ export const makeSingleTestResult = ( duration: test.duration, errors: errorsDetailed.map(getErrorStack), errorsDetailed, + failing: test.failing, invocations: test.invocations, location, numPassingAsserts: test.numPassingAsserts, @@ -502,6 +503,7 @@ export const parseSingleTestResult = ( return { ancestorTitles, duration: testResult.duration, + failing: testResult.failing, failureDetails: testResult.errorsDetailed, failureMessages: Array.from(testResult.errors), fullName, diff --git a/packages/jest-expect/src/types.ts b/packages/jest-expect/src/types.ts index 7cceee6d01b1..3ab9c5c05b4a 100644 --- a/packages/jest-expect/src/types.ts +++ b/packages/jest-expect/src/types.ts @@ -52,6 +52,8 @@ type PromiseMatchers = { declare module 'expect' { interface MatcherState { snapshotState: SnapshotState; + /** Whether the test was called with `test.failing()` */ + testFailing?: boolean; } interface BaseExpect { addSnapshotSerializer: typeof addSerializer; diff --git a/packages/jest-snapshot/src/State.ts b/packages/jest-snapshot/src/State.ts index e865abd89058..0403a96129ac 100644 --- a/packages/jest-snapshot/src/State.ts +++ b/packages/jest-snapshot/src/State.ts @@ -36,6 +36,7 @@ export type SnapshotMatchOptions = { readonly inlineSnapshot?: string; readonly isInline: boolean; readonly error?: Error; + readonly testFailing?: boolean; }; type SnapshotReturnOptions = { @@ -197,6 +198,7 @@ export default class SnapshotState { inlineSnapshot, isInline, error, + testFailing = false, }: SnapshotMatchOptions): SnapshotReturnOptions { this._counters.set(testName, (this._counters.get(testName) || 0) + 1); const count = Number(this._counters.get(testName)); @@ -230,6 +232,23 @@ export default class SnapshotState { this._snapshotData[key] = receivedSerialized; } + // In pure matching only runs, return the match result while skipping any updates + // reports. + if (testFailing) { + if (hasSnapshot && !isInline) { + // Retain current snapshot values. + this._addSnapshot(key, expected, {error, isInline}); + } + return { + actual: removeExtraLineBreaks(receivedSerialized), + count, + expected: + expected === undefined ? undefined : removeExtraLineBreaks(expected), + key, + pass, + }; + } + // These are the conditions on when to write snapshots: // * There's no snapshot file in a non-CI environment. // * There is a snapshot file and we decided to update the snapshot. diff --git a/packages/jest-snapshot/src/index.ts b/packages/jest-snapshot/src/index.ts index 452704f03567..d4e8a3ab8f26 100644 --- a/packages/jest-snapshot/src/index.ts +++ b/packages/jest-snapshot/src/index.ts @@ -278,7 +278,13 @@ const _toMatchSnapshot = (config: MatchSnapshotConfig) => { config; let {received} = config; - context.dontThrow && context.dontThrow(); + /** If a test was ran with `test.failing`. Passed by Jest Circus. */ + const {testFailing = false} = context; + + if (!testFailing && context.dontThrow) { + // Supress errors while running tests + context.dontThrow(); + } const {currentConcurrentTestName, isNot, snapshotState} = context; const currentTestName = @@ -360,6 +366,7 @@ const _toMatchSnapshot = (config: MatchSnapshotConfig) => { inlineSnapshot, isInline, received, + testFailing, testName: fullTestName, }); const {actual, count, expected, pass} = result; diff --git a/packages/jest-snapshot/src/types.ts b/packages/jest-snapshot/src/types.ts index ed052f4491ce..cf669b8be5c9 100644 --- a/packages/jest-snapshot/src/types.ts +++ b/packages/jest-snapshot/src/types.ts @@ -13,6 +13,7 @@ import type SnapshotState from './State'; export interface Context extends MatcherContext { snapshotState: SnapshotState; + testFailing?: boolean; } // This is typically implemented by `jest-haste-map`'s `HasteFS`, but we diff --git a/packages/jest-test-result/src/types.ts b/packages/jest-test-result/src/types.ts index acab4be765ad..6cd590a5a2db 100644 --- a/packages/jest-test-result/src/types.ts +++ b/packages/jest-test-result/src/types.ts @@ -91,6 +91,11 @@ export type TestResult = { console?: ConsoleBuffer; coverage?: CoverageMapData; displayName?: Config.DisplayName; + /** + * Whether [`test.failing()`](https://jestjs.io/docs/api#testfailingname-fn-timeout) + * was used. + */ + failing?: boolean; failureMessage?: string | null; leaks: boolean; memoryUsage?: number; diff --git a/packages/jest-types/src/Circus.ts b/packages/jest-types/src/Circus.ts index 595f6304f25e..700ca87c2613 100644 --- a/packages/jest-types/src/Circus.ts +++ b/packages/jest-types/src/Circus.ts @@ -203,6 +203,11 @@ export type TestResult = { duration?: number | null; errors: Array; errorsDetailed: Array; + /** + * Whether [`test.failing()`](https://jestjs.io/docs/api#testfailingname-fn-timeout) + * was used. + */ + failing?: boolean; invocations: number; status: TestStatus; location?: {column: number; line: number} | null; diff --git a/packages/jest-types/src/TestResult.ts b/packages/jest-types/src/TestResult.ts index 62f6c4239907..186e0ec6d6c8 100644 --- a/packages/jest-types/src/TestResult.ts +++ b/packages/jest-types/src/TestResult.ts @@ -23,6 +23,11 @@ type Callsite = { export type AssertionResult = { ancestorTitles: Array; duration?: number | null; + /** + * Whether [`test.failing()`](https://jestjs.io/docs/api#testfailingname-fn-timeout) + * was used. + */ + failing?: boolean; failureDetails: Array; failureMessages: Array; fullName: string;