From 34d15d11d161bc5b5a1bcf10e4574f184575c7e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Warda?= Date: Fri, 6 May 2022 12:23:43 +0200 Subject: [PATCH] feat: add `test.failing` method (#12610) --- CHANGELOG.md | 1 + docs/GlobalAPI.md | 56 +++++ .../circusDeclarationErrors.test.ts.snap | 4 +- .../__snapshots__/testFailing.test.ts.snap | 226 ++++++++++++++++++ .../testFailingJasmine.test.ts.snap | 146 +++++++++++ e2e/__tests__/testFailing.test.ts | 50 ++++ e2e/__tests__/testFailingJasmine.test.ts | 22 ++ e2e/test-failing/__tests__/statuses.test.js | 48 ++++ .../__tests__/worksWithConcurrentMode.test.js | 24 ++ .../worksWithConcurrentOnlyMode.test.js | 24 ++ .../__tests__/worksWithOnlyMode.test.js | 67 ++++++ .../__tests__/worksWithSkipMode.test.js | 42 ++++ e2e/test-failing/package.json | 5 + .../circusItFailingTestError.test.ts | 69 ++++++ packages/jest-circus/src/eventHandler.ts | 11 +- packages/jest-circus/src/index.ts | 18 ++ packages/jest-circus/src/run.ts | 18 +- packages/jest-circus/src/utils.ts | 2 + packages/jest-jasmine2/src/index.ts | 12 + .../jest-jasmine2/src/jasmineAsyncInstall.ts | 5 + .../jest-types/__typetests__/globals.test.ts | 39 +++ packages/jest-types/src/Circus.ts | 2 + packages/jest-types/src/Global.ts | 2 + 23 files changed, 888 insertions(+), 5 deletions(-) create mode 100644 e2e/__tests__/__snapshots__/testFailing.test.ts.snap create mode 100644 e2e/__tests__/__snapshots__/testFailingJasmine.test.ts.snap create mode 100644 e2e/__tests__/testFailing.test.ts create mode 100644 e2e/__tests__/testFailingJasmine.test.ts create mode 100644 e2e/test-failing/__tests__/statuses.test.js create mode 100644 e2e/test-failing/__tests__/worksWithConcurrentMode.test.js create mode 100644 e2e/test-failing/__tests__/worksWithConcurrentOnlyMode.test.js create mode 100644 e2e/test-failing/__tests__/worksWithOnlyMode.test.js create mode 100644 e2e/test-failing/__tests__/worksWithSkipMode.test.js create mode 100644 e2e/test-failing/package.json create mode 100644 packages/jest-circus/src/__tests__/circusItFailingTestError.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index aafd0572da72..0b21614d206f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features +- `[jest-circus]` Add `failing` test modifier that inverts the behaviour of tests ([#12610](https://github.com/facebook/jest/pull/12610)) - `[jest-environment-node, jest-environment-jsdom]` Allow specifying `customExportConditions` ([#12774](https://github.com/facebook/jest/pull/12774)) ### Fixes diff --git a/docs/GlobalAPI.md b/docs/GlobalAPI.md index 12f9f9a87c56..42cdd9f9c872 100644 --- a/docs/GlobalAPI.md +++ b/docs/GlobalAPI.md @@ -731,6 +731,62 @@ test.each` }); ``` +### `test.failing(name, fn, timeout)` + +Also under the alias: `it.failing(name, fn, timeout)` + +:::note + +This is only available with the default [jest-circus](https://github.com/facebook/jest/tree/main/packages/jest-circus) runner. + +::: + +Use `test.failing` when you are writing a test and expecting it to fail. These tests will behave the other way normal tests do. If `failing` test will throw any errors then it will pass. If it does not throw it will fail. + +:::tip + +You can use this type of tests i.e. when writing code in a BDD way. In that case the tests will not show up as failing until they pass. Then you can just remove the `failing` modifier to make them pass. + +It can also be a nice way to contribute failing tests to a project, even if you don't know how to fix the bug. + +::: + +Example: + +```js +test.failing('it is not equal', () => { + expect(5).toBe(6); // this test will pass +}); + +test.failing('it is equal', () => { + expect(10).toBe(10); // this test will fail +}); +``` + +### `test.only.failing(name, fn, timeout)` + +Also under the aliases: `it.only.failing(name, fn, timeout)`, `fit.failing(name, fn, timeout)` + +:::note + +This is only available with the default [jest-circus](https://github.com/facebook/jest/tree/main/packages/jest-circus) runner. + +::: + +Use `test.only.failing` if you want to only run a specific failing test. + +### `test.skip.failing(name, fn, timeout)` + +Also under the aliases: `it.skip.failing(name, fn, timeout)`, `xit.failing(name, fn, timeout)`, `xtest.failing(name, fn, timeout)` + +:::note + +This is only available with the default [jest-circus](https://github.com/facebook/jest/tree/main/packages/jest-circus) runner. + +::: + +Use `test.skip.failing` if you want to skip running a specific failing test. + ### `test.only(name, fn, timeout)` Also under the aliases: `it.only(name, fn, timeout)`, and `fit(name, fn, timeout)` diff --git a/e2e/__tests__/__snapshots__/circusDeclarationErrors.test.ts.snap b/e2e/__tests__/__snapshots__/circusDeclarationErrors.test.ts.snap index fb8e6dd705be..d541ca6e6fbe 100644 --- a/e2e/__tests__/__snapshots__/circusDeclarationErrors.test.ts.snap +++ b/e2e/__tests__/__snapshots__/circusDeclarationErrors.test.ts.snap @@ -16,7 +16,7 @@ exports[`defining tests and hooks asynchronously throws 1`] = ` 14 | }); 15 | }); - at eventHandler (../../packages/jest-circus/build/eventHandler.js:145:11) + at eventHandler (../../packages/jest-circus/build/eventHandler.js:153:11) at test (__tests__/asyncDefinition.test.js:12:5) ● Test suite failed to run @@ -46,7 +46,7 @@ exports[`defining tests and hooks asynchronously throws 1`] = ` 20 | }); 21 | - at eventHandler (../../packages/jest-circus/build/eventHandler.js:145:11) + at eventHandler (../../packages/jest-circus/build/eventHandler.js:153:11) at test (__tests__/asyncDefinition.test.js:18:3) ● Test suite failed to run diff --git a/e2e/__tests__/__snapshots__/testFailing.test.ts.snap b/e2e/__tests__/__snapshots__/testFailing.test.ts.snap new file mode 100644 index 000000000000..32a75730235d --- /dev/null +++ b/e2e/__tests__/__snapshots__/testFailing.test.ts.snap @@ -0,0 +1,226 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`works with all statuses 1`] = ` +"FAIL __tests__/statuses.test.js + ✓ passes + ✕ fails + ✓ failing fails = passes + ✓ failing fails = passes with test syntax + ✕ failing passes = fails + ○ skipped skips + ○ skipped skipped failing 1 + ○ skipped skipped failing 2 + ○ skipped skipped failing with different syntax + ○ skipped skipped failing with another different syntax + ✎ todo todo + + ● fails + + expect(received).toBe(expected) // Object.is equality + + Expected: 101 + Received: 10 + + 11 | + 12 | it('fails', () => { + > 13 | expect(10).toBe(101); + | ^ + 14 | }); + 15 | + 16 | it.skip('skips', () => { + + at Object.toBe (__tests__/statuses.test.js:13:14) + + ● failing passes = fails + + Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error. + + 36 | }); + 37 | + > 38 | it.failing('failing passes = fails', () => { + | ^ + 39 | expect(10).toBe(10); + 40 | }); + 41 | + + at Object.failing (__tests__/statuses.test.js:38:4)" +`; + +exports[`works with concurrent and only mode 1`] = ` +"FAIL __tests__/worksWithConcurrentOnlyMode.test.js + block with concurrent + ✕ failing passes = fails + ✓ failing fails = passes + ○ skipped skipped failing test + ○ skipped skipped failing fails + + ● block with concurrent › failing passes = fails + + Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error. + + 11 | }); + 12 | + > 13 | it.concurrent.only.failing('failing passes = fails', () => { + | ^ + 14 | expect(10).toBe(10); + 15 | }); + 16 | + + at failing (__tests__/worksWithConcurrentOnlyMode.test.js:13:22) + at Object.describe (__tests__/worksWithConcurrentOnlyMode.test.js:8:1)" +`; + +exports[`works with concurrent mode 1`] = ` +"FAIL __tests__/worksWithConcurrentMode.test.js + block with concurrent + ✕ failing test + ✕ failing passes = fails + ✓ failing fails = passes + ○ skipped skipped failing fails + + ● block with concurrent › failing test + + expect(received).toBe(expected) // Object.is equality + + Expected: 101 + Received: 10 + + 8 | describe('block with concurrent', () => { + 9 | it('failing test', () => { + > 10 | expect(10).toBe(101); + | ^ + 11 | }); + 12 | + 13 | it.concurrent.failing('failing passes = fails', () => { + + at Object.toBe (__tests__/worksWithConcurrentMode.test.js:10:16) + + ● block with concurrent › failing passes = fails + + Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error. + + 11 | }); + 12 | + > 13 | it.concurrent.failing('failing passes = fails', () => { + | ^ + 14 | expect(10).toBe(10); + 15 | }); + 16 | + + at failing (__tests__/worksWithConcurrentMode.test.js:13:17) + at Object.describe (__tests__/worksWithConcurrentMode.test.js:8:1)" +`; + +exports[`works with only mode 1`] = ` +"FAIL __tests__/worksWithOnlyMode.test.js + block with only, should pass + ✓ failing fails = passes, should pass + ○ skipped failing test but skipped + ○ skipped passing test but skipped + block with only, should fail + ✕ failing passes = fails, should fail + ○ skipped failing test but skipped + ○ skipped passing test but skipped + block with only in other it, should skip + ✕ failing test + ○ skipped failing passes = fails, should fail but skipped + ○ skipped passing test but skipped + block with only with different syntax, should fail + ✕ failing passes = fails, should fail 1 + ✕ failing passes = fails, should fail 2 + ○ skipped failing test but skipped + ○ skipped passing test but skipped + + ● block with only, should fail › failing passes = fails, should fail + + Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error. + + 21 | + 22 | describe('block with only, should fail', () => { + > 23 | it.only.failing('failing passes = fails, should fail', () => { + | ^ + 24 | expect(10).toBe(10); + 25 | }); + 26 | + + at failing (__tests__/worksWithOnlyMode.test.js:23:11) + at Object.describe (__tests__/worksWithOnlyMode.test.js:22:1) + + ● block with only in other it, should skip › failing test + + expect(received).toBe(expected) // Object.is equality + + Expected: 101 + Received: 10 + + 41 | // eslint-disable-next-line jest/no-focused-tests + 42 | it.only('failing test', () => { + > 43 | expect(10).toBe(101); + | ^ + 44 | }); + 45 | + 46 | it('passing test but skipped', () => { + + at Object.toBe (__tests__/worksWithOnlyMode.test.js:43:16) + + ● block with only with different syntax, should fail › failing passes = fails, should fail 1 + + Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error. + + 50 | + 51 | describe('block with only with different syntax, should fail', () => { + > 52 | fit.failing('failing passes = fails, should fail 1', () => { + | ^ + 53 | expect(10).toBe(10); + 54 | }); + 55 | + + at failing (__tests__/worksWithOnlyMode.test.js:52:7) + at Object.describe (__tests__/worksWithOnlyMode.test.js:51:1) + + ● block with only with different syntax, should fail › failing passes = fails, should fail 2 + + Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error. + + 54 | }); + 55 | + > 56 | test.only.failing('failing passes = fails, should fail 2', () => { + | ^ + 57 | expect(10).toBe(10); + 58 | }); + 59 | + + at failing (__tests__/worksWithOnlyMode.test.js:56:13) + at Object.describe (__tests__/worksWithOnlyMode.test.js:51:1)" +`; + +exports[`works with skip mode 1`] = ` +"FAIL __tests__/worksWithSkipMode.test.js + block with only, should pass + ✕ failing test + ✓ failing fails = passes + ○ skipped skipped failing fails = passes, should pass + ○ skipped passing test + block with only, should fail + ✓ passing test + ✓ failing passes = fails + ○ skipped failing passes = fails, should fail + ○ skipped failing test + + ● block with only, should pass › failing test + + expect(received).toBe(expected) // Object.is equality + + Expected: 101 + Received: 10 + + 12 | + 13 | it('failing test', () => { + > 14 | expect(10).toBe(101); + | ^ + 15 | }); + 16 | + 17 | it.skip('passing test', () => { + + at Object.toBe (__tests__/worksWithSkipMode.test.js:14:16)" +`; diff --git a/e2e/__tests__/__snapshots__/testFailingJasmine.test.ts.snap b/e2e/__tests__/__snapshots__/testFailingJasmine.test.ts.snap new file mode 100644 index 000000000000..b4afcb588c50 --- /dev/null +++ b/e2e/__tests__/__snapshots__/testFailingJasmine.test.ts.snap @@ -0,0 +1,146 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`throws an error about unsupported modifier 1`] = ` +"FAIL __tests__/statuses.test.js + ● Test suite failed to run + + Jest: \`failing\` tests are only supported in \`jest-circus\`. + + 20 | it.todo('todo'); + 21 | + > 22 | it.failing('failing fails = passes', () => { + | ^ + 23 | expect(10).toBe(101); + 24 | }); + 25 | + + at Object.failing (__tests__/statuses.test.js:22:4) + +FAIL __tests__/worksWithConcurrentMode.test.js + ● block with concurrent › failing test + + expect(received).toBe(expected) // Object.is equality + + Expected: 101 + Received: 10 + + 8 | describe('block with concurrent', () => { + 9 | it('failing test', () => { + > 10 | expect(10).toBe(101); + | ^ + 11 | }); + 12 | + 13 | it.concurrent.failing('failing passes = fails', () => { + + at Object.toBe (__tests__/worksWithConcurrentMode.test.js:10:16) + +FAIL __tests__/worksWithConcurrentOnlyMode.test.js + ● block with concurrent › skipped failing test + + expect(received).toBe(expected) // Object.is equality + + Expected: 101 + Received: 10 + + 8 | describe('block with concurrent', () => { + 9 | it('skipped failing test', () => { + > 10 | expect(10).toBe(101); + | ^ + 11 | }); + 12 | + 13 | it.concurrent.only.failing('failing passes = fails', () => { + + at Object.toBe (__tests__/worksWithConcurrentOnlyMode.test.js:10:16) + +FAIL __tests__/worksWithOnlyMode.test.js + ● block with only, should pass › encountered a declaration exception + + Jest: \`failing\` tests are only supported in \`jest-circus\`. + + 7 | + 8 | describe('block with only, should pass', () => { + > 9 | it.only.failing('failing fails = passes, should pass', () => { + | ^ + 10 | expect(10).toBe(101); + 11 | }); + 12 | + + at Suite.failing (__tests__/worksWithOnlyMode.test.js:9:11) + at Object.describe (__tests__/worksWithOnlyMode.test.js:8:1) + + ● block with only, should fail › encountered a declaration exception + + Jest: \`failing\` tests are only supported in \`jest-circus\`. + + 21 | + 22 | describe('block with only, should fail', () => { + > 23 | it.only.failing('failing passes = fails, should fail', () => { + | ^ + 24 | expect(10).toBe(10); + 25 | }); + 26 | + + at Suite.failing (__tests__/worksWithOnlyMode.test.js:23:11) + at Object.describe (__tests__/worksWithOnlyMode.test.js:22:1) + + ● block with only in other it, should skip › encountered a declaration exception + + Jest: \`failing\` tests are only supported in \`jest-circus\`. + + 35 | + 36 | describe('block with only in other it, should skip', () => { + > 37 | it.failing('failing passes = fails, should fail but skipped', () => { + | ^ + 38 | expect(10).toBe(10); + 39 | }); + 40 | + + at Suite.failing (__tests__/worksWithOnlyMode.test.js:37:6) + at Object.describe (__tests__/worksWithOnlyMode.test.js:36:1) + + ● block with only with different syntax, should fail › encountered a declaration exception + + Jest: \`failing\` tests are only supported in \`jest-circus\`. + + 50 | + 51 | describe('block with only with different syntax, should fail', () => { + > 52 | fit.failing('failing passes = fails, should fail 1', () => { + | ^ + 53 | expect(10).toBe(10); + 54 | }); + 55 | + + at Suite.failing (__tests__/worksWithOnlyMode.test.js:52:7) + at Object.describe (__tests__/worksWithOnlyMode.test.js:51:1) + +FAIL __tests__/worksWithSkipMode.test.js + ● block with only, should pass › encountered a declaration exception + + Jest: \`failing\` tests are only supported in \`jest-circus\`. + + 7 | + 8 | describe('block with only, should pass', () => { + > 9 | it.skip.failing('skipped failing fails = passes, should pass', () => { + | ^ + 10 | expect(10).toBe(101); + 11 | }); + 12 | + + at Suite.failing (__tests__/worksWithSkipMode.test.js:9:11) + at Object.describe (__tests__/worksWithSkipMode.test.js:8:1) + + ● block with only, should fail › encountered a declaration exception + + Jest: \`failing\` tests are only supported in \`jest-circus\`. + + 25 | + 26 | describe('block with only, should fail', () => { + > 27 | it.skip.failing('failing passes = fails, should fail', () => { + | ^ + 28 | expect(10).toBe(10); + 29 | }); + 30 | + + at Suite.failing (__tests__/worksWithSkipMode.test.js:27:11) + at Object.describe (__tests__/worksWithSkipMode.test.js:26:1)" +`; diff --git a/e2e/__tests__/testFailing.test.ts b/e2e/__tests__/testFailing.test.ts new file mode 100644 index 000000000000..da8584ba7b95 --- /dev/null +++ b/e2e/__tests__/testFailing.test.ts @@ -0,0 +1,50 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import * as path from 'path'; +import {skipSuiteOnJasmine} from '@jest/test-utils'; +import {extractSummary} from '../Utils'; +import runJest, {json as runWithJson} from '../runJest'; + +skipSuiteOnJasmine(); + +const dir = path.resolve(__dirname, '../test-failing'); + +test('works with all statuses', () => { + const result = runJest(dir, ['statuses.test.js']); + expect(result.exitCode).toBe(1); + const {rest} = extractSummary(result.stderr); + expect(rest).toMatchSnapshot(); +}); + +test('works with only mode', () => { + const result = runJest(dir, ['worksWithOnlyMode.test.js']); + expect(result.exitCode).toBe(1); + const {rest} = extractSummary(result.stderr); + expect(rest).toMatchSnapshot(); +}); + +test('works with skip mode', () => { + const result = runJest(dir, ['worksWithSkipMode.test.js']); + expect(result.exitCode).toBe(1); + const {rest} = extractSummary(result.stderr); + expect(rest).toMatchSnapshot(); +}); + +test('works with concurrent mode', () => { + const result = runWithJson(dir, ['worksWithConcurrentMode.test.js']); + expect(result.exitCode).toBe(1); + const {rest} = extractSummary(result.stderr); + expect(rest).toMatchSnapshot(); +}); + +test('works with concurrent and only mode', () => { + const result = runWithJson(dir, ['worksWithConcurrentOnlyMode.test.js']); + expect(result.exitCode).toBe(1); + const {rest} = extractSummary(result.stderr); + expect(rest).toMatchSnapshot(); +}); diff --git a/e2e/__tests__/testFailingJasmine.test.ts b/e2e/__tests__/testFailingJasmine.test.ts new file mode 100644 index 000000000000..7c6aeecd2f9d --- /dev/null +++ b/e2e/__tests__/testFailingJasmine.test.ts @@ -0,0 +1,22 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import * as path from 'path'; +import {skipSuiteOnJestCircus} from '@jest/test-utils'; +import {extractSortedSummary} from '../Utils'; +import runJest from '../runJest'; + +skipSuiteOnJestCircus(); + +const dir = path.resolve(__dirname, '../test-failing'); + +test('throws an error about unsupported modifier', () => { + const result = runJest(dir); + expect(result.exitCode).toBe(1); + const {rest} = extractSortedSummary(result.stderr); + expect(rest).toMatchSnapshot(); +}); diff --git a/e2e/test-failing/__tests__/statuses.test.js b/e2e/test-failing/__tests__/statuses.test.js new file mode 100644 index 000000000000..a75235f1367b --- /dev/null +++ b/e2e/test-failing/__tests__/statuses.test.js @@ -0,0 +1,48 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +it('passes', () => { + expect(10).toBe(10); +}); + +it('fails', () => { + expect(10).toBe(101); +}); + +it.skip('skips', () => { + expect(10).toBe(101); +}); + +it.todo('todo'); + +it.failing('failing fails = passes', () => { + expect(10).toBe(101); +}); + +test.failing('failing fails = passes with test syntax', () => { + expect(10).toBe(101); +}); + +it.skip.failing('skipped failing 1', () => { + expect(10).toBe(10); +}); + +test.skip.failing('skipped failing 2', () => { + expect(10).toBe(101); +}); + +it.failing('failing passes = fails', () => { + expect(10).toBe(10); +}); + +xit.failing('skipped failing with different syntax', () => { + expect(10).toBe(10); +}); + +xtest.failing('skipped failing with another different syntax', () => { + expect(10).toBe(10); +}); diff --git a/e2e/test-failing/__tests__/worksWithConcurrentMode.test.js b/e2e/test-failing/__tests__/worksWithConcurrentMode.test.js new file mode 100644 index 000000000000..950b01c29d08 --- /dev/null +++ b/e2e/test-failing/__tests__/worksWithConcurrentMode.test.js @@ -0,0 +1,24 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +describe('block with concurrent', () => { + it('failing test', () => { + expect(10).toBe(101); + }); + + it.concurrent.failing('failing passes = fails', () => { + expect(10).toBe(10); + }); + + it.concurrent.failing('failing fails = passes', () => { + expect(10).toBe(101); + }); + + it.concurrent.skip.failing('skipped failing fails', () => { + expect(10).toBe(101); + }); +}); diff --git a/e2e/test-failing/__tests__/worksWithConcurrentOnlyMode.test.js b/e2e/test-failing/__tests__/worksWithConcurrentOnlyMode.test.js new file mode 100644 index 000000000000..93d8850d90ed --- /dev/null +++ b/e2e/test-failing/__tests__/worksWithConcurrentOnlyMode.test.js @@ -0,0 +1,24 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +describe('block with concurrent', () => { + it('skipped failing test', () => { + expect(10).toBe(101); + }); + + it.concurrent.only.failing('failing passes = fails', () => { + expect(10).toBe(10); + }); + + it.concurrent.only.failing('failing fails = passes', () => { + expect(10).toBe(101); + }); + + it.concurrent.failing('skipped failing fails', () => { + expect(10).toBe(101); + }); +}); diff --git a/e2e/test-failing/__tests__/worksWithOnlyMode.test.js b/e2e/test-failing/__tests__/worksWithOnlyMode.test.js new file mode 100644 index 000000000000..47933f8c0234 --- /dev/null +++ b/e2e/test-failing/__tests__/worksWithOnlyMode.test.js @@ -0,0 +1,67 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +describe('block with only, should pass', () => { + it.only.failing('failing fails = passes, should pass', () => { + expect(10).toBe(101); + }); + + it('failing test but skipped', () => { + expect(10).toBe(101); + }); + + it('passing test but skipped', () => { + expect(10).toBe(10); + }); +}); + +describe('block with only, should fail', () => { + it.only.failing('failing passes = fails, should fail', () => { + expect(10).toBe(10); + }); + + it('failing test but skipped', () => { + expect(10).toBe(101); + }); + + it('passing test but skipped', () => { + expect(10).toBe(10); + }); +}); + +describe('block with only in other it, should skip', () => { + it.failing('failing passes = fails, should fail but skipped', () => { + expect(10).toBe(10); + }); + + // eslint-disable-next-line jest/no-focused-tests + it.only('failing test', () => { + expect(10).toBe(101); + }); + + it('passing test but skipped', () => { + expect(10).toBe(10); + }); +}); + +describe('block with only with different syntax, should fail', () => { + fit.failing('failing passes = fails, should fail 1', () => { + expect(10).toBe(10); + }); + + test.only.failing('failing passes = fails, should fail 2', () => { + expect(10).toBe(10); + }); + + it('failing test but skipped', () => { + expect(10).toBe(101); + }); + + it('passing test but skipped', () => { + expect(10).toBe(10); + }); +}); diff --git a/e2e/test-failing/__tests__/worksWithSkipMode.test.js b/e2e/test-failing/__tests__/worksWithSkipMode.test.js new file mode 100644 index 000000000000..36cb4d1e43bd --- /dev/null +++ b/e2e/test-failing/__tests__/worksWithSkipMode.test.js @@ -0,0 +1,42 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +describe('block with only, should pass', () => { + it.skip.failing('skipped failing fails = passes, should pass', () => { + expect(10).toBe(101); + }); + + it('failing test', () => { + expect(10).toBe(101); + }); + + it.skip('passing test', () => { + expect(10).toBe(10); + }); + + it.failing('failing fails = passes', () => { + expect(10).toBe(101); + }); +}); + +describe('block with only, should fail', () => { + it.skip.failing('failing passes = fails, should fail', () => { + expect(10).toBe(10); + }); + + it.skip('failing test', () => { + expect(10).toBe(101); + }); + + it('passing test', () => { + expect(10).toBe(10); + }); + + it.failing('failing passes = fails', () => { + expect(10).toBe(101); + }); +}); diff --git a/e2e/test-failing/package.json b/e2e/test-failing/package.json new file mode 100644 index 000000000000..148788b25446 --- /dev/null +++ b/e2e/test-failing/package.json @@ -0,0 +1,5 @@ +{ + "jest": { + "testEnvironment": "node" + } +} diff --git a/packages/jest-circus/src/__tests__/circusItFailingTestError.test.ts b/packages/jest-circus/src/__tests__/circusItFailingTestError.test.ts new file mode 100644 index 000000000000..65322ba71184 --- /dev/null +++ b/packages/jest-circus/src/__tests__/circusItFailingTestError.test.ts @@ -0,0 +1,69 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import type {Global} from '@jest/types'; + +let circusIt: Global.It; +let circusTest: Global.It; + +const aliasCircusIt = () => { + const {it, test} = require('../'); + circusIt = it; + circusTest = test; +}; + +aliasCircusIt(); + +describe('test/it.failing error throwing', () => { + it("it doesn't throw an error with valid arguments", () => { + expect(() => { + circusIt.failing('test1', () => {}); + }).not.toThrowError(); + }); + it('it throws error with missing callback function', () => { + expect(() => { + circusIt.failing('test2'); + }).toThrowError( + 'Missing second argument. It must be a callback function. Perhaps you want to use `test.todo` for a test placeholder.', + ); + }); + it("it throws an error when first argument isn't valid", () => { + expect(() => { + circusIt.failing(() => {}); + }).toThrowError( + 'Invalid first argument, () => {}. It must be a named class, named function, number, or string.', + ); + }); + it('it throws an error when callback function is not a function', () => { + expect(() => { + circusIt.failing('test4', 'test4b'); + }).toThrowError( + 'Invalid second argument, test4b. It must be a callback function.', + ); + }); + it('test throws error with missing callback function', () => { + expect(() => { + circusTest.failing('test5'); + }).toThrowError( + 'Missing second argument. It must be a callback function. Perhaps you want to use `test.todo` for a test placeholder.', + ); + }); + it("test throws an error when first argument isn't a string", () => { + expect(() => { + circusTest.failing(() => {}); + }).toThrowError( + 'Invalid first argument, () => {}. It must be a named class, named function, number, or string.', + ); + }); + it('test throws an error when callback function is not a function', () => { + expect(() => { + circusTest.failing('test7', 'test8b'); + }).toThrowError( + 'Invalid second argument, test8b. It must be a callback function.', + ); + }); +}); diff --git a/packages/jest-circus/src/eventHandler.ts b/packages/jest-circus/src/eventHandler.ts index 21c9b35ab2ba..904ebd383d7a 100644 --- a/packages/jest-circus/src/eventHandler.ts +++ b/packages/jest-circus/src/eventHandler.ts @@ -122,7 +122,15 @@ const eventHandler: Circus.EventHandler = (event, state) => { } case 'add_test': { const {currentDescribeBlock, currentlyRunningTest, hasStarted} = state; - const {asyncError, fn, mode, testName: name, timeout, concurrent} = event; + const { + asyncError, + fn, + mode, + testName: name, + timeout, + concurrent, + failing, + } = event; if (currentlyRunningTest) { currentlyRunningTest.errors.push( @@ -148,6 +156,7 @@ const eventHandler: Circus.EventHandler = (event, state) => { currentDescribeBlock, timeout, asyncError, + failing, ); if (currentDescribeBlock.mode !== 'skip' && test.mode === 'only') { state.hasFocusedTests = true; diff --git a/packages/jest-circus/src/index.ts b/packages/jest-circus/src/index.ts index cc8c63872cbe..7d49ba1ddc07 100644 --- a/packages/jest-circus/src/index.ts +++ b/packages/jest-circus/src/index.ts @@ -139,6 +139,15 @@ const test: Global.It = (() => { timeout?: number, ): void => _addTest(testName, 'only', true, fn, concurrentOnly, timeout); + const bindFailing = (concurrent: boolean, mode: Circus.TestMode) => { + const failing = ( + testName: Circus.TestNameLike, + fn?: Circus.TestFn, + timeout?: number, + ): void => _addTest(testName, mode, concurrent, fn, failing, timeout, true); + return failing; + }; + test.todo = (testName: Circus.TestNameLike, ...rest: Array): void => { if (rest.length > 0 || typeof testName !== 'string') { throw new ErrorWithStack( @@ -160,6 +169,7 @@ const test: Global.It = (() => { timeout?: number, ) => void, timeout?: number, + failing?: boolean, ) => { const asyncError = new ErrorWithStack(undefined, testFn); @@ -185,6 +195,7 @@ const test: Global.It = (() => { return dispatchSync({ asyncError, concurrent, + failing: failing === undefined ? false : failing, fn, mode, name: 'add_test', @@ -196,14 +207,21 @@ const test: Global.It = (() => { test.each = bindEach(test); only.each = bindEach(only); skip.each = bindEach(skip); + concurrentTest.each = bindEach(concurrentTest, false); concurrentOnly.each = bindEach(concurrentOnly, false); + only.failing = bindFailing(false, 'only'); + skip.failing = bindFailing(false, 'skip'); + + test.failing = bindFailing(false); test.only = only; test.skip = skip; test.concurrent = concurrentTest; concurrentTest.only = concurrentOnly; concurrentTest.skip = skip; + concurrentTest.failing = bindFailing(true); + concurrentOnly.failing = bindFailing(true, 'only'); return test; })(); diff --git a/packages/jest-circus/src/run.ts b/packages/jest-circus/src/run.ts index f22faea266ea..4a944d186c2d 100644 --- a/packages/jest-circus/src/run.ts +++ b/packages/jest-circus/src/run.ts @@ -224,9 +224,23 @@ const _callCircusTest = async ( isHook: false, timeout, }); - await dispatch({name: 'test_fn_success', test}); + if (test.failing) { + test.asyncError.message = + 'Failing test passed even though it was supposed to fail. Remove `.failing` to remove error.'; + await dispatch({ + error: test.asyncError, + name: 'test_fn_failure', + test, + }); + } else { + await dispatch({name: 'test_fn_success', test}); + } } catch (error) { - await dispatch({error, name: 'test_fn_failure', test}); + if (test.failing) { + await dispatch({name: 'test_fn_success', test}); + } else { + await dispatch({error, name: 'test_fn_failure', test}); + } } }; diff --git a/packages/jest-circus/src/utils.ts b/packages/jest-circus/src/utils.ts index 11060646a3be..4d787f83fa90 100644 --- a/packages/jest-circus/src/utils.ts +++ b/packages/jest-circus/src/utils.ts @@ -61,12 +61,14 @@ export const makeTest = ( parent: Circus.DescribeBlock, timeout: number | undefined, asyncError: Circus.Exception, + failing: boolean, ): Circus.TestEntry => ({ type: 'test', // eslint-disable-next-line sort-keys asyncError, concurrent, duration: null, errors: [], + failing, fn, invocations: 0, mode, diff --git a/packages/jest-jasmine2/src/index.ts b/packages/jest-jasmine2/src/index.ts index cf7c8bf5b4b1..e09cbcc941cd 100644 --- a/packages/jest-jasmine2/src/index.ts +++ b/packages/jest-jasmine2/src/index.ts @@ -12,6 +12,7 @@ import type {AssertionResult, TestResult} from '@jest/test-result'; import type {Config, Global} from '@jest/types'; import type Runtime from 'jest-runtime'; import type {SnapshotState} from 'jest-snapshot'; +import {ErrorWithStack} from 'jest-util'; import installEach from './each'; import {installErrorOnPrivate} from './errorOnPrivate'; import type Spec from './jasmine/Spec'; @@ -80,6 +81,17 @@ export default async function jasmine2( installEach(environment); + const failing = () => { + throw new ErrorWithStack( + 'Jest: `failing` tests are only supported in `jest-circus`.', + failing, + ); + }; + + environment.global.it.failing = failing; + environment.global.fit.failing = failing; + environment.global.xit.failing = failing; + environment.global.test = environment.global.it; environment.global.it.only = environment.global.fit; environment.global.it.todo = env.todo; diff --git a/packages/jest-jasmine2/src/jasmineAsyncInstall.ts b/packages/jest-jasmine2/src/jasmineAsyncInstall.ts index 836b0df29726..5e0cb5288bc6 100644 --- a/packages/jest-jasmine2/src/jasmineAsyncInstall.ts +++ b/packages/jest-jasmine2/src/jasmineAsyncInstall.ts @@ -223,6 +223,11 @@ function makeConcurrent( }; // each is binded after the function is made concurrent, so for now it is made noop concurrentFn.each = () => () => {}; + concurrentFn.failing = () => () => { + throw new Error( + 'Jest: `failing` tests are only supported in `jest-circus`.', + ); + }; return concurrentFn; } diff --git a/packages/jest-types/__typetests__/globals.test.ts b/packages/jest-types/__typetests__/globals.test.ts index fa8f3cf20940..e4bbd6fb6c6b 100644 --- a/packages/jest-types/__typetests__/globals.test.ts +++ b/packages/jest-types/__typetests__/globals.test.ts @@ -135,6 +135,42 @@ expectType(test.skip.each(list)(function named() {}, fn)); expectType(test.skip.each(list)(class {}, fn)); expectType(test.skip.each(list)(class Named {}, fn)); +expectType(test.failing(123, fn)); +expectType(test.failing(() => {}, fn)); +expectType(test.failing(function named() {}, fn)); +expectType(test.failing(class {}, fn)); +expectType(test.failing(class Named {}, fn)); + +expectType(test.skip.failing(123, fn)); +expectType(test.skip.failing(() => {}, fn)); +expectType(test.skip.failing(function named() {}, fn)); +expectType(test.skip.failing(class {}, fn)); +expectType(test.skip.failing(class Named {}, fn)); + +expectType(test.only.failing(123, fn)); +expectType(test.only.failing(() => {}, fn)); +expectType(test.only.failing(function named() {}, fn)); +expectType(test.only.failing(class {}, fn)); +expectType(test.only.failing(class Named {}, fn)); + +expectType(test.concurrent.failing(123, asyncFn)); +expectType(test.concurrent.failing(() => {}, asyncFn)); +expectType(test.concurrent.failing(function named() {}, asyncFn)); +expectType(test.concurrent.failing(class {}, asyncFn)); +expectType(test.concurrent.failing(class Named {}, asyncFn)); + +expectType(test.concurrent.skip.failing(123, asyncFn)); +expectType(test.concurrent.skip.failing(() => {}, asyncFn)); +expectType(test.concurrent.skip.failing(function named() {}, asyncFn)); +expectType(test.concurrent.skip.failing(class {}, asyncFn)); +expectType(test.concurrent.skip.failing(class Named {}, asyncFn)); + +expectType(test.concurrent.only.failing(123, asyncFn)); +expectType(test.concurrent.only.failing(() => {}, asyncFn)); +expectType(test.concurrent.only.failing(function named() {}, asyncFn)); +expectType(test.concurrent.only.failing(class {}, asyncFn)); +expectType(test.concurrent.only.failing(class Named {}, asyncFn)); + expectType( test.each` a | b | expected @@ -189,6 +225,9 @@ expectType( `(testName, fn, timeout), ); +expectType(test.concurrent(testName, asyncFn)); +expectType(test.concurrent(testName, asyncFn, timeout)); + expectType(test.concurrent.each(list)(testName, asyncFn)); expectType(test.concurrent.each(list)(testName, asyncFn, timeout)); expectType(test.concurrent.each(table)(testName, asyncFn)); diff --git a/packages/jest-types/src/Circus.ts b/packages/jest-types/src/Circus.ts index cf2306b12e20..7ed5023d6caf 100644 --- a/packages/jest-types/src/Circus.ts +++ b/packages/jest-types/src/Circus.ts @@ -74,6 +74,7 @@ export type SyncEvent = mode?: TestMode; concurrent: boolean; timeout: number | undefined; + failing: boolean; } | { // Any unhandled error that happened outside of test/hooks (unless it is @@ -250,4 +251,5 @@ export type TestEntry = { seenDone: boolean; status?: TestStatus | null; // whether the test has been skipped or run already timeout?: number; + failing: boolean; }; diff --git a/packages/jest-types/src/Global.ts b/packages/jest-types/src/Global.ts index aed1c4b2a8c9..aa17a4c8f851 100644 --- a/packages/jest-types/src/Global.ts +++ b/packages/jest-types/src/Global.ts @@ -69,6 +69,7 @@ export interface HookBase { export interface ItBase { (testName: TestNameLike, fn: TestFn, timeout?: number): void; each: Each; + failing(testName: TestNameLike, fn: TestFn, timeout?: number): void; } export interface It extends ItBase { @@ -80,6 +81,7 @@ export interface It extends ItBase { export interface ItConcurrentBase { (testName: TestNameLike, testFn: ConcurrentTestFn, timeout?: number): void; each: Each; + failing(testName: TestNameLike, fn: ConcurrentTestFn, timeout?: number): void; } export interface ItConcurrentExtended extends ItConcurrentBase {