feat: add summaryThreshold option to summary reporter (#13895)

CHANGELOG.md

@@ -9,6 +9,7 @@
 - `[@jest/create-cache-key-function]` Allow passing `length` argument to `createCacheKey()` function and set its default value to `16` on Windows ([#13827](https://github.com/facebook/jest/pull/13827))
 - `[jest-message-util]` Add support for [AggregateError](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AggregateError) ([#13946](https://github.com/facebook/jest/pull/13946) & [#13947](https://github.com/facebook/jest/pull/13947))
 - `[jest-message-util]` Add support for [Error causes](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error/cause) in `test` and `it` ([#13935](https://github.com/facebook/jest/pull/13935) & [#13966](https://github.com/facebook/jest/pull/13966))
+- `[jest-reporters]` Add `summaryThreshold` option to summary reporter to allow overriding the internal threshold that is used to print the summary of all failed tests when the number of test suites surpasses it ([#13895](https://github.com/facebook/jest/pull/13895))
 - `[jest-worker]` Add `start` method to worker farms ([#13937](https://github.com/facebook/jest/pull/13937))
 
 ### Fixes

docs/Configuration.md

@@ -1334,6 +1334,29 @@ const config: Config = {
 export default config;
 ```
 
+The `summary` reporter accepts options. Since it is included in the `default` reporter you may also pass the options there.
+
+```js tab
+/** @type {import('jest').Config} */
+const config = {
+  reporters: [['default', {summaryThreshold: 10}]],
+};
+
+module.exports = config;
+```
+
+```ts tab
+import type {Config} from 'jest';
+
+const config: Config = {
+  reporters: [['default', {summaryThreshold: 10}]],
+};
+
+export default config;
+```
+
+The `summaryThreshold` option behaves in the following way, if the total number of test suites surpasses this threshold, a detailed summary of all failed tests will be printed after executing all the tests. It defaults to `20`.
+
 #### Custom Reporters
 
 :::tip

e2e/__tests__/summaryThreshold.test.ts

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import runJest from '../runJest';
+
+['default', 'summary'].forEach(reporter => {
+  describe(`${reporter} reporter`, () => {
+    test('prints failure messages when total number of test suites is over summaryThreshold', () => {
+      const {exitCode, stderr} = runJest('summary-threshold', [
+        '--config',
+        JSON.stringify({
+          reporters: [[reporter, {summaryThreshold: 2}]],
+        }),
+      ]);
+
+      expect(exitCode).toBe(1);
+      expect(stderr).toMatch(
+        /Summary of all failing tests(\n|.)*expect\(1\)\.toBe\(0\)/,
+      );
+      expect(stderr).toMatch(
+        /Summary of all failing tests(\n|.)*expect\(2\)\.toBe\(0\)/,
+      );
+    });
+  });
+});

e2e/summary-threshold/__tests__/summarySuite1.test.js

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+test('fails', () => {
+  expect(1).toBe(0);
+});

e2e/summary-threshold/__tests__/summarySuite2.test.js

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+test('fails', () => {
+  expect(2).toBe(0);
+});

e2e/summary-threshold/__tests__/summarySuite3.test.js

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+test('passes', () => {
+  expect(1).toBe(1);
+});

e2e/summary-threshold/package.json

@@ -0,0 +1,5 @@
+{
+  "jest": {
+    "testEnvironment": "node"
+  }
+}

packages/jest-core/src/TestScheduler.ts

@@ -17,6 +17,7 @@ import {
   Reporter,
   ReporterContext,
   SummaryReporter,
+  SummaryReporterOptions,
   VerboseReporter,
 } from '@jest/reporters';
 import {
@@ -336,12 +337,12 @@ class TestScheduler {
   async _setupReporters() {
     const {collectCoverage: coverage, notify, verbose} = this._globalConfig;
     const reporters = this._globalConfig.reporters || [['default', {}]];
-    let summary = false;
+    let summaryOptions: SummaryReporterOptions | null = null;
 
     for (const [reporter, options] of reporters) {
       switch (reporter) {
         case 'default':
-          summary = true;
+          summaryOptions = options;
           verbose
             ? this.addReporter(new VerboseReporter(this._globalConfig))
             : this.addReporter(new DefaultReporter(this._globalConfig));
@@ -353,7 +354,7 @@ class TestScheduler {
             );
           break;
         case 'summary':
-          summary = true;
+          summaryOptions = options;
           break;
         default:
           await this._addCustomReporter(reporter, options);
@@ -368,8 +369,8 @@ class TestScheduler {
       this.addReporter(new CoverageReporter(this._globalConfig, this._context));
     }
 
-    if (summary) {
-      this.addReporter(new SummaryReporter(this._globalConfig));
+    if (summaryOptions != null) {
+      this.addReporter(new SummaryReporter(this._globalConfig, summaryOptions));
     }
   }
 

packages/jest-reporters/src/SummaryReporter.ts

@@ -19,8 +19,6 @@ import getSnapshotSummary from './getSnapshotSummary';
 import getSummary from './getSummary';
 import type {ReporterOnStartOptions} from './types';
 
-const TEST_SUMMARY_THRESHOLD = 20;
-
 const NPM_EVENTS = new Set([
   'prepublish',
   'publish',
@@ -51,16 +49,35 @@ const NPM_EVENTS = new Set([
 const {npm_config_user_agent, npm_lifecycle_event, npm_lifecycle_script} =
   process.env;
 
+export type SummaryReporterOptions = {
+  summaryThreshold?: number;
+};
+
 export default class SummaryReporter extends BaseReporter {
   private _estimatedTime: number;
   private readonly _globalConfig: Config.GlobalConfig;
+  private readonly _summaryThreshold: number;
 
   static readonly filename = __filename;
 
-  constructor(globalConfig: Config.GlobalConfig) {
+  constructor(
+    globalConfig: Config.GlobalConfig,
+    options?: SummaryReporterOptions,
+  ) {
     super();
     this._globalConfig = globalConfig;
     this._estimatedTime = 0;
+    this._validateOptions(options);
+    this._summaryThreshold = options?.summaryThreshold ?? 20;
+  }
+
+  private _validateOptions(options?: SummaryReporterOptions) {
+    if (
+      options?.summaryThreshold &&
+      typeof options.summaryThreshold !== 'number'
+    ) {
+      throw new TypeError('The option summaryThreshold should be a number');
+    }
   }
 
   // If we write more than one character at a time it is possible that
@@ -176,7 +193,7 @@ export default class SummaryReporter extends BaseReporter {
     const runtimeErrors = aggregatedResults.numRuntimeErrorTestSuites;
     if (
       failedTests + runtimeErrors > 0 &&
-      aggregatedResults.numTotalTestSuites > TEST_SUMMARY_THRESHOLD
+      aggregatedResults.numTotalTestSuites > this._summaryThreshold
     ) {
       this.log(chalk.bold('Summary of all failing tests'));
       aggregatedResults.testResults.forEach(testResult => {

packages/jest-reporters/src/__tests__/SummaryReporter.test.js

@@ -160,3 +160,78 @@ test('snapshots all have results (after update)', () => {
   testReporter.onRunComplete(new Set(), aggregatedResults);
   expect(results.join('').replace(/\\/g, '/')).toMatchSnapshot();
 });
+
+describe('summaryThreshold option', () => {
+  const aggregatedResults = {
+    numFailedTestSuites: 1,
+    numFailedTests: 1,
+    numPassedTestSuites: 2,
+    numRuntimeErrorTestSuites: 0,
+    numTotalTestSuites: 3,
+    numTotalTests: 3,
+    snapshot: {
+      filesRemovedList: [],
+      filesUnmatched: 0,
+      total: 0,
+      uncheckedKeysByFile: [],
+      unmatched: 0,
+    },
+    startTime: 0,
+    testResults: [
+      {
+        failureMessage: 'FailureMessage1',
+        numFailingTests: 1,
+        testFilePath: 'path1',
+      },
+      {
+        failureMessage: 'FailureMessage2',
+        numFailingTests: 1,
+        testFilePath: 'path2',
+      },
+    ],
+  };
+
+  it('Should print failure messages when number of test suites is over the threshold', () => {
+    const options = {
+      summaryThreshold: aggregatedResults.numTotalTestSuites - 1,
+    };
+
+    requireReporter();
+    const testReporter = new SummaryReporter(globalConfig, options);
+    testReporter.onRunComplete(new Set(), aggregatedResults);
+    expect(results.join('').replace(/\\/g, '/')).toMatchSnapshot();
+  });
+
+  it('Should not print failure messages when number of test suites is under the threshold', () => {
+    const options = {
+      summaryThreshold: aggregatedResults.numTotalTestSuites + 1,
+    };
+
+    requireReporter();
+    const testReporter = new SummaryReporter(globalConfig, options);
+    testReporter.onRunComplete(new Set(), aggregatedResults);
+    expect(results.join('').replace(/\\/g, '/')).toMatchSnapshot();
+  });
+
+  it('Should not print failure messages when number of test suites is equal to the threshold', () => {
+    const options = {
+      summaryThreshold: aggregatedResults.numTotalTestSuites,
+    };
+
+    requireReporter();
+    const testReporter = new SummaryReporter(globalConfig, options);
+    testReporter.onRunComplete(new Set(), aggregatedResults);
+    expect(results.join('').replace(/\\/g, '/')).toMatchSnapshot();
+  });
+
+  it('Should throw error if threshold is not a number', () => {
+    const options = {
+      summaryThreshold: 'not a number',
+    };
+
+    requireReporter();
+    expect(() => new SummaryReporter(globalConfig, options)).toThrow(
+      'The option summaryThreshold should be a number',
+    );
+  });
+});

packages/jest-reporters/src/__tests__/__snapshots__/SummaryReporter.test.js.snap

@@ -59,3 +59,36 @@ exports[`snapshots needs update with yarn test 1`] = `
 <dim>Ran all test suites</intensity><dim>.</intensity>
 "
 `;
+
+exports[`summaryThreshold option Should not print failure messages when number of test suites is equal to the threshold 1`] = `
+"<bold>Test Suites: </intensity><bold><red>1 failed</color></intensity>, <bold><green>2 passed</color></intensity>, 3 total
+<bold>Tests:       </intensity><bold><red>1 failed</color></intensity>, 3 total
+<bold>Snapshots:   </intensity>0 total
+<bold>Time:</intensity>        0.01 s
+<dim>Ran all test suites</intensity><dim>.</intensity>
+"
+`;
+
+exports[`summaryThreshold option Should not print failure messages when number of test suites is under the threshold 1`] = `
+"<bold>Test Suites: </intensity><bold><red>1 failed</color></intensity>, <bold><green>2 passed</color></intensity>, 3 total
+<bold>Tests:       </intensity><bold><red>1 failed</color></intensity>, 3 total
+<bold>Snapshots:   </intensity>0 total
+<bold>Time:</intensity>        0.01 s
+<dim>Ran all test suites</intensity><dim>.</intensity>
+"
+`;
+
+exports[`summaryThreshold option Should print failure messages when number of test suites is over the threshold 1`] = `
+"<bold>Summary of all failing tests</intensity>
+</><inverse><bold><red> FAIL </color></intensity></inverse></> <dim>../</intensity><bold>path1</intensity>
+FailureMessage1
+</><inverse><bold><red> FAIL </color></intensity></inverse></> <dim>../</intensity><bold>path2</intensity>
+FailureMessage2
+
+<bold>Test Suites: </intensity><bold><red>1 failed</color></intensity>, <bold><green>2 passed</color></intensity>, 3 total
+<bold>Tests:       </intensity><bold><red>1 failed</color></intensity>, 3 total
+<bold>Snapshots:   </intensity>0 total
+<bold>Time:</intensity>        0.01 s
+<dim>Ran all test suites</intensity><dim>.</intensity>
+"
+`;

packages/jest-reporters/src/index.ts

@@ -30,6 +30,7 @@ export {default as GitHubActionsReporter} from './GitHubActionsReporter';
 export {default as NotifyReporter} from './NotifyReporter';
 export {default as SummaryReporter} from './SummaryReporter';
 export {default as VerboseReporter} from './VerboseReporter';
+export type {SummaryReporterOptions} from './SummaryReporter';
 export type {
   Reporter,
   ReporterOnStartOptions,