Test retries (#6498)

* Enable configurable retries for failed test cases

* Update tests

* Remove testRetries CLI and config option. Add as jest.retryTimes with reporter integration.

* Add jest.retryTimes to CHANGELOG.md and JestObjectAPI.md

* Prettier fix on snapshot test

* Update runJest to support jest-circus environment override

* Update docs and use skipSuiteOnJasmine

* Update retryTimes tests

* Remove useJestCircus boolean on runTest

* Remove outdated comment. Revert runJest environment override logic.

* Removed outdated comment from test_retries test

* Update snapshot tests

* Update Jest Object docs for retryTimes. Use symbol for global lookup.

CHANGELOG.md

@@ -10,6 +10,7 @@
 
 - `[jest-each]` Add support for keyPaths in test titles ([#6457](https://github.com/facebook/jest/pull/6457))
 - `[jest-cli]` Add `jest --init` option that generates a basic configuration file with a short description for each option ([#6442](https://github.com/facebook/jest/pull/6442))
+- `[jest.retryTimes]` Add `jest.retryTimes()` option that allows failed tests to be retried n-times when using jest-circus. ([#6498](https://github.com/facebook/jest/pull/6498))
 
 ### Fixes
 

docs/JestObjectAPI.md

@@ -21,6 +21,7 @@ The `jest` object is automatically in scope within every test file. The methods
 - [`jest.resetAllMocks()`](#jestresetallmocks)
 - [`jest.restoreAllMocks()`](#jestrestoreallmocks)
 - [`jest.resetModules()`](#jestresetmodules)
+- [`jest.retryTimes()`](#jestretrytimes)
 - [`jest.runAllTicks()`](#jestrunallticks)
 - [`jest.runAllTimers()`](#jestrunalltimers)
 - [`jest.advanceTimersByTime(msToRun)`](#jestadvancetimersbytimemstorun)
@@ -312,6 +313,37 @@ test('works too', () => {
 
 Returns the `jest` object for chaining.
 
+### `jest.retryTimes()`
+
+Runs failed tests n-times until they pass or until the max number of retries are exhausted. This only works with jest-circus!
+
+Example in a test:
+
+```js
+jest.retryTimes(3);
+test('will fail', () => {
+  expect(true).toBe(false);
+});
+```
+
+To run with jest circus:
+
+Install jest-circus
+
+```
+yarn add --dev jest-circus
+```
+
+Then set as the testRunner in your jest config:
+
+```js
+module.exports = {
+  testRunner: 'jest-circus/runner',
+};
+```
+
+Returns the `jest` object for chaining.
+
 ### `jest.runAllTicks()`
 
 Exhausts the **micro**-task queue (usually interfaced in node via `process.nextTick`).

e2e/__tests__/test_retries.test.js

@@ -0,0 +1,92 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow
+ */
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const runJest = require('../runJest');
+
+const ConditionalTest = require('../../scripts/ConditionalTest');
+
+ConditionalTest.skipSuiteOnJasmine();
+
+describe('Test Retries', () => {
+  const outputFileName = 'retries.result.json';
+  const outputFilePath = path.join(
+    process.cwd(),
+    'e2e/test-retries/',
+    outputFileName,
+  );
+
+  afterAll(() => {
+    fs.unlinkSync(outputFilePath);
+  });
+
+  it('retries failed tests if configured', () => {
+    let jsonResult;
+
+    const reporterConfig = {
+      reporters: [
+        ['<rootDir>/reporters/RetryReporter.js', {output: outputFilePath}],
+      ],
+    };
+
+    runJest('test-retries', [
+      '--config',
+      JSON.stringify(reporterConfig),
+      'retry.test.js',
+    ]);
+
+    const testOutput = fs.readFileSync(outputFilePath, 'utf8');
+
+    try {
+      jsonResult = JSON.parse(testOutput);
+    } catch (err) {
+      throw new Error(
+        `Can't parse the JSON result from ${outputFileName}, ${err.toString()}`,
+      );
+    }
+
+    expect(jsonResult.numPassedTests).toBe(0);
+    expect(jsonResult.numFailedTests).toBe(1);
+    expect(jsonResult.numPendingTests).toBe(0);
+    expect(jsonResult.testResults[0].testResults[0].invocations).toBe(4);
+  });
+
+  it('does not retry by default', () => {
+    let jsonResult;
+
+    const reporterConfig = {
+      reporters: [
+        ['<rootDir>/reporters/RetryReporter.js', {output: outputFilePath}],
+      ],
+    };
+
+    runJest('test-retries', [
+      '--config',
+      JSON.stringify(reporterConfig),
+      'control.test.js',
+    ]);
+
+    const testOutput = fs.readFileSync(outputFilePath, 'utf8');
+
+    try {
+      jsonResult = JSON.parse(testOutput);
+    } catch (err) {
+      throw new Error(
+        `Can't parse the JSON result from ${outputFileName}, ${err.toString()}`,
+      );
+    }
+
+    expect(jsonResult.numPassedTests).toBe(0);
+    expect(jsonResult.numFailedTests).toBe(1);
+    expect(jsonResult.numPendingTests).toBe(0);
+    expect(jsonResult.testResults[0].testResults[0].invocations).toBe(1);
+  });
+});

e2e/runJest.js

@@ -55,6 +55,7 @@ function runJest(
         NODE_PATH: options.nodePath,
       })
     : process.env;
+
   const result = spawnSync(JEST_PATH, args || [], {
     cwd: dir,
     env,

e2e/test-retries/__tests__/control.test.js

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+it('retryTimes not set', () => {
+  expect(true).toBeFalsy();
+});

e2e/test-retries/__tests__/retry.test.js

@@ -0,0 +1,13 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+jest.retryTimes(3);
+
+it('retryTimes set', () => {
+  expect(true).toBeFalsy();
+});

e2e/test-retries/package.json

@@ -0,0 +1,5 @@
+{
+  "jest": {
+    "testEnvironment": "node"
+  }
+}

e2e/test-retries/reporters/RetryReporter.js

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+const fs = require('fs');
+
+/**
+ * RetryReporter
+ * Reporter for testing output of onRunComplete
+ */
+class RetryReporter {
+  constructor(globalConfig, options) {
+    this._options = options;
+  }
+
+  onRunComplete(contexts, results) {
+    if (this._options.output) {
+      fs.writeFileSync(this._options.output, JSON.stringify(results, null, 2), {
+        encoding: 'utf8',
+      });
+    }
+  }
+}
+
+module.exports = RetryReporter;

packages/jest-circus/src/event_handler.js

@@ -114,6 +114,7 @@ const handler: EventHandler = (event, state): void => {
     case 'test_start': {
       state.currentlyRunningTest = event.test;
       event.test.startedAt = Date.now();
+      event.test.invocations += 1;
       break;
     }
     case 'test_fn_failure': {

packages/jest-circus/src/legacy_code_todo_rewrite/jest_adapter_init.js

@@ -137,6 +137,7 @@ export const runAndTransformResultsToJestFormat = async ({
       duration: testResult.duration,
       failureMessages: testResult.errors,
       fullName: ancestorTitles.concat(title).join(' '),
+      invocations: testResult.invocations,
       location: testResult.location,
       numPassingAsserts: 0,
       status,

packages/jest-circus/src/run.js

@@ -46,8 +46,27 @@ const _runTestsForDescribeBlock = async (describeBlock: DescribeBlock) => {
   for (const hook of beforeAll) {
     await _callHook({describeBlock, hook});
   }
+
+  // Tests that fail and are retried we run after other tests
+  const retryTimes = parseInt(global[Symbol.for('RETRY_TIMES')], 10) || 0;
+  const deferredRetryTests = [];
+
   for (const test of describeBlock.tests) {
     await _runTest(test);
+
+    if (retryTimes > 0 && test.errors.length > 0) {
+      deferredRetryTests.push(test);
+    }
+  }
+
+  // Re-run failed tests n-times if configured
+  for (const test of deferredRetryTests) {
+    let numRetriesAvailable = retryTimes;
+
+    while (numRetriesAvailable > 0 && test.errors.length > 0) {
+      await _runTest(test);
+      numRetriesAvailable--;
+    }
   }
 
   for (const child of describeBlock.children) {

packages/jest-circus/src/utils.js

@@ -80,6 +80,7 @@ export const makeTest = (
     duration: null,
     errors: [],
     fn,
+    invocations: 0,
     mode: _mode,
     name: convertDescriptorToString(name),
     parent,
@@ -276,6 +277,7 @@ const makeTestResults = (describeBlock: DescribeBlock, config): TestResults => {
     testResults.push({
       duration: test.duration,
       errors: test.errors.map(_formatError),
+      invocations: test.invocations,
       location,
       status,
       testPath,

packages/jest-cli/src/cli/args.js

@@ -578,7 +578,7 @@ export const options = {
     description:
       'Allows the use of a custom results processor. ' +
       'This processor must be a node module that exports ' +
-      'a function expecting as the first argument the result object',
+      'a function expecting as the first argument the result object.',
     type: 'string',
   },
   testRunner: {

packages/jest-runtime/src/index.js

@@ -830,6 +830,11 @@ class Runtime {
       return jestObject;
     };
 
+    const retryTimes = (numTestRetries: number) => {
+      this._environment.global[Symbol.for('RETRY_TIMES')] = numTestRetries;
+      return jestObject;
+    };
+
     const jestObject = {
       addMatchers: (matchers: Object) =>
         this._environment.global.jasmine.addMatchers(matchers),
@@ -855,6 +860,7 @@ class Runtime {
       resetModuleRegistry: resetModules,
       resetModules,
       restoreAllMocks,
+      retryTimes,
       runAllImmediates: () => this._environment.fakeTimers.runAllImmediates(),
       runAllTicks: () => this._environment.fakeTimers.runAllTicks(),
       runAllTimers: () => this._environment.fakeTimers.runAllTimers(),

types/Circus.js

@@ -194,6 +194,7 @@ export type TestEntry = {|
   asyncError: Exception, // Used if the test failure contains no usable stack trace
   errors: TestError,
   fn: ?TestFn,
+  invocations: number,
   mode: TestMode,
   name: TestName,
   parent: DescribeBlock,

types/Jest.js

@@ -32,6 +32,7 @@ export type Jest = {|
   resetModuleRegistry(): Jest,
   resetModules(): Jest,
   restoreAllMocks(): Jest,
+  retryTimes(numRetries: number): Jest,
   runAllImmediates(): void,
   runAllTicks(): void,
   runAllTimers(): void,