jest-circus runs children in shuffled order (#12922)

CHANGELOG.md

@@ -3,6 +3,7 @@
 ### Features
 
 - `[jest-changed-files]` Support Sapling ([#13941](https://github.com/facebook/jest/pull/13941))
+- `[jest-circus, @jest/cli, jest-config]` Add feature to randomize order of tests via CLI flag or through the config file([#12922](https://github.com/facebook/jest/pull/12922))
 - `[jest-cli, jest-config, @jest/core, jest-haste-map, @jest/reporters, jest-runner, jest-runtime, @jest/types]` Add `workerThreads` configuration option to allow using [worker threads](https://nodejs.org/dist/latest/docs/api/worker_threads.html) for parallelization ([#13939](https://github.com/facebook/jest/pull/13939))
 - `[jest-config]` Add `openHandlesTimeout` option to configure possible open handles warning. ([#13875](https://github.com/facebook/jest/pull/13875))
 - `[@jest/create-cache-key-function]` Allow passing `length` argument to `createCacheKey()` function and set its default value to `16` on Windows ([#13827](https://github.com/facebook/jest/pull/13827))

docs/CLI.md

@@ -332,6 +332,22 @@ If configuration files are found in the specified paths, _all_ projects specifie
 
 :::
 
+### `--randomize`
+
+Shuffle the order of the tests within a file. The shuffling is based on the seed. See [`--seed=<num>`](#--seednum) for more info.
+
+Seed value is displayed when this option is set. Equivalent to setting the CLI option [`--showSeed`](#--showseed).
+
+```bash
+jest --randomize --seed 1234
+```
+
+:::note
+
+This option is only supported using the default `jest-circus` test runner.
+
+:::
+
 ### `--reporters`
 
 Run tests with specified reporters. [Reporter options](configuration#reporters-arraymodulename--modulename-options) are not available via CLI. Example with multiple reporters:

docs/Configuration.md

@@ -1222,6 +1222,12 @@ With the `projects` option enabled, Jest will copy the root-level configuration
 
 :::
 
+### `randomize` \[boolean]
+
+Default: `false`
+
+The equivalent of the [`--randomize`](CLI.md#--randomize) flag to randomize the order of the tests in a file.
+
 ### `reporters` \[array<moduleName | \[moduleName, options]>]
 
 Default: `undefined`

e2e/__tests__/__snapshots__/randomize.test.ts.snap

@@ -0,0 +1,89 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`works with each 1`] = `
+"  ✓ test1
+  ✓ test2
+  ✓ test3
+  describe2
+    ✓ test4
+    ✓ test6
+    ✓ test5
+  describe1
+    ✓ test4
+    ✓ test6
+    ✓ test5
+  describe3
+    ✓ test11
+    ✓ test12
+    ✓ test10
+    describe4
+      ✓ test14
+      ✓ test15
+      ✓ test13"
+`;
+
+exports[`works with hooks 1`] = `
+"  ✓ test1
+  ✓ test2
+  ✓ test3
+  describe2
+    ✓ test7
+    ✓ test9
+    ✓ test8
+  describe1
+    ✓ test4
+    ✓ test6
+    ✓ test5
+  describe3
+    ✓ test11
+    ✓ test12
+    ✓ test10
+    describe4
+      ✓ test14
+      ✓ test15
+      ✓ test13"
+`;
+
+exports[`works with passing tests 1`] = `
+"  ✓ test1
+  ✓ test2
+  ✓ test3
+  describe2
+    ✓ test7
+    ✓ test9
+    ✓ test8
+  describe1
+    ✓ test4
+    ✓ test6
+    ✓ test5
+  describe3
+    ✓ test11
+    ✓ test12
+    ✓ test10
+    describe4
+      ✓ test14
+      ✓ test15
+      ✓ test13"
+`;
+
+exports[`works with snapshots 1`] = `
+"  ✓ test1
+  ✓ test2
+  ✓ test3
+  describe2
+    ✓ test4
+    ✓ test6
+    ✓ test5
+  describe1
+    ✓ test4
+    ✓ test6
+    ✓ test5
+  describe3
+    ✓ test11
+    ✓ test12
+    ✓ test10
+    describe4
+      ✓ test14
+      ✓ test15
+      ✓ test13"
+`;

e2e/__tests__/randomize.test.ts

@@ -0,0 +1,85 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import * as path from 'path';
+import {skipSuiteOnJasmine} from '@jest/test-utils';
+import {extractSummary} from '../Utils';
+import runJest, {RunJestResult} from '../runJest';
+
+skipSuiteOnJasmine();
+
+const dir = path.resolve(__dirname, '../randomize');
+
+const trimFirstLine = (str: string): string =>
+  str.split('\n').slice(1).join('\n');
+
+function runJestTwice(
+  dir: string,
+  args: Array<string>,
+): [RunJestResult, RunJestResult] {
+  return [
+    runJest(dir, [...args, '--randomize']),
+    runJest(dir, [...args, '--config', 'different-config.json']),
+  ];
+}
+
+test('works with passing tests', () => {
+  const [result1, result2] = runJestTwice(dir, [
+    'success.test.js',
+    '--seed',
+    '123',
+  ]);
+
+  const rest1 = trimFirstLine(extractSummary(result1.stderr).rest);
+  const rest2 = trimFirstLine(extractSummary(result2.stderr).rest);
+
+  expect(rest1).toEqual(rest2);
+  expect(rest1).toMatchSnapshot();
+});
+
+test('works with each', () => {
+  const [result1, result2] = runJestTwice(dir, [
+    'each.test.js',
+    '--seed',
+    '123',
+  ]);
+
+  const rest1 = trimFirstLine(extractSummary(result1.stderr).rest);
+  const rest2 = trimFirstLine(extractSummary(result2.stderr).rest);
+
+  expect(rest1).toEqual(rest2);
+  expect(rest1).toMatchSnapshot();
+});
+
+test('works with hooks', () => {
+  const [result1, result2] = runJestTwice(dir, [
+    'hooks.test.js',
+    '--seed',
+    '123',
+  ]);
+
+  // Change in formatting could change this one
+  const rest1 = trimFirstLine(extractSummary(result1.stderr).rest);
+  const rest2 = trimFirstLine(extractSummary(result2.stderr).rest);
+
+  expect(rest1).toEqual(rest2);
+  expect(rest1).toMatchSnapshot();
+});
+
+test('works with snapshots', () => {
+  const [result1, result2] = runJestTwice(dir, [
+    'snapshots.test.js',
+    '--seed',
+    '123',
+  ]);
+
+  const rest1 = trimFirstLine(extractSummary(result1.stderr).rest);
+  const rest2 = trimFirstLine(extractSummary(result2.stderr).rest);
+
+  expect(rest1).toEqual(rest2);
+  expect(rest1).toMatchSnapshot();
+});

e2e/__tests__/showSeed.test.ts

@@ -14,39 +14,41 @@ const dir = path.resolve(__dirname, '../jest-object');
 const randomSeedValueRegExp = /Seed:\s+<<REPLACED>>/;
 const seedValueRegExp = /Seed:\s+1234/;
 
-test('--showSeed changes report to output seed', () => {
-  const {stderr} = runJest(dir, ['--showSeed', '--no-cache']);
+describe.each(['showSeed', 'randomize'])('Option %s', option => {
+  test(`--${option} changes report to output seed`, () => {
+    const {stderr} = runJest(dir, [`--${option}`, '--no-cache']);
 
-  const {summary} = extractSummary(stderr);
+    const {summary} = extractSummary(stderr);
 
-  expect(replaceSeed(summary)).toMatch(randomSeedValueRegExp);
-});
+    expect(replaceSeed(summary)).toMatch(randomSeedValueRegExp);
+  });
 
-test('if --showSeed is not present the report will not show the seed', () => {
-  const {stderr} = runJest(dir, ['--seed', '1234']);
+  test(`if --${option} is not present the report will not show the seed`, () => {
+    const {stderr} = runJest(dir, ['--seed', '1234']);
 
-  const {summary} = extractSummary(stderr);
+    const {summary} = extractSummary(stderr);
 
-  expect(replaceSeed(summary)).not.toMatch(randomSeedValueRegExp);
-});
+    expect(replaceSeed(summary)).not.toMatch(randomSeedValueRegExp);
+  });
 
-test('if showSeed is present in the config the report will show the seed', () => {
-  const {stderr} = runJest(dir, [
-    '--seed',
-    '1234',
-    '--config',
-    'different-config.json',
-  ]);
+  test(`if ${option} is present in the config the report will show the seed`, () => {
+    const {stderr} = runJest(dir, [
+      '--seed',
+      '1234',
+      '--config',
+      `${option}-config.json`,
+    ]);
 
-  const {summary} = extractSummary(stderr);
+    const {summary} = extractSummary(stderr);
 
-  expect(summary).toMatch(seedValueRegExp);
-});
+    expect(summary).toMatch(seedValueRegExp);
+  });
 
-test('--seed --showSeed will show the seed in the report', () => {
-  const {stderr} = runJest(dir, ['--showSeed', '--seed', '1234']);
+  test(`--seed --${option} will show the seed in the report`, () => {
+    const {stderr} = runJest(dir, [`--${option}`, '--seed', '1234']);
 
-  const {summary} = extractSummary(stderr);
+    const {summary} = extractSummary(stderr);
 
-  expect(summary).toMatch(seedValueRegExp);
+    expect(summary).toMatch(seedValueRegExp);
+  });
 });

e2e/jest-object/randomize-config.json

@@ -0,0 +1,4 @@
+{
+  "displayName": "Config from randomize-config.json file",
+  "randomize": true
+}

e2e/jest-object/showSeed-config.json

@@ -0,0 +1,4 @@
+{
+  "displayName": "Config from showSeed-config.json file",
+  "showSeed": true
+}

e2e/randomize/__tests__/__snapshots__/snapshots.test.js.snap

@@ -0,0 +1,31 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`describe1 test4 1`] = `4`;
+
+exports[`describe1 test5 1`] = `5`;
+
+exports[`describe1 test6 1`] = `6`;
+
+exports[`describe2 test4 1`] = `4`;
+
+exports[`describe2 test5 1`] = `5`;
+
+exports[`describe2 test6 1`] = `6`;
+
+exports[`describe3 describe4 test13 1`] = `13`;
+
+exports[`describe3 describe4 test14 1`] = `14`;
+
+exports[`describe3 describe4 test15 1`] = `15`;
+
+exports[`describe3 test10 1`] = `10`;
+
+exports[`describe3 test11 1`] = `11`;
+
+exports[`describe3 test12 1`] = `12`;
+
+exports[`test1 1`] = `1`;
+
+exports[`test2 1`] = `2`;
+
+exports[`test3 1`] = `3`;

e2e/randomize/__tests__/each.test.js

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+it.each([1, 2, 3])('test%d', () => {
+  expect(true).toBe(true);
+});
+
+describe.each([1, 2])('describe%d', () => {
+  it.each([4, 5, 6])('test%d', () => {
+    expect(true).toBe(true);
+  });
+});
+
+describe('describe3', () => {
+  it.each([10, 11, 12])('test%d', () => {
+    expect(true).toBe(true);
+  });
+
+  describe('describe4', () => {
+    it.each([13, 14, 15])('test%d', () => {
+      expect(true).toBe(true);
+    });
+  });
+});