From d25d0a4e31f200bd4df025dbea21e7129fa3ad4f Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Wed, 9 Oct 2024 17:59:32 +0900 Subject: [PATCH 1/8] feat(runner)!: support `TestOptions.shuffle` and inherit parent suite shuffle --- packages/runner/src/run.ts | 2 +- packages/runner/src/suite.ts | 13 ++++++------- packages/runner/src/types/tasks.ts | 4 ++++ test/core/test/random.test.ts | 4 +--- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 399931736cd2..50c26a364f15 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -409,7 +409,7 @@ export async function runSuite(suite: Suite, runner: VitestRunner): Promise group.type === 'suite', diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index c935c3301341..392c06395908 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -206,9 +206,8 @@ export function getRunner(): VitestRunner { } function createDefaultSuite(runner: VitestRunner) { - const config = runner.config.sequence - const api = config.shuffle ? suite.shuffle : suite - return api('', { concurrent: config.concurrent }, () => {}) + const { concurrent, shuffle } = runner.config.sequence + return suite('', { concurrent, shuffle }, () => {}) } export function clearCollectorContext( @@ -331,9 +330,7 @@ function createSuiteCollector( ) { task.concurrent = true } - if (shuffle) { - task.shuffle = true - } + task.shuffle = suiteOptions?.shuffle const context = createTestContext(task, runner) // create test context @@ -533,11 +530,13 @@ function createSuite() { options.concurrent = isConcurrent && !isSequential options.sequential = isSequential && !isConcurrent + options.shuffle = this.shuffle ?? options.shuffle + return createSuiteCollector( formatName(name), factory, mode, - this.shuffle, + options.shuffle, this.each, options, ) diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index 5cf376b69a06..fe424efcb7fe 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -388,6 +388,10 @@ export interface TestOptions { * Tests inherit `sequential` from `describe()` and nested `describe()` will inherit from parent's `sequential`. */ sequential?: boolean + /** + * Whether the tasks of the suite run in a random order. + */ + shuffle?: boolean /** * Whether the test should be skipped. */ diff --git a/test/core/test/random.test.ts b/test/core/test/random.test.ts index b4f32f44cece..83cba6901320 100644 --- a/test/core/test/random.test.ts +++ b/test/core/test/random.test.ts @@ -4,9 +4,7 @@ import { afterAll, describe, expect, test } from 'vitest' const numbers: number[] = [] describe.shuffle('random tests', () => { - describe('inside', () => { - // shuffle is not inherited from parent - + describe('inside', { shuffle: false }, () => { test('inside 1', () => { numbers.push(1) }) From 0f40ee39c3029fa222947c4886d2beee6f1ebd55 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Wed, 9 Oct 2024 18:30:57 +0900 Subject: [PATCH 2/8] test: shuffle and unshuffle --- test/core/test/random.test.ts | 37 ++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/test/core/test/random.test.ts b/test/core/test/random.test.ts index 83cba6901320..90e46b251c9b 100644 --- a/test/core/test/random.test.ts +++ b/test/core/test/random.test.ts @@ -1,16 +1,31 @@ -import { afterAll, describe, expect, test } from 'vitest' +import { describe, expect, test } from 'vitest' // tests use seed of 101, so they have deterministic random order const numbers: number[] = [] describe.shuffle('random tests', () => { - describe('inside', { shuffle: false }, () => { + describe('suite unshuffle', { shuffle: false }, () => { test('inside 1', () => { numbers.push(1) }) + test('inside 1.5', () => { + numbers.push(1.5) + }) test('inside 2', () => { numbers.push(2) }) + + describe('suite shuffle', { shuffle: true }, () => { + test('inside 2.1', () => { + numbers.push(2.1) + }) + test('inside 2.2', () => { + numbers.push(2.2) + }) + test('inside 2.3', () => { + numbers.push(2.3) + }) + }) }) test('test 1', () => { @@ -22,8 +37,20 @@ describe.shuffle('random tests', () => { test('test 3', () => { numbers.push(5) }) +}) - afterAll(() => { - expect(numbers).toStrictEqual([4, 5, 3, 1, 2]) - }) +test('assert', () => { + expect(numbers).toMatchInlineSnapshot(` + [ + 4, + 5, + 3, + 1, + 1.5, + 2, + 2.2, + 2.3, + 2.1, + ] + `) }) From f30543c87e69c256411d16afe0fc81dba7e334d4 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Wed, 9 Oct 2024 19:29:08 +0900 Subject: [PATCH 3/8] test: test `config.sequence.shuffle` --- packages/runner/src/collect.ts | 2 ++ packages/runner/src/suite.ts | 1 + test/config/fixtures/shuffle/basic.test.ts | 27 +++++++++++++++++++ test/config/fixtures/shuffle/vitest.config.ts | 10 +++++++ test/config/test/shuffle-options.test.ts | 15 +++++++++++ 5 files changed, 55 insertions(+) create mode 100644 test/config/fixtures/shuffle/basic.test.ts create mode 100644 test/config/fixtures/shuffle/vitest.config.ts diff --git a/packages/runner/src/collect.ts b/packages/runner/src/collect.ts index 57e0a65586ba..c6a7eaa9318f 100644 --- a/packages/runner/src/collect.ts +++ b/packages/runner/src/collect.ts @@ -29,6 +29,8 @@ export async function collectTests( for (const filepath of paths) { const file = createFileTask(filepath, config.root, config.name, runner.pool) + // TODO? + file.shuffle = config.sequence.shuffle runner.onCollectStart?.(file) diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index 392c06395908..d18030a99766 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -206,6 +206,7 @@ export function getRunner(): VitestRunner { } function createDefaultSuite(runner: VitestRunner) { + // TODO? const { concurrent, shuffle } = runner.config.sequence return suite('', { concurrent, shuffle }, () => {}) } diff --git a/test/config/fixtures/shuffle/basic.test.ts b/test/config/fixtures/shuffle/basic.test.ts new file mode 100644 index 000000000000..1ca2d761d64f --- /dev/null +++ b/test/config/fixtures/shuffle/basic.test.ts @@ -0,0 +1,27 @@ +import { afterAll, describe, expect, test } from 'vitest' + +const numbers: number[] = [] + +test.for([1, 2, 3, 4, 5])('test %s', (v) => { + numbers.push(10 + v) +}), + +describe("inherit shuffle", () => { + test.for([1, 2, 3, 4, 5])('test %s', (v) => { + numbers.push(20 + v) + }) +}) + +describe('unshuffle', { shuffle: false }, () => { + test.for([1, 2, 3, 4, 5])('test %s', (v) => { + numbers.push(30 + v) + }) +}) + +afterAll(() => { + expect(numbers).toEqual([ + 11, 14, 13, 15, 12, + 31, 32, 33, 34, 35, + 21, 24, 23, 25, 22 + ]) +}) diff --git a/test/config/fixtures/shuffle/vitest.config.ts b/test/config/fixtures/shuffle/vitest.config.ts new file mode 100644 index 000000000000..4f7af402e973 --- /dev/null +++ b/test/config/fixtures/shuffle/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + sequence: { + seed: 101, + shuffle: true, + } + } +}) diff --git a/test/config/test/shuffle-options.test.ts b/test/config/test/shuffle-options.test.ts index 63e5a12c7b10..77067fb7dde7 100644 --- a/test/config/test/shuffle-options.test.ts +++ b/test/config/test/shuffle-options.test.ts @@ -47,3 +47,18 @@ test.each([ const { ctx } = await run({ shuffle, sequencer: CustomSequencer }) expect(ctx?.config.sequence.sequencer.name).toBe('CustomSequencer') }) + +test('shuffle', async () => { + const { stderr, ctx } = await runVitest({ + root: './fixtures/shuffle', + }) + expect(stderr).toBe('') + expect(ctx?.state.getFiles().map(f => [f.name, f.result?.state])).toMatchInlineSnapshot(` + [ + [ + "basic.test.ts", + "pass", + ], + ] + `); +}) From a8299df2140d2a1a62bec704514fd7e2163c9b18 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 10 Oct 2024 08:49:43 +0900 Subject: [PATCH 4/8] chore: cleanup --- packages/runner/src/collect.ts | 1 - packages/runner/src/suite.ts | 1 - test/config/test/shuffle-options.test.ts | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/runner/src/collect.ts b/packages/runner/src/collect.ts index c6a7eaa9318f..985fd90061af 100644 --- a/packages/runner/src/collect.ts +++ b/packages/runner/src/collect.ts @@ -29,7 +29,6 @@ export async function collectTests( for (const filepath of paths) { const file = createFileTask(filepath, config.root, config.name, runner.pool) - // TODO? file.shuffle = config.sequence.shuffle runner.onCollectStart?.(file) diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index d18030a99766..392c06395908 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -206,7 +206,6 @@ export function getRunner(): VitestRunner { } function createDefaultSuite(runner: VitestRunner) { - // TODO? const { concurrent, shuffle } = runner.config.sequence return suite('', { concurrent, shuffle }, () => {}) } diff --git a/test/config/test/shuffle-options.test.ts b/test/config/test/shuffle-options.test.ts index 77067fb7dde7..f9659f04819e 100644 --- a/test/config/test/shuffle-options.test.ts +++ b/test/config/test/shuffle-options.test.ts @@ -60,5 +60,5 @@ test('shuffle', async () => { "pass", ], ] - `); + `) }) From 96c00f22ac4a9842c54f5c4a0498717a4059a536 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 10 Oct 2024 13:07:56 +0900 Subject: [PATCH 5/8] refactor: inherit --- packages/runner/src/suite.ts | 12 ++++++------ test/config/fixtures/shuffle/basic.test.ts | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index 392c06395908..e8084cfe779c 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -206,8 +206,8 @@ export function getRunner(): VitestRunner { } function createDefaultSuite(runner: VitestRunner) { - const { concurrent, shuffle } = runner.config.sequence - return suite('', { concurrent, shuffle }, () => {}) + const config = runner.config.sequence + return suite('', { concurrent: config.concurrent }, () => {}) } export function clearCollectorContext( @@ -518,8 +518,10 @@ function createSuite() { ) // inherit options from current suite - if (currentSuite?.options) { - options = { ...currentSuite.options, ...options } + options = { + ...currentSuite?.options, + ...options, + shuffle: this.shuffle ?? options.shuffle ?? currentSuite?.options?.shuffle ?? runner?.config.sequence.shuffle, } // inherit concurrent / sequential from suite @@ -530,8 +532,6 @@ function createSuite() { options.concurrent = isConcurrent && !isSequential options.sequential = isSequential && !isConcurrent - options.shuffle = this.shuffle ?? options.shuffle - return createSuiteCollector( formatName(name), factory, diff --git a/test/config/fixtures/shuffle/basic.test.ts b/test/config/fixtures/shuffle/basic.test.ts index 1ca2d761d64f..673479ea80cf 100644 --- a/test/config/fixtures/shuffle/basic.test.ts +++ b/test/config/fixtures/shuffle/basic.test.ts @@ -4,7 +4,7 @@ const numbers: number[] = [] test.for([1, 2, 3, 4, 5])('test %s', (v) => { numbers.push(10 + v) -}), +}) describe("inherit shuffle", () => { test.for([1, 2, 3, 4, 5])('test %s', (v) => { From d92b3bb537c520a21265b797e2b8fb3299c355e6 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 10 Oct 2024 13:10:44 +0900 Subject: [PATCH 6/8] refactor: more --- packages/runner/src/suite.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index e8084cfe779c..6b982ec2d1a2 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -291,7 +291,6 @@ function createSuiteCollector( name: string, factory: SuiteFactory = () => {}, mode: RunMode, - shuffle?: boolean, each?: boolean, suiteOptions?: TestOptions, ) { @@ -422,7 +421,7 @@ function createSuiteCollector( mode, each, file: undefined!, - shuffle, + shuffle: suiteOptions?.shuffle, tasks: [], meta: Object.create(null), concurrent: suiteOptions?.concurrent, @@ -536,7 +535,6 @@ function createSuite() { formatName(name), factory, mode, - options.shuffle, this.each, options, ) From 7befc12fc85869442a1142dc0595852dac09f142 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 10 Oct 2024 13:19:37 +0900 Subject: [PATCH 7/8] fix: remove shuffle from TestCollectorOptions --- packages/runner/src/types/tasks.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index fe424efcb7fe..ef66b4b782b8 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -275,16 +275,16 @@ interface EachFunctionReturn { ( name: string | Function, fn: (...args: T) => Awaitable, - options: TestOptions + options: TestCollectorOptions ): void ( name: string | Function, fn: (...args: T) => Awaitable, - options?: number | TestOptions + options?: number | TestCollectorOptions ): void ( name: string | Function, - options: TestOptions, + options: TestCollectorOptions, fn: (...args: T) => Awaitable ): void } @@ -305,7 +305,7 @@ interface TestForFunctionReturn { ): void ( name: string | Function, - options: TestOptions, + options: TestCollectorOptions, fn: (args: Arg, context: Context) => Awaitable ): void } @@ -336,16 +336,16 @@ interface TestCollectorCallable { ( name: string | Function, fn: TestFunction, - options: TestOptions + options: TestCollectorOptions ): void ( name: string | Function, fn?: TestFunction, - options?: number | TestOptions + options?: number | TestCollectorOptions ): void ( name: string | Function, - options?: TestOptions, + options?: TestCollectorOptions, fn?: TestFunction ): void } @@ -359,6 +359,8 @@ type ChainableTestAPI = ChainableFunction< } > +type TestCollectorOptions = Omit + export interface TestOptions { /** * Test timeout. From e9acfc5f567b07eb538630252f1d3a8f00383369 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 10 Oct 2024 13:29:29 +0900 Subject: [PATCH 8/8] docs: udpate `describe.shuffle` --- docs/api/index.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/api/index.md b/docs/api/index.md index 3a9b89debaa3..6546725ea6c6 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -797,10 +797,23 @@ Vitest provides a way to run all tests in random order via CLI flag [`--sequence ```ts import { describe, test } from 'vitest' +// or describe('suite', { shuffle: true }, ...) describe.shuffle('suite', () => { test('random test 1', async () => { /* ... */ }) test('random test 2', async () => { /* ... */ }) test('random test 3', async () => { /* ... */ }) + + // `shuffle` is inherited + describe('still random', () => { + test('random 4.1', async () => { /* ... */ }) + test('random 4.2', async () => { /* ... */ }) + }) + + // disable shuffle inside + describe('not random', { shuffle: false }, () => { + test('in order 5.1', async () => { /* ... */ }) + test('in order 5.2', async () => { /* ... */ }) + }) }) // order depends on sequence.seed option in config (Date.now() by default) ```