From 2597cd68c86c712522d6f8811e01ed6b416db4a4 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Mon, 1 Apr 2024 11:02:31 +0900 Subject: [PATCH 01/15] feat: support running suites in parallel --- examples/basic/test/repro.test.ts | 40 ++++++++++++++++++++++++++++++ packages/runner/src/suite.ts | 7 ++++-- packages/runner/src/types/tasks.ts | 4 +++ 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 examples/basic/test/repro.test.ts diff --git a/examples/basic/test/repro.test.ts b/examples/basic/test/repro.test.ts new file mode 100644 index 000000000000..49d649974d54 --- /dev/null +++ b/examples/basic/test/repro.test.ts @@ -0,0 +1,40 @@ +import { describe, test } from 'vitest' + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + +// finishes in 3 sec +// (1st and 2nd suites in parallel) +describe('example-1', () => { + // finishes in 3 sec + // (1st and 2nd cases in serial) + describe('1st suite', { concurrentSuite: true }, () => { + test('1st case', async () => { + await sleep(1000) + }) + test('2nd case', async () => { + await sleep(2000) + }) + }) + + describe('2nd suite', { concurrentSuite: true }, () => { + test('1st case', async () => { + await sleep(1000) + }) + test('2nd case', async () => { + await sleep(2000) + }) + }) +}) + +// finishes in 3 sec +// (same as example-1 but implemented as describe.each) +describe('example-2', () => { + describe.each(['1st suite', '2nd suite'])('%s', { concurrentSuite: true }, () => { + test('1st case', async () => { + await sleep(1000) + }) + test('2nd case', async () => { + await sleep(2000) + }) + }) +}) diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index b1761b5c20fc..c75edc50d75b 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -172,7 +172,8 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m if (typeof suiteOptions === 'object') options = Object.assign({}, suiteOptions, options) - // inherit concurrent / sequential from suite + // TODO: `this.concurrent/sequential` always undefined? + // inherit concurrent / sequential from current suite options.concurrent = this.concurrent || (!this.sequential && options?.concurrent) options.sequential = this.sequential || (!this.concurrent && options?.sequential) @@ -215,6 +216,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m tasks: [], meta: Object.create(null), projectName: '', + concurrent: suiteOptions?.concurrentSuite, } if (runner && includeLocation && runner.config.includeTaskLocation) { @@ -282,7 +284,8 @@ function createSuite() { if (currentSuite?.options) options = { ...currentSuite.options, ...options } - // inherit concurrent / sequential from current suite + // TODO: `this.concurrent/sequential` always undefined? + // inherit test concurrent / sequential from current suite options.concurrent = this.concurrent || (!this.sequential && options?.concurrent) options.sequential = this.sequential || (!this.concurrent && options?.sequential) diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index 9e1ac191dc92..063402f1bfff 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -185,6 +185,10 @@ export interface TestOptions { * Tests inherit `concurrent` from `describe()` and nested `describe()` will inherit from parent's `concurrent`. */ concurrent?: boolean + /** + * wip + */ + concurrentSuite?: boolean /** * Whether tests run sequentially. * Tests inherit `sequential` from `describe()` and nested `describe()` will inherit from parent's `sequential`. From de4fe6ccbcf18294cedfc0c7b48721a6cdac73ec Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Fri, 5 Apr 2024 12:25:23 +0900 Subject: [PATCH 02/15] chore: cleanup --- packages/runner/src/suite.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index c75edc50d75b..08940df9a04d 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -172,8 +172,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m if (typeof suiteOptions === 'object') options = Object.assign({}, suiteOptions, options) - // TODO: `this.concurrent/sequential` always undefined? - // inherit concurrent / sequential from current suite + // inherit concurrent / sequential from suite options.concurrent = this.concurrent || (!this.sequential && options?.concurrent) options.sequential = this.sequential || (!this.concurrent && options?.sequential) @@ -284,8 +283,7 @@ function createSuite() { if (currentSuite?.options) options = { ...currentSuite.options, ...options } - // TODO: `this.concurrent/sequential` always undefined? - // inherit test concurrent / sequential from current suite + // inherit concurrent / sequential from current suite options.concurrent = this.concurrent || (!this.sequential && options?.concurrent) options.sequential = this.sequential || (!this.concurrent && options?.sequential) From f409f8058b9f4a3367005dac3abc8f7bb62730bb Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Fri, 5 Apr 2024 13:04:15 +0900 Subject: [PATCH 03/15] test: add test --- examples/basic/test/repro.test.ts | 24 +++++++++ packages/runner/src/run.ts | 1 + packages/runner/src/types/tasks.ts | 2 +- test/core/test/concurrent-suite.test.ts | 72 +++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 test/core/test/concurrent-suite.test.ts diff --git a/examples/basic/test/repro.test.ts b/examples/basic/test/repro.test.ts index 49d649974d54..19950c2af7cf 100644 --- a/examples/basic/test/repro.test.ts +++ b/examples/basic/test/repro.test.ts @@ -38,3 +38,27 @@ describe('example-2', () => { }) }) }) + +describe.only('example-3', () => { + describe('nested', { concurrentSuite: true }, () => { + // finishes in 3 sec + // (1st and 2nd cases in serial) + describe('1st suite', () => { + test('1st case', async () => { + await sleep(1000) + }) + test('2nd case', async () => { + await sleep(2000) + }) + }) + + describe('2nd suite', () => { + test('1st case', async () => { + await sleep(1000) + }) + test('2nd case', async () => { + await sleep(2000) + }) + }) + }) +}) diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 5155621fb129..60a748d4a998 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -311,6 +311,7 @@ export async function runSuite(suite: Suite, runner: VitestRunner) { else { for (let tasksGroup of partitionSuiteChildren(suite)) { if (tasksGroup[0].concurrent === true) { + // TODO: p-limit needs to be shared globally to not go over maxConcurrency? const mutex = limit(runner.config.maxConcurrency) await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner)))) } diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index 063402f1bfff..312faa7a68e8 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -186,7 +186,7 @@ export interface TestOptions { */ concurrent?: boolean /** - * wip + * TODO: suite only options? */ concurrentSuite?: boolean /** diff --git a/test/core/test/concurrent-suite.test.ts b/test/core/test/concurrent-suite.test.ts new file mode 100644 index 000000000000..440667bb1e60 --- /dev/null +++ b/test/core/test/concurrent-suite.test.ts @@ -0,0 +1,72 @@ +import { createDefer } from '@vitest/utils' +import { afterAll, describe, test } from 'vitest' + +describe('basic', () => { + const defers = [ + createDefer(), + createDefer(), + createDefer(), + createDefer(), + ] + + afterAll(async () => { + await defers[3] + }) + + describe('1st suite', { concurrentSuite: true }, () => { + test('0', async () => { + defers[0].resolve() + }) + + test('1', async () => { + await defers[2] // this would deadlock if sequential + defers[1].resolve() + }) + }) + + describe('2nd suite', { concurrentSuite: true }, () => { + test('2', async () => { + await defers[0] + defers[2].resolve() + }) + test('3', async () => { + await defers[1] + defers[3].resolve() + }) + }) +}) + +describe('option affects deeply', { concurrentSuite: true }, () => { + const defers = [ + createDefer(), + createDefer(), + createDefer(), + createDefer(), + ] + + afterAll(async () => { + await defers[3] + }) + + describe('1st suite', () => { + test('0', async () => { + defers[0].resolve() + }) + + test('1', async () => { + await defers[2] // this would deadlock if sequential + defers[1].resolve() + }) + }) + + describe('2nd suite', () => { + test('2', async () => { + await defers[0] + defers[2].resolve() + }) + test('3', async () => { + await defers[1] + defers[3].resolve() + }) + }) +}) From 89aadc6b999ea6bd046f801fce2ff06428927b71 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Fri, 5 Apr 2024 13:04:30 +0900 Subject: [PATCH 04/15] chore: cleanup --- examples/basic/test/repro.test.ts | 64 ------------------------------- 1 file changed, 64 deletions(-) delete mode 100644 examples/basic/test/repro.test.ts diff --git a/examples/basic/test/repro.test.ts b/examples/basic/test/repro.test.ts deleted file mode 100644 index 19950c2af7cf..000000000000 --- a/examples/basic/test/repro.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { describe, test } from 'vitest' - -const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) - -// finishes in 3 sec -// (1st and 2nd suites in parallel) -describe('example-1', () => { - // finishes in 3 sec - // (1st and 2nd cases in serial) - describe('1st suite', { concurrentSuite: true }, () => { - test('1st case', async () => { - await sleep(1000) - }) - test('2nd case', async () => { - await sleep(2000) - }) - }) - - describe('2nd suite', { concurrentSuite: true }, () => { - test('1st case', async () => { - await sleep(1000) - }) - test('2nd case', async () => { - await sleep(2000) - }) - }) -}) - -// finishes in 3 sec -// (same as example-1 but implemented as describe.each) -describe('example-2', () => { - describe.each(['1st suite', '2nd suite'])('%s', { concurrentSuite: true }, () => { - test('1st case', async () => { - await sleep(1000) - }) - test('2nd case', async () => { - await sleep(2000) - }) - }) -}) - -describe.only('example-3', () => { - describe('nested', { concurrentSuite: true }, () => { - // finishes in 3 sec - // (1st and 2nd cases in serial) - describe('1st suite', () => { - test('1st case', async () => { - await sleep(1000) - }) - test('2nd case', async () => { - await sleep(2000) - }) - }) - - describe('2nd suite', () => { - test('1st case', async () => { - await sleep(1000) - }) - test('2nd case', async () => { - await sleep(2000) - }) - }) - }) -}) From 7b3ffeba3fcb4539ce2e68573782f152b5048def Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Fri, 5 Apr 2024 13:09:48 +0900 Subject: [PATCH 05/15] test: test with describe.each --- test/core/test/concurrent-suite.test.ts | 39 ++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/test/core/test/concurrent-suite.test.ts b/test/core/test/concurrent-suite.test.ts index 440667bb1e60..b4ad332464d5 100644 --- a/test/core/test/concurrent-suite.test.ts +++ b/test/core/test/concurrent-suite.test.ts @@ -36,7 +36,7 @@ describe('basic', () => { }) }) -describe('option affects deeply', { concurrentSuite: true }, () => { +describe('inherits option', { concurrentSuite: true }, () => { const defers = [ createDefer(), createDefer(), @@ -70,3 +70,40 @@ describe('option affects deeply', { concurrentSuite: true }, () => { }) }) }) + +describe('works with describe.each', () => { + const defers = [ + createDefer(), + createDefer(), + createDefer(), + createDefer(), + ] + + afterAll(async () => { + await defers[3] + }) + + describe.each([0, 1])('%s', { concurrentSuite: true }, (i) => { + if (i === 0) { + test('0', async () => { + defers[0].resolve() + }) + + test('1', async () => { + await defers[2] // this would deadlock if sequential + defers[1].resolve() + }) + } + + if (i === 1) { + test('2', async () => { + await defers[0] + defers[2].resolve() + }) + test('3', async () => { + await defers[1] + defers[3].resolve() + }) + } + }) +}) From 060a3ab9a5d8293d706ad986fcd7c871095c214f Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Fri, 5 Apr 2024 13:11:02 +0900 Subject: [PATCH 06/15] test: tweak --- test/core/test/concurrent-suite.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/core/test/concurrent-suite.test.ts b/test/core/test/concurrent-suite.test.ts index b4ad332464d5..afd19152d3c7 100644 --- a/test/core/test/concurrent-suite.test.ts +++ b/test/core/test/concurrent-suite.test.ts @@ -83,8 +83,8 @@ describe('works with describe.each', () => { await defers[3] }) - describe.each([0, 1])('%s', { concurrentSuite: true }, (i) => { - if (i === 0) { + describe.each(['1st suite', '2nd suite'])('%s', { concurrentSuite: true }, (s) => { + if (s === '1st suite') { test('0', async () => { defers[0].resolve() }) @@ -95,7 +95,7 @@ describe('works with describe.each', () => { }) } - if (i === 1) { + if (s === '2nd suite') { test('2', async () => { await defers[0] defers[2].resolve() From 71bfabd6433ccb8ee90c97b760172175ceae9a1c Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Fri, 5 Apr 2024 13:43:43 +0900 Subject: [PATCH 07/15] test: more --- test/core/test/concurrent-suite.test.ts | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/test/core/test/concurrent-suite.test.ts b/test/core/test/concurrent-suite.test.ts index afd19152d3c7..f94e4ffd6d16 100644 --- a/test/core/test/concurrent-suite.test.ts +++ b/test/core/test/concurrent-suite.test.ts @@ -1,5 +1,5 @@ import { createDefer } from '@vitest/utils' -import { afterAll, describe, test } from 'vitest' +import { afterAll, describe, expect, test } from 'vitest' describe('basic', () => { const defers = [ @@ -107,3 +107,23 @@ describe('works with describe.each', () => { } }) }) + +describe('tests are sequential', () => { + describe('1st suite', { concurrentSuite: true }, () => { + const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + let done = false + + test('0', () => { + expect(done).toBe(false) + }) + + test('1', async () => { + await sleep(100) + done = true + }) + + test('2', () => { + expect(done).toBe(true) + }) + }) +}) From 247ca42e19f1632a0670660c26e2a55e9167e67c Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Fri, 5 Apr 2024 14:00:53 +0900 Subject: [PATCH 08/15] test: tweak --- test/core/test/concurrent-suite.test.ts | 40 +++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/test/core/test/concurrent-suite.test.ts b/test/core/test/concurrent-suite.test.ts index f94e4ffd6d16..b5b9f184cf1e 100644 --- a/test/core/test/concurrent-suite.test.ts +++ b/test/core/test/concurrent-suite.test.ts @@ -108,12 +108,14 @@ describe('works with describe.each', () => { }) }) +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + describe('tests are sequential', () => { describe('1st suite', { concurrentSuite: true }, () => { - const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) let done = false - test('0', () => { + test('0', async () => { + await sleep(200) expect(done).toBe(false) }) @@ -127,3 +129,37 @@ describe('tests are sequential', () => { }) }) }) + +// TODO +describe('maxConcurrency', { concurrent: true, concurrentSuite: true }, () => { + const defers = [ + createDefer(), + createDefer(), + createDefer(), + createDefer(), + ] + + describe('1st suite', () => { + test('0', async () => { + defers[0].resolve() + await defers[3] + }) + + test('1', async () => { + await defers[0] + defers[1].resolve() + }) + }) + + describe('2nd suite', () => { + test('2', async () => { + await defers[1] + defers[2].resolve() + }) + + test('3', async () => { + await defers[2] + defers[3].resolve() + }) + }) +}) From 9dee912e34dc1c3b8e9e92201ff225845628bde2 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Fri, 5 Apr 2024 14:15:14 +0900 Subject: [PATCH 09/15] test: test maxConcurrency --- packages/runner/src/run.ts | 1 - .../test/concurrent-maxConcurrency.test.ts | 42 +++++++++++++++++++ test/core/test/concurrent-suite.test.ts | 34 --------------- 3 files changed, 42 insertions(+), 35 deletions(-) create mode 100644 test/core/test/concurrent-maxConcurrency.test.ts diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 60a748d4a998..5155621fb129 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -311,7 +311,6 @@ export async function runSuite(suite: Suite, runner: VitestRunner) { else { for (let tasksGroup of partitionSuiteChildren(suite)) { if (tasksGroup[0].concurrent === true) { - // TODO: p-limit needs to be shared globally to not go over maxConcurrency? const mutex = limit(runner.config.maxConcurrency) await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner)))) } diff --git a/test/core/test/concurrent-maxConcurrency.test.ts b/test/core/test/concurrent-maxConcurrency.test.ts new file mode 100644 index 000000000000..993f9095d0fb --- /dev/null +++ b/test/core/test/concurrent-maxConcurrency.test.ts @@ -0,0 +1,42 @@ +import { createDefer } from '@vitest/utils' +import { describe, test, vi } from 'vitest' + +vi.setConfig({ maxConcurrency: 2 }) + +describe('limit for each suite', () => { + // this test requires running 3 tests in parallel. + // but, currently p-limit is applied for each suite layer, + // so tests succeed. + // + // [0] [1] [2] + // * -> + // * -> + // <- * + // <------ + + const defers = [ + createDefer(), + createDefer(), + createDefer(), + ] + + describe('1st suite', { concurrent: true, concurrentSuite: true }, () => { + test('a', async () => { + defers[0].resolve() + await defers[2] + }) + + test('b', async () => { + await defers[0] + defers[1].resolve() + await defers[2] + }) + }) + + describe('2nd suite', { concurrent: true, concurrentSuite: true }, () => { + test('c', async () => { + await defers[1] + defers[2].resolve() + }) + }) +}) diff --git a/test/core/test/concurrent-suite.test.ts b/test/core/test/concurrent-suite.test.ts index b5b9f184cf1e..6d6faaf22faa 100644 --- a/test/core/test/concurrent-suite.test.ts +++ b/test/core/test/concurrent-suite.test.ts @@ -129,37 +129,3 @@ describe('tests are sequential', () => { }) }) }) - -// TODO -describe('maxConcurrency', { concurrent: true, concurrentSuite: true }, () => { - const defers = [ - createDefer(), - createDefer(), - createDefer(), - createDefer(), - ] - - describe('1st suite', () => { - test('0', async () => { - defers[0].resolve() - await defers[3] - }) - - test('1', async () => { - await defers[0] - defers[1].resolve() - }) - }) - - describe('2nd suite', () => { - test('2', async () => { - await defers[1] - defers[2].resolve() - }) - - test('3', async () => { - await defers[2] - defers[3].resolve() - }) - }) -}) From f52316bd2833615734ebc7234846cdcf9feaeea6 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 30 Apr 2024 15:51:22 +0900 Subject: [PATCH 10/15] wip: limitMaxConcurrency only test cases --- packages/runner/src/run.ts | 9 +++-- pnpm-lock.yaml | 1 + .../fails/concurrent-suite-deadlock.test.ts} | 27 +++++++------- .../fails/concurrent-test-deadlock.test.ts | 37 +++++++++++++++++++ .../cli/test/__snapshots__/fails.test.ts.snap | 4 ++ 5 files changed, 61 insertions(+), 17 deletions(-) rename test/{core/test/concurrent-maxConcurrency.test.ts => cli/fixtures/fails/concurrent-suite-deadlock.test.ts} (50%) create mode 100644 test/cli/fixtures/fails/concurrent-test-deadlock.test.ts diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index c0d55c414b21..3ed6ed96e025 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -325,8 +325,7 @@ export async function runSuite(suite: Suite, runner: VitestRunner) { else { for (let tasksGroup of partitionSuiteChildren(suite)) { if (tasksGroup[0].concurrent === true) { - const mutex = limit(runner.config.maxConcurrency) - await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner)))) + await Promise.all(tasksGroup.map(c => runSuiteChild(c, runner))) } else { const { sequence } = runner.config @@ -379,15 +378,19 @@ export async function runSuite(suite: Suite, runner: VitestRunner) { } } +let limitMaxConcurrency: ReturnType + async function runSuiteChild(c: Task, runner: VitestRunner) { if (c.type === 'test' || c.type === 'custom') - return runTest(c, runner) + return limitMaxConcurrency(() => runTest(c, runner)) else if (c.type === 'suite') return runSuite(c, runner) } export async function runFiles(files: File[], runner: VitestRunner) { + limitMaxConcurrency ??= limit(runner.config.maxConcurrency) + for (const file of files) { if (!file.tasks.length && !runner.config.passWithNoTests) { if (!file.result?.errors?.length) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0925345a1f7c..d3f2ced20b00 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29436,6 +29436,7 @@ packages: /workbox-google-analytics@7.0.0: resolution: {integrity: sha512-MEYM1JTn/qiC3DbpvP2BVhyIH+dV/5BjHk756u9VbwuAhu0QHyKscTnisQuz21lfRpOwiS9z4XdqeVAKol0bzg==} + deprecated: It is not compatible with newer versions of GA starting with v4, as long as you are using GAv3 it should be ok, but the package is not longer being maintained dependencies: workbox-background-sync: 7.0.0 workbox-core: 7.0.0 diff --git a/test/core/test/concurrent-maxConcurrency.test.ts b/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts similarity index 50% rename from test/core/test/concurrent-maxConcurrency.test.ts rename to test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts index 993f9095d0fb..5a970ce2fce7 100644 --- a/test/core/test/concurrent-maxConcurrency.test.ts +++ b/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts @@ -1,26 +1,25 @@ -import { createDefer } from '@vitest/utils' import { describe, test, vi } from 'vitest' +import { createDefer } from 'vitest/dist/utils.js' -vi.setConfig({ maxConcurrency: 2 }) +// 3 tests depend on each other, +// so they will deadlock when maxConcurrency < 3 +// +// [a] [b] [c] +// * -> +// * -> +// <- * +// <------ -describe('limit for each suite', () => { - // this test requires running 3 tests in parallel. - // but, currently p-limit is applied for each suite layer, - // so tests succeed. - // - // [0] [1] [2] - // * -> - // * -> - // <- * - // <------ +vi.setConfig({ maxConcurrency: 2 }) +describe('wrapper', { concurrent: true, timeout: 500 }, () => { const defers = [ createDefer(), createDefer(), createDefer(), ] - describe('1st suite', { concurrent: true, concurrentSuite: true }, () => { + describe('1st suite', () => { test('a', async () => { defers[0].resolve() await defers[2] @@ -33,7 +32,7 @@ describe('limit for each suite', () => { }) }) - describe('2nd suite', { concurrent: true, concurrentSuite: true }, () => { + describe('2nd suite', () => { test('c', async () => { await defers[1] defers[2].resolve() diff --git a/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts b/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts new file mode 100644 index 000000000000..430316be91c4 --- /dev/null +++ b/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts @@ -0,0 +1,37 @@ +import { describe, test, vi } from 'vitest' +import { createDefer } from 'vitest/dist/utils.js' + +// 3 tests depend on each other, +// so they will deadlock when maxConcurrency < 3 +// +// [a] [b] [c] +// * -> +// * -> +// <- * +// <------ + +vi.setConfig({ maxConcurrency: 2 }) + +describe('wrapper', { concurrent: true, timeout: 500 }, () => { + const defers = [ + createDefer(), + createDefer(), + createDefer(), + ] + + test('a', async () => { + defers[0].resolve() + await defers[2] + }) + + test('b', async () => { + await defers[0] + defers[1].resolve() + await defers[2] + }) + + test('c', async () => { + await defers[1] + defers[2].resolve() + }) +}) diff --git a/test/cli/test/__snapshots__/fails.test.ts.snap b/test/cli/test/__snapshots__/fails.test.ts.snap index 6fe18ff23435..9dbbe710ceb9 100644 --- a/test/cli/test/__snapshots__/fails.test.ts.snap +++ b/test/cli/test/__snapshots__/fails.test.ts.snap @@ -2,6 +2,10 @@ exports[`should fail .dot-folder/dot-test.test.ts > .dot-folder/dot-test.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`; +exports[`should fail concurrent-suite-deadlock.test.ts > concurrent-suite-deadlock.test.ts 1`] = `"Error: Test timed out in 500ms."`; + +exports[`should fail concurrent-test-deadlock.test.ts > concurrent-test-deadlock.test.ts 1`] = `"Error: Test timed out in 500ms."`; + exports[`should fail each-timeout.test.ts > each-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`; exports[`should fail empty.test.ts > empty.test.ts 1`] = `"Error: No test suite found in file /empty.test.ts"`; From 7bb58da4f822c9ace0b22e1717b6d7e7d712b1ca Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 14 May 2024 15:25:37 +0900 Subject: [PATCH 11/15] chore: remove concurrentSuite and use only concurrent --- packages/runner/src/suite.ts | 2 +- packages/runner/src/types/tasks.ts | 6 +---- test/core/test/concurrent-suite.test.ts | 32 ++++--------------------- 3 files changed, 7 insertions(+), 33 deletions(-) diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index 3e866048c855..2b2b4338b439 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -222,7 +222,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m shuffle, tasks: [], meta: Object.create(null), - concurrent: suiteOptions?.concurrentSuite, + concurrent: suiteOptions?.concurrent, } if (runner && includeLocation && runner.config.includeTaskLocation) { diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index fb3e9297a722..885e8e9bd32a 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -180,14 +180,10 @@ export interface TestOptions { */ repeats?: number /** - * Whether tests run concurrently. + * Whether suites and tests run concurrently. * Tests inherit `concurrent` from `describe()` and nested `describe()` will inherit from parent's `concurrent`. */ concurrent?: boolean - /** - * TODO: suite only options? - */ - concurrentSuite?: boolean /** * Whether tests run sequentially. * Tests inherit `sequential` from `describe()` and nested `describe()` will inherit from parent's `sequential`. diff --git a/test/core/test/concurrent-suite.test.ts b/test/core/test/concurrent-suite.test.ts index 6d6faaf22faa..ed1f29c47401 100644 --- a/test/core/test/concurrent-suite.test.ts +++ b/test/core/test/concurrent-suite.test.ts @@ -1,5 +1,5 @@ import { createDefer } from '@vitest/utils' -import { afterAll, describe, expect, test } from 'vitest' +import { afterAll, describe, test } from 'vitest' describe('basic', () => { const defers = [ @@ -13,7 +13,7 @@ describe('basic', () => { await defers[3] }) - describe('1st suite', { concurrentSuite: true }, () => { + describe('1st suite', { concurrent: true }, () => { test('0', async () => { defers[0].resolve() }) @@ -24,7 +24,7 @@ describe('basic', () => { }) }) - describe('2nd suite', { concurrentSuite: true }, () => { + describe('2nd suite', { concurrent: true }, () => { test('2', async () => { await defers[0] defers[2].resolve() @@ -36,7 +36,7 @@ describe('basic', () => { }) }) -describe('inherits option', { concurrentSuite: true }, () => { +describe('inherits option', { concurrent: true }, () => { const defers = [ createDefer(), createDefer(), @@ -83,7 +83,7 @@ describe('works with describe.each', () => { await defers[3] }) - describe.each(['1st suite', '2nd suite'])('%s', { concurrentSuite: true }, (s) => { + describe.each(['1st suite', '2nd suite'])('%s', { concurrent: true }, (s) => { if (s === '1st suite') { test('0', async () => { defers[0].resolve() @@ -107,25 +107,3 @@ describe('works with describe.each', () => { } }) }) - -const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) - -describe('tests are sequential', () => { - describe('1st suite', { concurrentSuite: true }, () => { - let done = false - - test('0', async () => { - await sleep(200) - expect(done).toBe(false) - }) - - test('1', async () => { - await sleep(100) - done = true - }) - - test('2', () => { - expect(done).toBe(true) - }) - }) -}) From 029a1cdca0a4a7ec2a0c617a34f120c9261cd501 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 14 May 2024 15:40:51 +0900 Subject: [PATCH 12/15] test: test override `concurrent: false` --- test/core/test/concurrent-suite.test.ts | 68 ++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/test/core/test/concurrent-suite.test.ts b/test/core/test/concurrent-suite.test.ts index ed1f29c47401..23339b78e926 100644 --- a/test/core/test/concurrent-suite.test.ts +++ b/test/core/test/concurrent-suite.test.ts @@ -1,5 +1,5 @@ import { createDefer } from '@vitest/utils' -import { afterAll, describe, test } from 'vitest' +import { afterAll, describe, expect, test } from 'vitest' describe('basic', () => { const defers = [ @@ -107,3 +107,69 @@ describe('works with describe.each', () => { } }) }) + +describe('override concurrent', { concurrent: true }, () => { + checkParallelSuites() + + describe('s-x', { concurrent: false }, () => { + checkSequentialTests() + }) + + describe('s-y', () => { + checkParallelTests() + }) +}) + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + +function checkSequentialTests() { + let x = 0 + + test('t1', async () => { + await sleep(200) + expect(x).toBe(0) + x++ + }) + + test('t2', async () => { + expect(x).toBe(1) + }) +} + +function checkParallelTests() { + const defers = [ + createDefer(), + createDefer(), + ] + + test('t1', async () => { + defers[0].resolve() + await defers[1] + }) + + test('t2', async () => { + await defers[0] + defers[1].resolve() + }) +} + +function checkParallelSuites() { + const defers = [ + createDefer(), + createDefer(), + ] + + describe('s1', () => { + test('t1', async () => { + defers[0].resolve() + await defers[1] + }) + }) + + describe('s2', () => { + test('t1', async () => { + await defers[0] + defers[1].resolve() + }) + }) +} From 7454aea2226d8d347596cd07ac8ccab6485e441e Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 14 May 2024 15:48:21 +0900 Subject: [PATCH 13/15] test: describe.sequential --- test/core/test/concurrent-suite.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/core/test/concurrent-suite.test.ts b/test/core/test/concurrent-suite.test.ts index 23339b78e926..56bac82a7818 100644 --- a/test/core/test/concurrent-suite.test.ts +++ b/test/core/test/concurrent-suite.test.ts @@ -115,6 +115,15 @@ describe('override concurrent', { concurrent: true }, () => { checkSequentialTests() }) + describe.sequential('s-x-1', () => { + checkSequentialTests() + }) + + // TODO: not working? + // describe('s-x-2', { sequential: true, }, () => { + // checkSequentialTests() + // }) + describe('s-y', () => { checkParallelTests() }) From 0e071e0c9440eefa4344a93f9152e3b67ded4210 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 14 May 2024 16:18:16 +0900 Subject: [PATCH 14/15] chore(docs): update describe.concurrent --- docs/api/index.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/api/index.md b/docs/api/index.md index 36817aea75b1..dc1addeef582 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -684,15 +684,18 @@ In order to do that run `vitest` with specific file containing the tests in ques - **Alias:** `suite.concurrent` -`describe.concurrent` in a suite marks every tests as concurrent +`describe.concurrent` runs all inner suites and tests in parallel ```ts twoslash import { describe, test } from 'vitest' // ---cut--- -// All tests within this suite will be run in parallel +// All suites and tests within this suite will be run in parallel describe.concurrent('suite', () => { test('concurrent test 1', async () => { /* ... */ }) - test('concurrent test 2', async () => { /* ... */ }) + describe('concurrent suite 2', async () => { + test('concurrent test inner 1', async () => { /* ... */ }) + test('concurrent test inner 2', async () => { /* ... */ }) + }) test.concurrent('concurrent test 3', async () => { /* ... */ }) }) ``` From 5ede155581e4c9e76a0f00558330d26b1d000cbb Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 14 May 2024 18:49:08 +0900 Subject: [PATCH 15/15] chore: fix createDefer --- pnpm-lock.yaml | 3 +++ test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts | 2 +- test/cli/fixtures/fails/concurrent-test-deadlock.test.ts | 2 +- test/cli/package.json | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f1e9b8f12f3b..b1170e57c378 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1094,6 +1094,9 @@ importers: '@vitest/runner': specifier: workspace:^ version: link:../../packages/runner + '@vitest/utils': + specifier: workspace:* + version: link:../../packages/utils debug: specifier: ^4.3.4 version: 4.3.4(supports-color@8.1.1) diff --git a/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts b/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts index 5a970ce2fce7..0c57d449dde4 100644 --- a/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts +++ b/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts @@ -1,5 +1,5 @@ +import { createDefer } from '@vitest/utils' import { describe, test, vi } from 'vitest' -import { createDefer } from 'vitest/dist/utils.js' // 3 tests depend on each other, // so they will deadlock when maxConcurrency < 3 diff --git a/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts b/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts index 430316be91c4..b56fd442dcdc 100644 --- a/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts +++ b/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts @@ -1,5 +1,5 @@ import { describe, test, vi } from 'vitest' -import { createDefer } from 'vitest/dist/utils.js' +import { createDefer } from '@vitest/utils' // 3 tests depend on each other, // so they will deadlock when maxConcurrency < 3 diff --git a/test/cli/package.json b/test/cli/package.json index d821f41e31f8..80e7e4629463 100644 --- a/test/cli/package.json +++ b/test/cli/package.json @@ -10,6 +10,7 @@ "@types/ws": "^8.5.9", "@vitejs/plugin-basic-ssl": "^1.0.2", "@vitest/runner": "workspace:^", + "@vitest/utils": "workspace:*", "debug": "^4.3.4", "execa": "^8.0.1", "unplugin-swc": "^1.4.4",