From d3c85c4d19a51fd329e95aef36b954757f39b425 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Tue, 2 Jul 2024 11:11:10 +0200 Subject: [PATCH 1/7] fix(vitest): show unrelated failed tests when rerunning a test --- packages/vitest/src/api/setup.ts | 2 +- packages/vitest/src/node/pools/typecheck.ts | 2 +- packages/vitest/src/node/reporters/default.ts | 11 +++++++++++ .../vitest/src/node/reporters/renderers/utils.ts | 13 +++++++++---- packages/vitest/src/types/reporter.ts | 4 ++-- test/core/vite.config.ts | 1 - 6 files changed, 24 insertions(+), 9 deletions(-) diff --git a/packages/vitest/src/api/setup.ts b/packages/vitest/src/api/setup.ts index bb04c48c60de..69ee138e9b3a 100644 --- a/packages/vitest/src/api/setup.ts +++ b/packages/vitest/src/api/setup.ts @@ -201,7 +201,7 @@ export class WebSocketReporter implements Reporter { }) } - onFinished(files?: File[], errors?: unknown[]) { + onFinished(files: File[], errors: unknown[]) { this.clients.forEach((client) => { client.onFinished?.(files, errors)?.catch?.(noop) }) diff --git a/packages/vitest/src/node/pools/typecheck.ts b/packages/vitest/src/node/pools/typecheck.ts index 9edb034b9993..3c86225e5db6 100644 --- a/packages/vitest/src/node/pools/typecheck.ts +++ b/packages/vitest/src/node/pools/typecheck.ts @@ -40,7 +40,7 @@ export function createTypecheckPool(ctx: Vitest): ProcessPool { // triggered by TSC watcher, not Vitest watcher, so we need to emulate what Vitest does in this case if (ctx.config.watch && !ctx.runningPromise) { - await ctx.report('onFinished', files) + await ctx.report('onFinished', files, []) await ctx.report('onWatcherStart', files, [ ...(project.config.typecheck.ignoreSourceErrors ? [] : sourceErrors), ...ctx.state.getUnhandledErrors(), diff --git a/packages/vitest/src/node/reporters/default.ts b/packages/vitest/src/node/reporters/default.ts index 1d19b46f2d35..5ea890903ce8 100644 --- a/packages/vitest/src/node/reporters/default.ts +++ b/packages/vitest/src/node/reporters/default.ts @@ -1,4 +1,5 @@ import c from 'picocolors' +import { hasFailed } from '@vitest/runner/utils' import type { UserConsoleLog } from '../../types/general' import { BaseReporter } from './base' import type { ListRendererOptions } from './renderers/listRenderer' @@ -56,6 +57,16 @@ export class DefaultReporter extends BaseReporter { files = this.ctx.state.getFiles(), errors = this.ctx.state.getUnhandledErrors(), ) { + // print failed tests without their errors to keep track of previously failed tests + // this can happen if there are multiple test errors, and user changed a file + // that triggered a rerun of unrelated tests - in that case they want to see + // the error for the test they are currently working on, but still keep track of + // the other failed tests + this.renderer?.update([ + ...this.ctx.state.getFiles().filter(file => hasFailed(file) && !files.includes(file)), + ...files, + ]) + this.stopListRender() this.ctx.logger.log() super.onFinished(files, errors) diff --git a/packages/vitest/src/node/reporters/renderers/utils.ts b/packages/vitest/src/node/reporters/renderers/utils.ts index 4686c1b1abf5..cee31f750697 100644 --- a/packages/vitest/src/node/reporters/renderers/utils.ts +++ b/packages/vitest/src/node/reporters/renderers/utils.ts @@ -19,6 +19,12 @@ export const hookSpinnerMap = new WeakMap string>>() export const pointer = c.yellow(F_POINTER) export const skipped = c.dim(c.gray(F_DOWN)) +const benchmarkPass = c.green(F_DOT) +const testPass = c.green(F_CHECK) +const taskFail = c.red(F_CROSS) +const suiteFail = c.red(F_POINTER) +const pending = c.gray('·') + export function getCols(delta = 0) { let length = process.stdout?.columns if (!length || Number.isNaN(length)) { @@ -154,10 +160,9 @@ export function getStateSymbol(task: Task) { } if (!task.result) { - return c.gray('·') + return pending } - // pending if (task.result.state === 'run') { if (task.type === 'suite') { return pointer @@ -171,11 +176,11 @@ export function getStateSymbol(task: Task) { } if (task.result.state === 'pass') { - return task.meta?.benchmark ? c.green(F_DOT) : c.green(F_CHECK) + return task.meta?.benchmark ? benchmarkPass : testPass } if (task.result.state === 'fail') { - return task.type === 'suite' ? pointer : c.red(F_CROSS) + return task.type === 'suite' ? suiteFail : taskFail } return ' ' diff --git a/packages/vitest/src/types/reporter.ts b/packages/vitest/src/types/reporter.ts index ff2518aa7171..968eb85da159 100644 --- a/packages/vitest/src/types/reporter.ts +++ b/packages/vitest/src/types/reporter.ts @@ -8,8 +8,8 @@ export interface Reporter { onSpecsCollected?: (specs?: SerializableSpec[]) => Awaitable onCollected?: (files?: File[]) => Awaitable onFinished?: ( - files?: File[], - errors?: unknown[], + files: File[], + errors: unknown[], coverage?: unknown ) => Awaitable onTaskUpdate?: (packs: TaskResultPack[]) => Awaitable diff --git a/test/core/vite.config.ts b/test/core/vite.config.ts index 4ebb0cb03994..cd0bf996e990 100644 --- a/test/core/vite.config.ts +++ b/test/core/vite.config.ts @@ -50,7 +50,6 @@ export default defineConfig({ port: 3022, }, test: { - reporters: ['dot'], api: { port: 3023, }, From b7091d2dfe6ba12bef6857fa1d905ce634f11bfd Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Tue, 2 Jul 2024 11:21:10 +0200 Subject: [PATCH 2/7] fix: support basic reporter --- packages/vitest/src/node/reporters/base.ts | 103 ++++++++++-------- packages/vitest/src/node/reporters/basic.ts | 13 +++ .../src/node/reporters/renderers/utils.ts | 10 +- 3 files changed, 73 insertions(+), 53 deletions(-) diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index 5fcb7f6b69e7..3d49e9b6741e 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -14,6 +14,7 @@ import { getFullName, getSafeTimers, getSuites, + getTestName, getTests, hasFailed, hasFailedSnapshot, @@ -33,8 +34,8 @@ import { formatTimeString, getStateString, getStateSymbol, - pointer, renderSnapshotSummary, + suiteFail, } from './renderers/utils' const BADGE_PADDING = ' ' @@ -115,59 +116,65 @@ export abstract class BaseReporter implements Reporter { if (this.isTTY) { return } - const logger = this.ctx.logger for (const pack of packs) { const task = this.ctx.state.idMap.get(pack[0]) - if ( - task - && 'filepath' in task - && task.result?.state - && task.result?.state !== 'run' - ) { - const tests = getTests(task) - const failed = tests.filter(t => t.result?.state === 'fail') - const skipped = tests.filter( - t => t.mode === 'skip' || t.mode === 'todo', - ) - let state = c.dim(`${tests.length} test${tests.length > 1 ? 's' : ''}`) - if (failed.length) { - state += ` ${c.dim('|')} ${c.red(`${failed.length} failed`)}` - } - if (skipped.length) { - state += ` ${c.dim('|')} ${c.yellow(`${skipped.length} skipped`)}` - } - let suffix = c.dim(' (') + state + c.dim(')') - if (task.result.duration) { - const color - = task.result.duration > this.ctx.config.slowTestThreshold - ? c.yellow - : c.gray - suffix += color(` ${Math.round(task.result.duration)}${c.dim('ms')}`) - } - if (this.ctx.config.logHeapUsage && task.result.heap != null) { - suffix += c.magenta( - ` ${Math.floor(task.result.heap / 1024 / 1024)} MB heap used`, - ) - } - - let title = ` ${getStateSymbol(task)} ` - if (task.projectName) { - title += formatProjectName(task.projectName) - } - title += `${task.name} ${suffix}` - logger.log(title) - - // print short errors, full errors will be at the end in summary - for (const test of failed) { - logger.log(c.red(` ${pointer} ${getFullName(test, c.dim(' > '))}`)) - test.result?.errors?.forEach((e) => { - logger.log(c.red(` ${F_RIGHT} ${(e as any)?.message}`)) - }) - } + if (task) { + this.printTask(task) } } } + protected printTask(task: Task) { + if ( + !('filepath' in task) + || !task.result?.state + || task.result?.state === 'run') { + return + } + const logger = this.ctx.logger + + const tests = getTests(task) + const failed = tests.filter(t => t.result?.state === 'fail') + const skipped = tests.filter( + t => t.mode === 'skip' || t.mode === 'todo', + ) + let state = c.dim(`${tests.length} test${tests.length > 1 ? 's' : ''}`) + if (failed.length) { + state += ` ${c.dim('|')} ${c.red(`${failed.length} failed`)}` + } + if (skipped.length) { + state += ` ${c.dim('|')} ${c.yellow(`${skipped.length} skipped`)}` + } + let suffix = c.dim(' (') + state + c.dim(')') + if (task.result.duration) { + const color + = task.result.duration > this.ctx.config.slowTestThreshold + ? c.yellow + : c.gray + suffix += color(` ${Math.round(task.result.duration)}${c.dim('ms')}`) + } + if (this.ctx.config.logHeapUsage && task.result.heap != null) { + suffix += c.magenta( + ` ${Math.floor(task.result.heap / 1024 / 1024)} MB heap used`, + ) + } + + let title = ` ${getStateSymbol(task)} ` + if (task.projectName) { + title += formatProjectName(task.projectName) + } + title += `${task.name} ${suffix}` + logger.log(title) + + // print short errors, full errors will be at the end in summary + for (const test of failed) { + logger.log(c.red(` ${suiteFail} ${getTestName(test, c.dim(' > '))}`)) + test.result?.errors?.forEach((e) => { + logger.log(c.red(` ${F_RIGHT} ${(e as any)?.message}`)) + }) + } + } + onWatcherStart( files = this.ctx.state.getFiles(), errors = this.ctx.state.getUnhandledErrors(), diff --git a/packages/vitest/src/node/reporters/basic.ts b/packages/vitest/src/node/reporters/basic.ts index a6b2b3632a90..7325b6217afa 100644 --- a/packages/vitest/src/node/reporters/basic.ts +++ b/packages/vitest/src/node/reporters/basic.ts @@ -1,9 +1,22 @@ +import { hasFailed } from '@vitest/runner/utils' import type { File } from '../../types/tasks' import { BaseReporter } from './base' export class BasicReporter extends BaseReporter { isTTY = false + onWatcherRerun(files: string[], trigger?: string) { + super.onWatcherRerun(files, trigger) + + const failedTasks = this.ctx.state.getFiles().filter((file) => { + return hasFailed(file) && !files.includes(file.filepath) + }) + + for (const task of failedTasks) { + this.printTask(task) + } + } + reportSummary(files: File[], errors: unknown[]) { // non-tty mode doesn't add a new line this.ctx.logger.log() diff --git a/packages/vitest/src/node/reporters/renderers/utils.ts b/packages/vitest/src/node/reporters/renderers/utils.ts index cee31f750697..ea8cab848137 100644 --- a/packages/vitest/src/node/reporters/renderers/utils.ts +++ b/packages/vitest/src/node/reporters/renderers/utils.ts @@ -19,11 +19,11 @@ export const hookSpinnerMap = new WeakMap string>>() export const pointer = c.yellow(F_POINTER) export const skipped = c.dim(c.gray(F_DOWN)) -const benchmarkPass = c.green(F_DOT) -const testPass = c.green(F_CHECK) -const taskFail = c.red(F_CROSS) -const suiteFail = c.red(F_POINTER) -const pending = c.gray('·') +export const benchmarkPass = c.green(F_DOT) +export const testPass = c.green(F_CHECK) +export const taskFail = c.red(F_CROSS) +export const suiteFail = c.red(F_POINTER) +export const pending = c.gray('·') export function getCols(delta = 0) { let length = process.stdout?.columns From abfb9b409e1730a5cecf41e78c53f3c03265b881 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Tue, 2 Jul 2024 13:28:33 +0200 Subject: [PATCH 3/7] fix: print correct errors count --- packages/vitest/src/node/reporters/base.ts | 16 ++++++++++++---- packages/vitest/src/node/reporters/basic.ts | 7 +------ packages/vitest/src/node/reporters/default.ts | 3 +-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index 3d49e9b6741e..f608c1775e16 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -35,7 +35,7 @@ import { getStateString, getStateSymbol, renderSnapshotSummary, - suiteFail, + taskFail, } from './renderers/utils' const BADGE_PADDING = ' ' @@ -64,6 +64,7 @@ export abstract class BaseReporter implements Reporter { start = 0 end = 0 watchFilters?: string[] + failedUnwatchedFiles: Task[] = [] isTTY: boolean ctx: Vitest = undefined! @@ -168,7 +169,7 @@ export abstract class BaseReporter implements Reporter { // print short errors, full errors will be at the end in summary for (const test of failed) { - logger.log(c.red(` ${suiteFail} ${getTestName(test, c.dim(' > '))}`)) + logger.log(c.red(` ${taskFail} ${getTestName(test, c.dim(' > '))}`)) test.result?.errors?.forEach((e) => { logger.log(c.red(` ${F_RIGHT} ${(e as any)?.message}`)) }) @@ -240,6 +241,9 @@ export abstract class BaseReporter implements Reporter { onWatcherRerun(files: string[], trigger?: string) { this.resetLastRunLog() this.watchFilters = files + this.failedUnwatchedFiles = this.ctx.state.getFiles().filter((file) => { + return hasFailed(file) && !files.includes(file.filepath) + }) files.forEach((filepath) => { let reruns = this._filesInWatchMode.get(filepath) ?? 0 @@ -382,7 +386,11 @@ export abstract class BaseReporter implements Reporter { } reportTestSummary(files: File[], errors: unknown[]) { - const tests = getTests(files) + const affectedFiles = [ + ...this.failedUnwatchedFiles, + ...files, + ] + const tests = getTests(affectedFiles) const logger = this.ctx.logger const executionTime = this.end - this.start @@ -444,7 +452,7 @@ export abstract class BaseReporter implements Reporter { } } - logger.log(padTitle('Test Files'), getStateString(files)) + logger.log(padTitle('Test Files'), getStateString(affectedFiles)) logger.log(padTitle('Tests'), getStateString(tests)) if (this.ctx.projects.some(c => c.config.typecheck.enabled)) { const failed = tests.filter( diff --git a/packages/vitest/src/node/reporters/basic.ts b/packages/vitest/src/node/reporters/basic.ts index 7325b6217afa..b1b85e5dea37 100644 --- a/packages/vitest/src/node/reporters/basic.ts +++ b/packages/vitest/src/node/reporters/basic.ts @@ -1,4 +1,3 @@ -import { hasFailed } from '@vitest/runner/utils' import type { File } from '../../types/tasks' import { BaseReporter } from './base' @@ -8,11 +7,7 @@ export class BasicReporter extends BaseReporter { onWatcherRerun(files: string[], trigger?: string) { super.onWatcherRerun(files, trigger) - const failedTasks = this.ctx.state.getFiles().filter((file) => { - return hasFailed(file) && !files.includes(file.filepath) - }) - - for (const task of failedTasks) { + for (const task of this.failedUnwatchedFiles) { this.printTask(task) } } diff --git a/packages/vitest/src/node/reporters/default.ts b/packages/vitest/src/node/reporters/default.ts index 5ea890903ce8..a6b3dc513665 100644 --- a/packages/vitest/src/node/reporters/default.ts +++ b/packages/vitest/src/node/reporters/default.ts @@ -1,5 +1,4 @@ import c from 'picocolors' -import { hasFailed } from '@vitest/runner/utils' import type { UserConsoleLog } from '../../types/general' import { BaseReporter } from './base' import type { ListRendererOptions } from './renderers/listRenderer' @@ -63,7 +62,7 @@ export class DefaultReporter extends BaseReporter { // the error for the test they are currently working on, but still keep track of // the other failed tests this.renderer?.update([ - ...this.ctx.state.getFiles().filter(file => hasFailed(file) && !files.includes(file)), + ...this.failedUnwatchedFiles, ...files, ]) From 9b6d6787318c35beebf2f7fb82e80f7c120d247d Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Tue, 2 Jul 2024 13:28:42 +0200 Subject: [PATCH 4/7] test: add tests --- .../fixtures/single-failed/basic.test.ts | 5 ++ .../fixtures/single-failed/failed.test.ts | 5 ++ .../fixtures/single-failed/vitest.config.ts | 11 ++++ test/watch/fixtures/vitest.config.ts | 6 +- test/watch/test/reporter-failed.test.ts | 61 +++++++++++++++++++ 5 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 test/watch/fixtures/single-failed/basic.test.ts create mode 100644 test/watch/fixtures/single-failed/failed.test.ts create mode 100644 test/watch/fixtures/single-failed/vitest.config.ts create mode 100644 test/watch/test/reporter-failed.test.ts diff --git a/test/watch/fixtures/single-failed/basic.test.ts b/test/watch/fixtures/single-failed/basic.test.ts new file mode 100644 index 000000000000..e765faa923f6 --- /dev/null +++ b/test/watch/fixtures/single-failed/basic.test.ts @@ -0,0 +1,5 @@ +import { expect, it } from 'vitest'; + +it('works correctly', () => { + expect(1).toBe(1) +}) diff --git a/test/watch/fixtures/single-failed/failed.test.ts b/test/watch/fixtures/single-failed/failed.test.ts new file mode 100644 index 000000000000..5cdd5163d644 --- /dev/null +++ b/test/watch/fixtures/single-failed/failed.test.ts @@ -0,0 +1,5 @@ +import { it } from 'vitest'; + +it('fails', () => { + throw new Error('failed') +}) diff --git a/test/watch/fixtures/single-failed/vitest.config.ts b/test/watch/fixtures/single-failed/vitest.config.ts new file mode 100644 index 000000000000..302f6020f057 --- /dev/null +++ b/test/watch/fixtures/single-failed/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'vitest/config' + +// Patch stdin on the process so that we can fake it to seem like a real interactive terminal and pass the TTY checks +process.stdin.isTTY = true +process.stdin.setRawMode = () => process.stdin + +export default defineConfig({ + test: { + watch: true, + }, +}) diff --git a/test/watch/fixtures/vitest.config.ts b/test/watch/fixtures/vitest.config.ts index 623e5e267490..184e5719a9d6 100644 --- a/test/watch/fixtures/vitest.config.ts +++ b/test/watch/fixtures/vitest.config.ts @@ -1,4 +1,4 @@ -import { defineConfig } from 'vitest/config' +import { defaultExclude, defineConfig } from 'vitest/config' // Patch stdin on the process so that we can fake it to seem like a real interactive terminal and pass the TTY checks process.stdin.isTTY = true @@ -7,6 +7,10 @@ process.stdin.setRawMode = () => process.stdin export default defineConfig({ test: { watch: true, + exclude: [ + ...defaultExclude, + '**/single-failed/**', + ], // This configuration is edited by tests reporters: 'verbose', diff --git a/test/watch/test/reporter-failed.test.ts b/test/watch/test/reporter-failed.test.ts new file mode 100644 index 000000000000..57e136b953e3 --- /dev/null +++ b/test/watch/test/reporter-failed.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, onTestFinished } from 'vitest' +import { editFile, runVitest } from '../../test-utils' + +describe.each(['default', 'basic'])('reporter: %s', (reporter) => { + it('prints previously failed tests on rerun', async () => { + const { vitest, ctx } = await runVitest({ + watch: true, + root: './fixtures/single-failed', + reporters: [reporter], + }) + onTestFinished(async () => { + await ctx?.close() + }) + + expect(vitest.stderr).toContain('failed.test.ts > fails') + expect(vitest.stdout).toContain('❯ failed.test.ts') + expect(vitest.stdout).toContain('× fails') + expect(vitest.stdout).toContain('1 failed') + expect(vitest.stdout).toContain('1 passed') + + editFile('./fixtures/single-failed/basic.test.ts', file => `${file}\n`) + + vitest.resetOutput() + + await vitest.waitForStdout('RERUN ../../basic.test.ts') + await vitest.waitForStdout('Waiting for file changes...') + + expect(vitest.stdout).toContain('❯ failed.test.ts') + expect(vitest.stdout).toContain('× fails') + expect(vitest.stdout).toContain('1 failed') + expect(vitest.stdout).toContain('1 passed') + }) + + it('prints tests once if changed test is the same', async () => { + const { vitest, ctx } = await runVitest({ + watch: true, + root: './fixtures/single-failed', + reporters: [reporter], + }) + onTestFinished(async () => { + await ctx?.close() + }) + + expect(vitest.stderr).toContain('failed.test.ts > fails') + expect(vitest.stdout).toContain('❯ failed.test.ts') + expect(vitest.stdout).toContain('× fails') + expect(vitest.stdout).toContain('1 failed') + + editFile('./fixtures/single-failed/failed.test.ts', file => `${file}\n`) + + vitest.resetOutput() + + await vitest.waitForStdout('RERUN ../../failed.test.ts') + await vitest.waitForStdout('Watching for file changes...') + + expect(vitest.stdout).toContain('❯ failed.test.ts') + expect(vitest.stdout).toContain('× fails') + expect(vitest.stdout).toContain('1 failed') + expect(vitest.stdout).not.toContain('1 passed') + }) +}) From c214e7d52f51cff921f63483df2776240985065c Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Tue, 2 Jul 2024 13:29:40 +0200 Subject: [PATCH 5/7] chore: fix snapshot --- test/reporters/tests/merge-reports.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/reporters/tests/merge-reports.test.ts b/test/reporters/tests/merge-reports.test.ts index be73e8dc03db..70de1b63fba0 100644 --- a/test/reporters/tests/merge-reports.test.ts +++ b/test/reporters/tests/merge-reports.test.ts @@ -89,13 +89,13 @@ test('merge reports', async () => { test 1-2 ❯ first.test.ts (2 tests | 1 failed)