diff --git a/docs/api/expect.md b/docs/api/expect.md index 04f4317b04d6..7b74129f6d53 100644 --- a/docs/api/expect.md +++ b/docs/api/expect.md @@ -836,7 +836,7 @@ test('spy function', () => { - **Type**: `(...args: any[]) => Awaitable` -This assertion checks if a function was called with certain parameters at it's last invocation. Requires a spy function to be passed to `expect`. +This assertion checks if a function was called with certain parameters at its last invocation. Requires a spy function to be passed to `expect`. ```ts import { expect, test, vi } from 'vitest' @@ -950,7 +950,7 @@ test('spy function returns a product', () => { - **Type**: `(returnValue: any) => Awaitable` -You can call this assertion to check if a function has successfully returned a value with certain parameters on it's last invoking. Requires a spy function to be passed to `expect`. +You can call this assertion to check if a function has successfully returned a value with certain parameters on its last invoking. Requires a spy function to be passed to `expect`. ```ts import { expect, test, vi } from 'vitest' diff --git a/docs/config/index.md b/docs/config/index.md index e9ebabc9a068..5c9c75c87be6 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -1137,6 +1137,10 @@ Clean coverage report on watch rerun - **Available for providers:** `'v8' | 'istanbul'` - **CLI:** `--coverage.reportsDirectory=` +::: warning +Vitest will delete this directory before running tests if `coverage.clean` is enabled (default value). +::: + Directory to write coverage report to. To preview the coverage report in the output of [HTML reporter](/guide/reporters.html#html-reporter), this option must be set as a sub-directory of the html report directory (for example `./html/coverage`). @@ -1760,13 +1764,29 @@ Sharding is happening before sorting, and only if `--shard` option is provided. #### sequence.shuffle -- **Type**: `boolean` +- **Type**: `boolean | { files?, tests? }` - **Default**: `false` - **CLI**: `--sequence.shuffle`, `--sequence.shuffle=false` -If you want tests to run randomly, you can enable it with this option, or CLI argument [`--sequence.shuffle`](/guide/cli). +If you want files and tests to run randomly, you can enable it with this option, or CLI argument [`--sequence.shuffle`](/guide/cli). + +Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your files and tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously. + +#### sequence.shuffle.files 1.4.0+ {#sequence-shuffle-files} -Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously. +- **Type**: `boolean` +- **Default**: `false` +- **CLI**: `--sequence.shuffle.files`, `--sequence.shuffle.files=false` + +Whether to randomize files, be aware that long running tests will not start earlier if you enable this option. + +#### sequence.shuffle.tests 1.4.0+ {#sequence-shuffle-tests} + +- **Type**: `boolean` +- **Default**: `false` +- **CLI**: `--sequence.shuffle.tests`, `--sequence.shuffle.tests=false` + +Whether to randomize tests. #### sequence.concurrent 0.32.2+ {#sequence-concurrent} @@ -2084,3 +2104,16 @@ Disabling this option might [improve performance](/guide/improving-performance) ::: tip You can disable isolation for specific pools by using [`poolOptions`](#pooloptions) property. ::: + +### includeTaskLocation 1.4.0+ {#includeTaskLocation} + +- **Type:** `boolean` +- **Default:** `false` + +Should `location` property be included when Vitest API receives tasks in [reporters](#reporters). If you have a lot of tests, this might cause a small performance regression. + +The `location` property has `column` and `line` values that correspond to the `test` or `describe` position in the original file. + +::: tip +This option has no effect if you do not use custom code that relies on this. +::: diff --git a/docs/guide/cli.md b/docs/guide/cli.md index 64f85d0a620c..424775c82dfc 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -103,10 +103,11 @@ Run only [benchmark](https://vitest.dev/guide/features.html#benchmarking-experim | `--exclude ` | Additional file globs to be excluded from test | | `--expand-snapshot-diff` | Show full diff when snapshot fails | | `--disable-console-intercept` | Disable automatic interception of console logging (default: `false`) | +| `--clearScreen` | Clear terminal screen when re-running tests during watch mode (default: `true`) | | `--typecheck [options]` | Custom options for typecheck pool. If passed without options, enables typechecking | | `--typecheck.enabled` | Enable typechecking alongside tests (default: `false`) | | `--typecheck.only` | Run only typecheck tests. This automatically enables typecheck (default: `false`) | -| `--project` | The name of the project to run if you are using Vitest workspace feature. This can be repeated for multiple projects: `--project=1 --project=2` | +| `--project` | The name of the project to run if you are using Vitest workspace feature. This can be repeated for multiple projects: `--project=1 --project=2`. You can also filter projects using wildcards like `--project=packages*` | | `-h, --help` | Display available CLI options | ::: tip @@ -137,6 +138,8 @@ vitest --api=false To run tests against changes made in the last commit, you can use `--changed HEAD~1`. You can also pass commit hash (e.g. `--changed 09a9920`) or branch name (e.g. `--changed origin/develop`). + When used with code coverage the report will contain only the files that were related to the changes. + If paired with the [`forceRerunTriggers`](/config/#forcereruntriggers) config option it will run the whole test suite if at least one of the files listed in the `forceRerunTriggers` list changes. By default, changes to the Vitest config file and `package.json` will always rerun the whole suite. ### shard diff --git a/eslint.config.js b/eslint.config.js index 0e308cd46160..2a7def112665 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -15,6 +15,8 @@ export default antfu( 'test/workspaces/results.json', 'test/reporters/fixtures/with-syntax-error.test.js', 'test/network-imports/public/slash@3.0.0.js', + 'test/coverage-test/src/transpiled.js', + 'test/coverage-test/src/original.ts', 'examples/**/mockServiceWorker.js', 'examples/sveltekit/.svelte-kit', ], diff --git a/examples/playwright/test/basic.test.ts b/examples/playwright/test/basic.test.ts index 44cee93046d7..4c6631c55b81 100644 --- a/examples/playwright/test/basic.test.ts +++ b/examples/playwright/test/basic.test.ts @@ -20,9 +20,10 @@ describe.runIf(process.platform !== 'win32')('basic', async () => { }) afterAll(async () => { - await browser.close() + // hook timed out and we already have another error + await browser?.close() await new Promise((resolve, reject) => { - server.httpServer.close(error => error ? reject(error) : resolve()) + server?.httpServer.close(error => error ? reject(error) : resolve()) }) }) diff --git a/packages/browser/src/client/runner.ts b/packages/browser/src/client/runner.ts index 63a9be827640..1725b93883d2 100644 --- a/packages/browser/src/client/runner.ts +++ b/packages/browser/src/client/runner.ts @@ -55,7 +55,13 @@ export function createBrowserRunner( } } - onCollected = (files: File[]): unknown => { + onCollected = async (files: File[]): Promise => { + if (this.config.includeTaskLocation) { + try { + await updateFilesLocations(files) + } + catch (_) {} + } return rpc().onCollected(files) } @@ -107,3 +113,28 @@ export async function initiateRunner() { cachedRunner = runner return runner } + +async function updateFilesLocations(files: File[]) { + const { loadSourceMapUtils } = await importId('vitest/utils') as typeof import('vitest/utils') + const { TraceMap, originalPositionFor } = await loadSourceMapUtils() + + const promises = files.map(async (file) => { + const result = await rpc().getBrowserFileSourceMap(file.filepath) + if (!result) + return null + const traceMap = new TraceMap(result as any) + function updateLocation(task: Task) { + if (task.location) { + const { line, column } = originalPositionFor(traceMap, task.location) + if (line != null && column != null) + task.location = { line, column: column + 1 } + } + if ('tasks' in task) + task.tasks.forEach(updateLocation) + } + file.tasks.forEach(updateLocation) + return null + }) + + await Promise.all(promises) +} diff --git a/packages/browser/src/client/tester.ts b/packages/browser/src/client/tester.ts index 1e926157ed80..88a842b207d9 100644 --- a/packages/browser/src/client/tester.ts +++ b/packages/browser/src/client/tester.ts @@ -146,7 +146,7 @@ async function runTests(files: string[]) { preparedData = await tryCall(() => prepareTestEnvironment(files)) } catch (error) { - debug('data cannot be loaded becuase it threw an error') + debug('data cannot be loaded because it threw an error') await client.rpc.onUnhandledError(serializeError(error), 'Preload Error') done(files) return diff --git a/packages/browser/src/node/index.ts b/packages/browser/src/node/index.ts index 0f25198e7b4f..0da28819870c 100644 --- a/packages/browser/src/node/index.ts +++ b/packages/browser/src/node/index.ts @@ -149,6 +149,7 @@ export default (project: WorkspaceProject, base = '/'): Plugin[] => { include: [ 'vitest > @vitest/utils > pretty-format', 'vitest > @vitest/snapshot > pretty-format', + 'vitest > @vitest/snapshot > magic-string', 'vitest > diff-sequences', 'vitest > pretty-format', 'vitest > pretty-format > ansi-styles', diff --git a/packages/coverage-istanbul/src/provider.ts b/packages/coverage-istanbul/src/provider.ts index 50c25c4c45e6..7b4676dc62e9 100644 --- a/packages/coverage-istanbul/src/provider.ts +++ b/packages/coverage-istanbul/src/provider.ts @@ -253,13 +253,16 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co } async getCoverageMapForUncoveredFiles(coveredFiles: string[]) { - // Load, instrument and collect empty coverages from all files which - // are not already in the coverage map - const includedFiles = await this.testExclude.glob(this.ctx.config.root) + const allFiles = await this.testExclude.glob(this.ctx.config.root) + let includedFiles = allFiles.map(file => resolve(this.ctx.config.root, file)) + + if (this.ctx.config.changed) + includedFiles = (this.ctx.config.related || []).filter(file => includedFiles.includes(file)) + const uncoveredFiles = includedFiles - .map(file => resolve(this.ctx.config.root, file)) .filter(file => !coveredFiles.includes(file)) + const cacheKey = new Date().getTime() const coverageMap = libCoverage.createCoverageMap({}) // Note that these cannot be run parallel as synchronous instrumenter.lastFileCoverage @@ -267,13 +270,8 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co for (const [index, filename] of uncoveredFiles.entries()) { debug('Uncovered file %s %d/%d', filename, index, uncoveredFiles.length) - // Make sure file is not served from cache - // so that instrumenter loads up requested file coverage - if (this.ctx.vitenode.fetchCache.has(filename)) - this.ctx.vitenode.fetchCache.delete(filename) - - await this.ctx.vitenode.transformRequest(filename) - + // Make sure file is not served from cache so that instrumenter loads up requested file coverage + await this.ctx.vitenode.transformRequest(`${filename}?v=${cacheKey}`) const lastCoverage = this.instrumenter.lastFileCoverage() coverageMap.addFileCoverage(lastCoverage) } diff --git a/packages/coverage-v8/package.json b/packages/coverage-v8/package.json index 569920243152..d01b01765276 100644 --- a/packages/coverage-v8/package.json +++ b/packages/coverage-v8/package.json @@ -55,6 +55,7 @@ "magicast": "^0.3.3", "picocolors": "^1.0.0", "std-env": "^3.5.0", + "strip-literal": "^2.0.0", "test-exclude": "^6.0.0", "v8-to-istanbul": "^9.2.0" }, diff --git a/packages/coverage-v8/src/provider.ts b/packages/coverage-v8/src/provider.ts index 517f446618cb..3f43c74ec131 100644 --- a/packages/coverage-v8/src/provider.ts +++ b/packages/coverage-v8/src/provider.ts @@ -14,6 +14,7 @@ import remapping from '@ampproject/remapping' import { normalize, resolve } from 'pathe' import c from 'picocolors' import { provider } from 'std-env' +import { stripLiteral } from 'strip-literal' import createDebug from 'debug' import { cleanUrl } from 'vite-node/utils' import type { EncodedSourceMap, FetchResult } from 'vite-node' @@ -245,9 +246,14 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage private async getUntestedFiles(testedFiles: string[]): Promise { const transformResults = normalizeTransformResults(this.ctx.vitenode.fetchCache) - const includedFiles = await this.testExclude.glob(this.ctx.config.root) + const allFiles = await this.testExclude.glob(this.ctx.config.root) + let includedFiles = allFiles.map(file => resolve(this.ctx.config.root, file)) + + if (this.ctx.config.changed) + includedFiles = (this.ctx.config.related || []).filter(file => includedFiles.includes(file)) + const uncoveredFiles = includedFiles - .map(file => pathToFileURL(resolve(this.ctx.config.root, file))) + .map(file => pathToFileURL(file)) .filter(file => !testedFiles.includes(file.pathname)) let merged: RawCoverage = { result: [] } @@ -260,7 +266,13 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage } const coverages = await Promise.all(chunk.map(async (filename) => { - const { source } = await this.getSources(filename.href, transformResults) + const transformResult = await this.ctx.vitenode.transformRequest(filename.pathname).catch(() => {}) + + // Ignore empty files, e.g. files that contain only typescript types and no runtime code + if (transformResult && stripLiteral(transformResult.code).trim() === '') + return null + + const { originalSource } = await this.getSources(filename.href, transformResults) const coverage = { url: filename.href, @@ -269,7 +281,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage functions: [{ ranges: [{ startOffset: 0, - endOffset: source.length, + endOffset: originalSource.length, count: 0, }], isBlockCoverage: true, @@ -281,7 +293,10 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage return { result: [coverage] } })) - merged = mergeProcessCovs([merged, ...coverages]) + merged = mergeProcessCovs([ + merged, + ...coverages.filter((cov): cov is NonNullable => cov != null), + ]) } return merged @@ -289,7 +304,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage private async getSources(url: string, transformResults: TransformResults, functions: Profiler.FunctionCoverage[] = []): Promise<{ source: string - originalSource?: string + originalSource: string sourceMap?: { sourcemap: EncodedSourceMap } }> { const filePath = normalize(fileURLToPath(url)) @@ -306,8 +321,16 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage }) // These can be uncovered files included by "all: true" or files that are loaded outside vite-node - if (!map) - return { source: code || sourcesContent } + if (!map) { + return { + source: code || sourcesContent, + originalSource: sourcesContent, + } + } + + const sources = [url] + if (map.sources && map.sources[0] && !url.endsWith(map.sources[0])) + sources[0] = new URL(map.sources[0], url).href return { originalSource: sourcesContent, @@ -316,7 +339,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage sourcemap: excludeGeneratedCode(code, { ...map, version: 3, - sources: [url], + sources, sourcesContent: [sourcesContent], }), }, diff --git a/packages/runner/src/collect.ts b/packages/runner/src/collect.ts index a32501b67a2c..0a9a69bb540d 100644 --- a/packages/runner/src/collect.ts +++ b/packages/runner/src/collect.ts @@ -28,7 +28,7 @@ export async function collectTests(paths: string[], runner: VitestRunner): Promi projectName: config.name, } - clearCollectorContext(runner) + clearCollectorContext(filepath, runner) try { const setupStart = now() diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 4087ab3b29fb..5155621fb129 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -393,7 +393,7 @@ export async function startTests(paths: string[], runner: VitestRunner) { const files = await collectTests(paths, runner) - runner.onCollected?.(files) + await runner.onCollected?.(files) await runner.onBeforeRunFiles?.(files) await runFiles(files, runner) diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index e3ae628cbfce..0f5feb6b787d 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -1,4 +1,5 @@ import { format, isObject, objDisplay, objectAttr } from '@vitest/utils' +import { parseSingleStack } from '@vitest/utils/source-map' import type { Custom, CustomAPI, File, Fixtures, RunMode, Suite, SuiteAPI, SuiteCollector, SuiteFactory, SuiteHooks, Task, TaskCustomOptions, Test, TestAPI, TestFunction, TestOptions } from './types' import type { VitestRunner } from './types/runner' import { createChainable } from './utils/chain' @@ -25,19 +26,25 @@ export const it = test let runner: VitestRunner let defaultSuite: SuiteCollector +let currentTestFilepath: string export function getDefaultSuite() { return defaultSuite } +export function getTestFilepath() { + return currentTestFilepath +} + export function getRunner() { return runner } -export function clearCollectorContext(currentRunner: VitestRunner) { +export function clearCollectorContext(filepath: string, currentRunner: VitestRunner) { if (!defaultSuite) defaultSuite = currentRunner.config.sequence.shuffle ? suite.shuffle('') : currentRunner.config.sequence.concurrent ? suite.concurrent('') : suite('') runner = currentRunner + currentTestFilepath = filepath collectorContext.tasks.length = 0 defaultSuite.clear() collectorContext.currentSuite = defaultSuite @@ -103,7 +110,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m let suite: Suite - initSuite() + initSuite(true) const task = function (name = '', options: TaskCustomOptions = {}) { const task: Custom = { @@ -140,6 +147,17 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m )) } + if (runner.config.includeTaskLocation) { + const limit = Error.stackTraceLimit + // custom can be called from any place, let's assume the limit is 10 stacks + Error.stackTraceLimit = 10 + const error = new Error('stacktrace').stack! + Error.stackTraceLimit = limit + const stack = findStackTrace(error) + if (stack) + task.location = stack + } + tasks.push(task) return task } @@ -183,7 +201,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m getHooks(suite)[name].push(...fn as any) } - function initSuite() { + function initSuite(includeLocation: boolean) { if (typeof suiteOptions === 'number') suiteOptions = { timeout: suiteOptions } @@ -199,13 +217,27 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m projectName: '', } + if (runner && includeLocation && runner.config.includeTaskLocation) { + const limit = Error.stackTraceLimit + Error.stackTraceLimit = 5 + const error = new Error('stacktrace').stack! + Error.stackTraceLimit = limit + const stack = parseSingleStack(error.split('\n')[5]) + if (stack) { + suite.location = { + line: stack.line, + column: stack.column, + } + } + } + setHooks(suite, createSuiteHooks()) } function clear() { tasks.length = 0 factoryQueue.length = 0 - initSuite() + initSuite(false) } async function collect(file?: File) { @@ -397,3 +429,18 @@ function formatTemplateString(cases: any[], args: any[]): any[] { } return res } + +function findStackTrace(error: string) { + // first line is the error message + // and the first 3 stacks are always from the collector + const lines = error.split('\n').slice(4) + for (const line of lines) { + const stack = parseSingleStack(line) + if (stack && stack.file === getTestFilepath()) { + return { + line: stack.line, + column: stack.column, + } + } + } +} diff --git a/packages/runner/src/types/runner.ts b/packages/runner/src/types/runner.ts index 6822c4903c1c..ce4d18f7a942 100644 --- a/packages/runner/src/types/runner.ts +++ b/packages/runner/src/types/runner.ts @@ -33,6 +33,7 @@ export interface VitestRunnerConfig { testTimeout: number hookTimeout: number retry: number + includeTaskLocation?: boolean diffOptions?: DiffOptions } diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index e2ceddfe7a50..afb822cb1e4f 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -18,6 +18,10 @@ export interface TaskBase { result?: TaskResult retry?: number repeats?: number + location?: { + line: number + column: number + } } export interface TaskPopulated extends TaskBase { diff --git a/packages/ui/client/components/views/ViewReportError.vue b/packages/ui/client/components/views/ViewReportError.vue index 9a9bf01d96a6..5a1de0a9fe2c 100644 --- a/packages/ui/client/components/views/ViewReportError.vue +++ b/packages/ui/client/components/views/ViewReportError.vue @@ -1,6 +1,7 @@