feat(vitest): run typecheck during tests (#4324)

docs/advanced/api.md

@@ -52,7 +52,6 @@ Vitest instance requires the current test mode. It can be either:
 
 - `test` when running runtime tests
 - `benchmark` when running benchmarks
-- `typecheck` when running type tests
 
 ### mode
 
@@ -64,10 +63,6 @@ Test mode will only call functions inside `test` or `it`, and throws an error wh
 
 Benchmark mode calls `bench` functions and throws an error, when it encounters `test` or `it`. This mode uses `benchmark.include` and `benchmark.exclude` options in the config to find benchmark files.
 
-#### typecheck
-
-Typecheck mode doesn't _run_ tests. It only analyses types and gives a summary. This mode uses `typecheck.include` and `typecheck.exclude` options in the config to find files to analyze.
-
 ### start
 
 You can start running tests or benchmarks with `start` method. You can pass an array of strings to filter test files.

docs/config/index.md

@@ -547,7 +547,7 @@ export default defineConfig({
 
 ### poolMatchGlobs
 
-- **Type:** `[string, 'threads' | 'forks' | 'vmThreads'][]`
+- **Type:** `[string, 'threads' | 'forks' | 'vmThreads' | 'typescript'][]`
 - **Default:** `[]`
 - **Version:** Since Vitest 0.29.4
 
@@ -1637,6 +1637,24 @@ Changes the order in which setup files are executed.
 
 Options for configuring [typechecking](/guide/testing-types) test environment.
 
+#### typecheck.enabled
+
+- **Type**: `boolean`
+- **Default**: `false`
+- **CLI**: `--typecheck`, `--typecheck.enabled`
+- **Version**: Since Vitest 1.0.0-beta.3
+
+Enable typechecking alongside your regular tests.
+
+#### typecheck.only
+
+- **Type**: `boolean`
+- **Default**: `false`
+- **CLI**: `--typecheck.only`
+- **Version**: Since Vitest 1.0.0-beta.3
+
+Run only typecheck tests, when typechecking is enabled. When using CLI, this option will automatically enable typechecking.
+
 #### typecheck.checker
 
 - **Type**: `'tsc' | 'vue-tsc' | string`

docs/guide/cli.md

@@ -96,6 +96,9 @@ Run only [benchmark](https://vitest.dev/guide/features.html#benchmarking-experim
 | `--inspect-brk` | Enables Node.js inspector with break |
 | `--bail <number>` | Stop test execution when given number of tests have failed |
 | `--retry <times>` | Retry the test specific number of times if it fails |
+| `--typecheck [options]` | Custom options for typecheck pool. If passed without options, enables typechecking |
+| `--typecheck.enabled` | Enable typechecking alongside tests (default: `false`) |
+| `--typecheck.only` | Run only typecheck tests. This automatically enables typecheck (default: `false`) |
 | `-h, --help` | Display available CLI options |
 
 ::: tip

docs/guide/testing-types.md

@@ -66,12 +66,12 @@ assertType<string>(answr) //
 
 ## Run typechecking
 
-Add this command to your `scripts` section in `package.json`:
+To enabled typechecking, just add `--typecheck` flag to your Vitest command in `package.json`:
 
 ```json
 {
   "scripts": {
-    "typecheck": "vitest typecheck"
+    "test": "vitest --typecheck"
   }
 }
 ```
@@ -80,13 +80,13 @@ Now you can run typecheck:
 
 ```sh
 # npm
-npm run typecheck
+npm run test
 
 # yarn
-yarn typecheck
+yarn test
 
 # pnpm
-pnpm run typecheck
+pnpm run test
 ```
 
 Vitest uses `tsc --noEmit` or `vue-tsc --noEmit`, depending on your configuration, so you can remove these scripts from your pipeline.

packages/runner/src/utils/tasks.ts

@@ -7,8 +7,8 @@ function isAtomTest(s: Task): s is Test | Custom {
 
 export function getTests(suite: Arrayable<Task>): (Test | Custom)[] {
   const tests: (Test | Custom)[] = []
-  const suite_arr = toArray(suite)
-  for (const s of suite_arr) {
+  const arraySuites = toArray(suite)
+  for (const s of arraySuites) {
     if (isAtomTest(s)) {
       tests.push(s)
     }

packages/ui/client/components/Suites.vue

@@ -23,6 +23,7 @@ async function onRunCurrent() {
     <TasksList :tasks="current.tasks" :nested="true">
       <template #header>
         <StatusIcon mx-1 :task="current" />
+        <div v-if="current.type === 'suite' && current.meta.typecheck" i-logos:typescript-icon flex-shrink-0 mr-1 />
         <span data-testid="filenames" font-bold text-sm flex-auto ws-nowrap overflow-hidden truncate>{{ name }}</span>
         <div class="flex text-lg">
           <IconButton

packages/ui/client/components/TaskItem.vue

@@ -22,6 +22,7 @@ const duration = computed(() => {
     hover="bg-active"
   >
     <StatusIcon :task="task" mr-2 />
+    <div v-if="task.type === 'suite' && task.meta.typecheck" i-logos:typescript-icon flex-shrink-0 mr-2 />
     <div flex items-end gap-2 :text="task?.result?.state === 'fail' ? 'red-500' : ''">
       <span text-sm truncate font-light>{{ task.name }}</span>
       <span v-if="typeof duration === 'number'" text="xs" op20 style="white-space: nowrap">

packages/ui/client/components/views/ViewEditor.vue

@@ -72,7 +72,7 @@ function createErrorElement(e: ErrorWithDiff) {
   div.className = 'op80 flex gap-x-2 items-center'
   const pre = document.createElement('pre')
   pre.className = 'c-red-600 dark:c-red-400'
-  pre.textContent = `${' '.repeat(stack.column)}^ ${e?.nameStr}: ${e?.message}`
+  pre.textContent = `${' '.repeat(stack.column)}^ ${e?.nameStr || e.name}: ${e?.message || ''}`
   div.appendChild(pre)
   const span = document.createElement('span')
   span.className = 'i-carbon-launch c-red-600 dark:c-red-400 hover:cursor-pointer min-w-1em min-h-1em'

packages/ui/client/composables/summary.ts

@@ -1,5 +1,5 @@
 import { hasFailedSnapshot } from '@vitest/ws-client'
-import type { Benchmark, Task, Test, TypeCheck } from 'vitest/src'
+import type { Custom, Task, Test } from 'vitest/src'
 import { files, testRunState } from '~/composables/client'
 
 type Nullable<T> = T | null | undefined
@@ -33,7 +33,7 @@ export const testsSkipped = computed(() => testsIgnore.value.filter(f => f.mode
 export const testsTodo = computed(() => testsIgnore.value.filter(f => f.mode === 'todo'))
 export const totalTests = computed(() => testsFailed.value.length + testsSuccess.value.length)
 export const time = computed(() => {
-  const t = getTests(tests.value).reduce((acc, t) => {
+  const t = files.value.reduce((acc, t) => {
     acc += Math.max(0, t.collectDuration || 0)
     acc += Math.max(0, t.setupDuration || 0)
     acc += Math.max(0, t.result?.duration || 0)
@@ -53,10 +53,25 @@ function toArray<T>(array?: Nullable<Arrayable<T>>): Array<T> {
   return [array]
 }
 
-function isAtomTest(s: Task): s is Test | Benchmark | TypeCheck {
-  return (s.type === 'test' || s.type === 'benchmark' || s.type === 'typecheck')
+function isAtomTest(s: Task): s is Test | Custom {
+  return (s.type === 'test' || s.type === 'custom')
 }
 
-function getTests(suite: Arrayable<Task>): (Test | Benchmark | TypeCheck)[] {
-  return toArray(suite).flatMap(s => isAtomTest(s) ? [s] : s.tasks.flatMap(c => isAtomTest(c) ? [c] : getTests(c)))
+function getTests(suite: Arrayable<Task>): (Test | Custom)[] {
+  const tests: (Test | Custom)[] = []
+  const arraySuites = toArray(suite)
+  for (const s of arraySuites) {
+    if (isAtomTest(s)) {
+      tests.push(s)
+    }
+    else {
+      for (const task of s.tasks) {
+        if (isAtomTest(task))
+          tests.push(task)
+        else
+          tests.push(...getTests(task))
+      }
+    }
+  }
+  return tests
 }

packages/ui/package.json

@@ -57,6 +57,7 @@
   },
   "devDependencies": {
     "@faker-js/faker": "^8.0.2",
+    "@iconify-json/logos": "^1.1.37",
     "@testing-library/cypress": "^9.0.0",
     "@types/codemirror": "^5.60.8",
     "@types/d3-force": "^3.0.4",

packages/vitest/src/node/cli-api.ts

@@ -56,6 +56,15 @@ export async function startVitest(
   if (typeof options.browser === 'object' && !('enabled' in options.browser))
     options.browser.enabled = true
 
+  if (typeof options.typecheck === 'boolean')
+    options.typecheck = { enabled: true }
+
+  if (typeof options.typecheck?.only === 'boolean') {
+    options.typecheck ??= {}
+    options.typecheck.only = true
+    options.typecheck.enabled = true
+  }
+
   const ctx = await createVitest(mode, options, viteOverrides)
 
   if (mode === 'test' && ctx.config.coverage.enabled) {

packages/vitest/src/node/cli.ts

@@ -51,6 +51,9 @@ cli
   .option('--bail <number>', 'Stop test execution when given number of tests have failed', { default: 0 })
   .option('--retry <times>', 'Retry the test specific number of times if it fails', { default: 0 })
   .option('--diff <path>', 'Path to a diff config that will be used to generate diff interface')
+  .option('--typecheck [options]', 'Custom options for typecheck pool')
+  .option('--typecheck.enabled', 'Enable typechecking alongside tests (default: false)')
+  .option('--typecheck.only', 'Run only typecheck tests. This automatically enables typecheck (default: false)')
   .help()
 
 cli
@@ -73,10 +76,6 @@ cli
   .command('bench [...filters]')
   .action(benchmark)
 
-cli
-  .command('typecheck [...filters]')
-  .action(typecheck)
-
 cli
   .command('[...filters]')
   .action((filters, options) => start('test', filters, options))
@@ -130,11 +129,6 @@ async function benchmark(cliFilters: string[], options: CliOptions): Promise<voi
   await start('benchmark', cliFilters, options)
 }
 
-async function typecheck(cliFilters: string[] = [], options: CliOptions = {}) {
-  console.warn(c.yellow('Testing types with tsc and vue-tsc is an experimental feature.\nBreaking changes might not follow semver, please pin Vitest\'s version when using it.'))
-  await start('typecheck', cliFilters, options)
-}
-
 function normalizeCliOptions(argv: CliOptions): CliOptions {
   if (argv.root)
     argv.root = normalize(argv.root)

packages/vitest/src/node/config.ts

@@ -378,10 +378,11 @@ export function resolveConfig(
 
   resolved.environmentMatchGlobs = (resolved.environmentMatchGlobs || []).map(i => [resolve(resolved.root, i[0]), i[1]])
 
-  if (mode === 'typecheck') {
-    resolved.include = resolved.typecheck.include
-    resolved.exclude = resolved.typecheck.exclude
-  }
+  resolved.typecheck ??= {} as any
+  resolved.typecheck.enabled ??= false
+
+  if (resolved.typecheck.enabled)
+    console.warn(c.yellow('Testing types with tsc and vue-tsc is an experimental feature.\nBreaking changes might not follow semver, please pin Vitest\'s version when using it.'))
 
   resolved.browser ??= {} as any
   resolved.browser.enabled ??= false

packages/vitest/src/node/core.ts

@@ -72,6 +72,7 @@ export class Vitest {
   }
 
   private _onRestartListeners: OnServerRestartHandler[] = []
+  private _onClose: (() => Awaited<unknown>)[] = []
   private _onSetServer: OnServerRestartHandler[] = []
   private _onCancelListeners: ((reason: CancelReason) => Promise<void> | void)[] = []
 
@@ -93,7 +94,7 @@ export class Vitest {
     this.cache = new VitestCache()
     this.snapshot = new SnapshotManager({ ...resolved.snapshotOptions })
 
-    if (this.config.watch && this.mode !== 'typecheck')
+    if (this.config.watch)
       this.registerWatcher()
 
     this.vitenode = new ViteNodeServer(server, this.config.server)
@@ -308,15 +309,8 @@ export class Vitest {
     return Promise.all(this.projects.map(w => w.initBrowserProvider()))
   }
 
-  typecheck(filters?: string[]) {
-    return Promise.all(this.projects.map(project => project.typecheck(filters)))
-  }
-
   async start(filters?: string[]) {
-    if (this.mode === 'typecheck') {
-      await this.typecheck(filters)
-      return
-    }
+    this._onClose = []
 
     try {
       await this.initCoverageProvider()
@@ -748,7 +742,7 @@ export class Vitest {
 
   async close() {
     if (!this.closingPromise) {
-      const closePromises = this.projects.map(w => w.close().then(() => w.server = undefined as any))
+      const closePromises: unknown[] = this.projects.map(w => w.close().then(() => w.server = undefined as any))
       // close the core workspace server only once
       // it's possible that it's not initialized at all because it's not running any tests
       if (!this.coreWorkspaceProject || !this.projects.includes(this.coreWorkspaceProject))
@@ -757,6 +751,8 @@ export class Vitest {
       if (this.pool)
         closePromises.push(this.pool.close().then(() => this.pool = undefined))
 
+      closePromises.push(...this._onClose.map(fn => fn()))
+
       this.closingPromise = Promise.allSettled(closePromises).then((results) => {
         results.filter(r => r.status === 'rejected').forEach((err) => {
           this.logger.error('error during close', (err as PromiseRejectedResult).reason)
@@ -834,4 +830,8 @@ export class Vitest {
   onCancel(fn: (reason: CancelReason) => void) {
     this._onCancelListeners.push(fn)
   }
+
+  onClose(fn: () => void) {
+    this._onClose.push(fn)
+  }
 }

packages/vitest/src/node/logger.ts

@@ -95,10 +95,21 @@ export class Logger {
     const comma = c.dim(', ')
     if (filters?.length)
       this.console.error(c.dim('filter:  ') + c.yellow(filters.join(comma)))
-    if (config.include)
-      this.console.error(c.dim('include: ') + c.yellow(config.include.join(comma)))
-    if (config.exclude)
-      this.console.error(c.dim('exclude:  ') + c.yellow(config.exclude.join(comma)))
+    this.ctx.projects.forEach((project) => {
+      const config = project.config
+      const name = project.getName()
+      const output = project.isCore() || !name ? '' : `[${name}]`
+      if (output)
+        this.console.error(c.bgCyan(`${output} Config`))
+      if (config.include)
+        this.console.error(c.dim('include: ') + c.yellow(config.include.join(comma)))
+      if (config.exclude)
+        this.console.error(c.dim('exclude:  ') + c.yellow(config.exclude.join(comma)))
+      if (config.typecheck.enabled) {
+        this.console.error(c.dim('typecheck include: ') + c.yellow(config.typecheck.include.join(comma)))
+        this.console.error(c.dim('typecheck exclude: ') + c.yellow(config.typecheck.exclude.join(comma)))
+      }
+    })
     if (config.watchExclude)
       this.console.error(c.dim('watch exclude:  ') + c.yellow(config.watchExclude.join(comma)))
 

packages/vitest/src/node/pool.ts

@@ -9,6 +9,7 @@ import { createThreadsPool } from './pools/threads'
 import { createBrowserPool } from './pools/browser'
 import { createVmThreadsPool } from './pools/vm-threads'
 import type { WorkspaceProject } from './workspace'
+import { createTypecheckPool } from './pools/typecheck'
 
 export type WorkspaceSpec = [project: WorkspaceProject, testFile: string]
 export type RunWithFiles = (files: WorkspaceSpec[], invalidates?: string[]) => Promise<void>
@@ -35,12 +36,20 @@ export function createPool(ctx: Vitest): ProcessPool {
     threads: null,
     browser: null,
     vmThreads: null,
+    typescript: null,
   }
 
-  function getDefaultPoolName(project: WorkspaceProject): Pool {
+  function getDefaultPoolName(project: WorkspaceProject, file: string): Pool {
     if (project.config.browser.enabled)
       return 'browser'
 
+    if (project.config.typecheck.enabled) {
+      for (const glob of project.config.typecheck.include) {
+        if (mm.isMatch(file, glob, { cwd: project.config.root }))
+          return 'typescript'
+      }
+    }
+
     return project.config.pool
   }
 
@@ -51,7 +60,7 @@ export function createPool(ctx: Vitest): ProcessPool {
       if (mm.isMatch(file, glob, { cwd: project.config.root }))
         return pool as Pool
     }
-    return getDefaultPoolName(project)
+    return getDefaultPoolName(project, file)
   }
 
   async function runTests(files: WorkspaceSpec[], invalidate?: string[]) {
@@ -93,6 +102,7 @@ export function createPool(ctx: Vitest): ProcessPool {
       threads: [],
       browser: [],
       vmThreads: [],
+      typescript: [],
     }
 
     for (const spec of files) {
@@ -123,6 +133,11 @@ export function createPool(ctx: Vitest): ProcessPool {
         return pools.threads.runTests(files, invalidate)
       }
 
+      if (pool === 'typescript') {
+        pools.typescript ??= createTypecheckPool(ctx)
+        return pools.typescript.runTests(files)
+      }
+
       pools.forks ??= createChildProcessPool(ctx, options)
       return pools.forks.runTests(files, invalidate)
     }))