feat(ui): show unhandled errors on the ui (#4380)

Co-authored-by: Vladimir <[email protected]>

packages/ui/client/auto-imports.d.ts

@@ -90,6 +90,7 @@ declare global {
   const onUpdated: typeof import('vue')['onUpdated']
   const openInEditor: typeof import('./composables/error')['openInEditor']
   const params: typeof import('./composables/params')['params']
+  const parseError: typeof import('./composables/error')['parseError']
   const pausableWatch: typeof import('@vueuse/core')['pausableWatch']
   const provide: typeof import('vue')['provide']
   const provideLocal: typeof import('@vueuse/core')['provideLocal']

packages/ui/client/components.d.ts

@@ -13,6 +13,7 @@ declare module 'vue' {
     Dashboard: typeof import('./components/Dashboard.vue')['default']
     DashboardEntry: typeof import('./components/dashboard/DashboardEntry.vue')['default']
     DetailsPanel: typeof import('./components/DetailsPanel.vue')['default']
+    ErrorEntry: typeof import('./components/dashboard/ErrorEntry.vue')['default']
     FileDetails: typeof import('./components/FileDetails.vue')['default']
     IconButton: typeof import('./components/IconButton.vue')['default']
     Modal: typeof import('./components/Modal.vue')['default']

packages/ui/client/components/DetailsPanel.vue

@@ -7,7 +7,7 @@ const open = ref(true)
 </script>
 
 <template>
-  <div :open="open" class="details-panel" @toggle="open = $event.target.open">
+  <div :open="open" class="details-panel" data-testid="details-panel" @toggle="open = $event.target.open">
     <div p="y1" text-sm bg-base items-center z-5 gap-2 :class="color" w-full flex select-none sticky top="-1">
       <div flex-1 h-1px border="base b" op80 />
       <slot name="summary" :open="open" />

packages/ui/client/components/dashboard/ErrorEntry.vue

@@ -0,0 +1,42 @@
+<script setup lang="ts">
+  const props = defineProps<{
+    error: ErrorWithDiff
+  }>()
+</script>
+
+<template>
+  <h4 bg="red500/10" p-1 mb-1 mt-2 rounded>
+    <span font-bold>
+      {{ error.name || error.nameStr || 'Unknown Error' }}<template v-if="error.message">:</template>
+    </span>
+    {{ error.message }}
+  </h4>
+  <p v-if="error.stacks?.length" class="scrolls" text="xs" font-mono mx-1 my-2 pb-2 overflow-auto>
+    <span v-for="(frame, i) in error.stacks" whitespace-pre :font-bold="i === 0 ? '' : null">❯ {{ frame.method}} {{ frame.file }}:<span text="red500/70">{{ frame.line }}:{{ frame.column }}</span><br></span>
+  </p>
+  <p v-if="error.VITEST_TEST_PATH" text="sm" mb-2>
+    This error originated in <span font-bold>{{ error.VITEST_TEST_PATH }}</span> test file. It doesn't mean the error was thrown inside the file itself, but while it was running.
+  </p>
+  <p v-if="error.VITEST_TEST_NAME" text="sm" mb-2>
+    The latest test that might've caused the error is <span font-bold>{{ error.VITEST_TEST_NAME }}</span>. It might mean one of the following:<br>
+    <ul>
+      <li>
+        The error was thrown, while Vitest was running this test.
+      </li>
+      <li>
+        If the error occurred after the test had been completed, this was the last documented test before it was thrown.
+      </li>
+    </ul>
+  </p>
+  <p v-if="error.VITEST_AFTER_ENV_TEARDOWN" text="sm" font-thin>
+    This error was caught after test environment was torn down. Make sure to cancel any running tasks before test finishes:<br>
+    <ul>
+      <li>
+        Cancel timeouts using clearTimeout and clearInterval.
+      </li>
+      <li>
+        Wait for promises to resolve using the await keyword.
+      </li>
+    </ul>
+  </p>
+</template>

packages/ui/client/components/dashboard/TestFilesEntry.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { files } from '../../composables/client'
+import { files, unhandledErrors } from '../../composables/client'
 import { filesFailed, filesSnapshotFailed, filesSuccess, time } from '../../composables/summary'
 </script>
 
@@ -44,17 +44,54 @@ import { filesFailed, filesSnapshotFailed, filesSuccess, time } from '../../comp
       </div>
     </template>
 
+    <template v-if="unhandledErrors.length">
+      <div i-carbon-checkmark-outline-error />
+      <div>
+        Errors
+      </div>
+      <div class="number" text-red5>
+        {{ unhandledErrors.length }}
+      </div>
+    </template>
+
     <div i-carbon-timer />
     <div>Time</div>
     <div class="number" data-testid="run-time">
       {{ time }}
     </div>
   </div>
+  <template v-if="unhandledErrors.length">
+    <div bg="red500/10" text="red500" p="x3 y2" max-w-xl m-2 rounded>
+      <h3 text-center mb-2>
+        Unhandled Errors
+      </h3>
+      <p text="sm" font-thin mb-2 data-testid="unhandled-errors">
+        Vitest caught {{ unhandledErrors.length }} error{{ unhandledErrors.length > 1 ? 's' : '' }} during the test run.<br>
+        This might cause false positive tests. Resolve unhandled errors to make sure your tests are not affected.
+      </p>
+      <details
+        data-testid="unhandled-errors-details"
+        class="scrolls unhandled-errors"
+        text="sm" font-thin pe-2.5 open:max-h-52 overflow-auto
+      >
+        <summary font-bold cursor-pointer>Errors</summary>
+        <ErrorEntry v-for="e in unhandledErrors" :error="e" />
+      </details>
+    </div>
+  </template>
 </template>
 
 <style scoped>
 .number {
   font-weight: 400;
   text-align: right;
 }
+
+.unhandled-errors {
+  --cm-ttc-c-thumb: #CCC;
+}
+
+html.dark .unhandled-errors {
+  --cm-ttc-c-thumb: #444;
+}
 </style>

packages/ui/client/composables/client/index.ts

@@ -1,16 +1,18 @@
 import { createClient, getTasks } from '@vitest/ws-client'
 import type { WebSocketStatus } from '@vueuse/core'
-import type { File, ResolvedConfig } from 'vitest'
+import type { ErrorWithDiff, File, ResolvedConfig } from 'vitest'
 import type { Ref } from 'vue'
 import { reactive } from 'vue'
 import type { RunState } from '../../../types'
 import { ENTRY_URL, isReport } from '../../constants'
+import { parseError } from '../error'
 import { activeFileId } from '../params'
 import { createStaticClient } from './static'
 
 export { ENTRY_URL, PORT, HOST, isReport } from '../../constants'
 
 export const testRunState: Ref<RunState> = ref('idle')
+export const unhandledErrors: Ref<ErrorWithDiff[]> = ref([])
 
 export const client = (function createVitestClient() {
   if (isReport) {
@@ -23,8 +25,9 @@ export const client = (function createVitestClient() {
         onTaskUpdate() {
           testRunState.value = 'running'
         },
-        onFinished() {
+        onFinished(_files, errors) {
           testRunState.value = 'idle'
+          unhandledErrors.value = (errors || []).map(parseError)
         },
       },
     })
@@ -70,11 +73,13 @@ watch(
     ws.addEventListener('open', async () => {
       status.value = 'OPEN'
       client.state.filesMap.clear()
-      const [files, _config] = await Promise.all([
+      const [files, _config, errors] = await Promise.all([
         client.rpc.getFiles(),
         client.rpc.getConfig(),
+        client.rpc.getUnhandledErrors(),
       ])
       client.state.collectFiles(files)
+      unhandledErrors.value = (errors || []).map(parseError)
       config.value = _config
     })
 

packages/ui/client/composables/client/static.ts

@@ -1,16 +1,16 @@
 import type { BirpcReturn } from 'birpc'
 import type { VitestClient } from '@vitest/ws-client'
-import type { WebSocketHandlers } from 'vitest/src/api/types'
+import type { File, ModuleGraphData, ResolvedConfig, WebSocketEvents, WebSocketHandlers } from 'vitest'
 import { parse } from 'flatted'
 import { decompressSync, strFromU8 } from 'fflate'
-import type { File, ModuleGraphData, ResolvedConfig } from 'vitest/src/types'
 import { StateManager } from '../../../../vitest/src/node/state'
 
 interface HTMLReportMetadata {
   paths: string[]
   files: File[]
   config: ResolvedConfig
   moduleGraph: Record<string, ModuleGraphData>
+  unhandledErrors: unknown[]
 }
 
 const noop: any = () => {}
@@ -42,6 +42,9 @@ export function createStaticClient(): VitestClient {
     getModuleGraph: async (id) => {
       return metadata.moduleGraph[id]
     },
+    getUnhandledErrors: () => {
+      return metadata.unhandledErrors
+    },
     getTransformResult: async (id) => {
       return {
         code: id,
@@ -66,9 +69,12 @@ export function createStaticClient(): VitestClient {
     saveSnapshotFile: asyncNoop,
     readTestFile: asyncNoop,
     removeSnapshotFile: asyncNoop,
+    onUnhandledError: noop,
+    saveTestFile: asyncNoop,
+    getProvidedContext: () => ({}),
   } as WebSocketHandlers
 
-  ctx.rpc = rpc as any as BirpcReturn<WebSocketHandlers>
+  ctx.rpc = rpc as any as BirpcReturn<WebSocketHandlers, WebSocketEvents>
 
   let openPromise: Promise<void>
 

packages/ui/client/composables/error.ts

@@ -1,4 +1,6 @@
 import Filter from 'ansi-to-html'
+import type { ErrorWithDiff } from 'vitest'
+import { parseStacktrace } from '@vitest/utils/source-map'
 
 export function shouldOpenInEditor(name: string, fileName?: string) {
   return fileName && name.endsWith(fileName)
@@ -15,3 +17,32 @@ export function createAnsiToHtmlFilter(dark: boolean) {
     bg: dark ? '#000' : '#FFF',
   })
 }
+
+function isPrimitive(value: unknown) {
+  return value === null || (typeof value !== 'function' && typeof value !== 'object')
+}
+
+export function parseError(e: unknown) {
+  let error = e as ErrorWithDiff
+
+  if (isPrimitive(e)) {
+    error = {
+      message: String(error).split(/\n/g)[0],
+      stack: String(error),
+      name: '',
+    }
+  }
+
+  if (!e) {
+    const err = new Error('unknown error')
+    error = {
+      message: err.message,
+      stack: err.stack,
+      name: '',
+    }
+  }
+
+  error.stacks = parseStacktrace(error.stack || error.stackStr || '', { ignoreStackEntries: [] })
+
+  return error
+}

packages/ui/client/composables/summary.ts

@@ -1,5 +1,5 @@
 import { hasFailedSnapshot } from '@vitest/ws-client'
-import type { Custom, Task, Test } from 'vitest/src'
+import type { Custom, Task, Test } from 'vitest'
 import { files, testRunState } from '~/composables/client'
 
 type Nullable<T> = T | null | undefined

packages/ui/node/reporter.ts

@@ -28,6 +28,7 @@ interface HTMLReportData {
   files: File[]
   config: ResolvedConfig
   moduleGraph: Record<string, ModuleGraphData>
+  unhandledErrors: unknown[]
 }
 
 const distDir = resolve(fileURLToPath(import.meta.url), '../../dist')
@@ -47,6 +48,7 @@ export default class HTMLReporter implements Reporter {
       paths: this.ctx.state.getPaths(),
       files: this.ctx.state.getFiles(),
       config: this.ctx.config,
+      unhandledErrors: this.ctx.state.getUnhandledErrors(),
       moduleGraph: {},
     }
     await Promise.all(

packages/vitest/src/api/setup.ts

@@ -147,6 +147,9 @@ export function setup(vitestOrWorkspace: Vitest | WorkspaceProject, _server?: Vi
         getProvidedContext() {
           return 'ctx' in vitestOrWorkspace ? vitestOrWorkspace.getProvidedContext() : ({} as any)
         },
+        getUnhandledErrors() {
+          return ctx.state.getUnhandledErrors()
+        },
       },
       {
         post: msg => ws.send(msg),
@@ -206,9 +209,9 @@ class WebSocketReporter implements Reporter {
     })
   }
 
-  onFinished(files?: File[] | undefined) {
+  onFinished(files?: File[], errors?: unknown[]) {
     this.clients.forEach((client) => {
-      client.onFinished?.(files)
+      client.onFinished?.(files, errors)
     })
   }
 

packages/vitest/src/api/types.ts

@@ -31,6 +31,7 @@ export interface WebSocketHandlers {
   rerun(files: string[]): Promise<void>
   updateSnapshot(file?: File): Promise<void>
   getProvidedContext(): ProvidedContext
+  getUnhandledErrors(): unknown[]
 }
 
 export interface WebSocketEvents extends Pick<Reporter, 'onCollected' | 'onFinished' | 'onTaskUpdate' | 'onUserConsoleLog' | 'onPathsCollected'> {

packages/vitest/src/node/error.ts

@@ -102,7 +102,7 @@ export async function printError(error: unknown, project: WorkspaceProject | und
   if (testName) {
     logger.error(c.red(`The latest test that might've caused the error is "${c.bold(testName)}". It might mean one of the following:`
     + '\n- The error was thrown, while Vitest was running this test.'
-    + '\n- This was the last recorded test before the error was thrown, if error originated after test finished its execution.'))
+    + '\n- If the error occurred after the test had been completed, this was the last documented test before it was thrown.'))
   }
   if (afterEnvTeardown) {
     logger.error(c.red('This error was caught after test environment was torn down. Make sure to cancel any running tasks before test finishes:'

packages/ws-client/src/index.ts

@@ -65,8 +65,8 @@ export function createClient(url: string, options: VitestClientOptions = {}) {
     onUserConsoleLog(log) {
       ctx.state.updateUserLog(log)
     },
-    onFinished(files) {
-      handlers.onFinished?.(files)
+    onFinished(files, errors) {
+      handlers.onFinished?.(files, errors)
     },
     onCancel(reason: CancelReason) {
       handlers.onCancel?.(reason)

test/reporters/tests/__snapshots__/html.test.ts.snap

@@ -104,6 +104,7 @@ exports[`html reporter > resolves to "failing" status for test file "json-fail"
   "paths": [
     "<rootDir>/test/reporters/fixtures/json-fail.test.ts",
   ],
+  "unhandledErrors": [],
 }
 `;
 
@@ -229,5 +230,6 @@ exports[`html reporter > resolves to "passing" status for test file "all-passing
   "paths": [
     "<rootDir>/test/reporters/fixtures/all-passing-or-skipped.test.ts",
   ],
+  "unhandledErrors": [],
 }
 `;

test/ui/fixtures/sample.test.ts

@@ -3,5 +3,11 @@ import { expect, it } from 'vitest'
 it('add', () => {
   // eslint-disable-next-line no-console
   console.log('log test')
+  setTimeout(() => {
+    throw new Error('error')
+  })
+  setTimeout(() => {
+    throw 1
+  })
   expect(1 + 1).toEqual(2)
 })

test/ui/test/html-report.spec.ts

@@ -34,8 +34,17 @@ test.describe('html report', () => {
     // dashbaord
     await expect(page.locator('[aria-labelledby=tests]')).toContainText('5 Pass 0 Fail 5 Total')
 
+    // unhandled errors
+    await expect(page.getByTestId('unhandled-errors')).toContainText(
+      'Vitest caught 2 errors during the test run. This might cause false positive tests. '
+      + 'Resolve unhandled errors to make sure your tests are not affected.',
+    )
+
+    await expect(page.getByTestId('unhandled-errors-details')).toContainText('Error: error')
+    await expect(page.getByTestId('unhandled-errors-details')).toContainText('Unknown Error: 1')
+
     // report
-    await page.getByText('sample.test.ts').click()
+    await page.getByTestId('details-panel').getByText('sample.test.ts').click()
     await page.getByText('All tests passed in this file').click()
     await expect(page.getByTestId('filenames')).toContainText('sample.test.ts')