From 2bdd9dc9692f04276d1809f0917af3342fc45ce7 Mon Sep 17 00:00:00 2001 From: Kyle Gach Date: Mon, 19 Aug 2024 23:00:11 -0600 Subject: [PATCH] WIP: Docs for Test runner with Vitest - Includes sidebar order updates - Use non-zero-indexed orders --- .../portable-stories-jest.mdx | 2 +- .../portable-stories-playwright.mdx | 2 +- .../portable-stories-vitest.mdx | 2 +- docs/writing-tests/accessibility-testing.mdx | 2 +- docs/writing-tests/addon-vitest.mdx | 346 ++++++++++++++++++ .../import-stories-in-tests/index.mdx | 2 +- .../stories-in-end-to-end-tests.mdx | 2 +- .../stories-in-unit-tests.mdx | 2 +- docs/writing-tests/interaction-testing.mdx | 2 +- docs/writing-tests/snapshot-testing/index.mdx | 2 +- .../snapshot-testing/snapshot-testing.mdx | 2 + .../storyshots-migration-guide.mdx | 2 + docs/writing-tests/test-coverage.mdx | 2 +- docs/writing-tests/visual-testing.mdx | 2 +- 14 files changed, 361 insertions(+), 11 deletions(-) create mode 100644 docs/writing-tests/addon-vitest.mdx diff --git a/docs/api/portable-stories/portable-stories-jest.mdx b/docs/api/portable-stories/portable-stories-jest.mdx index 64038d44dd7a..209aa2942e49 100644 --- a/docs/api/portable-stories/portable-stories-jest.mdx +++ b/docs/api/portable-stories/portable-stories-jest.mdx @@ -2,7 +2,7 @@ title: 'Portable stories in Jest' sidebar: title: Jest - order: 1 + order: 2 --- diff --git a/docs/api/portable-stories/portable-stories-playwright.mdx b/docs/api/portable-stories/portable-stories-playwright.mdx index 2cbae20f3dd9..a2becdc78b74 100644 --- a/docs/api/portable-stories/portable-stories-playwright.mdx +++ b/docs/api/portable-stories/portable-stories-playwright.mdx @@ -2,7 +2,7 @@ title: 'Portable stories in Playwright CT' sidebar: title: Playwright - order: 2 + order: 3 --- (⚠️ **Experimental**) diff --git a/docs/api/portable-stories/portable-stories-vitest.mdx b/docs/api/portable-stories/portable-stories-vitest.mdx index 11aa9b99475d..281d18b1adf9 100644 --- a/docs/api/portable-stories/portable-stories-vitest.mdx +++ b/docs/api/portable-stories/portable-stories-vitest.mdx @@ -2,7 +2,7 @@ title: 'Portable stories in Vitest' sidebar: title: Vitest - order: 0 + order: 1 --- diff --git a/docs/writing-tests/accessibility-testing.mdx b/docs/writing-tests/accessibility-testing.mdx index 5942811bfb27..2c2eebbe506a 100644 --- a/docs/writing-tests/accessibility-testing.mdx +++ b/docs/writing-tests/accessibility-testing.mdx @@ -1,7 +1,7 @@ --- title: 'Accessibility tests' sidebar: - order: 3 + order: 4 title: Accessibility tests --- diff --git a/docs/writing-tests/addon-vitest.mdx b/docs/writing-tests/addon-vitest.mdx new file mode 100644 index 000000000000..139f9e6d170e --- /dev/null +++ b/docs/writing-tests/addon-vitest.mdx @@ -0,0 +1,346 @@ +--- +title: 'Test runner with Vitest' +sidebar: + order: 2 + title: Test runner with Vitest +--- + +TK - Intro + +## Set up + +To get started, run the following command to install and configure the addon: + +{/* TODO: Snippetize */} +```sh +npx storybook@latest add @storybook/experimental-addon-vitest +``` + +That command will do the following: + +1. Install and register the Vitest addon, which contains the plugin to run your stories as tests +1. Inspect your project's Vite and Vitest setup + 1. If Vite is not installed and you're using the [`nextjs` framework](../get-started/frameworks/nextjs.mdx), it will install and configure Vite for you, as well as `vite-plugin-storybook-nextjs` (necessary to have your Next.js components function in Vitest). + 1. Otherwise, if Vite is not installed, it will stop and point you to these instructions to continue setting it up in your project. + 1. If Vite is installed, it will then check for Vitest. + 1. If Vitest is not installed, it will: + 1. Install `vitest`, `@vitest/browser`, and `playwright` + 1. Run `npx playwright install chromium` to install the Chromium browser engine + 1. Create a Vitest config file (`vitest.config.ts`) and a Vitest setup file (`storybook.setup.ts`) + 1. If Vitest is installed, it will stop and point you to these instructions to continue setting it up in your project. + + + + If your stories use template-based Vue components, you may need to [alias the `vue` module](https://vuejs.org/guide/scaling-up/tooling#note-on-in-browser-template-compilation) to resolve correctly in the Playwright CT environment. You can do this via the [`ctViteConfig` property](https://playwright.dev/docs/test-components#i-have-a-project-that-already-uses-vite-can-i-reuse-the-config): + +
+ Example Playwright configuration + +```ts +// playwright-config.ts +import { defineConfig } from '@playwright/experimental-ct-vue'; + +export default defineConfig({ + ctViteConfig: { + resolve: { + alias: { + vue: 'vue/dist/vue.esm-bundler.js', + }, + }, + }, +}); +``` +
+ + + +The configuration produced by the `add` command will attempt to set some sensible defaults for your project. However, you may need to adjust the configuration to fit your project's needs. The full configuration options can be found in the [API section](#options), below. + +### Example configuration files + +Here are configuration files generated by the `add` command. You can use these as a reference when setting up your project. + +
+ Example Vitest setup file + +```ts title="storybook.setup.ts" +TK +``` +
+ +
+ Example Vitest config file + +```ts title="vitest.config.ts" +import { defineConfig, mergeConfig } from 'vitest/config' +import viteConfig from '../vite.config' +import { storybookTest } from '@storybook/experimental-addon-vitest/plugin' + +export default mergeConfig( + viteConfig, + defineConfig({ + // ... TK + }) +) +``` +
+ +
+ Example Vitest workspace file + +{/* TODO: Nextjs & SvelteKit examples */} +```ts title="vitest.workspace.ts" +import { defineWorkspace } from 'vitest/config' +import { storybookTest } from '@storybook/experimental-addon-vitest/plugin' + +export default defineWorkspace([ + // This is the path to your existing Vitest config files + './vitest.config.ts', + { + name: 'storybook', + plugins: [ + storybookTest({ + storybookScript: 'yarn storybook --ci', + }), + ], + // Glob pattern to find story files + include: ['../src/**/*.stories.?(m)[jt]s?(x)'], + // Enable browser mode + browser: { + enabled: true, + name: 'chromium', + // Make sure to install Playwright + provider: 'playwright', + headless: true, + }, + setupFiles: ['./storybook.setup.ts'], + } +]) +``` +
+ +## How it works + +Before running tests using the plugin, it's helpful to understand how it works. + +First, the plugin does not need to run or build Storybook to test your stories. Instead, it transforms your stories into tests using Vite and [portable stories](../api/portable-stories/portable-stories-vitest.mdx). Portable stories are a mechanism to compose all of a story's configuration ([parameters](../writing-stories/parameters.mdx), [decorators](../writing-stories/decorators.mdx), etc.) with the story itself. This allows you to run your stories as tests without needing to run Storybook. + +Those tests are then run using Vitest. We recommend (and configure, by default) running Vitest in browser mode, using Playwright's Chromium browser. Browser mode ensures your components are tested in a real browser environment, which is more accurate than simulations like JSDom or HappyDom. This is especially important for testing components that rely on browser APIs or features. + +Stories are tested in two ways: a smoke test to ensure it renders and, if a [play function](../writing-stories/play-function.mdx) is defined, that function is run and any [assertions made](../writing-tests/interaction-testing.mdx#assert-tests-with-vitests-apis) within it are validated. + +### Debugging + +While the plugin does not require Storybook to run when testing, you may still want to run Storybook to debug your tests. To enable this, provide the [`storybookScript` option](#storybookscript) in the plugin configuration. When you run Vitest in watch mode, the plugin will start Storybook using this script and provide links to the story in the output on test failures. This allows you to quickly jump to the story in Storybook to debug the issue. + +You can also provide a [`storybookUrl` option](#storybookurl) to the plugin configuration. When you're not using watch mode and tests fail, the plugin will provide a link to the story using this URL in the output. This is useful when [running tests in CI](#in-ci) or other environments where Storybook is not already running. + +TK - Screenshot of test output with links to SB + +## Usage + +There are three primary ways to run tests using the Vitest plugin: + +### CLI + +The plugin transforms your stories into real Vitest tests, so you run those tests just like you run any other Vitest tests in your project. Typically, you will have a `test` script in your `package.json` that runs Vitest. When you run that script, the addon will find and run your story-based tests. Here's an example of running your tests (in [watch mode](https://vitest.dev/guide/cli.html#vitest-watch), by default) using the Vitest CLI: + +{/* TODO: Snippetize */} +```sh +npm run test +``` + +### Editor extension + +Transforming your stories into Vitest tests with the plugin also enables you to run and debug tests using Vitest [IDE integrations](https://vitest.dev/guide/ide.html). This allows you to run tests directly from your editor, such as VSCode and JetBrains IDE. + +TK - Screenshot of VS Code + +### In CI + +For the most part, running your Storybook tests in CI is done [via the CLI](#cli). However, to have the test output link to your published Storybook on test failures, you need to provide the [`storybookUrl` option](#storybookurl) in the plugin configuration. + +Here's an example using GitHub Actions. The steps are similar for other CI providers, though details in the syntax or configuration may vary. + +First, we run a command to build and publish Storybook. In this case, we'll use Chromatic. This gives us a URL to the published Storybook instance. We then pass that URL to the plugin configuration using an environment variable. Finally, we update the plugin configuration to use that environment variable in the `storybookUrl` option. + +```yaml +TK +``` + +```js title="vitest.workspace.ts" +process.env.SB_URL +``` + +## Configuration + +Most of the configuration for the Vitest plugin's behavior is done in the Vitest configuration files. However, you can also define configuration in your stories themselves, using [tags](../writing-stories/tags.mdx), to control how they are tested. + +In this example, the Default story will not be tested, and the Primary story will. + +{/* TODO: Snippetize */} +```js title="Button.stories.tsx" +import { Button } from './Button' + +export default { + component: Button, + // 👇 Apply `test` tag to all stories in this file + tags: ['test'], +} + +export const Default = { + // 👇 Remove `test` tag from this story + tags: ['!test'], +} + +export const Primary = { + args: { primary: true } +} +``` + +By default, the plugin will run all stories with the `test` tag. You can adjust this behavior by providing the [`tags` option](#tags) in the plugin configuration. This allows you to include, exclude, or skip stories based on their tags. + +Here's an example of how you might configure the plugin to only run stories with the `test` and `spec` tags, while excluding stories with the `docs-only` tag: + +{/* TODO: Snippetize */} +```js title="vitest.workspace.ts" +{ + plugins: [ + storybookTest({ + tags: { + include: ['test', 'spec'], + exclude: ['docs-only'], + }, + }), + ], +} +``` + +If the same tag is in both the `include` and `exclude` arrays, the `exclude` behavior takes precedence. + +## FAQ + +### How to ensure my tests can find assets in the public directory? + +If your stories use assets in the public directory and you're not using the default public directory location (`public`), you need to adjust the Vitest configuration to include the public directory. You can do this by providing the `publicDir` option in the Vitest configuration file. + +```ts +TK +``` + +### How to debug my tests in Storybook? + +The plugin will attempt to provide links to the story in Storybook when tests fail, for [debugging](#debugging) purposes. + +If the URLs are not working when running tests in watch mode, you should check two configuration options: + +- [`storybookUrl`](#storybookurl): Ensure this URL is correct and accessible. For example, the default is `http://localhost:6006`, which may not use the same port number you're using. +- [`storybookScript`](#storybookscript): Ensure this script is correctly starting Storybook. + +If the URLs are not working when running tests in CI, you should ensure the Storybook is built and published before running the tests. You can then provide the URL to the published Storybook using the `storybookUrl` option. See the [In CI](#in-ci) section for an example. + +### How to apply custom Vite configuration? + +If you have custom operations defined in [`viteFinal`](../api/main-config/main-config-vite-final.mdx) in your `.storybook/main.js|ts` file, you will need to translate those into the Vitest configuration. This is because the plugin does not use the Storybook Vite configuration. + +```ts +TK +``` + +### Why do we recommend browser mode? + +``` +1. It’s a real browser environment. JSDom/HappyDom are simulations with shortcomings. +2. https://vitest.dev/guide/browser/#motivation +``` + +### How to use WebDriver instead of Playwright? + +``` +https://vitest.dev/config/#browser-provider +``` + +### How to use a browser other than Chromium + +``` +https://vitest.dev/config/#browser-46-name +``` + +### How is this different from the test runner? + +``` +1. TR requires an SB instance to be running; this does not (except for debugging) +2. TR runs SB and listens to results; this transforms stories (using portable stories) into tests +3. TR is based on Jest; this is based on Vitest +4. This is more configurable and more simple than TR + 1. TR is always a separate command; this is just Vitest (`yarn test`) +5. This is faster than TR + 1. Needs benchmarks + 1. a sandbox run in our monorepo + 1. Vitest plugin: 1m 6s + 2. Test-runner: 1m 14s + SB build & publish time +``` + +### Why does the `add` command stop in some cases? + +TK + +## API + +### Exports + +`@storybook/experimental-addon-vitest/plugin` + +TK + +### Options + +#### `configDir` + +Type: `string` + +Default: `.storybook` + +The directory where the Storybook configuration is located, relative to CWD. If not provided, the plugin will use `.storybook` in the current working directory. + +#### `storybookScript` + +Type: `string` + +Optional script to run Storybook. If provided, Vitest will start Storybook using this script when run in watch mode. Only runs if the Storybook in `storybookUrl` is not already available. + +#### `storybookUrl` + +Type: `string` + +Default: `http://localhost:6006` + +The URL where Storybook is hosted. This is used for internal checks and to provide a link to the story in the test output on failures. + +#### `tags` + +Type: + +```ts +{ + include: string[]; + exclude: string[]; + skip: string[]; +} +``` + +Default: + +```ts +{ + include: ['test'], + exclude: [], + skip: [], +} +``` + +Tags to include, exclude, or skip. These tags are defined as annotations in your story, meta, or preview. + +- `include`: `string[]` - Tags to include. +- `exclude`: `string[]` - Tags to exclude. +- `skip`: `string[]` - Tags to skip. diff --git a/docs/writing-tests/import-stories-in-tests/index.mdx b/docs/writing-tests/import-stories-in-tests/index.mdx index c845f373267b..dc51d8da2e83 100644 --- a/docs/writing-tests/import-stories-in-tests/index.mdx +++ b/docs/writing-tests/import-stories-in-tests/index.mdx @@ -1,6 +1,6 @@ --- title: Import stories in tests sidebar: - order: 7 + order: 8 title: Import stories in tests --- \ No newline at end of file diff --git a/docs/writing-tests/import-stories-in-tests/stories-in-end-to-end-tests.mdx b/docs/writing-tests/import-stories-in-tests/stories-in-end-to-end-tests.mdx index 77167db0e88a..59997b7a38ee 100644 --- a/docs/writing-tests/import-stories-in-tests/stories-in-end-to-end-tests.mdx +++ b/docs/writing-tests/import-stories-in-tests/stories-in-end-to-end-tests.mdx @@ -2,7 +2,7 @@ title: 'Stories in end-to-end tests' sidebar: title: End-to-end tests - order: 1 + order: 2 --- Storybook seamlessly integrates with additional testing frameworks like [Cypress](https://www.cypress.io/) and [Playwright](https://playwright.dev/) to provide a comprehensive testing solution. By leveraging the Component Story Format (CSF), developers can write test cases that simulate user interactions and verify the behavior of individual components within the Storybook environment. This approach enables developers to thoroughly test their components' functionality, responsiveness, and visual appearance across different scenarios, resulting in more robust and reliable applications. diff --git a/docs/writing-tests/import-stories-in-tests/stories-in-unit-tests.mdx b/docs/writing-tests/import-stories-in-tests/stories-in-unit-tests.mdx index 7d16abf31b5f..67ab0e6585b2 100644 --- a/docs/writing-tests/import-stories-in-tests/stories-in-unit-tests.mdx +++ b/docs/writing-tests/import-stories-in-tests/stories-in-unit-tests.mdx @@ -2,7 +2,7 @@ title: 'Stories in unit tests' sidebar: title: Unit tests - order: 0 + order: 1 --- Teams test a variety of UI characteristics using different tools. Each tool requires you to replicate the same component state over and over. That’s a maintenance headache. Ideally, you’d set up your tests similarly and reuse that across tools. diff --git a/docs/writing-tests/interaction-testing.mdx b/docs/writing-tests/interaction-testing.mdx index a2ee5908316e..dee9c7409b11 100644 --- a/docs/writing-tests/interaction-testing.mdx +++ b/docs/writing-tests/interaction-testing.mdx @@ -1,7 +1,7 @@ --- title: 'Interaction tests' sidebar: - order: 4 + order: 5 title: Interaction tests --- diff --git a/docs/writing-tests/snapshot-testing/index.mdx b/docs/writing-tests/snapshot-testing/index.mdx index 6ec310332af2..4ce4b3630102 100644 --- a/docs/writing-tests/snapshot-testing/index.mdx +++ b/docs/writing-tests/snapshot-testing/index.mdx @@ -1,6 +1,6 @@ --- title: Snapshot testing sidebar: - order: 6 + order: 7 title: Snapshot testing --- \ No newline at end of file diff --git a/docs/writing-tests/snapshot-testing/snapshot-testing.mdx b/docs/writing-tests/snapshot-testing/snapshot-testing.mdx index bc9ca5b12225..7565090aff5a 100644 --- a/docs/writing-tests/snapshot-testing/snapshot-testing.mdx +++ b/docs/writing-tests/snapshot-testing/snapshot-testing.mdx @@ -1,5 +1,7 @@ --- title: 'Write snapshot tests' +sidebar: + order: 1 --- Snapshot tests compare the rendered markup of every story against known baselines. It’s a way to identify markup changes that trigger rendering errors and warnings. diff --git a/docs/writing-tests/snapshot-testing/storyshots-migration-guide.mdx b/docs/writing-tests/snapshot-testing/storyshots-migration-guide.mdx index 7c4e5ec96847..18c2b5022be3 100644 --- a/docs/writing-tests/snapshot-testing/storyshots-migration-guide.mdx +++ b/docs/writing-tests/snapshot-testing/storyshots-migration-guide.mdx @@ -1,5 +1,7 @@ --- title: 'Storyshots migration guide' +sidebar: + order: 2 --- diff --git a/docs/writing-tests/test-coverage.mdx b/docs/writing-tests/test-coverage.mdx index 63ae072264eb..0d09d2033565 100644 --- a/docs/writing-tests/test-coverage.mdx +++ b/docs/writing-tests/test-coverage.mdx @@ -1,7 +1,7 @@ --- title: 'Test coverage' sidebar: - order: 5 + order: 6 title: Test coverage --- diff --git a/docs/writing-tests/visual-testing.mdx b/docs/writing-tests/visual-testing.mdx index 33288d79a7f0..ba061e5f026f 100644 --- a/docs/writing-tests/visual-testing.mdx +++ b/docs/writing-tests/visual-testing.mdx @@ -2,7 +2,7 @@ title: Visual tests hideRendererSelector: true sidebar: - order: 2 + order: 3 title: Visual tests ---