ci: fix flaky loading on visual test runner (#1130)

.eslintignore

@@ -1,7 +1,4 @@
 # Force include
-!.storybook
-!.storybook-docs
-!.playground
 !.*.js
 
 # Ignore

.eslintrc.js

@@ -423,7 +423,7 @@ module.exports = {
       },
     },
     {
-      files: ['.playground/**/*.ts?(x)'],
+      files: ['playground/**/*.ts?(x)'],
       rules: {
         'react/prefer-stateless-function': 0,
       },

global.d.ts

@@ -18,3 +18,43 @@
  */
 
 import 'jest-extended'; // https://github.com/jest-community/jest-extended
+
+declare global {
+  namespace NodeJS {
+    interface ProcessEnv {
+      /**
+       * Node enviroment
+       */
+      NODE_ENV: 'development' | 'production' | 'test';
+      /**
+       * Port used for dev servers including:
+       *  - storybook
+       *  - playground
+       */
+      PORT?: string;
+      /**
+       * Timezone flag used on jest.ts.config.js
+       */
+      TZ: string;
+      /**
+       * Flag used to enable a more suitable version of storybook for visual regression tests.
+       *
+       * Including:
+       * - disabling animations
+       * - preloading icons
+       */
+      STORYBOOK_VRT: string;
+      /**
+       * Flag used to enable debug state on visual regression test runnner
+       */
+      DEBUG: string;
+      /**
+       * String used for seeding a random number generator used in storybook and test files
+       *
+       * When seeded all rng use a deterministic random set of numbers.
+       * When no see is provided a truely _random_ number set will be used.
+       */
+      RNG_SEED: string;
+    }
+  }
+}

integration/helpers.ts

@@ -73,7 +73,6 @@ function encodeString(string: string) {
  */
 const storiesToSkip: Record<string, string[]> = {
   // Interactions: ['Some story name'],
-  Legend: ['Actions'],
   'Test Cases': ['No Series'],
 };
 

integration/page_objects/common.ts

@@ -388,7 +388,6 @@ class CommonPage {
     options?: Omit<ScreenshotElementAtUrlOptions, 'action'>,
   ) {
     const action = async () => {
-      await this.disableAnimations();
       // click to focus within the chart
       await this.clickMouseRelativeToDOMElement({ top: 0, left: 0 }, this.chartSelector);
       // eslint-disable-next-line no-restricted-syntax
@@ -446,12 +445,6 @@ class CommonPage {
     });
   }
 
-  async disableAnimations() {
-    await page.evaluate(() => {
-      document.querySelector('#story-root')!.classList.add('disable-animations');
-    });
-  }
-
   /**
    * Wait for an element to be on the DOM
    *

integration/tests/all.test.ts

@@ -21,18 +21,18 @@ import { getStorybookInfo } from '../helpers';
 import { common } from '../page_objects';
 
 // mock required for importing trick, otherwise .scss files will throw an error
-jest.mock('../../.storybook/theme_service.ts', () => ({
+jest.mock('../../storybook/theme_service.ts', () => ({
   switchTheme: () => undefined,
 }));
 
 const storyGroups = getStorybookInfo();
 
 describe('Baseline Visual tests for all stories', () => {
   describe.each(storyGroups)('%s', (_group, encodedGroup, stories) => {
-    describe.each(stories)('%s', (_title, encodedTitle, delay) => {
+    describe.each(stories)('%s', (_title, encodedTitle) => {
       it('visually looks correct', async () => {
         const url = `http://localhost:9001?id=${encodedGroup}--${encodedTitle}`;
-        await common.expectChartAtUrlToMatchScreenshot(url, { delay });
+        await common.expectChartAtUrlToMatchScreenshot(url);
       });
     });
   });

integration/tests/area_stories.test.ts

@@ -89,7 +89,6 @@ describe('Area series stories', () => {
 
     it('shows only positive values when hiding negative one', async () => {
       const action = async () => {
-        await common.disableAnimations();
         await page.click('.echLegendItem:nth-child(2) .echLegendItem__label');
       };
       await common.expectChartAtUrlToMatchScreenshot(
@@ -100,7 +99,6 @@ describe('Area series stories', () => {
 
     it('shows only negative values when hiding positive one', async () => {
       const action = async () => {
-        await common.disableAnimations();
         await page.click('.echLegendItem:nth-child(1) .echLegendItem__label');
       };
       await common.expectChartAtUrlToMatchScreenshot(

integration/tests/legend_stories.test.ts

@@ -49,7 +49,6 @@ describe('Legend stories', () => {
 
   it('should render color picker on mouse click', async () => {
     const action = async () => {
-      await common.disableAnimations();
       await common.clickMouseRelativeToDOMElement({ left: 0, top: 0 }, '.echLegendItem__color');
     };
     await common.expectElementAtUrlToMatchScreenshot(
@@ -58,19 +57,16 @@ describe('Legend stories', () => {
       {
         action,
         waitSelector: common.chartWaitSelector,
-        delay: 500, // needed for popover animation to complete
       },
     );
   });
 
   it('should render legend action on mouse hover', async () => {
     const action = async () => {
-      await common.disableAnimations();
       await common.moveMouseRelativeToDOMElement({ left: 30, top: 10 }, '.echLegendItem');
     };
     await common.expectChartAtUrlToMatchScreenshot('http://localhost:9001/?path=/story/legend--actions', {
       action,
-      delay: 500, // needed for icon to load
     });
   });
 

integration/tests/test_cases_stories.test.ts

@@ -25,7 +25,6 @@ describe('Test cases stories', () => {
       'http://localhost:9001/?path=/story/test-cases--no-series&knob-Show custom no results=true',
       {
         waitSelector: '.echReactiveChart_noResults .euiIcon:not(.euiIcon-isLoading)',
-        delay: 500, // wait for icon to load
       },
     );
   });

package.json

@@ -33,14 +33,14 @@
     "lint": "NODE_ENV=production eslint --quiet --ext .tsx,.ts,.js .",
     "lint:fix": "yarn lint --fix",
     "prettier:check": "prettier --check \"**/*.{json,html,css,scss}\"",
-    "playground": "cd .playground && webpack-dev-server",
-    "playground:ie": "cd .playground && webpack-dev-server --host=0.0.0.0 --disable-host-check --useLocalIp",
+    "playground": "cd playground && webpack-dev-server",
+    "playground:ie": "cd playground && webpack-dev-server --host=0.0.0.0 --disable-host-check --useLocalIp",
     "pq": "pretty-quick",
     "semantic-release": "semantic-release",
     "start": "yarn storybook",
-    "start-docs": "start-storybook -p 8001 -c .storybook-docs --ci",
-    "storybook": "RNG_SEED='elastic-charts' start-storybook -s ./public -p 9001 -c .storybook --ci --no-version-updates",
-    "storybook:build": "rm -rf .out && build-storybook -s ./public -c .storybook -o .out",
+    "start-docs": "start-storybook -p 8001 -c storybook-docs --ci",
+    "storybook": "STORYBOOK_VRT='true' start-storybook -s ./public -p 9001 -c storybook --ci --no-version-updates",
+    "storybook:build": "rm -rf .out && build-storybook -s ./public -c storybook -o .out",
     "test": "jest --verbose --config jest.config.js",
     "test:tz": "yarn test:tz-utc && yarn test:tz-ny && yarn test:tz-jp",
     "test:tz-utc": "TZ=UTC jest --verbose --config=jest.tz.config.js",

src/mocks/utils.ts

@@ -32,12 +32,19 @@ import { DataGenerator, RandomNumberGenerator } from '../utils/data_generators/d
 export const forcedType = <T extends Record<string, unknown>>(obj: Partial<T>): T => obj as T;
 
 /**
- * Return rng function with optional `min`, `max` and `fractionDigits` params
+ * Returns rng seed from `process.env`
+ * @internal
+ */
+export const getRNGSeed = (fallback?: string): string | undefined =>
+  process.env.RNG_SEED ?? (process.env.STORYBOOK_VRT ? 'elastic-charts' : fallback);
+
+/**
+ * Returns rng function with optional `min`, `max` and `fractionDigits` params
  *
- * @param string process.env.RNG_SEED
+ * @param string seed for deterministic algorithm
  * @internal
  */
-export const getRandomNumberGenerator = (seed = process.env.RNG_SEED): RandomNumberGenerator => {
+export const getRandomNumberGenerator = (seed = getRNGSeed()): RandomNumberGenerator => {
   const rng = seedrandom(seed);
 
   /**

stories/interactions/14_crosshair_time.tsx

@@ -20,7 +20,6 @@
 import { boolean, select } from '@storybook/addon-knobs';
 import React from 'react';
 
-import { switchTheme } from '../../.storybook/theme_service';
 import {
   Axis,
   BarSeries,
@@ -36,6 +35,7 @@ import {
   TooltipType,
 } from '../../src';
 import { KIBANA_METRICS } from '../../src/utils/data_samples/test_dataset_kibana';
+import { switchTheme } from '../../storybook/theme_service';
 import { getChartRotationKnob } from '../utils/knobs';
 import { SB_KNOBS_PANEL } from '../utils/storybook';
 

stories/legend/13_inside_chart.tsx

@@ -20,7 +20,6 @@
 import { select, boolean, number } from '@storybook/addon-knobs';
 import React from 'react';
 
-import { switchTheme } from '../../.storybook/theme_service';
 import {
   AreaSeries,
   Axis,
@@ -37,6 +36,7 @@ import {
 } from '../../src';
 import { SeededDataGenerator } from '../../src/mocks/utils';
 import { KIBANA_METRICS } from '../../src/utils/data_samples/test_dataset_kibana';
+import { switchTheme } from '../../storybook/theme_service';
 import { SB_KNOBS_PANEL } from '../utils/storybook';
 
 const dg = new SeededDataGenerator();

stories/mixed/6_fitting_stacked.tsx

@@ -22,11 +22,11 @@ import numeral from 'numeral';
 import React from 'react';
 
 import { AreaSeries, Axis, Chart, CurveType, Position, ScaleType, Settings, Fit, StackMode } from '../../src';
-import { getRandomNumberGenerator } from '../../src/mocks/utils';
+import { getRandomNumberGenerator, getRNGSeed } from '../../src/mocks/utils';
 import { SB_KNOBS_PANEL } from '../utils/storybook';
 
 export const Example = () => {
-  const randomizeBoundingData = process.env.RNG_SEED !== null ? false : boolean('randomize bounding data', false);
+  const randomizeBoundingData = getRNGSeed() ? false : boolean('randomize bounding data', false);
   const dataTypes: Record<string, Array<{ x: number | string; y: number | null }>> = {
     isolated: [
       { x: 0, y: 3 },
@@ -159,7 +159,7 @@ export const Example = () => {
   const parsedEndValue: number | 'nearest' = Number.isNaN(Number(endValue)) ? 'nearest' : Number(endValue);
   const value = number('Explicit value (using Fit.Explicit)', 5);
   const xScaleType = dataKey === 'ordinal' ? ScaleType.Ordinal : ScaleType.Linear;
-  const rngSeed = randomizeBoundingData ? undefined : process.env.RNG_SEED ?? '__seed__';
+  const rngSeed = randomizeBoundingData ? undefined : getRNGSeed('__seed__');
   const rng = getRandomNumberGenerator(rngSeed);
   const tickFormat = stackMode === 'percentage' ? (d: any) => numeral(d).format('0[.]00%') : undefined;
   return (

stories/stylings/22_dark_theme.tsx

@@ -20,7 +20,6 @@
 import { boolean } from '@storybook/addon-knobs';
 import React from 'react';
 
-import { switchTheme } from '../../.storybook/theme_service';
 import {
   AreaSeries,
   Axis,
@@ -36,6 +35,7 @@ import {
   TooltipType,
 } from '../../src';
 import { SeededDataGenerator } from '../../src/mocks/utils';
+import { switchTheme } from '../../storybook/theme_service';
 
 const dg = new SeededDataGenerator();
 const data1 = dg.generateGroupedSeries(40, 4);

stories/stylings/4_theme_styling.tsx

@@ -20,7 +20,6 @@
 import { boolean, color, number, select } from '@storybook/addon-knobs';
 import React from 'react';
 
-import { switchTheme } from '../../.storybook/theme_service';
 import {
   AreaSeries,
   Axis,
@@ -38,6 +37,7 @@ import {
 } from '../../src';
 import { SeededDataGenerator } from '../../src/mocks/utils';
 import { palettes } from '../../src/utils/themes/colors';
+import { switchTheme } from '../../storybook/theme_service';
 
 function range(title: string, min: number, max: number, value: number, groupId?: string, step = 1) {
   return number(

storybook-docs/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "../tsconfig",
+  "include": ["../src/**/*", "../stories/**/*", "./**/*", "config.ts"],
+  "exclude": ["../**/*.test.*"]
+}

.storybook-docs/webpack.config.js → storybook-docs/webpack.config.js

@@ -37,8 +37,6 @@ const scssLoaders = [
 ];
 
 module.exports = async ({ config }) => {
-  // config.plugins.push(new webpack.EnvironmentPlugin({ RNG_SEED: null }));
-
   // Replace default css rules with nonce
   config.module.rules = config.module.rules.filter(({ test }) => !test.test('.css'));
   config.module.rules.push({
@@ -61,7 +59,7 @@ module.exports = async ({ config }) => {
 
   config.module.rules.push({
     test: /\.scss$/,
-    include: [path.resolve(__dirname, '../.storybook'), path.resolve(__dirname, '../node_modules/@elastic')],
+    include: [path.resolve(__dirname, '../storybook'), path.resolve(__dirname, '../node_modules/@elastic')],
     use: [
       {
         loader: 'style-loader',

.storybook/config.ts → storybook/config.ts

@@ -22,11 +22,17 @@ import { withKnobs } from '@storybook/addon-knobs';
 import { addDecorator, configure, addParameters } from '@storybook/react';
 import { create } from '@storybook/theming';
 
+import { preloadIcons } from './preload_icons';
 import { switchTheme } from './theme_service';
 import './style.scss';
 
 switchTheme('light');
 
+if (process.env.STORYBOOK_VRT) {
+  preloadIcons();
+  document.querySelector('html')?.classList.add('disable-animations');
+}
+
 addParameters({
   options: {
     theme: create({