Add combined Jest+Cypress code coverage reports (elastic#5262)

* Install Cypress code coverage dependencies

* Set up required babel+istanbul config

* Set up Cypress code coverage config

@see https://github.com/cypress-io/code-coverage#instrument-unit-tests

* Add script for combining cypress & jest code coverage
-into a single json & html report

* Misc cleanup/ignores

- Ignore new `.nyc_output` dir generated by cypress/istanbul

- Improve Jest config collectCoverageFrom !ignores:
  - DRY out component+services folder patterns with a {,} glob
  - Ignore new Cypress .spec files and .testenv files for jest coverage

- eslintignore the cypress folder, since they're all .js anyway and it has a decent amount of idiosyncracies
+ cleanup unused eslint-disable
+ cleanup cypress comment boilerplate
+ remove unused paths from webpack config

* Add wiki documentation

* [PR feedback] Move nyc config to own file

* [PR feedback] Switch combine coverage script to .sh

.babelrc.js

@@ -36,4 +36,10 @@ module.exports = {
       }
     ],
   ],
+  // Used for Cypress code coverage - note that the env has to be Cypress-specific, otherwise Jest --coverage throws errors
+  "env": {
+    "cypress_test": {
+      "plugins": ["istanbul"]
+    }
+  }
 };

.eslintignore

@@ -10,3 +10,4 @@ docs
 packages
 scripts
 generator-eui
+cypress

.gitignore

@@ -9,6 +9,7 @@ cypress/videos
 
 coverage/
 reports/
+.nyc_output/
 tmp/
 dist/
 lib/

cypress/plugins/index.js

@@ -7,27 +7,18 @@
  */
 
 /// <reference types="cypress" />
-// ***********************************************************
-// This example plugins/index.js can be used to load plugins
-//
-// You can change the location of this file or turn off loading
-// the plugins file with the 'pluginsFile' configuration option.
-//
-// You can read more here:
-// https://on.cypress.io/plugins-guide
-// ***********************************************************
 
 // This function is called when a project is opened or re-opened (e.g. due to
 // the project's config changing)
+// @see https://on.cypress.io/plugins-guide
 
-/**
- * @type {Cypress.PluginConfig}
- */
-// eslint-disable-next-line no-unused-vars
 module.exports = (on, config) => {
   // `on` is used to hook into various events Cypress emits
   // `config` is the resolved Cypress config
 
+  require('@cypress/code-coverage/task')(on, config);
+  on('file:preprocessor', require('@cypress/code-coverage/use-babelrc'));
+
   if (config.testingType === 'component') {
     const { startDevServer } = require('@cypress/webpack-dev-server');
 
@@ -38,4 +29,6 @@ module.exports = (on, config) => {
       startDevServer({ options, webpackConfig })
     );
   }
+
+  return config;
 };

cypress/plugins/webpack.config.js

@@ -11,9 +11,6 @@
 const path = require('path');
 const webpack = require('webpack');
 
-const SRC_DIR = path.resolve(__dirname, '..', 'src');
-const DIST_DIR = path.resolve(__dirname, '..', 'dist');
-
 const plugins = [
   // Force EuiIcon's dynamic imports to be included in the single eui.js build,
   // instead of being split out into multiple files
@@ -37,6 +34,9 @@ module.exports = {
         test: /\.(js|tsx?)$/,
         loader: 'babel-loader',
         exclude: /node_modules/,
+        options: {
+          plugins: ['istanbul'],
+        },
       },
       {
         test: /\.scss$/,

cypress/support/index.js

@@ -13,3 +13,4 @@
 // https://on.cypress.io/configuration
 // ***********************************************************
 
+import '@cypress/code-coverage/support';

nyc.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  'report-dir': 'reports/cypress-coverage',
+  reporter: ['json', 'html'],
+};

package.json

@@ -32,8 +32,9 @@
     "test-unit": "cross-env NODE_ENV=test jest --config ./scripts/jest/config.json",
     "test-a11y": "node ./scripts/a11y-testing",
     "test-staged": "yarn lint && node scripts/test-staged.js",
-    "test-cypress": "cypress run-ct",
-    "test-cypress-dev": "cypress open-ct",
+    "test-cypress": "cross-env NODE_ENV=cypress_test cypress run-ct",
+    "test-cypress-dev": "cross-env NODE_ENV=cypress_test cypress open-ct",
+    "combine-test-coverage": "sh ./scripts/combine-coverage.sh",
     "start-test-server": "BABEL_MODULES=false NODE_ENV=puppeteer NODE_OPTIONS=--max-old-space-size=4096 webpack-dev-server --config src-docs/webpack.config.js --port 9999",
     "yo-component": "yo ./generator-eui/app/component.js",
     "update-token-changelog": "node ./scripts/update-token-changelog.js",
@@ -105,6 +106,7 @@
     "@babel/preset-env": "^7.11.0",
     "@babel/preset-react": "^7.10.4",
     "@babel/preset-typescript": "^7.12.1",
+    "@cypress/code-coverage": "^3.9.11",
     "@cypress/react": "^5.9.4",
     "@cypress/webpack-dev-server": "^1.4.0",
     "@elastic/charts": "^38.0.1",
@@ -138,6 +140,7 @@
     "babel-plugin-add-module-exports": "^1.0.2",
     "babel-plugin-dynamic-import-node": "^2.3.3",
     "babel-plugin-inline-react-svg": "^1.1.1",
+    "babel-plugin-istanbul": "^6.0.0",
     "babel-plugin-pegjs-inline-precompile": "^0.1.1",
     "babel-template": "^6.26.0",
     "cache-loader": "^4.1.0",

scripts/combine-coverage.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# Combines Jest code coverage and Cypress code coverage into a single report %
+# NOTE: `yarn test-cypress` and `yarn test-unit --coverage` must have already been run
+# in order for the correct coverage JSON files to exist and be combined
+
+# Clean folder and re-import Jest/Cypress JSON coverage reports
+rm -rf reports/combined-coverage && mkdir reports/combined-coverage
+cp reports/jest-coverage/coverage-final.json reports/combined-coverage/jest-coverage.json
+cp reports/cypress-coverage/coverage-final.json reports/combined-coverage/cypress-coverage.json
+
+# Tell Istanbul to merge the reports in the folder and generate a HTML report
+yarn nyc merge reports/combined-coverage reports/combined-coverage/coverage-final.json
+yarn nyc report --reporter html --temp-dir reports/combined-coverage --report-dir reports/combined-coverage
+
+# Open the HTML report for convenience
+open reports/combined-coverage/index.html

scripts/jest/config.json

@@ -6,12 +6,11 @@
     "<rootDir>/packages/eslint-plugin"
   ],
   "collectCoverageFrom": [
-    "src/components/**/*.{ts,tsx,js,jsx}",
-    "!src/components/index.ts",
-    "!src/components/**/*/index.ts",
-    "src/services/**/*.{ts,tsx,js,jsx}",
-    "!src/services/index.ts",
-    "!src/services/**/*/index.ts"
+    "src/{components,services}/**/*.{ts,tsx,js,jsx}",
+    "!src/{components,services}/**/*.spec.{ts,tsx,js,jsx}",
+    "!src/{components,services}/**/*.testenv.{ts,tsx,js,jsx}",
+    "!src/{components,services}/index.ts",
+    "!src/{components,services}/**/*/index.ts"
   ],
   "moduleNameMapper": {
     "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "<rootDir>/scripts/jest/mocks/file_mock.js",
@@ -28,6 +27,7 @@
   ],
   "coverageDirectory": "<rootDir>/reports/jest-coverage",
   "coverageReporters": [
+    "json",
     "html"
   ],
   "moduleFileExtensions": [

wiki/cypress-testing.md

@@ -92,3 +92,19 @@ To track what Cypress is doing while running tests, you can pass in `--config vi
 Failing screenshot artifacts (as well as Cypress logs) are generated by our Jenkins CI.
 
 TODO: Instructions on where to click to see the relevant artifact(s)/screenshots
+
+## Code coverage
+
+Cypress has been configured to automatically output Cypress code coverage to the `reports/cypress-coverage` folder. If you would prefer to turn this off locally, you can add the `--env coverage=false` flag when running your your cypress commands.
+
+To view the generated HTML reports, you can either drag the `reports/cypress-coverage/index.html` file into a browser window, or in a terminal, run `open reports/cypress-coverage/index.html` from the EUI project root.
+
+### Combined Jest & Cypress code coverage
+
+If you're working on a component that has both Cypress and Jest tests, we have a command that allows you to combine the results of both coverage reports.
+
+1. Setup: On the component(s) you're looking for final coverage #s for,
+  - Ensure you have already run `yarn test-unit --coverage` to generate a Jest report
+  - Ensure you have already run `yarn test-cypress` to generate a Cypress report
+2. Run `yarn combine-test-coverage`
+  - This should automatically open a browser window with `reports/combined-coverage/index.html`

wiki/testing.md

@@ -112,3 +112,13 @@ Component mocking relies on using the `[name].testenv.*` namespace for identific
 ### Mapping all module exports
 
 The rendered output of a mocked component is at the author's discretion, however, all public exports from a module must be preserved in the mock file. Note that this does not apply to exported TypeScript types and interfaces, which will always be derived from the original component file.
+
+## Code coverage
+
+Jest code coverage reports can be generated by running `yarn test` or `yarn test-unit` with the `--coverage` flag. Generated HTML reports are output to the `reports/` folder.
+
+To view them, you can either drag the `reports/jest-coverage/index.html` file into a browser window, or in a terminal, run `open reports/jest-coverage/index.html` from the EUI project root.
+
+We currently do not use code coverage %s in CI for automated checks or reports. Code coverage is a tool intended to help devs quickly find lines/branches that still need to be covered and determine test cases to write.
+
+> If you're working on a component that has both Cypress and Jest tests, see the [Cypress testing doc](cypress-testing.md#code-coverage) and its code coverage section for instructions on how to get a combined code coverage report.