coverage: Switch to raw coverage files (#40013)

It turns out that there really aren't any really good generic coverage
reporting tools, the best you can get is something that highlights
covered lines without function/method stats, branch info, or anything
else fancy.

To provide useful coverage output, we're going to look instead at
generating separate coverage dashboards for PHP (PHPUnit) and JS
(jest/istanbul), each generated by those tools so we can get all the
useful info possible. To do that, we'll need all the coverage commands
to generate the tools' raw data formats rather than a lossy format like
clover.

As part of this conversion, we're adding `jest.config.cjs` everywhere so
we can consistently set `collectCoverageFrom`. By default the base
config will include non-test files under `src/`. Various packages need
additional dirs such as `components/`, and a root-level `index.jsx` or
`global.d.ts` seems common too. Then, since we have config files
everywhere, we can move `coverageDirectory` and `coverageReporters` into
the configs too.

docs/monorepo.md

@@ -309,7 +309,9 @@ If a project contains PHP or JavaScript tests, it should also define `.scripts.t
 
 Output should be written to the path specified via the `COVERAGE_DIR` environment variable. Subdirectories of that path may be used as desired.
 
-For PHP tests, you'll probably run PHPUnit as `php -dpcov.directory=. "$(command -v phpunit)" --coverage-clover "$COVERAGE_DIR/php/clover.xml"`.
+For PHP tests, you'll probably run PHPUnit as `php -dpcov.directory=. "$(command -v phpunit)" --coverage-php "$COVERAGE_DIR/php.cov"`. If you have multiple runs (e.g. unit and integration), be sure to write the `php.cov` files to separate subdirectories of `$COVERAGE_DIR`.
+
+For JS tests, you'll probably have a `test` script in package.json that runs `jest` with any needed options, and then a `test-coverage` script that does `pnpm run test --coverage`. If you have multiple runs (e.g. unit and integration), be sure each run writes to a different subdirectory of `$COVERAGE_DIR`.
 
 There's no need to be concerned about collisions with other projects' coverage files, a separate directory is used per project. The coverage files are also automatically copied to `ARTIFACTS_DIR`.
 

projects/github-actions/test-results-to-slack/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/github-actions/test-results-to-slack/package.json

@@ -28,6 +28,6 @@
 	"scripts": {
 		"build": "ncc build src/index.js -o dist --source-map --license licenses.txt",
 		"test": "jest --config=tests/jest.config.js --verbose --runInBand",
-		"test-coverage": "pnpm run test --coverage --coverageDirectory=\"$COVERAGE_DIR/js\" --coverageReporters=clover"
+		"test-coverage": "pnpm run test --coverage"
 	}
 }

projects/github-actions/test-results-to-slack/tests/jest.config.js

@@ -1,9 +1,10 @@
 const path = require( 'path' );
+const coverageConfig = require( 'jetpack-js-tools/jest/config.coverage.js' );
 
 module.exports = {
+	...coverageConfig,
 	rootDir: path.resolve( __dirname, '..' ),
 	roots: [ '<rootDir>/tests/' ],
-	collectCoverageFrom: [ '<rootDir>/src/**/*.js' ],
 	resolver: require.resolve( 'jetpack-js-tools/jest/jest-resolver.js' ),
 	clearMocks: true,
 	resetModules: true,

projects/js-packages/ai-client/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/js-packages/ai-client/package.json

@@ -19,7 +19,7 @@
 		"clean": "rm -rf build/",
 		"compile-ts": "tsc --pretty",
 		"test": "NODE_OPTIONS=--experimental-vm-modules jest",
-		"test-coverage": "pnpm run test --coverage --coverageDirectory=\"$COVERAGE_DIR/js\" --coverageReporters=clover",
+		"test-coverage": "pnpm run test --coverage",
 		"watch": "tsc --watch --pretty"
 	},
 	"type": "module",

projects/js-packages/analytics/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/js-packages/analytics/package.json

@@ -25,6 +25,6 @@
 	},
 	"scripts": {
 		"test": "NODE_OPTIONS=--experimental-vm-modules jest",
-		"test-coverage": "pnpm run test --coverageDirectory=\"$COVERAGE_DIR/js\" --coverage --coverageReporters=clover"
+		"test-coverage": "pnpm run test --coverage"
 	}
 }

projects/js-packages/api/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/js-packages/api/package.json

@@ -27,6 +27,6 @@
 	},
 	"scripts": {
 		"test": "NODE_OPTIONS=--experimental-vm-modules jest",
-		"test-coverage": "pnpm run test --coverageDirectory=\"$COVERAGE_DIR/js\" --coverage --coverageReporters=clover"
+		"test-coverage": "pnpm run test --coverage"
 	}
 }

projects/js-packages/babel-plugin-replace-textdomain/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/js-packages/babel-plugin-replace-textdomain/package.json

@@ -14,8 +14,8 @@
 	"license": "GPL-2.0-or-later",
 	"author": "Automattic",
 	"scripts": {
-		"test": "jest tests",
-		"test-coverage": "pnpm run test --coverage --collectCoverageFrom='src/**/*.js' --coverageDirectory=\"$COVERAGE_DIR/js\" --coverageReporters=clover"
+		"test": "jest --config=tests/jest.config.cjs",
+		"test-coverage": "pnpm run test --coverage"
 	},
 	"dependencies": {
 		"debug": "^4.3.2"

projects/js-packages/babel-plugin-replace-textdomain/tests/jest.config.cjs

@@ -0,0 +1,7 @@
+const path = require( 'path' );
+const coverageConfig = require( 'jetpack-js-tools/jest/config.coverage.js' );
+
+module.exports = {
+	...coverageConfig,
+	rootDir: path.join( __dirname, '..' ),
+};

projects/js-packages/boost-score-api/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/js-packages/boost-score-api/package.json

@@ -17,8 +17,8 @@
 		"build": "pnpm run clean && webpack",
 		"clean": "rm -rf build/",
 		"watch": "pnpm run build && pnpm webpack watch",
-		"test": "jest tests",
-		"test-coverage": "pnpm run test --coverage --coverageDirectory=\"$COVERAGE_DIR/js\" --coverageReporters=clover"
+		"test": "jest --config=tests/jest.config.cjs",
+		"test-coverage": "pnpm run test --coverage"
 	},
 	"dependencies": {
 		"@wordpress/i18n": "5.9.0",

projects/js-packages/boost-score-api/tests/jest.config.cjs

@@ -0,0 +1,7 @@
+const path = require( 'path' );
+const coverageConfig = require( 'jetpack-js-tools/jest/config.coverage.js' );
+
+module.exports = {
+	...coverageConfig,
+	rootDir: path.join( __dirname, '..' ),
+};

projects/js-packages/components/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/js-packages/components/jest.config.cjs

@@ -2,4 +2,10 @@ const baseConfig = require( 'jetpack-js-tools/jest/config.base.js' );
 
 module.exports = {
 	...baseConfig,
+	collectCoverageFrom: [
+		'<rootDir>/components/**/*.{js,mjs,cjs,jsx,ts,tsx,mts,cts}',
+		'<rootDir>/lib/**/*.{js,mjs,cjs,jsx,ts,tsx,mts,cts}',
+		'<rootDir>/tools/**/*.{js,mjs,cjs,jsx,ts,tsx,mts,cts}',
+		...baseConfig.collectCoverageFrom,
+	],
 };

projects/js-packages/components/package.json

@@ -81,6 +81,6 @@
 		"build": "pnpm run compile-ts",
 		"compile-ts": "tsc --pretty",
 		"test": "NODE_OPTIONS=--experimental-vm-modules jest",
-		"test-coverage": "pnpm run test --coverageDirectory=\"$COVERAGE_DIR/js\" --coverage --coverageReporters=clover"
+		"test-coverage": "pnpm run test --coverage"
 	}
 }

projects/js-packages/config/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/js-packages/config/jest.config.js

@@ -1,7 +1,10 @@
 // Note we intentionally don't use jetpack-js-tools/jest/config.base.js here.
 // This doesn't need jsdom or any of the other fancy stuff, but it does need to avoid the
 // standard jest-jetpack-config.js mocking.
+const coverageConfig = require( 'jetpack-js-tools/jest/config.coverage.js' );
+
 module.exports = {
+	...coverageConfig,
 	testMatch: [ '<rootDir>/**/test/*.[jt]s' ],
 	resolver: require.resolve( 'jetpack-js-tools/jest/jest-resolver.js' ),
 };

projects/js-packages/config/package.json

@@ -15,7 +15,7 @@
 	"author": "Automattic",
 	"scripts": {
 		"test": "jest",
-		"test-coverage": "pnpm run test --coverageDirectory=\"$COVERAGE_DIR/js\" --coverage --coverageReporters=clover"
+		"test-coverage": "pnpm run test --coverage"
 	},
 	"devDependencies": {
 		"jest": "29.7.0",

projects/js-packages/connection/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/js-packages/connection/jest.config.cjs

@@ -3,4 +3,11 @@ const baseConfig = require( 'jetpack-js-tools/jest/config.base.js' );
 module.exports = {
 	...baseConfig,
 	setupFilesAfterEnv: [ ...baseConfig.setupFilesAfterEnv, '<rootDir>/jest.setup.js' ],
+	collectCoverageFrom: [
+		'<rootDir>/components/**/*.{js,mjs,cjs,jsx,ts,tsx,mts,cts}',
+		'<rootDir>/helpers/**/*.{js,mjs,cjs,jsx,ts,tsx,mts,cts}',
+		'<rootDir>/hooks/**/*.{js,mjs,cjs,jsx,ts,tsx,mts,cts}',
+		'<rootDir>/state/**/*.{js,mjs,cjs,jsx,ts,tsx,mts,cts}',
+		...baseConfig.collectCoverageFrom,
+	],
 };

projects/js-packages/connection/package.json

@@ -60,6 +60,6 @@
 	],
 	"scripts": {
 		"test": "NODE_OPTIONS=--experimental-vm-modules jest",
-		"test-coverage": "pnpm run test --coverageDirectory=\"$COVERAGE_DIR/js\" --coverage --coverageReporters=clover"
+		"test-coverage": "pnpm run test --coverage"
 	}
 }

projects/js-packages/critical-css-gen/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/js-packages/critical-css-gen/package.json

@@ -20,7 +20,7 @@
 		"clean": "rm -rf build/",
 		"clean:test": "rm -rf tests/build/",
 		"test": "pnpm build && pnpm build:test && NODE_ENV=test NODE_PATH=./node_modules jest --forceExit --config=tests/config/jest.config.js",
-		"test-coverage": "pnpm run test --coverage --coverageDirectory=\"$COVERAGE_DIR/js\" --coverageReporters=clover"
+		"test-coverage": "pnpm run test --coverage"
 	},
 	"main": "./build/browser.js",
 	"devDependencies": {

projects/js-packages/critical-css-gen/tests/config/jest.config.js

@@ -1,9 +1,11 @@
+import coverageConfig from 'jetpack-js-tools/jest/config.coverage.js';
+
 export default {
+	...coverageConfig,
 	rootDir: '../',
 	testEnvironment: 'jest-environment-node',
 	testMatch: [ '**/?(*.)+(spec|test).js' ],
 	setupFilesAfterEnv: [ './config/jest-setup.js' ],
-	collectCoverageFrom: [ '../build-node/*.js' ],
 	testPathIgnorePatterns: [ '/node_modules/', 'config/jest-setup.js', 'build-node/*' ],
 	moduleDirectories: [ 'build-node', 'node_modules' ],
 };

projects/js-packages/eslint-changed/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/js-packages/eslint-changed/package.json

@@ -19,7 +19,7 @@
 	},
 	"scripts": {
 		"test": "NODE_OPTIONS=--experimental-vm-modules jest --config=tests/jest.config.cjs",
-		"test-coverage": "pnpm run test --coverage --coverageDirectory=\"$COVERAGE_DIR/js\" --coverageReporters=clover"
+		"test-coverage": "pnpm run test --coverage"
 	},
 	"dependencies": {
 		"chalk": "5.0.1",

projects/js-packages/eslint-changed/tests/jest.config.cjs

@@ -1,8 +1,13 @@
 const path = require( 'path' );
+const coverageConfig = require( 'jetpack-js-tools/jest/config.coverage.js' );
 
 module.exports = {
+	...coverageConfig,
 	rootDir: path.resolve( __dirname, '..' ),
 	roots: [ '<rootDir>/tests/' ],
-	collectCoverageFrom: [ '<rootDir>/src/**/*.js' ],
 	resolver: require.resolve( 'jetpack-js-tools/jest/jest-resolver.js' ),
+	collectCoverageFrom: [
+		'<rootDir>/bin/**/*.{js,mjs,cjs,jsx,ts,tsx,mts,cts}',
+		...coverageConfig.collectCoverageFrom,
+	],
 };

projects/js-packages/eslint-config-target-es/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/js-packages/eslint-config-target-es/package.json

@@ -14,8 +14,8 @@
 	"license": "GPL-2.0-or-later",
 	"author": "Automattic",
 	"scripts": {
-		"test": "jest tests",
-		"test-coverage": "pnpm run test --coverage --collectCoverageFrom='src/**/*.js' --coverageDirectory=\"$COVERAGE_DIR/js\" --coverageReporters=clover"
+		"test": "jest --config=tests/jest.config.cjs",
+		"test-coverage": "pnpm run test --coverage"
 	},
 	"dependencies": {
 		"@mdn/browser-compat-data": "5.5.49",

projects/js-packages/eslint-config-target-es/tests/jest.config.cjs

@@ -0,0 +1,7 @@
+const path = require( 'path' );
+const coverageConfig = require( 'jetpack-js-tools/jest/config.coverage.js' );
+
+module.exports = {
+	...coverageConfig,
+	rootDir: path.join( __dirname, '..' ),
+};

projects/js-packages/i18n-check-webpack-plugin/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+

projects/js-packages/i18n-check-webpack-plugin/package.json

@@ -14,8 +14,8 @@
 	"license": "GPL-2.0-or-later",
 	"author": "Automattic",
 	"scripts": {
-		"test": "jest tests/",
-		"test-coverage": "pnpm run test --coverage --collectCoverageFrom='src/**/*.js' --coverageDirectory=\"$COVERAGE_DIR/js\" --coverageReporters=clover"
+		"test": "jest --config=tests/jest.config.cjs",
+		"test-coverage": "pnpm run test --coverage"
 	},
 	"dependencies": {
 		"debug": "^4.3.2"

projects/js-packages/i18n-check-webpack-plugin/tests/jest.config.cjs

@@ -0,0 +1,7 @@
+const path = require( 'path' );
+const coverageConfig = require( 'jetpack-js-tools/jest/config.coverage.js' );
+
+module.exports = {
+	...coverageConfig,
+	rootDir: path.join( __dirname, '..' ),
+};

projects/js-packages/i18n-loader-webpack-plugin/changelog/update-switch-to-raw-coverage-files

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Generate raw phpunit and/or jest coverage data instead of clover.
+
+