From 99312a55f28b626b00b66251020b0268473e24aa Mon Sep 17 00:00:00 2001 From: Pulkit Gupta Date: Sun, 2 Apr 2023 22:48:48 +0530 Subject: [PATCH] test_runner: add code coverage support to spec reporter PR-URL: https://github.com/nodejs/node/pull/46674 Reviewed-By: Benjamin Gruenbaum Reviewed-By: Moshe Atlow --- lib/internal/test_runner/reporter/spec.js | 5 +- lib/internal/test_runner/reporter/tap.js | 39 +----- lib/internal/test_runner/utils.js | 52 +++++++- test/parallel/test-runner-coverage.js | 156 +++++++++++++++------- 4 files changed, 163 insertions(+), 89 deletions(-) diff --git a/lib/internal/test_runner/reporter/spec.js b/lib/internal/test_runner/reporter/spec.js index 2e3ba834351970..d991f563a315c8 100644 --- a/lib/internal/test_runner/reporter/spec.js +++ b/lib/internal/test_runner/reporter/spec.js @@ -15,7 +15,7 @@ const assert = require('assert'); const Transform = require('internal/streams/transform'); const { inspectWithNoCustomRetry } = require('internal/errors'); const { green, blue, red, white, gray } = require('internal/util/colors'); - +const { getCoverageReport } = require('internal/test_runner/utils'); const inspectOptions = { __proto__: null, colors: true, breakLength: Infinity }; @@ -30,6 +30,7 @@ const symbols = { 'test:fail': '\u2716 ', 'test:pass': '\u2714 ', 'test:diagnostic': '\u2139 ', + 'test:coverage': '\u2139 ', 'arrow:right': '\u25B6 ', 'hyphen:minus': '\uFE63 ', }; @@ -115,6 +116,8 @@ class SpecReporter extends Transform { break; case 'test:diagnostic': return `${colors[type]}${this.#indent(data.nesting)}${symbols[type]}${data.message}${white}\n`; + case 'test:coverage': + return getCoverageReport(this.#indent(data.nesting), data.summary, symbols['test:coverage'], blue); } } _transform({ type, data }, encoding, callback) { diff --git a/lib/internal/test_runner/reporter/tap.js b/lib/internal/test_runner/reporter/tap.js index c7d5dd9bdb037f..e4061ac93fca42 100644 --- a/lib/internal/test_runner/reporter/tap.js +++ b/lib/internal/test_runner/reporter/tap.js @@ -3,7 +3,6 @@ const { ArrayPrototypeForEach, ArrayPrototypeJoin, ArrayPrototypePush, - NumberPrototypeToFixed, ObjectEntries, RegExpPrototypeSymbolReplace, SafeMap, @@ -13,7 +12,7 @@ const { } = primordials; const { inspectWithNoCustomRetry } = require('internal/errors'); const { isError, kEmptyObject } = require('internal/util'); -const { relative } = require('path'); +const { getCoverageReport } = require('internal/test_runner/utils'); const kDefaultIndent = ' '; // 4 spaces const kFrameStartRegExp = /^ {4}at /; const kLineBreakRegExp = /\n|\r\n/; @@ -49,7 +48,7 @@ async function * tapReporter(source) { yield `${indent(data.nesting)}# ${tapEscape(data.message)}\n`; break; case 'test:coverage': - yield reportCoverage(data.nesting, data.summary); + yield getCoverageReport(indent(data.nesting), data.summary, '# ', ''); break; } } @@ -73,40 +72,6 @@ function reportTest(nesting, testNumber, status, name, skip, todo) { return line; } -function reportCoverage(nesting, summary) { - const pad = indent(nesting); - let report = `${pad}# start of coverage report\n`; - - report += `${pad}# file | line % | branch % | funcs % | uncovered lines\n`; - - for (let i = 0; i < summary.files.length; ++i) { - const { - path, - coveredLinePercent, - coveredBranchPercent, - coveredFunctionPercent, - uncoveredLineNumbers, - } = summary.files[i]; - const relativePath = relative(summary.workingDirectory, path); - const lines = NumberPrototypeToFixed(coveredLinePercent, 2); - const branches = NumberPrototypeToFixed(coveredBranchPercent, 2); - const functions = NumberPrototypeToFixed(coveredFunctionPercent, 2); - const uncovered = ArrayPrototypeJoin(uncoveredLineNumbers, ', '); - - report += `${pad}# ${relativePath} | ${lines} | ${branches} | ` + - `${functions} | ${uncovered}\n`; - } - - const { totals } = summary; - report += `${pad}# all files | ` + - `${NumberPrototypeToFixed(totals.coveredLinePercent, 2)} | ` + - `${NumberPrototypeToFixed(totals.coveredBranchPercent, 2)} | ` + - `${NumberPrototypeToFixed(totals.coveredFunctionPercent, 2)} |\n`; - - report += `${pad}# end of coverage report\n`; - return report; -} - function reportDetails(nesting, data = kEmptyObject) { const { error, duration_ms } = data; const _indent = indent(nesting); diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index 27e84aac941df7..48987f740e9a8a 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -1,20 +1,23 @@ 'use strict'; const { + ArrayPrototypeJoin, ArrayPrototypeMap, ArrayPrototypePush, ObjectCreate, ObjectGetOwnPropertyDescriptor, + NumberPrototypeToFixed, SafePromiseAllReturnArrayLike, RegExp, RegExpPrototypeExec, SafeMap, } = primordials; -const { basename } = require('path'); +const { basename, relative } = require('path'); const { createWriteStream } = require('fs'); const { pathToFileURL } = require('internal/url'); const { createDeferredPromise } = require('internal/util'); const { getOptionValue } = require('internal/options'); +const { green, red, white } = require('internal/util/colors'); const { codes: { @@ -247,6 +250,52 @@ function countCompletedTest(test, harness = test.root.harness) { harness.counters.all++; } + +function coverageThreshold(coverage, color) { + coverage = NumberPrototypeToFixed(coverage, 2); + if (color) { + if (coverage > 90) return `${green}${coverage}${color}`; + if (coverage < 50) return `${red}${coverage}${color}`; + } + return coverage; +} + +function getCoverageReport(pad, summary, symbol, color) { + let report = `${color}${pad}${symbol}start of coverage report\n`; + + report += `${pad}${symbol}file | line % | branch % | funcs % | uncovered lines\n`; + + for (let i = 0; i < summary.files.length; ++i) { + const { + path, + coveredLinePercent, + coveredBranchPercent, + coveredFunctionPercent, + uncoveredLineNumbers, + } = summary.files[i]; + const relativePath = relative(summary.workingDirectory, path); + const lines = coverageThreshold(coveredLinePercent, color); + const branches = coverageThreshold(coveredBranchPercent, color); + const functions = coverageThreshold(coveredFunctionPercent, color); + const uncovered = ArrayPrototypeJoin(uncoveredLineNumbers, ', '); + + report += `${pad}${symbol}${relativePath} | ${lines} | ${branches} | ` + + `${functions} | ${uncovered}\n`; + } + + const { totals } = summary; + report += `${pad}${symbol}all files | ` + + `${coverageThreshold(totals.coveredLinePercent, color)} | ` + + `${coverageThreshold(totals.coveredBranchPercent, color)} | ` + + `${coverageThreshold(totals.coveredFunctionPercent, color)} |\n`; + + report += `${pad}${symbol}end of coverage report\n`; + if (color) { + report += white; + } + return report; +} + module.exports = { convertStringToRegExp, countCompletedTest, @@ -256,4 +305,5 @@ module.exports = { isTestFailureError, parseCommandLine, setupTestReporters, + getCoverageReport, }; diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js index ae4a539cff4991..01fc8667199e50 100644 --- a/test/parallel/test-runner-coverage.js +++ b/test/parallel/test-runner-coverage.js @@ -18,7 +18,7 @@ function findCoverageFileForPid(pid) { }); } -function getCoverageFixtureReport() { +function getTapCoverageFixtureReport() { const report = [ '# start of coverage report', '# file | line % | branch % | funcs % | uncovered lines', @@ -37,64 +37,120 @@ function getCoverageFixtureReport() { return report; } -test('--experimental-test-coverage and --test cannot be combined', () => { - // TODO(cjihrig): This test can be removed once multi-process code coverage - // is supported. - const args = ['--test', '--experimental-test-coverage']; - const result = spawnSync(process.execPath, args); - - // 9 is the documented exit code for an invalid CLI argument. - assert.strictEqual(result.status, 9); - assert.match( - result.stderr.toString(), - /--experimental-test-coverage cannot be used with --test/ - ); -}); +function getSpecCoverageFixtureReport() { + const report = [ + '\u2139 start of coverage report', + '\u2139 file | line % | branch % | funcs % | uncovered lines', + '\u2139 test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12, ' + + '13, 16, 17, 18, 19, 20, 21, 22, 27, 39, 43, 44, 61, 62, 66, 67, 71, 72', + '\u2139 test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | ', + '\u2139 test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5, 6', + '\u2139 all files | 78.35 | 43.75 | 60.00 |', + '\u2139 end of coverage report', + ].join('\n'); -test('handles the inspector not being available', (t) => { - if (process.features.inspector) { - return; + if (common.isWindows) { + return report.replaceAll('/', '\\'); } - const fixture = fixtures.path('test-runner', 'coverage.js'); - const args = ['--experimental-test-coverage', fixture]; - const result = spawnSync(process.execPath, args); + return report; +} - assert(!result.stdout.toString().includes('# start of coverage report')); - assert(result.stderr.toString().includes('coverage could not be collected')); - assert.strictEqual(result.status, 0); - assert(!findCoverageFileForPid(result.pid)); -}); +test('test coverage report', async (t) => { + await t.test('--experimental-test-coverage and --test cannot be combined', () => { + // TODO(cjihrig): This test can be removed once multi-process code coverage + // is supported. + const args = ['--test', '--experimental-test-coverage']; + const result = spawnSync(process.execPath, args); + + // 9 is the documented exit code for an invalid CLI argument. + assert.strictEqual(result.status, 9); + assert.match( + result.stderr.toString(), + /--experimental-test-coverage cannot be used with --test/ + ); + }); -test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => { - if (!process.features.inspector) { - return; - } + await t.test('handles the inspector not being available', (t) => { + if (process.features.inspector) { + return; + } - const fixture = fixtures.path('test-runner', 'coverage.js'); - const args = ['--experimental-test-coverage', fixture]; - const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } }; - const result = spawnSync(process.execPath, args, options); - const report = getCoverageFixtureReport(); + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--experimental-test-coverage', fixture]; + const result = spawnSync(process.execPath, args); - assert(result.stdout.toString().includes(report)); - assert.strictEqual(result.stderr.toString(), ''); - assert.strictEqual(result.status, 0); - assert(findCoverageFileForPid(result.pid)); + assert(!result.stdout.toString().includes('# start of coverage report')); + assert(result.stderr.toString().includes('coverage could not be collected')); + assert.strictEqual(result.status, 0); + assert(!findCoverageFileForPid(result.pid)); + }); }); -test('coverage is reported without NODE_V8_COVERAGE present', (t) => { - if (!process.features.inspector) { - return; - } +test('test tap coverage reporter', async (t) => { + await t.test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => { + if (!process.features.inspector) { + return; + } + + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--experimental-test-coverage', '--test-reporter', 'tap', fixture]; + const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } }; + const result = spawnSync(process.execPath, args, options); + const report = getTapCoverageFixtureReport(); + + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.stderr.toString(), ''); + assert.strictEqual(result.status, 0); + assert(findCoverageFileForPid(result.pid)); + }); + + await t.test('coverage is reported without NODE_V8_COVERAGE present', (t) => { + if (!process.features.inspector) { + return; + } + + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--experimental-test-coverage', '--test-reporter', 'tap', fixture]; + const result = spawnSync(process.execPath, args); + const report = getTapCoverageFixtureReport(); - const fixture = fixtures.path('test-runner', 'coverage.js'); - const args = ['--experimental-test-coverage', fixture]; - const result = spawnSync(process.execPath, args); - const report = getCoverageFixtureReport(); + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.stderr.toString(), ''); + assert.strictEqual(result.status, 0); + assert(!findCoverageFileForPid(result.pid)); + }); +}); - assert(result.stdout.toString().includes(report)); - assert.strictEqual(result.stderr.toString(), ''); - assert.strictEqual(result.status, 0); - assert(!findCoverageFileForPid(result.pid)); +test('test spec coverage reporter', async (t) => { + await t.test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => { + if (!process.features.inspector) { + return; + } + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--experimental-test-coverage', '--test-reporter', 'spec', fixture]; + const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } }; + const result = spawnSync(process.execPath, args, options); + const report = getSpecCoverageFixtureReport(); + + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.stderr.toString(), ''); + assert.strictEqual(result.status, 0); + assert(findCoverageFileForPid(result.pid)); + }); + + await t.test('coverage is reported without NODE_V8_COVERAGE present', (t) => { + if (!process.features.inspector) { + return; + } + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--experimental-test-coverage', '--test-reporter', 'spec', fixture]; + const result = spawnSync(process.execPath, args); + const report = getSpecCoverageFixtureReport(); + + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.stderr.toString(), ''); + assert.strictEqual(result.status, 0); + assert(!findCoverageFileForPid(result.pid)); + }); });