From bc63b1583ec95abbf4c74df5f8346e0c8f048220 Mon Sep 17 00:00:00 2001 From: Nick Malaguti Date: Fri, 10 Apr 2015 17:13:11 -0400 Subject: [PATCH] feat(reporter): add check coverage thresholds Add check option to coverageReporter options. Supports similar options to istanbul's check-coverage feature. It will cause karma to return a non-zero exit code if coverage thresholds are not met. Closes #21 --- LICENSE-istanbul | 24 +++++++ README.md | 39 +++++++++++ lib/reporter.js | 138 +++++++++++++++++++++++++++++++++++++- test/reporter.spec.coffee | 73 +++++++++++++++++++- 4 files changed, 272 insertions(+), 2 deletions(-) create mode 100644 LICENSE-istanbul diff --git a/LICENSE-istanbul b/LICENSE-istanbul new file mode 100644 index 0000000..45a650b --- /dev/null +++ b/LICENSE-istanbul @@ -0,0 +1,24 @@ +Copyright 2012 Yahoo! Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the Yahoo! Inc. nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL YAHOO! INC. BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 066b380..34cbb15 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,45 @@ coverageReporter: { } ``` +#### check +**Type:** Object + +**Description:** This will be used to configure minimum threshold enforcement for coverage results. If the thresholds are not met, karma will return failure. Thresholds, when specified as a positive number are taken to be the minimum percentage required. When a threshold is specified as a negative number it represents the maximum number of uncovered entities allowed. + +For example, `statements: 90` implies minimum statement coverage is 90%. `statements: -10` implies that no more than 10 uncovered statements are allowed. + +`global` applies to all files together and `each` on a per-file basis. A list of files or patterns can be excluded from enforcement via the `exclude` property. On a per-file or pattern basis, per-file thresholds can be overridden via the `overrides` property. + +```javascript +coverageReporter: { + check: { + global: { + statements: 50, + branches: 50, + functions: 50, + lines: 50, + excludes: [ + 'foo/bar/**/*.js' + ] + }, + each: { + statements: 50, + branches: 50, + functions: 50, + lines: 50, + excludes: [ + 'other/directory/**/*.js' + ], + overrides: { + 'baz/component/**/*.js': { + statements: 98 + } + } + } + } +} +``` + #### watermarks **Type:** Object diff --git a/lib/reporter.js b/lib/reporter.js index 9039445..ca68615 100644 --- a/lib/reporter.js +++ b/lib/reporter.js @@ -1,4 +1,8 @@ // Coverage Reporter +// Part of this code is based on [1], which is licensed under the New BSD License. +// For more information see the See the accompanying LICENSE-istanbul file for terms. +// +// [1]: https://github.com/gotwarlost/istanbul/blob/master/lib/command/check-coverage.js // ===================== // // Generates the report @@ -8,11 +12,20 @@ var path = require('path') var istanbul = require('istanbul') +var minimatch = require('minimatch') var globalSourceCache = require('./source-cache') var coverageMap = require('./coverage-map') var SourceCacheStore = require('./source-cache-store') +function isAbsolute (file) { + if (path.isAbsolute) { + return path.isAbsolute(file) + } + + return path.resolve(file) === path.normalize(file) +} + // TODO(vojta): inject only what required (config.basePath, config.coverageReporter) var CoverageReporter = function (rootConfig, helper, logger) { var _ = helper._ @@ -64,6 +77,116 @@ var CoverageReporter = function (rootConfig, helper, logger) { } } + function normalize (key) { + // Exclude keys will always be relative, but covObj keys can be absolute or relative + var excludeKey = isAbsolute(key) ? path.relative(basePath, key) : key + // Also normalize for files that start with `./`, etc. + excludeKey = path.normalize(excludeKey) + + return excludeKey + } + + function removeFiles (covObj, patterns) { + var obj = {} + + Object.keys(covObj).forEach(function (key) { + // Do any patterns match the resolved key + var found = patterns.some(function (pattern) { + return minimatch(normalize(key), pattern, {dot: true}) + }) + + // if no patterns match, keep the key + if (!found) { + obj[key] = covObj[key] + } + }) + + return obj + } + + function overrideThresholds (key, overrides) { + var thresholds = {} + + // First match wins + Object.keys(overrides).some(function (pattern) { + if (minimatch(normalize(key), pattern, {dot: true})) { + thresholds = overrides[pattern] + return true + } + }) + + return thresholds + } + + function checkCoverage (browser, collector) { + var defaultThresholds = { + global: { + statements: 0, + branches: 0, + lines: 0, + functions: 0, + excludes: [] + }, + each: { + statements: 0, + branches: 0, + lines: 0, + functions: 0, + excludes: [], + overrides: {} + } + } + + var thresholds = helper.merge({}, defaultThresholds, config.check) + + var rawCoverage = collector.getFinalCoverage() + var globalResults = istanbul.utils.summarizeCoverage(removeFiles(rawCoverage, thresholds.global.excludes)) + var eachResults = removeFiles(rawCoverage, thresholds.each.excludes) + + // Summarize per-file results and mutate original results. + Object.keys(eachResults).forEach(function (key) { + eachResults[key] = istanbul.utils.summarizeFileCoverage(eachResults[key]) + }) + + var coverageFailed = false + + function check (name, thresholds, actuals) { + [ + 'statements', + 'branches', + 'lines', + 'functions' + ].forEach(function (key) { + var actual = actuals[key].pct + var actualUncovered = actuals[key].total - actuals[key].covered + var threshold = thresholds[key] + + if (threshold < 0) { + if (threshold * -1 < actualUncovered) { + coverageFailed = true + log.error(browser.name + ': Uncovered count for ' + key + ' (' + actualUncovered + + ') exceeds ' + name + ' threshold (' + -1 * threshold + ')') + } + } else { + if (actual < threshold) { + coverageFailed = true + log.error(browser.name + ': Coverage for ' + key + ' (' + actual + + '%) does not meet ' + name + ' threshold (' + threshold + '%)') + } + } + }) + } + + check('global', thresholds.global, globalResults) + + Object.keys(eachResults).forEach(function (key) { + var keyThreshold = helper.merge(thresholds.each, overrideThresholds(key, thresholds.each.overrides)) + check('per-file' + ' (' + key + ') ', keyThreshold, eachResults[key]) + }) + + return coverageFailed + } + // Generate the output directory from the `coverageReporter.dir` and // `coverageReporter.subdir` options. function generateOutputDir (browserName, dir, subdir) { @@ -109,7 +232,9 @@ var CoverageReporter = function (rootConfig, helper, logger) { collectors[browser.id].add(result.coverage) } - this.onRunComplete = function (browsers) { + this.onRunComplete = function (browsers, results) { + var checkedCoverage = {} + reporters.forEach(function (reporterConfig) { browsers.forEach(function (browser) { var collector = collectors[browser.id] @@ -118,6 +243,17 @@ var CoverageReporter = function (rootConfig, helper, logger) { return } + // If config.check is defined, check coverage levels for each browser + if (config.hasOwnProperty('check') && !checkedCoverage[browser.id]) { + checkedCoverage[browser.id] = true + var coverageFailed = checkCoverage(browser, collector) + if (coverageFailed) { + if (results) { + results.exitCode = 1 + } + } + } + pendingFileWritings++ var mainDir = reporterConfig.dir || config.dir diff --git a/test/reporter.spec.coffee b/test/reporter.spec.coffee index fb32143..ce7d286 100644 --- a/test/reporter.spec.coffee +++ b/test/reporter.spec.coffee @@ -26,10 +26,11 @@ describe 'reporter', -> mockAdd = sinon.spy() mockDispose = sinon.spy() + mockGetFinalCoverage = sinon.stub().returns {} mockCollector = class Collector add: mockAdd dispose: mockDispose - getFinalCoverage: -> null + getFinalCoverage: mockGetFinalCoverage mockWriteReport = sinon.spy() mockReportCreate = sinon.stub().returns writeReport: mockWriteReport mockMkdir = sinon.spy() @@ -48,6 +49,13 @@ describe 'reporter', -> functions: [50, 80] lines: [50, 80] + mockSummarizeCoverage = sinon.stub().returns { + lines: {total: 5, covered: 1, skipped: 0, pct: 20}, + statements: {total: 5, covered: 1, skipped: 0, pct: 20}, + functions: {total: 5, covered: 1, skipped: 0, pct: 20}, + branches: {total: 5, covered: 1, skipped: 0, pct: 20} + } + mocks = fs: mockFs istanbul: @@ -55,6 +63,9 @@ describe 'reporter', -> Collector: mockCollector Report: create: mockReportCreate config: defaultConfig: sinon.stub().returns(reporting: watermarks: mockDefaultWatermarks) + utils: + summarizeCoverage: mockSummarizeCoverage + summarizeFileCoverage: mockSummarizeCoverage dateformat: require 'dateformat' './coverage-map': mockCoverageMap @@ -377,3 +388,63 @@ describe 'reporter', -> mockMkdir.getCall(0).args[1]() expect(mockDispose).not.to.have.been.calledBefore mockWriteReport + + it 'should log errors on low coverage and fail the build', -> + customConfig = _.merge {}, rootConfig, + coverageReporter: + check: + each: + statements: 50 + + mockGetFinalCoverage.returns + './foo/bar.js': {} + './foo/baz.js': {} + + spy1 = sinon.spy() + + customLogger = create: (name) -> + debug: -> null + info: -> null + warn: -> null + error: spy1 + + results = exitCode: 0 + + reporter = new m.CoverageReporter customConfig, mockHelper, customLogger + reporter.onRunStart() + browsers.forEach (b) -> reporter.onBrowserStart b + reporter.onRunComplete browsers, results + + expect(spy1).to.have.been.called + + expect(results.exitCode).to.not.equal 0 + + it 'should not log errors on sufficient coverage and not fail the build', -> + customConfig = _.merge {}, rootConfig, + coverageReporter: + check: + each: + statements: 10 + + mockGetFinalCoverage.returns + './foo/bar.js': {} + './foo/baz.js': {} + + spy1 = sinon.spy() + + customLogger = create: (name) -> + debug: -> null + info: -> null + warn: -> null + error: spy1 + + results = exitCode: 0 + + reporter = new m.CoverageReporter customConfig, mockHelper, customLogger + reporter.onRunStart() + browsers.forEach (b) -> reporter.onBrowserStart b + reporter.onRunComplete browsers, results + + expect(spy1).to.not.have.been.called + + expect(results.exitCode).to.equal 0