From 800476394e4eb2078021825b5edbb635f46ad960 Mon Sep 17 00:00:00 2001 From: Christian Budde Christensen Date: Mon, 11 Jan 2016 18:16:06 +0100 Subject: [PATCH] feat: Do not fail on empty test suite Add option to configuration-file and cli-args to server (single run) and runner, for disabling failure on empty test suite. Will instead display a warning. Closes #926 --- lib/cli.js | 8 +++++ lib/config.js | 1 + lib/middleware/runner.js | 3 +- lib/runner.js | 20 ++++++++---- lib/server.js | 4 ++- test/unit/middleware/runner.spec.js | 50 ++++++++++++++++++++++++++++- test/unit/runner.spec.js | 23 ++++++++++--- 7 files changed, 95 insertions(+), 14 deletions(-) diff --git a/lib/cli.js b/lib/cli.js index 351b0848d..335aabc51 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -37,6 +37,10 @@ var processArgs = function (argv, options, fs, path) { options.colors = options.colors === 'true' } + if (helper.isString(options.failOnEmptyTestSuite)) { + options.failOnEmptyTestSuite = options.failOnEmptyTestSuite === 'true' + } + if (helper.isString(options.logLevel)) { var logConstant = constant['LOG_' + options.logLevel.toUpperCase()] if (helper.isDefined(logConstant)) { @@ -165,6 +169,8 @@ var describeStart = function () { .describe('single-run', 'Run the test when browsers captured and exit.') .describe('no-single-run', 'Disable single-run.') .describe('report-slower-than', ' Report tests that are slower than given time [ms].') + .describe('fail-on-empty-test-suite', 'Fail on empty test suite.') + .describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.') .describe('help', 'Print usage and options.') } @@ -176,6 +182,8 @@ var describeRun = function () { ' $0 run [] [] [ -- ]') .describe('port', ' Port where the server is listening.') .describe('no-refresh', 'Do not re-glob all the patterns.') + .describe('fail-on-empty-test-suite', 'Fail on empty test suite.') + .describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.') .describe('help', 'Print usage.') } diff --git a/lib/config.js b/lib/config.js index 61d4d7fd0..7346022e3 100644 --- a/lib/config.js +++ b/lib/config.js @@ -260,6 +260,7 @@ var Config = function () { this.browserDisconnectTolerance = 0 this.browserNoActivityTimeout = 10000 this.concurrency = Infinity + this.failOnEmptyTestSuite = true } var CONFIG_SYNTAX_HELP = ' module.exports = function(config) {\n' + diff --git a/lib/middleware/runner.js b/lib/middleware/runner.js index 8a2847260..377b1e39d 100644 --- a/lib/middleware/runner.js +++ b/lib/middleware/runner.js @@ -41,7 +41,8 @@ var createRunnerMiddleware = function (emitter, fileList, capturedBrowsers, repo // clean up, close runner response emitter.once('run_complete', function (browsers, results) { reporter.removeAdapter(responseWrite) - response.end(constant.EXIT_CODE + results.exitCode) + var emptyTestSuite = (results.failed + results.success) === 0 ? 0 : 1 + response.end(constant.EXIT_CODE + emptyTestSuite + results.exitCode) }) }) diff --git a/lib/runner.js b/lib/runner.js index 8d226228c..41c43bf75 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -3,9 +3,11 @@ var http = require('http') var constant = require('./constants') var helper = require('./helper') var cfg = require('./config') +var logger = require('./logger') +var log = logger.create('runner') -var parseExitCode = function (buffer, defaultCode) { - var tailPos = buffer.length - Buffer.byteLength(constant.EXIT_CODE) - 1 +var parseExitCode = function (buffer, defaultCode, failOnEmptyTestSuite) { + var tailPos = buffer.length - Buffer.byteLength(constant.EXIT_CODE) - 2 if (tailPos < 0) { return defaultCode @@ -14,9 +16,15 @@ var parseExitCode = function (buffer, defaultCode) { // tail buffer which might contain the message var tail = buffer.slice(tailPos) var tailStr = tail.toString() - if (tailStr.substr(0, tailStr.length - 1) === constant.EXIT_CODE) { + if (tailStr.substr(0, tailStr.length - 2) === constant.EXIT_CODE) { tail.fill('\x00') - return parseInt(tailStr.substr(-1), 10) + var emptyInt = parseInt(tailStr.substr(-2, 1), 10) + var exitCode = parseInt(tailStr.substr(-1), 10) + if (failOnEmptyTestSuite === false && emptyInt === 0) { + log.warn('Test suite was empty.') + return 0 + } + return exitCode } return defaultCode @@ -40,7 +48,7 @@ exports.run = function (config, done) { var request = http.request(options, function (response) { response.on('data', function (buffer) { - exitCode = parseExitCode(buffer, exitCode) + exitCode = parseExitCode(buffer, exitCode, config.failOnEmptyTestSuite) process.stdout.write(buffer) }) @@ -51,7 +59,7 @@ exports.run = function (config, done) { request.on('error', function (e) { if (e.code === 'ECONNREFUSED') { - console.error('There is no server listening on port %d', options.port) + log.error('There is no server listening on port %d', options.port) done(1, e.code) } else { throw e diff --git a/lib/server.js b/lib/server.js index 9de3de03e..96247b10b 100644 --- a/lib/server.js +++ b/lib/server.js @@ -251,8 +251,10 @@ Server.prototype._start = function (config, launcher, preprocess, fileList, webS var results = singleRunBrowsers.getResults() if (singleRunBrowserNotCaptured) { results.exitCode = 1 + } else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) { + results.exitCode = 0 + self.log.warn('Test suite was empty.') } - self.emit('run_complete', singleRunBrowsers, results) } } diff --git a/test/unit/middleware/runner.spec.js b/test/unit/middleware/runner.spec.js index 7b31849d2..d925bc5b7 100644 --- a/test/unit/middleware/runner.spec.js +++ b/test/unit/middleware/runner.spec.js @@ -73,7 +73,7 @@ describe('middleware.runner', () => { response.once('end', () => { expect(nextSpy).to.not.have.been.called - expect(response).to.beServedAs(200, 'result\x1FEXIT0') + expect(response).to.beServedAs(200, 'result\x1FEXIT10') done() }) @@ -83,6 +83,54 @@ describe('middleware.runner', () => { emitter.emit('run_complete', capturedBrowsers, {exitCode: 0}) }) + it('should set the empty to 0 if empty results', (done) => { + capturedBrowsers.add(new Browser()) + sinon.stub(capturedBrowsers, 'areAllReady', () => true) + + response.once('end', () => { + expect(nextSpy).to.not.have.been.called + expect(response).to.beServedAs(200, 'result\x1FEXIT00') + done() + }) + + handler(new HttpRequestMock('/__run__'), response, nextSpy) + + mockReporter.write('result') + emitter.emit('run_complete', capturedBrowsers, {exitCode: 0, success: 0, failed: 0}) + }) + + it('should set the empty to 1 if successful tests', (done) => { + capturedBrowsers.add(new Browser()) + sinon.stub(capturedBrowsers, 'areAllReady', () => true) + + response.once('end', () => { + expect(nextSpy).to.not.have.been.called + expect(response).to.beServedAs(200, 'result\x1FEXIT10') + done() + }) + + handler(new HttpRequestMock('/__run__'), response, nextSpy) + + mockReporter.write('result') + emitter.emit('run_complete', capturedBrowsers, {exitCode: 0, success: 3, failed: 0}) + }) + + it('should set the empty to 1 if failed tests', (done) => { + capturedBrowsers.add(new Browser()) + sinon.stub(capturedBrowsers, 'areAllReady', () => true) + + response.once('end', () => { + expect(nextSpy).to.not.have.been.called + expect(response).to.beServedAs(200, 'result\x1FEXIT10') + done() + }) + + handler(new HttpRequestMock('/__run__'), response, nextSpy) + + mockReporter.write('result') + emitter.emit('run_complete', capturedBrowsers, {exitCode: 0, success: 0, failed: 6}) + }) + it('should not run if there is no browser captured', (done) => { sinon.stub(fileListMock, 'refresh') diff --git a/test/unit/runner.spec.js b/test/unit/runner.spec.js index 5d5c7a785..141db0b41 100644 --- a/test/unit/runner.spec.js +++ b/test/unit/runner.spec.js @@ -12,14 +12,14 @@ describe('runner', () => { var EXIT = constant.EXIT_CODE it('should return 0 exit code if present in the buffer', () => { - expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}0`))).to.equal(0) + expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}10`))).to.equal(0) }) it('should null the exit code part of the buffer', () => { - var buffer = new Buffer(`some${EXIT}1`) + var buffer = new Buffer(`some${EXIT}01`) m.parseExitCode(buffer) - expect(buffer.toString()).to.equal('some\0\0\0\0\0\0') + expect(buffer.toString()).to.equal('some\0\0\0\0\0\0\0') }) it('should not touch buffer without exit code and return default', () => { @@ -41,8 +41,21 @@ describe('runner', () => { }) it('should parse any single digit exit code', () => { - expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}1`))).to.equal(1) - expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}7`))).to.equal(7) + expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}01`))).to.equal(1) + expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}17`))).to.equal(7) + }) + + it('should return exit code 0 if failOnEmptyTestSuite is false and and non-empty int is 0', () => { + expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}01`), undefined, false)).to.equal(0) + }) + + it('should return exit code if failOnEmptyTestSuite is true', () => { + expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}00`), undefined, true)).to.equal(0) + expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}01`), undefined, true)).to.equal(1) + expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}07`), undefined, true)).to.equal(7) + expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}10`), undefined, true)).to.equal(0) + expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}11`), undefined, true)).to.equal(1) + expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}17`), undefined, true)).to.equal(7) }) }) })