From 444f5d46309cf89aa46255ab0af7d5d5b4e980d3 Mon Sep 17 00:00:00 2001 From: Timo Tijhof Date: Mon, 5 Jul 2021 18:55:43 +0100 Subject: [PATCH] Spec: Define "error" event for global failures Ref https://github.com/qunitjs/qunit/issues/1633. --- lib/reporters/ConsoleReporter.js | 5 +++++ lib/reporters/SummaryReporter.js | 4 ++++ lib/reporters/TapReporter.js | 18 ++++++++++++++++++ spec/cri-draft.adoc | 30 ++++++++++++++++++++++++++++++ test/unit/console-reporter.js | 5 +++++ test/unit/summary-reporter.js | 12 ++++++++++++ test/unit/tap-reporter.js | 22 ++++++++++++++++++++++ 7 files changed, 96 insertions(+) diff --git a/lib/reporters/ConsoleReporter.js b/lib/reporters/ConsoleReporter.js index c6dfd68..1ccc215 100644 --- a/lib/reporters/ConsoleReporter.js +++ b/lib/reporters/ConsoleReporter.js @@ -5,6 +5,7 @@ module.exports = class ConsoleReporter { // https://github.com/js-reporters/js-reporters/issues/125 this.log = options.log || console.log.bind(console); + runner.on('error', this.onError.bind(this)); runner.on('runStart', this.onRunStart.bind(this)); runner.on('testStart', this.onTestStart.bind(this)); runner.on('testEnd', this.onTestEnd.bind(this)); @@ -15,6 +16,10 @@ module.exports = class ConsoleReporter { return new ConsoleReporter(runner); } + onError (error) { + this.log('error', error); + } + onRunStart (runStart) { this.log('runStart', runStart); } diff --git a/lib/reporters/SummaryReporter.js b/lib/reporters/SummaryReporter.js index 21e6a0d..bce0b27 100644 --- a/lib/reporters/SummaryReporter.js +++ b/lib/reporters/SummaryReporter.js @@ -21,6 +21,10 @@ module.exports = class SummaryReporter extends EventEmitter { this.summary = null; + runner.once('error', (error) => { + this.emit('error', error); + }); + runner.on('suiteEnd', (suiteEnd) => { const ownFull = suiteEnd.fullName.join('>'); const suiteName = suiteEnd.fullName.length >= 2 ? suiteEnd.fullName.slice(-2, -1)[0] : null; diff --git a/lib/reporters/TapReporter.js b/lib/reporters/TapReporter.js index 33f74bb..9ef7034 100644 --- a/lib/reporters/TapReporter.js +++ b/lib/reporters/TapReporter.js @@ -169,6 +169,7 @@ module.exports = class TapReporter { this.testCount = 0; + runner.on('error', this.onError.bind(this)); runner.on('runStart', this.onRunStart.bind(this)); runner.on('testEnd', this.onTestEnd.bind(this)); runner.on('runEnd', this.onRunEnd.bind(this)); @@ -182,6 +183,23 @@ module.exports = class TapReporter { this.log('TAP version 13'); } + onError (error) { + if (typeof error === 'object' && error !== null) { + let out = ' ---'; + const errYaml = prettyYamlValue(error); + if (errYaml !== '{}') { + out += `\n error: ${errYaml}`; + } + if (error.stack) { + // This property is non-enumerable by default, so include it manually. + out += `\n stack: ${prettyYamlValue(error.stack + '\n')}`; + } + out += '\n ...'; + this.log(out); + } + this.log('Bail out! ' + error); + } + onTestEnd (test) { this.testCount = this.testCount + 1; diff --git a/spec/cri-draft.adoc b/spec/cri-draft.adoc index 1e44007..d929abe 100644 --- a/spec/cri-draft.adoc +++ b/spec/cri-draft.adoc @@ -189,6 +189,26 @@ Callback parameters: producer.on('testEnd', (testEnd) => { … }); ---- +=== `error` event + +The **error** event indicates a global failure. It may be emitted at any time, including before a <> or <>. + +Reporters must interpret error events as the <> having failed. Reporters should not wait for or expect other events to be emitted afterwards. For example, any pending <> or <> might not be delivered. If other events do get emitted after this, reporters should doubt their accuracy or ignore the event entirely. For example, if a `runEnd` event is emitted _after_ an `error` event and the `runEnd` reports no failures, then reporters must still consider the run as having failed. + +Callback parameters: + +* <> **error**. + +[source,javascript] +---- +producer.on('error', (error) => { … }); +---- + +[TIP] +===== +The "error" event is analogous to "Bail out!" lines as defined in https://testanything.org/tap-version-13-specification.html#directives[TAP specification]. +===== + == Event data The following data structures must be implemented as objects that have the specified fields as own properties. The objects are not required to be an instance of any specific class. They may be null-inherited objects, plain objects, or an instance of any public or private class. @@ -286,6 +306,16 @@ Producers may set additional (non-standard) properties on `Assertion` objects. The properties of the Assertion object was decided in https://github.com/js-reporters/js-reporters/issues/79[issue #79], and later revised by https://github.com/js-reporters/js-reporters/issues/105[issue #105]. ===== +=== Error + +The **Error** value may be any JavaScript value. It is recommended to pass an object, such as an instance of the [ECMAScript Error Constructor](https://github.com/js-reporters/js-reporters/issues/123) (or a subclass of that). But it may also be a non-object, or a plain object with the following properties: + +`Error` object: + +* `string` **name**: Error class name, or other prefix. +* `string` **message**: Error message. +* `string|undefined|null` **stack**: Optional stack trace. + == Producer API The object on which the Producer API is implemented does not need to be exclusive or otherwise limited to the Producer API. Producers are encouraged to implement the API as transparently as possible. diff --git a/test/unit/console-reporter.js b/test/unit/console-reporter.js index 611598c..144a921 100644 --- a/test/unit/console-reporter.js +++ b/test/unit/console-reporter.js @@ -39,4 +39,9 @@ QUnit.module('ConsoleReporter', hooks => { emitter.emit('testEnd', {}); assert.equal(con.log.callCount, 1); }); + + test('Event "error"', assert => { + emitter.emit('error', {}); + assert.equal(con.log.callCount, 1); + }); }); diff --git a/test/unit/summary-reporter.js b/test/unit/summary-reporter.js index b549637..74a0664 100644 --- a/test/unit/summary-reporter.js +++ b/test/unit/summary-reporter.js @@ -210,4 +210,16 @@ QUnit.module('SummaryReporter', hooks => { done(); }); }); + + test('forward "error" event', assert => { + const done = assert.async(); + const obj = { message: 'Boo' }; + + reporter.once('error', (err) => { + assert.strictEqual(err, obj); + done(); + }); + + emitter.emit('error', obj); + }); }); diff --git a/test/unit/tap-reporter.js b/test/unit/tap-reporter.js index 10d5d36..41638ab 100644 --- a/test/unit/tap-reporter.js +++ b/test/unit/tap-reporter.js @@ -67,6 +67,28 @@ QUnit.module('TapReporter', hooks => { } }); + test('output global failure (string)', assert => { + emitter.emit('error', 'Boo'); + + assert.true(spy.calledWith('Bail out! Boo')); + }); + + test('output global failure (Error)', assert => { + const err = new ReferenceError('Boo is not defined'); + err.stack = `ReferenceError: Boo is not defined + at foo (foo.js:1:2) + at bar (bar.js:1:2)`; + emitter.emit('error', err); + + assert.true(spy.calledWith(` --- + stack: | + ReferenceError: Boo is not defined + at foo (foo.js:1:2) + at bar (bar.js:1:2) + ...`)); + assert.true(spy.calledWith('Bail out! ReferenceError: Boo is not defined')); + }); + test('output actual assertion value of undefined', assert => { emitter.emit('testEnd', data.actualUndefinedTest); assert.true(spy.calledWithMatch(/^ {2}actual {2}: undefined$/m));