From 6df223f560a095ad4c03879692a2727fefc517d5 Mon Sep 17 00:00:00 2001 From: charlierudolph Date: Fri, 30 Sep 2016 12:08:44 -0700 Subject: [PATCH] rerun formatter: output any scenario that doesn't pass --- docs/cli.md | 2 +- features/rerun_formatter.feature | 150 +++++------------- .../step_definitions/json_output_steps.js | 14 ++ lib/cucumber/listener/rerun_formatter.js | 2 +- .../cucumber/listener/rerun_formatter_spec.js | 43 ++++- 5 files changed, 100 insertions(+), 111 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 002a08e3a..45eaf67bc 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -50,7 +50,7 @@ Built-in formatters * json - prints the feature as JSON * pretty - prints the feature as is (default) * progress - prints one character per scenario -* rerun - prints the paths of the failing scenarios ([example](/features/rerun_formatter.feature)) +* rerun - prints the paths of any non passing scenarios ([example](/features/rerun_formatter.feature)) * suggested use: add the rerun formatter to your default profile and the output file to your `.gitignore` * snippets - prints just the code snippets for undefined steps * summary - prints a summary only, after all scenarios were executed diff --git a/features/rerun_formatter.feature b/features/rerun_formatter.feature index b19e8ef21..6e7270de0 100644 --- a/features/rerun_formatter.feature +++ b/features/rerun_formatter.feature @@ -7,29 +7,41 @@ Feature: Rerun Formatter Given a file named "features/a.feature" with: """ Feature: A - Scenario: 1 + Scenario: A - passing Given a passing step - Scenario: 2 + Scenario: A - failing Given a failing step - Scenario: 3 - Given a failing step + Scenario: A - ambiguous + Given an ambiguous step """ And a file named "features/b.feature" with: """ Feature: B - Scenario: 4 + Scenario: B - passing Given a passing step - Scenario: 5 - Given a failing step + Scenario: B - pending + Given a pending step """ + And a file named "features/c.feature" with: + """ + Feature: B + Scenario: C - passing + Given a passing step + + Scenario: C - undefined + Given an undefined step + """ And a file named "features/step_definitions/cucumber_steps.js" with: """ var cucumberSteps = function() { - this.When(/^a passing step$/, function() { }); - this.When(/^a failing step$/, function() { throw 'fail' }); + this.Given(/^a passing step$/, function() { }); + this.Given(/^a failing step$/, function() { throw 'fail' }); + this.Given(/^an ambiguous step$/, function() { }); + this.Given(/^an? ambiguous step$/, function() { }); + this.Given(/^a pending step$/, function() { return 'pending' }); }; module.exports = cucumberSteps; """ @@ -42,110 +54,34 @@ Feature: Rerun Formatter Scenario: passing When I run cucumber.js with `features/a.feature:2` - Then it outputs this text: - """ - Feature: A - - Scenario: 1 - ✔ Given a passing step - - 1 scenario (1 passed) - 1 step (1 passed) - - """ + Then the exit status should be 0 And the file "@rerun.txt" has the text: """ """ Scenario: multiple scenarios failing - When I run cucumber.js - Then it outputs this text: - """ - Feature: A - - Scenario: 1 - ✔ Given a passing step - - Scenario: 2 - ✖ Given a failing step - - Scenario: 3 - ✖ Given a failing step - - Feature: B - - Scenario: 4 - ✔ Given a passing step - - Scenario: 5 - ✖ Given a failing step - - Failures: - - 1) Scenario: 2 - features/a.feature:5 - Step: Given a failing step - features/a.feature:6 - Step Definition: features/step_definitions/cucumber_steps.js:3 - Message: - fail - - 2) Scenario: 3 - features/a.feature:8 - Step: Given a failing step - features/a.feature:9 - Step Definition: features/step_definitions/cucumber_steps.js:3 - Message: - fail - - 3) Scenario: 5 - features/b.feature:5 - Step: Given a failing step - features/b.feature:6 - Step Definition: features/step_definitions/cucumber_steps.js:3 - Message: - fail - - 5 scenarios (3 failed, 2 passed) - 5 steps (3 failed, 2 passed) - - """ + When I run cucumber.js with `-f json` + Then the exit status should be 1 + And the json output has the scenarios with names + | NAME | + | A - passing | + | A - failing | + | A - ambiguous | + | B - passing | + | B - pending | + | C - passing | + | C - undefined | And the file "@rerun.txt" has the text: """ features/a.feature:5:8 features/b.feature:5 - """ - When I run cucumber.js with `@rerun.txt` - Then it outputs this text: - """ - Feature: A - - Scenario: 2 - ✖ Given a failing step - - Scenario: 3 - ✖ Given a failing step - - Feature: B - - Scenario: 5 - ✖ Given a failing step - - Failures: - - 1) Scenario: 2 - features/a.feature:5 - Step: Given a failing step - features/a.feature:6 - Step Definition: features/step_definitions/cucumber_steps.js:3 - Message: - fail - - 2) Scenario: 3 - features/a.feature:8 - Step: Given a failing step - features/a.feature:9 - Step Definition: features/step_definitions/cucumber_steps.js:3 - Message: - fail - - 3) Scenario: 5 - features/b.feature:5 - Step: Given a failing step - features/b.feature:6 - Step Definition: features/step_definitions/cucumber_steps.js:3 - Message: - fail - - 3 scenarios (3 failed) - 3 steps (3 failed) - - """ + features/c.feature:5 + """ + When I run cucumber.js with `-f json @rerun.txt` + Then the exit status should be 1 + And the json output has the scenarios with names + | NAME | + | A - failing | + | A - ambiguous | + | B - pending | + | C - undefined | diff --git a/features/step_definitions/json_output_steps.js b/features/step_definitions/json_output_steps.js index bc454093c..0d76254f2 100644 --- a/features/step_definitions/json_output_steps.js +++ b/features/step_definitions/json_output_steps.js @@ -185,6 +185,20 @@ var jsonOutputSteps = function jsonOutputSteps() { assert.equal(scenario.name, name); }); + this.Then(/^the json output has the scenarios with names$/, function (table) { + var expectedNames = table.rows().map(function(row){ return row[0]; }); + var features = JSON.parse(this.lastRun.stdout); + var actualNames = []; + features.forEach(function(feature) { + feature.elements.forEach(function(element){ + if (element.type === 'scenario') { + actualNames.push(element.name); + } + }); + }); + assert.deepEqual(expectedNames, actualNames); + }); + this.Then(/^the json output's first scenario has the description "([^"]*)"$/, function (description) { var features = JSON.parse(this.lastRun.stdout); assert.equal(features[0].elements[0].description.trim(), description); diff --git a/lib/cucumber/listener/rerun_formatter.js b/lib/cucumber/listener/rerun_formatter.js index 38e970106..599c77ed2 100644 --- a/lib/cucumber/listener/rerun_formatter.js +++ b/lib/cucumber/listener/rerun_formatter.js @@ -8,7 +8,7 @@ function RerunFormatter(options) { self.handleScenarioResultEvent = function handleScenarioResultEvent(scenarioResult) { - if (scenarioResult.getStatus() === Cucumber.Status.FAILED) { + if (scenarioResult.getStatus() !== Cucumber.Status.PASSED) { var scenario = scenarioResult.getScenario(); var uri = path.relative(process.cwd(), scenario.getUri()); var line = scenario.getLine(); diff --git a/spec/cucumber/listener/rerun_formatter_spec.js b/spec/cucumber/listener/rerun_formatter_spec.js index 5ed726705..c7e5b1df0 100644 --- a/spec/cucumber/listener/rerun_formatter_spec.js +++ b/spec/cucumber/listener/rerun_formatter_spec.js @@ -15,7 +15,7 @@ describe("Cucumber.Listener.RerunFormatter", function () { } describe("handleAfterFeaturesEvent()", function () { - describe('no failed scenarios', function() { + describe('all passing scenarios', function() { beforeEach(function(done){ var scenarioResult = createScenarioResult(Cucumber.Status.PASSED); rerunFormatter.handleScenarioResultEvent(scenarioResult); @@ -35,7 +35,46 @@ describe("Cucumber.Listener.RerunFormatter", function () { rerunFormatter.handleAfterFeaturesEvent(null, done); }); - it("logs nothing", function () { + it("logs the path to the failed scenario", function () { + expect(rerunFormatter.getLogs()).toEqual(path.normalize('path/to/scenario:1')); + }); + }); + + describe('one ambiguous scenario', function() { + beforeEach(function(done) { + var scenarioResult = createScenarioResult(Cucumber.Status.AMBIGUOUS, 'path/to/scenario', 1); + rerunFormatter.handleScenarioResultEvent(scenarioResult); + + rerunFormatter.handleAfterFeaturesEvent(null, done); + }); + + it("logs the path to the ambiguous scenario", function () { + expect(rerunFormatter.getLogs()).toEqual(path.normalize('path/to/scenario:1')); + }); + }); + + describe('one undefined scenario', function() { + beforeEach(function(done) { + var scenarioResult = createScenarioResult(Cucumber.Status.UNDEFINED, 'path/to/scenario', 1); + rerunFormatter.handleScenarioResultEvent(scenarioResult); + + rerunFormatter.handleAfterFeaturesEvent(null, done); + }); + + it("logs the path to the undefined scenario", function () { + expect(rerunFormatter.getLogs()).toEqual(path.normalize('path/to/scenario:1')); + }); + }); + + describe('one pending scenario', function() { + beforeEach(function(done) { + var scenarioResult = createScenarioResult(Cucumber.Status.PENDING, 'path/to/scenario', 1); + rerunFormatter.handleScenarioResultEvent(scenarioResult); + + rerunFormatter.handleAfterFeaturesEvent(null, done); + }); + + it("logs the path to the pending scenario", function () { expect(rerunFormatter.getLogs()).toEqual(path.normalize('path/to/scenario:1')); }); });