Skip to content

Commit

Permalink
rerun formatter: output any scenario that doesn't pass
Browse files Browse the repository at this point in the history
  • Loading branch information
charlierudolph committed Sep 30, 2016
1 parent 93705c5 commit 6df223f
Show file tree
Hide file tree
Showing 5 changed files with 100 additions and 111 deletions.
2 changes: 1 addition & 1 deletion docs/cli.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ Built-in formatters
* json - prints the feature as JSON
* pretty - prints the feature as is (default)
* progress - prints one character per scenario
* rerun - prints the paths of the failing scenarios ([example](/features/rerun_formatter.feature))
* rerun - prints the paths of any non passing scenarios ([example](/features/rerun_formatter.feature))
* suggested use: add the rerun formatter to your default profile and the output file to your `.gitignore`
* snippets - prints just the code snippets for undefined steps
* summary - prints a summary only, after all scenarios were executed
Expand Down
150 changes: 43 additions & 107 deletions features/rerun_formatter.feature
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,29 +7,41 @@ Feature: Rerun Formatter
Given a file named "features/a.feature" with:
"""
Feature: A
Scenario: 1
Scenario: A - passing
Given a passing step
Scenario: 2
Scenario: A - failing
Given a failing step
Scenario: 3
Given a failing step
Scenario: A - ambiguous
Given an ambiguous step
"""
And a file named "features/b.feature" with:
"""
Feature: B
Scenario: 4
Scenario: B - passing
Given a passing step
Scenario: 5
Given a failing step
Scenario: B - pending
Given a pending step
"""
And a file named "features/c.feature" with:
"""
Feature: B
Scenario: C - passing
Given a passing step
Scenario: C - undefined
Given an undefined step
"""
And a file named "features/step_definitions/cucumber_steps.js" with:
"""
var cucumberSteps = function() {
this.When(/^a passing step$/, function() { });
this.When(/^a failing step$/, function() { throw 'fail' });
this.Given(/^a passing step$/, function() { });
this.Given(/^a failing step$/, function() { throw 'fail' });
this.Given(/^an ambiguous step$/, function() { });
this.Given(/^an? ambiguous step$/, function() { });
this.Given(/^a pending step$/, function() { return 'pending' });
};
module.exports = cucumberSteps;
"""
Expand All @@ -42,110 +54,34 @@ Feature: Rerun Formatter

Scenario: passing
When I run cucumber.js with `features/a.feature:2`
Then it outputs this text:
"""
Feature: A
Scenario: 1
✔ Given a passing step
1 scenario (1 passed)
1 step (1 passed)
<duration-stat>
"""
Then the exit status should be 0
And the file "@rerun.txt" has the text:
"""
"""

Scenario: multiple scenarios failing
When I run cucumber.js
Then it outputs this text:
"""
Feature: A
Scenario: 1
✔ Given a passing step
Scenario: 2
✖ Given a failing step
Scenario: 3
✖ Given a failing step
Feature: B
Scenario: 4
✔ Given a passing step
Scenario: 5
✖ Given a failing step
Failures:
1) Scenario: 2 - features/a.feature:5
Step: Given a failing step - features/a.feature:6
Step Definition: features/step_definitions/cucumber_steps.js:3
Message:
fail
2) Scenario: 3 - features/a.feature:8
Step: Given a failing step - features/a.feature:9
Step Definition: features/step_definitions/cucumber_steps.js:3
Message:
fail
3) Scenario: 5 - features/b.feature:5
Step: Given a failing step - features/b.feature:6
Step Definition: features/step_definitions/cucumber_steps.js:3
Message:
fail
5 scenarios (3 failed, 2 passed)
5 steps (3 failed, 2 passed)
<duration-stat>
"""
When I run cucumber.js with `-f json`
Then the exit status should be 1
And the json output has the scenarios with names
| NAME |
| A - passing |
| A - failing |
| A - ambiguous |
| B - passing |
| B - pending |
| C - passing |
| C - undefined |
And the file "@rerun.txt" has the text:
"""
features/a.feature:5:8
features/b.feature:5
"""
When I run cucumber.js with `@rerun.txt`
Then it outputs this text:
"""
Feature: A
Scenario: 2
✖ Given a failing step
Scenario: 3
✖ Given a failing step
Feature: B
Scenario: 5
✖ Given a failing step
Failures:
1) Scenario: 2 - features/a.feature:5
Step: Given a failing step - features/a.feature:6
Step Definition: features/step_definitions/cucumber_steps.js:3
Message:
fail
2) Scenario: 3 - features/a.feature:8
Step: Given a failing step - features/a.feature:9
Step Definition: features/step_definitions/cucumber_steps.js:3
Message:
fail
3) Scenario: 5 - features/b.feature:5
Step: Given a failing step - features/b.feature:6
Step Definition: features/step_definitions/cucumber_steps.js:3
Message:
fail
3 scenarios (3 failed)
3 steps (3 failed)
<duration-stat>
"""
features/c.feature:5
"""
When I run cucumber.js with `-f json @rerun.txt`
Then the exit status should be 1
And the json output has the scenarios with names
| NAME |
| A - failing |
| A - ambiguous |
| B - pending |
| C - undefined |
14 changes: 14 additions & 0 deletions features/step_definitions/json_output_steps.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,20 @@ var jsonOutputSteps = function jsonOutputSteps() {
assert.equal(scenario.name, name);
});

this.Then(/^the json output has the scenarios with names$/, function (table) {
var expectedNames = table.rows().map(function(row){ return row[0]; });
var features = JSON.parse(this.lastRun.stdout);
var actualNames = [];
features.forEach(function(feature) {
feature.elements.forEach(function(element){
if (element.type === 'scenario') {
actualNames.push(element.name);
}
});
});
assert.deepEqual(expectedNames, actualNames);
});

this.Then(/^the json output's first scenario has the description "([^"]*)"$/, function (description) {
var features = JSON.parse(this.lastRun.stdout);
assert.equal(features[0].elements[0].description.trim(), description);
Expand Down
2 changes: 1 addition & 1 deletion lib/cucumber/listener/rerun_formatter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ function RerunFormatter(options) {


self.handleScenarioResultEvent = function handleScenarioResultEvent(scenarioResult) {
if (scenarioResult.getStatus() === Cucumber.Status.FAILED) {
if (scenarioResult.getStatus() !== Cucumber.Status.PASSED) {
var scenario = scenarioResult.getScenario();
var uri = path.relative(process.cwd(), scenario.getUri());
var line = scenario.getLine();
Expand Down
43 changes: 41 additions & 2 deletions spec/cucumber/listener/rerun_formatter_spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ describe("Cucumber.Listener.RerunFormatter", function () {
}

describe("handleAfterFeaturesEvent()", function () {
describe('no failed scenarios', function() {
describe('all passing scenarios', function() {
beforeEach(function(done){
var scenarioResult = createScenarioResult(Cucumber.Status.PASSED);
rerunFormatter.handleScenarioResultEvent(scenarioResult);
Expand All @@ -35,7 +35,46 @@ describe("Cucumber.Listener.RerunFormatter", function () {
rerunFormatter.handleAfterFeaturesEvent(null, done);
});

it("logs nothing", function () {
it("logs the path to the failed scenario", function () {
expect(rerunFormatter.getLogs()).toEqual(path.normalize('path/to/scenario:1'));
});
});

describe('one ambiguous scenario', function() {
beforeEach(function(done) {
var scenarioResult = createScenarioResult(Cucumber.Status.AMBIGUOUS, 'path/to/scenario', 1);
rerunFormatter.handleScenarioResultEvent(scenarioResult);

rerunFormatter.handleAfterFeaturesEvent(null, done);
});

it("logs the path to the ambiguous scenario", function () {
expect(rerunFormatter.getLogs()).toEqual(path.normalize('path/to/scenario:1'));
});
});

describe('one undefined scenario', function() {
beforeEach(function(done) {
var scenarioResult = createScenarioResult(Cucumber.Status.UNDEFINED, 'path/to/scenario', 1);
rerunFormatter.handleScenarioResultEvent(scenarioResult);

rerunFormatter.handleAfterFeaturesEvent(null, done);
});

it("logs the path to the undefined scenario", function () {
expect(rerunFormatter.getLogs()).toEqual(path.normalize('path/to/scenario:1'));
});
});

describe('one pending scenario', function() {
beforeEach(function(done) {
var scenarioResult = createScenarioResult(Cucumber.Status.PENDING, 'path/to/scenario', 1);
rerunFormatter.handleScenarioResultEvent(scenarioResult);

rerunFormatter.handleAfterFeaturesEvent(null, done);
});

it("logs the path to the pending scenario", function () {
expect(rerunFormatter.getLogs()).toEqual(path.normalize('path/to/scenario:1'));
});
});
Expand Down

0 comments on commit 6df223f

Please sign in to comment.