From 6e55cdccbd0d618d314985284a6f2e836a234678 Mon Sep 17 00:00:00 2001 From: Julien Biezemans Date: Mon, 1 Aug 2011 22:02:59 +0200 Subject: [PATCH] Add failure reporting (close #20) - Exception messages and stack traces are now reported by the progress formatter - Failed scenario names and line numbers are also printed out --- Gemfile | 2 +- Gemfile.lock | 16 +- cucumber.js | 2 +- example/index.html | 2 +- features/cucumber-features | 2 +- features/legacy/progress_formatter.feature | 133 ---------- features/step_definitions/calculator_steps.js | 6 +- .../step_definitions/cucumber_js_mappings.rb | 18 +- features/step_definitions/cucumber_steps.js | 52 +++- .../step_definitions/legacy/cucumber_steps.js | 7 +- lib/cucumber.js | 2 +- lib/cucumber/ast/scenario.js | 6 +- lib/cucumber/debug.js | 10 +- lib/cucumber/debug/simple_ast_listener.js | 3 - lib/cucumber/listener/progress_formatter.js | 125 ++++++---- lib/cucumber/runtime/failed_step_result.js | 8 +- lib/cucumber/support_code/step_definition.js | 14 +- run_all_features.js | 5 +- spec/cucumber/ast/scenario_spec.js | 16 +- .../listener/progress_formatter_spec.js | 235 ++++++++++++++++-- .../runtime/failed_step_result_spec.js | 11 +- .../support_code/step_definition_spec.js | 7 +- spec/support/spec_helper.js | 5 +- 23 files changed, 435 insertions(+), 252 deletions(-) delete mode 100644 features/legacy/progress_formatter.feature diff --git a/Gemfile b/Gemfile index d5fca41b3..e520bdd59 100644 --- a/Gemfile +++ b/Gemfile @@ -1,2 +1,2 @@ source :rubygems -gem "aruba", "0.4.3" +gem "aruba", "0.4.5" diff --git a/Gemfile.lock b/Gemfile.lock index 66a63b9b7..60531b114 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GEM remote: http://rubygems.org/ specs: - aruba (0.4.3) + aruba (0.4.5) bcat (>= 0.6.1) childprocess (>= 0.1.9) cucumber (>= 0.10.7) @@ -10,20 +10,20 @@ GEM bcat (0.6.1) rack (~> 1.0) builder (3.0.0) - childprocess (0.1.9) + childprocess (0.2.0) ffi (~> 1.0.6) - cucumber (1.0.0) + cucumber (1.0.2) builder (>= 2.1.2) diff-lcs (>= 1.1.2) - gherkin (~> 2.4.1) + gherkin (~> 2.4.5) json (>= 1.4.6) term-ansicolor (>= 1.0.5) diff-lcs (1.1.2) ffi (1.0.9) - gherkin (2.4.1) + gherkin (2.4.5) json (>= 1.4.6) json (1.5.3) - rack (1.3.0) + rack (1.3.2) rdiscount (1.6.8) rspec (2.6.0) rspec-core (~> 2.6.0) @@ -33,10 +33,10 @@ GEM rspec-expectations (2.6.0) diff-lcs (~> 1.1.2) rspec-mocks (2.6.0) - term-ansicolor (1.0.5) + term-ansicolor (1.0.6) PLATFORMS ruby DEPENDENCIES - aruba (= 0.4.3) + aruba (= 0.4.5) diff --git a/cucumber.js b/cucumber.js index 68d4a28c0..712cf2bba 100755 --- a/cucumber.js +++ b/cucumber.js @@ -5,7 +5,7 @@ var featurePath = process.ARGV[2]; var supportCodePath = process.ARGV[3] ? process.cwd() + '/' + process.ARGV[3] : './features/step_definitions/cucumber_steps'; if (typeof(featurePath) == 'undefined') { - throw("Please give me a feature, try something like `" + process.ARGV[1] + " features/cucumber-features/core.feature`."); + throw(new Error("Please give me a feature, try something like `" + process.ARGV[1] + " features/cucumber-features/core.feature`.")); } var supportCode = require(supportCodePath); diff --git a/example/index.html b/example/index.html index a3a11b657..abf8ace76 100644 --- a/example/index.html +++ b/example/index.html @@ -36,7 +36,7 @@

Step definitions

Then(/^the variable should contain (\d+)$/, function(number, callback) { if (variable != parseInt(number)) - throw('Variable should contain '+number+' but it contains '+variable+'.'); + throw(new Error('Variable should contain '+number+' but it contains '+variable+'.')); callback(); }); diff --git a/features/cucumber-features b/features/cucumber-features index 066a82dee..65c6112cf 160000 --- a/features/cucumber-features +++ b/features/cucumber-features @@ -1 +1 @@ -Subproject commit 066a82deea93382cb8e185cf47c7647ca5ae1ec5 +Subproject commit 65c6112cfa6a6a27bae6d2e42b0b10824bd12bc9 diff --git a/features/legacy/progress_formatter.feature b/features/legacy/progress_formatter.feature deleted file mode 100644 index 39713431d..000000000 --- a/features/legacy/progress_formatter.feature +++ /dev/null @@ -1,133 +0,0 @@ -Feature: progress formatter - In order to get quick feedback when doing BDD - As a developer - I want to use a "progress" formatter - - Scenario: one scenario, one step, passing - Given a step definition matching /a passing step/ - When I run the following feature with the "progress" formatter: - """ - Feature: - Scenario: - Given a passing step - """ - Then the listener should output the following: - """ - . - - 1 scenario (1 passed) - 1 step (1 passed) - """ - - Scenario: one scenario, two steps, passing - Given a step definition matching /a passing step/ - When I run the following feature with the "progress" formatter: - """ - Feature: - Scenario: - Given a passing step - And a passing step - """ - Then the listener should output the following: - """ - .. - - 1 scenario (1 passed) - 2 steps (2 passed) - """ - - Scenario: two scenarios, five steps, passing - Given a step definition matching /a passing step/ - When I run the following feature with the "progress" formatter: - """ - Feature: - Scenario: - Given a passing step - And a passing step - Scenario: - Given a passing step - And a passing step - When a passing step - """ - Then the listener should output the following: - """ - .. - - 2 scenarios (2 passed) - 5 steps (5 passed) - """ - - Scenario: one scenario, one step, failing - Given a step definition failing with message "boom" matching /a failing step/ - When I run the following feature with the "progress" formatter: - """ - Feature: - Scenario: - Given a failing step - """ - Then the listener should output the following: - """ - F - - 1 scenario (1 failed) - 1 step (1 failed) - """ - - Scenario: one scenario, two steps, second failing - Given a step definition matching /a passing step/ - And a step definition failing with message "boom" matching /a failing step/ - When I run the following feature with the "progress" formatter: - """ - Feature: - Scenario: - Given a passing step - When a failing step - """ - Then the listener should output the following: - """ - .F - - 1 scenario (1 failed) - 2 steps (1 failed, 1 passed) - """ - - Scenario: one two-step passing scenario, one two-step scenario with latest step failing - Given a step definition matching /a passing step/ - And a step definition failing with message "boom" matching /a failing step/ - When I run the following feature with the "progress" formatter: - """ - Feature: - Scenario: - Given a passing step - When a passing step - Scenario: - Given a passing step - When a failing step - """ - Then the listener should output the following: - """ - ...F - - 2 scenarios (1 failed, 1 passed) - 4 steps (1 failed, 3 passed) - """ - - Scenario: one failing scenario with a skipped step - Given a step definition matching /a passing step/ - And a step definition matching /a skipped step/ - And a step definition failing with message "boom" matching /a failing step/ - When I run the following feature with the "progress" formatter: - """ - Feature: - Scenario: - Given a passing step - When a failing step - Then a skipped step - """ - Then the listener should output the following: - """ - .F- - - 1 scenario (1 failed) - 3 steps (1 failed, 1 skipped, 1 passed) - """ diff --git a/features/step_definitions/calculator_steps.js b/features/step_definitions/calculator_steps.js index 9cc08d634..59639f8b5 100644 --- a/features/step_definitions/calculator_steps.js +++ b/features/step_definitions/calculator_steps.js @@ -57,21 +57,21 @@ var calculatorSteps = function(Calculator) { Then(/^the calculator returns PI$/, function(callback) { var value = calc.value(); if (!isNumberWithinRangeOfValue(value, 0.00001, Math.PI)) - throw("Expected " + Math.PI + " (PI), got " + value); + throw(new Error("Expected " + Math.PI + " (PI), got " + value)); callback(); }); Then(/^the calculator returns "([^"]*)"$/, function(expected_number, callback) { var value = calc.value(); if (!isNumberWithinRangeOfValue(value, 0.00001, parseFloat(expected_number))) - throw("Expected calculator to return a value within 0.00001 of " + expected_number + ", got " + value); + throw(new Error("Expected calculator to return a value within 0.00001 of " + expected_number + ", got " + value)); callback(); }); Then(/^the calculator does not return ([\d\.]+)$/, function(unexpected_number, callback) { var value = calc.value(); if (isNumberWithinRangeOfValue(value, 0.00001, parseFloat(unexpected_number))) - throw("Expected calculator to not return a value within 0.00001 of " + unexpected_number + ", got " + value); + throw(new Error("Expected calculator to not return a value within 0.00001 of " + unexpected_number + ", got " + value)); callback(); }); }; diff --git a/features/step_definitions/cucumber_js_mappings.rb b/features/step_definitions/cucumber_js_mappings.rb index 126d65ef7..bd238b706 100644 --- a/features/step_definitions/cucumber_js_mappings.rb +++ b/features/step_definitions/cucumber_js_mappings.rb @@ -25,7 +25,7 @@ def cucumber_bin end def write_passing_mapping(step_name) - append_step_definition(step_name, "// no-op, pass gently") + append_step_definition(step_name, "// no-op, pass gently\ncallback();") end def write_pending_mapping(step_name) @@ -33,7 +33,11 @@ def write_pending_mapping(step_name) end def write_failing_mapping(step_name) - append_step_definition(step_name, "throw('Boom!');") + write_failing_mapping_with_message(step_name, "I was supposed to fail.") + end + + def write_failing_mapping_with_message(step_name, message) + append_step_definition(step_name, "throw(new Error('#{message}'));") end def write_calculator_code @@ -72,6 +76,15 @@ def assert_undefined_scenario assert_success true end + def assert_scenario_reported_as_failing(scenario_name) + assert_partial_output("# Scenario: #{scenario_name}", all_output) + assert_success false + end + + def assert_scenario_not_reported_as_failing(scenario_name) + assert_no_partial_output("# Scenario: #{scenario_name}", all_output) + end + def failed_output "failed" end @@ -84,7 +97,6 @@ def append_step_definition(step_name, code) Given(/#{step_name}/, function(callback) { fs.writeFileSync("#{step_file(step_name)}", ""); #{indented_code} - callback(); }); EOF end diff --git a/features/step_definitions/cucumber_steps.js b/features/step_definitions/cucumber_steps.js index 7616d0e33..27061adc5 100644 --- a/features/step_definitions/cucumber_steps.js +++ b/features/step_definitions/cucumber_steps.js @@ -27,7 +27,16 @@ var cucumberSteps = function() { prepare(); stepDefinitions += "Given(/^" + stepName + "$/, function(callback) {\ touchStep(\"" + stepName + "\");\ - throw('I was supposed to fail.');\ + throw(new Error('I was supposed to fail.'));\ +});\n"; + callback(); + }); + + Given(/^the step "([^"]*)" has a mapping failing with the message "([^"]*)"$/, function(stepName, message, callback) { + prepare(); + stepDefinitions += "Given(/^" + stepName + "$/, function(callback) {\ + touchStep(\"" + stepName + "\");\ + throw(new Error('" + message + "'));\ });\n"; callback(); }); @@ -63,7 +72,7 @@ var cucumberSteps = function() { Then(/^the scenario passes$/, function(callback) { if (!lastRunSucceeded) - throw("Expected the scenario to pass but it failed"); + throw(new Error("Expected the scenario to pass but it failed")); callback(); }); @@ -82,6 +91,16 @@ var cucumberSteps = function() { callback(); }); + Then(/^the scenario called "([^"]*)" is reported as failing$/, function(scenarioName, callback) { + assertScenarioReportedAsFailing(scenarioName); + callback(); + }); + + Then(/^the scenario called "([^"]*)" is not reported as failing$/, function(scenarioName, callback) { + assertScenarioNotReportedAsFailing(scenarioName); + callback(); + }); + Then(/^the step "([^"]*)" is skipped$/, function(stepName, callback) { assertSkippedStep(stepName); callback(); @@ -92,6 +111,11 @@ var cucumberSteps = function() { callback(); }); + Then(/^the failure message "([^"]*)" is output$/, function(message, callback) { + assertFailureMessage(message); + callback(); + }); + function prepare() { if (shouldPrepare) { shouldPrepare = false; @@ -150,29 +174,43 @@ var cucumberSteps = function() { assertSuccess(); } + function assertScenarioReportedAsFailing(scenarioName) { + assertPartialOutput("# Scenario: " + scenarioName, lastRunOutput); + assertFailure(); + } + + function assertScenarioNotReportedAsFailing(scenarioName) { + assertNoPartialOutput("# Scenario: " + scenarioName, lastRunOutput); + } + function assertSkippedStep(stepName) { if (isStepTouched(stepName)) - throw("Expected step \"" + stepName + "\" to have been skipped."); + throw(new Error("Expected step \"" + stepName + "\" to have been skipped.")); } function assertSuccess() { if (!lastRunSucceeded) - throw("Expected Cucumber to succeed but it failed."); + throw(new Error("Expected Cucumber to succeed but it failed.")); } function assertFailure() { if (lastRunSucceeded) - throw("Expected Cucumber to fail but it succeeded."); + throw(new Error("Expected Cucumber to fail but it succeeded.")); + } + + function assertFailureMessage(message) { + assertPartialOutput(message, lastRunOutput); + assertFailure(); } function assertPartialOutput(expected, actual) { if (actual.indexOf(expected) < 0) - throw("Expected:\n\"" + actual + "\"\nto match:\n\"" + expected + "\""); + throw(new Error("Expected:\n\"" + actual + "\"\nto match:\n\"" + expected + "\"")); } function assertNoPartialOutput(expected, actual) { if (actual.indexOf(expected) >= 0) - throw("Expected:\n\"" + actual + "\"\nnot to match:\n\"" + expected + "\""); + throw(new Error("Expected:\n\"" + actual + "\"\nnot to match:\n\"" + expected + "\"")); } }; module.exports = cucumberSteps; diff --git a/features/step_definitions/legacy/cucumber_steps.js b/features/step_definitions/legacy/cucumber_steps.js index 8999e4c51..6e7542b4d 100644 --- a/features/step_definitions/legacy/cucumber_steps.js +++ b/features/step_definitions/legacy/cucumber_steps.js @@ -38,7 +38,7 @@ var stepDefinitions = function() { // The created step definition body should: // 1. Fail all the time. Given(/^a(?: "(Given|When|Then)")? step definition failing with message "(.*)" matching \/(.*)\/$/, function(keyword, errorMessage, name, callback) { - var content = function(callback) { throw(errorMessage); }; + var content = function(callback) { throw(new Error(errorMessage)); }; _addStepDefinition(keyword, name, content); callback(); }); @@ -204,11 +204,6 @@ var stepDefinitions = function() { function _buildListener(listenerConstructor) { _listener = listenerConstructor({logToConsole: false}); - _listener.beforeEachScenarioDo(function() { - _stepDefs = []; - _recordedStepParameters = []; - _stepCallCount = 0; - }); }; function _addStepDefinition(keyword, name, content) { diff --git a/lib/cucumber.js b/lib/cucumber.js index 444ccf8e6..c6df693eb 100644 --- a/lib/cucumber.js +++ b/lib/cucumber.js @@ -4,7 +4,7 @@ var Cucumber = function(featuresSource, supportCodeDefinition) { var self = { start: function start(callback) { if (typeof(callback) !== 'function') - throw Cucumber.START_MISSING_CALLBACK_ERROR; + throw new Error(Cucumber.START_MISSING_CALLBACK_ERROR); var features = self.parseFeaturesSource(featuresSource); var supportCodeLibrary = self.initializeSupportCode(supportCodeDefinition); self.executeFeaturesAgainstSupportCodeLibrary(features, supportCodeLibrary, callback); diff --git a/lib/cucumber/ast/scenario.js b/lib/cucumber/ast/scenario.js index 89afcd19f..19dda5e4e 100644 --- a/lib/cucumber/ast/scenario.js +++ b/lib/cucumber/ast/scenario.js @@ -1,4 +1,4 @@ -var Scenario = function(keyword, name) { +var Scenario = function(keyword, name, description, line) { var Cucumber = require('../../cucumber'); var steps = Cucumber.Type.Collection(); @@ -12,6 +12,10 @@ var Scenario = function(keyword, name) { return name; }, + getLine: function getLine() { + return line; + }, + addStep: function addStep(step) { steps.add(step); }, diff --git a/lib/cucumber/debug.js b/lib/cucumber/debug.js index f0e664cd8..1308d5cf3 100644 --- a/lib/cucumber/debug.js +++ b/lib/cucumber/debug.js @@ -1,6 +1,6 @@ var Debug = { TODO: function TODO(description) { - return function() { throw("IMPLEMENT ME: " + description); }; + return function() { throw(new Error("IMPLEMENT ME: " + description)); }; }, warn: function warn(string, caption, level) { @@ -28,8 +28,12 @@ var Debug = { }, isMessageLeveltoBeDisplayed: function isMessageLeveltoBeDisplayed(level) { - level = level || 3; // default level - return (level <= process.env['DEBUG_LEVEL']); + if (process.env) { + level = level || 3; // default level + return (level <= process.env['DEBUG_LEVEL']); + } else { + return false; + } } }; Debug.SimpleAstListener = require('./debug/simple_ast_listener'); diff --git a/lib/cucumber/debug/simple_ast_listener.js b/lib/cucumber/debug/simple_ast_listener.js index 55481635a..7da2b6656 100644 --- a/lib/cucumber/debug/simple_ast_listener.js +++ b/lib/cucumber/debug/simple_ast_listener.js @@ -56,9 +56,6 @@ var SimpleAstListener = function(options) { log(currentStep.getDocString().getString(), 3); log('"""', 3); }; - if (!stepResult.isSuccessful()) { - log('--- FAILED ---', 3); - } callback(); }, diff --git a/lib/cucumber/listener/progress_formatter.js b/lib/cucumber/listener/progress_formatter.js index 8a3c0acd6..3bb877bf2 100644 --- a/lib/cucumber/listener/progress_formatter.js +++ b/lib/cucumber/listener/progress_formatter.js @@ -1,30 +1,27 @@ var ProgressFormatter = function(options) { var Cucumber = require('../../cucumber'); - var beforeEachScenarioUserFunctions = Cucumber.Type.Collection(); - var logs = ""; - var passedScenarios = 0; - var undefinedScenarios = 0; - var pendingScenarios = 0; - var failedScenarios = 0; - var passedSteps = 0; - var failedSteps = 0; - var skippedSteps = 0; - var undefinedSteps = 0; - var pendingSteps = 0; - var currentScenarioFailing = false; - var currentScenarioUndefined = false; - var currentScenarioPending = false; + var logs = ""; + var failedScenarioLogBuffer = ""; + var passedScenarioCount = 0; + var undefinedScenarioCount = 0; + var pendingScenarioCount = 0; + var failedScenarioCount = 0; + var passedStepCount = 0; + var failedStepCount = 0; + var skippedStepCount = 0; + var undefinedStepCount = 0; + var pendingStepCount = 0; + var currentScenarioFailing = false; + var currentScenarioUndefined = false; + var currentScenarioPending = false; + var failedStepResults = Cucumber.Type.Collection(); if (!options) options = {}; if (options['logToConsole'] == undefined) options['logToConsole'] = true; var self = { - beforeEachScenarioDo: function beforeEachScenarioDo(userFunction) { - beforeEachScenarioUserFunctions.add(userFunction); - }, - log: function log(string) { logs += string; if (options['logToConsole']) @@ -79,6 +76,7 @@ var ProgressFormatter = function(options) { self.markCurrentScenarioAsPending(); self.log(ProgressFormatter.PENDING_STEP_CHARACTER); } else { + self.storeFailedStepResult(stepResult); self.witnessFailedStep(); self.markCurrentScenarioAsFailing(); self.log(ProgressFormatter.FAILED_STEP_CHARACTER); @@ -106,6 +104,8 @@ var ProgressFormatter = function(options) { handleAfterScenarioEvent: function handleAfterScenarioEvent(event, callback) { if (self.isCurrentScenarioFailing()) { + var scenario = event.getPayloadItem('scenario'); + self.storeFailedScenario(scenario); self.witnessFailedScenario(); } else if (self.isCurrentScenarioUndefined()) { self.witnessUndefinedScenario(); @@ -147,13 +147,50 @@ var ProgressFormatter = function(options) { return currentScenarioPending; }, + storeFailedStepResult: function storeFailedStepResult(failedStepResult) { + failedStepResults.add(failedStepResult); + }, + + storeFailedScenario: function storeFailedScenario(failedScenario) { + var name = failedScenario.getName(); + var line = failedScenario.getLine(); + self.appendStringToFailedScenarioLogBuffer(":" + line + " # Scenario: " + name); + }, + + appendStringToFailedScenarioLogBuffer: function appendStringToFailedScenarioLogBuffer(string) { + failedScenarioLogBuffer += string + "\n"; + }, + + getFailedScenarioLogBuffer: function getFailedScenarioLogBuffer() { + return failedScenarioLogBuffer; + }, + logSummary: function logSummary() { self.log("\n\n"); + if (self.witnessedAnyFailedStep()) + self.logFailedStepResults(); self.logScenariosSummary(); self.logStepsSummary(); self.log("\n"); }, + logFailedStepResults: function logFailedStepResults() { + self.log("(::) failed steps (::)\n\n"); + failedStepResults.syncForEach(function(stepResult) { + self.logFailedStepResult(stepResult); + }); + self.log("Failing scenarios:\n"); + var failedScenarios = self.getFailedScenarioLogBuffer(); + self.log(failedScenarios); + self.log("\n"); + }, + + logFailedStepResult: function logFailedStepResult(stepResult) { + var failureMessage = stepResult.getFailureException(); + self.log(failureMessage.stack || failureMessage); + self.log("\n\n"); + }, + logScenariosSummary: function logScenariosSummary() { var scenarioCount = self.getScenarioCount(); var passedScenarioCount = self.getPassedScenarioCount(); @@ -189,79 +226,79 @@ var ProgressFormatter = function(options) { self.log(stepCount + " step" + (stepCount != 1 ? "s" : "")); if (stepCount > 0) { if (failedStepCount > 0) - details.push(failedStepCount + " failed"); + details.push(failedStepCount + " failed"); if (undefinedStepCount > 0) details.push(undefinedStepCount + " undefined"); if (pendingStepCount > 0) - details.push(pendingStepCount + " pending"); + details.push(pendingStepCount + " pending"); if (skippedStepCount > 0) - details.push(skippedStepCount + " skipped"); + details.push(skippedStepCount + " skipped"); if (passedStepCount > 0) - details.push(passedStepCount + " passed"); + details.push(passedStepCount + " passed"); self.log(" (" + details.join(', ') + ")"); } self.log("\n"); }, witnessPassedScenario: function witnessPassedScenario() { - passedScenarios++; + passedScenarioCount++; }, witnessUndefinedScenario: function witnessUndefinedScenario() { - undefinedScenarios++; + undefinedScenarioCount++; }, witnessPendingScenario: function witnessPendingScenario() { - pendingScenarios++; + pendingScenarioCount++; }, witnessFailedScenario: function witnessFailedScenario() { - failedScenarios++; + failedScenarioCount++; }, witnessPassedStep: function witnessPassedStep() { - passedSteps++; + passedStepCount++; }, witnessUndefinedStep: function witnessUndefinedStep() { - undefinedSteps++; + undefinedStepCount++; }, witnessPendingStep: function witnessPendingStep() { - pendingSteps++; + pendingStepCount++; }, witnessFailedStep: function witnessFailedStep() { - failedSteps++; + failedStepCount++; }, witnessSkippedStep: function witnessSkippedStep() { - skippedSteps++; + skippedStepCount++; }, getScenarioCount: function getScenarioCount() { var scenarioCount = - self.getPassedScenarioCount() + + self.getPassedScenarioCount() + self.getUndefinedScenarioCount() + - self.getPendingScenarioCount() + + self.getPendingScenarioCount() + self.getFailedScenarioCount(); return scenarioCount; }, getPassedScenarioCount: function getPassedScenarioCount() { - return passedScenarios; + return passedScenarioCount; }, getUndefinedScenarioCount: function getUndefinedScenarioCount() { - return undefinedScenarios; + return undefinedScenarioCount; }, getPendingScenarioCount: function getPendingScenarioCount() { - return pendingScenarios; + return pendingScenarioCount; }, getFailedScenarioCount: function getFailedScenarioCount() { - return failedScenarios; + return failedScenarioCount; }, getStepCount: function getStepCount() { @@ -275,23 +312,27 @@ var ProgressFormatter = function(options) { }, getPassedStepCount: function getPassedStepCount() { - return passedSteps; + return passedStepCount; }, getPendingStepCount: function getPendingStepCount() { - return pendingSteps; + return pendingStepCount; }, getFailedStepCount: function getFailedStepCount() { - return failedSteps; + return failedStepCount; }, getSkippedStepCount: function getSkippedStepCount() { - return skippedSteps; + return skippedStepCount; }, getUndefinedStepCount: function getUndefinedStepCount() { - return undefinedSteps; + return undefinedStepCount; + }, + + witnessedAnyFailedStep: function witnessedAnyFailedStep() { + return failedStepCount > 0; } }; return self; diff --git a/lib/cucumber/runtime/failed_step_result.js b/lib/cucumber/runtime/failed_step_result.js index 147bf6bbb..08bb78080 100644 --- a/lib/cucumber/runtime/failed_step_result.js +++ b/lib/cucumber/runtime/failed_step_result.js @@ -1,8 +1,12 @@ -var FailedStepResult = function() { +var FailedStepResult = function(failureException) { var self = { isSuccessful: function isSuccessful() { return false; }, isPending: function isPending() { return false; }, - isFailed: function isFailed() { return true; } + isFailed: function isFailed() { return true; }, + + getFailureException: function getFailureException() { + return failureException; + } }; return self; }; diff --git a/lib/cucumber/support_code/step_definition.js b/lib/cucumber/support_code/step_definition.js index 394de721c..5a8da73e5 100644 --- a/lib/cucumber/support_code/step_definition.js +++ b/lib/cucumber/support_code/step_definition.js @@ -18,12 +18,14 @@ var StepDefinition = function(regexp, code) { var parameters = self.buildInvocationParameters(stepName, docString, codeCallback); try { code.apply(undefined, parameters); - } catch (err) { - if (err) - Cucumber.Debug.warn(err, 'exception inside feature', 3); - var stepResult = (err instanceof Cucumber.Runtime.PendingStepException) ? - Cucumber.Runtime.PendingStepResult() : - Cucumber.Runtime.FailedStepResult(); + } catch (exception) { + if (exception) + Cucumber.Debug.warn(exception.stack || exception, 'exception inside feature', 3); + var stepResult; + if (exception instanceof Cucumber.Runtime.PendingStepException) + stepResult = Cucumber.Runtime.PendingStepResult() + else + stepResult = Cucumber.Runtime.FailedStepResult(exception); callback(stepResult); } }, diff --git a/run_all_features.js b/run_all_features.js index 306e467d7..f8f63858f 100755 --- a/run_all_features.js +++ b/run_all_features.js @@ -4,7 +4,10 @@ var util = require('util'); var spawn = require('child_process').spawn; runFeaturesInDir('features/legacy', 'features/step_definitions/legacy/cucumber_steps.js', function() { - runFeature('features/cucumber-features/core.feature', 'features/step_definitions/cucumber_steps.js', function() {}); + runFeature('features/cucumber-features/core.feature', 'features/step_definitions/cucumber_steps.js', function() { + runFeature('features/cucumber-features/failing_steps.feature', 'features/step_definitions/cucumber_steps.js', function() { + }); + }); }); diff --git a/spec/cucumber/ast/scenario_spec.js b/spec/cucumber/ast/scenario_spec.js index e181142e0..56ae90467 100644 --- a/spec/cucumber/ast/scenario_spec.js +++ b/spec/cucumber/ast/scenario_spec.js @@ -3,18 +3,20 @@ require('../../support/spec_helper'); describe("Cucumber.Ast.Scenario", function() { var Cucumber = require('cucumber'); var stepCollection, steps; - var scenario, keyword, name, lastStep; + var scenario, keyword, name, description, line, lastStep; beforeEach(function() { - keyword = createSpy("Feature keyword"); - name = createSpy("Feature name"); + keyword = createSpy("scenario keyword"); + name = createSpy("scenario name"); + description = createSpy("scenario description"); + line = createSpy("starting scenario line number"); lastStep = createSpy("Last step"); stepCollection = createSpy("Step collection"); spyOnStub(stepCollection, 'add'); spyOnStub(stepCollection, 'getLast').andReturn(lastStep); spyOnStub(stepCollection, 'forEach'); spyOn(Cucumber.Type, 'Collection').andReturn(stepCollection); - scenario = Cucumber.Ast.Scenario(keyword, name); + scenario = Cucumber.Ast.Scenario(keyword, name, description, line); }); describe("constructor", function() { @@ -35,6 +37,12 @@ describe("Cucumber.Ast.Scenario", function() { }); }); + describe("getLine()", function() { + it("returns the line on which the scenario starts", function() { + expect(scenario.getLine()).toBe(line); + }); + }); + describe("addStep()", function() { it("adds the step to the steps (collection)", function() { var step = createSpy("Step AST element"); diff --git a/spec/cucumber/listener/progress_formatter_spec.js b/spec/cucumber/listener/progress_formatter_spec.js index 91652e5a6..fa5f9b72d 100644 --- a/spec/cucumber/listener/progress_formatter_spec.js +++ b/spec/cucumber/listener/progress_formatter_spec.js @@ -2,32 +2,20 @@ require('../../support/spec_helper'); describe("Cucumber.Listener.ProgressFormatter", function() { var Cucumber = require('cucumber'); - var listener, beforeEachScenarioUserFunctions; + var listener, failedStepResults; beforeEach(function() { - beforeEachScenarioUserFunctions = createSpy("User Functions to call before each scenario"); - spyOn(Cucumber.Type, 'Collection').andReturn(beforeEachScenarioUserFunctions); - listener = Cucumber.Listener.ProgressFormatter(); + failedStepResults = createSpy("Failed steps"); + spyOn(Cucumber.Type, 'Collection').andReturn(failedStepResults); + listener = Cucumber.Listener.ProgressFormatter(); }); describe("constructor", function() { - it("creates a new collection to store user functions to call before each scenario", function() { + it("creates a collection to store the failed steps", function() { expect(Cucumber.Type.Collection).toHaveBeenCalled(); }); }); - describe("beforeEachScenarioDo()", function() { - beforeEach(function() { - spyOnStub(beforeEachScenarioUserFunctions, 'add'); - }); - - it("adds the user function to the collection of 'before each scenario' user functions", function() { - var userFunction = createSpy("A user function to call before each scenario"); - listener.beforeEachScenarioDo(userFunction); - expect(beforeEachScenarioUserFunctions.add).toHaveBeenCalledWith(userFunction); - }); - }); - describe("log()", function() { var logged, alsoLogged, loggedBuffer; @@ -252,6 +240,7 @@ describe("Cucumber.Listener.ProgressFormatter", function() { event = createSpyWithStubs("event", {getPayloadItem: stepResult}); callback = createSpy("Callback"); spyOn(listener, 'witnessPassedStep'); + spyOnStub(listener, 'storeFailedStepResult'); }); it("gets the step result from the event payload", function() { @@ -342,6 +331,11 @@ describe("Cucumber.Listener.ProgressFormatter", function() { expect(listener.log).not.toHaveBeenCalledWith(Cucumber.Listener.ProgressFormatter.PASSED_STEP_CHARACTER); }); + it("stores the failed step result", function() { + listener.handleStepResultEvent(event, callback); + expect(listener.storeFailedStepResult).toHaveBeenCalledWith(stepResult); + }); + it("witnesses a failed step", function() { listener.handleStepResultEvent(event, callback); expect(listener.witnessFailedStep).toHaveBeenCalled(); @@ -482,14 +476,29 @@ describe("Cucumber.Listener.ProgressFormatter", function() { }); describe("when the current scenario failed", function() { + var scenario; + beforeEach(function() { + scenario = createSpy("scenario"); listener.isCurrentScenarioFailing.andReturn(true); + spyOn(listener, 'storeFailedScenario'); + spyOnStub(event, 'getPayloadItem').andReturn(scenario); }); it("witnesses a failed scenario", function() { listener.handleAfterScenarioEvent(event, callback); expect(listener.witnessFailedScenario).toHaveBeenCalled(); }); + + it("gets the scenario from the payload", function() { + listener.handleAfterScenarioEvent(event, callback); + expect(event.getPayloadItem).toHaveBeenCalledWith('scenario'); + }); + + it("stores the failed scenario", function() { + listener.handleAfterScenarioEvent(event, callback); + expect(listener.storeFailedScenario).toHaveBeenCalledWith(scenario); + }); }); describe("when the current scenario did not fail", function() { @@ -607,12 +616,68 @@ describe("Cucumber.Listener.ProgressFormatter", function() { }); }); - describe("logSummary", function() { + describe("storeFailedStepResult()", function() { + var failedStepResult; + + beforeEach(function() { + failedStepResult = createSpy("failed step result"); + spyOnStub(failedStepResults, 'add'); + }); + + it("adds the result to the failed step result collection", function() { + listener.storeFailedStepResult(failedStepResult); + expect(failedStepResults.add).toHaveBeenCalledWith(failedStepResult); + }); + }); + + describe("storeFailedScenario()", function() { + var failedScenario, name, line; + + beforeEach(function() { + name = "some failed scenario"; + line = "123"; + string = ":" + line + " # Scenario: " + name; + failedScenario = createSpyWithStubs("failedScenario", {getName: name, getLine: line}); + spyOn(listener, 'appendStringToFailedScenarioLogBuffer'); + }); + + it("gets the name of the scenario", function() { + listener.storeFailedScenario(failedScenario); + expect(failedScenario.getName).toHaveBeenCalled(); + }); + + it("gets the line of the scenario", function() { + listener.storeFailedScenario(failedScenario); + expect(failedScenario.getLine).toHaveBeenCalled(); + }); + + it("appends the scenario details to the failed scenario log buffer", function() { + listener.storeFailedScenario(failedScenario); + expect(listener.appendStringToFailedScenarioLogBuffer).toHaveBeenCalledWith(string); + }); + }); + + describe("getFailedScenarioLogBuffer()", function() { + it("returns the logged failed scenario details", function() { + listener.appendStringToFailedScenarioLogBuffer("abc"); + expect(listener.getFailedScenarioLogBuffer()).toBe("abc\n"); + }); + + it("returns all logged failed scenario lines joined with a line break", function() { + listener.appendStringToFailedScenarioLogBuffer("abc"); + listener.appendStringToFailedScenarioLogBuffer("def"); + expect(listener.getFailedScenarioLogBuffer()).toBe("abc\ndef\n"); + }); + }); + + describe("logSummary()", function() { var scenarioCount, passedScenarioCount, failedScenarioCount; var stepCount, passedStepCount; beforeEach(function() { spyOn(listener, 'log'); + spyOn(listener, 'witnessedAnyFailedStep'); + spyOn(listener, 'logFailedStepResults'); spyOn(listener, 'logScenariosSummary'); spyOn(listener, 'logStepsSummary'); }); @@ -622,6 +687,33 @@ describe("Cucumber.Listener.ProgressFormatter", function() { expect(listener.log).toHaveBeenCalledWith("\n\n"); }); + it("checks if there are failed steps", function() { + listener.logSummary(); + expect(listener.witnessedAnyFailedStep).toHaveBeenCalled(); + }); + + describe("when there are failed steps", function() { + beforeEach(function() { + listener.witnessedAnyFailedStep.andReturn(true); + }); + + it("logs the failed steps", function() { + listener.logSummary(); + expect(listener.logFailedStepResults).toHaveBeenCalled(); + }); + }); + + describe("when there are no failed steps", function() { + beforeEach(function() { + listener.witnessedAnyFailedStep.andReturn(false); + }); + + it("does not log failed steps", function() { + listener.logSummary(); + expect(listener.logFailedStepResults).not.toHaveBeenCalled(); + }); + }); + it("logs the scenarios summary", function() { listener.logSummary(); expect(listener.logScenariosSummary).toHaveBeenCalled(); @@ -638,6 +730,102 @@ describe("Cucumber.Listener.ProgressFormatter", function() { }); }); + describe("logFailedStepResults()", function() { + var failedScenarioLogBuffer; + + beforeEach(function() { + failedScenarioLogBuffer = createSpy("failed scenario log buffer"); + spyOnStub(failedStepResults, 'syncForEach'); + spyOn(listener, 'log'); + spyOn(listener, 'getFailedScenarioLogBuffer').andReturn(failedScenarioLogBuffer); + }); + + it("logs a failed steps header", function() { + listener.logFailedStepResults(); + expect(listener.log).toHaveBeenCalledWith("(::) failed steps (::)\n\n"); + }); + + it("iterates synchronously over the failed step results", function() { + listener.logFailedStepResults(); + expect(failedStepResults.syncForEach).toHaveBeenCalled(); + expect(failedStepResults.syncForEach).toHaveBeenCalledWithAFunctionAsNthParameter(1); + }); + + describe("for each failed step result", function() { + var userFunction, failedStep, forEachCallback; + + beforeEach(function() { + listener.logFailedStepResults(); + userFunction = failedStepResults.syncForEach.mostRecentCall.args[0]; + failedStepResult = createSpy("failed step result"); + spyOn(listener, 'logFailedStepResult'); + }); + + it("tells the visitor to visit the feature and call back when finished", function() { + userFunction(failedStepResult); + expect(listener.logFailedStepResult).toHaveBeenCalledWith(failedStepResult); + }); + }); + + it("logs a failed scenarios header", function() { + listener.logFailedStepResults(); + expect(listener.log).toHaveBeenCalledWith("Failing scenarios:\n"); + }); + + it("gets the failed scenario details from its log buffer", function() { + listener.logFailedStepResults(); + expect(listener.getFailedScenarioLogBuffer).toHaveBeenCalled(); + }); + + it("logs the failed scenario details", function() { + listener.logFailedStepResults(); + expect(listener.log).toHaveBeenCalledWith(failedScenarioLogBuffer); + }); + + it("logs a line break", function() { + listener.logFailedStepResults(); + expect(listener.log).toHaveBeenCalledWith("\n"); + }); + }); + + describe("logFailedStepResult()", function() { + var stepResult, failureException; + + beforeEach(function() { + spyOn(listener, 'log'); + failureException = createSpy('caught exception'); + stepResult = createSpyWithStubs("failed step result", { getFailureException: failureException }); + }); + + it("gets the failure exception from the step result", function() { + listener.logFailedStepResult(stepResult); + expect(stepResult.getFailureException).toHaveBeenCalled(); + }); + + describe("when the failure exception has a stack", function() { + beforeEach(function() { + failureException.stack = createSpy('failure exception stack'); + }); + + it("logs the stack", function() { + listener.logFailedStepResult(stepResult); + expect(listener.log).toHaveBeenCalledWith(failureException.stack); + }); + }); + + describe("when the failure exception has no stack", function() { + it("logs the exception itself", function() { + listener.logFailedStepResult(stepResult); + expect(listener.log).toHaveBeenCalledWith(failureException); + }); + }); + + it("logs two line breaks", function() { + listener.logFailedStepResult(stepResult); + expect(listener.log).toHaveBeenCalledWith("\n\n"); + }); + }); + describe("logScenariosSummary()", function() { var scenarioCount, passedScenarioCount, pendingScenarioCount, failedScenarioCount; @@ -1422,4 +1610,15 @@ describe("Cucumber.Listener.ProgressFormatter", function() { }); }); }); + + describe("witnessedAnyFailedStep()", function() { + it("returns false when no failed step were encountered", function() { + expect(listener.witnessedAnyFailedStep()).toBeFalsy(); + }); + + it("returns true when one or more steps were witnessed", function() { + listener.witnessFailedStep(); + expect(listener.witnessedAnyFailedStep()).toBeTruthy(); + }); + }); }); diff --git a/spec/cucumber/runtime/failed_step_result_spec.js b/spec/cucumber/runtime/failed_step_result_spec.js index c29e7d669..f3322d4cb 100644 --- a/spec/cucumber/runtime/failed_step_result_spec.js +++ b/spec/cucumber/runtime/failed_step_result_spec.js @@ -2,10 +2,11 @@ require('../../support/spec_helper'); describe("Cucumber.Runtime.FailedStepResult", function() { var Cucumber = require('cucumber'); - var stepResult; + var stepResult, failureException; beforeEach(function() { - stepResult = Cucumber.Runtime.FailedStepResult(); + failureException = createSpy("failure exception"); + stepResult = Cucumber.Runtime.FailedStepResult(failureException); }); describe("isSuccessful()", function() { @@ -25,4 +26,10 @@ describe("Cucumber.Runtime.FailedStepResult", function() { expect(stepResult.isFailed()).toBeTruthy(); }); }); + + describe("getFailureException()", function() { + it("returns the exception passed to the constructor", function() { + expect(stepResult.getFailureException()).toBe(failureException); + }); + }); }); diff --git a/spec/cucumber/support_code/step_definition_spec.js b/spec/cucumber/support_code/step_definition_spec.js index 363a39035..e2f7817fa 100644 --- a/spec/cucumber/support_code/step_definition_spec.js +++ b/spec/cucumber/support_code/step_definition_spec.js @@ -136,18 +136,19 @@ describe("Cucumber.SupportCode.StepDefinition", function() { }); describe("when the step definition code fails", function() { - var failedStepResult; + var failedStepResult, failureException; beforeEach(function() { - stepDefinitionCode.apply.andThrow("I am a failing step definition"); + failureException = createSpy("I am a failing step definition"); failedStepResult = createSpy("failed step result"); + stepDefinitionCode.apply.andThrow(failureException); spyOn(Cucumber.Runtime, 'FailedStepResult').andReturn(failedStepResult); spyOn(Cucumber.Runtime, 'PendingStepResult'); }); it("creates a new failed step result", function() { stepDefinition.invoke(stepName, docString, callback); - expect(Cucumber.Runtime.FailedStepResult).toHaveBeenCalled(); + expect(Cucumber.Runtime.FailedStepResult).toHaveBeenCalledWith(failureException); }); it("does not create a new pending step result", function() { diff --git a/spec/support/spec_helper.js b/spec/support/spec_helper.js index 770e94b3d..d3f714c32 100644 --- a/spec/support/spec_helper.js +++ b/spec/support/spec_helper.js @@ -24,10 +24,11 @@ beforeEach(function() { return false; }, - toHaveBeenCalledWithStringMatching: function(regexp) { + toHaveBeenCalledWithStringMatching: function(pattern) { for(var i = 0; i < this.actual.callCount; i++) { var parameter = this.actual.argsForCall[i][0]; - if (regexp.test(parameter)) + if ((pattern.test && pattern.test(parameter)) || + (typeof(pattern) == 'string' && parameter.indexOf(pattern) >= 0)) return true; } return false;