From e9e509c29a6fb13fa122fb7a180c3abed8f65584 Mon Sep 17 00:00:00 2001 From: Jorg Jenni Date: Sun, 31 Jul 2022 12:41:17 +0100 Subject: [PATCH] add --retry-total option --- CHANGELOG.md | 1 + features/docs/cli/retry_failing_tests.feature | 28 +++++++++ lib/cucumber/cli/options.rb | 17 +++++- lib/cucumber/configuration.rb | 7 ++- lib/cucumber/filters/retry.rb | 21 ++++++- spec/cucumber/cli/options_spec.rb | 34 +++++++++++ spec/cucumber/filters/retry_spec.rb | 57 +++++++++++++++++-- 7 files changed, 154 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1773b418ee..fcbad7d056 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Please visit [cucumber/CONTRIBUTING.md](https://github.com/cucumber/cucumber/blo ## [Unreleased](https://github.com/cucumber/cucumber-ruby/compare/v8.0.0...main) ### Added +* Add option `--retry-total` ([PR#](https://github.com/cucumber/cucumber-ruby/pull/1669)) ### Fixed diff --git a/features/docs/cli/retry_failing_tests.feature b/features/docs/cli/retry_failing_tests.feature index ef6ef2b0ae..0e4dd970a8 100644 --- a/features/docs/cli/retry_failing_tests.feature +++ b/features/docs/cli/retry_failing_tests.feature @@ -95,3 +95,31 @@ Feature: Retry failing tests 3 scenarios (2 flaky, 1 passed) """ + + Scenario: Retry each test but suspend retrying after 2 failing tests, so later tests are not retried + Given a scenario "Fails-forever-1" that fails + And a scenario "Fails-forever-2" that fails + When I run `cucumber -q --retry 1 --retry-total 2 --format summary` + Then it should fail with: + """ + 5 scenarios (4 failed, 1 passed) + """ + And it should fail with: + """ + Fails-forever-1 + Fails-forever-1 ✗ + Fails-forever-1 ✗ + + Fails-forever-2 + Fails-forever-2 ✗ + Fails-forever-2 ✗ + + Fails-once feature + Fails-once ✗ + + Fails-twice feature + Fails-twice ✗ + + Solid + Solid ✓ + """ diff --git a/lib/cucumber/cli/options.rb b/lib/cucumber/cli/options.rb index 161de23c7f..5021b59154 100644 --- a/lib/cucumber/cli/options.rb +++ b/lib/cucumber/cli/options.rb @@ -56,11 +56,12 @@ class Options NO_PROFILE_LONG_FLAG = '--no-profile'.freeze FAIL_FAST_FLAG = '--fail-fast'.freeze RETRY_FLAG = '--retry'.freeze + RETRY_TOTAL_FLAG = '--retry-total'.freeze OPTIONS_WITH_ARGS = [ '-r', '--require', '--i18n-keywords', '-f', '--format', '-o', '--out', '-t', '--tags', '-n', '--name', '-e', '--exclude', - PROFILE_SHORT_FLAG, PROFILE_LONG_FLAG, RETRY_FLAG, '-l', - '--lines', '--port', '-I', '--snippet-type' + PROFILE_SHORT_FLAG, PROFILE_LONG_FLAG, RETRY_FLAG, RETRY_TOTAL_FLAG, + '-l', '--lines', '--port', '-I', '--snippet-type' ].freeze ORDER_TYPES = %w[defined random].freeze TAG_LIMIT_MATCHER = /(?@\w+):(?\d+)/x @@ -108,6 +109,7 @@ def parse!(args) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength opts.on('-j DIR', '--jars DIR', 'Load all the jars under DIR') { |jars| load_jars(jars) } if Cucumber::JRUBY opts.on("#{RETRY_FLAG} ATTEMPTS", *retry_msg) { |v| set_option :retry, v.to_i } + opts.on("#{RETRY_TOTAL_FLAG} TESTS", *retry_total_msg) { |v| set_option :retry_total, v.to_i } opts.on('--i18n-languages', *i18n_languages_msg) { list_languages_and_exit } opts.on('--i18n-keywords LANG', *i18n_keywords_msg) { |lang| language lang } opts.on(FAIL_FAST_FLAG, 'Exit immediately following the first failing scenario') { set_option :fail_fast } @@ -271,6 +273,13 @@ def retry_msg ['Specify the number of times to retry failing tests (default: 0)'] end + def retry_total_msg + [ + 'The total number of failing test after which retrying of tests is suspended.', + 'Example: --retry-total 10 -> Will stop retrying tests after 10 failing tests.' + ] + end + def name_msg [ 'Only execute the feature elements which match part of the given name.', @@ -543,6 +552,7 @@ def reverse_merge(other_options) # rubocop:disable Metrics/AbcSize end @options[:retry] = other_options[:retry] if @options[:retry].zero? + @options[:retry_total] = other_options[:retry_total] if @options[:retry_total].infinite? self end @@ -616,7 +626,8 @@ def default_options snippets: true, source: true, duration: true, - retry: 0 + retry: 0, + retry_total: Float::INFINITY } end end diff --git a/lib/cucumber/configuration.rb b/lib/cucumber/configuration.rb index ddcf3f9621..4935a7ecff 100644 --- a/lib/cucumber/configuration.rb +++ b/lib/cucumber/configuration.rb @@ -78,6 +78,10 @@ def retry_attempts @options[:retry] end + def retry_total_tests + @options[:retry_total] + end + def guess? @options[:guess] end @@ -273,7 +277,8 @@ def default_options snippets: true, source: true, duration: true, - event_bus: Cucumber::Events.make_event_bus + event_bus: Cucumber::Events.make_event_bus, + retry_total: Float::INFINITY } end diff --git a/lib/cucumber/filters/retry.rb b/lib/cucumber/filters/retry.rb index 57ba11b8b1..313be68614 100644 --- a/lib/cucumber/filters/retry.rb +++ b/lib/cucumber/filters/retry.rb @@ -7,6 +7,11 @@ module Cucumber module Filters class Retry < Core::Filter.new(:configuration) + def initialize(*_args) + super + @total_permanently_failed = 0 + end + def test_case(test_case) configuration.on_event(:test_case_finished) do |event| next unless retry_required?(test_case, event) @@ -21,7 +26,21 @@ def test_case(test_case) private def retry_required?(test_case, event) - event.test_case == test_case && event.result.failed? && test_case_counts[test_case] < configuration.retry_attempts + return false unless event.test_case == test_case + + return false unless event.result.failed? + + return false if @total_permanently_failed >= configuration.retry_total_tests + + retry_required = test_case_counts[test_case] < configuration.retry_attempts + if retry_required + # retry test + true + else + # test failed after max. attempts + @total_permanently_failed += 1 + false + end end def test_case_counts diff --git a/spec/cucumber/cli/options_spec.rb b/spec/cucumber/cli/options_spec.rb index 68644f493a..bb9d9e6cdd 100644 --- a/spec/cucumber/cli/options_spec.rb +++ b/spec/cucumber/cli/options_spec.rb @@ -379,6 +379,26 @@ def with_env(name, value) end end end + + context '--retry-total TESTS' do + context '--retry-total option not defined on the command line' do + it 'uses the --retry-total option from the profile' do + given_cucumber_yml_defined_as('foo' => '--retry-total 2') + options.parse!(%w[-p foo]) + + expect(options[:retry_total]).to be 2 + end + end + + context '--retry-total option defined on the command line' do + it 'ignores the --retry-total option from the profile' do + given_cucumber_yml_defined_as('foo' => '--retry-total 2') + options.parse!(%w[--retry-total 1 -p foo]) + + expect(options[:retry_total]).to be 1 + end + end + end end context '-P or --no-profile' do @@ -441,6 +461,20 @@ def with_env(name, value) end end + context '--retry-total TESTS' do + it 'is INFINITY by default' do + after_parsing('') do + expect(options[:retry_total]).to eql Float::INFINITY + end + end + + it 'sets the options[:retry_total] value' do + after_parsing('--retry 3 --retry-total 10') do + expect(options[:retry_total]).to eql 10 + end + end + end + it 'assigns any extra arguments as paths to features' do after_parsing('-f pretty my_feature.feature my_other_features') do expect(options[:paths]).to eq ['my_feature.feature', 'my_other_features'] diff --git a/spec/cucumber/filters/retry_spec.rb b/spec/cucumber/filters/retry_spec.rb index 0c983a434f..8345a35683 100644 --- a/spec/cucumber/filters/retry_spec.rb +++ b/spec/cucumber/filters/retry_spec.rb @@ -13,7 +13,8 @@ include Cucumber::Core include Cucumber::Events - let(:configuration) { Cucumber::Configuration.new(retry: 2) } + let(:configuration) { Cucumber::Configuration.new(retry: 2, retry_total: retry_total) } + let(:retry_total) { Float::INFINITY } let(:id) { double } let(:name) { double } let(:location) { double } @@ -55,12 +56,32 @@ context 'consistently failing test case' do let(:result) { Cucumber::Core::Test::Result::Failed.new(0, StandardError.new) } - it 'describes the test case the specified number of times' do - expect(receiver).to receive(:test_case) { |test_case| - configuration.notify :test_case_finished, test_case, result - }.exactly(3).times + shared_examples 'retries the test case the specified number of times' do |expected_nr_of_times| + it 'describes the test case the specified number of times' do + expect(receiver).to receive(:test_case) { |test_case| + configuration.notify :test_case_finished, test_case, result + }.exactly(expected_nr_of_times).times - filter.test_case(test_case) + filter.test_case(test_case) + end + end + + context 'when retry_total infinit' do + let(:retry_total) { Float::INFINITY } + + include_examples 'retries the test case the specified number of times', 3 + end + + context 'when retry_total 1' do + let(:retry_total) { 1 } + + include_examples 'retries the test case the specified number of times', 3 + end + + context 'when retry_total 0' do + let(:retry_total) { 0 } + + include_examples 'retries the test case the specified number of times', 1 end end @@ -100,4 +121,28 @@ end end end + + context 'too many failing tests' do + let(:retry_total) { 1 } + let(:always_failing_test_case1) do + Cucumber::Core::Test::Case.new(id, name, [double('test steps')], 'test.rb:1', tags, language) + end + let(:always_failing_test_case2) do + Cucumber::Core::Test::Case.new(id, name, [double('test steps')], 'test.rb:9', tags, language) + end + let(:fail_result) { Cucumber::Core::Test::Result::Failed.new(0, StandardError.new) } + + it 'stops retrying tests' do + expect(receiver).to receive(:test_case).with(always_failing_test_case1) { |test_case| + configuration.notify :test_case_finished, test_case, fail_result + }.ordered.exactly(3).times + + expect(receiver).to receive(:test_case).with(always_failing_test_case2) { |test_case| + configuration.notify :test_case_finished, test_case, fail_result + }.ordered.exactly(1).times + + filter.test_case(always_failing_test_case1) + filter.test_case(always_failing_test_case2) + end + end end