diff --git a/Rakefile b/Rakefile index 81d742e99..83e85470a 100644 --- a/Rakefile +++ b/Rakefile @@ -8,6 +8,7 @@ Bundler.setup require "cucumber/rake/task" require "rspec/core/rake_task" require "rubocop/rake_task" +require "rake/clean" Cucumber::Rake::Task.new do |t| t.cucumber_opts = %w(--format progress) @@ -21,8 +22,10 @@ end RSpec::Core::RakeTask.new +CLOBBER << "coverage/**/*" + desc "Run the whole test suite." -task test: [:spec, :cucumber] +task test: [:clobber, :spec, :cucumber] RuboCop::RakeTask.new diff --git a/features/02_configure_aruba/basics.feature b/features/02_configure_aruba/basics.feature index a95bca709..492e29555 100644 --- a/features/02_configure_aruba/basics.feature +++ b/features/02_configure_aruba/basics.feature @@ -55,7 +55,7 @@ Feature: Usage of configuration Given a file named "spec/support/aruba_config.rb" with: """ruby Aruba.configure do |config| - config.exit_timeout = 0.5 + config.exit_timeout = 0.7 end """ And a file named "spec/support/hooks.rb" with: @@ -96,7 +96,7 @@ Feature: Usage of configuration Given a file named "features/support/aruba_config.rb" with: """ruby Aruba.configure do |config| - config.exit_timeout = 0.5 + config.exit_timeout = 0.7 end """ And a file named "features/run.feature" with: @@ -107,7 +107,7 @@ Feature: Usage of configuration Then the exit status should be 0 Scenario: Slow command - When I run `aruba-test-cli 1.0` + When I run `aruba-test-cli 1.5` Then the exit status should be 128 """ When I run `cucumber` @@ -122,7 +122,7 @@ Feature: Usage of configuration Given a file named "features/support/aruba_config.rb" with: """ruby Aruba.configure do |config| - config.exit_timeout = 0.2 + config.exit_timeout = 0.7 end """ And a file named "features/support/hooks.rb" with: @@ -140,11 +140,11 @@ Feature: Usage of configuration @slow-command Scenario: Slow command known by the developer - When I run `aruba-test-cli 0.5` + When I run `aruba-test-cli 1` Then the exit status should be 0 Scenario: Slow command which might be a failure - When I run `aruba-test-cli 0.5` + When I run `aruba-test-cli 1` Then the exit status should be 128 """ When I run `cucumber` diff --git a/features/03_testing_frameworks/cucumber/steps/command/check_for_exit_statuses.feature b/features/03_testing_frameworks/cucumber/steps/command/check_for_exit_statuses.feature index d3405e762..9127ccfd1 100644 --- a/features/03_testing_frameworks/cucumber/steps/command/check_for_exit_statuses.feature +++ b/features/03_testing_frameworks/cucumber/steps/command/check_for_exit_statuses.feature @@ -95,7 +95,7 @@ Feature: Check exit status of commands """ Feature: Failing program Scenario: Run command - Given the default aruba exit timeout is 0.4 seconds + Given the default aruba exit timeout is 0.7 seconds When I successfully run `aruba-test-cli` """ When I run `cucumber` @@ -112,7 +112,7 @@ Feature: Check exit status of commands Feature: Failing program Scenario: Run command Given the default aruba exit timeout is 0 seconds - When I successfully run `aruba-test-cli` for up to 0.4 seconds + When I successfully run `aruba-test-cli` for up to 0.7 seconds """ When I run `cucumber` Then the features should all pass diff --git a/features/03_testing_frameworks/cucumber/steps/command/stop_command.feature b/features/03_testing_frameworks/cucumber/steps/command/stop_command.feature index 1e7995b74..6ac4f9a15 100644 --- a/features/03_testing_frameworks/cucumber/steps/command/stop_command.feature +++ b/features/03_testing_frameworks/cucumber/steps/command/stop_command.feature @@ -45,8 +45,8 @@ Feature: Stop commands """ Feature: Run it Scenario: Run command - Given the default aruba exit timeout is 0.2 seconds - And I wait 0.2 seconds for a command to start up + Given the default aruba exit timeout is 1.5 seconds + And I wait 0.7 seconds for a command to start up When I run `aruba-test-cli1` in background And I run `aruba-test-cli2` in background And I terminate the command started last @@ -97,8 +97,8 @@ Feature: Stop commands Background: Scenario: Run command - Given the default aruba exit timeout is 0.2 seconds - And I wait 0.2 seconds for a command to start up + Given the default aruba exit timeout is 0.7 seconds + And I wait 0.5 seconds for a command to start up When I run `aruba-test-cli1` in background And I run `aruba-test-cli2` in background And I stop the command started last @@ -145,10 +145,10 @@ Feature: Stop commands """ Feature: Run it Background: - Given the default aruba exit timeout is 0.2 seconds + Given the default aruba exit timeout is 0.3 seconds Scenario: Run command - Given I wait 0.2 seconds for a command to start up + Given I wait 0.3 seconds for a command to start up When I run `aruba-test-cli1` in background When I run `aruba-test-cli2` in background And I terminate the command "aruba-test-cli1" @@ -197,10 +197,10 @@ Feature: Stop commands """ Feature: Run it Background: - Given the default aruba exit timeout is 0.2 seconds + Given the default aruba exit timeout is 0.3 seconds Scenario: Run command - Given I wait 0.2 seconds for a command to start up + Given I wait 0.3 seconds for a command to start up When I run `aruba-test-cli1` in background And I run `aruba-test-cli2` in background When I stop the command "aruba-test-cli1" @@ -239,7 +239,7 @@ Feature: Stop commands Feature: Run it Scenario: Run command Given the default aruba stop signal is "HUP" - And the default aruba exit timeout is 0.2 seconds + And the default aruba exit timeout is 0.7 seconds When I run `aruba-test-cli` Then the exit status should be 155 """ @@ -271,8 +271,8 @@ Feature: Stop commands """ Feature: Run it Scenario: Run command - Given the default aruba exit timeout is 0.3 seconds - And I wait 0.2 seconds for a command to start up + Given the default aruba exit timeout is 0.7 seconds + And I wait 0.7 seconds for a command to start up When I run `aruba-test-cli1` in background And I terminate the command started last Then the exit status should be 100 diff --git a/features/04_aruba_api/command/run_command.feature b/features/04_aruba_api/command/run_command.feature index c7c655890..133a84a8f 100644 --- a/features/04_aruba_api/command/run_command.feature +++ b/features/04_aruba_api/command/run_command.feature @@ -116,7 +116,7 @@ Feature: Run command """ruby require 'spec_helper' - RSpec.describe 'Run command', type: :aruba, exit_timeout: 0.1, startup_wait_time: 0.4 do + RSpec.describe 'Run command', type: :aruba, exit_timeout: 0.5, startup_wait_time: 1.5 do before do run_command('aruba-test-cli') last_command_started.send_signal 'HUP' @@ -154,7 +154,7 @@ Feature: Run command """ruby require 'spec_helper' - RSpec.describe 'Run command', type: :aruba, exit_timeout: 0.4 do + RSpec.describe 'Run command', type: :aruba, exit_timeout: 1.0 do before { run_command('aruba-test-cli') } it 'runs the command with the expected results' do @@ -233,10 +233,10 @@ Feature: Run command """ruby require 'spec_helper' - RSpec.describe 'Run command', type: :aruba, exit_timeout: 0.1 do + RSpec.describe 'Run command', type: :aruba, exit_timeout: 1.0 do before do - run_command 'aruba-test-cli1', startup_wait_time: 0.5 - run_command 'aruba-test-cli2', startup_wait_time: 0.2 + run_command 'aruba-test-cli1', startup_wait_time: 1.0 + run_command 'aruba-test-cli2', startup_wait_time: 0.7 last_command_started.send_signal 'HUP' end @@ -289,8 +289,8 @@ Feature: Run command RSpec.describe 'Run command', type: :aruba do before do - run_command 'aruba-test-cli1', exit_timeout: 0.4 - run_command 'aruba-test-cli2', exit_timeout: 0.1 + run_command 'aruba-test-cli1', exit_timeout: 1.5 + run_command 'aruba-test-cli2', exit_timeout: 0.7 end it 'runs both commands with the expected results' do diff --git a/features/04_aruba_api/command/stop_single_command.feature b/features/04_aruba_api/command/stop_single_command.feature index 120f19221..18dfd15aa 100644 --- a/features/04_aruba_api/command/stop_single_command.feature +++ b/features/04_aruba_api/command/stop_single_command.feature @@ -27,7 +27,7 @@ Feature: Stop command """ruby require 'spec_helper' - RSpec.describe 'Run command', type: :aruba, exit_timeout: 0.2 do + RSpec.describe 'Run command', type: :aruba, exit_timeout: 0.5 do before { run_command('aruba-test-cli') } before { last_command_started.stop } it { expect(last_command_started).to be_successfully_executed } @@ -52,7 +52,7 @@ Feature: Stop command """ruby require 'spec_helper' - RSpec.describe 'Run command', type: :aruba, startup_wait_time: 0.1, exit_timeout: 0.3 do + RSpec.describe 'Run command', type: :aruba, startup_wait_time: 0.1, exit_timeout: 0.5 do before { run_command('aruba-test-cli') } before { find_command('aruba-test-cli').stop } it { expect(last_command_started).to be_successfully_executed } @@ -85,7 +85,7 @@ Feature: Stop command Aruba.configure do |config| config.stop_signal = 'HUP' - config.exit_timeout = 0.2 + config.exit_timeout = 0.5 end RSpec.describe 'Run command', type: :aruba do @@ -120,7 +120,7 @@ Feature: Stop command Aruba.configure do |config| config.stop_signal = 'HUP' - config.exit_timeout = 0.2 + config.exit_timeout = 0.5 end RSpec.describe 'Run command', type: :aruba do diff --git a/features/04_aruba_api/filesystem/use_fixtures.feature b/features/04_aruba_api/filesystem/use_fixtures.feature index 99075efed..29e62454d 100644 --- a/features/04_aruba_api/filesystem/use_fixtures.feature +++ b/features/04_aruba_api/filesystem/use_fixtures.feature @@ -10,7 +10,7 @@ Feature: Use fixtures in your tests Given I use a fixture named "cli-app" Scenario: Use a fixture for your tests - Given a file named "spec/fixture_spec.rb" with: + Given a file named "spec/fixture_a_spec.rb" with: """ require 'spec_helper' @@ -30,7 +30,7 @@ Feature: Use fixtures in your tests Then the specs should all pass Scenario: Pass file from fixtures to your command - Given a file named "spec/fixture_spec.rb" with: + Given a file named "spec/fixture_b_spec.rb" with: """ require 'spec_helper' @@ -56,7 +56,7 @@ Feature: Use fixtures in your tests Then the specs should all pass Scenario: Use a fixture for your tests in test/ - Given a file named "spec/fixture_spec.rb" with: + Given a file named "spec/fixture_c_spec.rb" with: """ require 'spec_helper' @@ -76,7 +76,7 @@ Feature: Use fixtures in your tests Then the specs should all pass Scenario: Use a fixture for your tests in spec/ - Given a file named "spec/fixture_spec.rb" with: + Given a file named "spec/fixture_d_spec.rb" with: """ require 'spec_helper' @@ -96,7 +96,7 @@ Feature: Use fixtures in your tests Then the specs should all pass Scenario: Use a fixture for your tests in features/ - Given a file named "spec/fixture_spec.rb" with: + Given a file named "spec/fixture_e_spec.rb" with: """ require 'spec_helper' diff --git a/features/support/env.rb b/features/support/env.rb index 4a4e7fb06..43bc1b0ad 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -1,29 +1,32 @@ $LOAD_PATH.unshift File.expand_path("../../lib", __dir__) # Has to be the first file required so that all other files show coverage information -require "simplecov" unless RUBY_PLATFORM.include?("java") +require_relative "simplecov/simplecov_setup" unless RUBY_PLATFORM.include?("java") # Standard Library require "fileutils" require "pathname" - # Gems require "aruba/cucumber" require "aruba/config/jruby" require "rspec/expectations" -Before do |test_case| +Around do |test_case, block| command_name = "#{test_case.location.file}:#{test_case.location.line} # #{test_case.name}" # Used in simplecov_setup so that each scenario has a different name and # their coverage results are merged instead of overwriting each other as # 'Cucumber Features' - ENV["SIMPLECOV_COMMAND_NAME"] = command_name.to_s + set_environment_variable "SIMPLECOV_COMMAND_NAME", command_name.to_s simplecov_setup_pathname = - Pathname.new(__FILE__).expand_path.parent.join("simplecov_setup") + Pathname.new(__FILE__).expand_path.parent.join("simplecov").to_s # set environment variable so child processes will merge their coverage data # with parent process's coverage data. - ENV["RUBYOPT"] = "-r#{simplecov_setup_pathname} #{ENV['RUBYOPT']}" + prepend_environment_variable "RUBYOPT", "-I#{simplecov_setup_pathname} -rsimplecov_setup " + + with_environment do + block.call + end end diff --git a/features/support/simplecov/simplecov_setup.rb b/features/support/simplecov/simplecov_setup.rb new file mode 100644 index 000000000..3c9c1e52f --- /dev/null +++ b/features/support/simplecov/simplecov_setup.rb @@ -0,0 +1,47 @@ +# @note this file is loaded in env.rb to setup simplecov using RUBYOPTs for +# child processes and @in-process +unless RUBY_PLATFORM.include?("java") + require "simplecov" + root = File.expand_path("../../..", __dir__) + command_name = ENV["SIMPLECOV_COMMAND_NAME"] || "Cucumber Features" + SimpleCov.command_name(command_name) + SimpleCov.root(root) + + # Run simplecov by default + SimpleCov.start unless ENV.key? "ARUBA_NO_COVERAGE" + + module SimpleCov + module Formatter + # @note this is a monkey patch to suppress console output generated by + # the Simplecov HTML formatter. + # + # The Simplecov HTML formatter, which is the default, writes a message to + # the console after it has stored coverage information the `coverage` + # directory. There is no built in way to suppress this message. See + # simplecov-ruby/simplecov-html#116 and + # https://github.com/simplecov-ruby/simplecov-html/blob/v0.12.3/lib/simplecov-html.rb#L23..L32. + # + # The console output generated by the default formatter causes failures + # for Aruba tests that are expecting exact command output. + # + # The approach employed here is to monkey patch the method so that it + # still generates the same message, but instead of writing to the console, + # it writes to a file in the `coverage` directory named `summary.txt`. + class HTMLFormatter + def format(result) + Dir[File.join(File.dirname(__FILE__), "../public/*")].each do |path| + FileUtils.cp_r(path, asset_output_path) + end + + File.open(File.join(output_path, "index.html"), "wb") do |file| + file.puts template("layout").result(binding) + end + + File.open(File.join(output_path, "summary.txt"), "w") do |file| + file.puts output_message(result) + end + end + end + end + end +end diff --git a/features/support/simplecov_setup.rb b/features/support/simplecov_setup.rb deleted file mode 100644 index 598dc8673..000000000 --- a/features/support/simplecov_setup.rb +++ /dev/null @@ -1,12 +0,0 @@ -# @note this file is loaded in env.rb to setup simplecov using RUBYOPTs for -# child processes and @in-process -unless RUBY_PLATFORM.include?("java") - require "simplecov" - root = File.expand_path("../..", __dir__) - command_name = ENV["SIMPLECOV_COMMAND_NAME"] || "Cucumber Features" - SimpleCov.command_name(command_name) - SimpleCov.root(root) - - # Run simplecov by default - SimpleCov.start unless ENV.key? "ARUBA_NO_COVERAGE" -end