From 7be181d3b4976ffe822ca30b89e34bc0ef1fa339 Mon Sep 17 00:00:00 2001 From: Naoto Kishino Date: Thu, 19 May 2022 15:31:51 +0900 Subject: [PATCH 1/2] Add json_report --- .circleci/config.yml | 2 +- exe/bucky | 38 +++++----- lib/bucky/core/test_core/test_case_loader.rb | 3 +- .../core/test_core/test_class_generator.rb | 7 +- lib/bucky/core/test_core/test_manager.rb | 72 ++++++++++++++++--- .../test_case/abst_test_case.rb | 7 ++ .../verifications/status_checker.rb | 2 +- spec/core/test_core/test_case_loader_spec.rb | 13 +--- .../test_core/test_class_generator_spec.rb | 10 +-- spec/core/test_core/test_manager_spec.rb | 2 +- .../verifications/status_checker_spec.rb | 4 +- 11 files changed, 105 insertions(+), 55 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 480aec8..ab4b674 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,7 +27,7 @@ jobs: command: docker load -q -i ~/caches/images.tar - run: name: docker up - command: docker-compose -f docker-compose.system-test.yml up --no-build -d + command: docker-compose -f docker-compose.system-test.yml up -d - run: name: execute system testing command: 'circleci tests glob system_testing/testing_code/*.bats | xargs -n 1 -I {} docker exec bucky-core bats "/bucky-core/"{} | circleci tests split' diff --git a/exe/bucky b/exe/bucky index 8115f20..985d138 100755 --- a/exe/bucky +++ b/exe/bucky @@ -89,7 +89,9 @@ end opts.on('-m', '--link_check_max_times') do |v| test_cond[:link_check_max_times] = v.to_i end - +opts.on('-o', '--out JSON_OUTPUT_FILE_PATH') do |v| + test_cond[:out] = v +end lint_cond = {} opts.on('-C', '--category CATEGORY_NAME') do |v| lint_cond[:lint_category] = v @@ -124,42 +126,34 @@ def bucky_home? File.exist?('.bucky_home') end -current_dir = Dir.pwd -gem_script_dir = __dir__ - -if ARGV == RUN_COMMAND - error_and_exit('Not bucky project dirctory here.') unless bucky_home? - - $bucky_home_dir = Dir.pwd +def setup_test_cond(test_cond) # Default conditions setting conditions setting test_cond[:test_category] ||= 'e2e' test_cond[:re_test_count] = test_cond[:re_test_count] ? test_cond[:re_test_count].to_i : 1 test_cond[:link_check_max_times] ||= 3 + test_cond[:out] ||= 'report.json' # Change to array e.g.--suite_id 1,2,3 -> :suite_id=>[1,2,3] - test_cond.each { |k, v| test_cond[k] = v.split(',') if v.instance_of?(String) } - require_relative '../lib/bucky/core/test_core/test_manager' - Bucky::Core::TestCore::TestManager.new(test_cond).run - Bucky::Core::TestCore::ExitHandler.instance.bucky_exit + %i[suite_name case label xlabel device].each do |k| + test_cond[k] = test_cond[k].split(',') unless test_cond[k].nil? + end + test_cond +end -elsif ARGV == RERUN_COMMAND +current_dir = Dir.pwd +gem_script_dir = __dir__ + +if ARGV[0].end_with? 'run' + error_and_exit('Not bucky project dirctory here.') unless bucky_home? $bucky_home_dir = Dir.pwd - # Default conditions setting conditions setting - test_cond[:test_category] ||= 'e2e' - test_cond[:link_check_max_times] ||= 3 - test_cond[:re_test_count] = test_cond[:re_test_count] ? test_cond[:re_test_count].to_i : 1 - # Change to array e.g.--suite_id 1,2,3 -> :suite_id=>[1,2,3] - test_cond.each { |k, v| test_cond[k] = v.split(',') if v.instance_of?(String) } require_relative '../lib/bucky/core/test_core/test_manager' - Bucky::Core::TestCore::TestManager.new(test_cond).rerun + Bucky::Core::TestCore::TestManager.new(setup_test_cond(test_cond)).send(ARGV[0]) Bucky::Core::TestCore::ExitHandler.instance.bucky_exit - elsif ARGV == LINT_COMMAND $bucky_home_dir = Dir.pwd # Default conditions setting lint_cond[:lint_category] ||= 'config' require_relative '../lib/bucky/tools/lint' Bucky::Tools::Lint.check(lint_cond[:lint_category]) - elsif ARGV[0..0] == NEW_COMMAND error_and_exit('No test app name.') if ARGV.length < 2 diff --git a/lib/bucky/core/test_core/test_case_loader.rb b/lib/bucky/core/test_core/test_case_loader.rb index 0ccfacf..905a75c 100644 --- a/lib/bucky/core/test_core/test_case_loader.rb +++ b/lib/bucky/core/test_core/test_case_loader.rb @@ -34,9 +34,8 @@ def load_testcode(test_cond) testcodes = [] service = (test_cond[:service] || ['*']).first device = (test_cond[:device] || ['*']).first - category = (test_cond[:test_category] || ['*']).first - Dir.glob("#{$bucky_home_dir}/services/#{service}/#{device}/scenarios/#{category}/*.yml").each do |testcode_file| + Dir.glob("#{$bucky_home_dir}/services/#{service}/#{device}/scenarios/#{test_cond[:test_category]}/*.yml").each do |testcode_file| testcodes << load_testcode_in_file(testcode_file, test_cond) end # Delete nil element diff --git a/lib/bucky/core/test_core/test_class_generator.rb b/lib/bucky/core/test_core/test_class_generator.rb index ff37cba..890547c 100644 --- a/lib/bucky/core/test_core/test_class_generator.rb +++ b/lib/bucky/core/test_core/test_class_generator.rb @@ -46,7 +46,7 @@ def initialize(test_cond) end # Genrate test class by test suite and test case data - def generate_test_class(data, link_status_url_log = {}) + def generate_test_class(data: [], linkstatus_url_log: {}, w_pipe: {}) test_cond = @test_cond # Common proccessing # e.g.) TestSampleAppPcE2e1, TestSampleAppPcHttpstatus1 @@ -63,6 +63,7 @@ def generate_test_class(data, link_status_url_log = {}) match_obj = /\Atest_(.+)\(.+::(Test.+)\)\z/.match(original_name) "#{match_obj[1]}(#{match_obj[2]})" end + define_method(:w_pipe, proc { w_pipe }) # Class structure is different for each test category case data[:test_category] @@ -74,8 +75,8 @@ def generate_test_class(data, link_status_url_log = {}) define_method(method_name) do puts "\n#{simple_test_class_name(name)}" t_case[:urls].each do |url| - link_status_check_args = { url: url, device: data[:suite][:device], exclude_urls: data[:suite][:exclude_urls], link_check_max_times: test_cond[:link_check_max_times], url_log: link_status_url_log } - link_status_check(link_status_check_args) + linkstatus_check_args = { url: url, device: data[:suite][:device], exclude_urls: data[:suite][:exclude_urls], link_check_max_times: test_cond[:link_check_max_times], url_log: linkstatus_url_log } + linkstatus_check(linkstatus_check_args) end end ) diff --git a/lib/bucky/core/test_core/test_manager.rb b/lib/bucky/core/test_core/test_manager.rb index 94444f7..adaa634 100644 --- a/lib/bucky/core/test_core/test_manager.rb +++ b/lib/bucky/core/test_core/test_manager.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require 'parallel' +require 'json' require_relative '../test_core/test_case_loader' require_relative '../../utils/config' require_relative './test_class_generator' @@ -26,6 +26,8 @@ def parallel_new_worker_each(data_set, max_processes, &block) # If child process dead, available workers increase Signal.trap('CLD') { available_workers += 1 } + r_pipe, w_pipe = IO.pipe + data_set.each do |data| # Wait until worker is available and handle exit code from previous process unless available_workers.positive? @@ -34,28 +36,51 @@ def parallel_new_worker_each(data_set, max_processes, &block) end # Workers decrease when start working available_workers -= 1 - fork { block.call(data) } + fork { block.call(data, w_pipe) } end # Handle all exit code in waitall Process.waitall.each do |child| Bucky::Core::TestCore::ExitHandler.instance.raise unless child[1].exitstatus.zero? end + + w_pipe.close + results_set = collect_results_set(r_pipe) + r_pipe.close + + results_set end def parallel_distribute_into_workers(data_set, max_processes, &block) # Group the data by remainder of index data_set_grouped = data_set.group_by.with_index { |_elem, index| index % max_processes } + r_pipe, w_pipe = IO.pipe # Use 'values' method to get only hash's key into an array data_set_grouped.values.each do |data_for_pre_worker| # Number of child process is equal to max_processes (or equal to data_set length when data_set length is less than max_processes) fork do - data_for_pre_worker.each { |data| block.call(data) } + data_for_pre_worker.each { |data| block.call(data, w_pipe) } end end # Handle all exit code in waitall Process.waitall.each do |child| Bucky::Core::TestCore::ExitHandler.instance.raise unless child[1].exitstatus.zero? end + + w_pipe.close + results_set = collect_results_set(r_pipe) + r_pipe.close + + results_set + end + + def collect_results_set(r_pipe) + results_set = {} + r_pipe.each_line do |line| + r = JSON.parse(line) + results_set[r['test_class_name']] = r + end + + results_set end end @@ -69,6 +94,20 @@ def initialize(test_cond) @tdo = Bucky::Core::Database::TestDataOperator.new @start_time = Time.now $job_id = @tdo.save_job_record_and_get_job_id(@start_time, @test_cond[:command_and_option]) + @json_report = { + summary: { + cases_count: 0, + success_count: 0, + failure_count: 0, + job_id: $job_id, + test_category: test_cond[:test_category], + device: test_cond[:device], + labels: test_cond[:label], + exclude_labels: test_cond[:xlabel], + rerun_job_id: test_cond[:job], + round_count: 0 + } + } end def run @@ -102,25 +141,42 @@ def do_test_suites(test_suite_data) e2e_parallel_num = Bucky::Utils::Config.instance[:e2e_parallel_num] linkstatus_parallel_num = Bucky::Utils::Config.instance[:linkstatus_parallel_num] tcg = Bucky::Core::TestCore::TestClassGenerator.new(@test_cond) - case @test_cond[:test_category][0] - when 'e2e' then parallel_new_worker_each(test_suite_data, e2e_parallel_num) { |data| tcg.generate_test_class(data) } + case @test_cond[:test_category] + when 'e2e' then results_set = parallel_new_worker_each(test_suite_data, e2e_parallel_num) { |data, w_pipe| tcg.generate_test_class(data: data, w_pipe: w_pipe) } when 'linkstatus' then - link_status_url_log = {} - parallel_distribute_into_workers(test_suite_data, linkstatus_parallel_num) { |data| tcg.generate_test_class(data, link_status_url_log) } + linkstatus_url_log = {} + results_set = parallel_distribute_into_workers(test_suite_data, linkstatus_parallel_num) { |data, w_pipe| tcg.generate_test_class(data: data, linkstatus_url_log: linkstatus_url_log, w_pipe: w_pipe) } end + + results_set end def execute_test + results_set = {} @re_test_count.times do |i| Bucky::Core::TestCore::ExitHandler.instance.reset $round = i + 1 + @json_report[:summary][:round_count] = $round test_suite_data = load_test_suites - do_test_suites(test_suite_data) + results_set = do_test_suites(test_suite_data) @test_cond[:re_test_cond] = @tdo.get_ng_test_cases_at_last_execution( is_error: 1, job_id: $job_id, round: $round ) break if @test_cond[:re_test_cond].empty? end + + return unless @test_cond[:out] + + results_set.each do |_class_name, res| + @json_report[:summary][:cases_count] += res['cases_count'] + @json_report[:summary][:success_count] += res['success_count'] + @json_report[:summary][:failure_count] += res['failure_count'] + end + + File.open(@test_cond[:out], 'w') do |f| + f.puts(@json_report.to_json) + puts "\nSave report : #{@test_cond[:out]}\n" + end end end end diff --git a/lib/bucky/test_equipment/test_case/abst_test_case.rb b/lib/bucky/test_equipment/test_case/abst_test_case.rb index f252936..ecb9daa 100644 --- a/lib/bucky/test_equipment/test_case/abst_test_case.rb +++ b/lib/bucky/test_equipment/test_case/abst_test_case.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'test/unit' +require 'json' require_relative '../../core/test_core/test_result' module Bucky @@ -25,6 +26,12 @@ def shutdown def run(result) super @@this_result.result = result unless $debug + w_pipe.puts({ + test_class_name: self.class.name, + cases_count: result.run_count, + success_count: result.pass_count, + failure_count: result.run_count - result.pass_count + }.to_json) end def setup diff --git a/lib/bucky/test_equipment/verifications/status_checker.rb b/lib/bucky/test_equipment/verifications/status_checker.rb index f8b83d2..d47a514 100644 --- a/lib/bucky/test_equipment/verifications/status_checker.rb +++ b/lib/bucky/test_equipment/verifications/status_checker.rb @@ -62,7 +62,7 @@ def http_status_check(args) end end - def link_status_check(args) + def linkstatus_check(args) url = args[:url] device = args[:device] exclude_urls = args[:exclude_urls] diff --git a/spec/core/test_core/test_case_loader_spec.rb b/spec/core/test_core/test_case_loader_spec.rb index 7d5844c..7a98161 100644 --- a/spec/core/test_core/test_case_loader_spec.rb +++ b/spec/core/test_core/test_case_loader_spec.rb @@ -11,16 +11,9 @@ $bucky_home_dir = bucky_home end - context 'In case there is no arguments' do - let(:test_cond) { {} } - it 'return test code object' do - expect(subject).not_to be_empty - end - end - context 'In case there are some arguments' do context 'when give args of test suite' do - let(:test_cond) { { suite_name: [expect_scenario] } } + let(:test_cond) { { suite_name: [expect_scenario], test_category: 'e2e' } } let(:expect_scenario) { 'scenario_a' } it 'return test code object' do expect(subject).not_to be_empty @@ -32,8 +25,8 @@ end end - context 'when give args of priority' do - let(:test_cond) { { priority: [expect_priority] } } + context 'When give args of priority' do + let(:test_cond) { { priority: [expect_priority], test_category: 'e2e' } } let(:expect_priority) { 'middle' } it 'return test code object' do expect(subject).not_to be_empty diff --git a/spec/core/test_core/test_class_generator_spec.rb b/spec/core/test_core/test_class_generator_spec.rb index d428583..ef43be9 100644 --- a/spec/core/test_core/test_class_generator_spec.rb +++ b/spec/core/test_core/test_class_generator_spec.rb @@ -10,8 +10,8 @@ allow(Bucky::Utils::Config).to receive(:instance).and_return(linkstatus_open_timeout: 60) allow(Bucky::Utils::Config).to receive(:instance).and_return(linkstatus_read_timeout: 60) end - let(:link_status_url_log) { { url: { code: 200, count: 1 } } } - let(:link_status_data) do + let(:linkstatus_url_log) { { url: { code: 200, count: 1 } } } + let(:linkstatus_data) do { test_class_name: 'SearchflowAreaTop', test_suite_name: 'searchflow_area_top', @@ -65,7 +65,7 @@ describe 'generate class' do let(:device) { 'pc' } it 'generate test class according to test suite data' do - subject.generate_test_class(link_status_data, link_status_url_log) + subject.generate_test_class(data: linkstatus_data, linkstatus_url_log: linkstatus_url_log) expect(subject.instance_variable_get(:@test_classes).const_get(:TestFooPcLinkstatusSearchflowAreaTop).name).to eq('Bucky::Core::TestCore::TestClasses::TestFooPcLinkstatusSearchflowAreaTop') end end @@ -74,14 +74,14 @@ let(:device) { 'sp' } context 'in case of linkstatus' do it 'generate test method according to test suite data' do - subject.generate_test_class(link_status_data, link_status_url_log) + subject.generate_test_class(data: linkstatus_data, linkstatus_url_log: linkstatus_url_log) expect(subject.instance_variable_get(:@test_classes).const_get(:TestFooSpLinkstatusSearchflowAreaTop).instance_methods).to include :test_foo_sp_linkstatus_searchflow_area_top_0 end end context 'in case of e2e' do it 'generate test method according to test suite data' do - subject.generate_test_class(e2e_data, link_status_url_log) + subject.generate_test_class(data: e2e_data) expect(subject.instance_variable_get(:@test_classes).const_get(:TestFooPcE2eSearchflowAreaTop).instance_methods).to include :test_foo_pc_e2e_searchflow_area_top_0 end end diff --git a/spec/core/test_core/test_manager_spec.rb b/spec/core/test_core/test_manager_spec.rb index 2bef6d6..11450f1 100644 --- a/spec/core/test_core/test_manager_spec.rb +++ b/spec/core/test_core/test_manager_spec.rb @@ -11,7 +11,7 @@ allow(Bucky::Core::Database::TestDataOperator).to receive(:new).and_return(tdo) allow(tdo).to receive(:save_job_record_and_get_job_id) allow(tdo).to receive(:get_ng_test_cases_at_last_execution).and_return(ng_case_data) - allow(tm).to receive(:do_test_suites) + allow(tm).to receive(:do_test_suites).and_return({}) end describe '#run' do diff --git a/spec/test_equipment/verifications/status_checker_spec.rb b/spec/test_equipment/verifications/status_checker_spec.rb index 6f1f61e..f2c5e93 100644 --- a/spec/test_equipment/verifications/status_checker_spec.rb +++ b/spec/test_equipment/verifications/status_checker_spec.rb @@ -106,7 +106,7 @@ end end end - describe 'link_status_check' do + describe 'linkstatus_check' do subject { test_class.new } let(:url) { 'https://example.com/' } let(:links) { ['https://example.com/hoge', 'https://example.com/fuga'] } @@ -122,7 +122,7 @@ end context 'no error base url, link url' do it 'not raise exception' do - expect { subject.link_status_check(args) }.not_to raise_error + expect { subject.linkstatus_check(args) }.not_to raise_error end end end From defbccb1418f5e96aeb422c52b54cb994f0c4f6f Mon Sep 17 00:00:00 2001 From: Naoto Kishino Date: Mon, 23 May 2022 17:15:43 +0900 Subject: [PATCH 2/2] Add --out option in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 81e3c7b..4278055 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ ENV_FOO=foo bucky run -t e2e -d -r, --re_test_count RE_TEST_COUNT # How many round you run tests -l, --label LABEL_NAME -m, --link_check_max_times MAX_TIMES # Works only with which category is linkstatus + -o, --out JSON_OUTPUT_FILE_PATH # Output summary report by json ``` ### Rerun test