Merge pull request #98 from naotospace/feature/json_format_report

Feature/json format report

.circleci/config.yml

@@ -27,7 +27,7 @@ jobs:
           command: docker load -q -i ~/caches/images.tar
       - run:
           name: docker up
-          command: docker-compose -f docker-compose.system-test.yml up --no-build -d
+          command: docker-compose -f docker-compose.system-test.yml up -d
       - run:
           name: execute system testing
           command: 'circleci tests glob system_testing/testing_code/*.bats | xargs -n 1 -I {} docker exec bucky-core bats "/bucky-core/"{} | circleci tests split'

README.md

@@ -98,6 +98,7 @@ ENV_FOO=foo bucky run -t e2e -d
     -r, --re_test_count RE_TEST_COUNT # How many round you run tests
     -l, --label LABEL_NAME
     -m, --link_check_max_times MAX_TIMES # Works only with which category is linkstatus
+    -o, --out JSON_OUTPUT_FILE_PATH # Output summary report by json
 ```
 
 ### Rerun test

exe/bucky

@@ -89,7 +89,9 @@ end
 opts.on('-m', '--link_check_max_times') do |v|
   test_cond[:link_check_max_times] = v.to_i
 end
-
+opts.on('-o', '--out JSON_OUTPUT_FILE_PATH') do |v|
+  test_cond[:out] = v
+end
 lint_cond = {}
 opts.on('-C', '--category CATEGORY_NAME') do |v|
   lint_cond[:lint_category] = v
@@ -124,42 +126,34 @@ def bucky_home?
   File.exist?('.bucky_home')
 end
 
-current_dir = Dir.pwd
-gem_script_dir = __dir__
-
-if ARGV == RUN_COMMAND
-  error_and_exit('Not bucky project dirctory here.') unless bucky_home?
-
-  $bucky_home_dir = Dir.pwd
+def setup_test_cond(test_cond)
   # Default conditions setting conditions setting
   test_cond[:test_category] ||= 'e2e'
   test_cond[:re_test_count] = test_cond[:re_test_count] ? test_cond[:re_test_count].to_i : 1
   test_cond[:link_check_max_times] ||= 3
+  test_cond[:out] ||= 'report.json'
   # Change to array e.g.--suite_id 1,2,3 -> :suite_id=>[1,2,3]
-  test_cond.each { |k, v| test_cond[k] = v.split(',') if v.instance_of?(String) }
-  require_relative '../lib/bucky/core/test_core/test_manager'
-  Bucky::Core::TestCore::TestManager.new(test_cond).run
-  Bucky::Core::TestCore::ExitHandler.instance.bucky_exit
+  %i[suite_name case label xlabel device].each do |k|
+    test_cond[k] = test_cond[k].split(',') unless test_cond[k].nil?
+  end
+  test_cond
+end
 
-elsif ARGV == RERUN_COMMAND
+current_dir = Dir.pwd
+gem_script_dir = __dir__
+
+if ARGV[0].end_with? 'run'
+  error_and_exit('Not bucky project dirctory here.') unless bucky_home?
   $bucky_home_dir = Dir.pwd
-  # Default conditions setting conditions setting
-  test_cond[:test_category] ||= 'e2e'
-  test_cond[:link_check_max_times] ||= 3
-  test_cond[:re_test_count] = test_cond[:re_test_count] ? test_cond[:re_test_count].to_i : 1
-  # Change to array e.g.--suite_id 1,2,3 -> :suite_id=>[1,2,3]
-  test_cond.each { |k, v| test_cond[k] = v.split(',') if v.instance_of?(String) }
   require_relative '../lib/bucky/core/test_core/test_manager'
-  Bucky::Core::TestCore::TestManager.new(test_cond).rerun
+  Bucky::Core::TestCore::TestManager.new(setup_test_cond(test_cond)).send(ARGV[0])
   Bucky::Core::TestCore::ExitHandler.instance.bucky_exit
-
 elsif ARGV == LINT_COMMAND
   $bucky_home_dir = Dir.pwd
   # Default conditions setting
   lint_cond[:lint_category] ||= 'config'
   require_relative '../lib/bucky/tools/lint'
   Bucky::Tools::Lint.check(lint_cond[:lint_category])
-
 elsif ARGV[0..0] == NEW_COMMAND
   error_and_exit('No test app name.') if ARGV.length < 2
 

lib/bucky/core/test_core/test_case_loader.rb

@@ -34,9 +34,8 @@ def load_testcode(test_cond)
             testcodes = []
             service = (test_cond[:service] || ['*']).first
             device = (test_cond[:device] || ['*']).first
-            category = (test_cond[:test_category] || ['*']).first
 
-            Dir.glob("#{$bucky_home_dir}/services/#{service}/#{device}/scenarios/#{category}/*.yml").each do |testcode_file|
+            Dir.glob("#{$bucky_home_dir}/services/#{service}/#{device}/scenarios/#{test_cond[:test_category]}/*.yml").each do |testcode_file|
               testcodes << load_testcode_in_file(testcode_file, test_cond)
             end
             # Delete nil element

lib/bucky/core/test_core/test_class_generator.rb

@@ -46,7 +46,7 @@ def initialize(test_cond)
         end
 
         # Genrate test class by test suite and test case data
-        def generate_test_class(data, link_status_url_log = {})
+        def generate_test_class(data: [], linkstatus_url_log: {}, w_pipe: {})
           test_cond = @test_cond
           # Common proccessing
           # e.g.) TestSampleAppPcE2e1, TestSampleAppPcHttpstatus1
@@ -63,6 +63,7 @@ def generate_test_class(data, link_status_url_log = {})
               match_obj = /\Atest_(.+)\(.+::(Test.+)\)\z/.match(original_name)
               "#{match_obj[1]}(#{match_obj[2]})"
             end
+            define_method(:w_pipe, proc { w_pipe })
 
             # Class structure is different for each test category
             case data[:test_category]
@@ -74,8 +75,8 @@ def generate_test_class(data, link_status_url_log = {})
                   define_method(method_name) do
                     puts "\n#{simple_test_class_name(name)}"
                     t_case[:urls].each do |url|
-                      link_status_check_args = { url: url, device: data[:suite][:device], exclude_urls: data[:suite][:exclude_urls], link_check_max_times: test_cond[:link_check_max_times], url_log: link_status_url_log }
-                      link_status_check(link_status_check_args)
+                      linkstatus_check_args = { url: url, device: data[:suite][:device], exclude_urls: data[:suite][:exclude_urls], link_check_max_times: test_cond[:link_check_max_times], url_log: linkstatus_url_log }
+                      linkstatus_check(linkstatus_check_args)
                     end
                   end
                 )

lib/bucky/core/test_core/test_manager.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'parallel'
+require 'json'
 require_relative '../test_core/test_case_loader'
 require_relative '../../utils/config'
 require_relative './test_class_generator'
@@ -26,6 +26,8 @@ def parallel_new_worker_each(data_set, max_processes, &block)
           # If child process dead, available workers increase
           Signal.trap('CLD') { available_workers += 1 }
 
+          r_pipe, w_pipe = IO.pipe
+
           data_set.each do |data|
             # Wait until worker is available and handle exit code from previous process
             unless available_workers.positive?
@@ -34,28 +36,51 @@ def parallel_new_worker_each(data_set, max_processes, &block)
             end
             # Workers decrease when start working
             available_workers -= 1
-            fork { block.call(data) }
+            fork { block.call(data, w_pipe) }
           end
           # Handle all exit code in waitall
           Process.waitall.each do |child|
             Bucky::Core::TestCore::ExitHandler.instance.raise unless child[1].exitstatus.zero?
           end
+
+          w_pipe.close
+          results_set = collect_results_set(r_pipe)
+          r_pipe.close
+
+          results_set
         end
 
         def parallel_distribute_into_workers(data_set, max_processes, &block)
           # Group the data by remainder of index
           data_set_grouped = data_set.group_by.with_index { |_elem, index| index % max_processes }
+          r_pipe, w_pipe = IO.pipe
           # Use 'values' method to get only hash's key into an array
           data_set_grouped.values.each do |data_for_pre_worker|
             # Number of child process is equal to max_processes (or equal to data_set length when data_set length is less than max_processes)
             fork do
-              data_for_pre_worker.each { |data| block.call(data) }
+              data_for_pre_worker.each { |data| block.call(data, w_pipe) }
             end
           end
           # Handle all exit code in waitall
           Process.waitall.each do |child|
             Bucky::Core::TestCore::ExitHandler.instance.raise unless child[1].exitstatus.zero?
           end
+
+          w_pipe.close
+          results_set = collect_results_set(r_pipe)
+          r_pipe.close
+
+          results_set
+        end
+
+        def collect_results_set(r_pipe)
+          results_set = {}
+          r_pipe.each_line do |line|
+            r = JSON.parse(line)
+            results_set[r['test_class_name']] = r
+          end
+
+          results_set
         end
       end
 
@@ -69,6 +94,20 @@ def initialize(test_cond)
           @tdo = Bucky::Core::Database::TestDataOperator.new
           @start_time = Time.now
           $job_id = @tdo.save_job_record_and_get_job_id(@start_time, @test_cond[:command_and_option])
+          @json_report = {
+            summary: {
+              cases_count: 0,
+              success_count: 0,
+              failure_count: 0,
+              job_id: $job_id,
+              test_category: test_cond[:test_category],
+              device: test_cond[:device],
+              labels: test_cond[:label],
+              exclude_labels: test_cond[:xlabel],
+              rerun_job_id: test_cond[:job],
+              round_count: 0
+            }
+          }
         end
 
         def run
@@ -102,25 +141,42 @@ def do_test_suites(test_suite_data)
           e2e_parallel_num = Bucky::Utils::Config.instance[:e2e_parallel_num]
           linkstatus_parallel_num = Bucky::Utils::Config.instance[:linkstatus_parallel_num]
           tcg = Bucky::Core::TestCore::TestClassGenerator.new(@test_cond)
-          case @test_cond[:test_category][0]
-          when 'e2e' then parallel_new_worker_each(test_suite_data, e2e_parallel_num) { |data| tcg.generate_test_class(data) }
+          case @test_cond[:test_category]
+          when 'e2e' then results_set = parallel_new_worker_each(test_suite_data, e2e_parallel_num) { |data, w_pipe| tcg.generate_test_class(data: data, w_pipe: w_pipe) }
           when 'linkstatus' then
-            link_status_url_log = {}
-            parallel_distribute_into_workers(test_suite_data, linkstatus_parallel_num) { |data| tcg.generate_test_class(data, link_status_url_log) }
+            linkstatus_url_log = {}
+            results_set = parallel_distribute_into_workers(test_suite_data, linkstatus_parallel_num) { |data, w_pipe| tcg.generate_test_class(data: data, linkstatus_url_log: linkstatus_url_log, w_pipe: w_pipe) }
           end
+
+          results_set
         end
 
         def execute_test
+          results_set = {}
           @re_test_count.times do |i|
             Bucky::Core::TestCore::ExitHandler.instance.reset
             $round = i + 1
+            @json_report[:summary][:round_count] = $round
             test_suite_data = load_test_suites
-            do_test_suites(test_suite_data)
+            results_set = do_test_suites(test_suite_data)
             @test_cond[:re_test_cond] = @tdo.get_ng_test_cases_at_last_execution(
               is_error: 1, job_id: $job_id, round: $round
             )
             break if @test_cond[:re_test_cond].empty?
           end
+
+          return unless @test_cond[:out]
+
+          results_set.each do |_class_name, res|
+            @json_report[:summary][:cases_count] += res['cases_count']
+            @json_report[:summary][:success_count] += res['success_count']
+            @json_report[:summary][:failure_count] += res['failure_count']
+          end
+
+          File.open(@test_cond[:out], 'w') do |f|
+            f.puts(@json_report.to_json)
+            puts "\nSave report : #{@test_cond[:out]}\n"
+          end
         end
       end
     end

lib/bucky/test_equipment/test_case/abst_test_case.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'test/unit'
+require 'json'
 require_relative '../../core/test_core/test_result'
 
 module Bucky
@@ -25,6 +26,12 @@ def shutdown
         def run(result)
           super
           @@this_result.result = result unless $debug
+          w_pipe.puts({
+            test_class_name: self.class.name,
+            cases_count: result.run_count,
+            success_count: result.pass_count,
+            failure_count: result.run_count - result.pass_count
+          }.to_json)
         end
 
         def setup

lib/bucky/test_equipment/verifications/status_checker.rb

@@ -62,7 +62,7 @@ def http_status_check(args)
           end
         end
 
-        def link_status_check(args)
+        def linkstatus_check(args)
           url = args[:url]
           device = args[:device]
           exclude_urls = args[:exclude_urls]

spec/core/test_core/test_case_loader_spec.rb

@@ -11,16 +11,9 @@
       $bucky_home_dir = bucky_home
     end
 
-    context 'In case there is no arguments' do
-      let(:test_cond) { {} }
-      it 'return test code object' do
-        expect(subject).not_to be_empty
-      end
-    end
-
     context 'In case there are some arguments' do
       context 'when give args of test suite' do
-        let(:test_cond) { { suite_name: [expect_scenario] } }
+        let(:test_cond) { { suite_name: [expect_scenario], test_category: 'e2e' } }
         let(:expect_scenario) { 'scenario_a' }
         it 'return test code object' do
           expect(subject).not_to be_empty
@@ -32,8 +25,8 @@
         end
       end
 
-      context 'when give args of priority' do
-        let(:test_cond) { { priority: [expect_priority] } }
+      context 'When give args of priority' do
+        let(:test_cond) { { priority: [expect_priority], test_category: 'e2e' } }
         let(:expect_priority) { 'middle' }
         it 'return test code object' do
           expect(subject).not_to be_empty

spec/core/test_core/test_class_generator_spec.rb

@@ -10,8 +10,8 @@
       allow(Bucky::Utils::Config).to receive(:instance).and_return(linkstatus_open_timeout: 60)
       allow(Bucky::Utils::Config).to receive(:instance).and_return(linkstatus_read_timeout: 60)
     end
-    let(:link_status_url_log) { { url: { code: 200, count: 1 } } }
-    let(:link_status_data) do
+    let(:linkstatus_url_log) { { url: { code: 200, count: 1 } } }
+    let(:linkstatus_data) do
       {
         test_class_name: 'SearchflowAreaTop',
         test_suite_name: 'searchflow_area_top',
@@ -65,7 +65,7 @@
     describe 'generate class' do
       let(:device) { 'pc' }
       it 'generate test class according to test suite data' do
-        subject.generate_test_class(link_status_data, link_status_url_log)
+        subject.generate_test_class(data: linkstatus_data, linkstatus_url_log: linkstatus_url_log)
         expect(subject.instance_variable_get(:@test_classes).const_get(:TestFooPcLinkstatusSearchflowAreaTop).name).to eq('Bucky::Core::TestCore::TestClasses::TestFooPcLinkstatusSearchflowAreaTop')
       end
     end
@@ -74,14 +74,14 @@
       let(:device) { 'sp' }
       context 'in case of linkstatus' do
         it 'generate test method according to test suite data' do
-          subject.generate_test_class(link_status_data, link_status_url_log)
+          subject.generate_test_class(data: linkstatus_data, linkstatus_url_log: linkstatus_url_log)
           expect(subject.instance_variable_get(:@test_classes).const_get(:TestFooSpLinkstatusSearchflowAreaTop).instance_methods).to include :test_foo_sp_linkstatus_searchflow_area_top_0
         end
       end
 
       context 'in case of e2e' do
         it 'generate test method according to test suite data' do
-          subject.generate_test_class(e2e_data, link_status_url_log)
+          subject.generate_test_class(data: e2e_data)
           expect(subject.instance_variable_get(:@test_classes).const_get(:TestFooPcE2eSearchflowAreaTop).instance_methods).to include :test_foo_pc_e2e_searchflow_area_top_0
         end
       end

spec/core/test_core/test_manager_spec.rb

@@ -11,7 +11,7 @@
     allow(Bucky::Core::Database::TestDataOperator).to receive(:new).and_return(tdo)
     allow(tdo).to receive(:save_job_record_and_get_job_id)
     allow(tdo).to receive(:get_ng_test_cases_at_last_execution).and_return(ng_case_data)
-    allow(tm).to receive(:do_test_suites)
+    allow(tm).to receive(:do_test_suites).and_return({})
   end
 
   describe '#run' do

spec/test_equipment/verifications/status_checker_spec.rb

@@ -106,7 +106,7 @@
       end
     end
   end
-  describe 'link_status_check' do
+  describe 'linkstatus_check' do
     subject { test_class.new }
     let(:url) { 'https://example.com/' }
     let(:links) { ['https://example.com/hoge', 'https://example.com/fuga'] }
@@ -122,7 +122,7 @@
     end
     context 'no error base url, link url' do
       it 'not raise exception' do
-        expect { subject.link_status_check(args) }.not_to raise_error
+        expect { subject.linkstatus_check(args) }.not_to raise_error
       end
     end
   end