test:Code coverage report (#1159)

.circleci/config.yml

@@ -162,6 +162,7 @@ orbs:
             environment:
               - BUNDLE_GEMFILE: /app/Gemfile
               - TEST_DATADOG_INTEGRATION: 1
+              - COVERAGE_BASE_DIR: coverage
           - *container_postgres
           - *container_presto
           - *container_mysql
@@ -178,7 +179,16 @@ orbs:
           - restore_cache:
               keys:
                 - '{{ .Environment.CIRCLE_CACHE_VERSION }}-bundle-<<parameters.ruby_version>>-{{ checksum ".circleci/bundle_checksum" }}'
+          - run:
+              name: Set coverage report directory
+              command: |
+                # Create a unique coverage directory for this job, to avoid conflicts when merging all results
+                echo 'export COVERAGE_DIR="$COVERAGE_BASE_DIR/versions/$CIRCLE_JOB/$CIRCLE_NODE_INDEX"' >> $BASH_ENV
           - *step_run_all_tests
+          - persist_to_workspace:
+              root: .
+              paths:
+                - coverage
       benchmark:
         <<: *job_defaults
         parameters:
@@ -232,6 +242,38 @@ orbs:
               keys:
                 - '{{ .Environment.CIRCLE_CACHE_VERSION }}-bundle-<<parameters.ruby_version>>-{{ checksum ".circleci/bundle_checksum" }}'
           - *step_rubocop
+      coverage:
+        <<: *job_defaults
+        parameters:
+          ruby_version:
+            description: Ruby version
+            type: string
+          image:
+            description: Docker image location
+            type: string
+        docker:
+          - <<: *container_base
+            image: <<parameters.image>>
+            environment:
+              - BUNDLE_GEMFILE: /app/Gemfile
+        steps:
+          - restore_cache:
+              keys:
+                - '{{ .Environment.CIRCLE_CACHE_VERSION }}-bundled-repo-<<parameters.ruby_version>>-{{ .Environment.CIRCLE_SHA1 }}'
+          - restore_cache:
+              keys:
+                - '{{ .Environment.CIRCLE_CACHE_VERSION }}-bundle-<<parameters.ruby_version>>-{{ checksum ".circleci/bundle_checksum" }}'
+          - attach_workspace:
+              at: /tmp/workspace
+          - run:
+              name: Generate coverage report artifact "coverage/index.html"
+              command: COVERAGE_DIR=/tmp/workspace/coverage bundle exec rake coverage:report
+          - run:
+              name: Generate coverage report artifact "coverage/versions/*/index.html"
+              command: COVERAGE_DIR=/tmp/workspace/coverage bundle exec rake coverage:report_per_ruby_version
+          - store_artifacts:
+              path: /tmp/workspace/coverage/report/
+              destination: coverage
     commands:
     executors:
 
@@ -340,6 +382,19 @@ workflows:
           name: lint
           requires:
             - build-2.6
+      - orb/coverage:
+          <<: *config-2_7
+          name: coverage
+          requires:
+            - test-2.0
+            - test-2.1
+            - test-2.2
+            - test-2.3
+            - test-2.4
+            - test-2.5
+            - test-2.6
+            - test-2.7
+            - test-jruby-9.2
       # MRI
       - orb/checkout:
           <<: *config-2_0

.simplecov

@@ -0,0 +1,38 @@
+SimpleCov.add_group 'contrib', '/lib/ddtrace/contrib'
+SimpleCov.add_group 'transport', '/lib/ddtrace/transport'
+SimpleCov.add_group 'spec', '/spec/'
+
+SimpleCov.coverage_dir ENV.fetch('COVERAGE_DIR', 'coverage')
+
+# Each test run requires its own unique command_name.
+# When running `rake spec:test_name`, the test process doesn't have access to the
+# rake task process, so we have come up with unique values ourselves.
+#
+# The current approach is to combine the ruby engine (ruby-2.7,jruby-9.2),
+# program name (rspec/test), command line arguments (--pattern spec/**/*_spec.rb),
+# and the loaded gemset.
+#
+# This should allow us to distinguish between runs with the same tests, but different gemsets:
+#   * appraisal rails5-mysql2 rake spec:rails
+#   * appraisal rails5-postgres rake spec:rails
+#
+# Subsequent runs of the same exact test suite should have the same command_name.
+command_line_arguments = ARGV.join(' ')
+gemset_hash = Digest::MD5.hexdigest Gem.loaded_specs.values.map { |x| "#{x.name}#{x.version}" }.sort.join
+ruby_engine = if defined?(RUBY_ENGINE_VERSION)
+  "#{RUBY_ENGINE}-#{RUBY_ENGINE_VERSION}"
+else
+  "#{RUBY_ENGINE}-#{RUBY_VERSION}" # For Ruby < 2.3
+end
+SimpleCov.command_name "#{ruby_engine}:#{gemset_hash}:#{$PROGRAM_NAME} #{command_line_arguments}"
+
+# A very large number to disable result merging timeout
+SimpleCov.merge_timeout 2 ** 31
+
+# DEV If we choose to enforce a hard minimum.
+# SimpleCov.minimum_coverage 95
+
+# DEV If we choose to enforce a maximum coverage drop.
+# DEV We have to figure out where to store previous coverage data
+# DEV in order to perform this comparison
+# SimpleCov.maximum_coverage_drop 1

Rakefile

@@ -759,4 +759,33 @@ task :ci do
   end
 end
 
+namespace :coverage do
+  # Generates one global report for all tracer tests
+  task :report do
+    require 'simplecov'
+
+    resultset_files = Dir["#{ENV.fetch('COVERAGE_DIR', 'coverage')}/.resultset.json"] +
+                      Dir["#{ENV.fetch('COVERAGE_DIR', 'coverage')}/versions/**/.resultset.json"]
+
+    SimpleCov.collate resultset_files do
+      coverage_dir "#{ENV.fetch('COVERAGE_DIR', 'coverage')}/report"
+      formatter SimpleCov::Formatter::HTMLFormatter
+    end
+  end
+
+  # Generates one report for each Ruby version
+  task :report_per_ruby_version do
+    require 'simplecov'
+
+    versions = Dir["#{ENV.fetch('COVERAGE_DIR', 'coverage')}/versions/*"].map { |f| File.basename(f) }
+    versions.map do |version|
+      puts "Generating report for: #{version}"
+      SimpleCov.collate Dir["#{ENV.fetch('COVERAGE_DIR', 'coverage')}/versions/#{version}/**/.resultset.json"] do
+        coverage_dir "#{ENV.fetch('COVERAGE_DIR', 'coverage')}/report/versions/#{version}"
+        formatter SimpleCov::Formatter::HTMLFormatter
+      end
+    end
+  end
+end
+
 task default: :test

ddtrace.gemspec

@@ -67,5 +67,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'redcarpet', '~> 3.4' if RUBY_PLATFORM != 'java'
   spec.add_development_dependency 'pry', '~> 0.10.4'
   spec.add_development_dependency 'pry-stack_explorer', '~> 0.4.9.2'
+  spec.add_development_dependency 'simplecov', '~> 0.17'
   spec.add_development_dependency 'warning', '~> 1' if RUBY_VERSION >= '2.5.0'
 end

docs/DevelopmentGuide.md

@@ -98,6 +98,22 @@ $ bundle exec appraisal contrib rake spec:redis'[--seed,1234]'
 
 This can be useful for replicating conditions from CI or isolating certain tests.
 
+**Checking test coverage**
+
+You can check test code coverage by creating a report _after_ running a test suite:
+```
+# Run the desired test suite
+$ bundle exec appraisal contrib rake spec:redis
+# Generate report for the suite executed
+$ bundle exec rake coverage:report
+```
+
+A webpage will be generated at `coverage/report/index.html` with the resulting report.
+
+Because you are likely not running all tests locally, your report will contain partial coverage results.
+You *must* check the CI step `coverage` for the complete test coverage report, ensuring coverage is not
+decreased.
+
 ### Checking code quality
 
 **Linting**

spec/spec_helper.rb

@@ -5,6 +5,12 @@
 require 'webmock/rspec'
 require 'climate_control'
 
+# +SimpleCov.start+ must be invoked before any application code is loaded
+require 'simplecov'
+SimpleCov.start do
+  formatter SimpleCov::Formatter::SimpleFormatter
+end
+
 require 'ddtrace/encoding'
 require 'ddtrace/tracer'
 require 'ddtrace/span'