diff --git a/.circleci/config.yml b/.circleci/config.yml index 4a85c284ea8..c46817ea075 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -162,6 +162,7 @@ orbs: environment: - BUNDLE_GEMFILE: /app/Gemfile - TEST_DATADOG_INTEGRATION: 1 + - COVERAGE_BASE_DIR: coverage - *container_postgres - *container_presto - *container_mysql @@ -178,7 +179,16 @@ orbs: - restore_cache: keys: - '{{ .Environment.CIRCLE_CACHE_VERSION }}-bundle-<>-{{ checksum ".circleci/bundle_checksum" }}' + - run: + name: Set coverage report directory + command: | + # Create a unique coverage directory for this job, to avoid conflicts when merging all results + echo 'export COVERAGE_DIR="$COVERAGE_BASE_DIR/versions/$CIRCLE_JOB/$CIRCLE_NODE_INDEX"' >> $BASH_ENV - *step_run_all_tests + - persist_to_workspace: + root: . + paths: + - coverage benchmark: <<: *job_defaults parameters: @@ -232,6 +242,38 @@ orbs: keys: - '{{ .Environment.CIRCLE_CACHE_VERSION }}-bundle-<>-{{ checksum ".circleci/bundle_checksum" }}' - *step_rubocop + coverage: + <<: *job_defaults + parameters: + ruby_version: + description: Ruby version + type: string + image: + description: Docker image location + type: string + docker: + - <<: *container_base + image: <> + environment: + - BUNDLE_GEMFILE: /app/Gemfile + steps: + - restore_cache: + keys: + - '{{ .Environment.CIRCLE_CACHE_VERSION }}-bundled-repo-<>-{{ .Environment.CIRCLE_SHA1 }}' + - restore_cache: + keys: + - '{{ .Environment.CIRCLE_CACHE_VERSION }}-bundle-<>-{{ checksum ".circleci/bundle_checksum" }}' + - attach_workspace: + at: /tmp/workspace + - run: + name: Generate coverage report artifact "coverage/index.html" + command: COVERAGE_DIR=/tmp/workspace/coverage bundle exec rake coverage:report + - run: + name: Generate coverage report artifact "coverage/versions/*/index.html" + command: COVERAGE_DIR=/tmp/workspace/coverage bundle exec rake coverage:report_per_ruby_version + - store_artifacts: + path: /tmp/workspace/coverage/report/ + destination: coverage commands: executors: @@ -340,6 +382,19 @@ workflows: name: lint requires: - build-2.6 + - orb/coverage: + <<: *config-2_7 + name: coverage + requires: + - test-2.0 + - test-2.1 + - test-2.2 + - test-2.3 + - test-2.4 + - test-2.5 + - test-2.6 + - test-2.7 + - test-jruby-9.2 # MRI - orb/checkout: <<: *config-2_0 diff --git a/.simplecov b/.simplecov new file mode 100644 index 00000000000..4a60e6082fa --- /dev/null +++ b/.simplecov @@ -0,0 +1,38 @@ +SimpleCov.add_group 'contrib', '/lib/ddtrace/contrib' +SimpleCov.add_group 'transport', '/lib/ddtrace/transport' +SimpleCov.add_group 'spec', '/spec/' + +SimpleCov.coverage_dir ENV.fetch('COVERAGE_DIR', 'coverage') + +# Each test run requires its own unique command_name. +# When running `rake spec:test_name`, the test process doesn't have access to the +# rake task process, so we have come up with unique values ourselves. +# +# The current approach is to combine the ruby engine (ruby-2.7,jruby-9.2), +# program name (rspec/test), command line arguments (--pattern spec/**/*_spec.rb), +# and the loaded gemset. +# +# This should allow us to distinguish between runs with the same tests, but different gemsets: +# * appraisal rails5-mysql2 rake spec:rails +# * appraisal rails5-postgres rake spec:rails +# +# Subsequent runs of the same exact test suite should have the same command_name. +command_line_arguments = ARGV.join(' ') +gemset_hash = Digest::MD5.hexdigest Gem.loaded_specs.values.map { |x| "#{x.name}#{x.version}" }.sort.join +ruby_engine = if defined?(RUBY_ENGINE_VERSION) + "#{RUBY_ENGINE}-#{RUBY_ENGINE_VERSION}" +else + "#{RUBY_ENGINE}-#{RUBY_VERSION}" # For Ruby < 2.3 +end +SimpleCov.command_name "#{ruby_engine}:#{gemset_hash}:#{$PROGRAM_NAME} #{command_line_arguments}" + +# A very large number to disable result merging timeout +SimpleCov.merge_timeout 2 ** 31 + +# DEV If we choose to enforce a hard minimum. +# SimpleCov.minimum_coverage 95 + +# DEV If we choose to enforce a maximum coverage drop. +# DEV We have to figure out where to store previous coverage data +# DEV in order to perform this comparison +# SimpleCov.maximum_coverage_drop 1 diff --git a/Rakefile b/Rakefile index c8681889046..d4be962a311 100644 --- a/Rakefile +++ b/Rakefile @@ -759,4 +759,33 @@ task :ci do end end +namespace :coverage do + # Generates one global report for all tracer tests + task :report do + require 'simplecov' + + resultset_files = Dir["#{ENV.fetch('COVERAGE_DIR', 'coverage')}/.resultset.json"] + + Dir["#{ENV.fetch('COVERAGE_DIR', 'coverage')}/versions/**/.resultset.json"] + + SimpleCov.collate resultset_files do + coverage_dir "#{ENV.fetch('COVERAGE_DIR', 'coverage')}/report" + formatter SimpleCov::Formatter::HTMLFormatter + end + end + + # Generates one report for each Ruby version + task :report_per_ruby_version do + require 'simplecov' + + versions = Dir["#{ENV.fetch('COVERAGE_DIR', 'coverage')}/versions/*"].map { |f| File.basename(f) } + versions.map do |version| + puts "Generating report for: #{version}" + SimpleCov.collate Dir["#{ENV.fetch('COVERAGE_DIR', 'coverage')}/versions/#{version}/**/.resultset.json"] do + coverage_dir "#{ENV.fetch('COVERAGE_DIR', 'coverage')}/report/versions/#{version}" + formatter SimpleCov::Formatter::HTMLFormatter + end + end + end +end + task default: :test diff --git a/ddtrace.gemspec b/ddtrace.gemspec index e2eb3ee9153..8a577e0471a 100644 --- a/ddtrace.gemspec +++ b/ddtrace.gemspec @@ -67,5 +67,6 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'redcarpet', '~> 3.4' if RUBY_PLATFORM != 'java' spec.add_development_dependency 'pry', '~> 0.10.4' spec.add_development_dependency 'pry-stack_explorer', '~> 0.4.9.2' + spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'warning', '~> 1' if RUBY_VERSION >= '2.5.0' end diff --git a/docs/DevelopmentGuide.md b/docs/DevelopmentGuide.md index 10534de556e..d7d94de9e44 100644 --- a/docs/DevelopmentGuide.md +++ b/docs/DevelopmentGuide.md @@ -98,6 +98,22 @@ $ bundle exec appraisal contrib rake spec:redis'[--seed,1234]' This can be useful for replicating conditions from CI or isolating certain tests. +**Checking test coverage** + +You can check test code coverage by creating a report _after_ running a test suite: +``` +# Run the desired test suite +$ bundle exec appraisal contrib rake spec:redis +# Generate report for the suite executed +$ bundle exec rake coverage:report +``` + +A webpage will be generated at `coverage/report/index.html` with the resulting report. + +Because you are likely not running all tests locally, your report will contain partial coverage results. +You *must* check the CI step `coverage` for the complete test coverage report, ensuring coverage is not +decreased. + ### Checking code quality **Linting** diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 7df62e93d6b..b61f4b0405d 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -5,6 +5,12 @@ require 'webmock/rspec' require 'climate_control' +# +SimpleCov.start+ must be invoked before any application code is loaded +require 'simplecov' +SimpleCov.start do + formatter SimpleCov::Formatter::SimpleFormatter +end + require 'ddtrace/encoding' require 'ddtrace/tracer' require 'ddtrace/span'