diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16b6c5a5..8ebac1ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,8 +21,6 @@ jobs: crystal: 1.9 - name: Install dependencies run: shards install - - name: Install coverage.py - run: "pip install --upgrade pip && pip install coverage && pip install pytest" - name: Run tests run: crystal spec --verbose --order random --error-on-warnings @@ -53,10 +51,8 @@ jobs: run: | sudo apt-get update sudo apt-get install kcov - - name: Install coverage.py - run: "pip install --upgrade pip && pip install coverage && pip install pytest" - name: Generate coverage - run: bin/crkcov --kcov-args --exclude-pattern=/usr/include,/usr/lib,lib/,spec/ --coverage-dir ${{ github.workspace }}/coverage + run: bin/crkcov --kcov-args --exclude-pattern=/usr/include,/usr/lib,lib/,spec/ - name: Report coverage env: COVERALLS_REPO_TOKEN: ${{ github.token }} diff --git a/.gitignore b/.gitignore index 35c618b3..6436ca2f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,3 @@ /docs/ /insecure_private_keys /lib/ -.coverage -coverage.xml diff --git a/shard.lock b/shard.lock index a0fececc..2c28e30d 100644 --- a/shard.lock +++ b/shard.lock @@ -14,7 +14,7 @@ shards: spectator: git: https://gitlab.com/arctic-fox/spectator.git - version: 0.12.0 + version: 0.11.6 sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git diff --git a/spec/coverage_reporter/api/jobs_spec.cr b/spec/coverage_reporter/api/jobs_spec.cr index db14806d..90066b40 100644 --- a/spec/coverage_reporter/api/jobs_spec.cr +++ b/spec/coverage_reporter/api/jobs_spec.cr @@ -75,7 +75,7 @@ Spectator.describe CoverageReporter::Api::Jobs do after_each do WebMock.reset - ENV.delete("COVERALLS_RUN_AT") + ENV.clear end it "calls the /jobs endpoint" do diff --git a/spec/coverage_reporter/api/webhook_spec.cr b/spec/coverage_reporter/api/webhook_spec.cr index 6e91bbcf..b3be7e21 100644 --- a/spec/coverage_reporter/api/webhook_spec.cr +++ b/spec/coverage_reporter/api/webhook_spec.cr @@ -15,8 +15,6 @@ Spectator.describe CoverageReporter::Api::Webhook do "X-Coveralls-Reporter" => "coverage-reporter", "X-Coveralls-Reporter-Version" => CoverageReporter::VERSION, "X-Coveralls-Source" => "cli", - "Accept" => "*/*", - "User-Agent" => "Crystal #{Crystal::VERSION}", } end diff --git a/spec/coverage_reporter/config_spec.cr b/spec/coverage_reporter/config_spec.cr index 0b51aafb..3e701f50 100644 --- a/spec/coverage_reporter/config_spec.cr +++ b/spec/coverage_reporter/config_spec.cr @@ -11,112 +11,15 @@ Spectator.describe CoverageReporter::Config do ) end + before_each { ENV.clear } + after_each { ENV.clear } + let(repo_token) { nil } let(path) { "" } let(job_flag_name) { nil } let(compare_ref) { nil } let(compare_sha) { nil } - before_each { delete_env_vars } - after_each { delete_env_vars } - - def delete_env_vars - ENV.delete("APPVEYOR") - ENV.delete("APPVEYOR_BUILD_VERSION") - ENV.delete("APPVEYOR_REPO_BRANCH") - ENV.delete("APPVEYOR_REPO_COMMIT") - ENV.delete("APPVEYOR_REPO_NAME") - ENV.delete("BUILDKITE") - ENV.delete("BUILDKITE_BUILD_NUMBER") - ENV.delete("BUILDKITE_BUILD_ID") - ENV.delete("BUILDKITE_PULL_REQUEST") - ENV.delete("BUILDKITE_BRANCH") - ENV.delete("BUILDKITE_COMMIT") - ENV.delete("CF_BRANCH") - ENV.delete("CF_BUILD_ID") - ENV.delete("CF_PULL_REQUEST_ID") - ENV.delete("CF_BRANCH") - ENV.delete("CF_REVISION") - ENV.delete("CI_BRANCH") - ENV.delete("CI_BUILD_NUMBER") - ENV.delete("CI_BUILD_URL") - ENV.delete("CI_COMMIT") - ENV.delete("CI_COMMIT_ID") - ENV.delete("CI_JOB_ID") - ENV.delete("CI_NAME") - ENV.delete("CI_PULL_REQUEST") - ENV.delete("CI_XCODE_PROJECT") - ENV.delete("CI_PULL_REQUEST_NUMBER") - ENV.delete("CIRCLECI") - ENV.delete("CIRCLE_WORKFLOW_ID") - ENV.delete("CIRCLE_BUILD_NUM") - ENV.delete("CIRCLE_BRANCH") - ENV.delete("CIRCLE_BUILD_URL") - ENV.delete("COVERALLS_REPO_TOKEN") - ENV.delete("COVERALLS_RUN_LOCALLY") - ENV.delete("COVERALLS_SERVICE_NAME") - ENV.delete("COVERALLS_SERVICE_NUMBER") - ENV.delete("COVERALLS_SERVICE_JOB_ID") - ENV.delete("COVERALLS_GIT_BRANCH") - ENV.delete("COVERALLS_GIT_COMMIT") - ENV.delete("DRONE") - ENV.delete("DRONE_BUILD_NUMBER") - ENV.delete("DRONE_PULL_REQUEST") - ENV.delete("DRONE_BRANCH") - ENV.delete("DRONE_COMMIT") - ENV.delete("GITHUB_ACTIONS") - ENV.delete("GITHUB_HEAD_REF") - ENV.delete("GITHUB_JOB") - ENV.delete("GITHUB_REF") - ENV.delete("GITHUB_REF_NAME") - ENV.delete("GITHUB_REPOSITORY") - ENV.delete("GITHUB_RUN_ATTEMPT") - ENV.delete("GITHUB_RUN_ID") - ENV.delete("GITHUB_SERVER_URL") - ENV.delete("GITHUB_SHA") - ENV.delete("GITLAB_CI") - ENV.delete("CI_JOB_NAME") - ENV.delete("CI_PIPELINE_IID") - ENV.delete("CI_COMMIT_REF_NAME") - ENV.delete("CI_COMMIT_SHA") - ENV.delete("CI_JOB_URL") - ENV.delete("CI_PIPELINE_URL") - ENV.delete("CI_MERGE_REQUEST_IID") - ENV.delete("JENKINS_HOME") - ENV.delete("BUILD_ID") - ENV.delete("BUILD_NUMBER") - ENV.delete("BRANCH_NAME") - ENV.delete("ghprbPullId") - ENV.delete("SEMAPHORE") - ENV.delete("SEMAPHORE_WORKFLOW_ID") - ENV.delete("SEMAPHORE_GIT_WORKING_BRANCH") - ENV.delete("SEMAPHORE_GIT_PR_NUMBER") - ENV.delete("SEMAPHORE_GIT_SHA") - ENV.delete("SEMAPHORE_ORGANIZATION_URL") - ENV.delete("SEMAPHORE_JOB_ID") - ENV.delete("SURF_SHA1") - ENV.delete("SURF_REF") - ENV.delete("TDDIUM") - ENV.delete("TDDIUM_SESSION_ID") - ENV.delete("TDDIUM_TID") - ENV.delete("TDDIUM_PR_ID") - ENV.delete("TDDIUM_CURRENT_BRANCH") - ENV.delete("TRAVIS") - ENV.delete("TRAVIS_PULL_REQUEST") - ENV.delete("TRAVIS_BRANCH") - ENV.delete("TRAVIS_JOB_NUMBER") - ENV.delete("TRAVIS_BUILD_NUMBER") - ENV.delete("TF_BUILD") - ENV.delete("BUILD_BUILDID") - ENV.delete("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER") - ENV.delete("BUILD_SOURCEBRANCHNAME") - ENV.delete("BUILD_SOURCEVERSION") - ENV.delete("WERCKER") - ENV.delete("WERCKER_BUILD_ID") - ENV.delete("WERCKER_GIT_BRANCH") - ENV.delete("WERCKER_GIT_COMMIT") - end - describe ".new" do context "without repo_token" do it "raises an exception" do @@ -165,7 +68,9 @@ Spectator.describe CoverageReporter::Config do end context "with ENV preset" do - before_each { ENV["COVERALLS_REPO_TOKEN"] = "env-token" } + before_each do + ENV["COVERALLS_REPO_TOKEN"] = "env-token" + end it "doesn't raise an exception" do expect { subject }.not_to raise_error @@ -245,6 +150,7 @@ Spectator.describe CoverageReporter::Config do end context "for generic CI" do + # Imagine we are on Circle before_each do ENV["CIRCLECI"] = "1" ENV["CIRCLE_WORKFLOW_ID"] = "circle-service-number" diff --git a/spec/coverage_reporter/parsers/coveragepy_parser_spec.cr b/spec/coverage_reporter/parsers/coveragepy_parser_spec.cr index d9066b5f..113a0ae9 100644 --- a/spec/coverage_reporter/parsers/coveragepy_parser_spec.cr +++ b/spec/coverage_reporter/parsers/coveragepy_parser_spec.cr @@ -3,25 +3,6 @@ require "../../spec_helper" Spectator.describe CoverageReporter::CoveragepyParser do subject { described_class.new(nil) } - before_all do - error = IO::Memory.new - output = IO::Memory.new - process_status = Process.run( - command: "coverage run -m pytest", - chdir: "spec/fixtures/python", - shell: true, - error: error, - output: output - ) - unless process_status.success? - raise "Failed: #{error}\n#{output}" - end - end - - after_all do - File.delete("spec/fixtures/python/.coverage") - end - describe "#matches?" do it "matches only SQLite3 db file" do expect(subject.matches?("spec/fixtures/python/.coverage")).to eq true @@ -33,24 +14,12 @@ Spectator.describe CoverageReporter::CoveragepyParser do describe "#parse" do let(filename) { "spec/fixtures/python/.coverage" } - context "with valid coverage file" do - it "reads the coverage" do - reports = subject.parse(filename) - - expect(reports.size).to eq 4 - expect(reports.map(&.to_h.transform_keys(&.to_s))) - .to eq YAML.parse(File.read("#{__DIR__}/coveragepy_results.yml")) - end - end - - context "with invalid coverage file" do - let(filename) { "spec/fixtures/simplecov/with-only-lines.resultset.json" } - - it "raises an error" do - io_memory = IO::Memory.new("some error") + it "reads the coverage" do + result = subject.parse(filename) - expect { subject.parse(filename, io_memory) }.to raise_error(CoverageReporter::CoveragepyParser::ParserError, "some error") - end + expect(result.size).to eq 20 + expect(result.map(&.to_h.transform_keys(&.to_s))) + .to eq YAML.parse(File.read("#{__DIR__}/coveragepy_results.yml")) end end end diff --git a/spec/coverage_reporter/parsers/coveragepy_results.yml b/spec/coverage_reporter/parsers/coveragepy_results.yml index dbbdfead..fa6f4c32 100644 --- a/spec/coverage_reporter/parsers/coveragepy_results.yml +++ b/spec/coverage_reporter/parsers/coveragepy_results.yml @@ -1,45 +1,526 @@ --- -- name: spec/fixtures/python/src/__init__.py - coverage: [] - branches: [] - source_digest: d41d8cd98f00b204e9800998ecf8427e -- name: spec/fixtures/python/src/boring_math.py - coverage: - - 1 - - 1 - - 1 - - 0 - - 0 - - - - 0 - - - - - - 1 - - 0 - - 0 - - - - 0 - branches: [] - source_digest: cc049f1b8d4db11de4a9d6171d65fde1 -- name: spec/fixtures/python/src/tests/__init__.py - coverage: [] - branches: [] - source_digest: d41d8cd98f00b204e9800998ecf8427e -- name: spec/fixtures/python/src/tests/test_boring_math.py - coverage: - - - - - - - - - - - - - - 1 - - - - 1 - - - - 1 - - 1 - - 1 - - 1 - branches: [] - source_digest: 4bff5ace6ccdf4a4e4659d0e5cc2276b +- name: spec/fixtures/python/conftest.py + coverage: + - 1 + - 1 + - + - + - 1 + - 1 + - 1 + - + - + - 1 + - 1 + - 1 + - + - 1 + - 1 + - + source_digest: b495247237e404918faf3b9b6f4fff45 +- name: spec/fixtures/python/tests/__init__.py + coverage: + - 1 + source_digest: dfc6c30bc1d49f27cafb5a99808189c7 +- name: spec/fixtures/python/tests/00_empty_test.py + coverage: + - 1 + - + - + - + - + - + - + - + - 1 + - + - + - 1 + - + - + - + - + - 0 + source_digest: 56c0d54db79148fce1d7ac64a9280426 +- name: spec/fixtures/python/tests/01_basic_test.py + coverage: + - 1 + - + - + - 1 + - + - + - + - 1 + - 1 + source_digest: f6acb1ce108b884c46874cf850df2c67 +- name: spec/fixtures/python/other_code/__init__.py + coverage: + - 1 + source_digest: dfc6c30bc1d49f27cafb5a99808189c7 +- name: spec/fixtures/python/other_code/services.py + coverage: + - 1 + - 1 + - + - + - 1 + - + - + - + - + - 1 + - 1 + - 1 + - 1 + - + - + - 1 + - + - + - 1 + - + - + - + - 0 + - + - 0 + - + - 0 + - + - 0 + - 0 + - + - + - 1 + - 1 + - + - 1 + - + - 1 + - + - 1 + - 1 + - + - + - 1 + - 1 + - 1 + - 1 + - + - + - 1 + - + - 1 + - 1 + - 1 + - 1 + - + source_digest: 5d790b16f5d4bdebf8c17b6c95204095 +- name: spec/fixtures/python/tests/02_special_assertions_test.py + coverage: + - 1 + - + - + - 1 + - + - + - + - 1 + - 1 + - + - + - 1 + - + - + - + - 1 + - + - 1 + - 1 + - + - + - 1 + - + - + - 1 + - + - + - + - + - 1 + source_digest: 01e00c9b8dd77cddb06846de87f1a444 +- name: spec/fixtures/python/tests/03_simple_fixture_test.py + coverage: + - 1 + - + - + - 1 + - + - + - + - + - 1 + - 1 + - + - + - 1 + - 1 + - + - + - + - 1 + - + - + - 1 + - + - + - + - 1 + source_digest: c90370431ed68b82228436f0befbec61 +- name: spec/fixtures/python/tests/04_fixture_returns_test.py + coverage: + - 1 + - + - + - 1 + - + - + - + - + - 1 + - 1 + - + - + - 1 + - 1 + - + - + - + - + - 1 + - 1 + source_digest: 81db404179c6cec7994af12047991760 +- name: spec/fixtures/python/tests/05_yield_fixture_test.py + coverage: + - 1 + - + - + - 1 + - 1 + - 1 + - + - + - 1 + - 1 + - + - + - + - + - 1 + - 1 + - + - + - 1 + - + - 1 + - 1 + source_digest: 1fe397b84c343175fe19bc2d98610db3 +- name: spec/fixtures/python/tests/06_request_test.py + coverage: + - 1 + - + - + - 1 + - 1 + - 1 + - + - + - 1 + - 1 + - + - + - + - + - 1 + - 1 + - 1 + - 1 + source_digest: 40c853cd4433aee28b2e5bb42adaca7a +- name: spec/fixtures/python/tests/07_request_finalizer_test.py + coverage: + - 1 + - + - + - 1 + - 1 + - 1 + - + - + - 1 + - 1 + - + - + - + - + - 1 + - 1 + - 1 + - + - + - 1 + - 1 + - + - + - 1 + - + - + - 1 + source_digest: c3c04f0a7ce3ce59ab3872df0481490a +- name: spec/fixtures/python/tests/08_params_test.py + coverage: + - 1 + - + - + - 1 + - 1 + - + - + - 1 + - 1 + - + - + - 1 + - 1 + - + - + - + - + - 1 + - + - + - 1 + - 1 + - + - + - + - + - 1 + source_digest: 0a2f136c649309a1ab7bf24693b15ade +- name: spec/fixtures/python/tests/09_params-ception_test.py + coverage: + - 1 + - + - + - 1 + - 1 + - + - + - + - 1 + - + - + - 1 + - 1 + - + - + - + - 1 + - + - + - 1 + - + - + - + - 1 + - + - 1 + source_digest: '08fcbfc6f3fe95931875f9adfc16db11' +- name: spec/fixtures/python/tests/10_advanced_params-ception_test.py + coverage: + - 1 + - + - + - 1 + - 1 + - + - + - + - 1 + - + - + - 1 + - 1 + - + - + - + - 1 + - 1 + - + - + - + - + - + - + - 1 + - 1 + - 1 + - + source_digest: 7f68fc4cf0936d2af5c06c6c22b97b51 +- name: spec/fixtures/python/tests/11_mark_test.py + coverage: + - 1 + - + - + - 1 + - 1 + - + - + - + - 1 + - + - + - 1 + - 1 + - 1 + - + - + - 1 + - 1 + - 1 + - + - + - + - 1 + - + - + - 1 + - 1 + - + - + - + - 0 + - 0 + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + source_digest: 7af52bb924bafde2a4fd30d206c94f6c +- name: spec/fixtures/python/tests/14_class_based_test.py + coverage: + - 1 + - + - + - + - + - + - 1 + - 1 + - + - 1 + - 0 + - + - 1 + - 1 + - 1 + - 1 + source_digest: b3448ac6917f9d05ad2dcd6b4d6ed5ea +- name: spec/fixtures/python/tests/15_advanced_class_test.py + coverage: + - 1 + - + - + - 1 + - 1 + - 1 + - + - + - 1 + - 1 + - 1 + - + - + - 1 + - 1 + - 1 + - 1 + - 1 + - + - 1 + - 1 + - 1 + - + - 1 + - 1 + - 1 + source_digest: ddcac19f774bb0306cb73e084f0e5abb +- name: spec/fixtures/python/tests/16_scoped_and_meta_fixtures_test.py + coverage: + - 1 + - 1 + - + - + - 1 + - 1 + - + - + - + - 1 + - 1 + - 1 + - + - + - 1 + - 1 + - + - + - + - 1 + source_digest: 2a3592da214eb09d98e00ab80e0d18c4 +- name: spec/fixtures/python/tests/19_re_usable_mock_test.py + coverage: + - 1 + - 1 + - + - + - 1 + - 1 + - + - + - + - 1 + - 1 + - 1 + - + - + - 1 + - 1 + - 1 + - 1 + - + - + - 1 + - 1 + - + - 1 + - 1 + source_digest: 700b6685d08b9438c8717bfd2c1efc29 diff --git a/spec/coverage_reporter/reporter_spec.cr b/spec/coverage_reporter/reporter_spec.cr index 595918f4..1add3ac4 100644 --- a/spec/coverage_reporter/reporter_spec.cr +++ b/spec/coverage_reporter/reporter_spec.cr @@ -51,7 +51,7 @@ Spectator.describe CoverageReporter::Reporter do ENV["COVERALLS_ENDPOINT"] = "https://example.com" end - after_each { ENV.delete("COVERALLS_ENDPOINT") } + after_each { ENV.clear } it "doesn't raise an error" do expect { subject.report }.not_to raise_error @@ -65,7 +65,7 @@ Spectator.describe CoverageReporter::Reporter do ENV["COVERALLS_DEVELOPMENT"] = "1" end - after_each { ENV.delete("COVERALLS_DEVELOPMENT") } + after_each { ENV.clear } it "doesn't raise an error" do expect { subject.report }.not_to raise_error @@ -118,7 +118,7 @@ Spectator.describe CoverageReporter::Reporter do ENV["COVERALLS_ENDPOINT"] = "https://example.com" end - after_each { ENV.delete("COVERALLS_ENDPOINT") } + after_each { ENV.clear } it "doesn't raise an error" do expect { subject.parallel_done }.not_to raise_error diff --git a/spec/fixtures/python/.coverage b/spec/fixtures/python/.coverage new file mode 100644 index 00000000..463ec423 Binary files /dev/null and b/spec/fixtures/python/.coverage differ diff --git a/spec/fixtures/python/conftest.py b/spec/fixtures/python/conftest.py new file mode 100644 index 00000000..d9b967e3 --- /dev/null +++ b/spec/fixtures/python/conftest.py @@ -0,0 +1,16 @@ +from __future__ import print_function +from pytest import fixture + + +@fixture +def global_fixture(): + print("\n(Doing global fixture setup stuff!)") + + +def pytest_configure(config): + config.addinivalue_line( + "markers", "db: Example marker for tagging Database related tests" + ) + config.addinivalue_line( + "markers", "slow: Example marker for tagging extremely slow tests" + ) diff --git a/spec/fixtures/python/other_code/__init__.py b/spec/fixtures/python/other_code/__init__.py new file mode 100644 index 00000000..350b53fa --- /dev/null +++ b/spec/fixtures/python/other_code/__init__.py @@ -0,0 +1 @@ +from __future__ import print_function diff --git a/spec/fixtures/python/other_code/services.py b/spec/fixtures/python/other_code/services.py new file mode 100644 index 00000000..60f66f58 --- /dev/null +++ b/spec/fixtures/python/other_code/services.py @@ -0,0 +1,56 @@ +import time +from collections import namedtuple + + +class ExpensiveClass(object): + """ + A fake Class that takes a long time to fully initialize + """ + + def __init__(self): + print("(Initializing ExpensiveClass instance...)") + time.sleep(0.2) + print("(ExpensiveClass instance complete!)") + + +FakeRow = namedtuple("FakeRow", ("id", "name", "value")) + + +def db_service(query_parameters): + """ + A fake DB service that takes a remarkably long time to yield results + """ + print("(Doing expensive database stuff!)") + + time.sleep(5.0) + + data = [FakeRow(0, "Foo", 19.95), FakeRow(1, "Bar", 1.99), FakeRow(2, "Baz", 9.99)] + + print("(Done doing expensive database stuff)") + return data + + +def count_service(query_parameters): + print("count_service: Performing a query (and counting the results)...") + + data = db_service(query_parameters) + + count = len(data) + + print("Found {} result(s)!".format(count)) + return count + + +DATA_SET_A = { + "Foo": "Bar", + "Baz": [5, 7, 11], + "Qux": {"A": "Boston", "B": "Python", "C": "TDD"}, +} + +DATA_SET_B = DATA_SET_A + +DATA_SET_C = { + "Foo": "Bar", + "Baz": [3, 5, 7], + "Qux": {"A": "Boston", "B": "Python", "C": "TDD"}, +} diff --git a/spec/fixtures/python/src/__init__.py b/spec/fixtures/python/src/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/spec/fixtures/python/src/boring_math.py b/spec/fixtures/python/src/boring_math.py deleted file mode 100644 index 3ddc3d5b..00000000 --- a/spec/fixtures/python/src/boring_math.py +++ /dev/null @@ -1,14 +0,0 @@ -def fib(n): - if n == 0: - return 1 - elif n == 1: - return 1 - else: - return fib(n - 1) + fib(n - 2) - - -def fac(n): - if n == 0: - return 1 - else: - return n * fac(n - 1) diff --git a/spec/fixtures/python/src/tests/__init__.py b/spec/fixtures/python/src/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/spec/fixtures/python/src/tests/test_boring_math.py b/spec/fixtures/python/src/tests/test_boring_math.py deleted file mode 100644 index e4665b99..00000000 --- a/spec/fixtures/python/src/tests/test_boring_math.py +++ /dev/null @@ -1,14 +0,0 @@ -# To re-generate the .coverage file: -# coverage run -m pytest -# This requires two Python libraries: -# pip install coverage -# pip install pytest - -from unittest import TestCase - -from src import boring_math - -class TestBoringMath(TestCase): - def test_fib_0(self): - result = boring_math.fib(0) - self.assertEqual(result, 1) diff --git a/spec/fixtures/python/tests/00_empty_test.py b/spec/fixtures/python/tests/00_empty_test.py new file mode 100644 index 00000000..5bf4013d --- /dev/null +++ b/spec/fixtures/python/tests/00_empty_test.py @@ -0,0 +1,17 @@ +def test_empty(): + """ + PyTest tests are callables whose names start with "test" + (by default) + + It looks for them in modules whose name starts with "test_" or ends with "_test" + (by default) + """ + pass + + +def empty_test(): + """ + My name doesn't start with "test", so I won't get run. + (by default ;-) + """ + pass diff --git a/spec/fixtures/python/tests/01_basic_test.py b/spec/fixtures/python/tests/01_basic_test.py new file mode 100644 index 00000000..8eb8558a --- /dev/null +++ b/spec/fixtures/python/tests/01_basic_test.py @@ -0,0 +1,9 @@ +from other_code.services import DATA_SET_A, DATA_SET_B, DATA_SET_C + + +def test_example(): + """ + But really, test cases should be callables containing assertions: + """ + print("\nRunning test_example...") + assert DATA_SET_A == DATA_SET_B diff --git a/spec/fixtures/python/tests/02_special_assertions_test.py b/spec/fixtures/python/tests/02_special_assertions_test.py new file mode 100644 index 00000000..3bec60d6 --- /dev/null +++ b/spec/fixtures/python/tests/02_special_assertions_test.py @@ -0,0 +1,30 @@ +import pytest + + +def test_div_zero_exception(): + """ + pytest.raises can assert that exceptions are raised (catching them) + """ + with pytest.raises(ZeroDivisionError): + x = 1 / 0 + + +def test_keyerror_details(): + """ + The raised exception can be referenced, and further inspected (or asserted) + """ + my_map = {"foo": "bar"} + + with pytest.raises(KeyError) as ke: + baz = my_map["baz"] + + # Our KeyError should reference the missing key, "baz" + assert "baz" in str(ke) + + +def test_approximate_matches(): + """ + pytest.approx can be used to assert "approximate" numerical equality + (compare to "assertAlmostEqual" in unittest.TestCase) + """ + assert 0.1 + 0.2 == pytest.approx(0.3) diff --git a/spec/fixtures/python/tests/03_simple_fixture_test.py b/spec/fixtures/python/tests/03_simple_fixture_test.py new file mode 100644 index 00000000..65fa0ce6 --- /dev/null +++ b/spec/fixtures/python/tests/03_simple_fixture_test.py @@ -0,0 +1,25 @@ +import pytest + + +def test_with_local_fixture(local_fixture): + """ + Fixtures can be invoked simply by having a positional arg + with the same name as a fixture: + """ + print("Running test_with_local_fixture...") + assert True + + +@pytest.fixture +def local_fixture(): + """ + Fixtures are callables decorated with @fixture + """ + print("\n(Doing Local Fixture setup stuff!)") + + +def test_with_global_fixture(global_fixture): + """ + Fixtures can also be shared across test files (see conftest.py) + """ + print("Running test_with_global_fixture...") diff --git a/spec/fixtures/python/tests/04_fixture_returns_test.py b/spec/fixtures/python/tests/04_fixture_returns_test.py new file mode 100644 index 00000000..49fba62c --- /dev/null +++ b/spec/fixtures/python/tests/04_fixture_returns_test.py @@ -0,0 +1,20 @@ +import pytest + + +def test_with_data_fixture(one_fixture): + """ + PyTest finds the fixture whose name matches the argument, + calls it, and passes that return value into our test case: + """ + print("\nRunning test_with_data_fixture: {}".format(one_fixture)) + assert one_fixture == 1 + + +@pytest.fixture +def one_fixture(): + """ + Beyond just "doing stuff", fixtures can return data, which + PyTest will pass to the test cases that refer to it... + """ + print("\n(Returning 1 from data_fixture)") + return 1 diff --git a/spec/fixtures/python/tests/05_yield_fixture_test.py b/spec/fixtures/python/tests/05_yield_fixture_test.py new file mode 100644 index 00000000..13e42815 --- /dev/null +++ b/spec/fixtures/python/tests/05_yield_fixture_test.py @@ -0,0 +1,22 @@ +import pytest + + +def test_with_yield_fixture(yield_fixture): + print("\n Running test_with_yield_fixture: {}".format(yield_fixture)) + assert "foo" in yield_fixture + + +@pytest.fixture +def yield_fixture(): + """ + Fixtures can yield their data + (additional code will run after the test) + """ + print("\n\n(Initializing yield_fixture)") + x = {"foo": "bar"} + + # Remember, unlike generators, fixtures should only yield once (if at all) + yield x + + print("\n(Cleaning up yield_fixture)") + del(x) diff --git a/spec/fixtures/python/tests/06_request_test.py b/spec/fixtures/python/tests/06_request_test.py new file mode 100644 index 00000000..ae70eb04 --- /dev/null +++ b/spec/fixtures/python/tests/06_request_test.py @@ -0,0 +1,18 @@ +import pytest + + +def test_with_introspection(introspective_fixture): + print("\nRunning test_with_introspection...") + assert True + + +@pytest.fixture +def introspective_fixture(request): + """ + The request fixture allows introspection into the + "requesting" test case + """ + print("\n\nintrospective_fixture:") + print("...Called at {}-level scope".format(request.scope)) + print(" ...In the {} module".format(request.module)) + print(" ...On the {} node".format(request.node)) diff --git a/spec/fixtures/python/tests/07_request_finalizer_test.py b/spec/fixtures/python/tests/07_request_finalizer_test.py new file mode 100644 index 00000000..85a279c7 --- /dev/null +++ b/spec/fixtures/python/tests/07_request_finalizer_test.py @@ -0,0 +1,27 @@ +import pytest + + +def test_with_safe_cleanup_fixture(safe_fixture): + print("\nRunning test_with_safe_cleanup_fixture...") + assert True + + +@pytest.fixture +def safe_fixture(request): + """ + The request can also be used to apply post-test callbacks + (these will run even if the Fixture itself fails!) + """ + print("\n(Begin setting up safe_fixture)") + request.addfinalizer(safe_cleanup) + risky_function() + + +def safe_cleanup(): + print("\n(Cleaning up after safe_fixture!)") + + +def risky_function(): + # # Uncomment to simulate a failure during Fixture setup! + # raise Exception("Whoops, I guess that risky function didn't work...") + print(" (Risky Function: Totally worth it!)") diff --git a/spec/fixtures/python/tests/08_params_test.py b/spec/fixtures/python/tests/08_params_test.py new file mode 100644 index 00000000..8f6203c8 --- /dev/null +++ b/spec/fixtures/python/tests/08_params_test.py @@ -0,0 +1,27 @@ +import pytest + + +def test_parameterization(letter): + print("\n Running test_parameterization with {}".format(letter)) + + +def test_modes(mode): + print("\n Running test_modes with {}".format(mode)) + + +@pytest.fixture(params=["a", "b", "c", "d", "e"]) +def letter(request): + """ + Fixtures with parameters will run once per param + (You can access the current param via the request fixture) + """ + yield request.param + + +@pytest.fixture(params=[1, 2, 3], ids=['foo', 'bar', 'baz']) +def mode(request): + """ + Fixtures with parameters will run once per param + (You can access the current param via the request fixture) + """ + yield request.param diff --git a/spec/fixtures/python/tests/09_params-ception_test.py b/spec/fixtures/python/tests/09_params-ception_test.py new file mode 100644 index 00000000..b85d4bec --- /dev/null +++ b/spec/fixtures/python/tests/09_params-ception_test.py @@ -0,0 +1,26 @@ +import pytest + + +@pytest.fixture(params=["a", "b", "c", "d"]) +def letters_fixture(request): + """ + Fixtures can cause tests to be run multiple times (once per parameter) + """ + yield request.param + + +@pytest.fixture(params=[1, 2, 3, 4]) +def numbers_fixture(request): + """ + Fixtures can invoke each other (producing cartesian products of parameters) + """ + yield request.param + + +def test_fixtureception(letters_fixture, numbers_fixture): + """ + Print out our combined fixture "product" + """ + coordinate = letters_fixture + str(numbers_fixture) + + print('\nRunning test_fixtureception with "{}"'.format(coordinate)) diff --git a/spec/fixtures/python/tests/10_advanced_params-ception_test.py b/spec/fixtures/python/tests/10_advanced_params-ception_test.py new file mode 100644 index 00000000..4db2f3a5 --- /dev/null +++ b/spec/fixtures/python/tests/10_advanced_params-ception_test.py @@ -0,0 +1,28 @@ +import pytest + + +@pytest.fixture(params=[1, 2, 3, 4]) +def numbers_fixture(request): + """ + Fixtures can cause tests to be run multiple times (once per parameter) + """ + yield request.param + + +@pytest.fixture(params=["a", "b", "c", "d"]) +def coordinates_fixture(request, numbers_fixture): + """ + Fixtures can invoke each other (producing cartesian products of params) + """ + coordinate = request.param + str(numbers_fixture) + yield coordinate + # # Uncomment for fun 80s board game reference (and fixture filtering) + # if coordinate == 'b2': + # print "(Don't sink my Battleship!)" + # pytest.skip() + + +def test_advanced_fixtureception(coordinates_fixture): + print( + '\nRunning test_advanced_fixtureception with "{}"'.format(coordinates_fixture) + ) diff --git a/spec/fixtures/python/tests/11_mark_test.py b/spec/fixtures/python/tests/11_mark_test.py new file mode 100644 index 00000000..d1f67c4f --- /dev/null +++ b/spec/fixtures/python/tests/11_mark_test.py @@ -0,0 +1,55 @@ +import pytest + + +@pytest.mark.db +def test_fake_query(): + """ + pytest.mark can be used to "tag" tests for later reference + """ + assert True + + +@pytest.mark.slow +def test_fake_stats_function(): + assert True + + +@pytest.mark.db +@pytest.mark.slow +def test_fake_multi_join_query(): + """ + Test cases can have multiple marks assigned + """ + assert True + + +@pytest.mark.db +def asserty_callable_thing(): + """ + PyTest still only runs "tests", not just any callabe with a mark + """ + print("This isn't even a test! And it fails!") + assert False + + +""" +Tags can be used to target (or omit) tests in the runner: + +# Run all three tests in this module (verbosely) +pytest -v 10_mark_test.py + +# Run one specific test by Node name: +pytest -v 10_mark_test.py::test_fake_query + +# Run all tests with "query" in their names +pytest -v -k query + +# Run all tests with "stats" or "join" in their names +pytest -v -k "stats or join" + +# Run all tests marked with "db" +pytest -v -m db + +# Run all tests marked with "db", but not with "slow" +pytest -v -m "db and not slow" +""" diff --git a/spec/fixtures/python/tests/12_special_marks.py b/spec/fixtures/python/tests/12_special_marks.py new file mode 100644 index 00000000..10b2ebab --- /dev/null +++ b/spec/fixtures/python/tests/12_special_marks.py @@ -0,0 +1,37 @@ +import pytest + +dev_s3_credentials = None + + +@pytest.mark.skip +def test_broken_feature(): + # Always skipped! + assert False + + +@pytest.mark.skipif(not dev_s3_credentials, reason="S3 creds not found!") +def test_s3_api(): + # Skipped if a certain condition is met + assert True + + +@pytest.mark.xfail +def test_where_failure_is_acceptable(): + # Allows failed assertions (returns "XPASS" if there are no failures) + assert True + + +@pytest.mark.xfail +def test_where_failure_is_accepted(): + # Allows failed assertions (returns "xfail" on failure) + assert False + + +@pytest.mark.xfail(strict=True) +def test_where_failure_is_mandatory(): + # Requires failed assertions! (returns "xfail" on failure; FAILs on pass!) + assert True + + +# # Uncomment to skip everything in the module +# pytest.skip("This whole Module is problematic at best!", allow_module_level=True) diff --git a/spec/fixtures/python/tests/13_mark_parametrization.py b/spec/fixtures/python/tests/13_mark_parametrization.py new file mode 100644 index 00000000..e3410acb --- /dev/null +++ b/spec/fixtures/python/tests/13_mark_parametrization.py @@ -0,0 +1,24 @@ +import pytest + + +@pytest.mark.parametrize("number", [1, 2, 3, 4, 5]) +def test_numbers(number): + """ + mark can be used to apply "inline" parameterization, without a fixture + """ + print("\nRunning test_numbers with {}".format(number)) + + +@pytest.mark.parametrize("x, y", [(1, 1), (1, 2), (2, 2)]) +def test_dimensions(x, y): + """ + mark.parametrize can even unpack tuples into named parameters + """ + print("\nRunning test_coordinates with {}x{}".format(x, y)) + +@pytest.mark.parametrize("mode", [1, 2, 3], ids=['foo', 'bar', 'baz']) +def test_modes(mode): + """ + The `ids` kwarg can be used to rename the parameters + """ + print("\nRunning test_modes with {}".format(mode)) diff --git a/spec/fixtures/python/tests/14_class_based_test.py b/spec/fixtures/python/tests/14_class_based_test.py new file mode 100644 index 00000000..49e49460 --- /dev/null +++ b/spec/fixtures/python/tests/14_class_based_test.py @@ -0,0 +1,16 @@ +class TestSimpleClass(object): + """ + Classes can still be used to organize collections of test cases, with + each test being a Method on the Class, rather than a standalone function. + """ + + x = 1 + y = 2 + + def regular_method(self): + print("\n(This is a regular, non-test-case method.)") + + def test_two_checking_method(self): + print("\nRunning TestSimpleClass.test_twos_method") + assert self.x != 2 + assert self.y == 2 diff --git a/spec/fixtures/python/tests/15_advanced_class_test.py b/spec/fixtures/python/tests/15_advanced_class_test.py new file mode 100644 index 00000000..7b245ee2 --- /dev/null +++ b/spec/fixtures/python/tests/15_advanced_class_test.py @@ -0,0 +1,26 @@ +from pytest import fixture, mark + + +@fixture +def class_fixture(): + print("\n (class_fixture)") + + +@fixture +def bonus_fixture(): + print("\n (bonus_fixture)") + + +@mark.usefixtures("class_fixture") +class TestIntermediateClass(object): + @fixture(autouse=True) + def method_fixture(self): + print("\n(autouse method_fixture)") + + def test1(self): + print("\n Running TestIntermediateClass.test1") + assert True + + def test2(self, bonus_fixture): + print("\n Running TestIntermediateClass.test2") + assert True diff --git a/spec/fixtures/python/tests/16_scoped_and_meta_fixtures_test.py b/spec/fixtures/python/tests/16_scoped_and_meta_fixtures_test.py new file mode 100644 index 00000000..d1b9f6d8 --- /dev/null +++ b/spec/fixtures/python/tests/16_scoped_and_meta_fixtures_test.py @@ -0,0 +1,20 @@ +from pytest import fixture, mark +from other_code.services import ExpensiveClass + + +@fixture(scope="module", autouse=True) +def scoped_fixture(): + """ + Scoping affects how often fixtures are (re)initialized + """ + print("\n(Begin Module-scoped fixture)") + yield ExpensiveClass() + print("\n(End Module-scoped fixture)") + + +@mark.parametrize("x", range(1, 51)) +def test_scoped_fixtures(x): + """ + A (hopefully fast!) test, to be run with fifty different parameters... + """ + print("\n Running test_scoped_fixture") diff --git a/spec/fixtures/python/tests/17_marked_meta_fixtures.py b/spec/fixtures/python/tests/17_marked_meta_fixtures.py new file mode 100644 index 00000000..4caf6fd8 --- /dev/null +++ b/spec/fixtures/python/tests/17_marked_meta_fixtures.py @@ -0,0 +1,24 @@ +from pytest import fixture, mark + + +@fixture(scope="module") +def meta_fixture(): + print("\n*** begin meta_fixture ***") + yield + print("\n*** end meta_fixture ***") + + +# Apply this fixture to everything in this module! +pytestmark = mark.usefixtures("meta_fixture") + + +def test_with_meta_fixtures_a(): + print("\n Running test_with_meta_fixtures_a") + + +def test_with_meta_fixtures_b(): + print("\n Running test_with_meta_fixtures_b") + + +# How could we tell meta_fixture to only run once, "around" our tests? +# (See 16_scoped_and_meta_fixtures_test.py for a hint...) diff --git a/spec/fixtures/python/tests/18_the_mocker_fixture.py b/spec/fixtures/python/tests/18_the_mocker_fixture.py new file mode 100644 index 00000000..4d697eff --- /dev/null +++ b/spec/fixtures/python/tests/18_the_mocker_fixture.py @@ -0,0 +1,20 @@ +from other_code.services import count_service + + +def test_simple_mocking(mocker): + """ + pytest-mock provides a fixture for easy, self-cleaning mocking + """ + mock_db_service = mocker.patch("other_code.services.db_service", autospec=True) + + mock_data = [(0, "fake row", 0.0)] + + mock_db_service.return_value = mock_data + + print("\n(Calling count_service with the DB mocked out...)") + + c = count_service("foo") + + mock_db_service.assert_called_with("foo") + + assert c == 1 diff --git a/spec/fixtures/python/tests/19_re_usable_mock_test.py b/spec/fixtures/python/tests/19_re_usable_mock_test.py new file mode 100644 index 00000000..d4fdfb94 --- /dev/null +++ b/spec/fixtures/python/tests/19_re_usable_mock_test.py @@ -0,0 +1,25 @@ +from other_code.services import count_service +from pytest import fixture, raises + + +@fixture +def re_usable_db_mocker(mocker): + """ + Fixtures can invoke mocker to yield "re-usable" mocks + """ + mock_db_service = mocker.patch("other_code.services.db_service", autospec=True) + mock_db_service.return_value = [(0, "fake row", 0.0)] + return mock_db_service + + +def test_re_usable_mocker(re_usable_db_mocker): + c = count_service("foo") + re_usable_db_mocker.assert_called_with("foo") + assert c == 1 + + +def test_mocker_with_exception(re_usable_db_mocker): + re_usable_db_mocker.side_effect = Exception("Oh noes!") + + with raises(Exception): + count_service("foo") diff --git a/spec/fixtures/python/tests/__init__.py b/spec/fixtures/python/tests/__init__.py new file mode 100644 index 00000000..350b53fa --- /dev/null +++ b/spec/fixtures/python/tests/__init__.py @@ -0,0 +1 @@ +from __future__ import print_function diff --git a/spec/fixtures/python/tests/other_stuff.py b/spec/fixtures/python/tests/other_stuff.py new file mode 100644 index 00000000..bdece7a7 --- /dev/null +++ b/spec/fixtures/python/tests/other_stuff.py @@ -0,0 +1,6 @@ +def test_in_non_test_module(): + """ + PyTest will recognize this function as a test... + But will not collect tests from this file (by default) + """ + print("\nRunning test_in_non_test_module...") diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index f0cb25d7..3c33c0ca 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -9,6 +9,7 @@ Spectator.configure do |config| config.randomize config.before_suite do + ENV.clear CoverageReporter::Log.set(CoverageReporter::Log::Level::Suppress) end end diff --git a/src/coverage_reporter/parsers/coveragepy_parser.cr b/src/coverage_reporter/parsers/coveragepy_parser.cr index 62a16b7f..7e2b297c 100644 --- a/src/coverage_reporter/parsers/coveragepy_parser.cr +++ b/src/coverage_reporter/parsers/coveragepy_parser.cr @@ -3,13 +3,16 @@ require "sqlite3" module CoverageReporter class CoveragepyParser < BaseParser - class ParserError < RuntimeError - end - def self.name "python" end + QUERY = <<-SQL + SELECT file.path, line_bits.numbits + FROM line_bits + INNER JOIN file ON (line_bits.file_id = file.id) + SQL + def globs : Array(String) [ ".coverage", @@ -27,25 +30,114 @@ module CoverageReporter false end - def parse(filename : String, error : Process::Stdio = IO::Memory.new) : Array(FileReport) - tmpfile = File.tempfile("coverage.xml") - process_status = Process.run( - command: "coverage xml --data-file #{filename} -o #{tmpfile.path}", - shell: true, - error: error - ) - - if process_status.success? - parser = CoberturaParser.new(@base_path) - parser.parse(tmpfile.path) - else - raise ParserError.new(error.to_s) + def parse(filename : String) : Array(FileReport) + lines = {} of String => Array(Hits) + + DB.open "sqlite3://#{filename}" do |db| + db.query(QUERY) do |rs| + rs.each do + name = rs.read(String) + numbits = rs.read(Slice(UInt8)) + nums = [] of Hits + numbits.each_with_index do |byte, byte_i| + 8.times do |bit_i| + if byte & (1 << bit_i) != 0 + nums << (byte_i * 8 + bit_i).to_u64 + end + end + end + lines[name] = nums + end + end end - ensure - begin - tmpfile && tmpfile.delete - rescue File::Error + + lines.map do |name, hits| + coverage = get_coverage(name, hits) + + file_report( + name: name, + coverage: coverage, + ) end end + + private def get_coverage(name : String, hits : Array(Hits)) : Array(Hits?) + coverage = {} of Line => Hits? + + line_no = 1.to_u64 + under_def = false + docstring = false + brackets = 0 + + File.each_line(name, chomp: true) do |line| + code = line.strip + + if code.ends_with?(/\(|\{|\[/) + brackets += code.count("({[") + brackets -= code.count(")}]") + end + + if !docstring && code.starts_with?("\"\"\"") + if under_def || hits.find { |i| i == line_no } + docstring = true + coverage[line_no] = nil + next + end + end + + # docstring + if docstring + if code.ends_with?("\"\"\"") + docstring = false + end + + coverage[line_no] = nil + next + end + + # comment + if code.starts_with?("#") + coverage[line_no] = nil + next + end + + # a hit + if hits.find { |i| i == line_no } + coverage[line_no] = 1 + next + end + + # inside brackets + if brackets > 0 + coverage[line_no] = nil + next + end + + # empty string + if code.empty? + coverage[line_no] = nil + next + end + + coverage[line_no] = 0 + ensure + line_no += 1 + + if code + if brackets > 0 && code.ends_with?(/\)|\}|\]/) + brackets += code.count("({[") + brackets -= code.count(")}]") + end + + under_def = code.starts_with?("def ") || code.starts_with?("class ") + end + end + + coverage.keys.sort!.map { |k| coverage[k] } + rescue File::NotFoundError + Log.error("Couldn't open file #{name}") + + [] of Hits? + end end end