From 8481e438bd3ec2a50a6818b2a660642dc411caaf Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Fri, 1 Mar 2019 14:55:09 +0100 Subject: [PATCH] ci: revisit coverage reporting This brings coverage back that got missing with 9dcd6f2. Continuation of #4839 / #4846. --- .travis.yml | 91 ++++++++++++++++++++++------------- azure-pipelines.yml | 42 ++++++++++------ testing/test_assertrewrite.py | 3 +- 3 files changed, 84 insertions(+), 52 deletions(-) diff --git a/.travis.yml b/.travis.yml index 58c7e67519d..03b502c806f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,45 +10,23 @@ python: - '3.7' install: - python -m pip install --upgrade --pre tox -env: - matrix: - - TOXENV=py27 PYTEST_COVERAGE=1 - # Specialized factors for py27. - - TOXENV=py27-nobyte-numpy-xdist PYTEST_COVERAGE=1 - - TOXENV=py27-pluggymaster-xdist - # Specialized factors for py37. - - TOXENV=py37-pexpect,py37-trial - - TOXENV=py37-pluggymaster-xdist - - TOXENV=py37-freeze - -matrix: - allow_failures: - - python: '3.8-dev' - env: TOXENV=py38-xdist jobs: include: - # Coverage tracking is slow with pypy, skip it. - - env: TOXENV=pypy-xdist - python: 'pypy2.7-6.0' - - env: TOXENV=pypy3-xdist - python: 'pypy3.5-6.0' - - - env: TOXENV=py34-xdist - python: '3.4' - - env: TOXENV=py35-xdist - python: '3.5' - - env: TOXENV=py36-xdist - python: '3.6' - - env: TOXENV=py37 + # OSX tests - first (in test stage), since they are the slower ones. - &test-macos + # NOTE: (tests with) pexpect appear to be buggy on Travis, + # at least with coverage. + # Log: https://travis-ci.org/pytest-dev/pytest/jobs/500358864 os: osx osx_image: xcode10.1 language: generic + # Coverage for: + # - py2 with symlink in test_cmdline_python_package_symlink. + env: TOXENV=py27-xdist PYTEST_COVERAGE=1 before_install: - python -V - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 27 - env: TOXENV=py27-xdist - <<: *test-macos env: TOXENV=py37-xdist before_install: @@ -58,16 +36,56 @@ jobs: - python -V - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 37 + # Full run of latest (major) supported versions, without xdist. + - env: TOXENV=py27 + python: '2.7' + - env: TOXENV=py37 + python: '3.7' + + # Coverage tracking is slow with pypy, skip it. + - env: TOXENV=pypy-xdist + python: 'pypy2.7-6.0' + - env: TOXENV=pypy3-xdist + python: 'pypy3.5-6.0' + + - env: TOXENV=py34-xdist + python: '3.4' + - env: TOXENV=py35-xdist + python: '3.5' + + # Coverage for: + # - pytester's LsofFdLeakChecker + # - TestArgComplete (linux only) + # - numpy + - env: TOXENV=py37-lsof-numpy-xdist PYTEST_COVERAGE=1 + + # Specialized factors for py27. + - env: TOXENV=py27-nobyte-numpy-xdist + python: '2.7' + - env: TOXENV=py27-pluggymaster-xdist + python: '2.7' + + # Specialized factors for py37. + # Coverage for: + # - test_sys_breakpoint_interception (via pexpect). + - env: TOXENV=py37-pexpect,py37-trial PYTEST_COVERAGE=1 + - env: TOXENV=py37-pluggymaster-xdist + - env: TOXENV=py37-freeze + # Jobs only run via Travis cron jobs (currently daily). - env: TOXENV=py38-xdist python: '3.8-dev' if: type = cron - stage: baseline - env: TOXENV=py27-pexpect,py27-trial - - env: TOXENV=py37-lsof-numpy-xdist PYTEST_COVERAGE=1 - - env: TOXENV=linting,docs,doctesting - python: '3.7' + # Coverage for: + # - _pytest.unittest._handle_skip (via pexpect). + env: TOXENV=py27-pexpect,py27-trial PYTEST_COVERAGE=1 + python: '2.7' + # Use py36 here for faster baseline. + - env: TOXENV=py36-xdist + python: '3.6' + - env: TOXENV=linting,docs,doctesting PYTEST_COVERAGE=1 - stage: deploy python: '3.6' @@ -84,6 +102,11 @@ jobs: tags: true repo: pytest-dev/pytest +matrix: + allow_failures: + - python: '3.8-dev' + env: TOXENV=py38-xdist + before_script: - | # Do not (re-)upload coverage with cron runs. @@ -109,7 +132,7 @@ after_success: coverage combine coverage xml coverage report -m - bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -n $TOXENV + bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -n $TOXENV-$TRAVIS_OS_NAME fi notifications: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bec7a6d7420..712106c94e2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,19 +20,28 @@ jobs: py27: python.version: '2.7' tox.env: 'py27' - PYTEST_COVERAGE: '1' - py27-nobyte-numpy-xdist: + py27-nobyte-lsof-numpy: python.version: '2.7' - tox.env: 'py27-nobyte-numpy-xdist' + tox.env: 'py27-lsof-nobyte-numpy' + # Coverage for: + # - test_supports_breakpoint_module_global + # - test_terminal_reporter_writer_attr (without xdist) + # - "if write" branch in _pytest.assertion.rewrite + # - numpy + # - pytester's LsofFdLeakChecker (being skipped) PYTEST_COVERAGE: '1' py27-trial: python.version: '2.7' tox.env: 'py27-trial' python.needs_vc: True - PYTEST_COVERAGE: '1' py27-pluggymaster-xdist: python.version: '2.7' tox.env: 'py27-pluggymaster-xdist' + # Coverage for: + # - except-IOError in _attempt_to_close_capture_file for py2. + # Also seen with py27-nobyte (using xdist), and py27-xdist. + # But no exception with py27-pexpect,py27-trial,py27-numpy. + PYTEST_COVERAGE: '1' pypy: python.version: 'pypy' tox.env: 'pypy' @@ -42,33 +51,34 @@ jobs: # python.version: 'pypy3' # tox.env: 'pypy3' # python.exe: 'pypy3' - py34: + py34-xdist: python.version: '3.4' - tox.env: 'py34' + tox.env: 'py34-xdist' + # Coverage for: + # - _pytest.compat._bytes_to_ascii PYTEST_COVERAGE: '1' - py35: + py35-xdist: python.version: '3.5' - tox.env: 'py35' + tox.env: 'py35-xdist' + # Coverage for: + # - test_supports_breakpoint_module_global PYTEST_COVERAGE: '1' - py36: + py36-xdist: python.version: '3.6' - tox.env: 'py36' - PYTEST_COVERAGE: '1' + tox.env: 'py36-xdist' py37: python.version: '3.7' tox.env: 'py37' + # Coverage for: + # - _py36_windowsconsoleio_workaround (with py36+) + # - test_request_garbage (no xdist) PYTEST_COVERAGE: '1' py37-linting/docs/doctesting: python.version: '3.7' tox.env: 'linting,docs,doctesting' - py37-xdist: - python.version: '3.7' - tox.env: 'py37-xdist' - PYTEST_COVERAGE: '1' py37-trial/numpy: python.version: '3.7' tox.env: 'py37-trial,py37-numpy' - PYTEST_COVERAGE: '1' py37-pluggymaster-xdist: python.version: '3.7' tox.env: 'py37-pluggymaster-xdist' diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py index bfb6acfabf5..098e106f93f 100644 --- a/testing/test_assertrewrite.py +++ b/testing/test_assertrewrite.py @@ -1313,8 +1313,7 @@ def test_cwd_changed(self, testdir, monkeypatch): # always (previously triggered via xdist only). # Ref: https://github.com/pytest-dev/py/pull/207 monkeypatch.setattr(sys, "path", [""] + sys.path) - if "pathlib" in sys.modules: - del sys.modules["pathlib"] + monkeypatch.delitem(sys.modules, "pathlib", raising=False) testdir.makepyfile( **{