Skip to content

Commit

Permalink
Allow platform to be spoofed for coverage
Browse files Browse the repository at this point in the history
  • Loading branch information
rmartin16 committed May 5, 2023
1 parent c88943f commit ca8d858
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 119 deletions.
174 changes: 79 additions & 95 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ jobs:
runs-on: ${{ matrix.platform }}-latest
continue-on-error: ${{ matrix.experimental }}
strategy:
fail-fast: false
matrix:
platform: [ "macOS", "Ubuntu", "Windows" ]
python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12-dev" ]
Expand All @@ -48,7 +49,7 @@ jobs:
experimental: true
# Run tests against the latest Windows Store Python
- platform: "Windows"
python-version: "winstore"
python-version: "winstore3.11"
experimental: false
steps:
- name: Checkout
Expand All @@ -63,8 +64,10 @@ jobs:
python-version: ${{ matrix.python-version }}

- name: Install Windows Store Python
if: matrix.python-version == 'winstore'
if: startswith(matrix.python-version, 'winstore')
uses: beeware/.github/.github/actions/install-win-store-python@main
with:
python-version: "3.11"

- name: Get Packages
uses: actions/[email protected]
Expand All @@ -81,26 +84,36 @@ jobs:
python -m pip install $(ls dist/briefcase-*.whl)[dev]
- name: Test
id: test
env:
COVERAGE_FILE: ".coverage.${{ matrix.platform }}.${{ matrix.python-version }}"
run: tox -e py --installpkg dist/briefcase-*.whl

- name: Store Coverage Data
if: success() || failure()
if: always() && (steps.test.outcome == 'failure' || steps.test.outcome == 'success')
uses: actions/[email protected]
with:
name: coverage-data
path: ".coverage.*"
if-no-files-found: ignore

coverage-platform:
name: Platform coverage - ${{ matrix.platform }}
runs-on: ${{ matrix.platform }}-latest
- name: Report Platform Coverage
if: always() && (steps.test.outcome == 'failure' || steps.test.outcome == 'success')
id: coverage
run: tox -qe coverage$(echo '${{ matrix.python-version }}' | tr -dc '0-9')-html

- name: Upload HTML Coverage Report
if: always() && steps.coverage.outcome == 'failure'
uses: actions/[email protected]
with:
name: html-coverage-report-${{ matrix.platform }}-${{ matrix.python-version }}
path: htmlcov

coverage:
name: Project coverage
runs-on: ubuntu-latest
if: always() && contains(join(needs.*.result, ','), 'success')
needs: unit-tests
strategy:
fail-fast: false
matrix:
platform: [ "macOS", "Ubuntu", "Windows" ]
steps:
- name: Checkout
uses: actions/[email protected]
Expand All @@ -110,12 +123,9 @@ jobs:
- name: Setup Python
uses: actions/[email protected]
with:
python-version: |
3.12-dev
3.11
3.10
3.9
3.8
# Use minimum version of python for coverage to avoid phantom branches
# https://github.com/nedbat/coveragepy/issues/1572#issuecomment-1522546425
python-version: "3.8"

- name: Install dev Dependencies
run: |
Expand All @@ -130,104 +140,78 @@ jobs:
with:
name: coverage-data

- name: ${{ matrix.platform }} Coverage Report
env:
COVERAGE_FILE: ".coverage.${{ matrix.platform }}"
run: tox -qe coverage-platform-html-keep

- name: Python 3.12 on ${{ matrix.platform }} Coverage Report
- name: Linux Coverage Report
id: linux-coverage
if: success() || failure()
continue-on-error: true
env:
COVERAGE_FILE: ".coverage.${{ matrix.platform }}.3.12-dev"
run: tox -qe coverage312-html-keep
COVERAGE_FILE: ".coverage.Ubuntu"
COVERAGE_PLATFORM: "linux"
run: tox -qe coverage-platform-html

- name: Python 3.11 on ${{ matrix.platform }} Coverage Report
if: success() || failure()
env:
COVERAGE_FILE: ".coverage.${{ matrix.platform }}.3.11"
run: tox -qe coverage311-html-keep
- name: Upload Linux Coverage HTML Report
if: always() && steps.linux-coverage.outcome == 'failure'
uses: actions/[email protected]
with:
name: html-coverage-report-Linux
path: htmlcov

- name: Python 3.10 on ${{ matrix.platform }} Coverage Report
- name: macOS Coverage Report
id: macos-coverage
if: success() || failure()
env:
COVERAGE_FILE: ".coverage.${{ matrix.platform }}.3.10"
run: tox -qe coverage310-html-keep
COVERAGE_FILE: ".coverage.macOS"
COVERAGE_PLATFORM: "darwin"
run: tox -qe coverage-platform-html

- name: Python 3.9 on ${{ matrix.platform }} Coverage Report
if: success() || failure()
env:
COVERAGE_FILE: ".coverage.${{ matrix.platform }}.3.9"
run: tox -qe coverage39-html-keep
- name: Upload macOS Coverage HTML Report
if: always() && steps.macos-coverage.outcome == 'failure'
uses: actions/[email protected]
with:
name: html-coverage-report-macOS
path: htmlcov

- name: Python 3.8 on ${{ matrix.platform }} Coverage Report
- name: Windows Coverage Report
id: windows-coverage
if: success() || failure()
env:
COVERAGE_FILE: ".coverage.${{ matrix.platform }}.3.8"
run: tox -qe coverage38-html-keep
COVERAGE_FILE: ".coverage.Windows"
COVERAGE_PLATFORM: "win32"
run: tox -qe coverage-platform-html

- name: Upload HTML Coverage Report
if: failure()
- name: Upload Windows Coverage HTML Report
if: always() && steps.windows-coverage.outcome == 'failure'
uses: actions/[email protected]
with:
name: html-platform-coverage-report-${{ matrix.platform }}
name: html-coverage-report-Windows
path: htmlcov

coverage-project:
name: Project coverage
runs-on: ubuntu-latest
needs: unit-tests
steps:
- name: Checkout
uses: actions/[email protected]
with:
fetch-depth: 0

- name: Setup Python
uses: actions/[email protected]
with:
# Use minimum version of python for coverage to avoid phantom branches
# https://github.com/nedbat/coveragepy/issues/1572#issuecomment-1522546425
python-version: "3.8"

- name: Install dev Dependencies
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade setuptools
# We don't actually want to install briefcase; we just
# want the dev extras so we have a known version of tox.
python -m pip install -e .[dev]
- name: Retrieve Coverage Data
uses: actions/[email protected]
with:
name: coverage-data

- name: Project Coverage Report
id: project-coverage
if: success() || failure()
run: tox -qe coverage-project-html

- name: Upload HTML Coverage Report
if: failure()
- name: Upload Project Coverage HTML Report
if: always() && steps.project-coverage.outcome == 'failure'
uses: actions/[email protected]
with:
name: html-project-coverage-report
name: html-coverage-report-project
path: htmlcov

# verify-apps:
# name: Build app
# needs: unit-tests
# uses: beeware/.github/.github/workflows/app-build-verify.yml@main
# with:
# # This *must* be the version of Python that is the system Python on the
# # Ubuntu version used to run Linux tests. We use a fixed ubuntu-22.04
# # rather than `-latest` because at some point, `-latest` will become
# # `-24.04`, but it will be a soft changeover, which will cause havoc with
# # the hard Python version requirement for local system packages.
# python-version: "3.10"
# runner-os: ${{ matrix.runner-os }}
# framework: ${{ matrix.framework }}
# strategy:
# fail-fast: false
# matrix:
# framework: [ "toga", "pyside2", "pyside6", "ppb", "pygame" ]
# runner-os: [ "macos-latest", "ubuntu-22.04", "windows-latest" ]
verify-apps:
name: Build app
needs: unit-tests
uses: beeware/.github/.github/workflows/app-build-verify.yml@main
with:
# This *must* be the version of Python that is the system Python on the
# Ubuntu version used to run Linux tests. We use a fixed ubuntu-22.04
# rather than `-latest` because at some point, `-latest` will become
# `-24.04`, but it will be a soft changeover, which will cause havoc with
# the hard Python version requirement for local system packages.
python-version: "3.10"
runner-os: ${{ matrix.runner-os }}
framework: ${{ matrix.framework }}
strategy:
fail-fast: false
matrix:
framework: [ "toga", "pyside2", "pyside6", "ppb", "pygame" ]
runner-os: [ "macos-latest", "ubuntu-22.04", "windows-latest" ]
38 changes: 19 additions & 19 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,32 +31,32 @@ exclude_lines = [
# Packages/Modules
no-cover-if-missing-setuptools_scm = "not is_installed('setuptools_scm')"
# Linux
no-cover-if-is-linux = "sys_platform == 'linux' and os_environ.get('COV_EXCLUDE_PLATFORM') != 'disable'"
no-cover-if-not-linux = "sys_platform != 'linux' and os_environ.get('COV_EXCLUDE_PLATFORM') != 'disable'"
no-cover-if-is-linux = "'linux' == os_environ.get('COVERAGE_PLATFORM', sys_platform) and os_environ.get('COVERAGE_EXCLUDE_PLATFORM') != 'disable'"
no-cover-if-not-linux = "'linux' != os_environ.get('COVERAGE_PLATFORM', sys_platform) and os_environ.get('COVERAGE_EXCLUDE_PLATFORM') != 'disable'"
# macOS
no-cover-if-is-macos = "sys_platform == 'darwin' and os_environ.get('COV_EXCLUDE_PLATFORM') != 'disable'"
no-cover-if-not-macos = "sys_platform != 'darwin' and os_environ.get('COV_EXCLUDE_PLATFORM') != 'disable'"
no-cover-if-is-macos = "'darwin' == os_environ.get('COVERAGE_PLATFORM', sys_platform) and os_environ.get('COVERAGE_EXCLUDE_PLATFORM') != 'disable'"
no-cover-if-not-macos = "'darwin' != os_environ.get('COVERAGE_PLATFORM', sys_platform) and os_environ.get('COVERAGE_EXCLUDE_PLATFORM') != 'disable'"
# Windows
no-cover-if-is-windows = "sys_platform == 'win32' and os_environ.get('COV_EXCLUDE_PLATFORM') != 'disable'"
no-cover-if-not-windows = "sys_platform != 'win32' and os_environ.get('COV_EXCLUDE_PLATFORM') != 'disable'"
no-cover-if-is-windows = "'win32' == os_environ.get('COVERAGE_PLATFORM', sys_platform) and os_environ.get('COVERAGE_EXCLUDE_PLATFORM') != 'disable'"
no-cover-if-not-windows = "'win32' != os_environ.get('COVERAGE_PLATFORM', sys_platform) and os_environ.get('COVERAGE_EXCLUDE_PLATFORM') != 'disable'"
# Python 3.12
no-cover-if-is-py312 = "python_version == '3.12' and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-lt-py312 = "sys_version_info < (3, 12) and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-gte-py312 = "sys_version_info > (3, 12) and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-is-py312 = "python_version == '3.12' and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-lt-py312 = "sys_version_info < (3, 12) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-gte-py312 = "sys_version_info > (3, 12) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
# Python 3.11
no-cover-if-is-py311 = "python_version == '3.11' and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-lt-py311 = "sys_version_info < (3, 11) and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-gte-py311 = "sys_version_info > (3, 11) and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-is-py311 = "python_version == '3.11' and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-lt-py311 = "sys_version_info < (3, 11) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-gte-py311 = "sys_version_info > (3, 11) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
# Python 3.10
no-cover-if-is-py310 = "python_version == '3.10' and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-lt-py310 = "sys_version_info < (3, 10) and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-gte-py310 = "sys_version_info > (3, 10) and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-is-py310 = "python_version == '3.10' and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-lt-py310 = "sys_version_info < (3, 10) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-gte-py310 = "sys_version_info > (3, 10) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
# Python 3.9
no-cover-if-is-py39 = "python_version == '3.9' and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-lt-py39 = "sys_version_info < (3, 9) and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-gte-py39 = "sys_version_info > (3, 9) and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-is-py39 = "python_version == '3.9' and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-lt-py39 = "sys_version_info < (3, 9) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-gte-py39 = "sys_version_info > (3, 9) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
# Python 3.8
no-cover-if-is-py38 = "python_version == '3.8' and os_environ.get('COV_EXCLUDE_PYTHON_VERSION') != 'disable'"
no-cover-if-is-py38 = "python_version == '3.8' and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"

[tool.isort]
profile = "black"
Expand Down
14 changes: 9 additions & 5 deletions tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -48,20 +48,24 @@ parallel_show_output = True
extras = dev
# 2023-04-22 see pkgenv ↑
download = {[pkgenv]download}
passenv =
COVERAGE_FILE
COVERAGE_PLATFORM
setenv =
COVERAGE_FILE = {env:COVERAGE_FILE:.coverage}
keep: COMBINE_FLAGS = --keep
!nofail: REPORT_FLAGS = --fail-under=100
# disable coverage exclusions for Python version to test entire platform
{platform,project}: COV_EXCLUDE_PYTHON_VERSION=disable
{platform,project}: COVERAGE_EXCLUDE_PYTHON_VERSION=disable
# disable coverage exclusions for host platform to test entire project
project: COV_EXCLUDE_PLATFORM=disable
project: COVERAGE_EXCLUDE_PLATFORM=disable
commands_pre =
python --version
python -c 'if 1: \
import os; \
print("COV_EXCLUDE_PYTHON_VERSION", os.environ.get("COV_EXCLUDE_PYTHON_VERSION", "<not set>")); \
print("COV_EXCLUDE_PLATFORM", os.environ.get("COV_EXCLUDE_PLATFORM", "<not set>"))'
print("COVERAGE_FILE", os.environ.get("COVERAGE_FILE", "<not set>")); \
print("COVERAGE_PLATFORM", os.environ.get("COVERAGE_PLATFORM", "<not set>")); \
print("COVERAGE_EXCLUDE_PYTHON_VERSION", os.environ.get("COVERAGE_EXCLUDE_PYTHON_VERSION", "<not set>")); \
print("COVERAGE_EXCLUDE_PLATFORM", os.environ.get("COVERAGE_EXCLUDE_PLATFORM", "<not set>"))'
commands =
-python -m coverage combine {env:COMBINE_FLAGS}
html: python -m coverage html --skip-covered --skip-empty
Expand Down

0 comments on commit ca8d858

Please sign in to comment.