diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 57f0d8b..67a1a25 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update - sudo apt-get install -y gdb + sudo apt-get install -y gdb lcov - name: Tests PyTest run: | pipx run nox --forcecolor -s test diff --git a/README.md b/README.md index 2c39126..f1df69a 100644 --- a/README.md +++ b/README.md @@ -247,7 +247,7 @@ fc-cache -f -v `spin` tests are invoked using: ``` -nox -s tests +nox -s test ``` ## History diff --git a/noxfile.py b/noxfile.py index ea40094..3b36a13 100644 --- a/noxfile.py +++ b/noxfile.py @@ -3,5 +3,5 @@ @nox.session def test(session: nox.Session) -> None: - session.install(".", "pytest", "build", "meson-python", "ninja") + session.install(".", "pytest", "build", "meson-python", "ninja", "gcovr") session.run("pytest", "spin", *session.posargs) diff --git a/spin/cmds/meson.py b/spin/cmds/meson.py index 6c6e62c..3eb0c82 100644 --- a/spin/cmds/meson.py +++ b/spin/cmds/meson.py @@ -2,8 +2,11 @@ import copy import json import os +import re import shutil import sys +from enum import Enum +from pathlib import Path import click @@ -11,6 +14,14 @@ from .util import run as _run install_dir = "build-install" +build_dir = "build" + + +class GcovReportFormat(str, Enum): + html = "html" + xml = "xml" + text = "text" + sonarqube = "sonarqube" # Allow specification of meson binary in configuration @@ -124,6 +135,54 @@ def _meson_version_configured(): pass +def _meson_coverage_configured() -> bool: + try: + build_options_fn = os.path.join( + "build", "meson-info", "intro-buildoptions.json" + ) + with open(build_options_fn) as f: + build_options = json.load(f) + for b in build_options: + if (b["name"] == "b_coverage") and (b["value"] is True): + return True + except: + pass + + return False + + +def _check_coverage_tool_installation(coverage_type: GcovReportFormat): + requirements = { # https://github.com/mesonbuild/meson/blob/6e381714c7cb15877e2bcaa304b93c212252ada3/docs/markdown/Unit-tests.md?plain=1#L49-L62 + GcovReportFormat.html: ["Gcovr/GenHTML", "lcov"], + GcovReportFormat.xml: ["Gcovr (version 3.3 or higher)"], + GcovReportFormat.text: ["Gcovr (version 3.3 or higher)"], + GcovReportFormat.sonarqube: ["Gcovr (version 4.2 or higher)"], + } + + # First check the presence of a valid build + if not (os.path.exists(build_dir)): + raise click.ClickException( + "`build` folder not found, cannot generate coverage reports. " + "Generate coverage artefacts by running `spin test --gcov`" + ) + + debug_files = Path(build_dir).rglob("*.gcno") + if len(list(debug_files)) == 0: + raise click.ClickException( + "Debug build not found, cannot generate coverage reports.\n\n" + "Please rebuild using `spin build --clean --gcov` first." + ) + + # Verify the tools are installed prior to the build + p = _run(["ninja", "-C", build_dir, "-t", "targets", "all"], output=False) + if f"coverage-{coverage_type.value}" not in p.stdout.decode("ascii"): + raise click.ClickException( + f"coverage-{coverage_type.value} is not supported... " + f"Ensure the following are installed: {', '.join(requirements[coverage_type])} " + "and rerun `spin test --gcov`" + ) + + @click.command() @click.option("-j", "--jobs", help="Number of parallel tasks to launch", type=int) @click.option("--clean", is_flag=True, help="Clean build directory before build") @@ -133,18 +192,7 @@ def _meson_version_configured(): @click.option( "--gcov", is_flag=True, - help="""Enable C code coverage via `gcov`. - - The meson-generated `build/build.ninja` has targets for compiling - coverage reports. - - E.g., to build an HTML report, in the `build` directory run - `ninja coverage-html`. - - To see a list all supported formats, run - `ninja -t targets | grep coverage-`. - - Also see https://mesonbuild.com/howtox.html#producing-a-coverage-report.""", + help="Enable C code coverage using `gcov`. Use `spin test --gcov` to generate reports.", ) @click.argument("meson_args", nargs=-1) def build(meson_args, jobs=None, clean=False, verbose=False, gcov=False, quiet=False): @@ -165,7 +213,6 @@ def build(meson_args, jobs=None, clean=False, verbose=False, gcov=False, quiet=F CFLAGS="-O0 -g" spin build """ - build_dir = "build" meson_args = list(meson_args) if gcov: @@ -191,7 +238,9 @@ def build(meson_args, jobs=None, clean=False, verbose=False, gcov=False, quiet=F # Build dir has been configured; check if it was configured by # current version of Meson - if _meson_version() != _meson_version_configured(): + if (_meson_version() != _meson_version_configured()) or ( + gcov and not _meson_coverage_configured() + ): _run(setup_cmd + ["--reconfigure"], output=not quiet) # Any other conditions that warrant a reconfigure? @@ -255,17 +304,30 @@ def _get_configured_command(command_name): "-c", "--coverage", is_flag=True, - help="Generate a coverage report of executed tests. An HTML copy of the report is written to `build/coverage`.", + help="Generate a Python coverage report of executed tests. An HTML copy of the report is written to `build/coverage`.", ) @click.option( "--gcov", is_flag=True, - help="Enable C code coverage via `gcov`. `gcov` output goes to `build/**/*.gc*`. " - "Reports can be generated using `ninja coverage*` commands. " - "See https://mesonbuild.com/howtox.html#producing-a-coverage-report", + help="Generate a C coverage report in `build/meson-logs/coveragereport`.", +) +@click.option( + "--gcov-format", + type=click.Choice(GcovReportFormat), + default="html", + help=f"Format of the gcov report. Can be one of {', '.join(e.value for e in GcovReportFormat)}.", ) @click.pass_context -def test(ctx, pytest_args, n_jobs, tests, verbose, coverage=False, gcov=False): +def test( + ctx, + pytest_args, + n_jobs, + tests, + verbose, + coverage=False, + gcov=None, + gcov_format=None, +): """🔧 Run tests PYTEST_ARGS are passed through directly to pytest, e.g.: @@ -315,7 +377,7 @@ def test(ctx, pytest_args, n_jobs, tests, verbose, coverage=False, gcov=False): click.secho( "Invoking `build` prior to running tests:", bold=True, fg="bright_green" ) - ctx.invoke(build_cmd, gcov=gcov) + ctx.invoke(build_cmd, gcov=bool(gcov)) package = cfg.get("tool.spin.package", None) if (not pytest_args) and (not tests): @@ -372,10 +434,44 @@ def test(ctx, pytest_args, n_jobs, tests, verbose, coverage=False, gcov=False): cmd = [sys.executable, "-P", "-m", "pytest"] else: cmd = ["pytest"] - _run( - cmd + list(pytest_args), - replace=True, - ) + p = _run(cmd + list(pytest_args)) + + if gcov: + # Verify the tools are present + click.secho( + "Verifying gcov dependencies...", + bold=True, + fg="bright_yellow", + ) + _check_coverage_tool_installation(gcov_format) + + # Generate report + click.secho( + f"Generating {gcov_format.value} coverage report...", + bold=True, + fg="bright_yellow", + ) + p = _run( + [ + "ninja", + "-C", + build_dir, + f"coverage-{gcov_format.value.lower()}", + ], + output=False, + ) + coverage_folder = click.style( + re.search(r"file://(.*)", p.stdout.decode("utf-8")).group(1), + bold=True, + fg="bright_yellow", + ) + click.secho( + f"Coverage report generated successfully and written to {coverage_folder}", + bold=True, + fg="bright_green", + ) + + raise SystemExit(p.returncode) @click.command() diff --git a/spin/tests/test_build_cmds.py b/spin/tests/test_build_cmds.py index dae17e5..d55ecdb 100644 --- a/spin/tests/test_build_cmds.py +++ b/spin/tests/test_build_cmds.py @@ -4,6 +4,7 @@ import tempfile from pathlib import Path +import pytest from testutil import skip_on_windows, skip_unless_linux, spin, stdout from spin.cmds.util import run @@ -27,6 +28,33 @@ def test_debug_builds(): assert len(list(debug_files)) != 0, "debug files not generated for gcov build" +def test_coverage_builds(): + """Does gcov test generate coverage files?""" + spin("test", "--gcov") + + coverage_files = Path(".").rglob("*.gcda") + assert len(list(coverage_files)) != 0, "coverage files not generated for gcov build" + + +@pytest.mark.parametrize( + "report_type,output_file", + [ + ("html", Path("coveragereport/index.html")), + ("xml", Path("coverage.xml")), + ("text", Path("coverage.txt")), + ("sonarqube", Path("sonarqube.xml")), + ], +) +def test_coverage_reports(report_type, output_file): + """Does gcov test generate coverage reports?""" + spin("test", "--gcov", f"--gcov-format={report_type}") + + coverage_report = Path("./build/meson-logs", output_file) + assert ( + coverage_report.exists() + ), f"coverage report not generated for gcov build ({report_type})" + + def test_expand_pythonpath(): """Does an $ENV_VAR get expanded in `spin run`?""" output = spin("run", "echo $PYTHONPATH")