Control Unit Testing verbosity with a new test_verbosity_level config…

…uration option // Resolve #4276

HISTORY.rst

@@ -4,6 +4,7 @@ Release Notes
 .. |PIOCONF| replace:: `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ configuration file
 .. |LDF| replace:: `LDF <https://docs.platformio.org/en/latest/librarymanager/ldf.html>`__
 .. |INTERPOLATION| replace:: `Interpolation of Values <https://docs.platformio.org/en/latest/projectconf/interpolation.html>`__
+.. |UNITTESTING| replace:: `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html>`__
 
 .. _release_notes_6:
 
@@ -15,7 +16,8 @@ PlatformIO Core 6
 6.0.2 (2022-??-??)
 ~~~~~~~~~~~~~~~~~~
 
-* Fixed an issue when the `build_src_flags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-flags>`__ were applied outside the project scope (`issue #4277 <https://github.com/platformio/platformio-core/issues/4277>`_)
+* Control |UNITTESTING| verbosity with a new `test_verbosity_level <https://docs.platformio.org/en/latest/projectconf/section_env_test.html#test_verbosity_level>`__ configuration option (`issue #4276 <https://github.com/platformio/platformio-core/issues/4276>`_)
+* Fixed an issue when the `build_src_flags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-flags>`__ option was applied outside the project scope (`issue #4277 <https://github.com/platformio/platformio-core/issues/4277>`_)
 
 6.0.1 (2022-05-17)
 ~~~~~~~~~~~~~~~~~~
@@ -58,7 +60,7 @@ Please check the `Migration guide from 5.x to 6.0 <https://docs.platformio.org/e
 
 * **Unit Testing**
 
-  - Refactored from scratch `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html>`_ solution and its documentation
+  - Refactored from scratch |UNITTESTING| solution and its documentation
   - New: `Test Hierarchy <https://docs.platformio.org/en/latest/advanced/unit-testing/structure.html>`_ (`issue #4135 <https://github.com/platformio/platformio-core/issues/4135>`_)
   - New: `Doctest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/doctest.html>`__ testing framework (`issue #4240 <https://github.com/platformio/platformio-core/issues/4240>`_)
   - New: `GoogleTest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/googletest.html>`__ testing and mocking framework (`issue #3572 <https://github.com/platformio/platformio-core/issues/3572>`_)

platformio/project/options.py

@@ -675,7 +675,8 @@ def get_default_core_dir():
             ConfigEnvOption(
                 group="test",
                 name="test_speed",
-                description="A connection speed (baud rate) to communicate with a target device",
+                description="A connection speed (baud rate) to communicate with "
+                "a target device",
                 type=click.INT,
                 default=115200,
             ),
@@ -696,6 +697,19 @@ def get_default_core_dir():
                     "and returns results to the standard output"
                 ),
             ),
+            ConfigEnvOption(
+                group="test",
+                name="test_verbosity_level",
+                description=(
+                    "Verbosity level: "
+                    "0=normal verbosity (default), "
+                    "1=raw testing output, "
+                    "2=base verbosity for buidling/uploading, "
+                    "3=extra verbosity for building/uploading"
+                ),
+                type=click.IntRange(min=0, max=3),
+                default=0,
+            ),
             # Debug
             ConfigEnvOption(
                 group="debug",

platformio/test/command.py

@@ -85,7 +85,12 @@
 @click.option("--list-tests", is_flag=True)
 @click.option("--json-output-path", type=click.Path(resolve_path=True))
 @click.option("--junit-output-path", type=click.Path(resolve_path=True))
-@click.option("--verbose", "-v", is_flag=True)
+@click.option(
+    "--verbose",
+    "-v",
+    count=True,
+    help="Increase verbosity level, maximum is 3 levels (-vvv), see docs for details",
+)
 @click.pass_context
 def test_cmd(  # pylint: disable=too-many-arguments,too-many-locals,redefined-builtin
     ctx,
@@ -121,7 +126,7 @@ def test_cmd(  # pylint: disable=too-many-arguments,too-many-locals,redefined-bu
         test_names = sorted(set(s.test_name for s in test_suites))
 
         if not verbose:
-            click.echo("Verbose mode can be enabled via `-v, --verbose` option")
+            click.echo("Verbosity level can be increased via `-v, --verbose` option")
         click.secho("Collected %d tests" % len(test_names), bold=True, nl=not verbose)
         if verbose:
             click.echo(" (%s)" % ", ".join(test_names))
@@ -134,7 +139,10 @@ def test_cmd(  # pylint: disable=too-many-arguments,too-many-locals,redefined-bu
                 test_suite,
                 project_config,
                 TestRunnerOptions(
-                    verbose=verbose,
+                    verbose=verbose
+                    or project_config.get(
+                        f"env:{test_suite.env_name}", "test_verbosity_level"
+                    ),
                     without_building=without_building,
                     without_uploading=without_uploading,
                     without_testing=without_testing,

platformio/test/runners/base.py

@@ -28,7 +28,7 @@
 class TestRunnerOptions:  # pylint: disable=too-many-instance-attributes
     def __init__(  # pylint: disable=too-many-arguments
         self,
-        verbose=False,
+        verbose=0,
         without_building=False,
         without_uploading=False,
         without_testing=False,
@@ -96,6 +96,8 @@ def start(self, cmd_ctx):
             self.setup()
             for stage in ("building", "uploading", "testing"):
                 getattr(self, f"stage_{stage}")()
+                if self.options.verbose:
+                    click.echo()
         except Exception as exc:  # pylint: disable=broad-except
             click.secho(str(exc), fg="red", err=True)
             self.test_suite.add_case(
@@ -126,7 +128,7 @@ def stage_building(self):
         except ReturnErrorCode:
             raise UnitTestSuiteError(
                 "Building stage has failed, see errors above. "
-                "Use `pio test --verbose` option to enable verbose output."
+                "Use `pio test -vvv` option to enable verbose output."
             )
 
     def stage_uploading(self):
@@ -145,7 +147,7 @@ def stage_uploading(self):
         except ReturnErrorCode:
             raise UnitTestSuiteError(
                 "Uploading stage has failed, see errors above. "
-                "Use `pio test --verbose` option to enable verbose output."
+                "Use `pio test -vvv` option to enable verbose output."
             )
 
     def stage_testing(self):
@@ -179,8 +181,8 @@ def run_project_targets(self, targets):
             run_cmd,
             project_conf=self.project_config.path,
             upload_port=self.options.upload_port,
-            verbose=self.options.verbose,
-            silent=not self.options.verbose,
+            verbose=self.options.verbose > 2,
+            silent=self.options.verbose < 2,
             environment=[self.test_suite.env_name],
             disable_auto_clean="nobuild" in targets,
             target=targets,

platformio/test/runners/doctest.py

@@ -119,7 +119,7 @@ def on_testing_line_output(self, line):
             click.echo(line, nl=False)
 
         test_case = self._tc_parser.parse(line)
-        if test_case:
+        if test_case and not self.options.verbose:
             click.echo(test_case.humanize())
             self.test_suite.add_case(test_case)
 

platformio/test/runners/googletest.py

@@ -110,7 +110,7 @@ def on_testing_line_output(self, line):
             click.echo(line, nl=False)
 
         test_case = self._tc_parser.parse(line)
-        if test_case:
+        if test_case and not self.options.verbose:
             click.echo(test_case.humanize())
             self.test_suite.add_case(test_case)
 

platformio/test/runners/unity.py

@@ -264,7 +264,7 @@ def on_testing_line_output(self, line):
             return
 
         test_case = self.parse_test_case(line)
-        if test_case:
+        if test_case and not self.options.verbose:
             click.echo(test_case.humanize())
 
         if all(s in line for s in ("Tests", "Failures", "Ignored")):

tests/commands/test_test.py

@@ -94,6 +94,7 @@ def test_group_and_custom_runner(clirunner, validate_cliresult, tmp_path: Path):
 [env:native]
 platform = native
 test_framework = custom
+test_verbosity_level = 1
 """
     )
     test_dir = project_dir / "test"
@@ -187,6 +188,7 @@ def teardown(self):
         ["-d", str(project_dir), "-e", "native", "--verbose"],
     )
     validate_cliresult(result)
+    assert "1 Tests 0 Failures 0 Ignored" in result.output
     assert "Called from my_extra_fun" in result.output
     assert "CustomTestRunner::TearDown called" in result.output
     assert "Disabled test suite" not in result.output