Merge pull request #4958 from bonzini/tap

Add initial TAP test support

docs/markdown/Reference-manual.md

@@ -1406,10 +1406,7 @@ executable to run.  The executable can be an [executable build target
 object](#build-target-object) returned by
 [`executable()`](#executable) or an [external program
 object](#external-program-object) returned by
-[`find_program()`](#find_program). The executable's exit code is used
-by the test harness to record the outcome of the test, for example
-exit code zero indicates success. For more on the Meson test harness
-protocol read [Unit Tests](Unit-tests.md).
+[`find_program()`](#find_program).
 
 Keyword arguments are the following:
 
@@ -1446,6 +1443,12 @@ Keyword arguments are the following:
   before test is executed even if they have `build_by_default : false`.
   Since 0.46.0
 
+- `protocol` specifies how the test results are parsed and can be one
+  of `exitcode` (the executable's exit code is used by the test harness
+  to record the outcome of the test) or `tap` ([Test Anything
+  Protocol](https://www.testanything.org/)). For more on the Meson test
+  harness protocol read [Unit Tests](Unit-tests.md). Since 0.50.0
+
 Defined tests can be run in a backend-agnostic way by calling
 `meson test` inside the build dir, or by using backend-specific
 commands, such as `ninja test` or `msbuild RUN_TESTS.vcxproj`.

docs/markdown/Unit-tests.md

@@ -51,9 +51,15 @@ By default Meson uses as many concurrent processes as there are cores on the tes
 $ MESON_TESTTHREADS=5 ninja test
 ```
 
-## Skipped tests
+## Skipped tests and hard errors
 
-Sometimes a test can only determine at runtime that it can not be run. The GNU standard approach in this case is to exit the program with error code 77. Meson will detect this and report these tests as skipped rather than failed. This behavior was added in version 0.37.0.
+Sometimes a test can only determine at runtime that it can not be run.
+
+For the default `exitcode` testing protocol, the GNU standard approach in this case is to exit the program with error code 77. Meson will detect this and report these tests as skipped rather than failed. This behavior was added in version 0.37.0.
+
+For TAP-based tests, skipped tests should print a single line starting with `1..0 # SKIP`.
+
+In addition, sometimes a test fails set up so that it should fail even if it is marked as an expected failure. The GNU standard approach in this case is to exit the program with error code 99. Again, Meson will detect this and report these tests as `ERROR`, ignoring the setting of `should_fail`. This behavior was added in version 0.50.0.
 
 ## Testing tool
 

mesonbuild/backend/backends.py

@@ -84,7 +84,7 @@ def __init__(self, name, fname, cmd_args, env, is_cross, exe_wrapper,
 
 class TestSerialisation:
     def __init__(self, name, project, suite, fname, is_cross_built, exe_wrapper, is_parallel,
-                 cmd_args, env, should_fail, timeout, workdir, extra_paths):
+                 cmd_args, env, should_fail, timeout, workdir, extra_paths, protocol):
         self.name = name
         self.project_name = project
         self.suite = suite
@@ -100,6 +100,7 @@ def __init__(self, name, project, suite, fname, is_cross_built, exe_wrapper, is_
         self.timeout = timeout
         self.workdir = workdir
         self.extra_paths = extra_paths
+        self.protocol = protocol
 
 class OptionProxy:
     def __init__(self, name, value):
@@ -756,7 +757,7 @@ def create_test_serialisation(self, tests):
                     raise MesonException('Bad object in test command.')
             ts = TestSerialisation(t.get_name(), t.project_name, t.suite, cmd, is_cross,
                                    exe_wrapper, t.is_parallel, cmd_args, t.env,
-                                   t.should_fail, t.timeout, t.workdir, extra_paths)
+                                   t.should_fail, t.timeout, t.workdir, extra_paths, t.protocol)
             arr.append(ts)
         return arr
 

mesonbuild/interpreter.py

@@ -850,7 +850,7 @@ def __repr__(self):
 
 class Test(InterpreterObject):
     def __init__(self, name, project, suite, exe, depends, is_parallel,
-                 cmd_args, env, should_fail, timeout, workdir):
+                 cmd_args, env, should_fail, timeout, workdir, protocol):
         InterpreterObject.__init__(self)
         self.name = name
         self.suite = suite
@@ -863,6 +863,7 @@ def __init__(self, name, project, suite, exe, depends, is_parallel,
         self.should_fail = should_fail
         self.timeout = timeout
         self.workdir = workdir
+        self.protocol = protocol
 
     def get_exe(self):
         return self.exe
@@ -1973,7 +1974,8 @@ def get_cross_property_method(self, args, kwargs):
                     'library': known_library_kwargs,
                     'subdir': {'if_found'},
                     'subproject': {'version', 'default_options', 'required'},
-                    'test': {'args', 'depends', 'env', 'is_parallel', 'should_fail', 'timeout', 'workdir', 'suite'},
+                    'test': {'args', 'depends', 'env', 'is_parallel', 'should_fail', 'timeout', 'workdir',
+                             'suite', 'protocol'},
                     'vcs_tag': {'input', 'output', 'fallback', 'command', 'replace_string'},
                     }
 
@@ -3269,6 +3271,9 @@ def add_test(self, node, args, kwargs, is_base_test):
             workdir = None
         if not isinstance(timeout, int):
             raise InterpreterException('Timeout must be an integer.')
+        protocol = kwargs.get('protocol', 'exitcode')
+        if protocol not in ('exitcode', 'tap'):
+            raise InterpreterException('Protocol must be "exitcode" or "tap".')
         suite = []
         prj = self.subproject if self.is_subproject() else self.build.project_name
         for s in mesonlib.stringlistify(kwargs.get('suite', '')):
@@ -3280,7 +3285,7 @@ def add_test(self, node, args, kwargs, is_base_test):
             if not isinstance(dep, (build.CustomTarget, build.BuildTarget)):
                 raise InterpreterException('Depends items must be build targets.')
         t = Test(args[0], prj, suite, exe.held_object, depends, par, cmd_args,
-                 env, should_fail, timeout, workdir)
+                 env, should_fail, timeout, workdir, protocol)
         if is_base_test:
             self.build.tests.append(t)
             mlog.debug('Adding test', mlog.bold(args[0], True))