Allow 'skip_*:' test.json keys at top-level or in matrix:

data/test.schema.json

@@ -127,6 +127,18 @@
         ]
       }
     },
+    "skip_on_env": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "skip_on_os": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
     "skip_on_jobname": {
       "type": "array",
       "items": {

docs/markdown/Contributing.md

@@ -305,20 +305,15 @@ project tests with different Meson options.
 
 In the `options` dict, all possible options and their values are
 specified. Each key in the `options` dict is a Meson option. It stores
-a list of all potential values in a dict format, which allows to skip
-specific values based on the current environment.
+a list of all potential values in a dict format.
 
 Each value must contain the `val` key for the value of the option.
 `null` can be used for adding matrix entries without the current
 option.
 
-The `skip_on_env` key can be used to specify a list of environment
-variables. If at least one environment variable in `skip_on_env`
-is present, the matrix entry containing this key is skipped.
-
-The `skip_on_os` key can be used to specify a list of OS names (or their
-negations, prefixed with a `!`).  If at least one item in the `skip_on_os` list
-is matched, the matrix entry containing this key is skipped.
+The `skip_on_env`, `skip_on_os` and `skip_on_jobname` keys (as described below)
+may be used in the value to skip that matrix entry, based on the current
+environment.
 
 Similarly, the `compilers` key can be used to define a mapping of
 compilers to languages that are required for this value.
@@ -383,6 +378,18 @@ matched:
 | `literal` | Literal match (default) |
 | `re`      | regex match             |
 
+#### skip_on_env
+
+The `skip_on_env` key can be used to specify a list of environment variables. If
+at least one environment variable in the `skip_on_env` list is present, the test
+is skipped.
+
+#### skip_on_os
+
+The `skip_on_os` key can be used to specify a list of OS names (or their
+negations, prefixed with a `!`).  If at least one item in the `skip_on_os` list
+is matched, the test is skipped.
+
 #### skip_on_jobname
 
 The `skip_on_jobname` key contains a list of strings. If the `MESON_CI_JOBNAME`

run_project_tests.py

@@ -725,6 +725,37 @@ def force_regenerate() -> None:
     return testresult
 
 
+# processing of test.json 'skip_*' keys, which can appear at top level, or in
+# matrix:
+def _skip_keys(test_def: T.Dict, mesonenv: environment.Environment) -> T.Tuple[bool, bool]:
+    # Test is expected to skip if MESON_CI_JOBNAME contains any of the list of
+    # substrings
+    if ('skip_on_jobname' in test_def) and (ci_jobname is not None):
+        skip_expected = any(s in ci_jobname for s in test_def['skip_on_jobname'])
+        print("skip_on_jobname: looking for %s in %s gave %s" % (test_def['skip_on_jobname'], ci_jobname, skip_expected))
+    else:
+        skip_expected = False
+
+    # Skip if environment variable is present
+    skip = False
+    if 'skip_on_env' in test_def:
+        for skip_env_var in test_def['skip_on_env']:
+            if skip_env_var in os.environ:
+                skip = True
+
+    # Skip if os
+    if 'skip_on_os' in test_def:
+        for skip_env_var in test_def['skip_on_os']:
+            if skip_env_var.startswith('!'):
+                if mesonenv.machines.host.system != skip_env_var[1:]:
+                    skip = True
+            else:
+                if mesonenv.machines.host.system == skip_env_var:
+                    skip = True
+
+    return (skip, skip_expected)
+
+
 def load_test_json(t: TestDef, stdout_mandatory: bool, mesonenv: environment.Environment) -> T.List[TestDef]:
     all_tests: T.List[TestDef] = []
     test_def = {}
@@ -759,6 +790,8 @@ def load_test_json(t: TestDef, stdout_mandatory: bool, mesonenv: environment.Env
     # Handle the do_not_set_opts list
     do_not_set_opts = test_def.get('do_not_set_opts', [])  # type: T.List[str]
 
+    (t.skip, t.skip_expected) = _skip_keys(test_def, mesonenv)
+
     # Skip tests if the tool requirements are not met
     if 'tools' in test_def:
         assert isinstance(test_def['tools'], dict)
@@ -768,52 +801,28 @@ def load_test_json(t: TestDef, stdout_mandatory: bool, mesonenv: environment.Env
             elif not mesonlib.version_compare(tool_vers_map[tool], vers_req):
                 t.skip = True
 
-    # Test is expected to skip if MESON_CI_JOBNAME contains any of the list of
-    # substrings
-    if ('skip_on_jobname' in test_def) and (ci_jobname is not None):
-        print("looking for %s in %s" % (test_def['skip_on_jobname'], ci_jobname))
-        skip_expected = any(s in ci_jobname for s in test_def['skip_on_jobname'])
-    else:
-        skip_expected = False
-
     # Skip the matrix code and just update the existing test
     if 'matrix' not in test_def:
         t.env.update(env)
         t.installed_files = installed
         t.do_not_set_opts = do_not_set_opts
         t.stdout = stdout
-        t.skip_expected = skip_expected
         return [t]
 
-    new_opt_list: T.List[T.List[T.Tuple[str, bool]]]
+    new_opt_list: T.List[T.List[T.Tuple[str, bool, bool]]]
 
     # 'matrix; entry is present, so build multiple tests from matrix definition
-    opt_list = []  # type: T.List[T.List[T.Tuple[str, bool]]]
+    opt_list = []  # type: T.List[T.List[T.Tuple[str, bool, bool]]]
     matrix = test_def['matrix']
     assert "options" in matrix
     for key, val in matrix["options"].items():
         assert isinstance(val, list)
-        tmp_opts = []  # type: T.List[T.Tuple[str, bool]]
+        tmp_opts = []  # type: T.List[T.Tuple[str, bool, bool]]
         for i in val:
             assert isinstance(i, dict)
             assert "val" in i
-            skip = False
-
-            # Skip the matrix entry if environment variable is present
-            if 'skip_on_env' in i:
-                for skip_env_var in i['skip_on_env']:
-                    if skip_env_var in os.environ:
-                        skip = True
 
-            # Skip the matrix entry if os
-            if 'skip_on_os' in i:
-                for skip_env_var in i['skip_on_os']:
-                    if skip_env_var.startswith('!'):
-                        if mesonenv.machines.host.system != skip_env_var[1:]:
-                            skip = True
-                    else:
-                        if mesonenv.machines.host.system == skip_env_var:
-                            skip = True
+            (skip, skip_expected) = _skip_keys(i, mesonenv)
 
             # Only run the test if all compiler ID's match
             if 'compilers' in i:
@@ -824,10 +833,10 @@ def load_test_json(t: TestDef, stdout_mandatory: bool, mesonenv: environment.Env
 
             # Add an empty matrix entry
             if i['val'] is None:
-                tmp_opts += [(None, skip)]
+                tmp_opts += [(None, skip, skip_expected)]
                 continue
 
-            tmp_opts += [('{}={}'.format(key, i['val']), skip)]
+            tmp_opts += [('{}={}'.format(key, i['val']), skip, skip_expected)]
 
         if opt_list:
             new_opt_list = []
@@ -860,12 +869,13 @@ def load_test_json(t: TestDef, stdout_mandatory: bool, mesonenv: environment.Env
         name = ' '.join([x[0] for x in i if x[0] is not None])
         opts = ['-D' + x[0] for x in i if x[0] is not None]
         skip = any([x[1] for x in i])
+        skip_expected = any([x[2] for x in i])
         test = TestDef(t.path, name, opts, skip or t.skip)
         test.env.update(env)
         test.installed_files = installed
         test.do_not_set_opts = do_not_set_opts
         test.stdout = stdout
-        test.skip_expected = skip_expected
+        test.skip_expected = skip_expected or t.skip_expected
         all_tests.append(test)
 
     return all_tests