From 3b26de80504f68591d2cee76b1ed9c3a2951e6dc Mon Sep 17 00:00:00 2001 From: Zoraaver Singh Date: Tue, 27 Feb 2024 19:32:18 +0000 Subject: [PATCH] Add support for test config overrides It can be useful to override the config for certain tests. The main use case for this is to override environment variables which control test assertions (e.g. NO_DANGLING_FILESYSTEM, ERRNO_MODE_WINDOWS). When running the tests in CI, it would be nicer to support this without manually modifying the JSON test config files. --- README.md | 15 ++++++++ examples/config_override.json | 16 ++++++++ test-runner/tests/test_test_suite_runner.py | 6 ++- test-runner/wasi_test_runner/__main__.py | 13 ++++++- test-runner/wasi_test_runner/harness.py | 7 +++- test-runner/wasi_test_runner/override.py | 38 +++++++++++++++++++ test-runner/wasi_test_runner/test_case.py | 19 ++++++---- .../wasi_test_runner/test_suite_runner.py | 21 ++++++++-- 8 files changed, 118 insertions(+), 17 deletions(-) create mode 100644 examples/config_override.json create mode 100644 test-runner/wasi_test_runner/override.py diff --git a/README.md b/README.md index 6649e8c4..e197f355 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,21 @@ python3 test-runner/wasi_test_runner.py -r adapters/wasmtime.py # path to a runtime adapter ``` +You can also optionally override test configs with the `--config-override` +option: + +```bash +python3 test-runner/wasi_test_runner.py \ + -t ./tests/assemblyscript/testsuite/ # path to folders containing .wasm test files \ + ./tests/c/testsuite/ \ + ./tests/rust/testsuite/ \ + --config-override examples/config_override.json \ + -r adapters/wasmtime.py # path to a runtime adapter +``` + +This can be useful for passing additional environment variables to certain +tests without modifying the test config files. + The default executable in the adapter used for test execution can be overridden using `TEST_RUNTIME_EXE` variable. This only works with adapters defined in [adapters/](adapters/), and might not work with 3rd party adapters. diff --git a/examples/config_override.json b/examples/config_override.json new file mode 100644 index 00000000..5e7ec607 --- /dev/null +++ b/examples/config_override.json @@ -0,0 +1,16 @@ +{ + "WASI C tests": { + "stat-dev-ino": { + "dirs": [ + "fs-tests.dir" + ], + "env": { + "TEST_VAR": "test" + }, + "args": [ + "fs-tests.dir", + "test_arg" + ] + } + } +} \ No newline at end of file diff --git a/test-runner/tests/test_test_suite_runner.py b/test-runner/tests/test_test_suite_runner.py index d73438de..17bd3dc3 100644 --- a/test-runner/tests/test_test_suite_runner.py +++ b/test-runner/tests/test_test_suite_runner.py @@ -67,7 +67,9 @@ def test_runner_end_to_end() -> None: filters = [filt] with patch("glob.glob", return_value=test_paths): - suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters, filters) # type: ignore + suite = tsr.run_tests_from_test_suite( + "my-path", runtime, validators, reporters, filters, None # type: ignore + ) # Assert manifest was read correctly assert suite.name == "test-suite" @@ -104,6 +106,6 @@ def test_runner_end_to_end() -> None: @patch("os.path.exists", Mock(return_value=False)) def test_runner_should_use_path_for_name_if_manifest_does_not_exist() -> None: - suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], []) + suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], [], None) assert suite.name == "my-path" diff --git a/test-runner/wasi_test_runner/__main__.py b/test-runner/wasi_test_runner/__main__.py index c2b92f45..1b63493b 100644 --- a/test-runner/wasi_test_runner/__main__.py +++ b/test-runner/wasi_test_runner/__main__.py @@ -1,6 +1,6 @@ import argparse import sys -from typing import List +from typing import List, Optional from .runtime_adapter import RuntimeAdapter @@ -11,6 +11,7 @@ from .reporters.console import ConsoleTestReporter from .reporters.json import JSONTestReporter from .validators import exit_code_validator, stdout_validator, Validator +from .override import ConfigOverride, JSONConfigOverride def main() -> int: @@ -46,6 +47,11 @@ def main() -> int: default=False, help="Disables color for console output reporter.", ) + parser.add_argument( + "--config-override", + required=False, + help="Location of JSON file containing overrides for the config used for each test", + ) options = parser.parse_args() @@ -59,12 +65,17 @@ def main() -> int: for filt in options.exclude_filter: filters.append(JSONTestExcludeFilter(filt)) + override: Optional[ConfigOverride] = None + if options.config_override: + override = JSONConfigOverride(options.config_override) + return run_all_tests( RuntimeAdapter(options.runtime_adapter), options.test_suite, validators, reporters, filters, + override, ) diff --git a/test-runner/wasi_test_runner/harness.py b/test-runner/wasi_test_runner/harness.py index 027b2bb5..d25a1d8c 100644 --- a/test-runner/wasi_test_runner/harness.py +++ b/test-runner/wasi_test_runner/harness.py @@ -1,24 +1,27 @@ -from typing import List +from typing import List, Optional from .filters import TestFilter from .reporters import TestReporter from .test_suite_runner import run_tests_from_test_suite from .runtime_adapter import RuntimeAdapter from .validators import Validator +from .override import ConfigOverride +# pylint: disable-msg=too-many-arguments def run_all_tests( runtime: RuntimeAdapter, test_suite_paths: List[str], validators: List[Validator], reporters: List[TestReporter], filters: List[TestFilter], + override: Optional[ConfigOverride], ) -> int: ret = 0 for test_suite_path in test_suite_paths: test_suite = run_tests_from_test_suite( - test_suite_path, runtime, validators, reporters, filters, + test_suite_path, runtime, validators, reporters, filters, override ) for reporter in reporters: reporter.report_test_suite(test_suite) diff --git a/test-runner/wasi_test_runner/override.py b/test-runner/wasi_test_runner/override.py new file mode 100644 index 00000000..dc6f1b6e --- /dev/null +++ b/test-runner/wasi_test_runner/override.py @@ -0,0 +1,38 @@ +import json + +from abc import ABC, abstractmethod +from typing import Any, Dict, Optional +from .test_case import ( + Config, +) + + +class ConfigOverride(ABC): + @abstractmethod + def get_test_override( + self, test_suite_name: str, test_name: str + ) -> Optional[Config]: + pass + + +class JSONConfigOverride(ConfigOverride): + overrides_dict: Dict[str, Dict[str, Dict[str, Any]]] + + def __init__(self, overrides_path: str) -> None: + with open(overrides_path, encoding="utf-8") as file: + self.overrides_dict = json.load(file) + + def get_test_override( + self, test_suite_name: str, test_name: str + ) -> Optional[Config]: + test_suite_overrides = self.overrides_dict.get(test_suite_name) + + if test_suite_overrides is None: + return None + + test_override = test_suite_overrides.get(test_name) + + if test_override is None: + return None + + return Config.from_dict(test_override) diff --git a/test-runner/wasi_test_runner/test_case.py b/test-runner/wasi_test_runner/test_case.py index eebb95d3..fc574f6e 100644 --- a/test-runner/wasi_test_runner/test_case.py +++ b/test-runner/wasi_test_runner/test_case.py @@ -37,10 +37,19 @@ class Config(NamedTuple): @classmethod def from_file(cls: Type[T], config_file: str) -> T: - default = cls() - with open(config_file, encoding="utf-8") as file: dict_config = json.load(file) + return cls.from_dict(dict_config) + + @classmethod + def _validate_dict(cls: Type[T], dict_config: Dict[str, Any]) -> None: + for field_name in dict_config: + if field_name not in cls._fields: + logging.warning("Unknown field in the config file: %s", field_name) + + @classmethod + def from_dict(cls: Type[T], dict_config: Dict[str, Any]) -> T: + default = cls() cls._validate_dict(dict_config) @@ -53,12 +62,6 @@ def from_file(cls: Type[T], config_file: str) -> T: stdout=dict_config.get("stdout", default.stdout), ) - @classmethod - def _validate_dict(cls: Type[T], dict_config: Dict[str, Any]) -> None: - for field_name in dict_config: - if field_name not in cls._fields: - logging.warning("Unknown field in the config file: %s", field_name) - class TestCase(NamedTuple): name: str diff --git a/test-runner/wasi_test_runner/test_suite_runner.py b/test-runner/wasi_test_runner/test_suite_runner.py index c9a26b14..838bae27 100644 --- a/test-runner/wasi_test_runner/test_suite_runner.py +++ b/test-runner/wasi_test_runner/test_suite_runner.py @@ -6,7 +6,7 @@ import time from datetime import datetime -from typing import List, cast +from typing import List, Optional, cast from .filters import TestFilter from .runtime_adapter import RuntimeAdapter @@ -19,14 +19,17 @@ from .reporters import TestReporter from .test_suite import TestSuite from .validators import Validator +from .override import ConfigOverride +# pylint: disable-msg=too-many-arguments, too-many-locals def run_tests_from_test_suite( test_suite_path: str, runtime: RuntimeAdapter, validators: List[Validator], reporters: List[TestReporter], filters: List[TestFilter], + config_override: Optional[ConfigOverride], ) -> TestSuite: test_cases: List[TestCase] = [] test_start = datetime.now() @@ -45,7 +48,14 @@ def run_tests_from_test_suite( test_case = _skip_single_test(runtime, validators, test_path) break else: - test_case = _execute_single_test(runtime, validators, test_path) + test_config_override = ( + config_override.get_test_override(test_suite_name, test_name) + if config_override + else None + ) + test_case = _execute_single_test( + runtime, validators, test_path, test_config_override + ) test_cases.append(test_case) for reporter in reporters: reporter.report_test(test_case) @@ -73,9 +83,12 @@ def _skip_single_test( def _execute_single_test( - runtime: RuntimeAdapter, validators: List[Validator], test_path: str + runtime: RuntimeAdapter, + validators: List[Validator], + test_path: str, + config_override: Optional[Config], ) -> TestCase: - config = _read_test_config(test_path) + config = config_override or _read_test_config(test_path) test_start = time.time() test_output = runtime.run_test(test_path, config.args, config.env, config.dirs) elapsed = time.time() - test_start