From 9ed593e829b14a0e154be2d05d726e239a605329 Mon Sep 17 00:00:00 2001 From: Lunderberg Date: Wed, 30 Jun 2021 02:47:52 -0700 Subject: [PATCH] [Unittests] Added a meta-test for tvm.testing.fixture behavior in case of a broken fixture. (#8343) In these cases, the test function should be marked as failing the setup, and should not run. This is pytest's default behavior, and should work whether or not a fixture is cached. Co-authored-by: Eric Lunderberg --- .../unittest/test_tvm_testing_features.py | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/python/unittest/test_tvm_testing_features.py b/tests/python/unittest/test_tvm_testing_features.py index 1a7595aac5c7..07b8c652bf1f 100644 --- a/tests/python/unittest/test_tvm_testing_features.py +++ b/tests/python/unittest/test_tvm_testing_features.py @@ -25,7 +25,10 @@ # This file tests features in tvm.testing, such as verifying that # cached fixtures are run an appropriate number of times. As a # result, the order of the tests is important. Use of --last-failed -# or --failed-first while debugging this file is not advised. +# or --failed-first while debugging this file is not advised. If +# these tests are distributed/parallelized using pytest-xdist or +# similar, all tests in this file should run sequentially on the same +# node. (See https://stackoverflow.com/a/59504228) class TestTargetAutoParametrization: @@ -145,5 +148,37 @@ def test_cached_count(self): assert self.cached_calls == len(self.param1_vals) +class TestBrokenFixture: + # Tests that use a fixture that throws an exception fail, and are + # marked as setup failures. The tests themselves are never run. + # This behavior should be the same whether or not the fixture + # results are cached. + + num_uses_broken_uncached_fixture = 0 + num_uses_broken_cached_fixture = 0 + + @tvm.testing.fixture + def broken_uncached_fixture(self): + raise RuntimeError("Intentionally broken fixture") + + @pytest.mark.xfail(True, reason="Broken fixtures should result in a failing setup", strict=True) + def test_uses_broken_uncached_fixture(self, broken_uncached_fixture): + type(self).num_uses_broken_fixture += 1 + + def test_num_uses_uncached(self): + assert self.num_uses_broken_uncached_fixture == 0 + + @tvm.testing.fixture(cache_return_value=True) + def broken_cached_fixture(self): + raise RuntimeError("Intentionally broken fixture") + + @pytest.mark.xfail(True, reason="Broken fixtures should result in a failing setup", strict=True) + def test_uses_broken_cached_fixture(self, broken_cached_fixture): + type(self).num_uses_broken_cached_fixture += 1 + + def test_num_uses_cached(self): + assert self.num_uses_broken_cached_fixture == 0 + + if __name__ == "__main__": sys.exit(pytest.main(sys.argv))