From d9ba5926bfe72c3a65bff85e8bc433bcea4e3d0c Mon Sep 17 00:00:00 2001
From: Alan Lawrence <alan.lawrence@cambridgequantum.com>
Date: Mon, 16 Sep 2024 14:25:26 +0100
Subject: [PATCH] test: only skip validation tests when given explicit flag
 (#472)

pytest previously skipped the validation tests if the validator was not
built, leading to the situation in #468 where many tests were
accidentally skipped in CI.

Instead, fail the tests if the validator is not installed, but allow
`pytest --no_validation` to skip them instead.

The possibility of doing similar for execution tests (somewhat-silently
skipped if `execute-llvm` isn't installed) is left for a later PR.

---------

Co-authored-by: Kartik Singhal <kartik.singhal@quantinuum.com>
---
 README.md                     |  4 +++-
 tests/conftest.py             |  7 +++++++
 tests/integration/conftest.py | 13 +++++++------
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 491cab0c..7948f1ca 100644
--- a/README.md
+++ b/README.md
@@ -96,9 +96,11 @@ uv run pre-commit install
 Run tests using
 
 ```sh
-uv run pytest -v
+uv run pytest [-v]  # -v just enables verbose test output
 ```
 
+(If you have not built the validator, you can do `uv run pytest --no_validation`.)
+
 You have to install extra dependencies to test automatic circuit conversion from `pytket`.
 
 ```sh
diff --git a/tests/conftest.py b/tests/conftest.py
index f5d7a394..aac6a94e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -16,3 +16,10 @@ def dir_path(s):
         type=dir_path,
         help="A directory to which to export test cases",
     )
+
+    parser.addoption(
+        "--no_validation",
+        dest="validation",
+        action="store_false",
+        help="Disable validation tests (run by default)",
+    )
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index b1cac1ce..a3b33d8a 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -6,7 +6,7 @@
 import subprocess
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def export_test_cases_dir(request):
     r = request.config.getoption("--export-test-cases")
     if r and not r.exists():
@@ -26,16 +26,17 @@ def get_validator() -> Path | None:
     return None
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def validate(request, export_test_cases_dir: Path):
-    def validate_json(hugr: str):
+    if request.config.getoption("validation"):
         # Check if the validator is installed
         validator = get_validator()
         if validator is None:
-            pytest.skip(
-                "Skipping validation: Run `cargo build` to install the validator"
-            )
+            pytest.fail("Run `cargo build -p release` to install the validator")
+    else:
+        pytest.skip("Skipping validation tests as requested")
 
+    def validate_json(hugr: str):
         # Executes `cargo run -p validator -- validate -`
         # passing the hugr JSON as stdin
         p = subprocess.run(  # noqa: S603