Merge pull request #3830 from tybug/assume-status-reason

add status reason to UnsatisfiedAssumption rejections

hypothesis-python/RELEASE.rst

@@ -0,0 +1,5 @@
+RELEASE_TYPE: patch
+
+This patch adds a :ref:`test statistics <statistics>` event when a generated example is rejected via :func:`assume <hypothesis.assume>`.
+
+This may also help with distinguishing ``gave_up`` examples in :doc:`observability <observability>` (:issue:`3827`).

hypothesis-python/src/hypothesis/control.py

@@ -8,6 +8,7 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import inspect
 import math
 from collections import defaultdict
 from typing import NoReturn, Union
@@ -25,14 +26,19 @@
 from hypothesis.vendor.pretty import IDKey
 
 
+def _calling_function_name(frame):
+    return frame.f_back.f_code.co_name
+
+
 def reject() -> NoReturn:
     if _current_build_context.value is None:
         note_deprecation(
             "Using `reject` outside a property-based test is deprecated",
             since="2023-09-25",
             has_codemod=False,
         )
-    raise UnsatisfiedAssumption
+    f = _calling_function_name(inspect.currentframe())
+    raise UnsatisfiedAssumption(f"reject() in {f}")
 
 
 def assume(condition: object) -> bool:
@@ -49,7 +55,8 @@ def assume(condition: object) -> bool:
             has_codemod=False,
         )
     if not condition:
-        raise UnsatisfiedAssumption
+        f = _calling_function_name(inspect.currentframe())
+        raise UnsatisfiedAssumption(f"failed to satisfy assume() in {f}")
     return True
 
 

hypothesis-python/src/hypothesis/core.py

@@ -1005,10 +1005,10 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None:
                     f"{self.test.__name__} returned {result!r} instead.",
                     HealthCheck.return_value,
                 )
-        except UnsatisfiedAssumption:
+        except UnsatisfiedAssumption as e:
             # An "assume" check failed, so instead we inform the engine that
             # this test run was invalid.
-            data.mark_invalid()
+            data.mark_invalid(e.reason)
         except StopTest:
             # The engine knows how to handle this control exception, so it's
             # OK to re-raise it.

hypothesis-python/src/hypothesis/errors.py

@@ -23,6 +23,9 @@ class UnsatisfiedAssumption(HypothesisException):
     If you're seeing this error something has gone wrong.
     """
 
+    def __init__(self, reason=None):
+        self.reason = reason
+
 
 class NoSuchExample(HypothesisException):
     """The condition we have been asked to satisfy appears to be always false.

hypothesis-python/tests/cover/test_control.py

@@ -27,15 +27,15 @@
     UnsatisfiedAssumption,
 )
 from hypothesis.internal.compat import ExceptionGroup
-from hypothesis.internal.conjecture.data import ConjectureData as TD
+from hypothesis.internal.conjecture.data import ConjectureData
 from hypothesis.stateful import RuleBasedStateMachine, rule
 from hypothesis.strategies import integers
 
 from tests.common.utils import capture_out
 
 
 def bc():
-    return BuildContext(TD.for_buffer(b""))
+    return BuildContext(ConjectureData.for_buffer(b""))
 
 
 def test_cannot_cleanup_with_no_context():

hypothesis-python/tests/cover/test_observability.py

@@ -65,3 +65,16 @@ def test_observability():
     assert {t["property"] for t in testcases} == {do_it_all.__name__}
     assert len({t["run_start"] for t in testcases}) == 2
     assert {t["status"] for t in testcases} == {"gave_up", "passed", "failed"}
+
+
+def test_assume_has_status_reason():
+    @given(st.booleans())
+    def f(b):
+        assume(b)
+
+    with capture_observations() as ls:
+        f()
+
+    gave_ups = [t for t in ls if t["type"] == "test_case" and t["status"] == "gave_up"]
+    for gave_up in gave_ups:
+        assert gave_up["status_reason"] == "failed to satisfy assume() in f"

hypothesis-python/tests/cover/test_regressions.py

@@ -129,6 +129,7 @@ def test_prng_state_unpolluted_by_given_issue_1266():
         runtime=timedelta(seconds=1.5), deadline=timedelta(seconds=1.0)
     ),
     errors.RewindRecursive(int),
+    errors.UnsatisfiedAssumption("reason for unsatisfied"),
 ]
 
 

hypothesis-python/tests/cover/test_statistical_events.py

@@ -20,6 +20,7 @@
     event,
     example,
     given,
+    reject,
     settings,
     stateful,
     strategies as st,
@@ -260,3 +261,28 @@ def test(value):
 @given(st.booleans())
 def test_event_with_non_weakrefable_keys(b):
     event((b,))
+
+
+def test_assume_adds_event_with_function_origin():
+    @given(st.integers())
+    def very_distinguishable_name(n):
+        assume(n > 100)
+
+    stats = call_for_statistics(very_distinguishable_name)
+
+    for tc in stats["generate-phase"]["test-cases"]:
+        for e in tc["events"]:
+            assert "failed to satisfy assume() in very_distinguishable_name" in e
+
+
+def test_reject_adds_event_with_function_origin():
+    @given(st.integers())
+    def very_distinguishable_name(n):
+        if n > 100:
+            reject()
+
+    stats = call_for_statistics(very_distinguishable_name)
+
+    for tc in stats["generate-phase"]["test-cases"]:
+        for e in tc["events"]:
+            assert "reject() in very_distinguishable_name" in e