Add note to errors from strategies

hypothesis-python/RELEASE.rst

@@ -0,0 +1,4 @@
+RELEASE_TYPE: patch
+
+This patch :pep:`adds a note <678>` to errors which occur while drawing from
+a strategy, to make it easier to tell why your test failed in such cases.

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -40,7 +40,7 @@
 
 from hypothesis.errors import Frozen, InvalidArgument, StopTest
 from hypothesis.internal.cache import LRUReusedCache
-from hypothesis.internal.compat import floor, int_from_bytes, int_to_bytes
+from hypothesis.internal.compat import add_note, floor, int_from_bytes, int_to_bytes
 from hypothesis.internal.conjecture.floats import float_to_lex, lex_to_float
 from hypothesis.internal.conjecture.junkdrawer import IntList, uniform
 from hypothesis.internal.conjecture.utils import (
@@ -1748,14 +1748,17 @@ def draw(
         try:
             if not at_top_level:
                 return strategy.do_draw(self)
-            else:
-                assert start_time is not None
+            assert start_time is not None
+            key = observe_as or f"generate:unlabeled_{len(self.draw_times)}"
+            try:
                 strategy.validate()
                 try:
                     return strategy.do_draw(self)
                 finally:
-                    key = observe_as or f"generate:unlabeled_{len(self.draw_times)}"
                     self.draw_times[key] = time.perf_counter() - start_time
+            except Exception as err:
+                add_note(err, f"while generating {key[9:]!r} from {strategy!r}")
+                raise
         finally:
             self.stop_example()
 

hypothesis-python/src/hypothesis/stateful.py

@@ -47,6 +47,7 @@
 from hypothesis.control import _current_build_context, current_build_context
 from hypothesis.core import TestFunc, given
 from hypothesis.errors import InvalidArgument, InvalidDefinition
+from hypothesis.internal.compat import add_note
 from hypothesis.internal.conjecture import utils as cu
 from hypothesis.internal.healthcheck import fail_health_check
 from hypothesis.internal.observability import TESTCASE_CALLBACKS
@@ -439,18 +440,16 @@ class Rule:
     bundles = attr.ib(init=False)
 
     def __attrs_post_init__(self):
-        arguments = {}
+        self.arguments_strategies = {}
         bundles = []
         for k, v in sorted(self.arguments.items()):
             assert not isinstance(v, BundleReferenceStrategy)
             if isinstance(v, Bundle):
                 bundles.append(v)
                 consume = isinstance(v, BundleConsumer)
-                arguments[k] = BundleReferenceStrategy(v.name, consume=consume)
-            else:
-                arguments[k] = v
+                v = BundleReferenceStrategy(v.name, consume=consume)
+            self.arguments_strategies[k] = v
         self.bundles = tuple(bundles)
-        self.arguments_strategy = st.fixed_dictionaries(arguments)
 
 
 self_strategy = st.runner()
@@ -978,7 +977,15 @@ def do_draw(self, data):
             .filter(lambda r: feature_flags.is_enabled(r.function.__name__))
         )
 
-        return (rule, data.draw(rule.arguments_strategy))
+        arguments = {}
+        for k, strat in rule.arguments_strategies.items():
+            try:
+                arguments[k] = data.draw(strat)
+            except Exception as err:
+                rname = rule.function.__name__
+                add_note(err, f"while generating {k!r} from {strat!r} for rule {rname}")
+                raise
+        return (rule, arguments)
 
     def is_valid(self, rule):
         predicates = self.machine._observability_predicates

hypothesis-python/tests/cover/test_error_in_draw.py

@@ -8,10 +8,13 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import re
+
 import pytest
 
 from hypothesis import given, strategies as st
 from hypothesis.errors import HypothesisWarning
+from hypothesis.stateful import RuleBasedStateMachine, rule, run_state_machine_as_test
 
 
 def test_error_is_in_finally():
@@ -36,3 +39,31 @@ def test_warns_on_bool_strategy(data):
     ):
         if st.booleans():  # 'forgot' to draw from the strategy
             pass
+
+
+def test_adds_note_showing_which_strategy():
+    class X:
+        def __init__(self, y: int) -> None:
+            assert y == 7
+
+    @given(...)
+    def inner(value: X):
+        assert isinstance(value, X)
+
+    rep = re.escape(repr(st.from_type(X)))
+    with pytest.raises(AssertionError, match=f".*while generating 'value' from {rep}"):
+        inner()
+
+
+def test_adds_note_showing_which_strategy_stateful():
+    strategy = st.integers().map(lambda x: x / 0)
+
+    class Machine(RuleBasedStateMachine):
+        @rule(value=strategy)
+        def take_a_step(self, value):
+            assert value
+
+    msg = f"while generating 'value' from {strategy!r} for rule take_a_step"
+    print(msg)
+    with pytest.raises(ZeroDivisionError, match=f".*{re.escape(msg)}"):
+        run_state_machine_as_test(Machine)