Warn on @given + function-scoped fixtures

hypothesis-python/RELEASE.rst

@@ -0,0 +1,11 @@
+RELEASE_TYPE: minor
+
+This release adds an explicit warning for tests that are both decorated with
+:func:`@given(...) <hypothesis.given>` and request a
+:doc:`function-scoped pytest fixture <pytest:fixture>`, because such fixtures
+are only executed once for *all* Hypothesis test cases and that often causes
+trouble (:issue:`377`).
+
+It's *very* difficult to fix this on the :pypi:`pytest` side, so since 2015
+our advice has been "just don't use function-scoped fixtures with Hypothesis".
+Now we detect and warn about the issue at runtime!

hypothesis-python/src/hypothesis/extra/pytestplugin.py

@@ -14,10 +14,12 @@
 # END HEADER
 
 from distutils.version import LooseVersion
+from inspect import signature
 
 import pytest
 
 from hypothesis import Verbosity, core, settings
+from hypothesis._settings import note_deprecation
 from hypothesis.errors import InvalidArgument
 from hypothesis.internal.detection import is_hypothesis_test
 from hypothesis.reporting import default as default_reporter, with_reporter
@@ -144,6 +146,25 @@ def pytest_runtest_call(item):
                     raise InvalidArgument(message % (name,))
             yield
         else:
+            # Warn about function-scoped fixtures, excluding autouse fixtures because
+            # the advice is probably not actionable and the status quo seems OK...
+            # See https://github.com/HypothesisWorks/hypothesis/issues/377 for detail.
+            argnames = None
+            for fx_defs in item._request._fixturemanager.getfixtureinfo(
+                node=item, func=item.function, cls=None
+            ).name2fixturedefs.values():
+                if argnames is None:
+                    argnames = frozenset(signature(item.function).parameters)
+                for fx in fx_defs:
+                    if fx.scope == "function" and fx.argname in argnames:
+                        note_deprecation(
+                            "%s uses the %r fixture, but function-scoped fixtures "
+                            "should not be used with @given(...) tests, because "
+                            "fixtures are not reset between generated examples!"
+                            % (item.nodeid, fx.argname),
+                            since="RELEASEDAY",
+                        )
+
             if item.get_closest_marker("parametrize") is not None:
                 # Give every parametrized test invocation a unique database key
                 key = item.nodeid.encode("utf-8")

hypothesis-python/tests/cover/test_mock.py

@@ -19,7 +19,7 @@
 import math
 from unittest import mock
 
-from _pytest.capture import CaptureFixture
+from _pytest.config import Config
 
 import hypothesis.strategies as st
 from hypothesis import given
@@ -34,15 +34,15 @@ def test_can_mock_inside_given_without_fixture(atan, thing):
 
 @mock.patch("math.atan")
 @given(thing=st.text())
-def test_can_mock_outside_given_with_fixture(atan, capsys, thing):
+def test_can_mock_outside_given_with_fixture(atan, pytestconfig, thing):
     assert isinstance(atan, mock.MagicMock)
     assert isinstance(math.atan, mock.MagicMock)
-    assert isinstance(capsys, CaptureFixture)
+    assert isinstance(pytestconfig, Config)
 
 
 @given(thing=st.text())
-def test_can_mock_within_test_with_fixture(capsys, thing):
-    assert isinstance(capsys, CaptureFixture)
+def test_can_mock_within_test_with_fixture(pytestconfig, thing):
+    assert isinstance(pytestconfig, Config)
     assert not isinstance(math.atan, mock.MagicMock)
     with mock.patch("math.atan") as atan:
         assert isinstance(atan, mock.MagicMock)

hypothesis-python/tests/pytest/test_fixtures.py

@@ -21,18 +21,20 @@
 from hypothesis.strategies import integers
 from tests.common.utils import fails
 
+pytest_plugins = "pytester"
 
-@pytest.fixture
+
+@pytest.fixture(scope="session")
 def infinity():
     return float("inf")
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
 def mock_fixture():
     return Mock()
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
 def spec_fixture():
     class Foo:
         def __init__(self):
@@ -77,3 +79,26 @@ def test_can_inject_mock_via_fixture(mock_fixture, xs):
 def test_can_inject_autospecced_mock_via_fixture(spec_fixture, xs):
     spec_fixture.bar.return_value = float("inf")
     assert xs <= spec_fixture.bar()
+
+
+TESTSUITE = """
+import pytest
+from hypothesis import given, strategies as st
+
+@pytest.fixture(scope="function", autouse=True)
+def autofix(request):
+    pass
+
+@given(x=st.integers())
+def test_requests_function_scoped_fixture(capsys, x):
+    pass
+
+@given(x=st.integers())
+def test_autouse_function_scoped_fixture(x):
+    pass
+"""
+
+
+def test_given_plus_function_scoped_non_autouse_fixtures_are_deprecated(testdir):
+    script = testdir.makepyfile(TESTSUITE)
+    testdir.runpytest(script, "-Werror").assert_outcomes(passed=1, failed=1)