diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
index 0702a85b8e..70394e3967 100644
--- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
+++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
@@ -508,6 +508,11 @@ def test_function(self, data: ConjectureData) -> None:
                 # to a buffer
                 data = ConjectureData.for_ir_tree(data.examples.ir_tree_nodes)
                 self.__stoppable_test_function(data)
+                data.freeze()
+                # should we raise Flaky here instead?
+                if data.status != Status.INTERESTING:
+                    self.exit_with(ExitReason.flaky)
+
                 self._cache(data)
 
             key = data.interesting_origin