diff --git a/userbenchmark/dynamo/dynamobench/_dynamo/utils.py b/userbenchmark/dynamo/dynamobench/_dynamo/utils.py
index 32305531c..e4aa58e0c 100644
--- a/userbenchmark/dynamo/dynamobench/_dynamo/utils.py
+++ b/userbenchmark/dynamo/dynamobench/_dynamo/utils.py
@@ -3599,4 +3599,6 @@ def set_feature_use(feature: str, usage: bool):
     Records whether we are using a feature
     Generally a feature is a JK.
     """
-    get_metrics_context().set_key_value("feature_usage", feature, usage)
+    # Note that sometimes (tests etc...) we're not in a context which we can record into
+    if get_metrics_context().in_progress():
+        get_metrics_context().set_key_value("feature_usage", feature, usage)