diff --git a/datasette/app.py b/datasette/app.py
index 8873ce2885..03d1dacc93 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -295,7 +295,9 @@ def __init__(
             # Validate those settings
             for key in settings:
                 if key not in DEFAULT_SETTINGS:
-                    raise StartupError("Invalid setting '{key}' in settings.json")
+                    raise StartupError(
+                        "Invalid setting '{}' in settings.json".format(key)
+                    )
         self._settings = dict(DEFAULT_SETTINGS, **(settings or {}))
         self.renderers = {}  # File extension -> (renderer, can_render) functions
         self.version_note = version_note
diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py
index e365515b7f..f5ecf0d6d7 100644
--- a/tests/test_config_dir.py
+++ b/tests/test_config_dir.py
@@ -86,8 +86,9 @@ def test_invalid_settings(config_dir):
         json.dumps({"invalid": "invalid-setting"}), "utf-8"
     )
     try:
-        with pytest.raises(StartupError):
+        with pytest.raises(StartupError) as ex:
             ds = Datasette([], config_dir=config_dir)
+        assert ex.value.args[0] == "Invalid setting 'invalid' in settings.json"
     finally:
         (config_dir / "settings.json").write_text(previous, "utf-8")