From 8f9509f00cceea8dc87403c28b2056db7b246ed4 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 22 Apr 2024 16:01:37 -0700
Subject: [PATCH 01/48] datasette, not self.ds, in internals documentation

---
 docs/internals.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/internals.rst b/docs/internals.rst
index b57a289a07..795856594d 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -386,7 +386,7 @@ This is useful when you need to check multiple permissions at once. For example,
 
 .. code-block:: python
 
-    await self.ds.ensure_permissions(
+    await datasette.ensure_permissions(
         request.actor,
         [
             ("view-table", (database, table)),
@@ -420,7 +420,7 @@ This example checks if the user can access a specific table, and sets ``private`
 
 .. code-block:: python
 
-    visible, private = await self.ds.check_visibility(
+    visible, private = await datasette.check_visibility(
         request.actor,
         action="view-table",
         resource=(database, table),
@@ -430,7 +430,7 @@ The following example runs three checks in a row, similar to :ref:`datasette_ens
 
 .. code-block:: python
 
-    visible, private = await self.ds.check_visibility(
+    visible, private = await datasette.check_visibility(
         request.actor,
         permissions=[
             ("view-table", (database, table)),

From e1bfab3fca22e4f8d06dcf419e945f13fc20aef1 Mon Sep 17 00:00:00 2001
From: Alex Garcia <alexsebastian.garcia@gmail.com>
Date: Tue, 11 Jun 2024 09:33:23 -0700
Subject: [PATCH 02/48] Move Metadata to `--internal` database

Refs:
- https://github.com/simonw/datasette/pull/2343
- https://github.com/simonw/datasette/issues/2341
---
 datasette/app.py                | 210 +++++++++++++++++++++-----------
 datasette/default_menu_links.py |   4 -
 datasette/facets.py             |  13 +-
 datasette/hookspecs.py          |   5 -
 datasette/renderer.py           |   1 -
 datasette/utils/internal_db.py  |  37 ++++++
 datasette/views/base.py         |   6 +-
 datasette/views/database.py     |   7 +-
 datasette/views/index.py        |  10 +-
 datasette/views/row.py          |   5 +-
 datasette/views/table.py        |  38 ++++--
 docs/codespell-ignore-words.txt |   3 +-
 docs/plugin_hooks.rst           |   7 +-
 tests/test_api.py               |  62 ++++++----
 tests/test_cli.py               |   4 +-
 tests/test_config_dir.py        |  17 ---
 tests/test_facets.py            |   9 +-
 tests/test_html.py              |  14 +--
 tests/test_permissions.py       |   1 -
 tests/test_plugins.py           |  41 +------
 tests/test_routes.py            |   2 -
 tests/test_table_html.py        |   4 +-
 22 files changed, 286 insertions(+), 214 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 23d21600c6..8020c5dac2 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -443,6 +443,37 @@ def __init__(
         self._root_token = secrets.token_hex(32)
         self.client = DatasetteClient(self)
 
+    async def apply_metadata_json(self):
+        # Apply any metadata entries from metadata.json to the internal tables
+        # step 1: top-level metadata
+        for key in self._metadata_local or {}:
+            if key == "databases":
+                continue
+            await self.set_instance_metadata(key, self._metadata_local[key])
+
+        # step 2: database-level metadata
+        for dbname, db in self._metadata_local.get("databases", {}).items():
+            for key, value in db.items():
+                if key == "tables":
+                    continue
+                await self.set_database_metadata(dbname, key, value)
+
+            # step 3: table-level metadata
+            for tablename, table in db.get("tables", {}).items():
+                for key, value in table.items():
+                    if key == "columns":
+                        continue
+                    await self.set_resource_metadata(dbname, tablename, key, value)
+
+                # step 4: column-level metadata (only descriptions in metadata.json)
+                for columnname, column_description in table.get("columns", {}).items():
+                    await self.set_column_metadata(
+                        dbname, tablename, columnname, "description", column_description
+                    )
+
+            # TODO(alex) is metadata.json was loaded in, and --internal is not memory, then log
+            # a warning to user that they should delete their metadata.json file
+
     def get_jinja_environment(self, request: Request = None) -> Environment:
         environment = self._jinja_env
         if request:
@@ -476,6 +507,7 @@ async def _refresh_schemas(self):
         internal_db = self.get_internal_database()
         if not self.internal_db_created:
             await init_internal_db(internal_db)
+            await self.apply_metadata_json()
             self.internal_db_created = True
         current_schema_versions = {
             row["database_name"]: row["schema_version"]
@@ -646,57 +678,113 @@ def _metadata_recursive_update(self, orig, updated):
                 orig[key] = upd_value
         return orig
 
-    def metadata(self, key=None, database=None, table=None, fallback=True):
-        """
-        Looks up metadata, cascading backwards from specified level.
-        Returns None if metadata value is not found.
-        """
-        assert not (
-            database is None and table is not None
-        ), "Cannot call metadata() with table= specified but not database="
-        metadata = {}
+    async def get_instance_metadata(self):
+        rows = await self.get_internal_database().execute(
+            """
+              SELECT
+                key,
+                value
+              FROM datasette_metadata_instance_entries
+            """
+        )
+        return dict(rows)
+
+    async def get_database_metadata(self, database_name: str):
+        rows = await self.get_internal_database().execute(
+            """
+              SELECT
+                key,
+                value
+              FROM datasette_metadata_database_entries
+              WHERE database_name = ?
+            """,
+            [database_name],
+        )
+        return dict(rows)
+
+    async def get_resource_metadata(self, database_name: str, resource_name: str):
+        rows = await self.get_internal_database().execute(
+            """
+              SELECT
+                key,
+                value
+              FROM datasette_metadata_resource_entries
+              WHERE database_name = ?
+                AND resource_name = ?
+            """,
+            [database_name, resource_name],
+        )
+        return dict(rows)
 
-        for hook_dbs in pm.hook.get_metadata(
-            datasette=self, key=key, database=database, table=table
-        ):
-            metadata = self._metadata_recursive_update(metadata, hook_dbs)
-
-        # security precaution!! don't allow anything in the local config
-        # to be overwritten. this is a temporary measure, not sure if this
-        # is a good idea long term or maybe if it should just be a concern
-        # of the plugin's implemtnation
-        metadata = self._metadata_recursive_update(metadata, self._metadata_local)
-
-        databases = metadata.get("databases") or {}
-
-        search_list = []
-        if database is not None:
-            search_list.append(databases.get(database) or {})
-        if table is not None:
-            table_metadata = ((databases.get(database) or {}).get("tables") or {}).get(
-                table
-            ) or {}
-            search_list.insert(0, table_metadata)
-
-        search_list.append(metadata)
-        if not fallback:
-            # No fallback allowed, so just use the first one in the list
-            search_list = search_list[:1]
-        if key is not None:
-            for item in search_list:
-                if key in item:
-                    return item[key]
-            return None
-        else:
-            # Return the merged list
-            m = {}
-            for item in search_list:
-                m.update(item)
-            return m
+    async def get_column_metadata(
+        self, database_name: str, resource_name: str, column_name: str
+    ):
+        rows = await self.get_internal_database().execute(
+            """
+              SELECT
+                key,
+                value
+              FROM datasette_metadata_column_entries
+              WHERE database_name = ?
+                AND resource_name = ?
+                AND column_name = ?
+            """,
+            [database_name, resource_name, column_name],
+        )
+        return dict(rows)
+
+    async def set_instance_metadata(self, key: str, value: str):
+        # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)
+        await self.get_internal_database().execute_write(
+            """
+              INSERT INTO datasette_metadata_instance_entries(key, value)
+                VALUES(?, ?)
+                ON CONFLICT(key) DO UPDATE SET value = excluded.value;
+            """,
+            [key, value],
+        )
 
-    @property
-    def _metadata(self):
-        return self.metadata()
+    async def set_database_metadata(self, database_name: str, key: str, value: str):
+        # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)
+        await self.get_internal_database().execute_write(
+            """
+              INSERT INTO datasette_metadata_database_entries(database_name, key, value)
+                VALUES(?, ?, ?)
+                ON CONFLICT(database_name, key) DO UPDATE SET value = excluded.value;
+            """,
+            [database_name, key, value],
+        )
+
+    async def set_resource_metadata(
+        self, database_name: str, resource_name: str, key: str, value: str
+    ):
+        # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)
+        await self.get_internal_database().execute_write(
+            """
+              INSERT INTO datasette_metadata_resource_entries(database_name, resource_name, key, value)
+                VALUES(?, ?, ?, ?)
+                ON CONFLICT(database_name, resource_name, key) DO UPDATE SET value = excluded.value;
+            """,
+            [database_name, resource_name, key, value],
+        )
+
+    async def set_column_metadata(
+        self,
+        database_name: str,
+        resource_name: str,
+        column_name: str,
+        key: str,
+        value: str,
+    ):
+        # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)
+        await self.get_internal_database().execute_write(
+            """
+              INSERT INTO datasette_metadata_column_entries(database_name, resource_name, column_name, key, value)
+                VALUES(?, ?, ?, ?, ?)
+                ON CONFLICT(database_name, resource_name, column_name, key) DO UPDATE SET value = excluded.value;
+            """,
+            [database_name, resource_name, column_name, key, value],
+        )
 
     def get_internal_database(self):
         return self._internal_database
@@ -774,20 +862,6 @@ async def get_canned_query(self, database_name, query_name, actor):
         if query:
             return query
 
-    def update_with_inherited_metadata(self, metadata):
-        # Fills in source/license with defaults, if available
-        metadata.update(
-            {
-                "source": metadata.get("source") or self.metadata("source"),
-                "source_url": metadata.get("source_url") or self.metadata("source_url"),
-                "license": metadata.get("license") or self.metadata("license"),
-                "license_url": metadata.get("license_url")
-                or self.metadata("license_url"),
-                "about": metadata.get("about") or self.metadata("about"),
-                "about_url": metadata.get("about_url") or self.metadata("about_url"),
-            }
-        )
-
     def _prepare_connection(self, conn, database):
         conn.row_factory = sqlite3.Row
         conn.text_factory = lambda x: str(x, "utf-8", "replace")
@@ -1079,11 +1153,6 @@ def absolute_url(self, request, path):
             url = "https://" + url[len("http://") :]
         return url
 
-    def _register_custom_units(self):
-        """Register any custom units defined in the metadata.json with Pint"""
-        for unit in self.metadata("custom_units") or []:
-            ureg.define(unit)
-
     def _connected_databases(self):
         return [
             {
@@ -1436,10 +1505,6 @@ def add_route(view, regex):
             ),
             r"/:memory:(?P<rest>.*)$",
         )
-        add_route(
-            JsonDataView.as_view(self, "metadata.json", lambda: self.metadata()),
-            r"/-/metadata(\.(?P<format>json))?$",
-        )
         add_route(
             JsonDataView.as_view(self, "versions.json", self._versions),
             r"/-/versions(\.(?P<format>json))?$",
@@ -1585,7 +1650,6 @@ async def resolve_row(self, request):
     def app(self):
         """Returns an ASGI app function that serves the whole of Datasette"""
         routes = self._routes()
-        self._register_custom_units()
 
         async def setup_db():
             # First time server starts up, calculate table counts for immutable databases
diff --git a/datasette/default_menu_links.py b/datasette/default_menu_links.py
index 56f481ef50..22e6e46abd 100644
--- a/datasette/default_menu_links.py
+++ b/datasette/default_menu_links.py
@@ -17,10 +17,6 @@ async def inner():
                 "href": datasette.urls.path("/-/versions"),
                 "label": "Version info",
             },
-            {
-                "href": datasette.urls.path("/-/metadata"),
-                "label": "Metadata",
-            },
             {
                 "href": datasette.urls.path("/-/settings"),
                 "label": "Settings",
diff --git a/datasette/facets.py b/datasette/facets.py
index f1cfc68f02..ccd854610c 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -103,10 +103,15 @@ def get_facet_size(self):
         max_returned_rows = self.ds.setting("max_returned_rows")
         table_facet_size = None
         if self.table:
-            tables_metadata = self.ds.metadata("tables", database=self.database) or {}
-            table_metadata = tables_metadata.get(self.table) or {}
-            if table_metadata:
-                table_facet_size = table_metadata.get("facet_size")
+            config_facet_size = (
+                self.ds.config.get("databases", {})
+                .get(self.database, {})
+                .get("tables", {})
+                .get(self.table, {})
+                .get("facet_size")
+            )
+            if config_facet_size:
+                table_facet_size = config_facet_size
         custom_facet_size = self.request.args.get("_facet_size")
         if custom_facet_size:
             if custom_facet_size == "max":
diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index 35468cc3ea..bcc2e22901 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -10,11 +10,6 @@ def startup(datasette):
     """Fires directly after Datasette first starts running"""
 
 
-@hookspec
-def get_metadata(datasette, key, database, table):
-    """Return metadata to be merged into Datasette's metadata dictionary"""
-
-
 @hookspec
 def asgi_wrapper(datasette):
     """Returns an ASGI middleware callable to wrap our ASGI application with"""
diff --git a/datasette/renderer.py b/datasette/renderer.py
index a446e69d0e..483c81e97a 100644
--- a/datasette/renderer.py
+++ b/datasette/renderer.py
@@ -56,7 +56,6 @@ def json_renderer(request, args, data, error, truncated=None):
 
     if truncated is not None:
         data["truncated"] = truncated
-
     if shape == "arrayfirst":
         if not data["rows"]:
             data = []
diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py
index dbfcceb47d..6a5e08cba5 100644
--- a/datasette/utils/internal_db.py
+++ b/datasette/utils/internal_db.py
@@ -63,6 +63,43 @@ async def init_internal_db(db):
     """
     ).strip()
     await db.execute_write_script(create_tables_sql)
+    await initialize_metadata_tables(db)
+
+
+async def initialize_metadata_tables(db):
+    await db.execute_write_script(
+        """
+              CREATE TABLE IF NOT EXISTS datasette_metadata_instance_entries(
+                key text,
+                value text,
+                unique(key)
+              );
+
+              CREATE TABLE IF NOT EXISTS datasette_metadata_database_entries(
+                database_name text,
+                key text,
+                value text,
+                unique(database_name, key)
+              );
+
+              CREATE TABLE IF NOT EXISTS datasette_metadata_resource_entries(
+                database_name text,
+                resource_name text,
+                key text,
+                value text,
+                unique(database_name, resource_name, key)
+              );
+
+              CREATE TABLE IF NOT EXISTS datasette_metadata_column_entries(
+                database_name text,
+                resource_name text,
+                column_name text,
+                key text,
+                value text,
+                unique(database_name, resource_name, column_name, key)
+              );
+            """
+    )
 
 
 async def populate_schema_tables(internal_db, db):
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 9d7a854c39..2e78b0a5a1 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -274,10 +274,6 @@ async def get(self, request):
 
         end = time.perf_counter()
         data["query_ms"] = (end - start) * 1000
-        for key in ("source", "source_url", "license", "license_url"):
-            value = self.ds.metadata(key)
-            if value:
-                data[key] = value
 
         # Special case for .jsono extension - redirect to _shape=objects
         if _format == "jsono":
@@ -385,7 +381,7 @@ async def get(self, request):
                 },
             }
             if "metadata" not in context:
-                context["metadata"] = self.ds.metadata()
+                context["metadata"] = await self.ds.get_instance_metadata()
             r = await self.render(templates, request=request, context=context)
             if status_code is not None:
                 r.status = status_code
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 851ae21fa1..2698a0ebaa 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -63,8 +63,7 @@ async def get(self, request, datasette):
         if format_ not in ("html", "json"):
             raise NotFound("Invalid format: {}".format(format_))
 
-        metadata = (datasette.metadata("databases") or {}).get(database, {})
-        datasette.update_with_inherited_metadata(metadata)
+        metadata = await datasette.get_database_metadata(database)
 
         sql_views = []
         for view_name in await db.view_names():
@@ -131,6 +130,7 @@ async def database_actions():
             "table_columns": (
                 await _table_columns(datasette, database) if allow_execute_sql else {}
             ),
+            "metadata": await datasette.get_database_metadata(database),
         }
 
         if format_ == "json":
@@ -625,8 +625,7 @@ async def fetch_data_for_csv(request, _next=None):
                     )
                 }
             )
-            metadata = (datasette.metadata("databases") or {}).get(database, {})
-            datasette.update_with_inherited_metadata(metadata)
+            metadata = await datasette.get_database_metadata(database)
 
             renderers = {}
             for key, (_, can_render) in datasette.renderers.items():
diff --git a/datasette/views/index.py b/datasette/views/index.py
index 6546b7ae83..a3178f5331 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -132,7 +132,13 @@ async def get(self, request):
             if self.ds.cors:
                 add_cors_headers(headers)
             return Response(
-                json.dumps({db["name"]: db for db in databases}, cls=CustomJSONEncoder),
+                json.dumps(
+                    {
+                        "databases": {db["name"]: db for db in databases},
+                        "metadata": await self.ds.get_instance_metadata(),
+                    },
+                    cls=CustomJSONEncoder,
+                ),
                 content_type="application/json; charset=utf-8",
                 headers=headers,
             )
@@ -151,7 +157,7 @@ async def get(self, request):
                 request=request,
                 context={
                     "databases": databases,
-                    "metadata": self.ds.metadata(),
+                    "metadata": await self.ds.get_instance_metadata(),
                     "datasette_version": __version__,
                     "private": not await self.ds.permission_allowed(
                         None, "view-instance"
diff --git a/datasette/views/row.py b/datasette/views/row.py
index 49d390f6be..6180446f1b 100644
--- a/datasette/views/row.py
+++ b/datasette/views/row.py
@@ -85,10 +85,6 @@ async def template_data():
                     "_table.html",
                 ],
                 "row_actions": row_actions,
-                "metadata": (self.ds.metadata("databases") or {})
-                .get(database, {})
-                .get("tables", {})
-                .get(table, {}),
                 "top_row": make_slot_function(
                     "top_row",
                     self.ds,
@@ -97,6 +93,7 @@ async def template_data():
                     table=resolved.table,
                     row=rows[0],
                 ),
+                "metadata": {},
             }
 
         data = {
diff --git a/datasette/views/table.py b/datasette/views/table.py
index ba03241dbc..e3bfb26060 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -147,7 +147,21 @@ async def display_columns_and_rows(
     """Returns columns, rows for specified table - including fancy foreign key treatment"""
     sortable_columns = sortable_columns or set()
     db = datasette.databases[database_name]
-    column_descriptions = datasette.metadata("columns", database_name, table_name) or {}
+    column_descriptions = dict(
+        await datasette.get_internal_database().execute(
+            """
+          SELECT
+            column_name,
+            value
+          FROM datasette_metadata_column_entries
+          WHERE database_name = ?
+            AND resource_name = ?
+            AND key = 'description'
+        """,
+            [database_name, table_name],
+        )
+    )
+
     column_details = {
         col.name: col for col in await db.table_column_details(table_name)
     }
@@ -1478,14 +1492,22 @@ async def extra_query():
 
     async def extra_metadata():
         "Metadata about the table and database"
-        metadata = (
-            (datasette.metadata("databases") or {})
-            .get(database_name, {})
-            .get("tables", {})
-            .get(table_name, {})
+        tablemetadata = await datasette.get_resource_metadata(database_name, table_name)
+
+        rows = await datasette.get_internal_database().execute(
+            """
+              SELECT
+                column_name,
+                value
+              FROM datasette_metadata_column_entries
+              WHERE database_name = ?
+                AND resource_name = ?
+                AND key = 'description'
+            """,
+            [database_name, table_name],
         )
-        datasette.update_with_inherited_metadata(metadata)
-        return metadata
+        tablemetadata["columns"] = dict(rows)
+        return tablemetadata
 
     async def extra_database():
         return database_name
diff --git a/docs/codespell-ignore-words.txt b/docs/codespell-ignore-words.txt
index 1959863f60..072f32b46d 100644
--- a/docs/codespell-ignore-words.txt
+++ b/docs/codespell-ignore-words.txt
@@ -2,4 +2,5 @@ alls
 fo
 ro
 te
-ths
\ No newline at end of file
+ths
+notin
\ No newline at end of file
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 972f3856b8..f4ef5d64d5 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -2002,6 +2002,7 @@ This example logs events to a `datasette_events` table in a database called `eve
     from datasette import hookimpl
     import json
 
+
     @hookimpl
     def startup(datasette):
         async def inner():
@@ -2031,7 +2032,11 @@ This example logs events to a `datasette_events` table in a database called `eve
                 insert into datasette_events (event_type, created, actor, properties)
                 values (?, strftime('%Y-%m-%d %H:%M:%S', 'now'), ?, ?)
             """,
-                (event.name, json.dumps(event.actor), json.dumps(properties)),
+                (
+                    event.name,
+                    json.dumps(event.actor),
+                    json.dumps(properties),
+                ),
             )
 
         return inner
diff --git a/tests/test_api.py b/tests/test_api.py
index 4ad55d7253..bd7346776c 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -29,8 +29,19 @@ async def test_homepage(ds_client):
     assert response.status_code == 200
     assert "application/json; charset=utf-8" == response.headers["content-type"]
     data = response.json()
-    assert data.keys() == {"fixtures": 0}.keys()
-    d = data["fixtures"]
+    assert sorted(list(data.get("metadata").keys())) == [
+        "about",
+        "about_url",
+        "description_html",
+        "license",
+        "license_url",
+        "source",
+        "source_url",
+        "title",
+    ]
+    databases = data.get("databases")
+    assert databases.keys() == {"fixtures": 0}.keys()
+    d = databases["fixtures"]
     assert d["name"] == "fixtures"
     assert isinstance(d["tables_count"], int)
     assert isinstance(len(d["tables_and_views_truncated"]), int)
@@ -45,7 +56,8 @@ async def test_homepage_sort_by_relationships(ds_client):
     response = await ds_client.get("/.json?_sort=relationships")
     assert response.status_code == 200
     tables = [
-        t["name"] for t in response.json()["fixtures"]["tables_and_views_truncated"]
+        t["name"]
+        for t in response.json()["databases"]["fixtures"]["tables_and_views_truncated"]
     ]
     assert tables == [
         "simple_primary_key",
@@ -590,21 +602,24 @@ def test_no_files_uses_memory_database(app_client_no_files):
     response = app_client_no_files.get("/.json")
     assert response.status == 200
     assert {
-        "_memory": {
-            "name": "_memory",
-            "hash": None,
-            "color": "a6c7b9",
-            "path": "/_memory",
-            "tables_and_views_truncated": [],
-            "tables_and_views_more": False,
-            "tables_count": 0,
-            "table_rows_sum": 0,
-            "show_table_row_counts": False,
-            "hidden_table_rows_sum": 0,
-            "hidden_tables_count": 0,
-            "views_count": 0,
-            "private": False,
-        }
+        "databases": {
+            "_memory": {
+                "name": "_memory",
+                "hash": None,
+                "color": "a6c7b9",
+                "path": "/_memory",
+                "tables_and_views_truncated": [],
+                "tables_and_views_more": False,
+                "tables_count": 0,
+                "table_rows_sum": 0,
+                "show_table_row_counts": False,
+                "hidden_table_rows_sum": 0,
+                "hidden_tables_count": 0,
+                "views_count": 0,
+                "private": False,
+            },
+        },
+        "metadata": {},
     } == response.json
     # Try that SQL query
     response = app_client_no_files.get(
@@ -768,12 +783,6 @@ def test_databases_json(app_client_two_attached_databases_one_immutable):
     assert False == fixtures_database["is_memory"]
 
 
-@pytest.mark.asyncio
-async def test_metadata_json(ds_client):
-    response = await ds_client.get("/-/metadata.json")
-    assert response.json() == ds_client.ds.metadata()
-
-
 @pytest.mark.asyncio
 async def test_threads_json(ds_client):
     response = await ds_client.get("/-/threads.json")
@@ -1039,8 +1048,8 @@ async def test_tilde_encoded_database_names(db_name):
     ds = Datasette()
     ds.add_memory_database(db_name)
     response = await ds.client.get("/.json")
-    assert db_name in response.json().keys()
-    path = response.json()[db_name]["path"]
+    assert db_name in response.json()["databases"].keys()
+    path = response.json()["databases"][db_name]["path"]
     # And the JSON for that database
     response2 = await ds.client.get(path + ".json")
     assert response2.status_code == 200
@@ -1083,6 +1092,7 @@ async def test_config_json(config, expected):
 
 
 @pytest.mark.asyncio
+@pytest.mark.skip(reason="rm?")
 @pytest.mark.parametrize(
     "metadata,expected_config,expected_metadata",
     (
diff --git a/tests/test_cli.py b/tests/test_cli.py
index bda17eed00..486852cd4b 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -159,8 +159,8 @@ def test_metadata_yaml():
         internal=None,
     )
     client = _TestClient(ds)
-    response = client.get("/-/metadata.json")
-    assert {"title": "Hello from YAML"} == response.json
+    response = client.get("/.json")
+    assert {"title": "Hello from YAML"} == response.json["metadata"]
 
 
 @mock.patch("datasette.cli.run_module")
diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py
index 66114a27db..46a6d341b4 100644
--- a/tests/test_config_dir.py
+++ b/tests/test_config_dir.py
@@ -99,12 +99,6 @@ def config_dir_client(config_dir):
     yield _TestClient(ds)
 
 
-def test_metadata(config_dir_client):
-    response = config_dir_client.get("/-/metadata.json")
-    assert 200 == response.status
-    assert METADATA == response.json
-
-
 def test_settings(config_dir_client):
     response = config_dir_client.get("/-/settings.json")
     assert 200 == response.status
@@ -149,17 +143,6 @@ def test_databases(config_dir_client):
         assert db["is_mutable"] == (expected_name != "immutable")
 
 
-@pytest.mark.parametrize("filename", ("metadata.yml", "metadata.yaml"))
-def test_metadata_yaml(tmp_path_factory, filename):
-    config_dir = tmp_path_factory.mktemp("yaml-config-dir")
-    (config_dir / filename).write_text("title: Title from metadata", "utf-8")
-    ds = Datasette([], config_dir=config_dir)
-    client = _TestClient(ds)
-    response = client.get("/-/metadata.json")
-    assert 200 == response.status
-    assert {"title": "Title from metadata"} == response.json
-
-
 def test_store_config_dir(config_dir_client):
     ds = config_dir_client.ds
 
diff --git a/tests/test_facets.py b/tests/test_facets.py
index 7634410864..023efcf092 100644
--- a/tests/test_facets.py
+++ b/tests/test_facets.py
@@ -584,9 +584,9 @@ async def test_facet_size():
     data5 = response5.json()
     assert len(data5["facet_results"]["results"]["city"]["results"]) == 20
     # Now try messing with facet_size in the table metadata
-    orig_metadata = ds._metadata_local
+    orig_config = ds.config
     try:
-        ds._metadata_local = {
+        ds.config = {
             "databases": {
                 "test_facet_size": {"tables": {"neighbourhoods": {"facet_size": 6}}}
             }
@@ -597,7 +597,7 @@ async def test_facet_size():
         data6 = response6.json()
         assert len(data6["facet_results"]["results"]["city"]["results"]) == 6
         # Setting it to max bumps it up to 50 again
-        ds._metadata_local["databases"]["test_facet_size"]["tables"]["neighbourhoods"][
+        ds.config["databases"]["test_facet_size"]["tables"]["neighbourhoods"][
             "facet_size"
         ] = "max"
         data7 = (
@@ -605,7 +605,7 @@ async def test_facet_size():
         ).json()
         assert len(data7["facet_results"]["results"]["city"]["results"]) == 20
     finally:
-        ds._metadata_local = orig_metadata
+        ds.config = orig_config
 
 
 def test_other_types_of_facet_in_metadata():
@@ -655,7 +655,6 @@ async def test_facet_against_in_memory_database():
     to_insert = [{"name": "one", "name2": "1"} for _ in range(800)] + [
         {"name": "two", "name2": "2"} for _ in range(300)
     ]
-    print(to_insert)
     await db.execute_write_many(
         "insert into t (name, name2) values (:name, :name2)", to_insert
     )
diff --git a/tests/test_html.py b/tests/test_html.py
index 42b290c86e..de732d2c47 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -446,7 +446,7 @@ async def test_database_metadata(ds_client):
         soup.find("div", {"class": "metadata-description"})
     )
     # The source/license should be inherited
-    assert_footer_links(soup)
+    # assert_footer_links(soup) TODO(alex) ensure
 
 
 @pytest.mark.asyncio
@@ -459,7 +459,7 @@ async def test_database_metadata_with_custom_sql(ds_client):
     # Description should be custom
     assert "Custom SQL query returning" in soup.find("h3").text
     # The source/license should be inherited
-    assert_footer_links(soup)
+    # assert_footer_links(soup)TODO(alex) ensure
 
 
 def test_database_download_for_immutable():
@@ -752,14 +752,6 @@ async def test_blob_download_invalid_messages(ds_client, path, expected_message)
     assert expected_message in response.text
 
 
-@pytest.mark.asyncio
-async def test_metadata_json_html(ds_client):
-    response = await ds_client.get("/-/metadata")
-    assert response.status_code == 200
-    pre = Soup(response.content, "html.parser").find("pre")
-    assert ds_client.ds.metadata() == json.loads(pre.text)
-
-
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "path",
@@ -931,7 +923,7 @@ def test_edit_sql_link_not_shown_if_user_lacks_permission(permission_allowed):
     [
         (None, None, None),
         ("test", None, ["/-/permissions"]),
-        ("root", ["/-/permissions", "/-/allow-debug", "/-/metadata"], None),
+        ("root", ["/-/permissions", "/-/allow-debug"], None),
     ],
 )
 async def test_navigation_menu_links(
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index 7279531911..fe91ec3de8 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -453,7 +453,6 @@ def view_instance_client():
         "/",
         "/fixtures",
         "/fixtures/facetable",
-        "/-/metadata",
         "/-/versions",
         "/-/plugins",
         "/-/settings",
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 9c8d02ecc4..cc2e16c83c 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -331,14 +331,14 @@ def test_hook_extra_template_vars(restore_working_directory):
     with make_app_client(
         template_dir=str(pathlib.Path(__file__).parent / "test_templates")
     ) as client:
-        response = client.get("/-/metadata")
+        response = client.get("/-/versions")
         assert response.status_code == 200
         extra_template_vars = json.loads(
             Soup(response.text, "html.parser").select("pre.extra_template_vars")[0].text
         )
         assert {
             "template": "show_json.html",
-            "scope_path": "/-/metadata",
+            "scope_path": "/-/versions",
             "columns": None,
         } == extra_template_vars
         extra_template_vars_from_awaitable = json.loads(
@@ -349,7 +349,7 @@ def test_hook_extra_template_vars(restore_working_directory):
         assert {
             "template": "show_json.html",
             "awaitable": True,
-            "scope_path": "/-/metadata",
+            "scope_path": "/-/versions",
         } == extra_template_vars_from_awaitable
 
 
@@ -357,7 +357,7 @@ def test_plugins_async_template_function(restore_working_directory):
     with make_app_client(
         template_dir=str(pathlib.Path(__file__).parent / "test_templates")
     ) as client:
-        response = client.get("/-/metadata")
+        response = client.get("/-/versions")
         assert response.status_code == 200
         extra_from_awaitable_function = (
             Soup(response.text, "html.parser")
@@ -422,7 +422,7 @@ def extra_template_vars(view_name):
         ("/fixtures", "database"),
         ("/fixtures/units", "table"),
         ("/fixtures/units/1", "row"),
-        ("/-/metadata", "json_data"),
+        ("/-/versions", "json_data"),
         ("/fixtures?sql=select+1", "database"),
     ),
 )
@@ -1073,36 +1073,6 @@ def test_hook_skip_csrf(app_client):
     assert second_missing_csrf_response.status_code == 403
 
 
-@pytest.mark.asyncio
-async def test_hook_get_metadata(ds_client):
-    try:
-        orig_metadata = ds_client.ds._metadata_local
-        ds_client.ds._metadata_local = {
-            "title": "Testing get_metadata hook!",
-            "databases": {"from-local": {"title": "Hello from local metadata"}},
-        }
-        og_pm_hook_get_metadata = pm.hook.get_metadata
-
-        def get_metadata_mock(*args, **kwargs):
-            return [
-                {
-                    "databases": {
-                        "from-hook": {"title": "Hello from the plugin hook"},
-                        "from-local": {"title": "This will be overwritten!"},
-                    }
-                }
-            ]
-
-        pm.hook.get_metadata = get_metadata_mock
-        meta = ds_client.ds.metadata()
-        assert "Testing get_metadata hook!" == meta["title"]
-        assert "Hello from local metadata" == meta["databases"]["from-local"]["title"]
-        assert "Hello from the plugin hook" == meta["databases"]["from-hook"]["title"]
-        pm.hook.get_metadata = og_pm_hook_get_metadata
-    finally:
-        ds_client.ds._metadata_local = orig_metadata
-
-
 def _extract_commands(output):
     lines = output.split("Commands:\n", 1)[1].split("\n")
     return {line.split()[0].replace("*", "") for line in lines if line.strip()}
@@ -1550,6 +1520,7 @@ async def test_hook_register_events():
     assert any(k.__name__ == "OneEvent" for k in datasette.event_classes)
 
 
+@pytest.mark.skip(reason="TODO")
 @pytest.mark.parametrize(
     "metadata,config,expected_metadata,expected_config",
     (
diff --git a/tests/test_routes.py b/tests/test_routes.py
index 85945decf0..b2d317598a 100644
--- a/tests/test_routes.py
+++ b/tests/test_routes.py
@@ -43,8 +43,6 @@ def routes():
             "RowView",
             {"format": "json", "database": "foo", "pks": "1", "table": "humbug"},
         ),
-        ("/-/metadata.json", "JsonDataView", {"format": "json"}),
-        ("/-/metadata", "JsonDataView", {"format": None}),
     ),
 )
 def test_routes(routes, path, expected_name, expected_matches):
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index 2a6586636c..fcf914a6d9 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -792,8 +792,6 @@ async def test_table_metadata(ds_client):
     assert "Simple <em>primary</em> key" == inner_html(
         soup.find("div", {"class": "metadata-description"})
     )
-    # The source/license should be inherited
-    assert_footer_links(soup)
 
 
 @pytest.mark.asyncio
@@ -1101,8 +1099,8 @@ async def test_column_metadata(ds_client):
     soup = Soup(response.text, "html.parser")
     dl = soup.find("dl")
     assert [(dt.text, dt.nextSibling.text) for dt in dl.findAll("dt")] == [
-        ("name", "The name of the attraction"),
         ("address", "The street address for the attraction"),
+        ("name", "The name of the attraction"),
     ]
     assert (
         soup.select("th[data-column=name]")[0]["data-column-description"]

From c698d008e08396ca0d288febbb9f656d96db83a9 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 11 Jun 2024 10:03:57 -0700
Subject: [PATCH 03/48] Only test first wheel, fixes surprise bug

https://github.com/simonw/datasette/issues/2351#issuecomment-2161211173
---
 test-in-pyodide-with-shot-scraper.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh
index 0c140818f4..acd9346595 100755
--- a/test-in-pyodide-with-shot-scraper.sh
+++ b/test-in-pyodide-with-shot-scraper.sh
@@ -6,7 +6,7 @@ set -e
 python3 -m build
 
 # Find name of wheel, strip off the dist/
-wheel=$(basename $(ls dist/*.whl))
+wheel=$(basename $(ls dist/*.whl) | head -n 1)
 
 # Create a blank index page
 echo '

From 9a3c3bfcc7d30d65514421bd315f1b8b5b735b86 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 11 Jun 2024 10:11:34 -0700
Subject: [PATCH 04/48] Fix for pyodide test failure, refs #2351

---
 test-in-pyodide-with-shot-scraper.sh | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh
index acd9346595..5845a0c3fd 100755
--- a/test-in-pyodide-with-shot-scraper.sh
+++ b/test-in-pyodide-with-shot-scraper.sh
@@ -18,6 +18,13 @@ cd dist
 python3 -m http.server 8529 &
 cd ..
 
+# Register the kill_server function to be called on script exit
+kill_server() {
+  pkill -f 'http.server 8529'
+}
+trap kill_server EXIT
+
+
 shot-scraper javascript http://localhost:8529/ "
 async () => {
   let pyodide = await loadPyodide();
@@ -26,6 +33,8 @@ async () => {
     import micropip
     await micropip.install('h11==0.12.0')
     await micropip.install('httpx==0.23')
+    # To avoid 'from typing_extensions import deprecated' error:
+    await micropip.install('typing-extensions>=4.12.2')
     await micropip.install('http://localhost:8529/$wheel')
     import ssl
     import setuptools
@@ -38,7 +47,4 @@ async () => {
   }
   return 'Test passed!';
 }
-"
-
-# Shut down the server
-pkill -f 'http.server 8529'
+"
\ No newline at end of file

From 7437d40e5dd4d614bb769e16c0c1b96c6c19647f Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 11 Jun 2024 10:17:02 -0700
Subject: [PATCH 05/48] <html lang="en">, closes #2348

---
 datasette/templates/base.html     | 2 +-
 datasette/templates/patterns.html | 2 +-
 tests/test_html.py                | 2 ++
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/datasette/templates/base.html b/datasette/templates/base.html
index afb8dcd3da..51f602b66e 100644
--- a/datasette/templates/base.html
+++ b/datasette/templates/base.html
@@ -1,5 +1,5 @@
 {% import "_crumbs.html" as crumbs with context %}<!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
     <title>{% block title %}{% endblock %}</title>
     <link rel="stylesheet" href="{{ urls.static('app.css') }}?{{ app_css_hash }}">
diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html
index cb0daf9a8e..24fe55314e 100644
--- a/datasette/templates/patterns.html
+++ b/datasette/templates/patterns.html
@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
     <title>Datasette: Pattern Portfolio</title>
     <link rel="stylesheet" href="{{ base_url }}-/static/app.css?{{ app_css_hash }}">
diff --git a/tests/test_html.py b/tests/test_html.py
index de732d2c47..a83f0e47db 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -21,6 +21,8 @@ def test_homepage(app_client_two_attached_databases):
     response = app_client_two_attached_databases.get("/")
     assert response.status_code == 200
     assert "text/html; charset=utf-8" == response.headers["content-type"]
+    # Should have a html lang="en" attribute
+    assert '<html lang="en">' in response.text
     soup = Soup(response.content, "html.parser")
     assert "Datasette Fixtures" == soup.find("h1").text
     assert (

From 2b6bfddafc1f3ad95bbb334e34a026964db4a813 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 11 Jun 2024 14:04:55 -0700
Subject: [PATCH 06/48] Workaround for #2353

---
 datasette/utils/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index e110891119..8075420278 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1146,7 +1146,7 @@ async def derive_named_parameters(db: "Database", sql: str) -> List[str]:
     try:
         results = await db.execute(explain, {p: None for p in possible_params})
         return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"]
-    except sqlite3.DatabaseError:
+    except (sqlite3.DatabaseError, AttributeError):
         return possible_params
 
 

From 780deaa275849a39a283ec7b57a0351aac18a047 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 12 Jun 2024 16:12:05 -0700
Subject: [PATCH 07/48] Reminder about how to deploy a release branch

---
 docs/contributing.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index b678e6379d..45330a8347 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -284,6 +284,10 @@ You can generate the list of issue references for a specific release by copying
 
 To create the tag for the release, create `a new release <https://github.com/simonw/datasette/releases/new>`__ on GitHub matching the new version number. You can convert the release notes to Markdown by copying and pasting the rendered HTML into this `Paste to Markdown tool <https://euangoddard.github.io/clipboard2markdown/>`__.
 
+Don't forget to create the release from the correct branch - usually ``main``, but sometimes ``0.64.x`` or similar for a bugfix release.
+
+While the release is running you can confirm that the correct commits made it into the release using the https://github.com/simonw/datasette/compare/0.64.6...0.64.7 URL.
+
 Finally, post a news item about the release on `datasette.io <https://datasette.io/>`__ by editing the `news.yaml <https://github.com/simonw/datasette.io/blob/main/news.yaml>`__ file in that site's repository.
 
 .. _contributing_alpha_beta:

From b39b01a89066b1322bb9f731529e636d707e19f6 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 12 Jun 2024 16:21:07 -0700
Subject: [PATCH 08/48] Copy across release notes from 0.64.7

Refs #2353
---
 docs/changelog.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index ebed499fef..90bf75e1d6 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,13 @@
 Changelog
 =========
 
+.. _v0_64_7:
+
+0.64.7 (2023-06-12)
+-------------------
+
+- Fixed a bug where canned queries with named parameters threw an error when run against SQLite 3.46.0. (:issue:`2353`)
+
 .. _v1_0_a13:
 
 1.0a13 (2024-03-12)

From d118d5c5bbb675f35baa5c376ee297b510dccfc0 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 12 Jun 2024 16:51:07 -0700
Subject: [PATCH 09/48] named_parameters(sql) sync function, refs #2354

Also refs #2353 and #2352
---
 datasette/utils/__init__.py | 33 ++++++++++++++++++++++++---------
 datasette/views/database.py |  6 ++----
 docs/internals.rst          | 10 +++++-----
 tests/test_utils.py         |  8 ++++++--
 4 files changed, 37 insertions(+), 20 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 8075420278..b3b51c5fa7 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1131,23 +1131,38 @@ class StartupError(Exception):
     pass
 
 
-_re_named_parameter = re.compile(":([a-zA-Z0-9_]+)")
+_single_line_comment_re = re.compile(r"--.*")
+_multi_line_comment_re = re.compile(r"/\*.*?\*/", re.DOTALL)
+_single_quote_re = re.compile(r"'(?:''|[^'])*'")
+_double_quote_re = re.compile(r'"(?:\"\"|[^"])*"')
+_named_param_re = re.compile(r":(\w+)")
 
 
 @documented
-async def derive_named_parameters(db: "Database", sql: str) -> List[str]:
+def named_parameters(sql: str) -> List[str]:
     """
     Given a SQL statement, return a list of named parameters that are used in the statement
 
     e.g. for ``select * from foo where id=:id`` this would return ``["id"]``
     """
-    explain = "explain {}".format(sql.strip().rstrip(";"))
-    possible_params = _re_named_parameter.findall(sql)
-    try:
-        results = await db.execute(explain, {p: None for p in possible_params})
-        return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"]
-    except (sqlite3.DatabaseError, AttributeError):
-        return possible_params
+    # Remove single-line comments
+    sql = _single_line_comment_re.sub("", sql)
+    # Remove multi-line comments
+    sql = _multi_line_comment_re.sub("", sql)
+    # Remove single-quoted strings
+    sql = _single_quote_re.sub("", sql)
+    # Remove double-quoted strings
+    sql = _double_quote_re.sub("", sql)
+    # Extract parameters from what is left
+    return _named_param_re.findall(sql)
+
+
+async def derive_named_parameters(db: "Database", sql: str) -> List[str]:
+    """
+    This undocumented but stable method exists for backwards compatibility
+    with plugins that were using it before it switched to named_parameters()
+    """
+    return named_parameters(sql)
 
 
 def add_cors_headers(headers):
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 2698a0ebaa..1d76c5e047 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -17,7 +17,7 @@
     add_cors_headers,
     await_me_maybe,
     call_with_supported_arguments,
-    derive_named_parameters,
+    named_parameters as derive_named_parameters,
     format_bytes,
     make_slot_function,
     tilde_decode,
@@ -484,9 +484,7 @@ async def get(self, request, datasette):
         if canned_query and canned_query.get("params"):
             named_parameters = canned_query["params"]
         if not named_parameters:
-            named_parameters = await derive_named_parameters(
-                datasette.get_database(database), sql
-            )
+            named_parameters = derive_named_parameters(sql)
         named_parameter_values = {
             named_parameter: params.get(named_parameter) or ""
             for named_parameter in named_parameters
diff --git a/docs/internals.rst b/docs/internals.rst
index 795856594d..38e66a57b7 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -1256,14 +1256,14 @@ Utility function for calling ``await`` on a return value if it is awaitable, oth
 
 .. autofunction:: datasette.utils.await_me_maybe
 
-.. _internals_utils_derive_named_parameters:
+.. _internals_utils_named_parameters:
 
-derive_named_parameters(db, sql)
---------------------------------
+named_parameters(sql)
+---------------------
 
-Derive the list of named parameters referenced in a SQL query, using an ``explain`` query executed against the provided database.
+Derive the list of ``:named`` parameters referenced in a SQL query.
 
-.. autofunction:: datasette.utils.derive_named_parameters
+.. autofunction:: datasette.utils.named_parameters
 
 .. _internals_tilde_encoding:
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 254b130055..88a4532ad1 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -612,10 +612,14 @@ def test_parse_metadata(content, expected):
         ("select this is invalid :one, :two, :three", ["one", "two", "three"]),
     ),
 )
-async def test_derive_named_parameters(sql, expected):
+@pytest.mark.parametrize("use_async_version", (False, True))
+async def test_named_parameters(sql, expected, use_async_version):
     ds = Datasette([], memory=True)
     db = ds.get_database("_memory")
-    params = await utils.derive_named_parameters(db, sql)
+    if use_async_version:
+        params = await utils.derive_named_parameters(db, sql)
+    else:
+        params = utils.named_parameters(sql)
     assert params == expected
 
 

From 64a125b860eda6f821ce5a67f48b13fd173d8671 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 12 Jun 2024 16:56:46 -0700
Subject: [PATCH 10/48] Removed unnecessary comments, refs #2354

---
 datasette/utils/__init__.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index b3b51c5fa7..c87de5f003 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1145,13 +1145,9 @@ def named_parameters(sql: str) -> List[str]:
 
     e.g. for ``select * from foo where id=:id`` this would return ``["id"]``
     """
-    # Remove single-line comments
     sql = _single_line_comment_re.sub("", sql)
-    # Remove multi-line comments
     sql = _multi_line_comment_re.sub("", sql)
-    # Remove single-quoted strings
     sql = _single_quote_re.sub("", sql)
-    # Remove double-quoted strings
     sql = _double_quote_re.sub("", sql)
     # Extract parameters from what is left
     return _named_param_re.findall(sql)

From 8f86d2af6a5b90bd0bf12812b4f37c0fbfc1bceb Mon Sep 17 00:00:00 2001
From: Alex Garcia <alexsebastian.garcia@gmail.com>
Date: Thu, 13 Jun 2024 10:09:45 -0700
Subject: [PATCH 11/48] Test against multiple SQLite versions (#2352)

* Use sqlite-versions action for testing multiple versions
---
 .github/workflows/test-sqlite-support.yml | 53 +++++++++++++++++++++++
 tests/test_csv.py                         |  1 +
 2 files changed, 54 insertions(+)
 create mode 100644 .github/workflows/test-sqlite-support.yml

diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml
new file mode 100644
index 0000000000..7882e05d54
--- /dev/null
+++ b/.github/workflows/test-sqlite-support.yml
@@ -0,0 +1,53 @@
+name: Test SQLite versions
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ${{ matrix.platform }}
+    continue-on-error: true
+    strategy:
+      matrix:
+        platform: [ubuntu-latest]
+        python-version: [  "3.8", "3.9", "3.10", "3.11", "3.12"]
+        sqlite-version: [
+          #"3", # latest version
+          "3.46",
+          #"3.45",
+          #"3.27",
+          #"3.26",
+          "3.25",
+          #"3.25.3", # 2018-09-25, window functions breaks test_upsert for some reason on 3.10, skip for now
+          #"3.24", # 2018-06-04, added UPSERT support
+          #"3.23.1" # 2018-04-10, before UPSERT
+        ]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        allow-prereleases: true
+        cache: pip
+        cache-dependency-path: setup.py
+    - name: Set up SQLite ${{ matrix.sqlite-version }}
+      uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6
+      with:
+        version: ${{ matrix.sqlite-version }}
+        cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1"
+    - run: python3 -c "import sqlite3; print(sqlite3.sqlite_version)"
+    - run: echo $LD_LIBRARY_PATH
+    - name: Build extension for --load-extension test
+      run: |-
+        (cd tests && gcc ext.c -fPIC -shared -o ext.so)
+    - name: Install dependencies
+      run: |
+        pip install -e '.[test]'
+        pip freeze
+    - name: Run tests
+      run: |
+        pytest -n auto -m "not serial"
+        pytest -m "serial"
diff --git a/tests/test_csv.py b/tests/test_csv.py
index 9f772f8985..d4c072c050 100644
--- a/tests/test_csv.py
+++ b/tests/test_csv.py
@@ -189,6 +189,7 @@ async def test_csv_with_non_ascii_characters(ds_client):
     assert response.text == "text,number\r\n𝐜𝐢𝐭𝐢𝐞𝐬,1\r\nbob,2\r\n"
 
 
+@pytest.mark.skip(reason="flakey")
 def test_max_csv_mb(app_client_csv_max_mb_one):
     # This query deliberately generates a really long string
     # should be 100*100*100*2 = roughly 2MB

From 45c27603d2b1f3536eccee7bdf20b9626a128bb3 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jun 2024 10:15:38 -0700
Subject: [PATCH 12/48] xfail two flaky tests, #2355, #2356

---
 tests/test_api_write.py | 1 +
 tests/test_csv.py       | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_api_write.py b/tests/test_api_write.py
index 6a7ddeb61d..2b57a99a0f 100644
--- a/tests/test_api_write.py
+++ b/tests/test_api_write.py
@@ -1359,6 +1359,7 @@ async def test_create_table_permissions(
 
 
 @pytest.mark.asyncio
+@pytest.mark.xfail(reason="Flaky, see https://github.com/simonw/datasette/issues/2356")
 @pytest.mark.parametrize(
     "input,expected_rows_after",
     (
diff --git a/tests/test_csv.py b/tests/test_csv.py
index d4c072c050..6dbe693b64 100644
--- a/tests/test_csv.py
+++ b/tests/test_csv.py
@@ -189,7 +189,7 @@ async def test_csv_with_non_ascii_characters(ds_client):
     assert response.text == "text,number\r\n𝐜𝐢𝐭𝐢𝐞𝐬,1\r\nbob,2\r\n"
 
 
-@pytest.mark.skip(reason="flakey")
+@pytest.mark.xfail(reason="Flaky, see https://github.com/simonw/datasette/issues/2355")
 def test_max_csv_mb(app_client_csv_max_mb_one):
     # This query deliberately generates a really long string
     # should be 100*100*100*2 = roughly 2MB

From 93534fd3d0c34aac9ff5f8677f1a8d8bddf886f9 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 13 Jun 2024 10:19:26 -0700
Subject: [PATCH 13/48] Show response.text on test_upsert failure, refs #2356

---
 tests/test_api_write.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_api_write.py b/tests/test_api_write.py
index 2b57a99a0f..7dbe12b555 100644
--- a/tests/test_api_write.py
+++ b/tests/test_api_write.py
@@ -537,7 +537,7 @@ async def test_upsert(ds_write, initial, input, expected_rows, should_return):
         json=input,
         headers=_headers(token),
     )
-    assert response.status_code == 200
+    assert response.status_code == 200, response.text
     assert response.json()["ok"] is True
 
     # Analytics event

From 62686114eee429d6f5e6badc4f11076b35c987df Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 21 Jun 2024 16:02:15 -0700
Subject: [PATCH 14/48] Do not show database name in Database Not Found error,
 refs #2359

---
 datasette/app.py        | 4 +---
 datasette/utils/asgi.py | 4 ++--
 tests/test_api_write.py | 2 +-
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 8020c5dac2..f732f95c0a 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1617,9 +1617,7 @@ async def resolve_database(self, request):
         try:
             return self.get_database(route=database_route)
         except KeyError:
-            raise DatabaseNotFound(
-                "Database not found: {}".format(database_route), database_route
-            )
+            raise DatabaseNotFound(database_route)
 
     async def resolve_table(self, request):
         db = await self.resolve_database(request)
diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index 2fad1d425b..b3f00bb3c5 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -23,9 +23,9 @@ class NotFound(Base400):
 
 
 class DatabaseNotFound(NotFound):
-    def __init__(self, message, database_name):
-        super().__init__(message)
+    def __init__(self, database_name):
         self.database_name = database_name
+        super().__init__("Database not found")
 
 
 class TableNotFound(NotFound):
diff --git a/tests/test_api_write.py b/tests/test_api_write.py
index 7dbe12b555..2cd8785808 100644
--- a/tests/test_api_write.py
+++ b/tests/test_api_write.py
@@ -133,7 +133,7 @@ async def test_insert_rows(ds_write, return_rows):
             {},
             None,
             404,
-            ["Database not found: data2"],
+            ["Database not found"],
         ),
         (
             "/data/docs2/-/insert",

From 7316dd4ac629d3b0e93a92b07cd6e565e2a66a07 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 21 Jun 2024 16:09:20 -0700
Subject: [PATCH 15/48] Fix for TableNotFound, refs #2359

---
 datasette/app.py         |  4 +---
 datasette/utils/asgi.py  |  4 ++--
 tests/test_api_write.py  | 10 +++++-----
 tests/test_table_api.py  |  2 +-
 tests/test_table_html.py |  2 +-
 5 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index f732f95c0a..001cf4c366 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1628,9 +1628,7 @@ async def resolve_table(self, request):
         if not table_exists:
             is_view = await db.view_exists(table_name)
         if not (table_exists or is_view):
-            raise TableNotFound(
-                "Table not found: {}".format(table_name), db.name, table_name
-            )
+            raise TableNotFound(db.name, table_name)
         return ResolvedTable(db, table_name, is_view)
 
     async def resolve_row(self, request):
diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index b3f00bb3c5..6bdba714e4 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -29,8 +29,8 @@ def __init__(self, database_name):
 
 
 class TableNotFound(NotFound):
-    def __init__(self, message, database_name, table):
-        super().__init__(message)
+    def __init__(self, database_name, table):
+        super().__init__("Table not found")
         self.database_name = database_name
         self.table = table
 
diff --git a/tests/test_api_write.py b/tests/test_api_write.py
index 2cd8785808..94eb902b2b 100644
--- a/tests/test_api_write.py
+++ b/tests/test_api_write.py
@@ -140,7 +140,7 @@ async def test_insert_rows(ds_write, return_rows):
             {},
             None,
             404,
-            ["Table not found: docs2"],
+            ["Table not found"],
         ),
         (
             "/data/docs/-/insert",
@@ -274,7 +274,7 @@ async def test_insert_rows(ds_write, return_rows):
             {"rows": [{"title": "Test"}]},
             None,
             404,
-            ["Table not found: badtable"],
+            ["Table not found"],
         ),
         # missing primary key
         (
@@ -598,7 +598,7 @@ async def test_delete_row_errors(ds_write, scenario):
     assert (
         response.json()["errors"] == ["Permission denied"]
         if scenario == "no_token"
-        else ["Table not found: bad_table"]
+        else ["Table not found"]
     )
     assert len((await ds_write.client.get("/data/docs.json?_shape=array")).json()) == 1
 
@@ -703,7 +703,7 @@ async def test_update_row_check_permission(ds_write, scenario):
     assert (
         response.json()["errors"] == ["Permission denied"]
         if scenario == "no_token"
-        else ["Table not found: bad_table"]
+        else ["Table not found"]
     )
 
 
@@ -830,7 +830,7 @@ async def test_drop_table(ds_write, scenario):
         assert response.json()["ok"] is False
         expected_error = "Permission denied"
         if scenario == "bad_table":
-            expected_error = "Table not found: bad_table"
+            expected_error = "Table not found"
         elif scenario == "immutable":
             expected_error = "Database is immutable"
         assert response.json()["errors"] == [expected_error]
diff --git a/tests/test_table_api.py b/tests/test_table_api.py
index 5893095045..8360157d7b 100644
--- a/tests/test_table_api.py
+++ b/tests/test_table_api.py
@@ -36,7 +36,7 @@ async def test_table_json(ds_client):
 async def test_table_not_exists_json(ds_client):
     assert (await ds_client.get("/fixtures/blah.json")).json() == {
         "ok": False,
-        "error": "Table not found: blah",
+        "error": "Table not found",
         "status": 404,
         "title": None,
     }
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index fcf914a6d9..1a9ed5404c 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -535,7 +535,7 @@ async def test_csv_json_export_links_include_labels_if_foreign_keys(ds_client):
 
 @pytest.mark.asyncio
 async def test_table_not_exists(ds_client):
-    assert "Table not found: blah" in (await ds_client.get("/fixtures/blah")).text
+    assert "Table not found" in (await ds_client.get("/fixtures/blah")).text
 
 
 @pytest.mark.asyncio

From 263788906aad8d53cb29369983f72a3b861bb4da Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 21 Jun 2024 16:10:16 -0700
Subject: [PATCH 16/48] Fix for RowNotFound, refs #2359

---
 datasette/app.py        | 4 +---
 datasette/utils/asgi.py | 4 ++--
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 001cf4c366..1c5d6a3727 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1638,9 +1638,7 @@ async def resolve_row(self, request):
         results = await db.execute(sql, params, truncate=True)
         row = results.first()
         if row is None:
-            raise RowNotFound(
-                "Row not found: {}".format(pk_values), db.name, table_name, pk_values
-            )
+            raise RowNotFound(db.name, table_name, pk_values)
         return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first())
 
     def app(self):
diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index 6bdba714e4..1699847e16 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -36,8 +36,8 @@ def __init__(self, database_name, table):
 
 
 class RowNotFound(NotFound):
-    def __init__(self, message, database_name, table, pk_values):
-        super().__init__(message)
+    def __init__(self, database_name, table, pk_values):
+        super().__init__("Row not found")
         self.database_name = database_name
         self.table_name = table
         self.pk_values = pk_values

From c2e8e5085b5ccc49121feaf11b57eb7d5af53bd9 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 21 Jun 2024 16:36:58 -0700
Subject: [PATCH 17/48] Release notes for 0.64.8 on main

---
 docs/changelog.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 90bf75e1d6..dab17bc64a 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,14 @@
 Changelog
 =========
 
+.. _v0_64_8:
+
+0.64.8 (2023-06-21)
+-------------------
+
+- Security improvement: 404 pages used to reflect content from the URL path, which could be used to display misleading information to Datasette users. 404 errors no longer display additional information from the URL. (:issue:`2359`)
+- Backported a better fix for correctly extracting named parameters from canned query SQL against SQLite 3.46.0. (:issue:`2353`)
+
 .. _v0_64_7:
 
 0.64.7 (2023-06-12)

From 56adfff8d2112c5890888af4da65d842ba7f6f86 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 2 Jul 2024 09:45:15 -0700
Subject: [PATCH 18/48] Bump the python-packages group across 1 directory with
 4 updates (#2362)

Bumps the python-packages group with 4 updates in the / directory: [sphinx](https://github.com/sphinx-doc/sphinx), [furo](https://github.com/pradyunsg/furo), [blacken-docs](https://github.com/adamchainz/blacken-docs) and [black](https://github.com/psf/black).


Updates `sphinx` from 7.2.6 to 7.3.7
- [Release notes](https://github.com/sphinx-doc/sphinx/releases)
- [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES.rst)
- [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.6...v7.3.7)

Updates `furo` from 2024.1.29 to 2024.5.6
- [Release notes](https://github.com/pradyunsg/furo/releases)
- [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md)
- [Commits](https://github.com/pradyunsg/furo/compare/2024.01.29...2024.05.06)

Updates `blacken-docs` from 1.16.0 to 1.18.0
- [Changelog](https://github.com/adamchainz/blacken-docs/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/adamchainz/blacken-docs/compare/1.16.0...1.18.0)

Updates `black` from 24.2.0 to 24.4.2
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/compare/24.2.0...24.4.2)

---
updated-dependencies:
- dependency-name: sphinx
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: python-packages
- dependency-name: furo
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: python-packages
- dependency-name: blacken-docs
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: python-packages
- dependency-name: black
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: python-packages
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/setup.py b/setup.py
index e51d824743..fa61729ff0 100644
--- a/setup.py
+++ b/setup.py
@@ -70,8 +70,8 @@ def get_version():
     """,
     extras_require={
         "docs": [
-            "Sphinx==7.2.6",
-            "furo==2024.1.29",
+            "Sphinx==7.3.7",
+            "furo==2024.5.6",
             "sphinx-autobuild",
             "codespell>=2.2.5",
             "blacken-docs",
@@ -84,8 +84,8 @@ def get_version():
             "pytest-xdist>=2.2.1",
             "pytest-asyncio>=0.17",
             "beautifulsoup4>=4.8.1",
-            "black==24.2.0",
-            "blacken-docs==1.16.0",
+            "black==24.4.2",
+            "blacken-docs==1.18.0",
             "pytest-timeout>=1.4.2",
             "trustme>=0.7",
             "cogapp>=3.3.0",

From a23c2aee006bea5f50c13e687dac77c66df1888b Mon Sep 17 00:00:00 2001
From: Alex Garcia <alexsebastian.garcia@gmail.com>
Date: Mon, 15 Jul 2024 10:33:51 -0700
Subject: [PATCH 19/48] Introduce new `/$DB/-/query` endpoint, soft replaces
 `/$DB?sql=...` (#2363)

* Introduce new default /$DB/-/query endpoint
* Fix a lot of tests
* Update pyodide test to use query endpoint
* Link to /fixtures/-/query in a few places
* Documentation for QueryView

---------

Co-authored-by: Simon Willison <swillison@gmail.com>
---
 datasette/app.py                     |  6 ++-
 datasette/templates/database.html    |  4 +-
 datasette/views/database.py          |  8 ++++
 docs/pages.rst                       | 15 +++++++
 test-in-pyodide-with-shot-scraper.sh |  4 +-
 tests/plugins/my_plugin.py           |  1 +
 tests/test_api.py                    | 38 ++++++++++++------
 tests/test_api_write.py              | 12 ++++--
 tests/test_canned_queries.py         |  2 +-
 tests/test_cli.py                    |  8 ++--
 tests/test_cli_serve_get.py          |  2 +-
 tests/test_crossdb.py                |  6 ++-
 tests/test_csv.py                    | 10 ++---
 tests/test_html.py                   | 59 ++++++++++++++++------------
 tests/test_load_extensions.py        | 12 +++---
 tests/test_messages.py               |  4 +-
 tests/test_permissions.py            |  8 ++--
 tests/test_plugins.py                | 22 ++++++-----
 tests/test_routes.py                 |  2 +-
 tests/test_table_api.py              |  4 +-
 tests/test_table_html.py             |  4 +-
 21 files changed, 148 insertions(+), 83 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 1c5d6a3727..5b8f910ce3 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -37,7 +37,7 @@
 from .events import Event
 from .views import Context
 from .views.base import ureg
-from .views.database import database_download, DatabaseView, TableCreateView
+from .views.database import database_download, DatabaseView, TableCreateView, QueryView
 from .views.index import IndexView
 from .views.special import (
     JsonDataView,
@@ -1578,6 +1578,10 @@ def add_route(view, regex):
             r"/(?P<database>[^\/\.]+)(\.(?P<format>\w+))?$",
         )
         add_route(TableCreateView.as_view(self), r"/(?P<database>[^\/\.]+)/-/create$")
+        add_route(
+            wrap_view(QueryView, self),
+            r"/(?P<database>[^\/\.]+)/-/query(\.(?P<format>\w+))?$",
+        )
         add_route(
             wrap_view(table_view, self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)(\.(?P<format>\w+))?$",
diff --git a/datasette/templates/database.html b/datasette/templates/database.html
index 4a4a05cbf2..f921bc2dbc 100644
--- a/datasette/templates/database.html
+++ b/datasette/templates/database.html
@@ -21,7 +21,7 @@ <h1>{{ metadata.title or database }}{% if private %} 🔒{% endif %}</h1>
 {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
 
 {% if allow_execute_sql %}
-    <form class="sql" action="{{ urls.database(database) }}" method="get">
+    <form class="sql" action="{{ urls.database(database) }}/-/query" method="get">
         <h3>Custom SQL query</h3>
         <p><textarea id="sql-editor" name="sql">{% if tables %}select * from {{ tables[0].name|escape_sqlite }}{% else %}select sqlite_version(){% endif %}</textarea></p>
         <p>
@@ -36,7 +36,7 @@ <h3>Custom SQL query</h3>
         <p>The following databases are attached to this connection, and can be used for cross-database joins:</p>
         <ul class="bullets">
             {% for db_name in attached_databases %}
-                <li><strong>{{ db_name }}</strong> - <a href="?sql=select+*+from+[{{ db_name }}].sqlite_master+where+type='table'">tables</a></li>
+                <li><strong>{{ db_name }}</strong> - <a href="{{ urls.database(db_name) }}/-/query?sql=select+*+from+[{{ db_name }}].sqlite_master+where+type='table'">tables</a></li>
             {% endfor %}
         </ul>
     </div>
diff --git a/datasette/views/database.py b/datasette/views/database.py
index 1d76c5e047..9ab061a106 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -58,6 +58,11 @@ async def get(self, request, datasette):
 
         sql = (request.args.get("sql") or "").strip()
         if sql:
+            redirect_url = "/" + request.url_vars.get("database") + "/-/query"
+            if request.url_vars.get("format"):
+                redirect_url += "." + request.url_vars.get("format")
+            redirect_url += "?" + request.query_string
+            return Response.redirect(redirect_url)
             return await QueryView()(request, datasette)
 
         if format_ not in ("html", "json"):
@@ -433,6 +438,8 @@ async def post(self, request, datasette):
     async def get(self, request, datasette):
         from datasette.app import TableNotFound
 
+        await datasette.refresh_schemas()
+
         db = await datasette.resolve_database(request)
         database = db.name
 
@@ -686,6 +693,7 @@ async def fetch_data_for_csv(request, _next=None):
             if allow_execute_sql and is_validated_sql and ":_" not in sql:
                 edit_sql_url = (
                     datasette.urls.database(database)
+                    + "/-/query"
                     + "?"
                     + urlencode(
                         {
diff --git a/docs/pages.rst b/docs/pages.rst
index 1c3e2c1e6e..239c9f8027 100644
--- a/docs/pages.rst
+++ b/docs/pages.rst
@@ -55,6 +55,21 @@ The following tables are hidden by default:
 - Tables relating to the inner workings of the SpatiaLite SQLite extension.
 - ``sqlite_stat`` tables used to store statistics used by the query optimizer.
 
+.. _QueryView:
+
+Queries
+=======
+
+The ``/database-name/-/query`` page can be used to execute an arbitrary SQL query against that database, if the :ref:`permissions_execute_sql` permission is enabled. This query is passed as the ``?sql=`` query string parameter.
+
+This means you can link directly to a query by constructing the following URL:
+
+``/database-name/-/query?sql=SELECT+*+FROM+table_name``
+
+Each configured :ref:`canned query <canned_queries>` has its own page, at ``/database-name/query-name``. Viewing this page will execute the query and display the results.
+
+In both cases adding a ``.json`` extension to the URL will return the results as JSON.
+
 .. _TableView:
 
 Table
diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh
index 5845a0c3fd..4d1c496857 100755
--- a/test-in-pyodide-with-shot-scraper.sh
+++ b/test-in-pyodide-with-shot-scraper.sh
@@ -40,11 +40,11 @@ async () => {
     import setuptools
     from datasette.app import Datasette
     ds = Datasette(memory=True, settings={'num_sql_threads': 0})
-    (await ds.client.get('/_memory.json?sql=select+55+as+itworks&_shape=array')).text
+    (await ds.client.get('/_memory/-/query.json?sql=select+55+as+itworks&_shape=array')).text
   \`);
   if (JSON.parse(output)[0].itworks != 55) {
     throw 'Got ' + output + ', expected itworks: 55';
   }
   return 'Test passed!';
 }
-"
\ No newline at end of file
+"
diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index 9ef10181d2..4ca4f989a0 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -411,6 +411,7 @@ def query_actions(datasette, database, query_name, sql):
     return [
         {
             "href": datasette.urls.database(database)
+            + "/-/query"
             + "?"
             + urllib.parse.urlencode(
                 {
diff --git a/tests/test_api.py b/tests/test_api.py
index bd7346776c..431ab5cee4 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -623,7 +623,7 @@ def test_no_files_uses_memory_database(app_client_no_files):
     } == response.json
     # Try that SQL query
     response = app_client_no_files.get(
-        "/_memory.json?sql=select+sqlite_version()&_shape=array"
+        "/_memory/-/query.json?sql=select+sqlite_version()&_shape=array"
     )
     assert 1 == len(response.json)
     assert ["sqlite_version()"] == list(response.json[0].keys())
@@ -653,7 +653,7 @@ def test_database_page_for_database_with_dot_in_name(app_client_with_dot):
 @pytest.mark.asyncio
 async def test_custom_sql(ds_client):
     response = await ds_client.get(
-        "/fixtures.json?sql=select+content+from+simple_primary_key"
+        "/fixtures/-/query.json?sql=select+content+from+simple_primary_key",
     )
     data = response.json()
     assert data == {
@@ -670,7 +670,9 @@ async def test_custom_sql(ds_client):
 
 
 def test_sql_time_limit(app_client_shorter_time_limit):
-    response = app_client_shorter_time_limit.get("/fixtures.json?sql=select+sleep(0.5)")
+    response = app_client_shorter_time_limit.get(
+        "/fixtures/-/query.json?sql=select+sleep(0.5)",
+    )
     assert 400 == response.status
     assert response.json == {
         "ok": False,
@@ -691,16 +693,22 @@ def test_sql_time_limit(app_client_shorter_time_limit):
 
 @pytest.mark.asyncio
 async def test_custom_sql_time_limit(ds_client):
-    response = await ds_client.get("/fixtures.json?sql=select+sleep(0.01)")
+    response = await ds_client.get(
+        "/fixtures/-/query.json?sql=select+sleep(0.01)",
+    )
     assert response.status_code == 200
-    response = await ds_client.get("/fixtures.json?sql=select+sleep(0.01)&_timelimit=5")
+    response = await ds_client.get(
+        "/fixtures/-/query.json?sql=select+sleep(0.01)&_timelimit=5",
+    )
     assert response.status_code == 400
     assert response.json()["title"] == "SQL Interrupted"
 
 
 @pytest.mark.asyncio
 async def test_invalid_custom_sql(ds_client):
-    response = await ds_client.get("/fixtures.json?sql=.schema")
+    response = await ds_client.get(
+        "/fixtures/-/query.json?sql=.schema",
+    )
     assert response.status_code == 400
     assert response.json()["ok"] is False
     assert "Statement must be a SELECT" == response.json()["error"]
@@ -883,9 +891,13 @@ async def test_json_columns(ds_client, extra_args, expected):
         select 1 as intval, "s" as strval, 0.5 as floatval,
         '{"foo": "bar"}' as jsonval
     """
-    path = "/fixtures.json?" + urllib.parse.urlencode({"sql": sql, "_shape": "array"})
+    path = "/fixtures/-/query.json?" + urllib.parse.urlencode(
+        {"sql": sql, "_shape": "array"}
+    )
     path += extra_args
-    response = await ds_client.get(path)
+    response = await ds_client.get(
+        path,
+    )
     assert response.json() == expected
 
 
@@ -917,7 +929,7 @@ def test_config_force_https_urls():
         ("/fixtures.json", 200),
         ("/fixtures/no_primary_key.json", 200),
         # A 400 invalid SQL query should still have the header:
-        ("/fixtures.json?sql=select+blah", 400),
+        ("/fixtures/-/query.json?sql=select+blah", 400),
         # Write APIs
         ("/fixtures/-/create", 405),
         ("/fixtures/facetable/-/insert", 405),
@@ -930,7 +942,9 @@ def test_cors(
     path,
     status_code,
 ):
-    response = app_client_with_cors.get(path)
+    response = app_client_with_cors.get(
+        path,
+    )
     assert response.status == status_code
     assert response.headers["Access-Control-Allow-Origin"] == "*"
     assert (
@@ -946,7 +960,9 @@ def test_cors(
     # should not have those headers - I'm using that fixture because
     # regular app_client doesn't have immutable fixtures.db which means
     # the test for /fixtures.db returns a 403 error
-    response = app_client_two_attached_databases_one_immutable.get(path)
+    response = app_client_two_attached_databases_one_immutable.get(
+        path,
+    )
     assert response.status == status_code
     assert "Access-Control-Allow-Origin" not in response.headers
     assert "Access-Control-Allow-Headers" not in response.headers
diff --git a/tests/test_api_write.py b/tests/test_api_write.py
index 94eb902b2b..b442113ba7 100644
--- a/tests/test_api_write.py
+++ b/tests/test_api_write.py
@@ -637,7 +637,9 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path):
     # Should be a single row
     assert (
         await ds_write.client.get(
-            "/data.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(table)
+            "/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(
+                table
+            )
         )
     ).json() == [1]
     # Now delete the row
@@ -645,7 +647,9 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path):
         # Special case for that rowid table
         delete_path = (
             await ds_write.client.get(
-                "/data.json?_shape=arrayfirst&sql=select+rowid+from+{}".format(table)
+                "/data/-/query.json?_shape=arrayfirst&sql=select+rowid+from+{}".format(
+                    table
+                )
             )
         ).json()[0]
 
@@ -663,7 +667,9 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path):
     assert event.pks == str(delete_path).split(",")
     assert (
         await ds_write.client.get(
-            "/data.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(table)
+            "/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(
+                table
+            )
         )
     ).json() == [0]
 
diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py
index 69ed5ff904..d1ed06d183 100644
--- a/tests/test_canned_queries.py
+++ b/tests/test_canned_queries.py
@@ -412,7 +412,7 @@ def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_js
 
 def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client):
     response = magic_parameters_client.get(
-        "/data.json?sql=select+:_header_host&_shape=array"
+        "/data/-/query.json?sql=select+:_header_host&_shape=array"
     )
     assert 400 == response.status
     assert response.json["error"].startswith("You did not supply a value for binding")
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 486852cd4b..d53e0a5f20 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -250,7 +250,7 @@ def test_plugin_s_overwrite():
             "--plugins-dir",
             plugins_dir,
             "--get",
-            "/_memory.json?sql=select+prepare_connection_args()",
+            "/_memory/-/query.json?sql=select+prepare_connection_args()",
         ],
     )
     assert result.exit_code == 0, result.output
@@ -265,7 +265,7 @@ def test_plugin_s_overwrite():
             "--plugins-dir",
             plugins_dir,
             "--get",
-            "/_memory.json?sql=select+prepare_connection_args()",
+            "/_memory/-/query.json?sql=select+prepare_connection_args()",
             "-s",
             "plugins.name-of-plugin",
             "OVERRIDE",
@@ -295,7 +295,7 @@ def test_setting_default_allow_sql(default_allow_sql):
             "default_allow_sql",
             "on" if default_allow_sql else "off",
             "--get",
-            "/_memory.json?sql=select+21&_shape=objects",
+            "/_memory/-/query.json?sql=select+21&_shape=objects",
         ],
     )
     if default_allow_sql:
@@ -309,7 +309,7 @@ def test_setting_default_allow_sql(default_allow_sql):
 
 def test_sql_errors_logged_to_stderr():
     runner = CliRunner(mix_stderr=False)
-    result = runner.invoke(cli, ["--get", "/_memory.json?sql=select+blah"])
+    result = runner.invoke(cli, ["--get", "/_memory/-/query.json?sql=select+blah"])
     assert result.exit_code == 1
     assert "sql = 'select blah', params = {}: no such column: blah\n" in result.stderr
 
diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py
index 1088d9069c..513669a178 100644
--- a/tests/test_cli_serve_get.py
+++ b/tests/test_cli_serve_get.py
@@ -31,7 +31,7 @@ def startup(datasette):
             "--plugins-dir",
             str(plugins_dir),
             "--get",
-            "/_memory.json?sql=select+sqlite_version()",
+            "/_memory/-/query.json?sql=select+sqlite_version()",
         ],
     )
     assert result.exit_code == 0, result.output
diff --git a/tests/test_crossdb.py b/tests/test_crossdb.py
index 01c5113041..58b81f705c 100644
--- a/tests/test_crossdb.py
+++ b/tests/test_crossdb.py
@@ -25,7 +25,8 @@ def test_crossdb_join(app_client_two_attached_databases_crossdb_enabled):
       fixtures.searchable
     """
     response = app_client.get(
-        "/_memory.json?" + urllib.parse.urlencode({"sql": sql, "_shape": "array"})
+        "/_memory/-/query.json?"
+        + urllib.parse.urlencode({"sql": sql, "_shape": "array"})
     )
     assert response.status == 200
     assert response.json == [
@@ -67,9 +68,10 @@ def test_crossdb_attached_database_list_display(
 ):
     app_client = app_client_two_attached_databases_crossdb_enabled
     response = app_client.get("/_memory")
+    response2 = app_client.get("/")
     for fragment in (
         "databases are attached to this connection",
         "<li><strong>fixtures</strong> - ",
-        "<li><strong>extra database</strong> - ",
+        '<li><strong>extra database</strong> - <a href="/extra+database/-/query?sql=',
     ):
         assert fragment in response.text
diff --git a/tests/test_csv.py b/tests/test_csv.py
index 6dbe693b64..f6d2a6b26d 100644
--- a/tests/test_csv.py
+++ b/tests/test_csv.py
@@ -146,14 +146,14 @@ async def test_table_csv_blob_columns(ds_client):
 @pytest.mark.asyncio
 async def test_custom_sql_csv_blob_columns(ds_client):
     response = await ds_client.get(
-        "/fixtures.csv?sql=select+rowid,+data+from+binary_data"
+        "/fixtures/-/query.csv?sql=select+rowid,+data+from+binary_data"
     )
     assert response.status_code == 200
     assert response.headers["content-type"] == "text/plain; charset=utf-8"
     assert response.text == (
         "rowid,data\r\n"
-        '1,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d"\r\n'
-        '2,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_blob_column=data&_blob_hash=b835b0483cedb86130b9a2c280880bf5fadc5318ddf8c18d0df5204d40df1724"\r\n'
+        '1,"http://localhost/fixtures/-/query.blob?sql=select+rowid,+data+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d"\r\n'
+        '2,"http://localhost/fixtures/-/query.blob?sql=select+rowid,+data+from+binary_data&_blob_column=data&_blob_hash=b835b0483cedb86130b9a2c280880bf5fadc5318ddf8c18d0df5204d40df1724"\r\n'
         "3,\r\n"
     )
 
@@ -161,7 +161,7 @@ async def test_custom_sql_csv_blob_columns(ds_client):
 @pytest.mark.asyncio
 async def test_custom_sql_csv(ds_client):
     response = await ds_client.get(
-        "/fixtures.csv?sql=select+content+from+simple_primary_key+limit+2"
+        "/fixtures/-/query.csv?sql=select+content+from+simple_primary_key+limit+2"
     )
     assert response.status_code == 200
     assert response.headers["content-type"] == "text/plain; charset=utf-8"
@@ -182,7 +182,7 @@ async def test_table_csv_download(ds_client):
 @pytest.mark.asyncio
 async def test_csv_with_non_ascii_characters(ds_client):
     response = await ds_client.get(
-        "/fixtures.csv?sql=select%0D%0A++%27%F0%9D%90%9C%F0%9D%90%A2%F0%9D%90%AD%F0%9D%90%A2%F0%9D%90%9E%F0%9D%90%AC%27+as+text%2C%0D%0A++1+as+number%0D%0Aunion%0D%0Aselect%0D%0A++%27bob%27+as+text%2C%0D%0A++2+as+number%0D%0Aorder+by%0D%0A++number"
+        "/fixtures/-/query.csv?sql=select%0D%0A++%27%F0%9D%90%9C%F0%9D%90%A2%F0%9D%90%AD%F0%9D%90%A2%F0%9D%90%9E%F0%9D%90%AC%27+as+text%2C%0D%0A++1+as+number%0D%0Aunion%0D%0Aselect%0D%0A++%27bob%27+as+text%2C%0D%0A++2+as+number%0D%0Aorder+by%0D%0A++number"
     )
     assert response.status_code == 200
     assert response.headers["content-type"] == "text/plain; charset=utf-8"
diff --git a/tests/test_html.py b/tests/test_html.py
index a83f0e47db..5b60d2f544 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -159,7 +159,7 @@ async def test_database_page(ds_client):
 
 @pytest.mark.asyncio
 async def test_invalid_custom_sql(ds_client):
-    response = await ds_client.get("/fixtures?sql=.schema")
+    response = await ds_client.get("/fixtures/-/query?sql=.schema")
     assert response.status_code == 400
     assert "Statement must be a SELECT" in response.text
 
@@ -167,7 +167,7 @@ async def test_invalid_custom_sql(ds_client):
 @pytest.mark.asyncio
 async def test_disallowed_custom_sql_pragma(ds_client):
     response = await ds_client.get(
-        "/fixtures?sql=SELECT+*+FROM+pragma_not_on_allow_list('idx52')"
+        "/fixtures/-/query?sql=SELECT+*+FROM+pragma_not_on_allow_list('idx52')"
     )
     assert response.status_code == 400
     pragmas = ", ".join("pragma_{}()".format(pragma) for pragma in allowed_pragmas)
@@ -180,7 +180,9 @@ async def test_disallowed_custom_sql_pragma(ds_client):
 
 
 def test_sql_time_limit(app_client_shorter_time_limit):
-    response = app_client_shorter_time_limit.get("/fixtures?sql=select+sleep(0.5)")
+    response = app_client_shorter_time_limit.get(
+        "/fixtures/-/query?sql=select+sleep(0.5)"
+    )
     assert 400 == response.status
     expected_html_fragments = [
         """
@@ -207,7 +209,7 @@ def test_row_page_does_not_truncate():
 def test_query_page_truncates():
     with make_app_client(settings={"truncate_cells_html": 5}) as client:
         response = client.get(
-            "/fixtures?"
+            "/fixtures/-/query?"
             + urllib.parse.urlencode(
                 {
                     "sql": "select 'this is longer than 5' as a, 'https://example.com/' as b"
@@ -229,7 +231,7 @@ def test_query_page_truncates():
     [
         ("/", ["index"]),
         ("/fixtures", ["db", "db-fixtures"]),
-        ("/fixtures?sql=select+1", ["query", "db-fixtures"]),
+        ("/fixtures/-/query?sql=select+1", ["query", "db-fixtures"]),
         (
             "/fixtures/simple_primary_key",
             ["table", "db-fixtures", "table-simple_primary_key"],
@@ -296,21 +298,24 @@ async def test_row_json_export_link(ds_client):
 
 @pytest.mark.asyncio
 async def test_query_json_csv_export_links(ds_client):
-    response = await ds_client.get("/fixtures?sql=select+1")
+    response = await ds_client.get("/fixtures/-/query?sql=select+1")
     assert response.status_code == 200
-    assert '<a href="/fixtures.json?sql=select+1">json</a>' in response.text
-    assert '<a href="/fixtures.csv?sql=select+1&amp;_size=max">CSV</a>' in response.text
+    assert '<a href="/fixtures/-/query.json?sql=select+1">json</a>' in response.text
+    assert (
+        '<a href="/fixtures/-/query.csv?sql=select+1&amp;_size=max">CSV</a>'
+        in response.text
+    )
 
 
 @pytest.mark.asyncio
 async def test_query_parameter_form_fields(ds_client):
-    response = await ds_client.get("/fixtures?sql=select+:name")
+    response = await ds_client.get("/fixtures/-/query?sql=select+:name")
     assert response.status_code == 200
     assert (
         '<label for="qp1">name</label> <input type="text" id="qp1" name="name" value="">'
         in response.text
     )
-    response2 = await ds_client.get("/fixtures?sql=select+:name&name=hello")
+    response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello")
     assert response2.status_code == 200
     assert (
         '<label for="qp1">name</label> <input type="text" id="qp1" name="name" value="hello">'
@@ -453,7 +458,9 @@ async def test_database_metadata(ds_client):
 
 @pytest.mark.asyncio
 async def test_database_metadata_with_custom_sql(ds_client):
-    response = await ds_client.get("/fixtures?sql=select+*+from+simple_primary_key")
+    response = await ds_client.get(
+        "/fixtures/-/query?sql=select+*+from+simple_primary_key"
+    )
     assert response.status_code == 200
     soup = Soup(response.text, "html.parser")
     # Page title should be the default
@@ -591,7 +598,7 @@ async def test_canned_query_with_custom_metadata(ds_client):
 
 @pytest.mark.asyncio
 async def test_urlify_custom_queries(ds_client):
-    path = "/fixtures?" + urllib.parse.urlencode(
+    path = "/fixtures/-/query?" + urllib.parse.urlencode(
         {"sql": "select ('https://twitter.com/' || 'simonw') as user_url;"}
     )
     response = await ds_client.get(path)
@@ -609,7 +616,7 @@ async def test_urlify_custom_queries(ds_client):
 
 @pytest.mark.asyncio
 async def test_show_hide_sql_query(ds_client):
-    path = "/fixtures?" + urllib.parse.urlencode(
+    path = "/fixtures/-/query?" + urllib.parse.urlencode(
         {"sql": "select ('https://twitter.com/' || 'simonw') as user_url;"}
     )
     response = await ds_client.get(path)
@@ -696,15 +703,15 @@ def test_canned_query_show_hide_metadata_option(
 
 @pytest.mark.asyncio
 async def test_binary_data_display_in_query(ds_client):
-    response = await ds_client.get("/fixtures?sql=select+*+from+binary_data")
+    response = await ds_client.get("/fixtures/-/query?sql=select+*+from+binary_data")
     assert response.status_code == 200
     table = Soup(response.content, "html.parser").find("table")
     expected_tds = [
         [
-            '<td class="col-data"><a class="blob-download" href="/fixtures.blob?sql=select+*+from+binary_data&amp;_blob_column=data&amp;_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d">&lt;Binary:\xa07\xa0bytes&gt;</a></td>'
+            '<td class="col-data"><a class="blob-download" href="/fixtures/-/query.blob?sql=select+*+from+binary_data&amp;_blob_column=data&amp;_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d">&lt;Binary:\xa07\xa0bytes&gt;</a></td>'
         ],
         [
-            '<td class="col-data"><a class="blob-download" href="/fixtures.blob?sql=select+*+from+binary_data&amp;_blob_column=data&amp;_blob_hash=b835b0483cedb86130b9a2c280880bf5fadc5318ddf8c18d0df5204d40df1724">&lt;Binary:\xa07\xa0bytes&gt;</a></td>'
+            '<td class="col-data"><a class="blob-download" href="/fixtures/-/query.blob?sql=select+*+from+binary_data&amp;_blob_column=data&amp;_blob_hash=b835b0483cedb86130b9a2c280880bf5fadc5318ddf8c18d0df5204d40df1724">&lt;Binary:\xa07\xa0bytes&gt;</a></td>'
         ],
         ['<td class="col-data">\xa0</td>'],
     ]
@@ -719,7 +726,7 @@ async def test_binary_data_display_in_query(ds_client):
     [
         ("/fixtures/binary_data/1.blob?_blob_column=data", "binary_data-1-data.blob"),
         (
-            "/fixtures.blob?sql=select+*+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d",
+            "/fixtures/-/query.blob?sql=select+*+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d",
             "data-f30889.blob",
         ),
     ],
@@ -758,7 +765,7 @@ async def test_blob_download_invalid_messages(ds_client, path, expected_message)
 @pytest.mark.parametrize(
     "path",
     [
-        "/fixtures?sql=select+*+from+[123_starts_with_digits]",
+        "/fixtures/-/query?sql=select+*+from+[123_starts_with_digits]",
         "/fixtures/123_starts_with_digits",
     ],
 )
@@ -771,7 +778,7 @@ async def test_zero_results(ds_client, path):
 
 @pytest.mark.asyncio
 async def test_query_error(ds_client):
-    response = await ds_client.get("/fixtures?sql=select+*+from+notatable")
+    response = await ds_client.get("/fixtures/-/query?sql=select+*+from+notatable")
     html = response.text
     assert '<p class="message-error">no such table: notatable</p>' in html
     assert '<textarea id="sql-editor" name="sql" style="height: 3em' in html
@@ -811,7 +818,7 @@ def test_debug_context_includes_extra_template_vars():
         "/fixtures/paginated_view",
         "/fixtures/facetable",
         "/fixtures/facetable?_facet=state",
-        "/fixtures?sql=select+1",
+        "/fixtures/-/query?sql=select+1",
     ],
 )
 @pytest.mark.parametrize("use_prefix", (True, False))
@@ -879,17 +886,17 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix):
     [
         (
             "/fixtures/neighborhood_search",
-            "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable._city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&amp;text=",
+            "/fixtures/-/query?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable._city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&amp;text=",
         ),
         (
             "/fixtures/neighborhood_search?text=ber",
-            "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable._city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&amp;text=ber",
+            "/fixtures/-/query?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable._city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&amp;text=ber",
         ),
         ("/fixtures/pragma_cache_size", None),
         (
             # /fixtures/𝐜𝐢𝐭𝐢𝐞𝐬
             "/fixtures/~F0~9D~90~9C~F0~9D~90~A2~F0~9D~90~AD~F0~9D~90~A2~F0~9D~90~9E~F0~9D~90~AC",
-            "/fixtures?sql=select+id%2C+name+from+facet_cities+order+by+id+limit+1%3B",
+            "/fixtures/-/query?sql=select+id%2C+name+from+facet_cities+order+by+id+limit+1%3B",
         ),
         ("/fixtures/magic_parameters", None),
     ],
@@ -960,7 +967,7 @@ async def test_navigation_menu_links(
 
 @pytest.mark.asyncio
 async def test_trace_correctly_escaped(ds_client):
-    response = await ds_client.get("/fixtures?sql=select+'<h1>Hello'&_trace=1")
+    response = await ds_client.get("/fixtures/-/query?sql=select+'<h1>Hello'&_trace=1")
     assert "select '<h1>Hello" not in response.text
     assert "select &#39;&lt;h1&gt;Hello" in response.text
 
@@ -989,8 +996,8 @@ async def test_trace_correctly_escaped(ds_client):
         ),
         # Custom query page
         (
-            "/fixtures?sql=select+*+from+facetable",
-            "http://localhost/fixtures.json?sql=select+*+from+facetable",
+            "/fixtures/-/query?sql=select+*+from+facetable",
+            "http://localhost/fixtures/-/query.json?sql=select+*+from+facetable",
         ),
         # Canned query page
         (
diff --git a/tests/test_load_extensions.py b/tests/test_load_extensions.py
index 4007e0be8a..cdadb091a6 100644
--- a/tests/test_load_extensions.py
+++ b/tests/test_load_extensions.py
@@ -25,15 +25,15 @@ async def test_load_extension_default_entrypoint():
     # should fail.
     ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])
 
-    response = await ds.client.get("/_memory.json?_shape=arrays&sql=select+a()")
+    response = await ds.client.get("/_memory/-/query.json?_shape=arrays&sql=select+a()")
     assert response.status_code == 200
     assert response.json()["rows"][0][0] == "a"
 
-    response = await ds.client.get("/_memory.json?_shape=arrays&sql=select+b()")
+    response = await ds.client.get("/_memory/-/query.json?_shape=arrays&sql=select+b()")
     assert response.status_code == 400
     assert response.json()["error"] == "no such function: b"
 
-    response = await ds.client.get("/_memory.json?_shape=arrays&sql=select+c()")
+    response = await ds.client.get("/_memory/-/query.json?_shape=arrays&sql=select+c()")
     assert response.status_code == 400
     assert response.json()["error"] == "no such function: c"
 
@@ -51,14 +51,14 @@ async def test_load_extension_multiple_entrypoints():
         ]
     )
 
-    response = await ds.client.get("/_memory.json?_shape=arrays&sql=select+a()")
+    response = await ds.client.get("/_memory/-/query.json?_shape=arrays&sql=select+a()")
     assert response.status_code == 200
     assert response.json()["rows"][0][0] == "a"
 
-    response = await ds.client.get("/_memory.json?_shape=arrays&sql=select+b()")
+    response = await ds.client.get("/_memory/-/query.json?_shape=arrays&sql=select+b()")
     assert response.status_code == 200
     assert response.json()["rows"][0][0] == "b"
 
-    response = await ds.client.get("/_memory.json?_shape=arrays&sql=select+c()")
+    response = await ds.client.get("/_memory/-/query.json?_shape=arrays&sql=select+c()")
     assert response.status_code == 200
     assert response.json()["rows"][0][0] == "c"
diff --git a/tests/test_messages.py b/tests/test_messages.py
index a7e4d04698..62d9f64795 100644
--- a/tests/test_messages.py
+++ b/tests/test_messages.py
@@ -12,7 +12,7 @@
     ],
 )
 async def test_add_message_sets_cookie(ds_client, qs, expected):
-    response = await ds_client.get(f"/fixtures.message?sql=select+1&{qs}")
+    response = await ds_client.get(f"/fixtures/-/query.message?sql=select+1&{qs}")
     signed = response.cookies["ds_messages"]
     decoded = ds_client.ds.unsign(signed, "messages")
     assert expected == decoded
@@ -22,7 +22,7 @@ async def test_add_message_sets_cookie(ds_client, qs, expected):
 async def test_messages_are_displayed_and_cleared(ds_client):
     # First set the message cookie
     set_msg_response = await ds_client.get(
-        "/fixtures.message?sql=select+1&add_msg=xmessagex"
+        "/fixtures/-/query.message?sql=select+1&add_msg=xmessagex"
     )
     # Now access a page that displays messages
     response = await ds_client.get("/", cookies=set_msg_response.cookies)
diff --git a/tests/test_permissions.py b/tests/test_permissions.py
index fe91ec3de8..0e9cd15b0a 100644
--- a/tests/test_permissions.py
+++ b/tests/test_permissions.py
@@ -268,7 +268,7 @@ def test_view_query(allow, expected_anon, expected_auth):
 def test_execute_sql(config):
     schema_re = re.compile("const schema = ({.*?});", re.DOTALL)
     with make_app_client(config=config) as client:
-        form_fragment = '<form class="sql" action="/fixtures"'
+        form_fragment = '<form class="sql" action="/fixtures/-/query"'
 
         # Anonymous users - should not display the form:
         anon_html = client.get("/fixtures").text
@@ -276,7 +276,7 @@ def test_execute_sql(config):
         # And const schema should be an empty object:
         assert "const schema = {};" in anon_html
         # This should 403:
-        assert client.get("/fixtures?sql=select+1").status == 403
+        assert client.get("/fixtures/-/query?sql=select+1").status == 403
         # ?_where= not allowed on tables:
         assert client.get("/fixtures/facet_cities?_where=id=3").status == 403
 
@@ -289,7 +289,7 @@ def test_execute_sql(config):
         assert set(schema["attraction_characteristic"]) == {"name", "pk"}
         assert schema["paginated_view"] == []
         assert form_fragment in response_text
-        query_response = client.get("/fixtures?sql=select+1", cookies=cookies)
+        query_response = client.get("/fixtures/-/query?sql=select+1", cookies=cookies)
         assert query_response.status == 200
         schema2 = json.loads(schema_re.search(query_response.text).group(1))
         assert set(schema2["attraction_characteristic"]) == {"name", "pk"}
@@ -337,7 +337,7 @@ def test_query_list_respects_view_query():
             ],
         ),
         (
-            "/fixtures?sql=select+1",
+            "/fixtures/-/query?sql=select+1",
             [
                 "view-instance",
                 ("view-database", "fixtures"),
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index cc2e16c83c..5fad03ad45 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -45,7 +45,7 @@ def test_plugin_hooks_have_tests(plugin_hook):
 @pytest.mark.asyncio
 async def test_hook_plugins_dir_plugin_prepare_connection(ds_client):
     response = await ds_client.get(
-        "/fixtures.json?_shape=arrayfirst&sql=select+convert_units(100%2C+'m'%2C+'ft')"
+        "/fixtures/-/query.json?_shape=arrayfirst&sql=select+convert_units(100%2C+'m'%2C+'ft')"
     )
     assert response.json()[0] == pytest.approx(328.0839)
 
@@ -53,7 +53,7 @@ async def test_hook_plugins_dir_plugin_prepare_connection(ds_client):
 @pytest.mark.asyncio
 async def test_hook_plugin_prepare_connection_arguments(ds_client):
     response = await ds_client.get(
-        "/fixtures.json?sql=select+prepare_connection_args()&_shape=arrayfirst"
+        "/fixtures/-/query.json?sql=select+prepare_connection_args()&_shape=arrayfirst"
     )
     assert [
         "database=fixtures, datasette.plugin_config(\"name-of-plugin\")={'depth': 'root'}"
@@ -176,7 +176,7 @@ async def test_hook_render_cell_link_from_json(ds_client):
     sql = """
         select '{"href": "http://example.com/", "label":"Example"}'
     """.strip()
-    path = "/fixtures?" + urllib.parse.urlencode({"sql": sql})
+    path = "/fixtures/-/query?" + urllib.parse.urlencode({"sql": sql})
     response = await ds_client.get(path)
     td = Soup(response.text, "html.parser").find("table").find("tbody").find("td")
     a = td.find("a")
@@ -205,7 +205,11 @@ async def test_hook_render_cell_demo(ds_client):
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
-    "path", ("/fixtures?sql=select+'RENDER_CELL_ASYNC'", "/fixtures/simple_primary_key")
+    "path",
+    (
+        "/fixtures/-/query?sql=select+'RENDER_CELL_ASYNC'",
+        "/fixtures/simple_primary_key",
+    ),
 )
 async def test_hook_render_cell_async(ds_client, path):
     response = await ds_client.get(path)
@@ -423,7 +427,7 @@ def extra_template_vars(view_name):
         ("/fixtures/units", "table"),
         ("/fixtures/units/1", "row"),
         ("/-/versions", "json_data"),
-        ("/fixtures?sql=select+1", "database"),
+        ("/fixtures/-/query?sql=select+1", "database"),
     ),
 )
 def test_view_names(view_names_client, path, view_name):
@@ -975,13 +979,13 @@ def get_actions_links(html):
 @pytest.mark.parametrize(
     "path,expected_url",
     (
-        ("/fixtures?sql=select+1", "/fixtures?sql=explain+select+1"),
+        ("/fixtures/-/query?sql=select+1", "/fixtures/-/query?sql=explain+select+1"),
         (
             "/fixtures/pragma_cache_size",
-            "/fixtures?sql=explain+PRAGMA+cache_size%3B",
+            "/fixtures/-/query?sql=explain+PRAGMA+cache_size%3B",
         ),
         # Don't attempt to explain an explain
-        ("/fixtures?sql=explain+select+1", None),
+        ("/fixtures/-/query?sql=explain+select+1", None),
     ),
 )
 async def test_hook_query_actions(ds_client, path, expected_url):
@@ -1475,7 +1479,7 @@ async def test_hook_top_row(ds_client):
 async def test_hook_top_query(ds_client):
     try:
         pm.register(SlotPlugin(), name="SlotPlugin")
-        response = await ds_client.get("/fixtures?sql=select+1&z=x")
+        response = await ds_client.get("/fixtures/-/query?sql=select+1&z=x")
         assert response.status_code == 200
         assert "Xtop_query:fixtures:select 1:x" in response.text
     finally:
diff --git a/tests/test_routes.py b/tests/test_routes.py
index b2d317598a..9866cc7645 100644
--- a/tests/test_routes.py
+++ b/tests/test_routes.py
@@ -95,7 +95,7 @@ async def test_db_with_route_databases(ds_with_route):
         ("/original-name/t", 404),
         ("/original-name/t/1", 404),
         ("/custom-route-name", 200),
-        ("/custom-route-name?sql=select+id+from+t", 200),
+        ("/custom-route-name/-/query?sql=select+id+from+t", 200),
         ("/custom-route-name/t", 200),
         ("/custom-route-name/t/1", 200),
     ),
diff --git a/tests/test_table_api.py b/tests/test_table_api.py
index 8360157d7b..11542cb056 100644
--- a/tests/test_table_api.py
+++ b/tests/test_table_api.py
@@ -57,7 +57,7 @@ async def test_table_shape_arrays(ds_client):
 @pytest.mark.asyncio
 async def test_table_shape_arrayfirst(ds_client):
     response = await ds_client.get(
-        "/fixtures.json?"
+        "/fixtures/-/query.json?"
         + urllib.parse.urlencode(
             {
                 "sql": "select content from simple_primary_key order by id",
@@ -699,7 +699,7 @@ async def test_table_through(ds_client):
 @pytest.mark.asyncio
 async def test_max_returned_rows(ds_client):
     response = await ds_client.get(
-        "/fixtures.json?sql=select+content+from+no_primary_key"
+        "/fixtures/-/query.json?sql=select+content+from+no_primary_key"
     )
     data = response.json()
     assert data["truncated"]
diff --git a/tests/test_table_html.py b/tests/test_table_html.py
index 1a9ed5404c..b7c52e016d 100644
--- a/tests/test_table_html.py
+++ b/tests/test_table_html.py
@@ -1199,7 +1199,9 @@ async def test_format_of_binary_links(size, title, length_bytes):
     expected = "{}>&lt;Binary:&nbsp;{}&nbsp;bytes&gt;</a>".format(title, length_bytes)
     assert expected in response.text
     # And test with arbitrary SQL query too
-    sql_response = await ds.client.get("/{}".format(db_name), params={"sql": sql})
+    sql_response = await ds.client.get(
+        "{}/-/query".format(db_name), params={"sql": sql}
+    )
     assert sql_response.status_code == 200
     assert expected in sql_response.text
 

From 2edf45b1b6d521f0d34e7fb9f0a2b85792c707a0 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 16 Jul 2024 14:20:54 -0700
Subject: [PATCH 20/48] Use isolation_level=IMMEDIATE, refs #2358

---
 datasette/database.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/datasette/database.py b/datasette/database.py
index ffe94ea725..71c134d1d9 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -85,12 +85,13 @@ def suggest_name(self):
             return "db"
 
     def connect(self, write=False):
+        extra_kwargs = {}
+        if write:
+            extra_kwargs["isolation_level"] = "IMMEDIATE"
         if self.memory_name:
             uri = "file:{}?mode=memory&cache=shared".format(self.memory_name)
             conn = sqlite3.connect(
-                uri,
-                uri=True,
-                check_same_thread=False,
+                uri, uri=True, check_same_thread=False, **extra_kwargs
             )
             if not write:
                 conn.execute("PRAGMA query_only=1")
@@ -111,7 +112,7 @@ def connect(self, write=False):
         if self.mode is not None:
             qs = f"?mode={self.mode}"
         conn = sqlite3.connect(
-            f"file:{self.path}{qs}", uri=True, check_same_thread=False
+            f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs
         )
         self._all_file_connections.append(conn)
         return conn

From feccfa2a4dc77e073416090004ed14da61352558 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 25 Jul 2024 16:04:29 -0700
Subject: [PATCH 21/48] Bump the python-packages group across 1 directory with
 2 updates (#2371)

Bumps the python-packages group with 2 updates in the / directory: [sphinx](https://github.com/sphinx-doc/sphinx) and [furo](https://github.com/pradyunsg/furo).


Updates `sphinx` from 7.3.7 to 7.4.7
- [Release notes](https://github.com/sphinx-doc/sphinx/releases)
- [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES.rst)
- [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.3.7...v7.4.7)

Updates `furo` from 2024.5.6 to 2024.7.18
- [Release notes](https://github.com/pradyunsg/furo/releases)
- [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md)
- [Commits](https://github.com/pradyunsg/furo/compare/2024.05.06...2024.07.18)

---
updated-dependencies:
- dependency-name: sphinx
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: python-packages
- dependency-name: furo
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: python-packages
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index fa61729ff0..c69404f884 100644
--- a/setup.py
+++ b/setup.py
@@ -70,8 +70,8 @@ def get_version():
     """,
     extras_require={
         "docs": [
-            "Sphinx==7.3.7",
-            "furo==2024.5.6",
+            "Sphinx==7.4.7",
+            "furo==2024.7.18",
             "sphinx-autobuild",
             "codespell>=2.2.5",
             "blacken-docs",

From 81b68a143a78b08d1ff7534ab7f00a647a7a0781 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 26 Jul 2024 14:09:20 -0700
Subject: [PATCH 22/48] /-/auth-token as root redirects to /, closes #2375

---
 datasette/views/special.py | 3 +++
 tests/test_auth.py         | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/datasette/views/special.py b/datasette/views/special.py
index c80e816f5a..e997b788a5 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -75,6 +75,9 @@ class AuthTokenView(BaseView):
     has_json_alternate = False
 
     async def get(self, request):
+        # If already signed in as root, redirect
+        if request.actor and request.actor.get("id") == "root":
+            return Response.redirect(self.ds.urls.instance())
         token = request.args.get("token") or ""
         if not self.ds._root_token:
             raise Forbidden("Root token has already been used")
diff --git a/tests/test_auth.py b/tests/test_auth.py
index 74e8283a76..fd7fcc65e5 100644
--- a/tests/test_auth.py
+++ b/tests/test_auth.py
@@ -26,6 +26,12 @@ async def test_auth_token(ds_client):
     # Check that a second with same token fails
     assert ds_client.ds._root_token is None
     assert (await ds_client.get(path)).status_code == 403
+    # But attempting with same token while logged in as root should redirect to /
+    response = await ds_client.get(
+        path, cookies={"ds_actor": ds_client.actor_cookie({"id": "root"})}
+    )
+    assert response.status_code == 302
+    assert response.headers["Location"] == "/"
 
 
 @pytest.mark.asyncio

From 169ee5d710b58d0d53a0c14486a2c1c4c8987e86 Mon Sep 17 00:00:00 2001
From: Alex Garcia <alexsebastian.garcia@gmail.com>
Date: Mon, 5 Aug 2024 10:35:38 -0700
Subject: [PATCH 23/48] Initial upgrade guide for v0.XX to v1

---
 docs/index.rst            |   1 +
 docs/internals.rst        | 122 ++++++++++++++++++++++++++++++++++++++
 docs/upgrade_guide_v1.rst |  91 ++++++++++++++++++++++++++++
 3 files changed, 214 insertions(+)
 create mode 100644 docs/upgrade_guide_v1.rst

diff --git a/docs/index.rst b/docs/index.rst
index e3036618ef..bf1bc0858d 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -65,5 +65,6 @@ Contents
    testing_plugins
    internals
    events
+   upgrade_guide_v1
    contributing
    changelog
diff --git a/docs/internals.rst b/docs/internals.rst
index 38e66a57b7..0eb95e8938 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -516,6 +516,128 @@ Returns the specified database object. Raises a ``KeyError`` if the database doe
 
 Returns a database object for reading and writing to the private :ref:`internal database <internals_internal>`.
 
+
+.. _datasette_get_instance_metadata:
+
+await .get_instance_metadata(self)
+----------------------------------
+
+Returns metadata keys and values for the entire Datasette instance as a dictionary.
+Internally queries the ``datasette_metadata_instance_entries`` table inside the :ref:`internal database <internals_internal>`.
+
+.. _datasette_get_database_metadata:
+
+await .get_database_metadata(self, database_name)
+------------------------------------------------------
+
+``database_name`` - string
+    The name of the database to query.
+
+Returns metadata keys and values for the specified database as a dictionary.
+Internally queries the ``datasette_metadata_database_entries`` table inside the :ref:`internal database <internals_internal>`.
+
+.. _datasette_get_resource_metadata:
+
+await .get_resource_metadata(self, database_name, resource_name)
+--------------------------------------------------------------------------
+
+``database_name`` - string
+    The name of the database to query.
+``resource_name`` - string
+    The name of the resource (table, view, or canned query) inside ``database_name`` to query.
+
+Returns metadata keys and values for the specified "resource" as a dictionary.
+A "resource" in this context can be a table, view, or canned query.
+Internally queries the ``datasette_metadata_resource_entries`` table inside the :ref:`internal database <internals_internal>`.
+
+.. _datasette_get_column_metadata:
+
+await .get_column_metadata(self, database_name, resource_name, column_name)
+------------------------------------------------------------------------------------------
+
+``database_name`` - string
+    The name of the database to query.
+``resource_name`` - string
+    The name of the resource (table, view, or canned query) inside ``database_name`` to query.
+``column_name`` - string
+    The name of the column inside ``resource_name`` to query.
+
+
+Returns metadata keys and values for the specified column, resource, and table as a dictionary.
+Internally queries the ``datasette_metadata_column_entries`` table inside the :ref:`internal database <internals_internal>`.
+
+.. _datasette_set_instance_metadata:
+
+await .set_instance_metadata(self, key, value)
+--------------------------------------------------------
+
+``key`` - string
+    The metadata entry key to insert (ex ``title``, ``description``, etc.)
+``value`` - string
+    The value of the metadata entry to insert.
+
+Adds a new metadata entry for the entire Datasette instance.
+Any previous instance-level metadata entry with the same ``key`` will be overwritten.
+Internally upserts the value into the  the ``datasette_metadata_instance_entries`` table inside the :ref:`internal database <internals_internal>`.
+
+.. _datasette_set_database_metadata:
+
+await .set_database_metadata(self, database_name, key, value)
+----------------------------------------------------------------------------
+
+``database_name`` - string
+    The database the metadata entry belongs to.
+``key`` - string
+    The metadata entry key to insert (ex ``title``, ``description``, etc.)
+``value`` - string
+    The value of the metadata entry to insert.
+
+Adds a new metadata entry for the specified database.
+Any previous database-level metadata entry with the same ``key`` will be overwritten.
+Internally upserts the value into the  the ``datasette_metadata_database_entries`` table inside the :ref:`internal database <internals_internal>`.
+
+
+.. _datasette_set_resource_metadata:
+
+await .set_resource_metadata(self, database_name, resource_name, key, value)
+------------------------------------------------------------------------------------------------
+
+``database_name`` - string
+    The database the metadata entry belongs to.
+``resource_name`` - string
+    The resource (table, view, or canned query) the metadata entry belongs to.
+``key`` - string
+    The metadata entry key to insert (ex ``title``, ``description``, etc.)
+``value`` - string
+    The value of the metadata entry to insert.
+
+Adds a new metadata entry for the specified "resource".
+Any previous resource-level metadata entry with the same ``key`` will be overwritten.
+Internally upserts the value into the  the ``datasette_metadata_resource_entries`` table inside the :ref:`internal database <internals_internal>`.
+
+
+.. _datasette_set_column_metadata:
+
+await .set_column_metadata(self, database_name, resource_name, column_name, key, value)
+------------------------------------------------------------------------------------------------------------------
+
+``database_name`` - string
+    The database the metadata entry belongs to.
+``resource_name`` - string
+    The resource (table, view, or canned query) the metadata entry belongs to.
+``column-name`` - string
+    The column the metadata entry belongs to.
+``key`` - string
+    The metadata entry key to insert (ex ``title``, ``description``, etc.)
+``value`` - string
+    The value of the metadata entry to insert.
+
+Adds a new metadata entry for the specified column.
+Any previous column-level metadata entry with the same ``key`` will be overwritten.
+Internally upserts the value into the  the ``datasette_metadata_column_entries`` table inside the :ref:`internal database <internals_internal>`.
+
+
+
 .. _datasette_add_database:
 
 .add_database(db, name=None, route=None)
diff --git a/docs/upgrade_guide_v1.rst b/docs/upgrade_guide_v1.rst
new file mode 100644
index 0000000000..90024ff5bc
--- /dev/null
+++ b/docs/upgrade_guide_v1.rst
@@ -0,0 +1,91 @@
+.. upgrade_guide_v1:
+
+====================================
+ Datasette 0.X -> 1.0 Upgrade Guide
+====================================
+
+
+This document specifically reviews what breaking changes Datasette ``1.0`` has when upgrading from a ``0.XX`` version.
+For new features that ``1.0`` offers, see the [Changelog](https://docs.datasette.io/en/latest/changelog.html).
+
+
+Metadata changes
+================
+
+Metadata in Datasette.10 was completely revamped.
+There are a number of related breaking changes, from the ``metadata.yaml`` file to Python APIs that you'll need to consider when upgrading.
+
+``metadata.yaml`` split into ``datasette.yaml``
+-----------------------------------------------
+
+Before Datasette 1.0, the ``metadata.yaml`` file became a kitchen sink if a mix of metadata, configuration, and settings.
+Now ``metadata.yaml`` is strictly for metaata (ex title and descriptions of database and tables, licensing info, etc).
+
+
+Metadata "fallback" has been removed
+------------------------------------
+
+Certain keys in metadata like ``license`` used to "fallback" up the chain of ownership.
+For example, if you set an ``MIT`` to a database and a table within that database did not have a specified license,
+then that table would inherit an ``MIT`` license.
+
+This behavior has been removed in Datasette 1.0. Now license fields must be placed on all items, including individual databases and tables.
+
+The ``get_metadata()`` Plugin hook has been removed
+---------------------------------------------------
+
+As of Datasette ``1.0a14`` (2024-XX-XX), the ``get_metadata()`` hook has been deprecated.
+
+.. code-block:: python
+
+    # ❌ DEPRECATED in Datasette 1.0
+    @hookimpl
+    def get_metadata(datasette, key, database, table):
+        pass
+
+Instead, one should use the following methods on a Datasette class:
+
+- :ref:`get_instance_metadata() <datasette_get_instance_metadata>` and  :ref:`set_instance_metadata() <datasette_set_instance_metadata>`
+- :ref:`get_database_metadata() <datasette_get_database_metadata>` and  :ref:`set_database_metadata() <datasette_set_database_metadata>`
+- :ref:`get_resource_metadata() <datasette_get_resource_metadata>` and  :ref:`set_resource_metadata() <datasette_set_resource_metadata>`
+- :ref:`get_column_metadata() <datasette_get_column_metadata>` and  :ref:`set_column_metadata() <datasette_set_column_metadata>`
+
+The ``/metadata.json`` endpoint has been removed
+------------------------------------------------
+
+As of Datasette `1.0a14`, the root level ``/metadata.json`` endpoint has been removed.
+
+The ``metadata()`` method on the Datasette class has been removed
+-----------------------------------------------------------------
+
+As of Datasette ``1.0a14``, the ``.metadata()`` method on the Datasette Python API has been removed.
+
+Instead, one should use the following methods on a Datasette class:
+
+
+- :ref:`get_instance_metadata() <datasette_get_instance_metadata>`
+- :ref:`get_database_metadata() <datasette_get_database_metadata>`
+- :ref:`get_resource_metadata() <datasette_get_resource_metadata>`
+- :ref:`get_column_metadata() <datasette_get_column_metadata>`
+
+
+New endpoint for SQL queries
+============================
+
+Previously, if you wanted to run SQL code using the Datasette HTTP API, you could call an endpoint that looked like:
+
+::
+
+    # DEPRECATED: Older endpoint for Datasette 0.XX
+    curl http://localhost:8001/_memory?sql=select+123
+
+However, in Datasette 1.0, the endpoint was slightly changed to:
+
+::
+
+    # ✅ Datasette 1.0 and beyond
+    curl http://localhost:8001/_memory/-/query?sql=select+123
+
+Specifically, now there's a ``/-/query`` "action" that should be used.
+
+**This isn't a breaking change.** API calls to the older ``/database?sql=...`` endpoint will redirect to the new ``database/-/query?sql=...`` endpoint. However, documentations and example will use the new query endpoint, so it is recommended to use that instead.

From bd7d3bb70f28804e66b487fa5d04cc476ddb650c Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Aug 2024 12:11:17 -0700
Subject: [PATCH 24/48] Tweaks and improvements to upgrade guide, refs ##2374

Also refs #2381
---
 docs/index.rst            |   2 +-
 docs/plugin_hooks.rst     |  44 --------------
 docs/upgrade_guide.rst    | 125 ++++++++++++++++++++++++++++++++++++++
 docs/upgrade_guide_v1.rst |  91 ---------------------------
 4 files changed, 126 insertions(+), 136 deletions(-)
 create mode 100644 docs/upgrade_guide.rst
 delete mode 100644 docs/upgrade_guide_v1.rst

diff --git a/docs/index.rst b/docs/index.rst
index bf1bc0858d..4d4d1d96e4 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -65,6 +65,6 @@ Contents
    testing_plugins
    internals
    events
-   upgrade_guide_v1
+   upgrade_guide
    contributing
    changelog
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index f4ef5d64d5..d3764dcc98 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -1463,50 +1463,6 @@ This example will disable CSRF protection for that specific URL path:
 
 If any of the currently active ``skip_csrf()`` plugin hooks return ``True``, CSRF protection will be skipped for the request.
 
-.. _plugin_hook_get_metadata:
-
-get_metadata(datasette, key, database, table)
----------------------------------------------
-
-``datasette`` - :ref:`internals_datasette`
-    You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``.
-
-``actor`` - dictionary or None
-    The currently authenticated :ref:`actor <authentication_actor>`.
-
-``database`` - string or None
-    The name of the database metadata is being asked for.
-
-``table`` - string or None
-    The name of the table.
-
-``key`` - string or None
-    The name of the key for which data is being asked for.
-
-This hook is responsible for returning a dictionary corresponding to Datasette :ref:`metadata`. This function is passed the ``database``, ``table`` and ``key`` which were passed to the upstream internal request for metadata. Regardless, it is important to return a global metadata object, where ``"databases": []`` would be a top-level key. The dictionary returned here, will be merged with, and overwritten by, the contents of the physical ``metadata.yaml`` if one is present.
-
-.. warning::
-    The design of this plugin hook does not currently provide a mechanism for interacting with async code, and may change in the future. See `issue 1384 <https://github.com/simonw/datasette/issues/1384>`__.
-
-.. code-block:: python
-
-    @hookimpl
-    def get_metadata(datasette, key, database, table):
-        metadata = {
-            "title": "This will be the Datasette landing page title!",
-            "description": get_instance_description(datasette),
-            "databases": [],
-        }
-        for db_name, db_data_dict in get_my_database_meta(
-            datasette, database, table, key
-        ):
-            metadata["databases"][db_name] = db_data_dict
-        # whatever we return here will be merged with any other plugins using this hook and
-        # will be overwritten by a local metadata.yaml if one exists!
-        return metadata
-
-Example: `datasette-remote-metadata plugin <https://datasette.io/plugins/datasette-remote-metadata>`__
-
 .. _plugin_hook_menu_links:
 
 menu_links(datasette, actor, request)
diff --git a/docs/upgrade_guide.rst b/docs/upgrade_guide.rst
new file mode 100644
index 0000000000..09023166f4
--- /dev/null
+++ b/docs/upgrade_guide.rst
@@ -0,0 +1,125 @@
+.. _upgrade_guide:
+
+===============
+ Upgrade guide
+===============
+
+.. _upgrade_guide_v1:
+
+Datasette 0.X -> 1.0
+====================
+
+This section reviews breaking changes Datasette ``1.0`` has when upgrading from a ``0.XX`` version.
+For new features that ``1.0`` offers, see the :ref:`changelog`.
+
+.. _upgrade_guide_v1_metadata:
+
+Metadata changes
+----------------
+
+Metadata was completely revamped for Datasette 1.0.  There are a number of related breaking changes, from the ``metadata.yaml`` file to Python APIs, that you'll need to consider when upgrading.
+
+.. _upgrade_guide_v1_metadata_split:
+
+``metadata.yaml`` split into ``datasette.yaml``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Before Datasette 1.0, the ``metadata.yaml`` file became a kitchen sink if a mix of metadata, configuration, and settings. Now ``metadata.yaml`` is strictly for metaata (ex title and descriptions of database and tables, licensing info, etc). Other settings have been moved to a ``datasette.yml`` configuration file, described in :ref:`configuration`.
+
+To start Datasette with both metadata and configuration files, run it like this:
+
+.. code-block:: bash
+
+    datasette --metadata metadata.yaml --config datasette.yaml
+    # Or the shortened version:
+    datasette -m metadata.yml -c datasette.yml
+
+.. _upgrade_guide_v1_metadata_upgrade:
+
+Upgrading an existing ``metadata.yaml`` file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `datasette-upgrade plugin <https://github.com/datasette/datasette-upgrade>`__ can be used to split a Datasette 0.x.x ``metadata.yaml`` (or ``.json``) file into separate ``metadata.yaml`` and ``datasette.yaml`` files. First, install the plugin:
+
+.. code-block:: bash
+
+    datasette install datasette-upgrade
+
+Then run it like this to produce the two new files:
+
+.. code-block:: bash
+
+    datasette upgrade metadata-to-config metadata.json -m metadata.yml -c datasette.yml
+
+Metadata "fallback" has been removed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Certain keys in metadata like ``license`` used to "fallback" up the chain of ownership.
+For example, if you set an ``MIT`` to a database and a table within that database did not have a specified license, then that table would inherit an ``MIT`` license.
+
+This behavior has been removed in Datasette 1.0. Now license fields must be placed on all items, including individual databases and tables.
+
+.. _upgrade_guide_v1_metadata_removed:
+
+The ``get_metadata()`` plugin hook has been removed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In Datasette ``0.x`` plugins could implement a ``get_metadata()`` plugin hook to customize how metadata was retrieved for different instances, databases and tables.
+
+This hook could be inefficient, since some pages might load metadata for many different items (to list a large number of tables, for example) which could result in a large number of calls to potentially expensive plugin hook implementations.
+
+As of Datasette ``1.0a14`` (2024-08-05), the ``get_metadata()`` hook has been deprecated:
+
+.. code-block:: python
+
+    # ❌ DEPRECATED in Datasette 1.0
+    @hookimpl
+    def get_metadata(datasette, key, database, table):
+        pass
+
+Instead, plugins are encouraged to interact directly with Datasette's in-memory metadata tables in SQLite using the following methods on the :ref:`internals_datasette`:
+
+- :ref:`get_instance_metadata() <datasette_get_instance_metadata>` and  :ref:`set_instance_metadata() <datasette_set_instance_metadata>`
+- :ref:`get_database_metadata() <datasette_get_database_metadata>` and  :ref:`set_database_metadata() <datasette_set_database_metadata>`
+- :ref:`get_resource_metadata() <datasette_get_resource_metadata>` and  :ref:`set_resource_metadata() <datasette_set_resource_metadata>`
+- :ref:`get_column_metadata() <datasette_get_column_metadata>` and  :ref:`set_column_metadata() <datasette_set_column_metadata>`
+
+A plugin that stores or calculates its own metadata can implement the :ref:`plugin_hook_startup` hook to populate those items on startup, and then call those methods while it is running to persist any new metadata changes.
+
+The ``/metadata.json`` endpoint has been removed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As of Datasette ``1.0a14``, the root level ``/metadata.json`` endpoint has been removed. Metadata for tables will become available through currently in-development extras in a future alpha.
+
+The ``metadata()`` method on the Datasette class has been removed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As of Datasette ``1.0a14``, the ``.metadata()`` method on the Datasette Python API has been removed.
+
+Instead, one should use the following methods on a Datasette class:
+
+- :ref:`get_instance_metadata() <datasette_get_instance_metadata>`
+- :ref:`get_database_metadata() <datasette_get_database_metadata>`
+- :ref:`get_resource_metadata() <datasette_get_resource_metadata>`
+- :ref:`get_column_metadata() <datasette_get_column_metadata>`
+
+New endpoint for SQL queries
+----------------------------
+
+Previously, if you wanted to run SQL code using the Datasette HTTP API, you could call an endpoint that looked like:
+
+::
+
+    # DEPRECATED: Older endpoint for Datasette 0.XX
+    curl http://localhost:8001/_memory?sql=select+123
+
+However, in Datasette 1.0, the endpoint was slightly changed to:
+
+::
+
+    # ✅ Datasette 1.0 and beyond
+    curl http://localhost:8001/_memory/-/query?sql=select+123
+
+Specifically, now there's a ``/-/query`` "action" that should be used.
+
+**This isn't a breaking change.** API calls to the older ``/database?sql=...`` endpoint will redirect to the new ``database/-/query?sql=...`` endpoint. However, documentations and example will use the new query endpoint, so it is recommended to use that instead.
diff --git a/docs/upgrade_guide_v1.rst b/docs/upgrade_guide_v1.rst
deleted file mode 100644
index 90024ff5bc..0000000000
--- a/docs/upgrade_guide_v1.rst
+++ /dev/null
@@ -1,91 +0,0 @@
-.. upgrade_guide_v1:
-
-====================================
- Datasette 0.X -> 1.0 Upgrade Guide
-====================================
-
-
-This document specifically reviews what breaking changes Datasette ``1.0`` has when upgrading from a ``0.XX`` version.
-For new features that ``1.0`` offers, see the [Changelog](https://docs.datasette.io/en/latest/changelog.html).
-
-
-Metadata changes
-================
-
-Metadata in Datasette.10 was completely revamped.
-There are a number of related breaking changes, from the ``metadata.yaml`` file to Python APIs that you'll need to consider when upgrading.
-
-``metadata.yaml`` split into ``datasette.yaml``
------------------------------------------------
-
-Before Datasette 1.0, the ``metadata.yaml`` file became a kitchen sink if a mix of metadata, configuration, and settings.
-Now ``metadata.yaml`` is strictly for metaata (ex title and descriptions of database and tables, licensing info, etc).
-
-
-Metadata "fallback" has been removed
-------------------------------------
-
-Certain keys in metadata like ``license`` used to "fallback" up the chain of ownership.
-For example, if you set an ``MIT`` to a database and a table within that database did not have a specified license,
-then that table would inherit an ``MIT`` license.
-
-This behavior has been removed in Datasette 1.0. Now license fields must be placed on all items, including individual databases and tables.
-
-The ``get_metadata()`` Plugin hook has been removed
----------------------------------------------------
-
-As of Datasette ``1.0a14`` (2024-XX-XX), the ``get_metadata()`` hook has been deprecated.
-
-.. code-block:: python
-
-    # ❌ DEPRECATED in Datasette 1.0
-    @hookimpl
-    def get_metadata(datasette, key, database, table):
-        pass
-
-Instead, one should use the following methods on a Datasette class:
-
-- :ref:`get_instance_metadata() <datasette_get_instance_metadata>` and  :ref:`set_instance_metadata() <datasette_set_instance_metadata>`
-- :ref:`get_database_metadata() <datasette_get_database_metadata>` and  :ref:`set_database_metadata() <datasette_set_database_metadata>`
-- :ref:`get_resource_metadata() <datasette_get_resource_metadata>` and  :ref:`set_resource_metadata() <datasette_set_resource_metadata>`
-- :ref:`get_column_metadata() <datasette_get_column_metadata>` and  :ref:`set_column_metadata() <datasette_set_column_metadata>`
-
-The ``/metadata.json`` endpoint has been removed
-------------------------------------------------
-
-As of Datasette `1.0a14`, the root level ``/metadata.json`` endpoint has been removed.
-
-The ``metadata()`` method on the Datasette class has been removed
------------------------------------------------------------------
-
-As of Datasette ``1.0a14``, the ``.metadata()`` method on the Datasette Python API has been removed.
-
-Instead, one should use the following methods on a Datasette class:
-
-
-- :ref:`get_instance_metadata() <datasette_get_instance_metadata>`
-- :ref:`get_database_metadata() <datasette_get_database_metadata>`
-- :ref:`get_resource_metadata() <datasette_get_resource_metadata>`
-- :ref:`get_column_metadata() <datasette_get_column_metadata>`
-
-
-New endpoint for SQL queries
-============================
-
-Previously, if you wanted to run SQL code using the Datasette HTTP API, you could call an endpoint that looked like:
-
-::
-
-    # DEPRECATED: Older endpoint for Datasette 0.XX
-    curl http://localhost:8001/_memory?sql=select+123
-
-However, in Datasette 1.0, the endpoint was slightly changed to:
-
-::
-
-    # ✅ Datasette 1.0 and beyond
-    curl http://localhost:8001/_memory/-/query?sql=select+123
-
-Specifically, now there's a ``/-/query`` "action" that should be used.
-
-**This isn't a breaking change.** API calls to the older ``/database?sql=...`` endpoint will redirect to the new ``database/-/query?sql=...`` endpoint. However, documentations and example will use the new query endpoint, so it is recommended to use that instead.

From 2ad51baa31bfba7940c739e99d4270f563a77290 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Aug 2024 12:16:30 -0700
Subject: [PATCH 25/48] Move /db?sql= redirect to top of changes

Refs #2381
---
 docs/upgrade_guide.rst | 50 +++++++++++++++++++++++-------------------
 1 file changed, 27 insertions(+), 23 deletions(-)

diff --git a/docs/upgrade_guide.rst b/docs/upgrade_guide.rst
index 09023166f4..add2f9db99 100644
--- a/docs/upgrade_guide.rst
+++ b/docs/upgrade_guide.rst
@@ -9,8 +9,29 @@
 Datasette 0.X -> 1.0
 ====================
 
-This section reviews breaking changes Datasette ``1.0`` has when upgrading from a ``0.XX`` version.
-For new features that ``1.0`` offers, see the :ref:`changelog`.
+This section reviews breaking changes Datasette ``1.0`` has when upgrading from a ``0.XX`` version. For new features that ``1.0`` offers, see the :ref:`changelog`.
+
+.. _upgrade_guide_v1_sql_queries:
+
+New URL for SQL queries
+-----------------------
+
+Prior to ``1.0a14`` the URL for executing a SQL query looked like this:
+
+::
+    /databasename?sql=select+1
+    # Or for JSON:
+    /databasename.json?sql=select+1
+
+This endpoint served two purposes: without a ``?sql=`` it would list the tables in the database, but with that option it would return results of a query instead.
+
+The URL for executing a SQL query now looks like this::
+
+    /databasename/-/query?sql=select+1
+    # Or for JSON:
+    /databasename/-/query.json?sql=select+1
+
+**This isn't a breaking change.** API calls to the older ``/databasename?sql=...`` endpoint will redirect to the new ``databasename/-/query?sql=...`` endpoint. Upgrading to the new URL is recommended to avoid the overhead of the additional redirect.
 
 .. _upgrade_guide_v1_metadata:
 
@@ -86,11 +107,15 @@ Instead, plugins are encouraged to interact directly with Datasette's in-memory
 
 A plugin that stores or calculates its own metadata can implement the :ref:`plugin_hook_startup` hook to populate those items on startup, and then call those methods while it is running to persist any new metadata changes.
 
+.. _upgrade_guide_v1_metadata_json_removed:
+
 The ``/metadata.json`` endpoint has been removed
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 As of Datasette ``1.0a14``, the root level ``/metadata.json`` endpoint has been removed. Metadata for tables will become available through currently in-development extras in a future alpha.
 
+.. _upgrade_guide_v1_metadata_method_removed:
+
 The ``metadata()`` method on the Datasette class has been removed
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -102,24 +127,3 @@ Instead, one should use the following methods on a Datasette class:
 - :ref:`get_database_metadata() <datasette_get_database_metadata>`
 - :ref:`get_resource_metadata() <datasette_get_resource_metadata>`
 - :ref:`get_column_metadata() <datasette_get_column_metadata>`
-
-New endpoint for SQL queries
-----------------------------
-
-Previously, if you wanted to run SQL code using the Datasette HTTP API, you could call an endpoint that looked like:
-
-::
-
-    # DEPRECATED: Older endpoint for Datasette 0.XX
-    curl http://localhost:8001/_memory?sql=select+123
-
-However, in Datasette 1.0, the endpoint was slightly changed to:
-
-::
-
-    # ✅ Datasette 1.0 and beyond
-    curl http://localhost:8001/_memory/-/query?sql=select+123
-
-Specifically, now there's a ``/-/query`` "action" that should be used.
-
-**This isn't a breaking change.** API calls to the older ``/database?sql=...`` endpoint will redirect to the new ``database/-/query?sql=...`` endpoint. However, documentations and example will use the new query endpoint, so it is recommended to use that instead.

From 8dc9bfa2ab878a8ea4827f5dec547aa6747430d3 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Aug 2024 12:58:10 -0700
Subject: [PATCH 26/48] Markup tweak for track_event docs

---
 docs/plugin_hooks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index d3764dcc98..5f735a31c9 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -1951,7 +1951,7 @@ This example plugin logs details of all events to standard error:
 
 The function can also return an async function which will be awaited. This is useful for writing to a database.
 
-This example logs events to a `datasette_events` table in a database called `events`. It uses the `startup()` hook to create that table if it does not exist.
+This example logs events to a ``datasette_events`` table in a database called ``events``. It uses the :ref:`plugin_hook_startup` hook to create that table if it does not exist.
 
 .. code-block:: python
 

From 2b0a61ee192c8c0b85a4ddbf9c76b1b6e61e9fc0 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Aug 2024 13:53:55 -0700
Subject: [PATCH 27/48] Rename metadata tables and add schema to docs, refs
 #2382

---
 datasette/app.py               |  16 ++---
 datasette/utils/internal_db.py |  62 +++++++++---------
 datasette/views/table.py       |   4 +-
 docs/internals.rst             | 116 ++++++++++++++++++++++++++++++---
 docs/metadata_doc.py           |  23 +++++++
 5 files changed, 173 insertions(+), 48 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 5b8f910ce3..1f9e9d3009 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -684,7 +684,7 @@ async def get_instance_metadata(self):
               SELECT
                 key,
                 value
-              FROM datasette_metadata_instance_entries
+              FROM metadata_instance
             """
         )
         return dict(rows)
@@ -695,7 +695,7 @@ async def get_database_metadata(self, database_name: str):
               SELECT
                 key,
                 value
-              FROM datasette_metadata_database_entries
+              FROM metadata_databases
               WHERE database_name = ?
             """,
             [database_name],
@@ -708,7 +708,7 @@ async def get_resource_metadata(self, database_name: str, resource_name: str):
               SELECT
                 key,
                 value
-              FROM datasette_metadata_resource_entries
+              FROM metadata_resources
               WHERE database_name = ?
                 AND resource_name = ?
             """,
@@ -724,7 +724,7 @@ async def get_column_metadata(
               SELECT
                 key,
                 value
-              FROM datasette_metadata_column_entries
+              FROM metadata_columns
               WHERE database_name = ?
                 AND resource_name = ?
                 AND column_name = ?
@@ -737,7 +737,7 @@ async def set_instance_metadata(self, key: str, value: str):
         # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)
         await self.get_internal_database().execute_write(
             """
-              INSERT INTO datasette_metadata_instance_entries(key, value)
+              INSERT INTO metadata_instance(key, value)
                 VALUES(?, ?)
                 ON CONFLICT(key) DO UPDATE SET value = excluded.value;
             """,
@@ -748,7 +748,7 @@ async def set_database_metadata(self, database_name: str, key: str, value: str):
         # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)
         await self.get_internal_database().execute_write(
             """
-              INSERT INTO datasette_metadata_database_entries(database_name, key, value)
+              INSERT INTO metadata_databases(database_name, key, value)
                 VALUES(?, ?, ?)
                 ON CONFLICT(database_name, key) DO UPDATE SET value = excluded.value;
             """,
@@ -761,7 +761,7 @@ async def set_resource_metadata(
         # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)
         await self.get_internal_database().execute_write(
             """
-              INSERT INTO datasette_metadata_resource_entries(database_name, resource_name, key, value)
+              INSERT INTO metadata_resources(database_name, resource_name, key, value)
                 VALUES(?, ?, ?, ?)
                 ON CONFLICT(database_name, resource_name, key) DO UPDATE SET value = excluded.value;
             """,
@@ -779,7 +779,7 @@ async def set_column_metadata(
         # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)
         await self.get_internal_database().execute_write(
             """
-              INSERT INTO datasette_metadata_column_entries(database_name, resource_name, column_name, key, value)
+              INSERT INTO metadata_columns(database_name, resource_name, column_name, key, value)
                 VALUES(?, ?, ?, ?, ?)
                 ON CONFLICT(database_name, resource_name, column_name, key) DO UPDATE SET value = excluded.value;
             """,
diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py
index 6a5e08cba5..626dd1377f 100644
--- a/datasette/utils/internal_db.py
+++ b/datasette/utils/internal_db.py
@@ -68,37 +68,39 @@ async def init_internal_db(db):
 
 async def initialize_metadata_tables(db):
     await db.execute_write_script(
-        """
-              CREATE TABLE IF NOT EXISTS datasette_metadata_instance_entries(
-                key text,
-                value text,
-                unique(key)
-              );
-
-              CREATE TABLE IF NOT EXISTS datasette_metadata_database_entries(
-                database_name text,
-                key text,
-                value text,
-                unique(database_name, key)
-              );
-
-              CREATE TABLE IF NOT EXISTS datasette_metadata_resource_entries(
-                database_name text,
-                resource_name text,
-                key text,
-                value text,
-                unique(database_name, resource_name, key)
-              );
-
-              CREATE TABLE IF NOT EXISTS datasette_metadata_column_entries(
-                database_name text,
-                resource_name text,
-                column_name text,
-                key text,
-                value text,
-                unique(database_name, resource_name, column_name, key)
-              );
+        textwrap.dedent(
+            """
+        CREATE TABLE IF NOT EXISTS metadata_instance (
+            key text,
+            value text,
+            unique(key)
+        );
+
+        CREATE TABLE IF NOT EXISTS metadata_databases (
+            database_name text,
+            key text,
+            value text,
+            unique(database_name, key)
+        );
+
+        CREATE TABLE IF NOT EXISTS metadata_resources (
+            database_name text,
+            resource_name text,
+            key text,
+            value text,
+            unique(database_name, resource_name, key)
+        );
+
+        CREATE TABLE IF NOT EXISTS metadata_columns (
+            database_name text,
+            resource_name text,
+            column_name text,
+            key text,
+            value text,
+            unique(database_name, resource_name, column_name, key)
+        );
             """
+        )
     )
 
 
diff --git a/datasette/views/table.py b/datasette/views/table.py
index e3bfb26060..fa2c80deae 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -153,7 +153,7 @@ async def display_columns_and_rows(
           SELECT
             column_name,
             value
-          FROM datasette_metadata_column_entries
+          FROM metadata_columns
           WHERE database_name = ?
             AND resource_name = ?
             AND key = 'description'
@@ -1499,7 +1499,7 @@ async def extra_metadata():
               SELECT
                 column_name,
                 value
-              FROM datasette_metadata_column_entries
+              FROM metadata_columns
               WHERE database_name = ?
                 AND resource_name = ?
                 AND key = 'description'
diff --git a/docs/internals.rst b/docs/internals.rst
index 0eb95e8938..8ecf432669 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -523,7 +523,7 @@ await .get_instance_metadata(self)
 ----------------------------------
 
 Returns metadata keys and values for the entire Datasette instance as a dictionary.
-Internally queries the ``datasette_metadata_instance_entries`` table inside the :ref:`internal database <internals_internal>`.
+Internally queries the ``metadata_instance`` table inside the :ref:`internal database <internals_internal>`.
 
 .. _datasette_get_database_metadata:
 
@@ -534,7 +534,7 @@ await .get_database_metadata(self, database_name)
     The name of the database to query.
 
 Returns metadata keys and values for the specified database as a dictionary.
-Internally queries the ``datasette_metadata_database_entries`` table inside the :ref:`internal database <internals_internal>`.
+Internally queries the ``metadata_databases`` table inside the :ref:`internal database <internals_internal>`.
 
 .. _datasette_get_resource_metadata:
 
@@ -548,7 +548,7 @@ await .get_resource_metadata(self, database_name, resource_name)
 
 Returns metadata keys and values for the specified "resource" as a dictionary.
 A "resource" in this context can be a table, view, or canned query.
-Internally queries the ``datasette_metadata_resource_entries`` table inside the :ref:`internal database <internals_internal>`.
+Internally queries the ``metadata_resources`` table inside the :ref:`internal database <internals_internal>`.
 
 .. _datasette_get_column_metadata:
 
@@ -564,7 +564,7 @@ await .get_column_metadata(self, database_name, resource_name, column_name)
 
 
 Returns metadata keys and values for the specified column, resource, and table as a dictionary.
-Internally queries the ``datasette_metadata_column_entries`` table inside the :ref:`internal database <internals_internal>`.
+Internally queries the ``metadata_columns`` table inside the :ref:`internal database <internals_internal>`.
 
 .. _datasette_set_instance_metadata:
 
@@ -578,7 +578,7 @@ await .set_instance_metadata(self, key, value)
 
 Adds a new metadata entry for the entire Datasette instance.
 Any previous instance-level metadata entry with the same ``key`` will be overwritten.
-Internally upserts the value into the  the ``datasette_metadata_instance_entries`` table inside the :ref:`internal database <internals_internal>`.
+Internally upserts the value into the  the ``metadata_instance`` table inside the :ref:`internal database <internals_internal>`.
 
 .. _datasette_set_database_metadata:
 
@@ -594,7 +594,7 @@ await .set_database_metadata(self, database_name, key, value)
 
 Adds a new metadata entry for the specified database.
 Any previous database-level metadata entry with the same ``key`` will be overwritten.
-Internally upserts the value into the  the ``datasette_metadata_database_entries`` table inside the :ref:`internal database <internals_internal>`.
+Internally upserts the value into the  the ``metadata_databases`` table inside the :ref:`internal database <internals_internal>`.
 
 
 .. _datasette_set_resource_metadata:
@@ -613,7 +613,7 @@ await .set_resource_metadata(self, database_name, resource_name, key, value)
 
 Adds a new metadata entry for the specified "resource".
 Any previous resource-level metadata entry with the same ``key`` will be overwritten.
-Internally upserts the value into the  the ``datasette_metadata_resource_entries`` table inside the :ref:`internal database <internals_internal>`.
+Internally upserts the value into the  the ``metadata_resources`` table inside the :ref:`internal database <internals_internal>`.
 
 
 .. _datasette_set_column_metadata:
@@ -634,7 +634,7 @@ await .set_column_metadata(self, database_name, resource_name, column_name, key,
 
 Adds a new metadata entry for the specified column.
 Any previous column-level metadata entry with the same ``key`` will be overwritten.
-Internally upserts the value into the  the ``datasette_metadata_column_entries`` table inside the :ref:`internal database <internals_internal>`.
+Internally upserts the value into the  the ``metadata_columns`` table inside the :ref:`internal database <internals_internal>`.
 
 
 
@@ -1338,6 +1338,8 @@ Datasette maintains an "internal" SQLite database used for configuration, cachin
 
 Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases.
 
+Metadata is stored in tables ``metadata_instance``, ``metadata_databases``, ``metadata_resources`` and ``metadata_columns``. Plugins can interact with these tables via the ``get_*_metadata`` and ``set_*_metadata`` methods.
+
 The internal database is not exposed in the Datasette application by default, which means private data can safely be stored without worry of accidentally leaking information through the default Datasette interface and API. However, other plugins do have full read and write access to the internal database.
 
 Plugins can access this database by calling ``internal_db = datasette.get_internal_database()`` and then executing queries using the :ref:`Database API <internals_database>`.
@@ -1349,6 +1351,104 @@ Plugin authors are asked to practice good etiquette when using the internal data
 3. Use temporary tables or shared in-memory attached databases when possible.
 4. Avoid implementing features that could expose private data stored in the internal database by other plugins.
 
+.. _internals_internal_schema:
+
+Internal database schema
+------------------------
+
+The internal database schema is as follows:
+
+.. [[[cog
+    from metadata_doc import internal_schema
+    internal_schema(cog)
+.. ]]]
+
+.. code-block:: sql
+
+    CREATE TABLE catalog_databases (
+        database_name TEXT PRIMARY KEY,
+        path TEXT,
+        is_memory INTEGER,
+        schema_version INTEGER
+    );
+    CREATE TABLE catalog_tables (
+        database_name TEXT,
+        table_name TEXT,
+        rootpage INTEGER,
+        sql TEXT,
+        PRIMARY KEY (database_name, table_name),
+        FOREIGN KEY (database_name) REFERENCES databases(database_name)
+    );
+    CREATE TABLE catalog_columns (
+        database_name TEXT,
+        table_name TEXT,
+        cid INTEGER,
+        name TEXT,
+        type TEXT,
+        "notnull" INTEGER,
+        default_value TEXT, -- renamed from dflt_value
+        is_pk INTEGER, -- renamed from pk
+        hidden INTEGER,
+        PRIMARY KEY (database_name, table_name, name),
+        FOREIGN KEY (database_name) REFERENCES databases(database_name),
+        FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name)
+    );
+    CREATE TABLE catalog_indexes (
+        database_name TEXT,
+        table_name TEXT,
+        seq INTEGER,
+        name TEXT,
+        "unique" INTEGER,
+        origin TEXT,
+        partial INTEGER,
+        PRIMARY KEY (database_name, table_name, name),
+        FOREIGN KEY (database_name) REFERENCES databases(database_name),
+        FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name)
+    );
+    CREATE TABLE catalog_foreign_keys (
+        database_name TEXT,
+        table_name TEXT,
+        id INTEGER,
+        seq INTEGER,
+        "table" TEXT,
+        "from" TEXT,
+        "to" TEXT,
+        on_update TEXT,
+        on_delete TEXT,
+        match TEXT,
+        PRIMARY KEY (database_name, table_name, id, seq),
+        FOREIGN KEY (database_name) REFERENCES databases(database_name),
+        FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name)
+    );
+    CREATE TABLE metadata_instance (
+        key text,
+        value text,
+        unique(key)
+    );
+    CREATE TABLE metadata_databases (
+        database_name text,
+        key text,
+        value text,
+        unique(database_name, key)
+    );
+    CREATE TABLE metadata_resources (
+        database_name text,
+        resource_name text,
+        key text,
+        value text,
+        unique(database_name, resource_name, key)
+    );
+    CREATE TABLE metadata_columns (
+        database_name text,
+        resource_name text,
+        column_name text,
+        key text,
+        value text,
+        unique(database_name, resource_name, column_name, key)
+    );
+
+.. [[[end]]]
+
 .. _internals_utils:
 
 The datasette.utils module
diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py
index ad85bf5295..031b3ddd21 100644
--- a/docs/metadata_doc.py
+++ b/docs/metadata_doc.py
@@ -40,3 +40,26 @@ def config_example(
     cog.out("    .. code-block:: json\n\n")
     cog.out(textwrap.indent(json.dumps(data, indent=2), "        "))
     cog.out("\n")
+
+
+def internal_schema(cog):
+    import asyncio
+    from datasette.app import Datasette
+    from sqlite_utils import Database
+
+    ds = Datasette()
+    db = ds.get_internal_database()
+
+    def get_schema(conn):
+        return Database(conn).schema
+
+    async def inner():
+        await ds.invoke_startup()
+        await ds._refresh_schemas()
+        return await db.execute_fn(get_schema)
+
+    schema = asyncio.run(inner())
+    cog.out("\n.. code-block:: sql")
+    cog.out("\n\n")
+    cog.out(textwrap.indent(schema, "    "))
+    cog.out("\n\n")

From 78ce105413b36f43c3a2dc8589444d814196740a Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Aug 2024 14:07:16 -0700
Subject: [PATCH 28/48] Markup fix

---
 docs/upgrade_guide.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/upgrade_guide.rst b/docs/upgrade_guide.rst
index add2f9db99..f983fb2d1e 100644
--- a/docs/upgrade_guide.rst
+++ b/docs/upgrade_guide.rst
@@ -19,6 +19,7 @@ New URL for SQL queries
 Prior to ``1.0a14`` the URL for executing a SQL query looked like this:
 
 ::
+
     /databasename?sql=select+1
     # Or for JSON:
     /databasename.json?sql=select+1

From 5bd6853bf489b32c49d9c610182de603604c4ffd Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Aug 2024 14:08:15 -0700
Subject: [PATCH 29/48] Release notes for 1.0a14, refs #2381

Refs #2306, #2307, #2311, #2319, #2341, #2348, #2352, #2353, #2358, #2359, #2360, #2375
---
 docs/changelog.rst | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index dab17bc64a..09d7d88853 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,26 @@
 Changelog
 =========
 
+.. _v1_0_a14:
+
+1.0a14 (2024-08-05)
+-------------------
+
+This alpha introduces significant changes to Datasette's :ref:`metadata` system, some of which represent breaking changes in advance of the full 1.0 release. The new :ref:`upgrade_guide` document provides detailed coverage of those breaking changes and how they affect plugin authors and Datasette API consumers.
+
+- The ``/databasename?sql=`` interface and JSON API for executing arbitrary SQL queries can now be found at ``/databasename/-/query?sql=``. Requests with a ``?sql=`` parameter to the old endpoints will be redirected. Thanks, `Alex Garcia <https://github.com/asg017>`__. (:issue:`2360`)
+- Metadata about tables, databases, instances and columns is now stored in :ref:`internals_internal`. Thanks, Alex Garcia. (:issue:`2341`)
+- Database write connections now execute using the ``IMMEDIATE`` isolation level for SQLite. This should help avoid a rare ``SQLITE_BUSY`` error that could occur when a transaction upgraded to a write mid-flight. (:issue:`2358`)
+- Fix for a bug where canned queries with named parameters could fail against SQLite 3.46. (:issue:`2353`)
+- Datasette now serves ``E-Tag`` headers for static files. Thanks, `Agustin Bacigalup <https://github.com/redraw>`__. (`#2306 <https://github.com/simonw/datasette/pull/2306>`__)
+- Dropdown menus now use a ``z-index`` that should avoid them being hidden by plugins. (:issue:`2311`)
+- Incorrect table and row names are no longer reflected back on the resulting 404 page. (:issue:`2359`)
+- Improved documentation for async usage of the :ref:`plugin_hook_track_event` hook. (:issue:`2319`)
+- Fixed some HTTPX deprecation warnings. (:issue:`2307`)
+- Datasette now serves a ``<html lange="en">`` attribute. Thanks, `Charles Nepote <https://github.com/CharlesNepote>`__. (:issue:`2348`)
+- Datasette's automated tests now run against the maximum and minimum supported versions of SQLite: 3.25 (from September 2018) and 3.46 (from May 2024). Thanks, Alex Garcia. (`#2352 <https://github.com/simonw/datasette/pull/2352>`__)
+- Fixed an issue where clicking twice on the URL output by ``datasette --root`` produced a confusing error. (:issue:`2375`)
+
 .. _v0_64_8:
 
 0.64.8 (2023-06-21)
@@ -671,7 +691,7 @@ Other small fixes
 
 - New ``datasette --uds /tmp/datasette.sock`` option for binding Datasette to a Unix domain socket, see :ref:`proxy documentation <deploying_proxy>` (:issue:`1388`)
 - ``"searchmode": "raw"`` table metadata option for defaulting a table to executing SQLite full-text search syntax without first escaping it, see :ref:`full_text_search_advanced_queries`. (:issue:`1389`)
-- New plugin hook: :ref:`plugin_hook_get_metadata`, for returning custom metadata for an instance, database or table. Thanks, Brandon Roberts! (:issue:`1384`)
+- New plugin hook: ``get_metadata()``, for returning custom metadata for an instance, database or table. Thanks, Brandon Roberts! (:issue:`1384`)
 - New plugin hook: :ref:`plugin_hook_skip_csrf`, for opting out of CSRF protection based on the incoming request. (:issue:`1377`)
 - The :ref:`menu_links() <plugin_hook_menu_links>`, :ref:`table_actions() <plugin_hook_table_actions>` and :ref:`database_actions() <plugin_hook_database_actions>` plugin hooks all gained a new optional ``request`` argument providing access to the current request. (:issue:`1371`)
 - Major performance improvement for Datasette faceting. (:issue:`1394`)

From ff710ed7ebb3de1660c68d667a46095771c00548 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Aug 2024 14:09:12 -0700
Subject: [PATCH 30/48] Typo fix, refs #2381

---
 docs/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 09d7d88853..8ffa66bdf7 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -20,7 +20,7 @@ This alpha introduces significant changes to Datasette's :ref:`metadata` system,
 - Incorrect table and row names are no longer reflected back on the resulting 404 page. (:issue:`2359`)
 - Improved documentation for async usage of the :ref:`plugin_hook_track_event` hook. (:issue:`2319`)
 - Fixed some HTTPX deprecation warnings. (:issue:`2307`)
-- Datasette now serves a ``<html lange="en">`` attribute. Thanks, `Charles Nepote <https://github.com/CharlesNepote>`__. (:issue:`2348`)
+- Datasette now serves a ``<html lang="en">`` attribute. Thanks, `Charles Nepote <https://github.com/CharlesNepote>`__. (:issue:`2348`)
 - Datasette's automated tests now run against the maximum and minimum supported versions of SQLite: 3.25 (from September 2018) and 3.46 (from May 2024). Thanks, Alex Garcia. (`#2352 <https://github.com/simonw/datasette/pull/2352>`__)
 - Fixed an issue where clicking twice on the URL output by ``datasette --root`` produced a confusing error. (:issue:`2375`)
 

From e9f598609bc0c22c296e62f52041d6032e8e8046 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Aug 2024 14:16:11 -0700
Subject: [PATCH 31/48] Fix for codespell recipe

---
 Justfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Justfile b/Justfile
index c8a06b5226..172de444e2 100644
--- a/Justfile
+++ b/Justfile
@@ -15,7 +15,7 @@ export DATASETTE_SECRET := "not_a_secret"
   pipenv run codespell README.md --ignore-words docs/codespell-ignore-words.txt
   pipenv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt
   pipenv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt
-  pipenv run tests --ignore-words docs/codespell-ignore-words.txt
+  pipenv run codespell tests --ignore-words docs/codespell-ignore-words.txt
 
 # Run linters: black, flake8, mypy, cog
 @lint: codespell

From 2e82eb108c82cdd0581c870a509a6e3bbf179643 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Aug 2024 14:16:34 -0700
Subject: [PATCH 32/48] Group docs on get_/set_ metadata methods, refs #2381

---
 docs/internals.rst | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/docs/internals.rst b/docs/internals.rst
index 8ecf432669..4289c8152f 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -516,11 +516,17 @@ Returns the specified database object. Raises a ``KeyError`` if the database doe
 
 Returns a database object for reading and writing to the private :ref:`internal database <internals_internal>`.
 
+.. _datasette_get_set_metadata:
+
+Getting and setting metadata
+----------------------------
+
+Metadata about the instance, databases, tables and columns is stored in tables in :ref:`internals_internal`. The following methods are the supported API for plugins to read and update that stored metadata.
 
 .. _datasette_get_instance_metadata:
 
 await .get_instance_metadata(self)
-----------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Returns metadata keys and values for the entire Datasette instance as a dictionary.
 Internally queries the ``metadata_instance`` table inside the :ref:`internal database <internals_internal>`.
@@ -528,7 +534,7 @@ Internally queries the ``metadata_instance`` table inside the :ref:`internal dat
 .. _datasette_get_database_metadata:
 
 await .get_database_metadata(self, database_name)
-------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``database_name`` - string
     The name of the database to query.
@@ -539,7 +545,7 @@ Internally queries the ``metadata_databases`` table inside the :ref:`internal da
 .. _datasette_get_resource_metadata:
 
 await .get_resource_metadata(self, database_name, resource_name)
---------------------------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``database_name`` - string
     The name of the database to query.
@@ -553,7 +559,7 @@ Internally queries the ``metadata_resources`` table inside the :ref:`internal da
 .. _datasette_get_column_metadata:
 
 await .get_column_metadata(self, database_name, resource_name, column_name)
-------------------------------------------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``database_name`` - string
     The name of the database to query.
@@ -569,7 +575,7 @@ Internally queries the ``metadata_columns`` table inside the :ref:`internal data
 .. _datasette_set_instance_metadata:
 
 await .set_instance_metadata(self, key, value)
---------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``key`` - string
     The metadata entry key to insert (ex ``title``, ``description``, etc.)
@@ -583,7 +589,7 @@ Internally upserts the value into the  the ``metadata_instance`` table inside th
 .. _datasette_set_database_metadata:
 
 await .set_database_metadata(self, database_name, key, value)
-----------------------------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``database_name`` - string
     The database the metadata entry belongs to.
@@ -596,11 +602,10 @@ Adds a new metadata entry for the specified database.
 Any previous database-level metadata entry with the same ``key`` will be overwritten.
 Internally upserts the value into the  the ``metadata_databases`` table inside the :ref:`internal database <internals_internal>`.
 
-
 .. _datasette_set_resource_metadata:
 
 await .set_resource_metadata(self, database_name, resource_name, key, value)
-------------------------------------------------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``database_name`` - string
     The database the metadata entry belongs to.
@@ -615,11 +620,10 @@ Adds a new metadata entry for the specified "resource".
 Any previous resource-level metadata entry with the same ``key`` will be overwritten.
 Internally upserts the value into the  the ``metadata_resources`` table inside the :ref:`internal database <internals_internal>`.
 
-
 .. _datasette_set_column_metadata:
 
 await .set_column_metadata(self, database_name, resource_name, column_name, key, value)
-------------------------------------------------------------------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``database_name`` - string
     The database the metadata entry belongs to.
@@ -636,8 +640,6 @@ Adds a new metadata entry for the specified column.
 Any previous column-level metadata entry with the same ``key`` will be overwritten.
 Internally upserts the value into the  the ``metadata_columns`` table inside the :ref:`internal database <internals_internal>`.
 
-
-
 .. _datasette_add_database:
 
 .add_database(db, name=None, route=None)
@@ -1338,7 +1340,7 @@ Datasette maintains an "internal" SQLite database used for configuration, cachin
 
 Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases.
 
-Metadata is stored in tables ``metadata_instance``, ``metadata_databases``, ``metadata_resources`` and ``metadata_columns``. Plugins can interact with these tables via the ``get_*_metadata`` and ``set_*_metadata`` methods.
+Metadata is stored in tables ``metadata_instance``, ``metadata_databases``, ``metadata_resources`` and ``metadata_columns``. Plugins can interact with these tables via the :ref:`get_*_metadata() and set_*_metadata() methods <datasette_get_set_metadata>`.
 
 The internal database is not exposed in the Datasette application by default, which means private data can safely be stored without worry of accidentally leaking information through the default Datasette interface and API. However, other plugins do have full read and write access to the internal database.
 

From f6bd2bf8b025dcee49248ae7224e242b448f558c Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Mon, 5 Aug 2024 14:30:02 -0700
Subject: [PATCH 33/48] Release 1.0a14

Refs #2306, #2307, #2311, #2319, #2341, #2348, #2352, #2353, #2358, #2359, #2360, #2375

Closes #2381
---
 datasette/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index 38a52c01ec..30f98bd5eb 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "1.0a13"
+__version__ = "1.0a14"
 __version_info__ = tuple(__version__.split("."))

From bf953628bb629d0c287ecb6c93407b29d0239e27 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 14 Aug 2024 14:28:48 -0700
Subject: [PATCH 34/48] Fix bug where -s could reset settings to defaults,
 closes #2389

---
 datasette/cli.py            |  5 ++++-
 datasette/utils/__init__.py |  9 +++++++++
 tests/test_cli.py           | 25 +++++++++++++++++++++++++
 tests/test_utils.py         | 32 ++++++++++++++++++++++++++++++++
 4 files changed, 70 insertions(+), 1 deletion(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 0c8a854102..0f7ce712d9 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -25,6 +25,7 @@
     LoadExtension,
     StartupError,
     check_connection,
+    deep_dict_update,
     find_spatialite,
     parse_metadata,
     ConnectionProblem,
@@ -552,7 +553,9 @@ def serve(
     # Merge in settings from -s/--setting
     if settings:
         settings_updates = pairs_to_nested_config(settings)
-        config_data.update(settings_updates)
+        # Merge recursively, to avoid over-writing nested values
+        # https://github.com/simonw/datasette/issues/2389
+        deep_dict_update(config_data, settings_updates)
 
     kwargs = dict(
         immutables=immutable,
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index c87de5f003..073d6e866a 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1451,3 +1451,12 @@ async def calculate_etag(filepath, chunk_size=4096):
     _etag_cache[filepath] = etag
 
     return etag
+
+
+def deep_dict_update(dict1, dict2):
+    for key, value in dict2.items():
+        if isinstance(value, dict):
+            dict1[key] = deep_dict_update(dict1.get(key, type(value)()), value)
+        else:
+            dict1[key] = value
+    return dict1
diff --git a/tests/test_cli.py b/tests/test_cli.py
index d53e0a5f20..bfb34c576f 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -240,6 +240,31 @@ def test_setting(args):
     assert settings["default_page_size"] == 5
 
 
+def test_setting_compatible_with_config(tmp_path):
+    # https://github.com/simonw/datasette/issues/2389
+    runner = CliRunner()
+    config_path = tmp_path / "config.json"
+    config_path.write_text(
+        '{"settings": {"default_page_size": 5, "sql_time_limit_ms": 50}}', "utf-8"
+    )
+    result = runner.invoke(
+        cli,
+        [
+            "--get",
+            "/-/settings.json",
+            "--config",
+            str(config_path),
+            "--setting",
+            "default_page_size",
+            "10",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    settings = json.loads(result.output)
+    assert settings["default_page_size"] == 10
+    assert settings["sql_time_limit_ms"] == 50
+
+
 def test_plugin_s_overwrite():
     runner = CliRunner()
     plugins_dir = str(pathlib.Path(__file__).parent / "plugins")
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 88a4532ad1..b8d047e93e 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -722,3 +722,35 @@ async def test_calculate_etag(tmp_path):
     utils._etag_cache[path] = "hash"
     assert "hash" == await utils.calculate_etag(path)
     utils._etag_cache.clear()
+
+
+@pytest.mark.parametrize(
+    "dict1,dict2,expected",
+    [
+        # Basic update
+        ({"a": 1, "b": 2}, {"b": 3, "c": 4}, {"a": 1, "b": 3, "c": 4}),
+        # Nested dictionary update
+        (
+            {"a": 1, "b": {"x": 10, "y": 20}},
+            {"b": {"y": 30, "z": 40}},
+            {"a": 1, "b": {"x": 10, "y": 30, "z": 40}},
+        ),
+        # Deep nested update
+        (
+            {"a": {"b": {"c": 1}}},
+            {"a": {"b": {"d": 2}}},
+            {"a": {"b": {"c": 1, "d": 2}}},
+        ),
+        # Update with mixed types
+        (
+            {"a": 1, "b": {"x": 10}},
+            {"b": {"y": 20}, "c": [1, 2, 3]},
+            {"a": 1, "b": {"x": 10, "y": 20}, "c": [1, 2, 3]},
+        ),
+    ],
+)
+def test_deep_dict_update(dict1, dict2, expected):
+    result = utils.deep_dict_update(dict1, dict2)
+    assert result == expected
+    # Check that the original dict1 was modified
+    assert dict1 == expected

From 93067668fe3cba5576fe430c1e547635989ebb9a Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 14 Aug 2024 17:57:13 -0700
Subject: [PATCH 35/48] /-/ alternative URL for homepage, closes #2393

---
 datasette/app.py               |  2 ++
 datasette/templates/index.html |  4 ++++
 datasette/views/index.py       |  4 +++-
 docs/pages.rst                 |  2 ++
 tests/test_html.py             | 22 ++++++++++++++++++++++
 5 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/datasette/app.py b/datasette/app.py
index 1f9e9d3009..8f69ee9881 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1476,6 +1476,8 @@ def add_route(view, regex):
             routes.append((regex, view))
 
         add_route(IndexView.as_view(self), r"/(\.(?P<format>jsono?))?$")
+        add_route(IndexView.as_view(self), r"/-/(\.(?P<format>jsono?))?$")
+        add_route(permanent_redirect("/-/"), r"/-$")
         # TODO: /favicon.ico and /-/static/ deserve far-future cache expires
         add_route(favicon, "/favicon.ico")
 
diff --git a/datasette/templates/index.html b/datasette/templates/index.html
index 6e95126ddc..a3595a39d8 100644
--- a/datasette/templates/index.html
+++ b/datasette/templates/index.html
@@ -2,6 +2,10 @@
 
 {% block title %}{{ metadata.title or "Datasette" }}: {% for database in databases %}{{ database.name }}{% if not loop.last %}, {% endif %}{% endfor %}{% endblock %}
 
+{% block extra_head %}
+{% if noindex %}<meta name="robots" content="noindex">{% endif %}
+{% endblock %}
+
 {% block body_class %}index{% endblock %}
 
 {% block content %}
diff --git a/datasette/views/index.py b/datasette/views/index.py
index a3178f5331..63cc067d58 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -152,8 +152,9 @@ async def get(self, request):
                 extra_links = await await_me_maybe(hook)
                 if extra_links:
                     homepage_actions.extend(extra_links)
+            alternative_homepage = request.path == "/-/"
             return await self.render(
-                ["index.html"],
+                ["default:index.html" if alternative_homepage else "index.html"],
                 request=request,
                 context={
                     "databases": databases,
@@ -166,5 +167,6 @@ async def get(self, request):
                         "top_homepage", self.ds, request
                     ),
                     "homepage_actions": homepage_actions,
+                    "noindex": request.path == "/-/",
                 },
             )
diff --git a/docs/pages.rst b/docs/pages.rst
index 239c9f8027..78d5520fb3 100644
--- a/docs/pages.rst
+++ b/docs/pages.rst
@@ -23,6 +23,8 @@ Add ``/.json`` to the end of the URL for the JSON version of the underlying data
 * `global-power-plants.datasettes.com/.json <https://global-power-plants.datasettes.com/.json>`_
 * `register-of-members-interests.datasettes.com/.json <https://register-of-members-interests.datasettes.com/.json>`_
 
+The index page can also be accessed at ``/-/``, useful for if the default index page has been replaced using an :ref:`index.html custom template <customization_custom_templates>`. The ``/-/`` page will always render the default Datasette ``index.html`` template.
+
 .. _DatabaseView:
 
 Database
diff --git a/tests/test_html.py b/tests/test_html.py
index 5b60d2f544..d648bdf04e 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -1,4 +1,5 @@
 from bs4 import BeautifulSoup as Soup
+from datasette.app import Datasette
 from datasette.utils import allowed_pragmas
 from .fixtures import (  # noqa
     app_client,
@@ -51,6 +52,27 @@ def test_homepage(app_client_two_attached_databases):
     ] == table_links
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("path", ("/", "/-/"))
+async def test_homepage_alternative_location(path, tmp_path_factory):
+    template_dir = tmp_path_factory.mktemp("templates")
+    (template_dir / "index.html").write_text("Custom homepage", "utf-8")
+    datasette = Datasette(template_dir=str(template_dir))
+    response = await datasette.client.get(path)
+    assert response.status_code == 200
+    html = response.text
+    if path == "/":
+        assert html == "Custom homepage"
+    else:
+        assert '<meta name="robots" content="noindex">' in html
+
+
+@pytest.mark.asyncio
+async def test_homepage_alternative_redirect(ds_client):
+    response = await ds_client.get("/-")
+    assert response.status_code == 301
+
+
 @pytest.mark.asyncio
 async def test_http_head(ds_client):
     response = await ds_client.head("/")

From 06d4ffb92e768fe7b088bb18281718dd5e42ad18 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 14 Aug 2024 21:29:16 -0700
Subject: [PATCH 36/48] Custom error on CSRF failures, closes #2390

Uses https://github.com/simonw/asgi-csrf/issues/28
---
 datasette/app.py   | 12 ++++++++++++
 setup.py           |  2 +-
 tests/test_html.py | 14 ++++++++++++++
 3 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/datasette/app.py b/datasette/app.py
index 8f69ee9881..1363bc5c92 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1,3 +1,4 @@
+from asgi_csrf import Errors
 import asyncio
 from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union
 import asgi_csrf
@@ -1657,6 +1658,16 @@ async def setup_db():
                 if not database.is_mutable:
                     await database.table_counts(limit=60 * 60 * 1000)
 
+        async def custom_csrf_error(scope, send, message_id):
+            await asgi_send(
+                send,
+                await self.render_template(
+                    "csrf_error.html",
+                    {"message_id": message_id, "message_name": Errors(message_id).name},
+                ),
+                403,
+            )
+
         asgi = asgi_csrf.asgi_csrf(
             DatasetteRouter(self, routes),
             signing_secret=self._secret,
@@ -1664,6 +1675,7 @@ async def setup_db():
             skip_if_scope=lambda scope: any(
                 pm.hook.skip_csrf(datasette=self, scope=scope)
             ),
+            send_csrf_failed=custom_csrf_error,
         )
         if self.setting("trace_debug"):
             asgi = AsgiTracer(asgi)
diff --git a/setup.py b/setup.py
index c69404f884..923bc82661 100644
--- a/setup.py
+++ b/setup.py
@@ -55,7 +55,7 @@ def get_version():
         "uvicorn>=0.11",
         "aiofiles>=0.4",
         "janus>=0.6.2",
-        "asgi-csrf>=0.9",
+        "asgi-csrf>=0.10",
         "PyYAML>=5.3",
         "mergedeep>=1.1.1",
         "itsdangerous>=1.1",
diff --git a/tests/test_html.py b/tests/test_html.py
index d648bdf04e..c559f0c295 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -1,3 +1,4 @@
+from asgi_csrf import Errors
 from bs4 import BeautifulSoup as Soup
 from datasette.app import Datasette
 from datasette.utils import allowed_pragmas
@@ -1158,3 +1159,16 @@ async def test_database_color(ds_client):
 
             pdb.set_trace()
         assert any(fragment in response.text for fragment in expected_fragments)
+
+
+@pytest.mark.asyncio
+async def test_custom_csrf_error(ds_client):
+    response = await ds_client.post(
+        "/-/messages",
+        data={
+            "message": "A message",
+        },
+        cookies={"csrftoken": "x"},
+    )
+    assert response.status_code == 403
+    assert "Error code is FORM_URLENCODED_MISMATCH." in response.text

From e9d34a99b84762bef4ed6a5dafd86e18b507b32d Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 14 Aug 2024 21:32:57 -0700
Subject: [PATCH 37/48] Missing template from previous commit, refs #2389

---
 datasette/templates/csrf_error.html | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 datasette/templates/csrf_error.html

diff --git a/datasette/templates/csrf_error.html b/datasette/templates/csrf_error.html
new file mode 100644
index 0000000000..7cd4b42b28
--- /dev/null
+++ b/datasette/templates/csrf_error.html
@@ -0,0 +1,13 @@
+{% extends "base.html" %}
+{% block title %}CSRF check failed){% endblock %}
+{% block content %}
+<h1>Form origin check failed</h1>
+
+<p>Your request's origin could not be validated. Please return to the form and submit it again.</p>
+
+<details><summary>Technical details</summary>
+  <p>Developers: consult Datasette's <a href="https://docs.datasette.io/en/latest/internals.html#csrf-protection">CSRF protection documentation</a>.</p>
+  <p>Error code is {{ message_name }}.</p>
+</details>
+
+{% endblock %}

From cf4274f2a3a0e6cddfdd7fe03f526cf9fe8b21a4 Mon Sep 17 00:00:00 2001
From: Alex Garcia <alexsebastian.garcia@gmail.com>
Date: Wed, 14 Aug 2024 21:33:58 -0700
Subject: [PATCH 38/48] less strict requirements to
 content-type=application/json (#2392)

---
 datasette/views/table.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index fa2c80deae..ba0dd4f34e 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -364,7 +364,7 @@ async def _validate_data(self, request, db, table_name, pks, upsert):
         def _errors(errors):
             return None, errors, {}
 
-        if request.headers.get("content-type") != "application/json":
+        if not request.headers.get("content-type").startswith("application/json"):
             # TODO: handle form-encoded data
             return _errors(["Invalid content-type, must be application/json"])
         body = await request.post_body()

From 492378c2a081fedabc1e2fd26bcc0b4ed50f2f83 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Wed, 14 Aug 2024 21:37:40 -0700
Subject: [PATCH 39/48] Test for application/json; charset=utf-8

Refs #2384, #2392
---
 tests/test_api_write.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tests/test_api_write.py b/tests/test_api_write.py
index b442113ba7..9c2b9b4584 100644
--- a/tests/test_api_write.py
+++ b/tests/test_api_write.py
@@ -37,12 +37,22 @@ def _headers(token):
 
 
 @pytest.mark.asyncio
-async def test_insert_row(ds_write):
+@pytest.mark.parametrize(
+    "content_type",
+    (
+        "application/json",
+        "application/json; charset=utf-8",
+    ),
+)
+async def test_insert_row(ds_write, content_type):
     token = write_token(ds_write)
     response = await ds_write.client.post(
         "/data/docs/-/insert",
         json={"row": {"title": "Test", "score": 1.2, "age": 5}},
-        headers=_headers(token),
+        headers={
+            "Authorization": "Bearer {}".format(token),
+            "Content-Type": content_type,
+        },
     )
     expected_row = {"id": 1, "title": "Test", "score": 1.2, "age": 5}
     assert response.status_code == 201

From 160d82f06e670c4d3f780a382faf1e8e7b6263a8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 14 Aug 2024 21:38:33 -0700
Subject: [PATCH 40/48] Bump furo and black (#2385)

Updates `furo` from 2024.7.18 to 2024.8.6
- [Release notes](https://github.com/pradyunsg/furo/releases)
- [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md)
- [Commits](https://github.com/pradyunsg/furo/compare/2024.07.18...2024.08.06)

Updates `black` from 24.4.2 to 24.8.0
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/compare/24.4.2...24.8.0)

---
updated-dependencies:

  dependency-group: python-packages
- dependency-name: furo
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: python-packages
- dependency-name: black
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: python-packages
...

Signed-off-by: dependabot[bot] <support@github.com>

* Pin Sphinx==7.4.7

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Simon Willison <swillison@gmail.com>
---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 923bc82661..22ec796363 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ def get_version():
     extras_require={
         "docs": [
             "Sphinx==7.4.7",
-            "furo==2024.7.18",
+            "furo==2024.8.6",
             "sphinx-autobuild",
             "codespell>=2.2.5",
             "blacken-docs",
@@ -84,7 +84,7 @@ def get_version():
             "pytest-xdist>=2.2.1",
             "pytest-asyncio>=0.17",
             "beautifulsoup4>=4.8.1",
-            "black==24.4.2",
+            "black==24.8.0",
             "blacken-docs==1.18.0",
             "pytest-timeout>=1.4.2",
             "trustme>=0.7",

From 05dfd34fd0dff34b64fb47e0dd1716c8bdbddfac Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 15 Aug 2024 08:48:47 -0700
Subject: [PATCH 41/48] Use text/html for CSRF error page, refs #2390

---
 datasette/app.py   | 5 +++--
 tests/test_html.py | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 1363bc5c92..fa5e90e335 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1661,11 +1661,12 @@ async def setup_db():
         async def custom_csrf_error(scope, send, message_id):
             await asgi_send(
                 send,
-                await self.render_template(
+                content=await self.render_template(
                     "csrf_error.html",
                     {"message_id": message_id, "message_name": Errors(message_id).name},
                 ),
-                403,
+                status=403,
+                content_type="text/html; charset=utf-8",
             )
 
         asgi = asgi_csrf.asgi_csrf(
diff --git a/tests/test_html.py b/tests/test_html.py
index c559f0c295..ae27048633 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -1171,4 +1171,5 @@ async def test_custom_csrf_error(ds_client):
         cookies={"csrftoken": "x"},
     )
     assert response.status_code == 403
+    assert response.headers["content-type"] == "text/html; charset=utf-8"
     assert "Error code is FORM_URLENCODED_MISMATCH." in response.text

From 6d91d082e0a7b1a275fe72549c7e132382986342 Mon Sep 17 00:00:00 2001
From: Alex Garcia <alexsebastian.garcia@gmail.com>
Date: Thu, 15 Aug 2024 13:19:22 -0700
Subject: [PATCH 42/48] Hide shadow tables, don't hide virtual tables

Closes #2296
---
 datasette/database.py            | 115 +++++++++++++++++++++++--------
 tests/test_api.py                |  46 ++++++-------
 tests/test_html.py               |   3 +-
 tests/test_internals_database.py |  49 ++++++++++++-
 4 files changed, 161 insertions(+), 52 deletions(-)

diff --git a/datasette/database.py b/datasette/database.py
index 71c134d1d9..8d51befda7 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -20,6 +20,7 @@
     table_columns,
     table_column_details,
 )
+from .utils.sqlite import sqlite_version
 from .inspect import inspect_hash
 
 connections = threading.local()
@@ -459,22 +460,95 @@ async def foreign_keys_for_table(self, table):
         )
 
     async def hidden_table_names(self):
-        # Mark tables 'hidden' if they relate to FTS virtual tables
-        hidden_tables = [
-            r[0]
-            for r in (
-                await self.execute(
+        hidden_tables = []
+        # Add any tables marked as hidden in config
+        db_config = self.ds.config.get("databases", {}).get(self.name, {})
+        if "tables" in db_config:
+            hidden_tables += [
+                t for t in db_config["tables"] if db_config["tables"][t].get("hidden")
+            ]
+
+        if sqlite_version()[1] >= 37:
+            hidden_tables += [
+                x[0]
+                for x in await self.execute(
+                    """
+                      with shadow_tables as (
+                        select name
+                        from pragma_table_list
+                        where [type] = 'shadow'
+                        order by name
+                      ),
+                      core_tables as (
+                        select name
+                        from sqlite_master
+                        WHERE  name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4')
+                          OR substr(name, 1, 1) == '_'
+                      ),
+                      combined as (
+                        select name from shadow_tables
+                        union all
+                        select name from core_tables
+                      )
+                      select name from combined order by 1
+                    """
+                )
+            ]
+        else:
+            hidden_tables += [
+                x[0]
+                for x in await self.execute(
+                    """
+                      WITH base AS (
+                        SELECT name
+                        FROM sqlite_master
+                        WHERE  name IN ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4')
+                          OR substr(name, 1, 1) == '_'
+                      ),
+                      fts_suffixes AS (
+                        SELECT column1 AS suffix
+                        FROM (VALUES ('_data'), ('_idx'), ('_docsize'), ('_content'), ('_config'))
+                      ),
+                      fts5_names AS (
+                        SELECT name
+                        FROM sqlite_master
+                        WHERE sql LIKE '%VIRTUAL TABLE%USING FTS%'
+                      ),
+                      fts5_shadow_tables AS (
+                        SELECT
+                          printf('%s%s', fts5_names.name, fts_suffixes.suffix) AS name
+                        FROM fts5_names
+                        JOIN fts_suffixes
+                      ),
+                      fts3_suffixes AS (
+                        SELECT column1 AS suffix
+                        FROM (VALUES ('_content'), ('_segdir'), ('_segments'), ('_stat'), ('_docsize'))
+                      ),
+                      fts3_names AS (
+                        SELECT name
+                        FROM sqlite_master
+                        WHERE sql LIKE '%VIRTUAL TABLE%USING FTS3%'
+                          OR sql LIKE '%VIRTUAL TABLE%USING FTS4%'
+                      ),
+                      fts3_shadow_tables AS (
+                        SELECT
+                          printf('%s%s', fts3_names.name, fts3_suffixes.suffix) AS name
+                        FROM fts3_names
+                        JOIN fts3_suffixes
+                      ),
+                      final AS (
+                        SELECT name FROM base
+                        UNION ALL
+                        SELECT name FROM fts5_shadow_tables
+                        UNION ALL
+                        SELECT name FROM fts3_shadow_tables
+                      )
+                      SELECT name FROM final ORDER BY 1
+
                     """
-                select name from sqlite_master
-                where rootpage = 0
-                and (
-                    sql like '%VIRTUAL TABLE%USING FTS%'
-                ) or name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4')
-                or name like '\\_%' escape '\\'
-            """
                 )
-            ).rows
-        ]
+            ]
+
         has_spatialite = await self.execute_fn(detect_spatialite)
         if has_spatialite:
             # Also hide Spatialite internal tables
@@ -503,19 +577,6 @@ async def hidden_table_names(self):
                     )
                 ).rows
             ]
-        # Add any tables marked as hidden in config
-        db_config = self.ds.config.get("databases", {}).get(self.name, {})
-        if "tables" in db_config:
-            hidden_tables += [
-                t for t in db_config["tables"] if db_config["tables"][t].get("hidden")
-            ]
-        # Also mark as hidden any tables which start with the name of a hidden table
-        # e.g. "searchable_fts" implies "searchable_fts_content" should be hidden
-        for table_name in await self.table_names():
-            for hidden_table in hidden_tables[:]:
-                if table_name.startswith(hidden_table):
-                    hidden_tables.append(table_name)
-                    continue
 
         return hidden_tables
 
diff --git a/tests/test_api.py b/tests/test_api.py
index 431ab5cee4..01c9bb79ad 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -389,6 +389,29 @@ async def test_database_page(ds_client):
             },
             "private": False,
         },
+        {
+            "name": "searchable_fts",
+            "columns": [
+                "text1",
+                "text2",
+                "name with . and spaces",
+            ]
+            + (
+                [
+                    "searchable_fts",
+                    "docid",
+                    "__langid",
+                ]
+                if supports_table_xinfo()
+                else []
+            ),
+            "primary_keys": [],
+            "count": 2,
+            "hidden": False,
+            "fts_table": "searchable_fts",
+            "foreign_keys": {"incoming": [], "outgoing": []},
+            "private": False,
+        },
         {
             "name": "searchable_tags",
             "columns": ["searchable_id", "tag"],
@@ -525,29 +548,6 @@ async def test_database_page(ds_client):
             "foreign_keys": {"incoming": [], "outgoing": []},
             "private": False,
         },
-        {
-            "name": "searchable_fts",
-            "columns": [
-                "text1",
-                "text2",
-                "name with . and spaces",
-            ]
-            + (
-                [
-                    "searchable_fts",
-                    "docid",
-                    "__langid",
-                ]
-                if supports_table_xinfo()
-                else []
-            ),
-            "primary_keys": [],
-            "count": 2,
-            "hidden": True,
-            "fts_table": "searchable_fts",
-            "foreign_keys": {"incoming": [], "outgoing": []},
-            "private": False,
-        },
         {
             "name": "searchable_fts_docsize",
             "columns": ["docid", "size"],
diff --git a/tests/test_html.py b/tests/test_html.py
index ae27048633..4d95a8fa07 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -41,13 +41,14 @@ def test_homepage(app_client_two_attached_databases):
     assert "extra database" == h2.text.strip()
     counts_p, links_p = h2.find_all_next("p")[:2]
     assert (
-        "2 rows in 1 table, 5 rows in 4 hidden tables, 1 view" == counts_p.text.strip()
+        "4 rows in 2 tables, 3 rows in 3 hidden tables, 1 view" == counts_p.text.strip()
     )
     # We should only show visible, not hidden tables here:
     table_links = [
         {"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a")
     ]
     assert [
+        {"href": r"/extra+database/searchable_fts", "text": "searchable_fts"},
         {"href": r"/extra+database/searchable", "text": "searchable"},
         {"href": r"/extra+database/searchable_view", "text": "searchable_view"},
     ] == table_links
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index 1c155cf3b3..bc3c8fcfad 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -4,7 +4,7 @@
 
 from datasette.app import Datasette
 from datasette.database import Database, Results, MultipleValues
-from datasette.utils.sqlite import sqlite3
+from datasette.utils.sqlite import sqlite3, sqlite_version
 from datasette.utils import Column
 from .fixtures import app_client, app_client_two_attached_databases_crossdb_enabled
 import pytest
@@ -664,3 +664,50 @@ async def test_in_memory_databases_forbid_writes(app_client):
     # Using db.execute_write() should work:
     await db.execute_write("create table foo (t text)")
     assert await db.table_names() == ["foo"]
+
+
+def pragma_table_list_supported():
+    return sqlite_version()[1] >= 37
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(
+    not pragma_table_list_supported(), reason="Requires PRAGMA table_list support"
+)
+async def test_hidden_tables(app_client):
+    ds = app_client.ds
+    db = ds.add_database(Database(ds, is_memory=True, is_mutable=True))
+    assert await db.hidden_table_names() == []
+    await db.execute("create virtual table f using fts5(a)")
+    assert await db.hidden_table_names() == [
+        "f_config",
+        "f_content",
+        "f_data",
+        "f_docsize",
+        "f_idx",
+    ]
+
+    await db.execute("create virtual table r using rtree(id, amin, amax)")
+    assert await db.hidden_table_names() == [
+        "f_config",
+        "f_content",
+        "f_data",
+        "f_docsize",
+        "f_idx",
+        "r_node",
+        "r_parent",
+        "r_rowid",
+    ]
+
+    await db.execute("create table _hideme(_)")
+    assert await db.hidden_table_names() == [
+        "_hideme",
+        "f_config",
+        "f_content",
+        "f_data",
+        "f_docsize",
+        "f_idx",
+        "r_node",
+        "r_parent",
+        "r_rowid",
+    ]

From 9cb5700d605b8c72930c8bce8b3eaa8a0763cca6 Mon Sep 17 00:00:00 2001
From: Seb Bacon <seb.bacon@gmail.com>
Date: Thu, 15 Aug 2024 21:20:26 +0100
Subject: [PATCH 43/48] bugfix: correctly detect json1 in versions.json (#2327)

Fixes #2326
---
 datasette/app.py  | 4 ++--
 tests/test_api.py | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index fa5e90e335..c2e685eec6 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -69,6 +69,7 @@
     async_call_with_supported_arguments,
     await_me_maybe,
     call_with_supported_arguments,
+    detect_json1,
     display_actor,
     escape_css_string,
     escape_sqlite,
@@ -1172,9 +1173,8 @@ def _versions(self):
         conn = sqlite3.connect(":memory:")
         self._prepare_connection(conn, "_memory")
         sqlite_version = conn.execute("select sqlite_version()").fetchone()[0]
-        sqlite_extensions = {}
+        sqlite_extensions = {"json1": detect_json1(conn)}
         for extension, testsql, hasversion in (
-            ("json1", "SELECT json('{}')", False),
             ("spatialite", "SELECT spatialite_version()", True),
         ):
             try:
diff --git a/tests/test_api.py b/tests/test_api.py
index 01c9bb79ad..fbbe3f675a 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -835,6 +835,10 @@ async def test_versions_json(ds_client):
     assert "version" in data["sqlite"]
     assert "fts_versions" in data["sqlite"]
     assert "compile_options" in data["sqlite"]
+    # By default, the json1 extension is enabled in the SQLite 
+    # provided by the `ubuntu-latest` github actions runner, and 
+    # all versions of SQLite from 3.38.0 onwards
+    assert data["sqlite"]["extensions"]["json1"]
 
 
 @pytest.mark.asyncio

From 53a8ae1871f9650559e0281fd86c6ca8779deec4 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 15 Aug 2024 17:16:47 -0700
Subject: [PATCH 44/48] Applied Black, refs #2327, #2326

---
 tests/test_api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_api.py b/tests/test_api.py
index fbbe3f675a..8a3fcc9245 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -835,8 +835,8 @@ async def test_versions_json(ds_client):
     assert "version" in data["sqlite"]
     assert "fts_versions" in data["sqlite"]
     assert "compile_options" in data["sqlite"]
-    # By default, the json1 extension is enabled in the SQLite 
-    # provided by the `ubuntu-latest` github actions runner, and 
+    # By default, the json1 extension is enabled in the SQLite
+    # provided by the `ubuntu-latest` github actions runner, and
     # all versions of SQLite from 3.38.0 onwards
     assert data["sqlite"]["extensions"]["json1"]
 

From 0dd41efce6a41d19d52308dbd191c94098250b7a Mon Sep 17 00:00:00 2001
From: Alex Garcia <alexsebastian.garcia@gmail.com>
Date: Thu, 15 Aug 2024 21:48:07 -0700
Subject: [PATCH 45/48] skip over "queries" blocks when processing
 database-level metadata items (#2386)

---
 datasette/app.py                  |  2 +-
 tests/test_internals_datasette.py | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/datasette/app.py b/datasette/app.py
index c2e685eec6..1c730a7379 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -456,7 +456,7 @@ async def apply_metadata_json(self):
         # step 2: database-level metadata
         for dbname, db in self._metadata_local.get("databases", {}).items():
             for key, value in db.items():
-                if key == "tables":
+                if key in ("tables", "queries"):
                     continue
                 await self.set_database_metadata(dbname, key, value)
 
diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py
index 2614e02e63..135a90996b 100644
--- a/tests/test_internals_datasette.py
+++ b/tests/test_internals_datasette.py
@@ -173,3 +173,23 @@ async def test_get_permission(ds_client):
     # And test KeyError
     with pytest.raises(KeyError):
         ds.get_permission("missing-permission")
+
+
+@pytest.mark.asyncio
+async def test_apply_metadata_json():
+    ds = Datasette(
+        metadata={
+            "databases": {
+                "legislators": {
+                    "tables": {"offices": {"summary": "office address or sumtin"}},
+                    "queries": {
+                        "millenntial_represetatives": {
+                            "summary": "Social media accounts for current legislators"
+                        }
+                    },
+                }
+            }
+        },
+    )
+    await ds.invoke_startup()
+    assert (await ds.client.get("/")).status_code == 200

From 7d8dd2ac7fbbc53f484e8d383ac7f3656b28aa86 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Thu, 15 Aug 2024 22:03:41 -0700
Subject: [PATCH 46/48] Release 1.0a15

Refs #2296, #2326, #2384, #2386, #2389, #2390, #2393, #2394
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index 30f98bd5eb..cb5d34bf1c 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "1.0a14"
+__version__ = "1.0a15"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 8ffa66bdf7..089ae42518 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,19 @@
 Changelog
 =========
 
+.. _v1_0_a15:
+
+1.0a15 (2024-08-15)
+-------------------
+
+- Datasette now defaults to hiding SQLite "shadow" tables, as seen in extensions such as SQLite FTS and `sqlite-vec <https://github.com/asg017/sqlite-vec>`__. Virtual tables that it makes sense to display, such as FTS core tables, are no longer hidden. Thanks, `Alex Garcia <https://github.com/asg017>`__. (:issue:`2296`)
+- Fixed bug where running Datasette with one or more ``-s/--setting`` options could over-ride settings that were present in ``datasette.yml``. (:issue:`2389`)
+- The Datasette homepage is now duplicated at ``/-/``, using the default ``index.html`` template. This ensures that the information on that page is still accessible even if the Datasette homepage has been customized using a custom ``index.html`` template, for example on sites like `datasette.io <https://datasette.io/>`__. (:issue:`2393`)
+- Failed CSRF checks now display a more user-friendly error page. (:issue:`2390`)
+- Fixed a bug where the ``json1`` extension was not correctly detected on the ``/-/versions`` page. Thanks, `Seb Bacon <https://github.com/sebbacon>`__. (:issue:`2326`)
+- Fixed a bug where the Datasette write API did not correctly accept ``Content-Type: application/json; charset=utf-8``. (:issue:`2384`)
+- Fixed a bug where Datasette would fail to start if ``metadata.yml`` contained a ``queries`` block. (`#2386 <https://github.com/simonw/datasette/pull/2386>`__)
+
 .. _v1_0_a14:
 
 1.0a14 (2024-08-05)

From d444b6aad568e3743199b44d4ae978f5a9ce36a4 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 20 Aug 2024 09:34:53 -0700
Subject: [PATCH 47/48] Fix for spacing on index page, closes #2399

---
 datasette/templates/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/templates/index.html b/datasette/templates/index.html
index a3595a39d8..0334927983 100644
--- a/datasette/templates/index.html
+++ b/datasette/templates/index.html
@@ -21,7 +21,7 @@ <h1>{{ metadata.title or "Datasette" }}{% if private %} 🔒{% endif %}</h1>
 {% for database in databases %}
     <h2 style="padding-left: 10px; border-left: 10px solid #{{ database.color }}"><a href="{{ urls.database(database.name) }}">{{ database.name }}</a>{% if database.private %} 🔒{% endif %}</h2>
     <p>
-        {% if database.show_table_row_counts %}{{ "{:,}".format(database.table_rows_sum) }} rows in {% endif %}{{ database.tables_count }} table{% if database.tables_count != 1 %}s{% endif %}{% if database.tables_count and database.hidden_tables_count %}, {% endif -%}
+        {% if database.show_table_row_counts %}{{ "{:,}".format(database.table_rows_sum) }} rows in {% endif %}{{ database.tables_count }} table{% if database.tables_count != 1 %}s{% endif %}{% if database.hidden_tables_count %}, {% endif -%}
         {% if database.hidden_tables_count -%}
             {% if database.show_table_row_counts %}{{ "{:,}".format(database.hidden_table_rows_sum) }} rows in {% endif %}{{ database.hidden_tables_count }} hidden table{% if database.hidden_tables_count != 1 %}s{% endif -%}
         {% endif -%}

From 39dfc7d7d77b901d7fef5481e91465fa48b88799 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Tue, 20 Aug 2024 19:03:33 -0700
Subject: [PATCH 48/48] Removed units functionality and Pint dependency

Closes #2400, unblocks #2320
---
 datasette/app.py                 |  1 -
 datasette/filters.py             | 24 +-------
 datasette/utils/__init__.py      |  1 -
 datasette/views/base.py          |  4 --
 datasette/views/row.py           |  1 -
 datasette/views/table.py         | 13 +----
 docs/metadata.rst                | 94 --------------------------------
 setup.py                         |  1 -
 tests/fixtures.py                | 11 ----
 tests/plugins/my_plugin.py       | 13 +++--
 tests/test_api.py                | 12 ----
 tests/test_internals_database.py |  1 -
 tests/test_plugins.py            |  4 +-
 tests/test_table_api.py          | 16 ------
 14 files changed, 14 insertions(+), 182 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 1c730a7379..d7d20016c0 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -37,7 +37,6 @@
 
 from .events import Event
 from .views import Context
-from .views.base import ureg
 from .views.database import database_download, DatabaseView, TableCreateView, QueryView
 from .views.index import IndexView
 from .views.special import (
diff --git a/datasette/filters.py b/datasette/filters.py
index 585d4865b6..67d4170bfc 100644
--- a/datasette/filters.py
+++ b/datasette/filters.py
@@ -368,12 +368,8 @@ class Filters:
     )
     _filters_by_key = {f.key: f for f in _filters}
 
-    def __init__(self, pairs, units=None, ureg=None):
-        if units is None:
-            units = {}
+    def __init__(self, pairs):
         self.pairs = pairs
-        self.units = units
-        self.ureg = ureg
 
     def lookups(self):
         """Yields (lookup, display, no_argument) pairs"""
@@ -413,20 +409,6 @@ def selections(self):
     def has_selections(self):
         return bool(self.pairs)
 
-    def convert_unit(self, column, value):
-        """If the user has provided a unit in the query, convert it into the column unit, if present."""
-        if column not in self.units:
-            return value
-
-        # Try to interpret the value as a unit
-        value = self.ureg(value)
-        if isinstance(value, numbers.Number):
-            # It's just a bare number, assume it's the column unit
-            return value
-
-        column_unit = self.ureg(self.units[column])
-        return value.to(column_unit).magnitude
-
     def build_where_clauses(self, table):
         sql_bits = []
         params = {}
@@ -434,9 +416,7 @@ def build_where_clauses(self, table):
         for column, lookup, value in self.selections():
             filter = self._filters_by_key.get(lookup, None)
             if filter:
-                sql_bit, param = filter.where_clause(
-                    table, column, self.convert_unit(column, value), i
-                )
+                sql_bit, param = filter.where_clause(table, column, value, i)
                 sql_bits.append(sql_bit)
                 if param is not None:
                     if not isinstance(param, list):
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 073d6e866a..7d248ee584 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1368,7 +1368,6 @@ def recursive_move(src, dest, path=None):
     "fts_table",
     "fts_pk",
     "searchmode",
-    "units",
 )
 
 
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 2e78b0a5a1..aee06b01af 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -8,8 +8,6 @@
 from markupsafe import escape
 
 
-import pint
-
 from datasette.database import QueryInterrupted
 from datasette.utils.asgi import Request
 from datasette.utils import (
@@ -32,8 +30,6 @@
     BadRequest,
 )
 
-ureg = pint.UnitRegistry()
-
 
 class DatasetteError(Exception):
     def __init__(
diff --git a/datasette/views/row.py b/datasette/views/row.py
index 6180446f1b..d802994e40 100644
--- a/datasette/views/row.py
+++ b/datasette/views/row.py
@@ -103,7 +103,6 @@ async def template_data():
             "columns": columns,
             "primary_keys": resolved.pks,
             "primary_key_values": pk_values,
-            "units": (await self.ds.table_config(database, table)).get("units", {}),
         }
 
         if "foreign_key_tables" in (request.args.get("_extras") or "").split(","):
diff --git a/datasette/views/table.py b/datasette/views/table.py
index ba0dd4f34e..d71efeb0ec 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -43,7 +43,7 @@
 from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response
 from datasette.filters import Filters
 import sqlite_utils
-from .base import BaseView, DatasetteError, ureg, _error, stream_csv
+from .base import BaseView, DatasetteError, _error, stream_csv
 from .database import QueryView
 
 LINK_WITH_LABEL = (
@@ -292,14 +292,6 @@ async def display_columns_and_rows(
                         ),
                     )
                 )
-            elif column in table_config.get("units", {}) and value != "":
-                # Interpret units using pint
-                value = value * ureg(table_config["units"][column])
-                # Pint uses floating point which sometimes introduces errors in the compact
-                # representation, which we have to round off to avoid ugliness. In the vast
-                # majority of cases this rounding will be inconsequential. I hope.
-                value = round(value.to_compact(), 6)
-                display_value = markupsafe.Markup(f"{value:~P}".replace(" ", "&nbsp;"))
             else:
                 display_value = str(value)
                 if truncate_cells and len(display_value) > truncate_cells:
@@ -1017,7 +1009,6 @@ async def table_view_data(
         nofacet = True
 
     table_metadata = await datasette.table_config(database_name, table_name)
-    units = table_metadata.get("units", {})
 
     # Arguments that start with _ and don't contain a __ are
     # special - things like ?_search= - and should not be
@@ -1029,7 +1020,7 @@ async def table_view_data(
                 filter_args.append((key, v))
 
     # Build where clauses from query string arguments
-    filters = Filters(sorted(filter_args), units, ureg)
+    filters = Filters(sorted(filter_args))
     where_clauses, params = filters.build_where_clauses(table_name)
 
     # Execute filters_from_request plugin hooks - including the default
diff --git a/docs/metadata.rst b/docs/metadata.rst
index f3ca68ac87..a3fa404091 100644
--- a/docs/metadata.rst
+++ b/docs/metadata.rst
@@ -205,100 +205,6 @@ These will be displayed at the top of the table page, and will also show in the
 
 You can see an example of how these look at `latest.datasette.io/fixtures/roadside_attractions <https://latest.datasette.io/fixtures/roadside_attractions>`__.
 
-Specifying units for a column
------------------------------
-
-Datasette supports attaching units to a column, which will be used when displaying
-values from that column. SI prefixes will be used where appropriate.
-
-Column units are configured in the metadata like so:
-
-.. [[[cog
-    metadata_example(cog, {
-        "databases": {
-            "database1": {
-                "tables": {
-                    "example_table": {
-                        "units": {
-                            "column1": "metres",
-                            "column2": "Hz"
-                        }
-                    }
-                }
-            }
-        }
-    })
-.. ]]]
-
-.. tab:: metadata.yaml
-
-    .. code-block:: yaml
-
-        databases:
-          database1:
-            tables:
-              example_table:
-                units:
-                  column1: metres
-                  column2: Hz
-
-
-.. tab:: metadata.json
-
-    .. code-block:: json
-
-        {
-          "databases": {
-            "database1": {
-              "tables": {
-                "example_table": {
-                  "units": {
-                    "column1": "metres",
-                    "column2": "Hz"
-                  }
-                }
-              }
-            }
-          }
-        }
-.. [[[end]]]
-
-
-Units are interpreted using Pint_, and you can see the full list of available units in
-Pint's `unit registry`_. You can also add `custom units`_ to the metadata, which will be
-registered with Pint:
-
-.. [[[cog
-    metadata_example(cog, {
-        "custom_units": [
-            "decibel = [] = dB"
-        ]
-    })
-.. ]]]
-
-.. tab:: metadata.yaml
-
-    .. code-block:: yaml
-
-        custom_units:
-        - decibel = [] = dB
-
-
-.. tab:: metadata.json
-
-    .. code-block:: json
-
-        {
-          "custom_units": [
-            "decibel = [] = dB"
-          ]
-        }
-.. [[[end]]]
-
-.. _Pint: https://pint.readthedocs.io/
-.. _unit registry: https://github.com/hgrecco/pint/blob/master/pint/default_en.txt
-.. _custom units: http://pint.readthedocs.io/en/latest/defining.html
-
 .. _metadata_default_sort:
 
 Setting a default sort order
diff --git a/setup.py b/setup.py
index 22ec796363..47d796a3da 100644
--- a/setup.py
+++ b/setup.py
@@ -50,7 +50,6 @@ def get_version():
         "httpx>=0.20",
         'importlib_resources>=1.3.1; python_version < "3.9"',
         'importlib_metadata>=4.6; python_version < "3.10"',
-        "pint>=0.9",
         "pluggy>=1.0",
         "uvicorn>=0.11",
         "aiofiles>=0.4",
diff --git a/tests/fixtures.py b/tests/fixtures.py
index af6b610ba2..0539b7c895 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -379,7 +379,6 @@ def generate_sortable_rows(num):
                     ],
                 },
                 "no_primary_key": {"sortable_columns": [], "hidden": True},
-                "units": {"units": {"distance": "m", "frequency": "Hz"}},
                 "primary_key_multiple_columns_explicit_label": {
                     "label_column": "content2"
                 },
@@ -507,16 +506,6 @@ def generate_sortable_rows(num):
   FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id)
 );
 
-CREATE TABLE units (
-  pk integer primary key,
-  distance int,
-  frequency int
-);
-
-INSERT INTO units VALUES (1, 1, 100);
-INSERT INTO units VALUES (2, 5000, 2500);
-INSERT INTO units VALUES (3, 100000, 75000);
-
 CREATE TABLE tags (
     tag TEXT PRIMARY KEY
 );
diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py
index 4ca4f989a0..e87353ea25 100644
--- a/tests/plugins/my_plugin.py
+++ b/tests/plugins/my_plugin.py
@@ -5,18 +5,21 @@
 from datasette.utils import path_with_added_args
 from datasette.utils.asgi import asgi_send_json, Response
 import base64
-import pint
 import json
-import urllib
-
-ureg = pint.UnitRegistry()
+import urllib.parse
 
 
 @hookimpl
 def prepare_connection(conn, database, datasette):
     def convert_units(amount, from_, to_):
         """select convert_units(100, 'm', 'ft');"""
-        return (amount * ureg(from_)).to(to_).to_tuple()[0]
+        # Convert meters to feet
+        if from_ == "m" and to_ == "ft":
+            return amount * 3.28084
+        # Convert feet to meters
+        if from_ == "ft" and to_ == "m":
+            return amount / 3.28084
+        assert False, "Unsupported conversion"
 
     conn.create_function("convert_units", 3, convert_units)
 
diff --git a/tests/test_api.py b/tests/test_api.py
index 8a3fcc9245..91f07563a7 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -528,16 +528,6 @@ async def test_database_page(ds_client):
             },
             "private": False,
         },
-        {
-            "name": "units",
-            "columns": ["pk", "distance", "frequency"],
-            "primary_keys": ["pk"],
-            "count": 3,
-            "hidden": False,
-            "fts_table": None,
-            "foreign_keys": {"incoming": [], "outgoing": []},
-            "private": False,
-        },
         {
             "name": "no_primary_key",
             "columns": ["content", "a", "b", "c"],
@@ -1133,7 +1123,6 @@ async def test_config_json(config, expected):
                                 ],
                             },
                             "no_primary_key": {"sortable_columns": [], "hidden": True},
-                            "units": {"units": {"distance": "m", "frequency": "Hz"}},
                             "primary_key_multiple_columns_explicit_label": {
                                 "label_column": "content2"
                             },
@@ -1168,7 +1157,6 @@ async def test_config_json(config, expected):
                                     "text",
                                 ]
                             },
-                            "units": {"units": {"distance": "m", "frequency": "Hz"}},
                             # These one get redacted:
                             "no_primary_key": "***",
                             "primary_key_multiple_columns_explicit_label": "***",
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index bc3c8fcfad..0020668a4b 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -422,7 +422,6 @@ async def test_table_names(db):
         "table/with/slashes.csv",
         "complex_foreign_keys",
         "custom_foreign_key_label",
-        "units",
         "tags",
         "searchable",
         "searchable_tags",
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 5fad03ad45..aa8f1578c3 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -424,8 +424,8 @@ def extra_template_vars(view_name):
     (
         ("/", "index"),
         ("/fixtures", "database"),
-        ("/fixtures/units", "table"),
-        ("/fixtures/units/1", "row"),
+        ("/fixtures/facetable", "table"),
+        ("/fixtures/facetable/1", "row"),
         ("/-/versions", "json_data"),
         ("/fixtures/-/query?sql=select+1", "database"),
     ),
diff --git a/tests/test_table_api.py b/tests/test_table_api.py
index 11542cb056..615b36ebab 100644
--- a/tests/test_table_api.py
+++ b/tests/test_table_api.py
@@ -720,22 +720,6 @@ async def test_view(ds_client):
     ]
 
 
-@pytest.mark.xfail
-@pytest.mark.asyncio
-async def test_unit_filters(ds_client):
-    response = await ds_client.get(
-        "/fixtures/units.json?_shape=arrays&distance__lt=75km&frequency__gt=1kHz"
-    )
-    assert response.status_code == 200
-    data = response.json()
-
-    assert data["units"]["distance"] == "m"
-    assert data["units"]["frequency"] == "Hz"
-
-    assert len(data["rows"]) == 1
-    assert data["rows"][0][0] == 2
-
-
 def test_page_size_matching_max_returned_rows(
     app_client_returned_rows_matches_page_size,
 ):