Skip to content

Commit

Permalink
Stop using parallel SQL queries for tables
Browse files Browse the repository at this point in the history
Refs:
- #2189
  • Loading branch information
simonw committed Sep 21, 2023
1 parent a4c96d0 commit 12395ba
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 10 deletions.
16 changes: 6 additions & 10 deletions datasette/views/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,10 @@ def __str__(self):
return json.dumps(d, default=repr, indent=2)


async def _gather_parallel(*args):
return await asyncio.gather(*args)


async def _gather_sequential(*args):
async def run_sequential(*args):
# This used to be swappable for asyncio.gather() to run things in
# parallel, but this lead to hard-to-debug locking issues with
# in-memory databases: https://github.com/simonw/datasette/issues/2189
results = []
for fn in args:
results.append(await fn)
Expand Down Expand Up @@ -1183,9 +1182,6 @@ async def table_view_data(
)
rows = rows[:page_size]

# For performance profiling purposes, ?_noparallel=1 turns off asyncio.gather
gather = _gather_sequential if request.args.get("_noparallel") else _gather_parallel

# Resolve extras
extras = _get_extras(request)
if any(k for k in request.args.keys() if k == "_facet" or k.startswith("_facet_")):
Expand Down Expand Up @@ -1249,7 +1245,7 @@ async def extra_facet_results(facet_instances):
if not nofacet:
# Run them in parallel
facet_awaitables = [facet.facet_results() for facet in facet_instances]
facet_awaitable_results = await gather(*facet_awaitables)
facet_awaitable_results = await run_sequential(*facet_awaitables)
for (
instance_facet_results,
instance_facets_timed_out,
Expand Down Expand Up @@ -1282,7 +1278,7 @@ async def extra_suggested_facets(facet_instances):
):
# Run them in parallel
facet_suggest_awaitables = [facet.suggest() for facet in facet_instances]
for suggest_result in await gather(*facet_suggest_awaitables):
for suggest_result in await run_sequential(*facet_suggest_awaitables):
suggested_facets.extend(suggest_result)
return suggested_facets

Expand Down
1 change: 1 addition & 0 deletions docs/internals.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1317,6 +1317,7 @@ This example uses the :ref:`register_routes() <plugin_register_routes>` plugin h
(r"/parallel-queries$", parallel_queries),
]
Note that running parallel SQL queries in this way has `been known to cause problems in the past <https://github.com/simonw/datasette/issues/2189>`__, so treat this example with caution.

Adding ``?_trace=1`` will show that the trace covers both of those child tasks.

Expand Down

0 comments on commit 12395ba

Please sign in to comment.