Skip to content

Commit

Permalink
force reschedule after putting something in the query pool
Browse files Browse the repository at this point in the history
  • Loading branch information
lonvia committed Jul 13, 2024
1 parent 7343259 commit 836f15f
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 4 deletions.
9 changes: 8 additions & 1 deletion src/nominatim_db/db/query_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@
QueueItem = Optional[Tuple[psycopg.abc.Query, Any]]

class QueryPool:
""" Pool to run SQL queries in parallel asynchronous execution.
The results of the queries is discarded.
"""
def __init__(self, dsn: str, pool_size: int = 1, **conn_args: Any) -> None:
self.wait_time = 0.0
self.query_queue: 'asyncio.Queue[QueueItem]' = asyncio.Queue()
Expand All @@ -29,12 +32,17 @@ def __init__(self, dsn: str, pool_size: int = 1, **conn_args: Any) -> None:


async def put_query(self, query: psycopg.abc.Query, params: Any) -> None:
""" Schedule a query for execution.
"""
tstart = time.time()
await self.query_queue.put((query, params))
self.wait_time += time.time() - tstart
await asyncio.sleep(0)


async def finish(self) -> None:
""" Wait for all queries to finish and close the pool.
"""
for _ in self.pool:
await self.query_queue.put(None)

Expand Down Expand Up @@ -69,4 +77,3 @@ async def __aenter__(self) -> 'QueryPool':

async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
await self.finish()

4 changes: 1 addition & 3 deletions src/nominatim_db/indexer/indexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,11 @@
"""
Main work horse for indexing (computing addresses) the database.
"""
from typing import Optional, List, cast
from typing import cast
import logging
import time
import asyncio

import psycopg
from psycopg_pool import AsyncConnectionPool

from ..db.connection import connect, execute_scalar
from ..db.query_pool import QueryPool
Expand Down

0 comments on commit 836f15f

Please sign in to comment.