Skip to content

Commit

Permalink
Log dead link verification request timings (#4508)
Browse files Browse the repository at this point in the history
* Reduce logging from dead link checks

Each call to `get_aiohttp_session` logs a line about whether a new session is created or being reused. This adds a huge number of log lines with zero value, as for each request, they will be identical. Instead, they just fill up the logs and make them harder to read and visually scan

* Log dead link verification request timings

* Use perf counter
  • Loading branch information
sarayourfriend authored Jun 19, 2024
1 parent 1b9aa0c commit d562f47
Showing 1 changed file with 17 additions and 5 deletions.
22 changes: 17 additions & 5 deletions api/api/utils/check_dead_links/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,22 +43,34 @@ def _get_expiry(status, default):
_timeout = aiohttp.ClientTimeout(total=2)


async def _head(url: str) -> tuple[str, int]:
async def _head(url: str, session: aiohttp.ClientSession) -> tuple[str, int]:
start_time = time.perf_counter()

try:
session = await get_aiohttp_session()
response = await session.head(
url, allow_redirects=False, headers=HEADERS, timeout=_timeout
)
return url, response.status
status = response.status
except (aiohttp.ClientError, asyncio.TimeoutError) as exception:
_log_validation_failure(exception)
return url, -1
status = -1

end_time = time.perf_counter()
logger.info(
"dead_link_validation_timing",
url=url,
status=status,
time=end_time - start_time,
)

return url, status


# https://stackoverflow.com/q/55259755
@async_to_sync
async def _make_head_requests(urls: list[str]) -> list[tuple[str, int]]:
tasks = [asyncio.ensure_future(_head(url)) for url in urls]
session = await get_aiohttp_session()
tasks = [asyncio.ensure_future(_head(url, session)) for url in urls]
responses = asyncio.gather(*tasks)
await responses
return responses.result()
Expand Down

0 comments on commit d562f47

Please sign in to comment.