Skip to content

Commit

Permalink
Merge pull request #675 from biglocalnews/md-674
Browse files Browse the repository at this point in the history
Patch MD #674
  • Loading branch information
stucka authored Nov 7, 2024
2 parents 87dbf79 + b60b9ca commit 078a6e1
Showing 1 changed file with 5 additions and 2 deletions.
7 changes: 5 additions & 2 deletions warn/scrapers/md.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def scrape(
for href in href_list:
# Request the HTML
url = f"https://www.dllr.state.md.us/employment/{href}"
filename = cache_dir / f"md/{href}.html"
filename = f"md/{href}.html"

if href not in old_pages:
sleep(naptime) # Try to stop blocked connections by being less aggressive
Expand All @@ -92,7 +92,10 @@ def scrape(
cache.write(filename, html)
else:
r = utils.fetch_if_not_cached(
filename, url, headers=request_headers, verify=request_verify
cache_dir / filename,
url,
headers=request_headers,
verify=request_verify,
)
html = cache.read(filename)

Expand Down

0 comments on commit 078a6e1

Please sign in to comment.