Skip to content

Commit

Permalink
Merge pull request #691 from biglocalnews/va-679v3
Browse files Browse the repository at this point in the history
Change VA URL
  • Loading branch information
stucka authored Jan 6, 2025
2 parents 0a9e318 + 8595100 commit c7ca12a
Showing 1 changed file with 6 additions and 3 deletions.
9 changes: 6 additions & 3 deletions warn/scrapers/va.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ def scrape(
Returns: the Path where the file is written
"""
cache = Cache(cache_dir)
csv_url = "https://vec.virginia.gov/warn-notices-csv.csv"
# csv_url = "https://vec.virginia.gov/warn-notices-csv.csv"
csv_url = "https://vec.virginia.gov/warn_notices.csv"

"""
This scraper originally tried to parse HTML to find a CSV download link.
Expand Down Expand Up @@ -145,7 +146,9 @@ def scrape(
driver = webdriver.Chrome(options=chromeoptionsholder, service=service)
logger.debug(f"Attempting to fetch {csv_url}")
driver.get(csv_url)
sleep(30) # Give it plenty of time to evaluate Javascript
sleep(45) # Give it plenty of time to evaluate Javascript
driver.get(csv_url)
sleep(10)
driver.quit()

download_dir = os.path.expanduser("~") + "/Downloads"
Expand All @@ -154,7 +157,7 @@ def scrape(
logger.error(f"The download directory is not {download_dir}.")

# get the list of files
list_of_files = glob(download_dir + "/warn-notices-csv*.csv")
list_of_files = glob(download_dir + "/warn_notices*.csv")
if len(list_of_files) == 0:
logger.error(f"No matching files found in {download_dir}.")

Expand Down

0 comments on commit c7ca12a

Please sign in to comment.