From 859510017f7f1efc3a6a8afbfc4af4a955a2e529 Mon Sep 17 00:00:00 2001 From: Mike Stucka Date: Mon, 6 Jan 2025 09:47:35 -0500 Subject: [PATCH] Change VA URL --- warn/scrapers/va.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/warn/scrapers/va.py b/warn/scrapers/va.py index 13618ac..554065f 100644 --- a/warn/scrapers/va.py +++ b/warn/scrapers/va.py @@ -49,7 +49,8 @@ def scrape( Returns: the Path where the file is written """ cache = Cache(cache_dir) - csv_url = "https://vec.virginia.gov/warn-notices-csv.csv" + # csv_url = "https://vec.virginia.gov/warn-notices-csv.csv" + csv_url = "https://vec.virginia.gov/warn_notices.csv" """ This scraper originally tried to parse HTML to find a CSV download link. @@ -145,7 +146,9 @@ def scrape( driver = webdriver.Chrome(options=chromeoptionsholder, service=service) logger.debug(f"Attempting to fetch {csv_url}") driver.get(csv_url) - sleep(30) # Give it plenty of time to evaluate Javascript + sleep(45) # Give it plenty of time to evaluate Javascript + driver.get(csv_url) + sleep(10) driver.quit() download_dir = os.path.expanduser("~") + "/Downloads" @@ -154,7 +157,7 @@ def scrape( logger.error(f"The download directory is not {download_dir}.") # get the list of files - list_of_files = glob(download_dir + "/warn-notices-csv*.csv") + list_of_files = glob(download_dir + "/warn_notices*.csv") if len(list_of_files) == 0: logger.error(f"No matching files found in {download_dir}.")