implement fake_useragent headers to solve #154

biglocalnews · Oct 8, 2021 · 0b4a6b3 · 0b4a6b3
1 parent 9160158
commit 0b4a6b3
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/court_scraper/platforms/oscn/pages/search.py b/court_scraper/platforms/oscn/pages/search.py
@@ -1,16 +1,19 @@
 import logging
 import requests
+from fake_useragent import UserAgent
 
 from court_scraper.utils import dates_for_range
-
 from .base_search import BaseSearch
 from .search_results import SearchResultsPage
 
 
 logger = logging.getLogger(__name__)
-
+ua = UserAgent()
 
 class Search(BaseSearch):
+
+
+
     """General search page for all OK counties.
 
     Supports searches by date, case type and a variety of other
@@ -70,9 +73,10 @@ def _run_search(self, search_params):
         params = self._default_params
         # Always add place to search
         params['db'] = self._place
+        headers = {'User-Agent': ua.random}
         # Add any extra params (typically will include filing date)
         params.update(search_params)
-        response = requests.get(self.url, params=params)
+        response = requests.get(self.url, params=params, headers=headers)
         html = response.text
         page = SearchResultsPage(self.place_id, html)
         return html, page.results

diff --git a/requirements.txt b/requirements.txt
@@ -9,3 +9,4 @@ retrying
 selenium
 sqlalchemy
 typing-extensions
+fake_useragent