Workaround Geofabrik 404

modules/download.py

@@ -52,7 +52,8 @@ def dl(url, local, logger=OsmoseLog.logger(), min_file_size=10*1024):
         headers["If-Modified-Since"] = open(file_ts).read()
 
     # request fails with a 304 error when the file wasn't modified
-    answer = downloader.get(url, headers=headers)
+    # Retry on 404, workaround Geofabrik update in progress
+    answer = downloader.get(url, headers=headers, session=downloader.requests_retry_session(status_forcelist=downloader.DEFAULT_RETRY_ON + (404, )))
     if answer.status_code == 304:
         logger.log(u"not newer")
         return False

modules/downloader.py

@@ -37,7 +37,9 @@
 HTTP_DATE_FMT = "%a, %d %b %Y %H:%M:%S GMT"
 
 
-def requests_retry_session(retries=3, backoff_factor=1, status_forcelist=(500, 502, 503, 504)):
+DEFAULT_RETRY_ON = (500, 502, 503, 504)
+
+def requests_retry_session(retries=3, backoff_factor=1, status_forcelist=DEFAULT_RETRY_ON):
     session = requests.Session()
     retry = Retry(
         total=retries,
@@ -52,9 +54,11 @@ def requests_retry_session(retries=3, backoff_factor=1, status_forcelist=(500, 5
     return session
 
 
-def get(url, headers={}):
+def get(url, headers={}, session=None):
     headers['User-Agent'] = 'Wget/1.9.1 - http://osmose.openstreetmap.fr'  # Add "Wget" for Dropbox user-agent checker
-    return requests_retry_session().get(url, headers=headers, stream=True)
+    if not session:
+        session = requests_retry_session()
+    return session.get(url, headers=headers, stream=True)
 
 def http_get(url, tmp_file, date_string=None, get=get):
     headers = {}