Skip to content

Commit

Permalink
Add another mutation of source_url to try when downloading (#54)
Browse files Browse the repository at this point in the history
  • Loading branch information
diversemix authored May 24, 2023
1 parent 116b9a8 commit c0a56c1
Showing 1 changed file with 11 additions and 3 deletions.
14 changes: 11 additions & 3 deletions src/navigator_data_ingest/base/api_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,13 +139,21 @@ def upload_document(
def _download_from_source(
session: requests.Session, source_url: str
) -> requests.Response:
# Try the orginal source url
download_response = session.get(source_url, allow_redirects=True, timeout=5)

# TODO this is a hack and we should handle source urls upstream in the backend
if download_response.status_code == 404:
download_response_altered = session.get(source_url.replace("%", ""), allow_redirects=True, timeout=5)
if download_response_altered.status_code == 200:
download_response = download_response_altered
# mutation 1 - remove %
download_response = session.get(
source_url.replace("%", ""), allow_redirects=True, timeout=5
)

if download_response.status_code == 404:
# mutation 2 - replace % with the encoded version, i.e. %25
download_response = session.get(
source_url.replace("%", "%25"), allow_redirects=True, timeout=5
)

if download_response.status_code >= 300:
raise Exception(
Expand Down

0 comments on commit c0a56c1

Please sign in to comment.