Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

added retry logic for all HTTP requests #216

Closed
wants to merge 1 commit into from
Closed

added retry logic for all HTTP requests #216

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions NAMESPACE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@ export(unname_osmdata_sf)
import(sp)
importFrom(Rcpp,evalCpp)
importFrom(curl,has_internet)
importFrom(httr,GET)
importFrom(httr,POST)
importFrom(httr,RETRY)
importFrom(httr,content)
importFrom(httr,stop_for_status)
importFrom(jsonlite,fromJSON)
Expand Down
4 changes: 2 additions & 2 deletions R/features.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ available_features <- function() {
url_ftrs <- "https://wiki.openstreetmap.org/wiki/Map_Features"
if (curl::has_internet ())
{
pg <- xml2::read_html (httr::GET (url_ftrs))
pg <- xml2::read_html (httr::RETRY ("GET", url_ftrs, terminate_on = c(403, 404)))
keys <- xml2::xml_attr (rvest::html_nodes (pg, "a[href^='/wiki/Key']"), #nolint
"title")
unique (sort (gsub ("^Key:", "", keys)))
Expand Down Expand Up @@ -48,7 +48,7 @@ available_tags <- function(feature) {
if (missing (feature))
stop ("Please specify feature")

pg <- xml2::read_html (httr::GET (url_ftrs))
pg <- xml2::read_html (httr::RETRY ("GET", url_ftrs, terminate_on = c(403, 404)))
#tags <- xml2::xml_attr (rvest::html_nodes (pg,
# sprintf("a[title^='Tag:%s']", feature)), "title")
#unique (sort (gsub (sprintf ("Tag:%s=", feature), "",
Expand Down
2 changes: 1 addition & 1 deletion R/getbb.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ getbb <- function(place_name,
if (!silent)
print(q_url)

#res <- httr::POST(base_url, query = query, httr::timeout (100))
#res <- httr::RETRY("POST", base_url, query = query, httr::timeout (100))
res <- httr::RETRY("POST", q_url, times = 10)
txt <- httr::content(res, as = "text", encoding = "UTF-8",
type = "application/xml")
Expand Down
2 changes: 1 addition & 1 deletion R/osmdata-package.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
#' @author Mark Padgham, Bob Rudis, Robin Lovelace, Maëlle Salmon
#' @import sp
#' @importFrom curl has_internet
#' @importFrom httr content GET POST stop_for_status
#' @importFrom httr content RETRY stop_for_status
#' @importFrom jsonlite fromJSON
#' @importFrom lubridate force_tz ymd_hms wday day month year
#' @importFrom magrittr %>%
Expand Down
4 changes: 2 additions & 2 deletions R/overpass-query.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ overpass_query <- function (query, quiet = FALSE, wait = TRUE, pad_wait = 5,
units = 'secs'))) + pad_wait
message (sprintf ('Waiting %s seconds', wait))
Sys.sleep (wait)
res <- httr::POST (overpass_url, body = query)
res <- httr::RETRY ("POST", overpass_url, body = query, terminate_on = c(403, 404))
} else {
stop ('Overpass query unavailable', call. = FALSE)
}
Expand All @@ -190,7 +190,7 @@ overpass_query <- function (query, quiet = FALSE, wait = TRUE, pad_wait = 5,
else
doc <- httr::content (res, as = 'text', encoding = encoding,
type = "application/xml")
# TODO: Just return the direct httr::POST result here and convert in the
# TODO: Just return the direct httr::RETRY() result here and convert in the
# subsequent functions (`osmdata_xml/csv/sp/sf`)?
check_for_error (doc)

Expand Down
6 changes: 3 additions & 3 deletions tests/testthat/test-features.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ if (get_local)
cfm_output_af <<- returnValue()
}
)
res <- httr::GET (url_ftrs)
res <- httr::RETRY ("GET", url_ftrs, terminate_on = c(403, 404))
untrace (curl::curl_fetch_memory)
save (cfm_output_af, file = "../cfm_output_af.rda")
}
Expand All @@ -42,7 +42,7 @@ test_that ("available_features", {
if (!test_all)
{
load ("../cfm_output_af.rda")
stub (available_features, 'httr::GET', function (x)
stub (available_features, 'httr::RETRY', function (...)
cfm_output_af$content )
}
expect_is (available_features (), "character")
Expand All @@ -57,7 +57,7 @@ test_that ("available_tags", {
if (!test_all)
{
load ("../cfm_output_af.rda")
stub (available_tags, 'httr::GET', function (x)
stub (available_tags, 'httr::RETRY', function (...)
cfm_output_af$content )
}
expect_that (length (available_tags ("junk")), equals (0))
Expand Down
6 changes: 3 additions & 3 deletions tests/testthat/test-getbb.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ if (get_local) {
exit = function() {
cfm_output <<- returnValue()
})
res <- httr::GET (the_url)
res <- httr::RETRY ("POST", the_url, terminate_on = c(403, 404))
class (cfm_output) <- 'response'
untrace (curl::curl_fetch_memory)
return (cfm_output)
Expand Down Expand Up @@ -58,7 +58,7 @@ test_that ('getbb-place_name', {
if (!test_all)
{
load("../cfm_output_bb1.rda")
stub (getbb, 'httr::GET', function (x) cfm_output_bb )
stub (getbb, 'httr::RETRY', function (...) cfm_output_bb )
}
res <- getbb (place_name = "Salzburg")
expect_is (res, "matrix")
Expand Down Expand Up @@ -86,7 +86,7 @@ test_that ('getbb-polygon', {
if (!test_all)
{
load("../cfm_output_bb2.rda")
stub (getbb, 'httr::GET', function (x) cfm_output_bb )
stub (getbb, 'httr::RETRY', function (...) cfm_output_bb )
}
res <- getbb (place_name = "Salzburg", format_out = "polygon")
expect_is (res, "list")
Expand Down
4 changes: 2 additions & 2 deletions tests/testthat/test-osmdata.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ get_local <- FALSE
if (get_local)
{
# This test needs to return the results of overpass_query(), not the direct
# httr::POST call, so can't be grabbed with curl_fetch_memory
# httr::RETRY call, so can't be grabbed with curl_fetch_memory
qry <- opq (bbox = c(-0.118, 51.514, -0.115, 51.517))
qry <- add_osm_feature (qry, key = 'highway')
overpass_query_result <- overpass_query (opq_string (qry),
Expand All @@ -33,7 +33,7 @@ if (get_local)
exit = function() {
cfm_output_overpass_query <<- returnValue()
})
res <- httr::POST (base_url, body = opq_string (qry))
res <- httr::RETRY ("POST", base_url, body = opq_string (qry), terminate_on = c(403, 404))
untrace (curl::curl_fetch_memory)
class (cfm_output_overpass_query) <- 'response'
save (cfm_output_overpass_query, file = '../cfm_output_overpass_query.rda')
Expand Down