replace httr with curl::multi_download

r-package/DESCRIPTION

@@ -31,10 +31,10 @@ LazyData: TRUE
 Depends:
     R (>= 3.5.0)
 Imports:
-      curl,
+      curl (>= 5.0.0),
       dplyr (>= 0.8-3),
       data.table,
-      httr (>= 1.4.1),
+      fs,
       methods,
       sf (>= 0.9-3),
       utils

r-package/NEWS.md

@@ -1,3 +1,15 @@
+# geobr v1.9.19999 dev
+
+
+**Minor changes**
+
+- Now using `curl::multi_download()` to download files in parallel
+- {geobr} now imports {fs} to use robust cross-platform file system operations
+- Removed dependency on the {httr} package
+- Simplified internal functions
+
+
+
 # geobr v1.9.0
 
 **Major changes**

r-package/R/geobr.R

@@ -42,7 +42,8 @@ if(getRversion() >= "2.15.1")  utils::globalVariables(c('brazil_2010',
                                                         'temp_meta',
                                                         'group_by',
                                                         'showProgress',
-                                                        'year'
+                                                        'year',
+                                                        'url2'
                                                         ))
 
 .onLoad <- function(lib, pkg) {

r-package/R/list_geobr.R

@@ -14,7 +14,7 @@
 list_geobr <- function(){
 
 # Get readme.md file
-tempf <- file.path(tempdir(), "readme.md")
+tempf <- fs::path(fs::path_temp(), "readme.md")
 
 # check if metadata has already been downloaded
 if (file.exists(tempf) & file.info(tempf)$size != 0) {
@@ -30,9 +30,15 @@ if (file.exists(tempf) & file.info(tempf)$size != 0) {
       )
   if(is.null(check_con) | isFALSE(check_con)){ return(invisible(NULL)) }
 
-  try(silent = TRUE,
-      httr::GET(url= git_url, httr::write_disk(tempf, overwrite = T))
-      )
+  try( silent = TRUE,
+       downloaded_files <- curl::multi_download(
+         urls = git_url,
+         destfiles = tempf,
+         resume = TRUE,
+         progress = FALSE
+       )
+  )
+
   readme <- readLines(tempf, encoding = "UTF-8")
 }
 

r-package/R/lookup_muni.R

@@ -33,7 +33,7 @@
 lookup_muni <- function(name_muni = NULL, code_muni = NULL) {
 
   # create tempfile to save metadata
-  tempf <- file.path(tempdir(), "lookup_muni_2010.csv")
+  tempf <- fs::path(fs::path_temp(), "lookup_muni_2010.csv")
 
   # IF metadata has already been downloaded
   if (file.exists(tempf) &  file.info(tempf)$size != 0) {
@@ -43,7 +43,7 @@ lookup_muni <- function(name_muni = NULL, code_muni = NULL) {
   } else {
 
   # Get metadata with data url addresses
-  temp_meta <- select_metadata(geography="lookup_muni", year=2010, simplified=F)
+  temp_meta <- select_metadata(geography="lookup_muni", year=2010, simplified=FALSE)
 
   # check if download failed
   if (is.null(temp_meta)) { return(invisible(NULL)) }
@@ -69,16 +69,24 @@ lookup_muni <- function(name_muni = NULL, code_muni = NULL) {
   }
 
   # download data
-  try( httr::GET(url=file_url,
-                 httr::write_disk(tempf, overwrite = T),
-                 config = httr::config(ssl_verifypeer = FALSE)
-                 ), silent = TRUE)
+  try( silent = TRUE,
+       downloaded_files <- curl::multi_download(
+         urls = file_url,
+         destfiles = tempf,
+         resume = TRUE,
+         progress = FALSE
+       )
+  )
+
+  # if anything fails, return NULL
+  if (any(!downloaded_files$success | is.na(downloaded_files$success))) {
+    msg <- paste("File cached locally seems to be corrupted. Please download it again.")
+    message(msg)
+    return(invisible(NULL))
+  }
 
   }
 
-  # check if download failed
-  msg <- "Problem connecting to data server. Please try it again in a few minutes."
-  if (!file.exists(tempf) | file.info(tempf)$size == 0) {message(msg); return(invisible(NULL)) }
 
   ### read/return lookup data
   lookup_table_2010 <- utils::read.csv(tempf, stringsAsFactors = F, encoding = 'UTF-8')

r-package/R/read_amazon.R

@@ -30,7 +30,7 @@ read_amazon <- function(year=2012, simplified=TRUE, showProgress=TRUE){
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_biomes.R

@@ -28,7 +28,7 @@ read_biomes <- function(year=2019, simplified=TRUE, showProgress=TRUE){
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_census_tract.R

@@ -70,7 +70,7 @@ read_census_tract <- function(code_tract, year=2010, zone = "urban", simplified=
       file_url <- as.character(temp_meta$download_path)
 
       # download files
-      temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+      temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
       # check if download failed
       if (is.null(temp_sf)) { return(invisible(NULL)) }
@@ -105,7 +105,7 @@ read_census_tract <- function(code_tract, year=2010, zone = "urban", simplified=
 
         }
       # download files
-      sf <- download_gpkg(file_url, progress_bar = showProgress)
+      sf <- download_gpkg(file_url, showProgress = showProgress)
 
       # check if download failed
       if (is.null(sf)) { return(invisible(NULL)) }

r-package/R/read_comparable_areas.R

@@ -55,7 +55,7 @@ read_comparable_areas <- function(start_year=1970, end_year=2010, simplified=TRU
   file_url <- file_url[ file_url %like% target_year]
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_conservation_units.R

@@ -27,7 +27,7 @@ read_conservation_units <- function(date=201909, simplified=TRUE, showProgress=T
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_country.R

@@ -28,7 +28,7 @@ read_country <- function(year=2010, simplified=TRUE, showProgress=TRUE){
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_disaster_risk_area.R

@@ -33,7 +33,7 @@ read_disaster_risk_area <- function(year=2010, simplified=TRUE, showProgress=TRU
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_health_facilities.R

@@ -39,7 +39,7 @@ read_health_facilities <- function(date = 202303, showProgress = TRUE){
     file_url <- as.character(temp_meta$download_path)
 
   # download files
-    temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+    temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_health_region.R

@@ -41,7 +41,7 @@ read_health_region <- function(year=2013, macro=FALSE, simplified=TRUE, showProg
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_immediate_region.R

@@ -42,7 +42,7 @@ read_immediate_region <- function(code_immediate="all", year=2019, simplified=TR
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_indigenous_land.R

@@ -30,7 +30,7 @@ read_indigenous_land <- function(date=201907, simplified=TRUE, showProgress=TRUE
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_intermediate_region.R

@@ -44,7 +44,7 @@ read_intermediate_region <- function(code_intermediate="all", year=2019, simplif
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_meso_region.R

@@ -50,7 +50,7 @@ read_meso_region <- function(code_meso = "all",
     file_url <- as.character(temp_meta$download_path)
 
     # download files
-    temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+    temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
     # check if download failed
     if (is.null(temp_sf)) { return(invisible(NULL)) }
@@ -71,7 +71,7 @@ read_meso_region <- function(code_meso = "all",
 
 
     # download files
-    temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+    temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
     # check if download failed
     if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_metro_area.R

@@ -37,7 +37,7 @@ read_metro_area <- function(year = 2018,
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_micro_region.R

@@ -28,7 +28,10 @@
 #' # Read all micro regions at a given year
 #'   micro <- read_micro_region(code_micro="all", year=2010)
 #'
-read_micro_region <- function(code_micro="all", year=2010, simplified=TRUE, showProgress=TRUE){
+read_micro_region <- function(code_micro="all",
+                              year=2010,
+                              simplified=TRUE,
+                              showProgress=TRUE){
 
   # Get metadata with data url addresses
   temp_meta <- select_metadata(geography="micro_region", year=year, simplified=simplified)
@@ -45,7 +48,7 @@ read_micro_region <- function(code_micro="all", year=2010, simplified=TRUE, show
     file_url <- as.character(temp_meta$download_path)
 
     # download files
-    temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+    temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
     # check if download failed
     if (is.null(temp_sf)) { return(invisible(NULL)) }
@@ -65,7 +68,7 @@ read_micro_region <- function(code_micro="all", year=2010, simplified=TRUE, show
 
 
     # download files
-    sf <- download_gpkg(file_url, progress_bar = showProgress)
+    sf <- download_gpkg(file_url, showProgress = showProgress)
 
     # check if download failed
     if (is.null(sf)) { return(invisible(NULL)) }

r-package/R/read_municipal_seat.R

@@ -27,7 +27,7 @@ read_municipal_seat <- function(year=2010, showProgress=TRUE){
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_municipality.R

@@ -13,7 +13,7 @@
 #'        the `geobr::lookup_muni()` function.
 #' @template simplified
 #' @template showProgress
-
+#' @template cache
 #'
 #' @return An `"sf" "data.frame"` object
 #'
@@ -34,7 +34,8 @@
 read_municipality <- function(code_muni = "all",
                               year = 2010,
                               simplified = TRUE,
-                              showProgress = TRUE) {
+                              showProgress = TRUE,
+                              cache = TRUE) {
 
   # Get metadata with data url addresses
   temp_meta <- select_metadata(geography="municipality", year=year, simplified=simplified)
@@ -51,7 +52,9 @@ read_municipality <- function(code_muni = "all",
       file_url <- as.character(temp_meta$download_path)
 
       # download gpkg
-      temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+      temp_sf <- download_gpkg(file_url = file_url,
+                               showProgress = showProgress,
+                               cache = cache)
 
       # check if download failed
       if (is.null(temp_sf)) { return(invisible(NULL)) }
@@ -112,7 +115,9 @@ read_municipality <- function(code_muni = "all",
       file_url <- as.character(temp_meta$download_path)
 
       # download files
-      temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+      temp_sf <- download_gpkg(file_url = file_url,
+                               showProgress = showProgress,
+                               cache = cache)
 
       # check if download failed
       if (is.null(temp_sf)) { return(invisible(NULL)) }
@@ -131,7 +136,7 @@ read_municipality <- function(code_muni = "all",
     if (is.character(code_muni)){ file_url <- as.character(subset(temp_meta, code_abbrev==substr(code_muni, 1, 2))$download_path) }
 
     # download files
-    sf <- download_gpkg(file_url, progress_bar = showProgress)
+    sf <- download_gpkg(file_url, showProgress = showProgress)
 
     # check if download failed
     if (is.null(sf)) { return(invisible(NULL)) }

r-package/R/read_neighborhood.R

@@ -27,7 +27,7 @@ read_neighborhood <- function(year=2010, simplified=TRUE, showProgress=TRUE){
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_pop_arrangements.R

@@ -28,7 +28,7 @@ read_pop_arrangements <- function(year=2015, simplified=TRUE, showProgress=TRUE)
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_region.R

@@ -28,7 +28,7 @@ read_region <- function(year = 2010,
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_schools.R

@@ -31,7 +31,7 @@ read_schools <- function(year=2020, showProgress=TRUE ){
     file_url <- as.character(temp_meta$download_path)
 
   # download files
-    temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+    temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
     # check if download failed
     if (is.null(temp_sf)) { return(invisible(NULL)) }

r-package/R/read_semiarid.R

@@ -28,7 +28,7 @@ read_semiarid <- function(year=2017, simplified=TRUE, showProgress=TRUE){
   file_url <- as.character(temp_meta$download_path)
 
   # download files
-  temp_sf <- download_gpkg(file_url, progress_bar = showProgress)
+  temp_sf <- download_gpkg(file_url, showProgress = showProgress)
 
   # check if download failed
   if (is.null(temp_sf)) { return(invisible(NULL)) }