be explicit with encoding and content-type

being processed by `httr::content()`.

closes #269; closes #270

NEWS.md

@@ -8,13 +8,22 @@ In the future:
 Version 0.11 (Released ?)
 ==========================================================
 
+### Breaking Changes
+
+* A possible, but unlikely, breaking change is that `kernel_api()` defaults to "text/csv" and UTF-8 encoding.  Formerly, the function would decide on the content-type and encoding.  More details are below in the 'Stability Features' subsection.
 
 ### New Features
 
 * It's now possible to specify the exact `col_types` (a [`readr::cols`](https://readr.tidyverse.org/reference/cols.html) object) that is passed to `readr::read_csv()` inside [`redcap_read_oneshot()`](https://github.com/OuhscBbmc/REDCapR/blob/master/R/redcap-read-oneshot.R). (#258)
 
 * [`reader::type_convert()`](https://readr.tidyverse.org/reference/type_convert.html) is used *after* all the batches are stacked on top of each other.  This way, batches cannot have incompatible data types as they're combined. (#257; thanks @isaactpetersen #245)  Consequently, the `guess_max` parameter in `redcap_read()` no longer serves a purpose, and has been soft-deprecated. (#267)
 
+### Stability Features
+
+* `httr::content()` (which is inside `kernel_api()`) now processes the returned value as "text/csv", by default.  This should prevent strange characters from tricking the process as the internal variable `raw_text` is being formed. See the [httr::content()`](https://httr.r-lib.org/reference/content.html) documentation for a list of possible values for the `content_type` parameter.  (Thanks to great debugging by @vortexing, #269)
+
+* Similarly, `kernel_api()` now has an `encoding` parameter, which defaults to "UTF-8".  (#270)
+
 ### Corrections
 
 * 'checkmate' package is now imported, not suggested (Thanks @dtenenba, #255)

R/kernel-api.R

@@ -5,6 +5,8 @@
 #' @param redcap_uri The URI (uniform resource identifier) of the REDCap project.  Required.
 #' @param post_body List of contents expected by the REDCap API.  Required.
 #' @param config_options  A list of options to pass to `POST` method in the `httr` package.  See the details below.  Optional.
+#' @param encoding  The encoding value passed to [httr::content()].  Defaults to 'UTF-8'.
+#' @param content_type The MIME value passed to [httr::content()].  Defaults to 'text/csv'.
 #'
 #' @return A [utils::packageVersion].
 #'
@@ -28,7 +30,13 @@
 #' read.csv(text=kernel$raw_text, stringsAsFactors=FALSE)
 #' as.list(read.csv(text=kernel$raw_text, stringsAsFactors=FALSE))
 
-kernel_api <- function( redcap_uri, post_body, config_options ) {
+kernel_api <- function(
+  redcap_uri,
+  post_body,
+  config_options,
+  encoding            = "UTF-8",
+  content_type        = "text/csv"
+  ) {
 
   start_time <- Sys.time()
 
@@ -41,8 +49,13 @@ kernel_api <- function( redcap_uri, post_body, config_options ) {
   status_code           <- result$status
   success               <- (status_code==200L)
   # raw_text              <- as.character(httr::content(result, as = "text"))
-  raw_text              <- as.character(httr::content(result, as = "text", type = "text/csv"))
-  # raw_text              <- as.character(httr::content(result, as = "parsed", type = "text/csv"))
+  raw_text              <- httr::content(
+    x           = result,
+    as          = "text",
+    encoding    = encoding,     # UTF-8 is the default parameter value (above)
+    type        = content_type  # text/csv is the default parameter value (above)
+  )
+
   raw_text              <- gsub("\r\n", "\n", raw_text) # Convert all line-endings to linux-style
   elapsed_seconds       <- as.numeric(difftime(Sys.time(), start_time, units="secs"))
 

inst/misc/example.credentials

@@ -1,7 +1,7 @@
 # This is an example credential file for the REDCap API.
 # Each record represents one accessible project for the current user.
 # Make sure that the real file with the sensitive password-like tokens
-# are stored securely.
+# is stored securely.
 #
 redcap_uri,username,project_id,token,comment
 "https://bbmc.ouhsc.edu/redcap/api/","myusername","153","9A81268476645C4E5F03428B8AC3AA7B","simple static (read-only) test project"
@@ -18,3 +18,4 @@ redcap_uri,username,project_id,token,comment
 "https://bbmc.ouhsc.edu/redcap/api/","myusername","999","06DEFB601F9B46847DAA9DF0CFA951B4","DAG"
 "https://bbmc.ouhsc.edu/redcap/api/","myusername","1396","14A41597332864D74460CBBF52EE49A6","potentially problematic values"
 "https://bbmc.ouhsc.edu/redcap/api/","myusername","1400","F187271FC6FD72C3BFCE37990A6BF6A7","Repeating Instruments"
+"https://bbmc.ouhsc.edu/redcap/api/","myusername","1425","221E86DABFEEA233067C6889991B7FBB","Potentially problematic dictionary"

inst/test-data/problematic-dictionary/problematic-dictionary.csv

@@ -0,0 +1,3 @@
+"Variable / Field Name","Form Name","Section Header","Field Type","Field Label","Choices, Calculations, OR Slider Labels","Field Note","Text Validation Type OR Show Slider Number","Text Validation Min","Text Validation Max",Identifier?,"Branching Logic (Show field only if...)","Required Field?","Custom Alignment","Question Number (surveys only)","Matrix Group Name","Matrix Ranking?","Field Annotation"
+record_id,form_1,,text,"Record ID",,,,,,,,,,,,,
+v1,form_1,,dropdown,"Is the laser mounted on the shark?�?�","0, No | 1, Yes | 3, I should find out",,,,,,,,,,,,

tests/testthat/test-metadata-read.R

@@ -8,6 +8,10 @@ credential_super_wide <- REDCapR::retrieve_credential_local(
   path_credential = system.file("misc/example.credentials", package="REDCapR"),
   project_id      = 753
 )
+credential_problem <- REDCapR::retrieve_credential_local(
+  path_credential = system.file("misc/example.credentials", package="REDCapR"),
+  project_id      = 1425
+)
 
 test_that("Metadata Smoke Test", {
   testthat::skip_on_cran()
@@ -34,6 +38,23 @@ test_that("Super-wide", {
   expect_equal(sum(is.na(returned_object$data)), expected=expected_na_cells)
 })
 
+test_that("Problematic Dictionary", {
+  testthat::skip_on_cran()
+  expected_outcome_message <- "The data dictionary describing 2 fields was read from REDCap in \\d+(\\.\\d+\\W|\\W)seconds\\.  The http status code was 200\\."
+  expected_row_count    <- 2L
+  expected_column_count <- 18L
+  expected_na_cells     <- 27L
+
+  expect_message(
+    regexp           = expected_outcome_message,
+    returned_object <- redcap_metadata_read(redcap_uri=credential_problem$redcap_uri, token=credential_problem$token)
+  )
+
+  expect_equal(nrow(returned_object$data), expected=expected_row_count) # dput(returned_object$data)
+  expect_equal(ncol(returned_object$data), expected=expected_column_count)
+  expect_equal(sum(is.na(returned_object$data)), expected=expected_na_cells)
+})
+
 test_that("Metadata Normal", {
   testthat::skip_on_cran()
   expected_data_frame <- structure(list(field_name = c("record_id", "name_first", "name_last",