diff --git a/DESCRIPTION b/DESCRIPTION index c1103c1d..79758a8a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -57,5 +57,6 @@ Remotes: License: GPL-2 LazyData: TRUE VignetteBuilder: knitr -RoxygenNote: 6.0.1 +Encoding: UTF-8 +RoxygenNote: 6.1.0 Roxygen: list(markdown = TRUE) diff --git a/docs/CONDUCT.html b/docs/CONDUCT.html index 25e8a5a3..8074687d 100644 --- a/docs/CONDUCT.html +++ b/docs/CONDUCT.html @@ -89,6 +89,9 @@ diff --git a/docs/articles/BasicREDCapROperations.html b/docs/articles/BasicREDCapROperations.html index 623a3d83..6aaaadfa 100644 --- a/docs/articles/BasicREDCapROperations.html +++ b/docs/articles/BasicREDCapROperations.html @@ -55,6 +55,9 @@ @@ -96,7 +96,7 @@

Basic REDCapR Operations

-

2018-07-12

+

2018-08-10

Source: vignettes/BasicREDCapROperations.Rmd @@ -115,1183 +115,205 @@

Set project-wide values.

There is some information that is specific to the REDCap project, as opposed to an individual operation. This includes the (1) uri of the server, and the (2) token for the user’s project.

-
library(REDCapR) #Load the package into the current R session.
-uri <- "https://bbmc.ouhsc.edu/redcap/api/"
-token <- "9A81268476645C4E5F03428B8AC3AA7B" #`UnitTestPhiFree` user and simple project (pid 153)
+
library(REDCapR) #Load the package into the current R session.
+uri <- "https://bbmc.ouhsc.edu/redcap/api/"
+token <- "9A81268476645C4E5F03428B8AC3AA7B" #`UnitTestPhiFree` user and simple project (pid 153)

Read all records and fields.

If no information is passed about the desired records or fields, then the entire data set is returned. Only two parameters are required, redcap_uri and token. Unless the verbose parameter is set to FALSE, a message will be printed on the R console with the number of records and fields returned.

-
#Return all records and all variables.
-ds_all_rows_all_fields <- redcap_read(redcap_uri=uri, token=token)$data
+
#Return all records and all variables.
+ds_all_rows_all_fields <- redcap_read(redcap_uri=uri, token=token)$data
The data dictionary describing 16 fields was read from REDCap in 0.4 seconds.  The http status code was 200.
-
5 records and 1 columns were read from REDCap in 0.7 seconds.  The http status code was 200.
-
Starting to read 5 records  at 2018-07-12 00:22:13.
+
5 records and 1 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
+
Starting to read 5 records  at 2018-08-10 20:36:45.
Reading batch 1 of 1, with subjects 1 through 5 (ie, 5 unique subject records).
5 records and 24 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
-
ds_all_rows_all_fields #Inspect the returned dataset
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-record id - -name first - -name last - -address - -telephone - -email - -dob - -age - -sex - -demographics complete - -height - -weight - -bmi - -comments - -mugshot - -health complete - -race 1 - -race 2 - -race 3 - -race 4 - -race 5 - -race 6 - -ethnicity - -race and ethnicity complete -
-1 - -Nutmeg - -Nutmouse - -14 Rose Cottage St. Kenning UK, 323232 - -
  1. 321-1111 -
-		 -nutty@mouse.com - -2003-08-30 - -11 - -0 - -2 - -7.00 - -1 - -204.1 - -Character in a book, with some guessing - -[document] - -1 - -0 - -0 - -0 - -0 - -1 - -0 - -1 - -2 -
-2 - -Tumtum - -Nutmouse - -14 Rose Cottage Blvd. Kenning UK 34243 - -
  1. 321-2222 -
-		 -tummy@mouse.comm - -2003-03-10 - -11 - -1 - -2 - -6.00 - -1 - -277.8 - -A mouse character from a good book - -[document] - -0 - -0 - -0 - -1 - -0 - -1 - -0 - -1 - -0 -
-3 - -Marcus - -Wood - -243 Hill St. Guthrie OK 73402 - -
  1. 321-3333 -
-		 -mw@mwood.net - -1934-04-09 - -80 - -1 - -2 - -180.00 - -80 - -24.7 - -completely made up - -[document] - -2 - -0 - -0 - -0 - -1 - -1 - -0 - -0 - -2 -
-4 - -Trudy - -DAG - -342 Elm Duncanville TX, 75116 - -
  1. 321-4444 -
-		 -peroxide@blonde.com - -1952-11-02 - -61 - -0 - -2 - -165.00 - -54 - -19.8 - -This record doesn’t have a DAG assigned - - - - -So call up Trudy on the telephone Send her a letter in the mail - -[document] - -2 - -0 - -1 - -0 - -0 - -1 - -0 - -1 - -2 -
-5 - -John Lee - -Walker - -Hotel Suite New Orleans LA, 70115 - -
  1. 321-5555 -
-		 -left@hippocket.com - -1955-04-15 - -59 - -1 - -2 - -193.04 - -104 - -27.9 - -Had a hand for trouble and a eye for cash - -He had a gold watch chain and a black mustache - -[document] - -0 - -1 - -0 - -0 - -0 - -0 - -1 - -2 - -2 -
+
ds_all_rows_all_fields #Inspect the returned dataset
+
  record_id name_first name_last                                 address
+1         1     Nutmeg  Nutmouse 14 Rose Cottage St.\nKenning UK, 323232
+2         2     Tumtum  Nutmouse 14 Rose Cottage Blvd.\nKenning UK 34243
+3         3     Marcus      Wood          243 Hill St.\nGuthrie OK 73402
+4         4      Trudy       DAG          342 Elm\nDuncanville TX, 75116
+5         5   John Lee    Walker      Hotel Suite\nNew Orleans LA, 70115
+       telephone               email        dob age sex
+1 (405) 321-1111     nutty@mouse.com 2003-08-30  11   0
+2 (405) 321-2222    tummy@mouse.comm 2003-03-10  11   1
+3 (405) 321-3333        mw@mwood.net 1934-04-09  80   1
+4 (405) 321-4444 peroxide@blonde.com 1952-11-02  61   0
+5 (405) 321-5555  left@hippocket.com 1955-04-15  59   1
+  demographics_complete height weight   bmi
+1                     2   7.00      1 204.1
+2                     2   6.00      1 277.8
+3                     2 180.00     80  24.7
+4                     2 165.00     54  19.8
+5                     2 193.04    104  27.9
+                                                                                                     comments
+1                                                                     Character in a book, with some guessing
+2                                                                          A mouse character from a good book
+3                                                                                          completely made up
+4 This record doesn't have a DAG assigned\n\nSo call up Trudy on the telephone\nSend her a letter in the mail
+5                 Had a hand for trouble and a eye for cash\n\nHe had a gold watch chain and a black mustache
+     mugshot health_complete race___1 race___2 race___3 race___4 race___5
+1 [document]               1        0        0        0        0        1
+2 [document]               0        0        0        1        0        1
+3 [document]               2        0        0        0        1        1
+4 [document]               2        0        1        0        0        1
+5 [document]               0        1        0        0        0        0
+  race___6 ethnicity race_and_ethnicity_complete
+1        0         1                           2
+2        0         1                           0
+3        0         0                           2
+4        0         1                           2
+5        1         2                           2

Read a subset of the records.

If only a subset of the records is desired, the two approaches are shown below. The first is to pass an array (where each element is an ID) to the records parameter. The second is to pass a single string (where the elements are separated by commas) to the records_collapsed parameter.

The first format is more natural for more R users. The second format is what is expected by the REDCap API. If a value for records is specified, but records_collapsed is not specified, then redcap_read_oneshot automatically converts the array into the format needed by the API.

-
#Return only records with IDs of 1 and 3
-desired_records_v1 <- c(1, 3)
-ds_some_rows_v1 <- redcap_read(
-   redcap_uri = uri, 
-   token      = token, 
-   records    = desired_records_v1
-)$data
-
The data dictionary describing 16 fields was read from REDCap in 0.3 seconds.  The http status code was 200.
+
#Return only records with IDs of 1 and 3
+desired_records_v1 <- c(1, 3)
+ds_some_rows_v1 <- redcap_read(
+   redcap_uri = uri, 
+   token      = token, 
+   records    = desired_records_v1
+)$data
+
The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
2 records and 1 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
-
Starting to read 2 records  at 2018-07-12 00:22:15.
+
Starting to read 2 records  at 2018-08-10 20:36:46.
Reading batch 1 of 1, with subjects 1 through 3 (ie, 2 unique subject records).
-
2 records and 24 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
-
#Return only records with IDs of 1 and 3 (alternate way)
-desired_records_v2 <- "1, 3"
-ds_some_rows_v2 <- redcap_read(
-   redcap_uri        = uri, 
-   token             = token, 
-   records_collapsed = desired_records_v2
-)$data
-
The data dictionary describing 16 fields was read from REDCap in 0.3 seconds.  The http status code was 200.
-
2 records and 1 columns were read from REDCap in 0.4 seconds.  The http status code was 200.
-
Starting to read 2 records  at 2018-07-12 00:22:16.
+
2 records and 24 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
+
#Return only records with IDs of 1 and 3 (alternate way)
+desired_records_v2 <- "1, 3"
+ds_some_rows_v2 <- redcap_read(
+   redcap_uri        = uri, 
+   token             = token, 
+   records_collapsed = desired_records_v2
+)$data
+
The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
+
2 records and 1 columns were read from REDCap in 0.5 seconds.  The http status code was 200.
+
Starting to read 2 records  at 2018-08-10 20:36:47.
Reading batch 1 of 1, with subjects 1 through 3 (ie, 2 unique subject records).
2 records and 24 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
-
ds_some_rows_v2 #Inspect the returned dataset
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-record id - -name first - -name last - -address - -telephone - -email - -dob - -age - -sex - -demographics complete - -height - -weight - -bmi - -comments - -mugshot - -health complete - -race 1 - -race 2 - -race 3 - -race 4 - -race 5 - -race 6 - -ethnicity - -race and ethnicity complete -
-1 - -Nutmeg - -Nutmouse - -14 Rose Cottage St. Kenning UK, 323232 - -
  1. 321-1111 -
-		 -nutty@mouse.com - -2003-08-30 - -11 - -0 - -2 - -7 - -1 - -204.1 - -Character in a book, with some guessing - -[document] - -1 - -0 - -0 - -0 - -0 - -1 - -0 - -1 - -2 -
-3 - -Marcus - -Wood - -243 Hill St. Guthrie OK 73402 - -
  1. 321-3333 -
-		 -mw@mwood.net - -1934-04-09 - -80 - -1 - -2 - -180 - -80 - -24.7 - -completely made up - -[document] - -2 - -0 - -0 - -0 - -1 - -1 - -0 - -0 - -2 -
+
ds_some_rows_v2 #Inspect the returned dataset
+
  record_id name_first name_last                                 address
+1         1     Nutmeg  Nutmouse 14 Rose Cottage St.\nKenning UK, 323232
+2         3     Marcus      Wood          243 Hill St.\nGuthrie OK 73402
+       telephone           email        dob age sex demographics_complete
+1 (405) 321-1111 nutty@mouse.com 2003-08-30  11   0                     2
+2 (405) 321-3333    mw@mwood.net 1934-04-09  80   1                     2
+  height weight   bmi                                comments    mugshot
+1      7      1 204.1 Character in a book, with some guessing [document]
+2    180     80  24.7                      completely made up [document]
+  health_complete race___1 race___2 race___3 race___4 race___5 race___6
+1               1        0        0        0        0        1        0
+2               2        0        0        0        1        1        0
+  ethnicity race_and_ethnicity_complete
+1         1                           2
+2         0                           2

Read a subset of the fields.

If only a subset of the fields is desired, then two approaches exist. The first is to pass an array (where each element is an field) to the fields parameter. The second is to pass a single string (where the elements are separated by commas) to the fields_collapsed parameter. Like with records and records_collapsed described above, this function converts the more natural format (ie, fields) to the format required by the API (ie, fields_collapsed) if fields is specified and fields_collapsed is not.

-
#Return only the fields record_id, name_first, and age
-desired_fields_v1 <- c("record_id", "name_first", "age")
-ds_some_fields_v1 <- redcap_read(
-   redcap_uri = uri, 
-   token      = token, 
-   fields     = desired_fields_v1
-)$data
-
The data dictionary describing 16 fields was read from REDCap in 0.3 seconds.  The http status code was 200.
+
#Return only the fields record_id, name_first, and age
+desired_fields_v1 <- c("record_id", "name_first", "age")
+ds_some_fields_v1 <- redcap_read(
+   redcap_uri = uri, 
+   token      = token, 
+   fields     = desired_fields_v1
+)$data
+
The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
5 records and 1 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
-
Starting to read 5 records  at 2018-07-12 00:22:18.
+
Starting to read 5 records  at 2018-08-10 20:36:49.
Reading batch 1 of 1, with subjects 1 through 5 (ie, 5 unique subject records).
-
5 records and 3 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
-
#Return only the fields record_id, name_first, and age (alternate way)
-desired_fields_v2 <- "record_id, name_first, age"
-ds_some_fields_v2 <- redcap_read(
-   redcap_uri       = uri, 
-   token            = token, 
-   fields_collapsed = desired_fields_v2
-)$data
-
The data dictionary describing 16 fields was read from REDCap in 0.3 seconds.  The http status code was 200.
-
5 records and 1 columns were read from REDCap in 0.4 seconds.  The http status code was 200.
-
Starting to read 5 records  at 2018-07-12 00:22:20.
+
5 records and 3 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
+
#Return only the fields record_id, name_first, and age (alternate way)
+desired_fields_v2 <- "record_id, name_first, age"
+ds_some_fields_v2 <- redcap_read(
+   redcap_uri       = uri, 
+   token            = token, 
+   fields_collapsed = desired_fields_v2
+)$data
+
The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
+
5 records and 1 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
+
Starting to read 5 records  at 2018-08-10 20:36:50.
Reading batch 1 of 1, with subjects 1 through 5 (ie, 5 unique subject records).
-
5 records and 3 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
-
ds_some_fields_v2 #Inspect the returned dataset
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-record id - -name first - -age -
-1 - -Nutmeg - -11 -
-2 - -Tumtum - -11 -
-3 - -Marcus - -80 -
-4 - -Trudy - -61 -
-5 - -John Lee - -59 -
+
5 records and 3 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
+
ds_some_fields_v2 #Inspect the returned dataset
+
  record_id name_first age
+1         1     Nutmeg  11
+2         2     Tumtum  11
+3         3     Marcus  80
+4         4      Trudy  61
+5         5   John Lee  59

Read a subset of records, conditioned on the values in some variables.

The two techniques above can be combined when your datasets are large and you don’t want to pull records with certain values. Suppose you want to select subjects from the previous dataset if the were born before 1960 and their weight was over 70kg. Two calls to the server are required. The first call to REDCap pulls all the records, but for only three columns: record_id, dob, and weight. From this subset, identify the records that you want to pull all the data for; in this case, the desired record_id values are 3 & 5. The second call to REDCap pulls all the columns, but only for the identified records.

-
######
-## Step 1: First call to REDCap
-desired_fields_v3 <- c("record_id", "dob", "weight")
-ds_some_fields_v3 <- redcap_read(
-   redcap_uri = uri, 
-   token      = token, 
-   fields     = desired_fields_v3
-)$data
-
The data dictionary describing 16 fields was read from REDCap in 0.4 seconds.  The http status code was 200.
+
######
+## Step 1: First call to REDCap
+desired_fields_v3 <- c("record_id", "dob", "weight")
+ds_some_fields_v3 <- redcap_read(
+   redcap_uri = uri, 
+   token      = token, 
+   fields     = desired_fields_v3
+)$data
+
The data dictionary describing 16 fields was read from REDCap in 0.5 seconds.  The http status code was 200.
5 records and 1 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
-
Starting to read 5 records  at 2018-07-12 00:22:21.
+
Starting to read 5 records  at 2018-08-10 20:36:52.
Reading batch 1 of 1, with subjects 1 through 5 (ie, 5 unique subject records).
-
5 records and 3 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
-
ds_some_fields_v3 #Examine the these three variables.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-record id - -dob - -weight -
-1 - -2003-08-30 - -1 -
-2 - -2003-03-10 - -1 -
-3 - -1934-04-09 - -80 -
-4 - -1952-11-02 - -54 -
-5 - -1955-04-15 - -104 -
-
######
-## Step 2: identify desired records, based on age & weight
-before_1960 <- (ds_some_fields_v3$dob <= as.Date("1960-01-01"))
-heavier_than_70_kg <- (ds_some_fields_v3$weight > 70)
-desired_records_v3 <- ds_some_fields_v3[before_1960 & heavier_than_70_kg, ]$record_id
-
-desired_records_v3 #Peek at IDs of the identified records
+
5 records and 3 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
+
ds_some_fields_v3 #Examine the these three variables.
+
  record_id        dob weight
+1         1 2003-08-30      1
+2         2 2003-03-10      1
+3         3 1934-04-09     80
+4         4 1952-11-02     54
+5         5 1955-04-15    104
+
######
+## Step 2: identify desired records, based on age & weight
+before_1960 <- (ds_some_fields_v3$dob <= as.Date("1960-01-01"))
+heavier_than_70_kg <- (ds_some_fields_v3$weight > 70)
+desired_records_v3 <- ds_some_fields_v3[before_1960 & heavier_than_70_kg, ]$record_id
+
+desired_records_v3 #Peek at IDs of the identified records
[1] 3 5
-
######
-## Step 3: second call to REDCap
-#Return only records that met the age & weight criteria.
-ds_some_rows_v3 <- redcap_read(
-   redcap_uri = uri, 
-   token      = token, 
-   records    = desired_records_v3
-)$data
-
The data dictionary describing 16 fields was read from REDCap in 0.3 seconds.  The http status code was 200.
-
2 records and 1 columns were read from REDCap in 0.4 seconds.  The http status code was 200.
-
Starting to read 2 records  at 2018-07-12 00:22:23.
+
######
+## Step 3: second call to REDCap
+#Return only records that met the age & weight criteria.
+ds_some_rows_v3 <- redcap_read(
+   redcap_uri = uri, 
+   token      = token, 
+   records    = desired_records_v3
+)$data
+
The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
+
2 records and 1 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
+
Starting to read 2 records  at 2018-08-10 20:36:53.
Reading batch 1 of 1, with subjects 3 through 5 (ie, 2 unique subject records).
2 records and 24 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
-
ds_some_rows_v3 #Examine the results.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-record id - -name first - -name last - -address - -telephone - -email - -dob - -age - -sex - -demographics complete - -height - -weight - -bmi - -comments - -mugshot - -health complete - -race 1 - -race 2 - -race 3 - -race 4 - -race 5 - -race 6 - -ethnicity - -race and ethnicity complete -
-3 - -Marcus - -Wood - -243 Hill St. Guthrie OK 73402 - -
  1. 321-3333 -
-		 -mw@mwood.net - -1934-04-09 - -80 - -1 - -2 - -180.00 - -80 - -24.7 - -completely made up - -[document] - -2 - -0 - -0 - -0 - -1 - -1 - -0 - -0 - -2 -
-5 - -John Lee - -Walker - -Hotel Suite New Orleans LA, 70115 - -
  1. 321-5555 -
-		 -left@hippocket.com - -1955-04-15 - -59 - -1 - -2 - -193.04 - -104 - -27.9 - -Had a hand for trouble and a eye for cash - - -He had a gold watch chain and a black mustache - -[document] - -0 - -1 - -0 - -0 - -0 - -0 - -1 - -2 - -2 -
+
ds_some_rows_v3 #Examine the results.
+
  record_id name_first name_last                            address
+1         3     Marcus      Wood     243 Hill St.\nGuthrie OK 73402
+2         5   John Lee    Walker Hotel Suite\nNew Orleans LA, 70115
+       telephone              email        dob age sex
+1 (405) 321-3333       mw@mwood.net 1934-04-09  80   1
+2 (405) 321-5555 left@hippocket.com 1955-04-15  59   1
+  demographics_complete height weight  bmi
+1                     2 180.00     80 24.7
+2                     2 193.04    104 27.9
+                                                                                     comments
+1                                                                          completely made up
+2 Had a hand for trouble and a eye for cash\n\nHe had a gold watch chain and a black mustache
+     mugshot health_complete race___1 race___2 race___3 race___4 race___5
+1 [document]               2        0        0        0        1        1
+2 [document]               0        1        0        0        0        0
+  race___6 ethnicity race_and_ethnicity_complete
+1        0         0                           2
+2        1         2                           2

@@ -1306,18 +328,18 @@

  • The fields_collapsed fields passed to the API. This shows which field subsets, if any, were requested.
  • The elapsed_seconds measures the duration of the call.
    • -
    #Return only the fields record_id, name_first, and age
-all_information <- redcap_read(
-   redcap_uri = uri, 
-   token      = token, 
-   fields     = desired_fields_v1
-)
    -
    The data dictionary describing 16 fields was read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    5 records and 1 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    Starting to read 5 records  at 2018-07-12 00:22:24.
    + +
    The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    5 records and 1 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    Starting to read 5 records  at 2018-08-10 20:36:54.
    Reading batch 1 of 1, with subjects 1 through 5 (ie, 5 unique subject records).
    -
    5 records and 3 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    all_information #Inspect the additional information
    +
    5 records and 3 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    + 
    $data
   record_id name_first age
 1         1     Nutmeg  11
@@ -1333,7 +355,7 @@ 
    
 [1] "200"
 
 $outcome_messages
-[1] "5 records and 3 columns were read from REDCap in 0.3 seconds.  The http status code was 200."
+[1] "5 records and 3 columns were read from REDCap in 0.2 seconds.  The http status code was 200."
 
 $records_collapsed
 [1] ""
@@ -1351,81 +373,78 @@ 
    
 [1] ""
 
 $elapsed_seconds
-[1] 1.503323
    +[1] 1.211472

    Session Information

    For the sake of documentation and reproducibility, the current report was rendered in the following environment. Click the line below to expand.

    -

    Environment

    -
    ─ Session info ──────────────────────────────────────────────────────────
- setting  value                       
- version  R version 3.4.4 (2018-03-15)
- os       Ubuntu 18.04 LTS            
- system   x86_64, linux-gnu           
- ui       X11                         
- language (EN)                        
- collate  en_US.UTF-8                 
- tz       America/Chicago             
- date     2018-07-12                  
+
    Environment 
    
+
    - Session info ----------------------------------------------------------
+ setting  value                                      
+ version  R version 3.5.1 Patched (2018-08-06 r75070)
+ os       Windows >= 8 x64                           
+ system   x86_64, mingw32                            
+ ui       RTerm                                      
+ language (EN)                                       
+ collate  English_United States.1252                 
+ tz       America/Chicago                            
+ date     2018-08-10                                 
 
-─ Packages ──────────────────────────────────────────────────────────────
+- Packages --------------------------------------------------------------
  package     * version     date       source                          
- assertthat    0.2.0       2017-04-11 cran (@0.2.0)                   
- backports     1.1.2       2017-12-13 cran (@1.1.2)                   
- bindr         0.1.1       2018-03-13 CRAN (R 3.4.3)                  
- bindrcpp      0.2.2       2018-03-29 CRAN (R 3.4.3)                  
- checkmate     1.8.6       2018-04-10 Github (mllg/checkmate@489319a) 
- clisymbols    1.2.0       2017-05-21 CRAN (R 3.4.3)                  
- colorspace    1.3-2       2016-12-14 CRAN (R 3.4.3)                  
- commonmark    1.5         2018-04-28 CRAN (R 3.4.4)                  
- crayon        1.3.4       2017-09-16 CRAN (R 3.4.3)                  
- curl          3.2         2018-03-28 CRAN (R 3.4.4)                  
- desc          1.2.0       2018-05-01 CRAN (R 3.4.4)                  
- devtools      1.13.6      2018-06-27 CRAN (R 3.4.4)                  
- digest        0.6.15      2018-01-28 CRAN (R 3.4.3)                  
- dplyr         0.7.6       2018-06-29 CRAN (R 3.4.4)                  
- evaluate      0.10.1      2017-06-24 CRAN (R 3.4.3)                  
- fs            1.2.3       2018-06-08 CRAN (R 3.4.4)                  
- glue          1.2.0       2017-10-29 cran (@1.2.0)                   
- highr         0.7         2018-06-09 CRAN (R 3.4.4)                  
- hms           0.4.2.9000  2018-05-26 Github (tidyverse/hms@14e74ab)  
- htmltools     0.3.6       2017-04-28 CRAN (R 3.4.3)                  
- httr          1.3.1       2017-08-20 CRAN (R 3.4.3)                  
- kableExtra    0.9.0       2018-05-21 CRAN (R 3.4.4)                  
- knitr       * 1.20        2018-02-20 CRAN (R 3.4.3)                  
- magrittr    * 1.5         2014-11-22 cran (@1.5)                     
- MASS          7.3-50      2018-04-30 CRAN (R 3.4.4)                  
- memoise       1.1.0       2017-04-21 CRAN (R 3.4.3)                  
- munsell       0.5.0       2018-06-12 CRAN (R 3.4.4)                  
- pillar        1.2.3       2018-05-25 CRAN (R 3.4.4)                  
- pkgconfig     2.0.1       2017-03-21 cran (@2.0.1)                   
- pkgdown       1.1.0       2018-06-02 CRAN (R 3.4.4)                  
- plyr          1.8.4       2016-06-08 CRAN (R 3.4.3)                  
- purrr         0.2.5       2018-05-29 CRAN (R 3.4.4)                  
- R6            2.2.2       2017-06-17 CRAN (R 3.4.3)                  
- Rcpp          0.12.17     2018-05-18 CRAN (R 3.4.4)                  
- readr         1.2.0       2018-05-26 Github (tidyverse/readr@d6d622b)
- REDCapR     * 0.9.10.9001 2018-07-11 local                           
- rlang         0.2.1       2018-05-30 CRAN (R 3.4.4)                  
- rmarkdown     1.10        2018-06-11 CRAN (R 3.4.4)                  
- roxygen2      6.0.1       2017-02-06 CRAN (R 3.4.4)                  
- rprojroot     1.3-2       2018-01-03 CRAN (R 3.4.3)                  
- rstudioapi    0.7         2017-09-07 CRAN (R 3.4.3)                  
- rvest         0.3.2       2016-06-17 CRAN (R 3.4.3)                  
- scales        0.5.0.9000  2018-03-29 Github (hadley/scales@d767915)  
- sessioninfo   1.0.0       2017-06-21 CRAN (R 3.4.3)                  
- stringi       1.2.3       2018-06-12 CRAN (R 3.4.4)                  
- stringr       1.3.1       2018-05-10 CRAN (R 3.4.4)                  
- tibble        1.4.2       2018-01-22 CRAN (R 3.4.3)                  
- tidyselect    0.2.4       2018-02-26 CRAN (R 3.4.3)                  
- viridisLite   0.3.0       2018-02-01 CRAN (R 3.4.3)                  
- withr         2.1.2       2018-03-29 Github (jimhester/withr@79d7b0d)
- xml2          1.2.0       2018-01-24 CRAN (R 3.4.3)                  
- yaml          2.1.19      2018-05-01 CRAN (R 3.4.4)                  
    
-
    
-
    Report rendered by wibeasley at 2018-07-12, 00:22 -0500 in 15 seconds.
    
+ assertthat    0.2.0       2017-04-11 CRAN (R 3.5.0)                  
+ backports     1.1.2       2017-12-13 CRAN (R 3.5.0)                  
+ bindr         0.1.1       2018-03-13 CRAN (R 3.5.0)                  
+ bindrcpp      0.2.2       2018-03-29 CRAN (R 3.5.0)                  
+ checkmate     1.8.9-9000  2018-08-09 Github (mllg/checkmate@29a1fb9) 
+ clisymbols    1.2.0       2017-05-21 CRAN (R 3.5.0)                  
+ colorspace    1.3-2       2016-12-14 CRAN (R 3.5.0)                  
+ commonmark    1.5         2018-04-28 CRAN (R 3.5.0)                  
+ crayon        1.3.4       2017-09-16 CRAN (R 3.5.0)                  
+ curl          3.2         2018-03-28 CRAN (R 3.5.0)                  
+ desc          1.2.0       2018-05-01 CRAN (R 3.5.0)                  
+ devtools      1.13.6      2018-06-27 CRAN (R 3.5.0)                  
+ digest        0.6.15      2018-01-28 CRAN (R 3.5.0)                  
+ dplyr         0.7.6       2018-06-29 CRAN (R 3.5.1)                  
+ evaluate      0.11        2018-07-17 CRAN (R 3.5.1)                  
+ fs            1.2.5       2018-07-30 CRAN (R 3.5.1)                  
+ glue          1.3.0       2018-07-17 CRAN (R 3.5.1)                  
+ hms           0.4.2.9001  2018-08-09 Github (tidyverse/hms@979286f)  
+ htmltools     0.3.6       2017-04-28 CRAN (R 3.5.0)                  
+ httr          1.3.1       2017-08-20 CRAN (R 3.5.0)                  
+ kableExtra    0.9.0       2018-05-21 CRAN (R 3.5.0)                  
+ knitr       * 1.20        2018-02-20 CRAN (R 3.5.0)                  
+ magrittr    * 1.5         2014-11-22 CRAN (R 3.5.0)                  
+ MASS          7.3-50      2018-04-30 CRAN (R 3.5.1)                  
+ memoise       1.1.0       2017-04-21 CRAN (R 3.5.0)                  
+ munsell       0.5.0       2018-06-12 CRAN (R 3.5.0)                  
+ pillar        1.3.0       2018-07-14 CRAN (R 3.5.1)                  
+ pkgconfig     2.0.1       2017-03-21 CRAN (R 3.5.0)                  
+ pkgdown       1.1.0       2018-06-02 CRAN (R 3.5.1)                  
+ purrr         0.2.5       2018-05-29 CRAN (R 3.5.0)                  
+ R6            2.2.2       2017-06-17 CRAN (R 3.5.0)                  
+ Rcpp          0.12.18     2018-07-23 CRAN (R 3.5.1)                  
+ readr         1.2.0       2018-08-09 Github (tidyverse/readr@4b2e93a)
+ REDCapR     * 0.9.10.9001 2018-08-11 local                           
+ rlang         0.2.1       2018-05-30 CRAN (R 3.5.0)                  
+ rmarkdown     1.10        2018-06-11 CRAN (R 3.5.0)                  
+ roxygen2      6.1.0       2018-07-27 CRAN (R 3.5.1)                  
+ rprojroot     1.3-2       2018-01-03 CRAN (R 3.5.0)                  
+ rstudioapi    0.7         2017-09-07 CRAN (R 3.5.0)                  
+ rvest         0.3.2       2016-06-17 CRAN (R 3.5.0)                  
+ scales        1.0.0       2018-08-09 CRAN (R 3.5.1)                  
+ sessioninfo   1.0.0       2017-06-21 CRAN (R 3.5.0)                  
+ stringi       1.2.4       2018-07-20 CRAN (R 3.5.1)                  
+ stringr       1.3.1       2018-05-10 CRAN (R 3.5.0)                  
+ tibble        1.4.2       2018-01-22 CRAN (R 3.5.0)                  
+ tidyselect    0.2.4       2018-02-26 CRAN (R 3.5.0)                  
+ viridisLite   0.3.0       2018-02-01 CRAN (R 3.5.0)                  
+ withr         2.1.2       2018-03-15 CRAN (R 3.5.0)                  
+ xml2          1.2.0       2018-01-24 CRAN (R 3.5.0)                  
+ yaml          2.2.0       2018-07-25 CRAN (R 3.5.1)
    +

    Report rendered by Will at 2018-08-10, 20:36 -0500 in 12 seconds.

    diff --git a/docs/articles/SecurityDatabase.html b/docs/articles/SecurityDatabase.html index 76e2c077..975beba9 100644 --- a/docs/articles/SecurityDatabase.html +++ b/docs/articles/SecurityDatabase.html @@ -55,6 +55,9 @@ @@ -96,7 +96,7 @@

    Security Database

    -

    2018-07-12

    +

    2018-08-10

    Source: vignettes/SecurityDatabase.Rmd @@ -108,303 +108,333 @@

    2018-07-12

    Description

    -

    The SQL code below adds schemas, a table and two stored procedures to an existing Microsoft SQL Database. This second database is not essential to calling the REDCap API, but it helps manage tokens securely.

    +

    The SQL code below adds schemas, a table and two stored procedures to an existing Microsoft SQL Server database. This second database is not essential to calling the REDCap API, but it helps manage tokens securely.

    This database contains the tokens and other sensitive content (such as passwords, API tokens, and file paths) that should not be stored in a Git repository (even a private Git repository). These passwords can be retrieved by REDCapR::retrieve_credential_mssql().

    Create a DSN on each client

    -

    After executing the SQL code in an existing database, create an ODBC DSN on each client machine that calls the database. Download the most recent drivers (as of Aug 2016, the most recent version is 13.1 for Windows and Linux, then run the wizard. Many values in the wizard will remain at the default values. Here are the important ones to change.

    +

    After executing the SQL code in an existing database, create an ODBC DSN on each client machine that calls the database. Download the most recent drivers (as of Aug 2018, the most recent version is 17 for Windows and Linux), then run the wizard. Many values in the wizard will remain at the default values. Here are the important ones to change.

    1. Set the DSN’s name field to whatever is used in the repository’s R code.
    2. Set the authenticity method to Integrated Windows authentication.
      3. -
    3. Set the default database to the name of the database that containing the tokens i.e., corresponding to the SQL code below in the example).
      4. +
    4. Set the default database to the name of the database that containing the tokens
    +

    In our code below, both DSN and database are named auxiliary_security.

    Note

    -

    We use Microsoft SQL Server, because that fits our University’s infrastructure the easiest. But this approach theoretically can work with any LDAP-enabled database server. Please contact us if your institution is using something other than SQL Server (or a different configuration of these components), and would like help adapting this approach to your infrastructure.

    +

    We use Microsoft SQL Server, because that fits our university’s infrastructure most easily. But this approach theoretically can work with any LDAP-enabled database server. Please contact us if your institution is using something other than SQL Server (or a different configuration of these components), and would like help adapting this approach.

    Create Database

    -

    This SQL code is run once inside an existing database to establish the schemas, table, and stored procedure used by REDCapR::retrieve_credential_mssql().

    -
    ------- SQL code to create necessary components in a Microsoft SQL Sever database -------
-
------------------------------------------------------------------------
--- Create two schemas.
--- The first scehma is accessible by all REDCap API users.
--- The second scehma is restricted to administrators.
---
-CREATE SCHEMA [redcap]
-CREATE SCHEMA [redcap_private]
-GO
-
------------------------------------------------------------------------
--- Create a table to contain the token
---
-CREATE TABLE [redcap_private].[tbl_credential](
-  [id]              [smallint]          NOT NULL,
-  [username]        [varchar](30)       NOT NULL,
-  [project_id]      [smallint]          NOT NULL,
-  [instance]        [varchar](30)       NOT NULL,
-  [token]           [char](32)          NOT NULL,
-  [redcap_uri]      [varchar](255)      NOT NULL,
- CONSTRAINT [PK_credential] PRIMARY KEY CLUSTERED
-(
-  [id] ASC
-)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
-) ON [PRIMARY]
-
-CREATE UNIQUE NONCLUSTERED INDEX [IX_tbl_credential_unique] ON [redcap_private].[tbl_credential]
-(
-  [instance]        ASC,
-  [project_id]      ASC,
-  [username]        ASC
-)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
-GO
-
------------------------------------------------------------------------
--- Create a stored procedure for users to call to retrieve the token.
--- Notice it should a different (and more permissive) schema than the table.
---
-CREATE PROCEDURE [redcap].[prc_credential]
-  -- Add the parameters for the stored procedure here
-
-  @project_id smallint,
-  @instance varchar(30)
-AS
-BEGIN
-  -- SET NOCOUNT ON added to prevent extra result sets from
-  -- interfering with SELECT statements.
-  SET NOCOUNT ON;
-
-  SELECT username, project_id, token, redcap_uri FROM [redcap_private].[tbl_credential]
-  WHERE username=system_user AND project_id=@project_id AND instance=@instance
-END
    +

    This SQL code is run once inside an existing database to establish the schemas, table, and stored procedure used by REDCapR::retrieve_credential_mssql(). In this example, we’ve arbitrarily called the database auxiliary_security.

    +
    ------- SQL code to create necessary components in a Microsoft SQL Sever database -------
+
+-----------------------------------------------------------------------
+-- Create two schemas.
+-- The first scehma is accessible by all REDCap API users.
+-- The second scehma is restricted to administrators.
+--
+CREATE SCHEMA [redcap]
+CREATE SCHEMA [redcap_private]
+GO
+
+-----------------------------------------------------------------------
+-- Create a table to contain the token
+--
+CREATE TABLE [redcap_private].[tbl_credential](
+  id            smallint        NOT NULL,
+  username      varchar(30)     NOT NULL,
+  project_id    smallint        NOT NULL,
+  instance      varchar(30)     NOT NULL,
+  token         char(32)        NOT NULL,
+  redcap_uri    varchar(255)    NOT NULL,
+ CONSTRAINT [PK_credential] PRIMARY KEY CLUSTERED
+(
+  [id] ASC
+)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
+) ON [PRIMARY]
+
+CREATE UNIQUE NONCLUSTERED INDEX [IX_tbl_credential_unique] ON [redcap_private].[tbl_credential]
+(
+  instance        ASC,
+  project_id      ASC,
+  username        ASC
+)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
+GO
+
+-----------------------------------------------------------------------
+-- Create a stored procedure for users to call to retrieve the token.
+-- Notice it should a different (and more permissive) schema than the table.
+--
+CREATE PROCEDURE [redcap].[prc_credential]
+  @project_id smallint,
+  @instance   varchar(30)
+AS
+BEGIN
+  SET NOCOUNT ON;
+
+  SELECT username, project_id, token, redcap_uri FROM [redcap_private].[tbl_credential]
+  WHERE
+    username   = system_user -- The username from the server's OS.
+    AND
+    project_id = @project_id -- Restricts to the desired REDCap project.
+    AND
+    instance   = @instance   -- System accommodates multiple REDCap instances.
+END

    Create user credentials to the auxiliary database

    -

    Add a user’s LDAP account to the SecurityAuxiliary database so that they can query the tables to retrieve their API.

    -

    Notice that this only gives the permissions to retrieve the token. You must still: 1. grant them API privileges to each appropriate REDCap project, and 2. copy the API from the REDCap database into the SecurityAuxiliary database.

    -

    In the future REDCapR may expose a function that allows the user to perform the second step (through a stored procedure).

    -

    Also, do not give typical users authorization for the ‘redcap_private’ schema. The current system allows the to view only their own tokens.

    -
    -----------------------------------------------------------------------
--- Add a OUHSC's user account to the auxiliary_security database so that they can query the tables to retrieve their API.
--- Notice that this only gives the permissions to retrieve the token.  You must still:
---   1) grant them API privileges to each appropriate REDCap project, and
---   2) copy the API from the REDCap database into the  auxiliary_security database.
--- Also, do not give typical users authorization for the 'redcap_private' schema.  The current system allows the to view only their own tokens.
------------------------------------------------------------------------
-
--- STEP #1: Declare the user name.  If everything runs correctly, this should be the only piece of code that you need to modify.
-print 'Step #1 executing....'
-USE [master]
-GO
-DECLARE @qualified_user_name varchar(255); SET @qualified_user_name = '[OUHSC\lsuarez3]'
-print 'Resulting login name: ' + @qualified_user_name; print ''
-
---EXEC sp_helplogins @LoginNamePattern=@qualified_user_name
---SELECT * FROM master..syslogins WHERE name = @qualified_user_name
---SELECT * FROM auxiliary_security.sys.sysusers
---SELECT * FROM sys.database_permissions
---SELECT * FROM sys.server_principals
-
------------------------------------------------------------------------
--- STEP #2: Create a login for the *server*.
-print 'Step #2 executing....'
-DECLARE @sql_create_login nvarchar(max)
-SET @sql_create_login = 'CREATE LOGIN ' + @qualified_user_name + ' FROM WINDOWS WITH DEFAULT_DATABASE=[auxiliary_security]'
-EXECUTE sp_executesql @sql_create_login
-DECLARE @login_count AS INT; SET @login_count = (SELECT COUNT(*) AS login_count FROM master..syslogins WHERE '[' + loginname + ']' = @qualified_user_name)
-print 'Logins matching desired name should equal 1.  It equals: ' + CONVERT(varchar, @login_count); print ''
-
------------------------------------------------------------------------
--- STEP #3: Create a user account for the *data base*, after switching the database under focus to auxiliary_security.
-print 'Step #3 executing....'
-USE [auxiliary_security]
-DECLARE @sql_create_user nvarchar(max)
-SET @sql_create_user = 'CREATE USER ' + @qualified_user_name + ' FOR LOGIN ' + @qualified_user_name
-EXECUTE sp_executesql @sql_create_user
-DECLARE @user_count AS INT; SET @user_count = (SELECT COUNT(*) AS user_count FROM auxiliary_security.sys.sysusers WHERE '[' + name + ']' = @qualified_user_name)
-print 'User accounts matching desired name should equal 1.  It equals: ' + CONVERT(varchar, @user_count); print ''
-
------------------------------------------------------------------------
--- STEP #4: Grant appropriate privileges for the 'redcap' schema.
-print 'Step #4 executing....'
-DECLARE @sql_grant_schema_redcap nvarchar(max)
-SET @sql_grant_schema_redcap = 'GRANT EXECUTE ON SCHEMA::[redcap] TO ' + @qualified_user_name
-EXECUTE sp_executesql @sql_grant_schema_redcap
-print 'Step #4 executed'; print ''
-
------------------------------------------------------------------------
--- STEP #5: Grant appropriate privileges for the 'Security' schema.
-print 'Step #5 executing....'
-DECLARE @sql_grant_schema_security nvarchar(max)
-SET @sql_grant_schema_security = 'GRANT EXECUTE ON SCHEMA::[security] TO ' + @qualified_user_name
-EXECUTE sp_executesql @sql_grant_schema_security
-print 'Step #5 executed'; print ''
-
------------------------------------------------------------------------
--- OPTIONAL STEP: Delete the user from the database (the first line) and then the server (the second line).
--- The person's other database user accounts (besides with the auxiliary_security database) will NOT be automatically deleted by these two lines.
---USE [auxiliary_security]; DROP USER [OUHSC\lsuarez3]
---USE [master]; DROP LOGIN [OUHSC\lsuarez3]
-
------------------------------------------------------------------------
--- REFERENCES & NOTES
-  --The @qualified_user_name must have both (a) the 'OUHSC' domain qualification, and (b) the square brackets (to escape the backslash).
-  --Using sp_executesql to add users: http://www.sqlservercentral.com/Forums/Topic497615-359-1.aspx
-  --Check if a server login exists: http://stackoverflow.com/questions/37275/sql-query-for-logins
-  --Retrieve database users: http://stackoverflow.com/questions/2445444/how-to-get-a-list-of-users-for-all-instances-databases
-  --Concatenating strings: http://blog.sqlauthority.com/2010/11/25/sql-server-concat-function-in-sql-server-sql-concatenation/
-  --DROP USER from database: http://msdn.microsoft.com/en-us/library/ms189438.aspx
-  --DROP LOGIN from server: http://msdn.microsoft.com/en-us/library/ms188012.aspx
-  --Declaring variables (eg, the username above): http://technet.microsoft.com/en-us/library/aa258839.aspx
-  --A different (& non-dynamic) way to establish a user: http://pic.dhe.ibm.com/infocenter/dmndhelp/v8r5m0/index.jsp?topic=%2Fcom.ibm.wbpm.imuc.sbpm.doc%2Ftopics%2Fdb_create_users_nd_aix.html
-  --If the variable has to cross a 'GO' (which the current version of the script doesn't need): http://stackoverflow.com/questions/937336/is-there-a-way-to-persist-a-variable-across-a-go
    +

    Add a user’s LDAP account to the auxiliary_security database so that they can query the tables to retrieve their API.

    +

    Notice that this only gives the permissions to retrieve the token. You still must grant them API privileges to each appropriate REDCap project. The automation in the R file below will copy the API token from the MySQL database into the auxiliary_security database (see the ‘Transfer Credentials’ section).

    +

    Only database admins should have authorization for the ‘redcap_private’ schema. Typical users should not be authorized for this schema. The current system allows typical users to view only their own tokens.

    +
    -----------------------------------------------------------------------
+-- Add a user account to the auxiliary_security database so that they can query the tables to retrieve their API.
+-- Notice that this only gives the permissions to retrieve the token.  You must still:
+--   1) grant them API privileges to each appropriate REDCap project, and
+--   2) copy the API from the REDCap database into the  auxiliary_security database.
+-- Also, do not give typical users authorization for the 'redcap_private' schema.  The current system allows them to view only their own tokens.
+-----------------------------------------------------------------------
+
+-- STEP #1: Declare the user name.  If everything runs correctly, this should be the only piece of code that you need to modify.
+print 'Step #1 executing....'
+USE [master]
+GO
+DECLARE @qualified_user_name varchar(255); SET @qualified_user_name = '[OUHSC\lsuarez3]'
+print 'Resulting login name: ' + @qualified_user_name; print ''
+
+--EXEC sp_helplogins @LoginNamePattern=@qualified_user_name
+--SELECT * FROM master..syslogins WHERE name = @qualified_user_name
+--SELECT * FROM auxiliary_security.sys.sysusers
+--SELECT * FROM sys.database_permissions
+--SELECT * FROM sys.server_principals
+
+-----------------------------------------------------------------------
+-- STEP #2: Create a login for the *server*.
+print 'Step #2 executing....'
+DECLARE @sql_create_login nvarchar(max)
+SET @sql_create_login = 'CREATE LOGIN ' + @qualified_user_name + ' FROM WINDOWS WITH DEFAULT_DATABASE=[auxiliary_security]'
+EXECUTE sp_executesql @sql_create_login
+DECLARE @login_count AS INT; SET @login_count = (SELECT COUNT(*) AS login_count FROM master..syslogins WHERE '[' + loginname + ']' = @qualified_user_name)
+print 'Logins matching desired name should equal 1.  It equals: ' + CONVERT(varchar, @login_count); print ''
+
+-----------------------------------------------------------------------
+-- STEP #3: Create a user account for the *database*, after switching the database under focus to auxiliary_security.
+print 'Step #3 executing....'
+USE [auxiliary_security]
+DECLARE @sql_create_user nvarchar(max)
+SET @sql_create_user = 'CREATE USER ' + @qualified_user_name + ' FOR LOGIN ' + @qualified_user_name
+EXECUTE sp_executesql @sql_create_user
+DECLARE @user_count AS INT; SET @user_count = (SELECT COUNT(*) AS user_count FROM auxiliary_security.sys.sysusers WHERE '[' + name + ']' = @qualified_user_name)
+print 'User accounts matching desired name should equal 1.  It equals: ' + CONVERT(varchar, @user_count); print ''
+
+-----------------------------------------------------------------------
+-- STEP #4: Grant appropriate privileges for the 'redcap' schema.
+print 'Step #4 executing....'
+DECLARE @sql_grant_schema_redcap nvarchar(max)
+SET @sql_grant_schema_redcap = 'GRANT EXECUTE ON SCHEMA::[redcap] TO ' + @qualified_user_name
+EXECUTE sp_executesql @sql_grant_schema_redcap
+print 'Step #4 executed'; print ''
+
+-----------------------------------------------------------------------
+-- STEP #5: Grant appropriate privileges for the 'Security' schema.
+print 'Step #5 executing....'
+DECLARE @sql_grant_schema_security nvarchar(max)
+SET @sql_grant_schema_security = 'GRANT EXECUTE ON SCHEMA::[security] TO ' + @qualified_user_name
+EXECUTE sp_executesql @sql_grant_schema_security
+print 'Step #5 executed'; print ''
+
+-----------------------------------------------------------------------
+-- OPTIONAL STEP: Delete the user from the database (the first line) and then the server (the second line).
+-- The person's other database user accounts (besides with the auxiliary_security database) will NOT be automatically deleted by these two lines.
+--USE [auxiliary_security]; DROP USER [OUHSC\lsuarez3]
+--USE [master]; DROP LOGIN [OUHSC\lsuarez3]
+
+-----------------------------------------------------------------------
+-- REFERENCES & NOTES
+  --The @qualified_user_name must have both (a) the 'OUHSC' domain qualification, and (b) the square brackets (to escape the backslash).
+  --Using sp_executesql to add users: http://www.sqlservercentral.com/Forums/Topic497615-359-1.aspx
+  --Check if a server login exists: http://stackoverflow.com/questions/37275/sql-query-for-logins
+  --Retrieve database users: http://stackoverflow.com/questions/2445444/how-to-get-a-list-of-users-for-all-instances-databases
+  --Concatenating strings: http://blog.sqlauthority.com/2010/11/25/sql-server-concat-function-in-sql-server-sql-concatenation/
+  --DROP USER from database: http://msdn.microsoft.com/en-us/library/ms189438.aspx
+  --DROP LOGIN from server: http://msdn.microsoft.com/en-us/library/ms188012.aspx
+  --Declaring variables (eg, the username above): http://technet.microsoft.com/en-us/library/aa258839.aspx
+  --A different (& non-dynamic) way to establish a user: http://pic.dhe.ibm.com/infocenter/dmndhelp/v8r5m0/index.jsp?topic=%2Fcom.ibm.wbpm.imuc.sbpm.doc%2Ftopics%2Fdb_create_users_nd_aix.html
+  --If the variable has to cross a 'GO' (which the current version of the script doesn't need): http://stackoverflow.com/questions/937336/is-there-a-way-to-persist-a-variable-across-a-go

    Transfer Credentials

    -

    Manually transfer tokens to the auxiliary server becomes unmanageable as your institution’s collection of API users grows. This script demonstrates how to progamatically transfer all tokens from multiple REDCap instances on your network. The basic steps are:

    +

    Manually transferring tokens to the auxiliary server becomes unmanageable as your institution’s collection of API users grows. This script demonstrates how to progamatically transfer all tokens from multiple REDCap instances. The basic steps are:

      -
    1. Read from the MySQL database underneath each REDCap instance.
      2. +
    2. Read from the MySQL database(s) underneath each REDCap instance on your campus.
    3. Combine & groom the credentials.
      4. -
    4. Upload to SQL Server.
      5. +
    5. Upload to SQL Server (called auxiliary_security here).
    -
    rm(list=ls(all=TRUE)) #Clear the memory for any variables set from any previous runs.
-
-# ---- load-sources ------------------------------------------------------------
-
-# ---- load-packages -----------------------------------------------------------
-library(magrittr)
-requireNamespace("RODBC")
-requireNamespace("dplyr")
-requireNamespace("readr")
-requireNamespace("tibble")
-requireNamespace("checkmate")
-
-# ---- declare-globals ---------------------------------------------------------
-
-# The Activity Directory name that should precede each username.
-#   This should correspond with the result of `SYSTEM_USER`
-#   (https://msdn.microsoft.com/en-us/library/ms179930.aspx)
-ldap_prefix <- "OUHSC\\"
-
-#Create a SQL statement for each REDCap instance.  Only the `instance` value should change.
-sql <- "
-  SELECT username, project_id, api_token
-  FROM redcap_user_rights
-  WHERE api_token IS NOT NULL"
-
-#Update this ad-hoc CSV.  Each row should represent one REDCap instance.
-#   Choose any casual name for the first variable, consistent with the `tweak-data` chunk below.
-#   Enter the exact URL for the second variable.
-ds_url <- readr::read_csv(paste(
-  "instance,redcap_uri",
-  "production,https://redcap-production.ouhsc.edu/redcap/api/",
-  "dev,https://redcap-dev.ouhsc.edu/redcap/api/",
-sep="\n"))
-
-
-# ---- load-data ---------------------------------------------------------------
-
-# Load the credentials from the first instance.
-channel <- RODBC::odbcConnect("redcap-production") # odbcGetInfo(channel)
-ds_prod <- RODBC::sqlQuery(channel, query=sql, stringsAsFactors=F)
-RODBC::odbcClose(channel); rm(channel)
-
-# Load the credentials from the second instance.
-#   Duplicate or remove this block, dependending on the number of instances.
-channel <- RODBC::odbcConnect("redcap-dev") # odbcGetInfo(channel)
-ds_dev  <- RODBC::sqlQuery(channel, query=sql, stringsAsFactors=F)
-RODBC::odbcClose(channel); rm(channel, sql)
-
-# Assert these variables contain valid datasets (instead of a character error message), and
-#   that at least some rows were returned.
-#   Adjust this to smaller values if necessary.  It's really just to catch blatant retrieval problems.
-checkmate::assert_data_frame(ds_bbmc, min.rows=5)
-checkmate::assert_data_frame(ds_dev , min.rows=5)
-
-
-# ---- tweak-data --------------------------------------------------------------
-
-#Label each instance, so they're distinguishable later.  Add/remove lines, depending on the number of campus instances
-ds_prod$instance <- "production"
-ds_dev$instance  <- "dev"
-
-#Combine the token collection from each instance.  Then prefix the username and include the URL of each instance.
-ds <- ds_prod %>%
-  dplyr::union(ds_dev) %>%                                     # Add/remove unions, based on the number of REDCap instances.
-  dplyr::select_(
-    "username"             = "`username`"
-    , "project_id"         = "`project_id`"
-    , "instance"           = "`instance`"
-    , "token"              = "`api_token`"
-  ) %>%
-  dplyr::arrange(instance, project_id, username) %>%
-  dplyr::mutate(
-    username               = paste0(ldap_prefix, username),    # Qualify for the Active Directory.
-    id                     = seq_len(n())                      # For the sake of a clustered primary key.
-  ) %>%
-  dplyr::left_join( ds_url, by="instance")                     # Include the instance URL.
-
-rm(ds_prod, ds_dev, ds_url)
-
-
-# ---- verify-values -----------------------------------------------------------
-# devtools::install_github("OuhscBbmc/OuhscMunge"); OuhscMunge::verify_value_headstart(ds)
-# Assert that the dataset is well-behaved.
-checkmate::assert_integer(  ds$id         , any.missing=F , lower=1, upper=2^31-1 , unique=T)
-checkmate::assert_character(ds$username   , any.missing=F , pattern="^.{1,255}$"            )
-checkmate::assert_integer(  ds$project_id , any.missing=F , lower=1, upper=2^31-1           )
-checkmate::assert_character(ds$token      , any.missing=F , pattern="^.{32}$"     , unique=T)
-checkmate::assert_character(ds$instance   , any.missing=F , pattern="^.{1,255}$"            )
-checkmate::assert_character(ds$redcap_uri , any.missing=F , pattern="^.{1,255}$"            )
-
-testit::assert(
-  "The `username` x `project_id` x `instance` must be unique.",
-  sum(duplicated(paste0(ds$username, "-", ds$project_id, "-", ds$instance))) == 0L
-)
-
-testit::assert("There should be at least 10 tokens written." , 10L <= nrow(ds))
-
-
-# ---- specify-columns-to-upload -----------------------------------------------
-
-# Dictate the exact columns and order that will be uploaded.
-columns_to_write <- c("id", "username", "project_id", "instance", "token", "redcap_uri")
-ds_slim <- ds[, columns_to_write]
-
-rm(columns_to_write)
-
-
-# ---- upload-to-db-credential ------------------------------------------------------------
-
-#Upload to SQL Server through ODBC.
-(start_time <- Sys.time())
-
-db_table         <- "redcap_private.tbl_credential"
-channel          <- RODBC::odbcConnect("auxiliary_security") #getSqlTypeInfo("Microsoft SQL Server") #;odbcGetInfo(channel)
-
-column_info      <- RODBC::sqlColumns(channel, db_table)
-var_types        <- as.character(column_info$TYPE_NAME)
-names(var_types) <- as.character(column_info$COLUMN_NAME)  #var_types
-
-RODBC::sqlClear(channel, db_table)
-RODBC::sqlSave(channel, ds_slim, db_table, append=TRUE, rownames=FALSE, fast=TRUE, varTypes=var_types)
-RODBC::odbcClose(channel); rm(channel)
-
-(elapsed_duration <-  Sys.time() - start_time) #0.6026149 secs 2016-08-29.
-rm(db_table, column_info, var_types, start_time, elapsed_duration)
    +
    rm(list=ls(all=TRUE)) #Clear the memory for any variables set from any previous runs.
+
+# ---- load-sources ------------------------------------------------------------
+
+# ---- load-packages -----------------------------------------------------------
+if( !require(OuhscMunge) )
+  stop('The `OuhscMunge` package needs to be installed with `devtools::install_github("OuhscBbmc/OuhscMunge")`.')
+
+testit::assert(
+  "The `OuhscMunge` package should meet a minimal version.",
+  compareVersion( as.character(packageVersion("OuhscMunge")), "0.1.9.9009") >= 0L
+)
+
+library(magrittr)
+requireNamespace("DBI")
+requireNamespace("odbc")
+requireNamespace("dplyr")
+requireNamespace("readr")
+requireNamespace("tibble")
+requireNamespace("testit")
+requireNamespace("checkmate")
+requireNamespace("OuhscMunge")  # devtools::install_github("OuhscBbmc/OuhscMunge")
+
+
+# ---- declare-globals ---------------------------------------------------------
+# This file assume your campus has two REDCap instances.
+# Modify each (a) database name, (b) REDCap URL, and (c) DSN name.
+
+name_production <- "production"
+name_dev        <- "dev"
+
+uri_production  <- "https://redcap-production.ouhsc.edu/redcap/api/",
+uri_dev         <- "https://redcap-dev.ouhsc.edu/redcap/api/"
+
+dsn_production  <- "redcap-production"
+dsn_dev         <- "redcap-dev"
+dsn_source      <- "auxiliary_security" # The DSN of the token server.
+
+# The Activity Directory name that should precede each username.
+#   This should correspond with the result of SQL Server's `SYSTEM_USER` function
+#   (https://msdn.microsoft.com/en-us/library/ms179930.aspx)
+ldap_prefix <- "OUHSC\\"
+
+####
+# Nothing below this line should need to change, assuming:
+# 1. the vignette was followed exactly (https://ouhscbbmc.github.io/REDCapR/articles/SecurityDatabase.html),
+# 2. your campus has exactly two REDCap instances.
+
+# SQL sent to the MySQL database underneath each REDCap instance.
+sql <- "
+  SELECT username, project_id, api_token
+  FROM redcap_user_rights
+  WHERE api_token IS NOT NULL
+"
+
+# Update this ad-hoc CSV.  Each row should represent one REDCap instance.
+ds_url <- tibble::tribble(
+  ~instance         , ~redcap_uri,
+  name_production   , uri_production,
+  name_dev          , uri_dev
+)
+
+# Remove variables that aren't used below.
+rm(uri_production, uri_dev)
+
+
+# ---- load-data ---------------------------------------------------------------
+
+# Load the credentials from the first/production REDCap instance.
+cnn_production  <- DBI::dbConnect(odbc::odbc(), dsn=dsn_production)
+ds_production   <- DBI::dbGetQuery(cnn_production, sql)
+DBI::dbDisconnect(cnn_production); rm(cnn_production, dsn_production)
+
+# Load the credentials from the second/dev REDCap instance.
+cnn_dev         <- DBI::dbConnect(odbc::odbc(), dsn=dsn_dev)
+ds_dev          <- DBI::dbGetQuery(cnn_dev, sql)
+DBI::dbDisconnect(cnn_dev); rm(cnn_dev, dsn_dev)
+
+rm(sql)
+
+# Assert these are valid datasets and contain at least 5 rows.
+#   Adjust '5' to smaller value if necessary.  It's just to catch blatant retrieval problems.
+checkmate::assert_data_frame(ds_production, min.rows=5)
+checkmate::assert_data_frame(ds_dev       , min.rows=5)
+
+
+# ---- tweak-data --------------------------------------------------------------
+
+# Label each instance, so they're distinguishable later.
+ds_production$instance <- name_production
+ds_dev$instance        <- name_dev
+
+# Stack the token collection from each instance.  Then prefix the username and include the URL of each instance.
+ds <- ds_production %>%
+  dplyr::union(ds_dev) %>%                                # Remove union if the dev instance isn't included.
+  tibble::as_tibble() %>%
+  dplyr::select(
+    username             = username,
+    project_id           = project_id,
+    instance             = instance,
+    token                = api_token
+  ) %>%
+  dplyr::mutate(
+    username             = paste0(ldap_prefix, username), # Qualify for the Active Directory.
+  ) %>%
+  dplyr::left_join( ds_url, by="instance") %>%            # Include the instance URL.
+  dplyr::arrange(instance, project_id, username) %>%
+  tibble::rowid_to_column("id")                           # For the sake of a clustered primary key.
+
+rm(ds_production, ds_dev, ds_url)
+rm(name_production, name_dev)
+rm(ldap_prefix)
+
+
+# ---- verify-values -----------------------------------------------------------
+
+# Assert that the dataset is well-behaved.
+# OuhscMunge::verify_value_headstart(ds)
+checkmate::assert_integer(  ds$id         , any.missing=F, lower=1, upper=.Machine$integer.max, unique=T)
+checkmate::assert_character(ds$username   , any.missing=F, pattern="^.{1,255}$"                         )
+checkmate::assert_integer(  ds$project_id , any.missing=F, lower=1, upper=.Machine$integer.max          )
+checkmate::assert_character(ds$token      , any.missing=F, pattern="^[A-Z0-9]{32}$"           , unique=T)
+checkmate::assert_character(ds$instance   , any.missing=F, pattern="^.{1,255}$"                         )
+checkmate::assert_character(ds$redcap_uri , any.missing=F, pattern="^.{1,255}$"                         )
+
+testit::assert(
+  "The `username` x `project_id` x `instance` must be unique.",
+  sum(duplicated(paste0(ds$username, "-", ds$project_id, "-", ds$instance))) == 0L
+)
+
+testit::assert("At least 10 tokens should be ready to write." , 10L <= nrow(ds))
+
+
+# ---- specify-columns-to-upload -----------------------------------------------
+
+# Dictate the exact columns and order that will be uploaded.
+columns_to_write <- c("id", "username", "project_id", "instance", "token", "redcap_uri")
+ds_slim          <- ds[, columns_to_write]
+rm(columns_to_write)
+
+
+# ---- upload-to-db ------------------------------------------------------------------
+
+OuhscMunge::upload_sqls_odbc(
+  d               = ds_slim,
+  schema_name     = "redcap_private",
+  table_name      = "tbl_credential",
+  dsn_name        = dsn_source,
+  create_table    = FALSE,
+  clear_table     = TRUE,
+  transaction     = TRUE,
+  verbose         = TRUE
+)
+# Uploading 252 tokens takes 0.004 minutes.

    Document Info

    -

    This document is primarily based on REDCap version 6.11.5, and was last updated 2016-08-30. A development version of the document is available on GitHub: https://ouhscbbmc.github.io/REDCapR/articles/SecurityDatabase.html

    +

    This document is primarily based on REDCap version 8.4.0, and was last updated 2018-08-10. A development version of the document is available on GitHub: https://ouhscbbmc.github.io/REDCapR/articles/SecurityDatabase.html

    diff --git a/docs/articles/TroubleshootingApiCalls.html b/docs/articles/TroubleshootingApiCalls.html index e6bb8bb8..c3007bed 100644 --- a/docs/articles/TroubleshootingApiCalls.html +++ b/docs/articles/TroubleshootingApiCalls.html @@ -55,6 +55,9 @@ @@ -96,7 +96,7 @@

    Troubleshooting REDCap API Calls

    -

    2018-07-12

    +

    2018-08-10

    Source: vignettes/TroubleshootingApiCalls.Rmd @@ -159,10 +159,7 @@

  • Can an administrator query the API successfully with Postman with the admin token? As an administrator, create an account for yourself, and verify that your token works on your server and project.

  • Can an administrator query the API successfully with Postman with the user’s token? Use Postman as before, but replace your token with the user’s token. Once the whole problem is solved, reissue new API tokens to both you and the user.

  • Can an user query the API successfully with Postman with the their own token? The values they enter should be exactly the same as those entered in the previous step. A failure here (assuming the previous step was successful) suggests a network or firewall issue. If the server is behind your institution’s firewall, verify the user is connecting successfully through the VPN.

    • -
  • -

    Can a user query the API with cURL? cURL is a command line tool that’s underneath a lot of libraries. If it’s installed correctly on your location machine, it can be executed from the terminal or command line.

    -
    curl -X POST -H "Cache-Control: no-cache" -F "token=9A81268476645C4E5F03428B8AC3AA7B" -F "content=record" -F "format=csv" "https://bbmc.ouhsc.edu/redcap/api/"
    -
    • +

  • Can a user query the API with cURL? cURL is a command line tool that’s underneath a lot of libraries. If it’s installed correctly on your location machine, it can be executed from the terminal or command line. shell curl -X POST -H "Cache-Control: no-cache" -F "token=9A81268476645C4E5F03428B8AC3AA7B" -F "content=record" -F "format=csv" "https://bbmc.ouhsc.edu/redcap/api/"

    • @@ -172,84 +169,84 @@

    1. Is httr installed on the user’s local machine? If so, running library(httr) should not produce any error messages if you’re starting with a fresh session of R:

      -
      > library(httr)
      +

    2. Does the user have the most recent version of httr? There are several ways to do this, but the easiest is probably to run update.packages(ask=FALSE, repos="http://cran.rstudio.com"). The optional argument ask prevents the user from needing to respond ‘Y’ to each outdated package.

    3. Can the user query a test project using httr? Both the redcapAPI and REDCapR packages employ something similar to the following function in httr. If you’re curious, here is the relevant source code for redcapAPI and REDCapR.

      If this check fails, consider attempting again with the uri and token used above in the Postman example.

      This check avoids checking the SSL certificate in order to simplify the troubleshooting. SSL verification is supported by default in the PyCap, redcapAPI, and REDCapR packages.

      -
      redcap_uri <- "https://bbmc.ouhsc.edu/redcap/api/"
-token      <- "9A81268476645C4E5F03428B8AC3AA7B"
-
-raw_text <- RCurl::postForm(
-  uri                         = redcap_uri
-  , token                     = token
-  , content                   = 'record'
-  , format                    = 'csv'
-  , type                      = 'flat'
-  , rawOrLabel                = 'raw'
-  , exportDataAccessGroups    = 'true'
-  , .opts                     = RCurl::curlOptions(ssl.verifypeer=FALSE)
-)
      +

      Alternatively, you can try using the httr package, which uses RCurl underneath. REDCapR and a recent fork of redcap actually uses httr directly, instead of RCurl. As of 2014-07-06, this works with the Windows 8 version for libcurl (which is underneath `RCurl), but not with some Linux versions; in this case pass the location of the SSL cert file.

      -
      post_body <- list(
-  token                       = token,
-  content                     = 'record',
-  format                      = 'csv',
-  type                        = 'flat',
-  rawOrLabel                  = 'raw',
-  exportDataAccessGroups      = 'true'
-)
-
-raw_text <- httr::POST(
-  url                         = redcap_uri,
-  body                        = post_body,
-  config                      = httr::config(ssl.verifypeer=FALSE),
-  httr::verbose() #Remove this line to suppress the frequent console updates.
-)
      +

    4. Can the user query a subset of their project using RCurl? This step is like the previous one, but with two differences. First, it’s using their REDCap project (instead of the test project). Second, it pulls fewer records, and a smaller collection of fields. Subsetting can help troubleshoot by avoiding (and thus identifying) cells with problematic values.

      Notice this call to RCurl::postForm() now passes values to the records and fields parameters. Also notice the value is a single long string, rather a vector of shorter strings (which is more natural to most R users).

      -
      redcap_uri         <- "https://the.urlofyourinsitution.edu/api/"
-token              <- "your-secret-token"
-records_collapsed  <- "1,2,3"                             # Assumes dataset contains ID values of 1-3.
-fields_collapsed   <- "record_id,name_first,name_last"    # Assumes dataset contains these variables.
-
-raw_text <- RCurl::postForm(
-  uri                          = redcap_uri
-  , token                      = token
-  , content                    = 'record'
-  , format                     = 'csv'
-  , type                       = 'flat'
-  , rawOrLabel                 = 'raw'
-  , exportDataAccessGroups     = 'true'
-  , records                    = records_collapsed
-  , fields                     = fields_collapsed
-  , .opts                      = RCurl::curlOptions(ssl.verifypeer=FALSE)
-)
      +

    5. Can the user query an entire project using RCurl? There are two advantages of trying a subset of the data. First, small datasets avoid the time-out errors that plague large datasets. Second, it may avoid problematic values being passed through the pipeline. If the current check fails but the previous check succeedes, then experiment with different expanses of records and fields. This should help determine which values are causing the problems, or if there’s simply too much data being pulled in one pass.

      If the desired dataset is too large, consider if you can prune unnecessary records or fields. If not, one solution is to pull smaller, multiple batches using the API, then reassemble them. The redcap_read() function in REDCapR does this automatically, and allows the user to specify a batch_size.

      -
      redcap_uri                  <- "https://the.urlofyourinsitution.edu/api/"
-token                       <- "your-secret-token"
-records_collapsed           <- NULL
-fields_collapsed            <- NULL
-
-raw_text <- RCurl::postForm(
-  uri                        = redcap_uri
-  , token                    = token
-  , content                  = 'record'
-  , format                   = 'csv'
-  , type                     = 'flat'
-  , rawOrLabel               = 'raw'
-  , exportDataAccessGroups   = 'true'
-  , records                  = records_collapsed
-  , fields                   = fields_collapsed
-  , .opts                    = RCurl::curlOptions(ssl.verifypeer=FALSE)
-)
      +

    @@ -259,75 +256,24 @@

    REDCapR is a package that uses cURL (via httr) to communicate with REDCap, and wraps convenience functions around it to reduce the size and complexity of the user’s code. The package’s basic functions are demonstrated in a vignette and are documented in its reference manual (a downloadable pdf of the functions are also available).

    If you’re not using REDCapR, you can skip this section and proceed to ‘Importing into REDCap from R’ below.

      +

    1. Is REDCapR installed on the user’s machine? Currently the easiest way to install REDCapR is with the devtools. The follow code installs devtools, then installs REDCapR. r install.packages("devtools", repos="http://cran.rstudio.com") devtools::install_github(repo="OuhscBbmc/REDCapR")

      2. +

    2. Does REDCapR load successfully on the user’s machine? If so, running library(REDCapR) should produce the following output if you’re starting with a fresh session of R: r library(REDCapR) ## Loading required package: REDCapR

    3. -

      Is REDCapR installed on the user’s machine? Currently the easiest way to install REDCapR is with the devtools. The follow code installs devtools, then installs REDCapR.

      -
      install.packages("devtools", repos="http://cran.rstudio.com")
-devtools::install_github(repo="OuhscBbmc/REDCapR")
      +

      Can the user export from an example project? This is the same fake data hosted by the OUHSC BBMC as in the previous section. r library(REDCapR) #Load the package into the current R session. uri <- "https://bbmc.ouhsc.edu/redcap/api/" token <- "9A81268476645C4E5F03428B8AC3AA7B" redcap_read(redcap_uri=uri, token=token)$data

      +

      The previous code should produce similar output. Notice there are five rows and the columns will wrap around if your console window is too narrow. ``` 5 records and 1 columns were read from REDCap in 0.41 seconds. Starting to read 5 records at 2014-06-27 17:19:49 Reading batch 1 of 1, with ids 1 through 5. 5 records and 16 columns were read from REDCap in 0.42 seconds.

      +

      record_id name_first name_last address telephone email 1 1 Nutmeg Nutmouse 14 Rose Cottage St.UK, 323232 (432) 456-4848 nutty@mouse.com 2 2 Tumtum Nutmouse 14 Rose Cottage Blvd.UK 34243 (234) 234-2343 tummy@mouse.comm 3 3 Marcus Wood 243 Hill St.OK 73402 (433) 435-9865 mw@mwood.net 4 4 Trudy DAG 342 ElmTX, 75116 (987) 654-3210 peroxide@blonde.com 5 5 John Lee Walker Hotel SuiteOrleans LA, 70115 (333) 333-4444 left@hippocket.com

      +
            dob age ethnicity race sex height weight   bmi
      +

      1 2003-08-30 10 1 2 0 5.00 1 400.0 2 2003-03-10 10 1 6 1 6.00 1 277.8 3 1934-04-09 79 0 4 1 180.00 80 24.7 4 1952-11-02 61 1 4 0 165.00 54 19.8 5 1955-04-15 58 1 4 1 193.04 104 27.9

      +
                                                                                                        comments
      +

      1 Character in a book, with some guessing 2 A mouse character from a good book 3 completely made up 4 This record doesn’t have a DAG assignedcall up Trudy on the telephoneher a letter in the mail 5 Had a hand for trouble and a eye for cashhad a gold watch chain and a black mustache

      +

      demographics_complete 1 2 2 2 3 2 4 2 5 2 ```

    4. -

      Does REDCapR load successfully on the user’s machine? If so, running library(REDCapR) should produce the following output if you’re starting with a fresh session of R:

      -
      library(REDCapR)
-## Loading required package: REDCapR
      -
      5. -
    5. -

      Can the user export from an example project? This is the same fake data hosted by the OUHSC BBMC as in the previous section.

      -
      library(REDCapR) #Load the package into the current R session.
-uri   <- "https://bbmc.ouhsc.edu/redcap/api/"
-token <- "9A81268476645C4E5F03428B8AC3AA7B"
-redcap_read(redcap_uri=uri, token=token)$data
      -

      The previous code should produce similar output. Notice there are five rows and the columns will wrap around if your console window is too narrow.

      -
      5 records and 1 columns were read from REDCap in 0.41 seconds.
-Starting to read 5 records  at 2014-06-27 17:19:49
-Reading batch 1 of 1, with ids 1 through 5.
-5 records and 16 columns were read from REDCap in 0.42 seconds.
-
-  record_id name_first name_last                                 address      telephone               email
-1         1     Nutmeg  Nutmouse 14 Rose Cottage St.\nKenning UK, 323232 (432) 456-4848     nutty@mouse.com
-2         2     Tumtum  Nutmouse 14 Rose Cottage Blvd.\nKenning UK 34243 (234) 234-2343    tummy@mouse.comm
-3         3     Marcus      Wood          243 Hill St.\nGuthrie OK 73402 (433) 435-9865        mw@mwood.net
-4         4      Trudy       DAG          342 Elm\nDuncanville TX, 75116 (987) 654-3210 peroxide@blonde.com
-5         5   John Lee    Walker      Hotel Suite\nNew Orleans LA, 70115 (333) 333-4444  left@hippocket.com
-
-         dob age ethnicity race sex height weight   bmi
-1 2003-08-30  10         1    2   0   5.00      1 400.0
-2 2003-03-10  10         1    6   1   6.00      1 277.8
-3 1934-04-09  79         0    4   1 180.00     80  24.7
-4 1952-11-02  61         1    4   0 165.00     54  19.8
-5 1955-04-15  58         1    4   1 193.04    104  27.9
-
-                                                                                                     comments
-1                                                                     Character in a book, with some guessing
-2                                                                          A mouse character from a good book
-3                                                                                          completely made up
-4 This record doesn't have a DAG assigned\n\nSo call up Trudy on the telephone\nSend her a letter in the mail
-5                 Had a hand for trouble and a eye for cash\n\nHe had a gold watch chain and a black mustache
-
-  demographics_complete
-1                     2
-2                     2
-3                     2
-4                     2
-5                     2
      -
      6. -
    6. -

      Can the user export from their own project? The code is similar to the previous check, but the uri and token values will need to be modified.

      -
      library(REDCapR) #Load the package into the current R session, if you haven't already.
-redcap_uri       <- "https://the.urlofyourinsitution.edu/api/"
-token            <- "your-secret-token"
-redcap_read(redcap_uri=uri, token=token)$data
      -

      Alternatively, a redcap_project object can be declared initially, which makes subsequent calls cleaner when the token and url are required only the when the object is declared.

      -
      library(REDCapR) #Load the package into the current R session, if you haven't already.
-uri               <- "https://bbmc.ouhsc.edu/redcap/api/"
-token             <- "9A81268476645C4E5F03428B8AC3AA7B"
-project           <- redcap_project$new(redcap_uri=uri, token=token)
-
-ds_three_columns  <- project$read(fields=c("record_id", "sex", "age"))$data
-
-ids_of_males      <- ds_three_columns$record_id[ds_three_columns$sex==1]
-ids_of_minors     <- ds_three_columns$record_id[ds_three_columns$age < 18]
-
-ds_males          <- project$read(records=ids_of_males, batch_size=2)$data
-ds_minors         <- project$read(records=ids_of_minors)$data
      +

      Can the user export from their own project? The code is similar to the previous check, but the uri and token values will need to be modified. r library(REDCapR) #Load the package into the current R session, if you haven't already. redcap_uri <- "https://the.urlofyourinsitution.edu/api/" token <- "your-secret-token" redcap_read(redcap_uri=uri, token=token)$data

      +

      Alternatively, a redcap_project object can be declared initially, which makes subsequent calls cleaner when the token and url are required only the when the object is declared. ```r library(REDCapR) #Load the package into the current R session, if you haven’t already. uri <- “https://bbmc.ouhsc.edu/redcap/api/” token <- “9A81268476645C4E5F03428B8AC3AA7B” project <- redcap_project$new(redcap_uri=uri, token=token)

      +

      ds_three_columns <- project\(read(fields=c("record_id", "sex", "age"))\)data

      +

      ids_of_males <- ds_three_columns\(record_id[ds_three_columns\)sex==1] ids_of_minors <- ds_three_columns\(record_id[ds_three_columns\)age < 18]

      +

      ds_males <- project\(read(records=ids_of_males, batch_size=2)\)data ds_minors <- project\(read(records=ids_of_minors)\)data ```

    7. Is the export operation still unsuccessful using REDCapR? If so the “Can the user query a entire REDCap project using RCurl?” check succeeded, but the REDCapR checks did not, consider posting a new GitHub issue to the package developers.

    @@ -339,63 +285,20 @@

    If you’re not using redcapAPI, you can skip this section and proceed to ‘Importing into REDCap from R’ below. More specific discussion about redcapAPI can be found at the package’s wiki.

    1. -

      Is redcapAPI installed on the user’s machine? Currently, the easiest way to install redcapAPI is from CRAN.

      -
      install.packages("redcapAPI")
      -

      Developmental versions may be available on GitHub.

      -
      install.packages("devtools", repos="http://cran.rstudio.com")
-devtools::install_github(repo="nutterb/redcapAPI")
      +

      Is redcapAPI installed on the user’s machine? Currently, the easiest way to install redcapAPI is from CRAN. r install.packages("redcapAPI")

      +

      Developmental versions may be available on GitHub. r install.packages("devtools", repos="http://cran.rstudio.com") devtools::install_github(repo="nutterb/redcapAPI")

      2. +

    2. Does redcapAPI load successfully on the user’s machine? If so, running library(redcapAPI) should produce the following output if you’re starting with a fresh session of R: r library(redcapAPI) ## Loading required package: redcapAPI

    3. -

      Does redcapAPI load successfully on the user’s machine? If so, running library(redcapAPI) should produce the following output if you’re starting with a fresh session of R:

      -
      library(redcapAPI)
-## Loading required package: redcapAPI
      -
      4. -
    4. -

      Can the user export from an example project? This is the same fake data hosted by the OUHSC BBMC as in the previous section.

      -
      library(redcapAPI) #Load the package into the current R session.
-rcon <- redcapConnection(
-  url   = "https://bbmc.ouhsc.edu/redcap/api/",
-  token = "9A81268476645C4E5F03428B8AC3AA7B"
-)
-exportRecords(rcon)
      -

      The previous code should produce similar output. Notice there are five rows and the columns will wrap around if your console window is too narrow.

      -
      record_id name_first name_last                                 address      telephone               email
-1         1     Nutmeg  Nutmouse 14 Rose Cottage St.\nKenning UK, 323232 (432) 456-4848     nutty@mouse.com
-2         2     Tumtum  Nutmouse 14 Rose Cottage Blvd.\nKenning UK 34243 (234) 234-2343    tummy@mouse.comm
-3         3     Marcus      Wood          243 Hill St.\nGuthrie OK 73402 (433) 435-9865        mw@mwood.net
-4         4      Trudy       DAG          342 Elm\nDuncanville TX, 75116 (987) 654-3210 peroxide@blonde.com
-5         5   John Lee    Walker      Hotel Suite\nNew Orleans LA, 70115 (333) 333-4444  left@hippocket.com
-
-       dob age ethnicity race sex height weight   bmi
-1 2003-08-30  10         1    2   0   5.00      1 400.0
-2 2003-03-10  10         1    6   1   6.00      1 277.8
-3 1934-04-09  79         0    4   1 180.00     80  24.7
-4 1952-11-02  61         1    4   0 165.00     54  19.8
-5 1955-04-15  58         1    4   1 193.04    104  27.9
-
-                                                                                                   comments
-1                                                                     Character in a book, with some guessing
-2                                                                          A mouse character from a good book
-3                                                                                          completely made up
-4 This record doesn't have a DAG assigned\n\nSo call up Trudy on the telephone\nSend her a letter in the mail
-5                 Had a hand for trouble and a eye for cash\n\nHe had a gold watch chain and a black mustache
-
-demographics_complete
-1                     2
-2                     2
-3                     2
-4                     2
-5                     2
      -
      5. -
    5. -

      Can the user export from their own project? The code is similar to the previous check, but the uri and token values will need to be modified.

      -
      library(redcapAPI) #Load the package into the current R session, if you haven't already.
-rcon <- redcapConnection(
-  url   = "https://the.urlofyourinsitution.edu/api/", # Adapt this to your server.
-  token = "your-secret-token"                         # Adapt this to your user's token.
-)
-exportRecords(rcon)
      +

      Can the user export from an example project? This is the same fake data hosted by the OUHSC BBMC as in the previous section. r library(redcapAPI) #Load the package into the current R session. rcon <- redcapConnection( url = "https://bbmc.ouhsc.edu/redcap/api/", token = "9A81268476645C4E5F03428B8AC3AA7B" ) exportRecords(rcon)

      +

      The previous code should produce similar output. Notice there are five rows and the columns will wrap around if your console window is too narrow. ``` record_id name_first name_last address telephone email 1 1 Nutmeg Nutmouse 14 Rose Cottage St.UK, 323232 (432) 456-4848 nutty@mouse.com 2 2 Tumtum Nutmouse 14 Rose Cottage Blvd.UK 34243 (234) 234-2343 tummy@mouse.comm 3 3 Marcus Wood 243 Hill St.OK 73402 (433) 435-9865 mw@mwood.net 4 4 Trudy DAG 342 ElmTX, 75116 (987) 654-3210 peroxide@blonde.com 5 5 John Lee Walker Hotel SuiteOrleans LA, 70115 (333) 333-4444 left@hippocket.com

      +
          dob age ethnicity race sex height weight   bmi
      +

      1 2003-08-30 10 1 2 0 5.00 1 400.0 2 2003-03-10 10 1 6 1 6.00 1 277.8 3 1934-04-09 79 0 4 1 180.00 80 24.7 4 1952-11-02 61 1 4 0 165.00 54 19.8 5 1955-04-15 58 1 4 1 193.04 104 27.9

      +
                                                                                                      comments
      +

      1 Character in a book, with some guessing 2 A mouse character from a good book 3 completely made up 4 This record doesn’t have a DAG assignedcall up Trudy on the telephoneher a letter in the mail 5 Had a hand for trouble and a eye for cashhad a gold watch chain and a black mustache

      +

      demographics_complete 1 2 2 2 3 2 4 2 5 2 ```

      6. +

    6. Can the user export from their own project? The code is similar to the previous check, but the uri and token values will need to be modified. r library(redcapAPI) #Load the package into the current R session, if you haven't already. rcon <- redcapConnection( url = "https://the.urlofyourinsitution.edu/api/", # Adapt this to your server. token = "your-secret-token" # Adapt this to your user's token. ) exportRecords(rcon)

    7. Is the export operation still unsuccessful using redcapAPI? If so the “Can the user query a entire REDCap project using RCurl?” check succeeded, but the redcapAPI checks did not, consider posting a new GitHub issue to the package developers.

    @@ -428,16 +331,14 @@

    Other good resources

    diff --git a/docs/articles/advanced-redcapr-operations.html b/docs/articles/advanced-redcapr-operations.html index b37a62b1..8076f403 100644 --- a/docs/articles/advanced-redcapr-operations.html +++ b/docs/articles/advanced-redcapr-operations.html @@ -55,6 +55,9 @@ @@ -96,7 +96,7 @@

    Advanced REDCapR Operations

    -

    2018-07-12

    +

    2018-08-10

    Source: vignettes/advanced-redcapr-operations.Rmd @@ -114,988 +114,209 @@

    Set project-wide values.

    There is some information that is specific to a REDCap project, as opposed to an individual operation. This includes the (1) uri of the server, and the (2) token for the user’s project. This is hosted on a machine used in REDCapR’s public test suite, so you can run this example from any computer. Unless tests are running.

    -
    library(REDCapR) #Load the package into the current R session.
-uri                   <- "https://bbmc.ouhsc.edu/redcap/api/"
-token_simple          <- "9A81268476645C4E5F03428B8AC3AA7B"
-token_longitudinal    <- "0434F0E9CF53ED0587847AB6E51DE762"
    +

    Converting from tall/long to wide

    Disclaimer: Occasionally we’re asked for a longitudinal dataset to be converted from a “long/tall format” (where typically each row is one observation for a participant) to a “wide format” (where each row is on participant). Usually we advise against it. Besides all the database benefits of a long structure, a wide structure restricts your options with the stat routine. No modern longitudinal analysis procedures (eg, growth curve models or multilevel/hierarchical models) accept wide. You’re pretty much stuck with repeated measures anova, which is very inflexible for real-world medical-ish analyses. It requires a patient to have a measurement at every time point; otherwise the anova excludes the patient entirely.

    However we like going wide to produce visual tables for publications, and here’s one way to do it in R. First retrieve the dataset from REDCap.

    -
    library(magrittr); 
-suppressPackageStartupMessages(requireNamespace("dplyr"))
-suppressPackageStartupMessages(requireNamespace("tidyr"))
-events_to_retain  <- c("dose_1_arm_1", "visit_1_arm_1", "dose_2_arm_1", "visit_2_arm_1")
-
-ds_long <- REDCapR::redcap_read_oneshot(redcap_uri=uri, token=token_longitudinal)$data
    -
    #> 18 records and 125 columns were read from REDCap in 0.6 seconds.  The http status code was 200.
    -
    ds_long %>% 
-  dplyr::select(study_id, redcap_event_name, pmq1, pmq2, pmq3, pmq4)
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -study id - -redcap event name - -pmq1 - -pmq2 - -pmq3 - -pmq4 -
    -100 - -enrollment_arm_1 - -NA - -NA - -NA - -NA -
    -100 - -dose_1_arm_1 - -2 - -2 - -1 - -1 -
    -100 - -visit_1_arm_1 - -1 - -0 - -0 - -0 -
    -100 - -dose_2_arm_1 - -3 - -1 - -0 - -0 -
    -100 - -visit_2_arm_1 - -0 - -1 - -0 - -0 -
    -100 - -final_visit_arm_1 - -NA - -NA - -NA - -NA -
    -220 - -enrollment_arm_1 - -NA - -NA - -NA - -NA -
    -220 - -dose_1_arm_1 - -0 - -1 - -0 - -2 -
    -220 - -visit_1_arm_1 - -0 - -3 - -1 - -0 -
    -220 - -dose_2_arm_1 - -1 - -2 - -0 - -1 -
    -220 - -visit_2_arm_1 - -3 - -4 - -1 - -0 -
    -220 - -final_visit_arm_1 - -NA - -NA - -NA - -NA -
    -304 - -enrollment_arm_2 - -NA - -NA - -NA - -NA -
    -304 - -deadline_to_opt_ou_arm_2 - -NA - -NA - -NA - -NA -
    -304 - -first_dose_arm_2 - -0 - -1 - -0 - -0 -
    -304 - -first_visit_arm_2 - -2 - -0 - -0 - -0 -
    -304 - -final_visit_arm_2 - -NA - -NA - -NA - -NA -
    -304 - -deadline_to_return_arm_2 - -NA - -NA - -NA - -NA -
    + +
    #> 18 records and 125 columns were read from REDCap in 0.5 seconds.  The http status code was 200.
    +
    ds_long %>% 
+  dplyr::select(study_id, redcap_event_name, pmq1, pmq2, pmq3, pmq4)
    +
    #>    study_id        redcap_event_name pmq1 pmq2 pmq3 pmq4
+#> 1       100         enrollment_arm_1   NA   NA   NA   NA
+#> 2       100             dose_1_arm_1    2    2    1    1
+#> 3       100            visit_1_arm_1    1    0    0    0
+#> 4       100             dose_2_arm_1    3    1    0    0
+#> 5       100            visit_2_arm_1    0    1    0    0
+#> 6       100        final_visit_arm_1   NA   NA   NA   NA
+#> 7       220         enrollment_arm_1   NA   NA   NA   NA
+#> 8       220             dose_1_arm_1    0    1    0    2
+#> 9       220            visit_1_arm_1    0    3    1    0
+#> 10      220             dose_2_arm_1    1    2    0    1
+#> 11      220            visit_2_arm_1    3    4    1    0
+#> 12      220        final_visit_arm_1   NA   NA   NA   NA
+#> 13      304         enrollment_arm_2   NA   NA   NA   NA
+#> 14      304 deadline_to_opt_ou_arm_2   NA   NA   NA   NA
+#> 15      304         first_dose_arm_2    0    1    0    0
+#> 16      304        first_visit_arm_2    2    0    0    0
+#> 17      304        final_visit_arm_2   NA   NA   NA   NA
+#> 18      304 deadline_to_return_arm_2   NA   NA   NA   NA

    When widening only one variable (eg, pmq1), the code’s pretty simple:

    -
    ds_wide <- ds_long %>% 
-  dplyr::select(study_id, redcap_event_name, pmq1) %>% 
-  dplyr::filter(redcap_event_name %in% events_to_retain) %>% 
-  tidyr::spread(key=redcap_event_name, value=pmq1)
-ds_wide
    - - - - - - - - - - - - - - - - - - - - - - - - -
    -study id - -dose 1 arm 1 - -dose 2 arm 1 - -visit 1 arm 1 - -visit 2 arm 1 -
    -100 - -2 - -3 - -1 - -0 -
    -220 - -0 - -1 - -0 - -3 -
    +
    ds_wide <- ds_long %>% 
+  dplyr::select(study_id, redcap_event_name, pmq1) %>% 
+  dplyr::filter(redcap_event_name %in% events_to_retain) %>% 
+  tidyr::spread(key=redcap_event_name, value=pmq1)
+ds_wide
    +
    #>   study_id dose_1_arm_1 dose_2_arm_1 visit_1_arm_1 visit_2_arm_1
+#> 1      100            2            3             1             0
+#> 2      220            0            1             0             3

    When widening more than one variable (eg, pmq1 - pmq4), it’s usually easiest to go even longer/taller (eg, ds_eav) before reversing direction and going wide:

    -
    pattern <- "^(\\w+?)_arm_(\\d)$"
-
-ds_eav <- ds_long %>% 
-  dplyr::select(study_id, redcap_event_name, pmq1, pmq2, pmq3, pmq4) %>% 
-  dplyr::mutate(
-    event      = sub(pattern, "\\1", redcap_event_name),
-    arm        = as.integer(sub(pattern, "\\2", redcap_event_name))
-  ) %>% 
-  dplyr::select(study_id, event, arm, pmq1, pmq2, pmq3, pmq4) %>% 
-  tidyr::gather(key=key, value=value, pmq1, pmq2, pmq3, pmq4) %>% 
-  dplyr::filter(!(event %in% c(
-    "enrollment", "final_visit", "deadline_to_return", "deadline_to_opt_ou")
-  )) %>% 
-  dplyr::mutate( # Simulate correcting for mismatched names across arms:
-    event = dplyr::recode(event, "first_dose"="dose_1", "first_visit"="visit_1"),
-    key = paste0(event, "_", key)
-  ) %>% 
-  dplyr::select(-event)
-
-# Show the first 10 rows of the EAV table.
-ds_eav %>% 
-  head(10)
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -study id - -arm - -key - -value -
    -100 - -1 - -dose_1_pmq1 - -2 -
    -100 - -1 - -visit_1_pmq1 - -1 -
    -100 - -1 - -dose_2_pmq1 - -3 -
    -100 - -1 - -visit_2_pmq1 - -0 -
    -220 - -1 - -dose_1_pmq1 - -0 -
    -220 - -1 - -visit_1_pmq1 - -0 -
    -220 - -1 - -dose_2_pmq1 - -1 -
    -220 - -1 - -visit_2_pmq1 - -3 -
    -304 - -2 - -dose_1_pmq1 - -0 -
    -304 - -2 - -visit_1_pmq1 - -2 -
    -
    # Spread the EAV to wide.
-ds_wide <- ds_eav %>% 
-  tidyr::spread(key=key, value=value)
-ds_wide
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -study id - -arm - -dose 1 pmq1 - -dose 1 pmq2 - -dose 1 pmq3 - -dose 1 pmq4 - -dose 2 pmq1 - -dose 2 pmq2 - -dose 2 pmq3 - -dose 2 pmq4 - -visit 1 pmq1 - -visit 1 pmq2 - -visit 1 pmq3 - -visit 1 pmq4 - -visit 2 pmq1 - -visit 2 pmq2 - -visit 2 pmq3 - -visit 2 pmq4 -
    -100 - -1 - -2 - -2 - -1 - -1 - -3 - -1 - -0 - -0 - -1 - -0 - -0 - -0 - -0 - -1 - -0 - -0 -
    -220 - -1 - -0 - -1 - -0 - -2 - -1 - -2 - -0 - -1 - -0 - -3 - -1 - -0 - -3 - -4 - -1 - -0 -
    -304 - -2 - -0 - -1 - -0 - -0 - -NA - -NA - -NA - -NA - -2 - -0 - -0 - -0 - -NA - -NA - -NA - -NA -
    +
    pattern <- "^(\\w+?)_arm_(\\d)$"
+
+ds_eav <- ds_long %>% 
+  dplyr::select(study_id, redcap_event_name, pmq1, pmq2, pmq3, pmq4) %>% 
+  dplyr::mutate(
+    event      = sub(pattern, "\\1", redcap_event_name),
+    arm        = as.integer(sub(pattern, "\\2", redcap_event_name))
+  ) %>% 
+  dplyr::select(study_id, event, arm, pmq1, pmq2, pmq3, pmq4) %>% 
+  tidyr::gather(key=key, value=value, pmq1, pmq2, pmq3, pmq4) %>% 
+  dplyr::filter(!(event %in% c(
+    "enrollment", "final_visit", "deadline_to_return", "deadline_to_opt_ou")
+  )) %>% 
+  dplyr::mutate( # Simulate correcting for mismatched names across arms:
+    event = dplyr::recode(event, "first_dose"="dose_1", "first_visit"="visit_1"),
+    key = paste0(event, "_", key)
+  ) %>% 
+  dplyr::select(-event)
+
+# Show the first 10 rows of the EAV table.
+ds_eav %>% 
+  head(10)
    +
    #>    study_id arm          key value
+#> 1       100   1  dose_1_pmq1     2
+#> 2       100   1 visit_1_pmq1     1
+#> 3       100   1  dose_2_pmq1     3
+#> 4       100   1 visit_2_pmq1     0
+#> 5       220   1  dose_1_pmq1     0
+#> 6       220   1 visit_1_pmq1     0
+#> 7       220   1  dose_2_pmq1     1
+#> 8       220   1 visit_2_pmq1     3
+#> 9       304   2  dose_1_pmq1     0
+#> 10      304   2 visit_1_pmq1     2
    + +
    #>   study_id arm dose_1_pmq1 dose_1_pmq2 dose_1_pmq3 dose_1_pmq4 dose_2_pmq1
+#> 1      100   1           2           2           1           1           3
+#> 2      220   1           0           1           0           2           1
+#> 3      304   2           0           1           0           0          NA
+#>   dose_2_pmq2 dose_2_pmq3 dose_2_pmq4 visit_1_pmq1 visit_1_pmq2
+#> 1           1           0           0            1            0
+#> 2           2           0           1            0            3
+#> 3          NA          NA          NA            2            0
+#>   visit_1_pmq3 visit_1_pmq4 visit_2_pmq1 visit_2_pmq2 visit_2_pmq3
+#> 1            0            0            0            1            0
+#> 2            1            0            3            4            1
+#> 3            0            0           NA           NA           NA
+#>   visit_2_pmq4
+#> 1            0
+#> 2            0
+#> 3           NA

    SSL Options

    The official cURL site discusses the process of using SSL to verify the server being connected to.

    Use the SSL cert file that come with the openssl package.

    -
    cert_location <- system.file("cacert.pem", package="openssl")
-if( file.exists(cert_location) ) {
-  config_options         <- list(cainfo=cert_location)
-  ds_different_cert_file <- redcap_read_oneshot(
-    redcap_uri     = uri,
-    token          = token_simple,
-    config_options = config_options
-  )$data
-}
    -
    #> 5 records and 24 columns were read from REDCap in 0.7 seconds.  The http status code was 200.
    + +
    #> 5 records and 24 columns were read from REDCap in 0.2 seconds.  The http status code was 200.

    Force the connection to use SSL=3 (which is not preferred, and possibly insecure).

    -
    config_options <- list(sslversion=3)
-ds_ssl_3 <- redcap_read_oneshot(
-  redcap_uri     = uri,
-  token          = token_simple,
-  config_options = config_options
-)$data
    -
    #> 5 records and 24 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    config_options <- list(ssl.verifypeer=FALSE)
-ds_no_ssl <- redcap_read_oneshot(
-   redcap_uri     = uri,
-   token          = token_simple,
-   config_options = config_options
-)$data
    + +
    #> 5 records and 24 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    + 
    #> 5 records and 24 columns were read from REDCap in 0.3 seconds.  The http status code was 200.

    Session Information

    For the sake of documentation and reproducibility, the current report was rendered in the following environment. Click the line below to expand.

    -

    Environment

    -
    #> ─ Session info ──────────────────────────────────────────────────────────
-#>  setting  value                       
-#>  version  R version 3.4.4 (2018-03-15)
-#>  os       Ubuntu 18.04 LTS            
-#>  system   x86_64, linux-gnu           
-#>  ui       X11                         
-#>  language (EN)                        
-#>  collate  en_US.UTF-8                 
-#>  tz       America/Chicago             
-#>  date     2018-07-12                  
+
    Environment 
    
+
    #> - Session info ----------------------------------------------------------
+#>  setting  value                                      
+#>  version  R version 3.5.1 Patched (2018-08-06 r75070)
+#>  os       Windows >= 8 x64                           
+#>  system   x86_64, mingw32                            
+#>  ui       RTerm                                      
+#>  language (EN)                                       
+#>  collate  English_United States.1252                 
+#>  tz       America/Chicago                            
+#>  date     2018-08-10                                 
 #> 
-#> ─ Packages ──────────────────────────────────────────────────────────────
+#> - Packages --------------------------------------------------------------
 #>  package     * version     date       source                          
-#>  assertthat    0.2.0       2017-04-11 cran (@0.2.0)                   
-#>  backports     1.1.2       2017-12-13 cran (@1.1.2)                   
-#>  bindr         0.1.1       2018-03-13 CRAN (R 3.4.3)                  
-#>  bindrcpp    * 0.2.2       2018-03-29 CRAN (R 3.4.3)                  
-#>  checkmate     1.8.6       2018-04-10 Github (mllg/checkmate@489319a) 
-#>  clisymbols    1.2.0       2017-05-21 CRAN (R 3.4.3)                  
-#>  colorspace    1.3-2       2016-12-14 CRAN (R 3.4.3)                  
-#>  commonmark    1.5         2018-04-28 CRAN (R 3.4.4)                  
-#>  crayon        1.3.4       2017-09-16 CRAN (R 3.4.3)                  
-#>  curl          3.2         2018-03-28 CRAN (R 3.4.4)                  
-#>  desc          1.2.0       2018-05-01 CRAN (R 3.4.4)                  
-#>  devtools      1.13.6      2018-06-27 CRAN (R 3.4.4)                  
-#>  digest        0.6.15      2018-01-28 CRAN (R 3.4.3)                  
-#>  dplyr         0.7.6       2018-06-29 CRAN (R 3.4.4)                  
-#>  evaluate      0.10.1      2017-06-24 CRAN (R 3.4.3)                  
-#>  fs            1.2.3       2018-06-08 CRAN (R 3.4.4)                  
-#>  glue          1.2.0       2017-10-29 cran (@1.2.0)                   
-#>  highr         0.7         2018-06-09 CRAN (R 3.4.4)                  
-#>  hms           0.4.2.9000  2018-05-26 Github (tidyverse/hms@14e74ab)  
-#>  htmltools     0.3.6       2017-04-28 CRAN (R 3.4.3)                  
-#>  httr          1.3.1       2017-08-20 CRAN (R 3.4.3)                  
-#>  kableExtra    0.9.0       2018-05-21 CRAN (R 3.4.4)                  
-#>  knitr       * 1.20        2018-02-20 CRAN (R 3.4.3)                  
-#>  magrittr    * 1.5         2014-11-22 cran (@1.5)                     
-#>  MASS          7.3-50      2018-04-30 CRAN (R 3.4.4)                  
-#>  memoise       1.1.0       2017-04-21 CRAN (R 3.4.3)                  
-#>  munsell       0.5.0       2018-06-12 CRAN (R 3.4.4)                  
-#>  pillar        1.2.3       2018-05-25 CRAN (R 3.4.4)                  
-#>  pkgconfig     2.0.1       2017-03-21 cran (@2.0.1)                   
-#>  pkgdown       1.1.0       2018-06-02 CRAN (R 3.4.4)                  
-#>  plyr          1.8.4       2016-06-08 CRAN (R 3.4.3)                  
-#>  purrr         0.2.5       2018-05-29 CRAN (R 3.4.4)                  
-#>  R6            2.2.2       2017-06-17 CRAN (R 3.4.3)                  
-#>  Rcpp          0.12.17     2018-05-18 CRAN (R 3.4.4)                  
-#>  readr         1.2.0       2018-05-26 Github (tidyverse/readr@d6d622b)
-#>  REDCapR     * 0.9.10.9001 2018-07-11 local                           
-#>  rlang         0.2.1       2018-05-30 CRAN (R 3.4.4)                  
-#>  rmarkdown     1.10        2018-06-11 CRAN (R 3.4.4)                  
-#>  roxygen2      6.0.1       2017-02-06 CRAN (R 3.4.4)                  
-#>  rprojroot     1.3-2       2018-01-03 CRAN (R 3.4.3)                  
-#>  rstudioapi    0.7         2017-09-07 CRAN (R 3.4.3)                  
-#>  rvest         0.3.2       2016-06-17 CRAN (R 3.4.3)                  
-#>  scales        0.5.0.9000  2018-03-29 Github (hadley/scales@d767915)  
-#>  sessioninfo   1.0.0       2017-06-21 CRAN (R 3.4.3)                  
-#>  stringi       1.2.3       2018-06-12 CRAN (R 3.4.4)                  
-#>  stringr       1.3.1       2018-05-10 CRAN (R 3.4.4)                  
-#>  tibble        1.4.2       2018-01-22 CRAN (R 3.4.3)                  
-#>  tidyr         0.8.1       2018-05-18 CRAN (R 3.4.4)                  
-#>  tidyselect    0.2.4       2018-02-26 CRAN (R 3.4.3)                  
-#>  viridisLite   0.3.0       2018-02-01 CRAN (R 3.4.3)                  
-#>  withr         2.1.2       2018-03-29 Github (jimhester/withr@79d7b0d)
-#>  xml2          1.2.0       2018-01-24 CRAN (R 3.4.3)                  
-#>  yaml          2.1.19      2018-05-01 CRAN (R 3.4.4)
    
-
    
-
    Report rendered by wibeasley at 2018-07-12, 00:22 -0500 in 5 seconds.
    
+#>  assertthat    0.2.0       2017-04-11 CRAN (R 3.5.0)                  
+#>  backports     1.1.2       2017-12-13 CRAN (R 3.5.0)                  
+#>  bindr         0.1.1       2018-03-13 CRAN (R 3.5.0)                  
+#>  bindrcpp    * 0.2.2       2018-03-29 CRAN (R 3.5.0)                  
+#>  checkmate     1.8.9-9000  2018-08-09 Github (mllg/checkmate@29a1fb9) 
+#>  clisymbols    1.2.0       2017-05-21 CRAN (R 3.5.0)                  
+#>  colorspace    1.3-2       2016-12-14 CRAN (R 3.5.0)                  
+#>  commonmark    1.5         2018-04-28 CRAN (R 3.5.0)                  
+#>  crayon        1.3.4       2017-09-16 CRAN (R 3.5.0)                  
+#>  curl          3.2         2018-03-28 CRAN (R 3.5.0)                  
+#>  desc          1.2.0       2018-05-01 CRAN (R 3.5.0)                  
+#>  devtools      1.13.6      2018-06-27 CRAN (R 3.5.0)                  
+#>  digest        0.6.15      2018-01-28 CRAN (R 3.5.0)                  
+#>  dplyr         0.7.6       2018-06-29 CRAN (R 3.5.1)                  
+#>  evaluate      0.11        2018-07-17 CRAN (R 3.5.1)                  
+#>  fs            1.2.5       2018-07-30 CRAN (R 3.5.1)                  
+#>  glue          1.3.0       2018-07-17 CRAN (R 3.5.1)                  
+#>  hms           0.4.2.9001  2018-08-09 Github (tidyverse/hms@979286f)  
+#>  htmltools     0.3.6       2017-04-28 CRAN (R 3.5.0)                  
+#>  httr          1.3.1       2017-08-20 CRAN (R 3.5.0)                  
+#>  kableExtra    0.9.0       2018-05-21 CRAN (R 3.5.0)                  
+#>  knitr       * 1.20        2018-02-20 CRAN (R 3.5.0)                  
+#>  magrittr    * 1.5         2014-11-22 CRAN (R 3.5.0)                  
+#>  MASS          7.3-50      2018-04-30 CRAN (R 3.5.1)                  
+#>  memoise       1.1.0       2017-04-21 CRAN (R 3.5.0)                  
+#>  munsell       0.5.0       2018-06-12 CRAN (R 3.5.0)                  
+#>  pillar        1.3.0       2018-07-14 CRAN (R 3.5.1)                  
+#>  pkgconfig     2.0.1       2017-03-21 CRAN (R 3.5.0)                  
+#>  pkgdown       1.1.0       2018-06-02 CRAN (R 3.5.1)                  
+#>  purrr         0.2.5       2018-05-29 CRAN (R 3.5.0)                  
+#>  R6            2.2.2       2017-06-17 CRAN (R 3.5.0)                  
+#>  Rcpp          0.12.18     2018-07-23 CRAN (R 3.5.1)                  
+#>  readr         1.2.0       2018-08-09 Github (tidyverse/readr@4b2e93a)
+#>  REDCapR     * 0.9.10.9001 2018-08-11 local                           
+#>  rlang         0.2.1       2018-05-30 CRAN (R 3.5.0)                  
+#>  rmarkdown     1.10        2018-06-11 CRAN (R 3.5.0)                  
+#>  roxygen2      6.1.0       2018-07-27 CRAN (R 3.5.1)                  
+#>  rprojroot     1.3-2       2018-01-03 CRAN (R 3.5.0)                  
+#>  rstudioapi    0.7         2017-09-07 CRAN (R 3.5.0)                  
+#>  rvest         0.3.2       2016-06-17 CRAN (R 3.5.0)                  
+#>  scales        1.0.0       2018-08-09 CRAN (R 3.5.1)                  
+#>  sessioninfo   1.0.0       2017-06-21 CRAN (R 3.5.0)                  
+#>  stringi       1.2.4       2018-07-20 CRAN (R 3.5.1)                  
+#>  stringr       1.3.1       2018-05-10 CRAN (R 3.5.0)                  
+#>  tibble        1.4.2       2018-01-22 CRAN (R 3.5.0)                  
+#>  tidyr         0.8.1       2018-05-18 CRAN (R 3.5.0)                  
+#>  tidyselect    0.2.4       2018-02-26 CRAN (R 3.5.0)                  
+#>  viridisLite   0.3.0       2018-02-01 CRAN (R 3.5.0)                  
+#>  withr         2.1.2       2018-03-15 CRAN (R 3.5.0)                  
+#>  xml2          1.2.0       2018-01-24 CRAN (R 3.5.0)                  
+#>  yaml          2.2.0       2018-07-25 CRAN (R 3.5.1)
    +

    Report rendered by Will at 2018-08-10, 20:36 -0500 in 3 seconds.

    diff --git a/docs/articles/index.html b/docs/articles/index.html index 592f0fec..2e67174b 100644 --- a/docs/articles/index.html +++ b/docs/articles/index.html @@ -89,6 +89,9 @@ diff --git a/docs/authors.html b/docs/authors.html index 91a3b195..3e381db6 100644 --- a/docs/authors.html +++ b/docs/authors.html @@ -89,6 +89,9 @@ diff --git a/docs/favicon.ico b/docs/favicon.ico index 15798f1f..c712399a 100644 Binary files a/docs/favicon.ico and b/docs/favicon.ico differ diff --git a/docs/index.html b/docs/index.html index 522cd8cd..5603ee95 100644 --- a/docs/index.html +++ b/docs/index.html @@ -60,6 +60,9 @@ @@ -103,33 +103,33 @@ REDCapR

    We’ve been using R with REDCap’s API since 2012 and have developed REDCapR. Before encapsulating these functions in a package, we were replicating 50+ lines of code to contact REDCap and robustly transform the returned csv into an R data.frame; it took twice that much to implement batching. All this can be done in one call to redcap_read():

    -
    ds <- redcap_read(redcap_uri=uri, token=token)$data
    +
    ds <- redcap_read(redcap_uri=uri, token=token)$data

    The redcap_read() function also accepts values for subsetting/filtering the records and fields. Here are two examples; the first selects only a portion of the rows, while the second selects only a portion of the columns. Documentation for the additional 20+ functions are found at ouhscbbmc.github.io/REDCapR/reference.

    -
    # Return only records with IDs of 1 and 4
-desired_records <- c(1, 4)
-ds_some_rows <- redcap_read(
-  redcap_uri   = uri,
-  token        = token,
-  records      = desired_records
-)$data
-
-# Return only the fields record_id, name_first, and age
-desired_fields <- c("record_id", "name_first", "age")
-ds_some_fields <- redcap_read(
-  redcap_uri  = uri,
-  token       = token,
-  fields      = desired_fields
-)$data
    +
    # Return only records with IDs of 1 and 4
+desired_records <- c(1, 4)
+ds_some_rows <- redcap_read(
+  redcap_uri   = uri,
+  token        = token,
+  records      = desired_records
+)$data
+
+# Return only the fields record_id, name_first, and age
+desired_fields <- c("record_id", "name_first", "age")
+ds_some_fields <- redcap_read(
+  redcap_uri  = uri,
+  token       = token,
+  fields      = desired_fields
+)$data

    The REDCapR package includes the SSL certificate retrieved by httr::find_cert_bundle(). Your REDCap server’s identity is always verified, unless the setting is overridden (alternative certificates can also be provided).

    To keep our maintenance efforts manageable, the package implements only the REDCap API functions that have been requested. If there’s a feature that would help your projects, please tell us in a new issue in REDCapR’s GitHub repository. A troubleshooting document helps diagnose issues with the API.

    Installation and Documentation

    The release version can be installed from CRAN.

    -
    install.packages("REDCapR")
    +
    install.packages("REDCapR")

    The development version can be installed from GitHub after installing the devtools package.

    -
    install.packages("devtools") # Run this line if the 'devtools' package isn't installed already.
-devtools::install_github(repo="OuhscBbmc/REDCapR")
    +
    install.packages("devtools") # Run this line if the 'devtools' package isn't installed already.
+devtools::install_github(repo="OuhscBbmc/REDCapR")

    The ouhscbbmc.github.io/REDCapR site describes the package functions, and includes documents involving basic operations, advanced operations, token security, and troubleshooting.

    Also checkout the other packages that exist for communicating with REDCap, which are listed in the REDCap Tools directory.

    diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml index 22697aa6..a574ae7c 100644 --- a/docs/pkgdown.yml +++ b/docs/pkgdown.yml @@ -1,11 +1,11 @@ -pandoc: 1.19.2.1 +pandoc: 2.2.1 pkgdown: 1.1.0 pkgdown_sha: ~ articles: + advanced-redcapr-operations: advanced-redcapr-operations.html BasicREDCapROperations: BasicREDCapROperations.html SecurityDatabase: SecurityDatabase.html TroubleshootingApiCalls: TroubleshootingApiCalls.html - advanced-redcapr-operations: advanced-redcapr-operations.html urls: reference: https://ouhscbbmc.github.io/REDCapR/reference article: https://ouhscbbmc.github.io/REDCapR/articles diff --git a/docs/reference/REDCapR-package.html b/docs/reference/REDCapR-package.html index 511b6c03..44c2ea5e 100644 --- a/docs/reference/REDCapR-package.html +++ b/docs/reference/REDCapR-package.html @@ -100,6 +100,9 @@ diff --git a/docs/reference/collapse_vector.html b/docs/reference/collapse_vector.html index e86a29df..c5814781 100644 --- a/docs/reference/collapse_vector.html +++ b/docs/reference/collapse_vector.html @@ -93,6 +93,9 @@ diff --git a/docs/reference/constant.html b/docs/reference/constant.html index 0d9562a9..13e32a3d 100644 --- a/docs/reference/constant.html +++ b/docs/reference/constant.html @@ -91,6 +91,9 @@ diff --git a/docs/reference/create_batch_glossary.html b/docs/reference/create_batch_glossary.html index 31954963..9daef37b 100644 --- a/docs/reference/create_batch_glossary.html +++ b/docs/reference/create_batch_glossary.html @@ -93,6 +93,9 @@ diff --git a/docs/reference/index.html b/docs/reference/index.html index b824cab1..f8429810 100644 --- a/docs/reference/index.html +++ b/docs/reference/index.html @@ -89,6 +89,9 @@ diff --git a/docs/reference/kernel_api.html b/docs/reference/kernel_api.html index c8d52d98..a67bac8f 100644 --- a/docs/reference/kernel_api.html +++ b/docs/reference/kernel_api.html @@ -91,6 +91,9 @@ @@ -186,7 +186,7 @@

    Examp # Consume the results in a few different ways. kernel$result
    #> Response [https://bbmc.ouhsc.edu/redcap/api/] -#> Date: 2018-07-12 05:22 +#> Date: 2018-08-11 01:36 #> Status: 200 #> Content-Type: text/csv; charset=utf-8 #> Size: 557 B diff --git a/docs/reference/metadata_utilities.html b/docs/reference/metadata_utilities.html index 01121038..a9e9a75e 100644 --- a/docs/reference/metadata_utilities.html +++ b/docs/reference/metadata_utilities.html @@ -91,6 +91,9 @@ @@ -209,7 +209,7 @@

    Examp #> 5 5 White #> 6 6 Unknown / Not Reported

    #This function is designed specifically for the checkbox values. -REDCapR::checkbox_choices(select_choices=choices_1)
    #> id label +REDCapR::checkbox_choices(select_choices=choices_1)
    #> id label #> 1 1 American Indian/Alaska Native #> 2 2 Asian #> 3 3 Native Hawaiian or Other Pacific Islander diff --git a/docs/reference/redcap_column_sanitize.html b/docs/reference/redcap_column_sanitize.html index 7c98e3ce..1034ff5e 100644 --- a/docs/reference/redcap_column_sanitize.html +++ b/docs/reference/redcap_column_sanitize.html @@ -91,6 +91,9 @@ @@ -181,10 +181,10 @@

    Details

    Examples

    dirty <- data.frame(id=1:3, names=c("Ekstr\xf8m", "J\xf6reskog", "bi\xdfchen Z\xfcrcher"))
-REDCapR::redcap_column_sanitize(dirty)
    #>   id            names
-#> 1  1          Ekstrom
-#> 2  2         Joreskog
-#> 3  3 bisschen Zurcher
    # Produces the dataset:
+REDCapR::redcap_column_sanitize(dirty)
    #>   id           names
+#> 1  1         Ekstrom
+#> 2  2        Joreskog
+#> 3  3 bi?chen Zurcher
    # Produces the dataset:
 #  id            names
 #1  1          Ekstr?m
 #2  2         Joreskog
diff --git a/docs/reference/redcap_download_file_oneshot.html b/docs/reference/redcap_download_file_oneshot.html
index 37c0672b..69115f65 100644
--- a/docs/reference/redcap_download_file_oneshot.html
+++ b/docs/reference/redcap_download_file_oneshot.html
@@ -91,6 +91,9 @@
     
   
   
 
       
diff --git a/docs/reference/redcap_metadata_read.html b/docs/reference/redcap_metadata_read.html
index d1c132c4..5985c471 100644
--- a/docs/reference/redcap_metadata_read.html
+++ b/docs/reference/redcap_metadata_read.html
@@ -92,6 +92,9 @@
     
   
   
 
       
@@ -145,9 +145,9 @@ 
    Export the metadata of a REDCap project.
    
     
     
    
 
-    
    redcap_metadata_read(redcap_uri, token, forms = NULL, forms_collapsed = "",
-  fields = NULL, fields_collapsed = "", verbose = TRUE,
-  config_options = NULL)
    
+    
    redcap_metadata_read(redcap_uri, token, forms = NULL,
+  forms_collapsed = "", fields = NULL, fields_collapsed = "",
+  verbose = TRUE, config_options = NULL)
    
     
     
    Arguments
    
     
diff --git a/docs/reference/redcap_project.html b/docs/reference/redcap_project.html
index 429d6a56..55ea15e8 100644
--- a/docs/reference/redcap_project.html
+++ b/docs/reference/redcap_project.html
@@ -93,6 +93,9 @@
     
diff --git a/docs/reference/redcap_read.html b/docs/reference/redcap_read.html
index 471648cf..3dab7e3d 100644
--- a/docs/reference/redcap_read.html
+++ b/docs/reference/redcap_read.html
@@ -94,6 +94,9 @@
     
diff --git a/docs/reference/redcap_read_oneshot.html b/docs/reference/redcap_read_oneshot.html
index 1e1e5834..7fd7bffb 100644
--- a/docs/reference/redcap_read_oneshot.html
+++ b/docs/reference/redcap_read_oneshot.html
@@ -91,6 +91,9 @@
     
diff --git a/docs/reference/redcap_read_oneshot_eav.html b/docs/reference/redcap_read_oneshot_eav.html
index 026317fe..46617cb4 100644
--- a/docs/reference/redcap_read_oneshot_eav.html
+++ b/docs/reference/redcap_read_oneshot_eav.html
@@ -91,6 +91,9 @@
     
diff --git a/docs/reference/redcap_upload_file_oneshot.html b/docs/reference/redcap_upload_file_oneshot.html
index 2a2a5d9e..5c45bccf 100644
--- a/docs/reference/redcap_upload_file_oneshot.html
+++ b/docs/reference/redcap_upload_file_oneshot.html
@@ -91,6 +91,9 @@
     
diff --git a/docs/reference/redcap_variables.html b/docs/reference/redcap_variables.html
index 450a07ea..e643da5a 100644
--- a/docs/reference/redcap_variables.html
+++ b/docs/reference/redcap_variables.html
@@ -91,6 +91,9 @@
     
@@ -143,7 +143,8 @@ 
    Enumerate the exported variables.
    
 
-    
    redcap_variables(redcap_uri, token, verbose = TRUE, config_options = NULL)
    
+    
    redcap_variables(redcap_uri, token, verbose = TRUE,
+  config_options = NULL)
    Arguments
    
   
   
 
       
   
   
 
       
   
   
 
       
   
   
 
       
   
   
 
       
   
   
 
       
     
     
     
     
     
    
diff --git a/docs/reference/redcap_version.html b/docs/reference/redcap_version.html
index 4d5679c4..f3c6cad9 100644
--- a/docs/reference/redcap_version.html
+++ b/docs/reference/redcap_version.html
@@ -91,6 +91,9 @@
     
@@ -143,7 +143,8 @@ 
    Determine version of REDCap instance
    
 
-    
    redcap_version(redcap_uri, token, verbose = TRUE, config_options = NULL)
    
+    
    redcap_version(redcap_uri, token, verbose = TRUE,
+  config_options = NULL)
    Arguments
    
   
   
 
       
     
     
     
     
     
    
@@ -180,7 +181,7 @@ 
    Details
     
    Examples
    uri      <- "https://bbmc.ouhsc.edu/redcap/api/"
 token    <- "9A81268476645C4E5F03428B8AC3AA7B"
-REDCapR::redcap_version(redcap_uri=uri, token=token)
    #> The REDCap version was successfully determined in 0.4 seconds.  The http status code was 200.  It is 8.4.0.
    #> [1] ‘8.4.0’
    
+REDCapR::redcap_version(redcap_uri=uri, token=token)
    #> The REDCap version was successfully determined in 0.2 seconds.  The http status code was 200.  It is 8.4.0.
    #> [1] '8.4.0'
    #> # A tibble: 2 x 4
-#>   field_name     field_index concern                 suggestion                
-#>   <chr>                <int> <chr>                   <chr>                     
-#> 1 flag_logical             2 The REDCap API does no… Convert the variable with…
-#> 2 flag_Uppercase           3 A REDCap project does … Change the uppercase lett…
    
+#>   field_name   field_index concern                  suggestion                 
+#>   <chr>              <int> <chr>                    <chr>                      
+#> 1 flag_logical           2 The REDCap API does not~ Convert the variable with ~
+#> 2 flag_Upperc~           3 A REDCap project does n~ Change the uppercase lette~
    
     
   
   
   
   
    
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
    
-record id
-
-name first
-
-name last
-
-address
-
-telephone
-
-email
-
-dob
-
-age
-
-sex
-
-demographics complete
-
-height
-
-weight
-
-bmi
-
-comments
-
-mugshot
-
-health complete
-
-race 1
-
-race 2
-
-race 3
-
-race 4
-
-race 5
-
-race 6
-
-ethnicity
-
-race and ethnicity complete
-
    
-1
-
-Nutmeg
-
-Nutmouse
-
-14 Rose Cottage St. Kenning UK, 323232
-
-
      
-
    1. 321-1111
-
    		
-nutty@mouse.com
-
-2003-08-30
-
-11
-
-0
-
-2
-
-7.00
-
-1
-
-204.1
-
-Character in a book, with some guessing
-
-[document]
-
-1
-
-0
-
-0
-
-0
-
-0
-
-1
-
-0
-
-1
-
-2
-
    
-2
-
-Tumtum
-
-Nutmouse
-
-14 Rose Cottage Blvd. Kenning UK 34243
-
-
      
-
    1. 321-2222
-
    		
-tummy@mouse.comm
-
-2003-03-10
-
-11
-
-1
-
-2
-
-6.00
-
-1
-
-277.8
-
-A mouse character from a good book
-
-[document]
-
-0
-
-0
-
-0
-
-1
-
-0
-
-1
-
-0
-
-1
-
-0
-
    
-3
-
-Marcus
-
-Wood
-
-243 Hill St. Guthrie OK 73402
-
-
      
-
    1. 321-3333
-
    		
-mw@mwood.net
-
-1934-04-09
-
-80
-
-1
-
-2
-
-180.00
-
-80
-
-24.7
-
-completely made up
-
-[document]
-
-2
-
-0
-
-0
-
-0
-
-1
-
-1
-
-0
-
-0
-
-2
-
    
-4
-
-Trudy
-
-DAG
-
-342 Elm Duncanville TX, 75116
-
-
      
-
    1. 321-4444
-
    		
-peroxide@blonde.com
-
-1952-11-02
-
-61
-
-0
-
-2
-
-165.00
-
-54
-
-19.8
-
-This record doesn’t have a DAG assigned
-
-
-
-
-So call up Trudy on the telephone Send her a letter in the mail
-
-[document]
-
-2
-
-0
-
-1
-
-0
-
-0
-
-1
-
-0
-
-1
-
-2
-
    
-5
-
-John Lee
-
-Walker
-
-Hotel Suite New Orleans LA, 70115
-
-
      
-
    1. 321-5555
-
    		
-left@hippocket.com
-
-1955-04-15
-
-59
-
-1
-
-2
-
-193.04
-
-104
-
-27.9
-
-Had a hand for trouble and a eye for cash
-
-He had a gold watch chain and a black mustache
-
-[document]
-
-0
-
-1
-
-0
-
-0
-
-0
-
-0
-
-1
-
-2
-
-2
-
    
+
+
      record_id name_first name_last                                 address
+1         1     Nutmeg  Nutmouse 14 Rose Cottage St.\nKenning UK, 323232
+2         2     Tumtum  Nutmouse 14 Rose Cottage Blvd.\nKenning UK 34243
+3         3     Marcus      Wood          243 Hill St.\nGuthrie OK 73402
+4         4      Trudy       DAG          342 Elm\nDuncanville TX, 75116
+5         5   John Lee    Walker      Hotel Suite\nNew Orleans LA, 70115
+       telephone               email        dob age sex
+1 (405) 321-1111     nutty@mouse.com 2003-08-30  11   0
+2 (405) 321-2222    tummy@mouse.comm 2003-03-10  11   1
+3 (405) 321-3333        mw@mwood.net 1934-04-09  80   1
+4 (405) 321-4444 peroxide@blonde.com 1952-11-02  61   0
+5 (405) 321-5555  left@hippocket.com 1955-04-15  59   1
+  demographics_complete height weight   bmi
+1                     2   7.00      1 204.1
+2                     2   6.00      1 277.8
+3                     2 180.00     80  24.7
+4                     2 165.00     54  19.8
+5                     2 193.04    104  27.9
+                                                                                                     comments
+1                                                                     Character in a book, with some guessing
+2                                                                          A mouse character from a good book
+3                                                                                          completely made up
+4 This record doesn't have a DAG assigned\n\nSo call up Trudy on the telephone\nSend her a letter in the mail
+5                 Had a hand for trouble and a eye for cash\n\nHe had a gold watch chain and a black mustache
+     mugshot health_complete race___1 race___2 race___3 race___4 race___5
+1 [document]               1        0        0        0        0        1
+2 [document]               0        0        0        1        0        1
+3 [document]               2        0        0        0        1        1
+4 [document]               2        0        1        0        0        1
+5 [document]               0        1        0        0        0        0
+  race___6 ethnicity race_and_ethnicity_complete
+1        0         1                           2
+2        0         1                           0
+3        0         0                           2
+4        0         1                           2
+5        1         2                           2

    Read a subset of the records.

    If only a subset of the records is desired, the two approaches are shown below. The first is to pass an array (where each element is an ID) to the records parameter. The second is to pass a single string (where the elements are separated by commas) to the records_collapsed parameter.

    The first format is more natural for more R users. The second format is what is expected by the REDCap API. If a value for records is specified, but records_collapsed is not specified, then redcap_read_oneshot automatically converts the array into the format needed by the API.

    -
    #Return only records with IDs of 1 and 3
-desired_records_v1 <- c(1, 3)
-ds_some_rows_v1 <- redcap_read(
-   redcap_uri = uri, 
-   token      = token, 
-   records    = desired_records_v1
-)$data
    -
    The data dictionary describing 16 fields was read from REDCap in 0.4 seconds.  The http status code was 200.
    -
    2 records and 1 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    Starting to read 2 records  at 2018-07-11 11:04:15.
    -
    Reading batch 1 of 1, with subjects 1 through 3 (ie, 2 unique subject records).
    -
    2 records and 24 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    #Return only records with IDs of 1 and 3 (alternate way)
-desired_records_v2 <- "1, 3"
-ds_some_rows_v2 <- redcap_read(
-   redcap_uri        = uri, 
-   token             = token, 
-   records_collapsed = desired_records_v2
-)$data
    -
    The data dictionary describing 16 fields was read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    2 records and 1 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    Starting to read 2 records  at 2018-07-11 11:04:16.
    + +
    The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    2 records and 1 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    Starting to read 2 records  at 2018-08-10 20:36:07.
    Reading batch 1 of 1, with subjects 1 through 3 (ie, 2 unique subject records).
    2 records and 24 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    -
    ds_some_rows_v2 #Inspect the returned dataset
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -record id - -name first - -name last - -address - -telephone - -email - -dob - -age - -sex - -demographics complete - -height - -weight - -bmi - -comments - -mugshot - -health complete - -race 1 - -race 2 - -race 3 - -race 4 - -race 5 - -race 6 - -ethnicity - -race and ethnicity complete -
    -1 - -Nutmeg - -Nutmouse - -14 Rose Cottage St. Kenning UK, 323232 - -
      -
    1. 321-1111 -
    		 -nutty@mouse.com - -2003-08-30 - -11 - -0 - -2 - -7 - -1 - -204.1 - -Character in a book, with some guessing - -[document] - -1 - -0 - -0 - -0 - -0 - -1 - -0 - -1 - -2 -
    -3 - -Marcus - -Wood - -243 Hill St. Guthrie OK 73402 - -
      -
    1. 321-3333 -
    		 -mw@mwood.net - -1934-04-09 - -80 - -1 - -2 - -180 - -80 - -24.7 - -completely made up - -[document] - -2 - -0 - -0 - -0 - -1 - -1 - -0 - -0 - -2 -
    - - + +
    The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    2 records and 1 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    Starting to read 2 records  at 2018-08-10 20:36:08.
    +
    Reading batch 1 of 1, with subjects 1 through 3 (ie, 2 unique subject records).
    +
    2 records and 24 columns were read from REDCap in 0.4 seconds.  The http status code was 200.
    + +
      record_id name_first name_last                                 address
+1         1     Nutmeg  Nutmouse 14 Rose Cottage St.\nKenning UK, 323232
+2         3     Marcus      Wood          243 Hill St.\nGuthrie OK 73402
+       telephone           email        dob age sex demographics_complete
+1 (405) 321-1111 nutty@mouse.com 2003-08-30  11   0                     2
+2 (405) 321-3333    mw@mwood.net 1934-04-09  80   1                     2
+  height weight   bmi                                comments    mugshot
+1      7      1 204.1 Character in a book, with some guessing [document]
+2    180     80  24.7                      completely made up [document]
+  health_complete race___1 race___2 race___3 race___4 race___5 race___6
+1               1        0        0        0        0        1        0
+2               2        0        0        0        1        1        0
+  ethnicity race_and_ethnicity_complete
+1         1                           2
+2         0                           2

    Read a subset of the fields.

    If only a subset of the fields is desired, then two approaches exist. The first is to pass an array (where each element is an field) to the fields parameter. The second is to pass a single string (where the elements are separated by commas) to the fields_collapsed parameter. Like with records and records_collapsed described above, this function converts the more natural format (ie, fields) to the format required by the API (ie, fields_collapsed) if fields is specified and fields_collapsed is not.

    -
    #Return only the fields record_id, name_first, and age
-desired_fields_v1 <- c("record_id", "name_first", "age")
-ds_some_fields_v1 <- redcap_read(
-   redcap_uri = uri, 
-   token      = token, 
-   fields     = desired_fields_v1
-)$data
    -
    The data dictionary describing 16 fields was read from REDCap in 0.4 seconds.  The http status code was 200.
    -
    5 records and 1 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    Starting to read 5 records  at 2018-07-11 11:04:18.
    + +
    The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    5 records and 1 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    Starting to read 5 records  at 2018-08-10 20:36:09.
    Reading batch 1 of 1, with subjects 1 through 5 (ie, 5 unique subject records).
    5 records and 3 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    -
    #Return only the fields record_id, name_first, and age (alternate way)
-desired_fields_v2 <- "record_id, name_first, age"
-ds_some_fields_v2 <- redcap_read(
-   redcap_uri       = uri, 
-   token            = token, 
-   fields_collapsed = desired_fields_v2
-)$data
    -
    The data dictionary describing 16 fields was read from REDCap in 0.4 seconds.  The http status code was 200.
    -
    5 records and 1 columns were read from REDCap in 0.4 seconds.  The http status code was 200.
    -
    Starting to read 5 records  at 2018-07-11 11:04:19.
    + +
    The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    5 records and 1 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    Starting to read 5 records  at 2018-08-10 20:36:10.
    Reading batch 1 of 1, with subjects 1 through 5 (ie, 5 unique subject records).
    -
    5 records and 3 columns were read from REDCap in 0.4 seconds.  The http status code was 200.
    -
    ds_some_fields_v2 #Inspect the returned dataset
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -record id - -name first - -age -
    -1 - -Nutmeg - -11 -
    -2 - -Tumtum - -11 -
    -3 - -Marcus - -80 -
    -4 - -Trudy - -61 -
    -5 - -John Lee - -59 -
    +
    5 records and 3 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    + +
      record_id name_first age
+1         1     Nutmeg  11
+2         2     Tumtum  11
+3         3     Marcus  80
+4         4      Trudy  61
+5         5   John Lee  59

    Read a subset of records, conditioned on the values in some variables.

    The two techniques above can be combined when your datasets are large and you don’t want to pull records with certain values. Suppose you want to select subjects from the previous dataset if the were born before 1960 and their weight was over 70kg. Two calls to the server are required. The first call to REDCap pulls all the records, but for only three columns: record_id, dob, and weight. From this subset, identify the records that you want to pull all the data for; in this case, the desired record_id values are 3 & 5. The second call to REDCap pulls all the columns, but only for the identified records.

    -
    ######
-## Step 1: First call to REDCap
-desired_fields_v3 <- c("record_id", "dob", "weight")
-ds_some_fields_v3 <- redcap_read(
-   redcap_uri = uri, 
-   token      = token, 
-   fields     = desired_fields_v3
-)$data
    -
    The data dictionary describing 16 fields was read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    5 records and 1 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    Starting to read 5 records  at 2018-07-11 11:04:21.
    + +
    The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    5 records and 1 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    Starting to read 5 records  at 2018-08-10 20:36:11.
    Reading batch 1 of 1, with subjects 1 through 5 (ie, 5 unique subject records).
    -
    5 records and 3 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    ds_some_fields_v3 #Examine the these three variables.
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -record id - -dob - -weight -
    -1 - -2003-08-30 - -1 -
    -2 - -2003-03-10 - -1 -
    -3 - -1934-04-09 - -80 -
    -4 - -1952-11-02 - -54 -
    -5 - -1955-04-15 - -104 -
    -
    ######
-## Step 2: identify desired records, based on age & weight
-before_1960 <- (ds_some_fields_v3$dob <= as.Date("1960-01-01"))
-heavier_than_70_kg <- (ds_some_fields_v3$weight > 70)
-desired_records_v3 <- ds_some_fields_v3[before_1960 & heavier_than_70_kg, ]$record_id
-
-desired_records_v3 #Peek at IDs of the identified records
    +
    5 records and 3 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    + +
      record_id        dob weight
+1         1 2003-08-30      1
+2         2 2003-03-10      1
+3         3 1934-04-09     80
+4         4 1952-11-02     54
+5         5 1955-04-15    104
    + 
    [1] 3 5
    -
    ######
-## Step 3: second call to REDCap
-#Return only records that met the age & weight criteria.
-ds_some_rows_v3 <- redcap_read(
-   redcap_uri = uri, 
-   token      = token, 
-   records    = desired_records_v3
-)$data
    + 
    The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
    -
    2 records and 1 columns were read from REDCap in 0.4 seconds.  The http status code was 200.
    -
    Starting to read 2 records  at 2018-07-11 11:04:22.
    +
    2 records and 1 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    +
    Starting to read 2 records  at 2018-08-10 20:36:12.
    Reading batch 1 of 1, with subjects 3 through 5 (ie, 2 unique subject records).
    -
    2 records and 24 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    ds_some_rows_v3 #Examine the results.
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -record id - -name first - -name last - -address - -telephone - -email - -dob - -age - -sex - -demographics complete - -height - -weight - -bmi - -comments - -mugshot - -health complete - -race 1 - -race 2 - -race 3 - -race 4 - -race 5 - -race 6 - -ethnicity - -race and ethnicity complete -
    -3 - -Marcus - -Wood - -243 Hill St. Guthrie OK 73402 - -
      -
    1. 321-3333 -
    		 -mw@mwood.net - -1934-04-09 - -80 - -1 - -2 - -180.00 - -80 - -24.7 - -completely made up - -[document] - -2 - -0 - -0 - -0 - -1 - -1 - -0 - -0 - -2 -
    -5 - -John Lee - -Walker - -Hotel Suite New Orleans LA, 70115 - -
      -
    1. 321-5555 -
    		 -left@hippocket.com - -1955-04-15 - -59 - -1 - -2 - -193.04 - -104 - -27.9 - -Had a hand for trouble and a eye for cash - - -He had a gold watch chain and a black mustache - -[document] - -0 - -1 - -0 - -0 - -0 - -0 - -1 - -2 - -2 -
    +
    2 records and 24 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    + +
      record_id name_first name_last                            address
+1         3     Marcus      Wood     243 Hill St.\nGuthrie OK 73402
+2         5   John Lee    Walker Hotel Suite\nNew Orleans LA, 70115
+       telephone              email        dob age sex
+1 (405) 321-3333       mw@mwood.net 1934-04-09  80   1
+2 (405) 321-5555 left@hippocket.com 1955-04-15  59   1
+  demographics_complete height weight  bmi
+1                     2 180.00     80 24.7
+2                     2 193.04    104 27.9
+                                                                                     comments
+1                                                                          completely made up
+2 Had a hand for trouble and a eye for cash\n\nHe had a gold watch chain and a black mustache
+     mugshot health_complete race___1 race___2 race___3 race___4 race___5
+1 [document]               2        0        0        0        1        1
+2 [document]               0        1        0        0        0        0
+  race___6 ethnicity race_and_ethnicity_complete
+1        0         0                           2
+2        1         2                           2

    Additional Returned Information

    @@ -1277,18 +497,18 @@

    Additional Returned Information

  • The fields_collapsed fields passed to the API. This shows which field subsets, if any, were requested.
  • The elapsed_seconds measures the duration of the call.
    • -
    #Return only the fields record_id, name_first, and age
-all_information <- redcap_read(
-   redcap_uri = uri, 
-   token      = token, 
-   fields     = desired_fields_v1
-)
    -
    The data dictionary describing 16 fields was read from REDCap in 0.3 seconds.  The http status code was 200.
    + +
    The data dictionary describing 16 fields was read from REDCap in 0.2 seconds.  The http status code was 200.
    5 records and 1 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    -
    Starting to read 5 records  at 2018-07-11 11:04:24.
    +
    Starting to read 5 records  at 2018-08-10 20:36:14.
    Reading batch 1 of 1, with subjects 1 through 5 (ie, 5 unique subject records).
    -
    5 records and 3 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    all_information #Inspect the additional information
    +
    5 records and 3 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    + 
    $data
   record_id name_first age
 1         1     Nutmeg  11
@@ -1304,7 +524,7 @@ 
    Additional Returned Information
    
 [1] "200"
 
 $outcome_messages
-[1] "5 records and 3 columns were read from REDCap in 0.3 seconds.  The http status code was 200."
+[1] "5 records and 3 columns were read from REDCap in 0.2 seconds.  The http status code was 200."
 
 $records_collapsed
 [1] ""
@@ -1322,80 +542,81 @@ 
    Additional Returned Information
    
 [1] ""
 
 $elapsed_seconds
-[1] 1.348229
    +[1] 1.185983

    Session Information

    For the sake of documentation and reproducibility, the current report was rendered in the following environment. Click the line below to expand.

    -

    Environment

    -
    ─ Session info ──────────────────────────────────────────────────────────
- setting  value                       
- version  R version 3.4.4 (2018-03-15)
- os       Ubuntu 18.04 LTS            
- system   x86_64, linux-gnu           
- ui       RStudio                     
- language (EN)                        
- collate  en_US.UTF-8                 
- tz       America/Chicago             
- date     2018-07-11                  
-
-─ Packages ──────────────────────────────────────────────────────────────
+
    
+
    Environment 
    
+
    - Session info ----------------------------------------------------------
+ setting  value                                      
+ version  R version 3.5.1 Patched (2018-08-06 r75070)
+ os       Windows >= 8 x64                           
+ system   x86_64, mingw32                            
+ ui       RStudio                                    
+ language (EN)                                       
+ collate  English_United States.1252                 
+ tz       America/Chicago                            
+ date     2018-08-10                                 
+
+- Packages --------------------------------------------------------------
  package     * version     date       source                          
- assertthat    0.2.0       2017-04-11 cran (@0.2.0)                   
- backports     1.1.2       2017-12-13 cran (@1.1.2)                   
- bindr         0.1.1       2018-03-13 CRAN (R 3.4.3)                  
- bindrcpp    * 0.2.2       2018-03-29 CRAN (R 3.4.3)                  
- checkmate     1.8.6       2018-04-10 Github (mllg/checkmate@489319a) 
- clisymbols    1.2.0       2017-05-21 CRAN (R 3.4.3)                  
- codetools     0.2-15      2016-10-05 CRAN (R 3.4.3)                  
- colorspace    1.3-2       2016-12-14 CRAN (R 3.4.3)                  
- commonmark    1.5         2018-04-28 CRAN (R 3.4.4)                  
- crayon        1.3.4       2017-09-16 CRAN (R 3.4.3)                  
- curl          3.2         2018-03-28 CRAN (R 3.4.4)                  
- desc          1.2.0       2018-05-01 CRAN (R 3.4.4)                  
- devtools      1.13.6      2018-06-27 CRAN (R 3.4.4)                  
- digest        0.6.15      2018-01-28 CRAN (R 3.4.3)                  
- dplyr         0.7.6       2018-06-29 CRAN (R 3.4.4)                  
- evaluate      0.10.1      2017-06-24 CRAN (R 3.4.3)                  
- glue          1.2.0       2017-10-29 cran (@1.2.0)                   
- highr         0.7         2018-06-09 CRAN (R 3.4.4)                  
- hms           0.4.2.9000  2018-05-26 Github (tidyverse/hms@14e74ab)  
- htmltools     0.3.6       2017-04-28 CRAN (R 3.4.3)                  
- httr          1.3.1       2017-08-20 CRAN (R 3.4.3)                  
- kableExtra    0.9.0       2018-05-21 CRAN (R 3.4.4)                  
- knitr       * 1.20        2018-02-20 CRAN (R 3.4.3)                  
- magrittr    * 1.5         2014-11-22 cran (@1.5)                     
- memoise       1.1.0       2017-04-21 CRAN (R 3.4.3)                  
- munsell       0.5.0       2018-06-12 CRAN (R 3.4.4)                  
- pillar        1.2.3       2018-05-25 CRAN (R 3.4.4)                  
- pkgconfig     2.0.1       2017-03-21 cran (@2.0.1)                   
- plyr          1.8.4       2016-06-08 CRAN (R 3.4.3)                  
- purrr         0.2.5       2018-05-29 CRAN (R 3.4.4)                  
- R6            2.2.2       2017-06-17 CRAN (R 3.4.3)                  
- Rcpp          0.12.17     2018-05-18 CRAN (R 3.4.4)                  
- readr         1.2.0       2018-05-26 Github (tidyverse/readr@d6d622b)
+ assertthat    0.2.0       2017-04-11 CRAN (R 3.5.0)                  
+ backports     1.1.2       2017-12-13 CRAN (R 3.5.0)                  
+ bindr         0.1.1       2018-03-13 CRAN (R 3.5.0)                  
+ bindrcpp    * 0.2.2       2018-03-29 CRAN (R 3.5.0)                  
+ checkmate     1.8.9-9000  2018-08-09 Github (mllg/checkmate@29a1fb9) 
+ clisymbols    1.2.0       2017-05-21 CRAN (R 3.5.0)                  
+ codetools     0.2-15      2016-10-05 CRAN (R 3.5.1)                  
+ colorspace    1.3-2       2016-12-14 CRAN (R 3.5.0)                  
+ commonmark    1.5         2018-04-28 CRAN (R 3.5.0)                  
+ crayon        1.3.4       2017-09-16 CRAN (R 3.5.0)                  
+ curl          3.2         2018-03-28 CRAN (R 3.5.0)                  
+ desc          1.2.0       2018-05-01 CRAN (R 3.5.0)                  
+ devtools      1.13.6      2018-06-27 CRAN (R 3.5.0)                  
+ digest        0.6.15      2018-01-28 CRAN (R 3.5.0)                  
+ dplyr         0.7.6       2018-06-29 CRAN (R 3.5.1)                  
+ evaluate      0.11        2018-07-17 CRAN (R 3.5.1)                  
+ git2r         0.23.0      2018-07-17 CRAN (R 3.5.1)                  
+ glue          1.3.0       2018-07-17 CRAN (R 3.5.1)                  
+ hms           0.4.2.9001  2018-08-09 Github (tidyverse/hms@979286f)  
+ htmltools     0.3.6       2017-04-28 CRAN (R 3.5.0)                  
+ httr          1.3.1       2017-08-20 CRAN (R 3.5.0)                  
+ kableExtra    0.9.0       2018-05-21 CRAN (R 3.5.0)                  
+ knitr       * 1.20        2018-02-20 CRAN (R 3.5.0)                  
+ magrittr    * 1.5         2014-11-22 CRAN (R 3.5.0)                  
+ memoise       1.1.0       2017-04-21 CRAN (R 3.5.0)                  
+ munsell       0.5.0       2018-06-12 CRAN (R 3.5.0)                  
+ packrat       0.4.9-3     2018-06-01 CRAN (R 3.5.0)                  
+ pillar        1.3.0       2018-07-14 CRAN (R 3.5.1)                  
+ pkgconfig     2.0.1       2017-03-21 CRAN (R 3.5.0)                  
+ purrr         0.2.5       2018-05-29 CRAN (R 3.5.0)                  
+ R6            2.2.2       2017-06-17 CRAN (R 3.5.0)                  
+ Rcpp          0.12.18     2018-07-23 CRAN (R 3.5.1)                  
+ readr         1.2.0       2018-08-09 Github (tidyverse/readr@4b2e93a)
  REDCapR     * 0.9.10.9001 <NA>       local                           
- rlang         0.2.1       2018-05-30 CRAN (R 3.4.4)                  
- rmarkdown     1.10        2018-06-11 CRAN (R 3.4.4)                  
- roxygen2      6.0.1       2017-02-06 CRAN (R 3.4.4)                  
- rprojroot     1.3-2       2018-01-03 CRAN (R 3.4.3)                  
- rstudioapi    0.7         2017-09-07 CRAN (R 3.4.3)                  
- rvest         0.3.2       2016-06-17 CRAN (R 3.4.3)                  
- scales        0.5.0.9000  2018-03-29 Github (hadley/scales@d767915)  
- sessioninfo   1.0.0       2017-06-21 CRAN (R 3.4.3)                  
- stringi       1.2.3       2018-06-12 CRAN (R 3.4.4)                  
- stringr       1.3.1       2018-05-10 CRAN (R 3.4.4)                  
- testthat      2.0.0       2017-12-13 CRAN (R 3.4.3)                  
- tibble        1.4.2       2018-01-22 CRAN (R 3.4.3)                  
- tidyr         0.8.1       2018-05-18 CRAN (R 3.4.4)                  
- tidyselect    0.2.4       2018-02-26 CRAN (R 3.4.3)                  
- viridisLite   0.3.0       2018-02-01 CRAN (R 3.4.3)                  
- withr         2.1.2       2018-03-29 Github (jimhester/withr@79d7b0d)
- xml2          1.2.0       2018-01-24 CRAN (R 3.4.3)                  
- yaml          2.1.19      2018-05-01 CRAN (R 3.4.4)                  
    
-
    
-
    Report rendered by wibeasley at 2018-07-11, 11:04 -0500 in 12 seconds.
    
+ rlang         0.2.1       2018-05-30 CRAN (R 3.5.0)                  
+ rmarkdown     1.10        2018-06-11 CRAN (R 3.5.0)                  
+ roxygen2      6.1.0       2018-07-27 CRAN (R 3.5.1)                  
+ rprojroot     1.3-2       2018-01-03 CRAN (R 3.5.0)                  
+ rstudioapi    0.7         2017-09-07 CRAN (R 3.5.0)                  
+ rvest         0.3.2       2016-06-17 CRAN (R 3.5.0)                  
+ scales        1.0.0       2018-08-09 CRAN (R 3.5.1)                  
+ sessioninfo   1.0.0       2017-06-21 CRAN (R 3.5.0)                  
+ stringi       1.2.4       2018-07-20 CRAN (R 3.5.1)                  
+ stringr       1.3.1       2018-05-10 CRAN (R 3.5.0)                  
+ testthat      2.0.0       2017-12-13 CRAN (R 3.5.0)                  
+ tibble        1.4.2       2018-01-22 CRAN (R 3.5.0)                  
+ tidyr         0.8.1       2018-05-18 CRAN (R 3.5.0)                  
+ tidyselect    0.2.4       2018-02-26 CRAN (R 3.5.0)                  
+ viridisLite   0.3.0       2018-02-01 CRAN (R 3.5.0)                  
+ withr         2.1.2       2018-03-15 CRAN (R 3.5.0)                  
+ xml2          1.2.0       2018-01-24 CRAN (R 3.5.0)                  
+ yaml          2.2.0       2018-07-25 CRAN (R 3.5.1)
    +
    +

    Report rendered by Will at 2018-08-10, 20:36 -0500 in 10 seconds.

    diff --git a/inst/doc/SecurityDatabase.Rmd b/inst/doc/SecurityDatabase.Rmd index f2bfa046..fef69fc6 100644 --- a/inst/doc/SecurityDatabase.Rmd +++ b/inst/doc/SecurityDatabase.Rmd @@ -11,7 +11,7 @@ vignette: > Description ======================================== -The SQL code below adds schemas, a table and two stored procedures to an existing Microsoft SQL Database. This second database is not essential to calling the REDCap API, but it helps manage tokens securely. +The SQL code below adds schemas, a table and two stored procedures to an existing Microsoft SQL Server database. This second database is not essential to calling the REDCap API, but it helps manage tokens securely. This database contains the tokens and other sensitive content (such as passwords, API tokens, and file paths) that should not be stored in a Git repository (even a private Git repository). These passwords can be retrieved by `REDCapR::retrieve_credential_mssql()`. @@ -19,23 +19,25 @@ This database contains the tokens and other sensitive content (such as passwords Create a DSN on each client ======================================== -After executing the SQL code in an existing database, create an ODBC [DSN](http://en.wikipedia.org/wiki/Data_source_name) on *each* client machine that calls the database. Download the most recent drivers (as of Aug 2016, the [most recent version is 13.1](https://msdn.microsoft.com/library/mt703139.aspx) for Windows and Linux, then run the wizard. Many values in the wizard will remain at the default values. Here are the important ones to change. +After executing the SQL code in an existing database, create an ODBC [DSN](http://en.wikipedia.org/wiki/Data_source_name) on *each* client machine that calls the database. Download the most recent drivers (as of Aug 2018, the [most recent version is 17](https://docs.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server) for Windows and Linux), then run the wizard. Many values in the wizard will remain at the default values. Here are the important ones to change. 1. Set the DSN's `name` field to whatever is used in the repository's R code. 2. Set the authenticity method to `Integrated Windows authentication`. -3. Set the `default database` to the name of the database that containing the tokens *i.e.*, corresponding to the SQL code below in the example). +3. Set the `default database` to the name of the database that containing the tokens + +In our code below, both DSN and database are named `auxiliary_security`. Note ======================================== -We use Microsoft SQL Server, because that fits our University's infrastructure the easiest. But this approach theoretically can work with any LDAP-enabled database server. Please contact us if your institution is using something other than SQL Server (or a different configuration of these components), and would like help adapting this approach to your infrastructure. +We use Microsoft SQL Server, because that fits our university's infrastructure most easily. But this approach theoretically can work with any LDAP-enabled database server. Please contact us if your institution is using something other than SQL Server (or a different configuration of these components), and would like help adapting this approach. Create Database ======================================== -This SQL code is run once inside an existing database to establish the schemas, table, and stored procedure used by `REDCapR::retrieve_credential_mssql()`. +This SQL code is run once inside an existing database to establish the schemas, table, and stored procedure used by `REDCapR::retrieve_credential_mssql()`. In this example, we've arbitrarily called the database `auxiliary_security`. ```sql ------- SQL code to create necessary components in a Microsoft SQL Sever database ------- @@ -53,12 +55,12 @@ GO -- Create a table to contain the token -- CREATE TABLE [redcap_private].[tbl_credential]( - [id] [smallint] NOT NULL, - [username] [varchar](30) NOT NULL, - [project_id] [smallint] NOT NULL, - [instance] [varchar](30) NOT NULL, - [token] [char](32) NOT NULL, - [redcap_uri] [varchar](255) NOT NULL, + id smallint NOT NULL, + username varchar(30) NOT NULL, + project_id smallint NOT NULL, + instance varchar(30) NOT NULL, + token char(32) NOT NULL, + redcap_uri varchar(255) NOT NULL, CONSTRAINT [PK_credential] PRIMARY KEY CLUSTERED ( [id] ASC @@ -67,9 +69,9 @@ CREATE TABLE [redcap_private].[tbl_credential]( CREATE UNIQUE NONCLUSTERED INDEX [IX_tbl_credential_unique] ON [redcap_private].[tbl_credential] ( - [instance] ASC, - [project_id] ASC, - [username] ASC + instance ASC, + project_id ASC, + username ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] GO @@ -78,18 +80,19 @@ GO -- Notice it should a different (and more permissive) schema than the table. -- CREATE PROCEDURE [redcap].[prc_credential] - -- Add the parameters for the stored procedure here - @project_id smallint, - @instance varchar(30) + @instance varchar(30) AS BEGIN - -- SET NOCOUNT ON added to prevent extra result sets from - -- interfering with SELECT statements. SET NOCOUNT ON; SELECT username, project_id, token, redcap_uri FROM [redcap_private].[tbl_credential] - WHERE username=system_user AND project_id=@project_id AND instance=@instance + WHERE + username = system_user -- The username from the server's OS. + AND + project_id = @project_id -- Restricts to the desired REDCap project. + AND + instance = @instance -- System accommodates multiple REDCap instances. END ``` @@ -97,23 +100,19 @@ END Create user credentials to the auxiliary database ======================================== -Add a user's LDAP account to the `SecurityAuxiliary` database so that they can query the tables to retrieve their API. - -Notice that this only gives the permissions to retrieve the token. You must still: - 1. grant them API privileges to each appropriate REDCap project, and - 2. copy the API from the REDCap database into the SecurityAuxiliary database. +Add a user's LDAP account to the `auxiliary_security` database so that they can query the tables to retrieve their API. -In the future `REDCapR` may expose a function that allows the user to perform the second step (through a stored procedure). +Notice that this only gives the permissions to retrieve the token. You still must grant them API privileges to each appropriate REDCap project. The automation in the R file below will copy the API token from the MySQL database into the `auxiliary_security` database (see the 'Transfer Credentials' section). -Also, do not give typical users authorization for the 'redcap_private' schema. The current system allows the to view only their own tokens. +Only database admins should have authorization for the 'redcap_private' schema. Typical users should not be authorized for this schema. The current system allows typical users to view only their own tokens. ```sql ----------------------------------------------------------------------- --- Add a OUHSC's user account to the auxiliary_security database so that they can query the tables to retrieve their API. +-- Add a user account to the auxiliary_security database so that they can query the tables to retrieve their API. -- Notice that this only gives the permissions to retrieve the token. You must still: -- 1) grant them API privileges to each appropriate REDCap project, and -- 2) copy the API from the REDCap database into the auxiliary_security database. --- Also, do not give typical users authorization for the 'redcap_private' schema. The current system allows the to view only their own tokens. +-- Also, do not give typical users authorization for the 'redcap_private' schema. The current system allows them to view only their own tokens. ----------------------------------------------------------------------- -- STEP #1: Declare the user name. If everything runs correctly, this should be the only piece of code that you need to modify. @@ -139,7 +138,7 @@ DECLARE @login_count AS INT; SET @login_count = (SELECT COUNT(*) AS login_count print 'Logins matching desired name should equal 1. It equals: ' + CONVERT(varchar, @login_count); print '' ----------------------------------------------------------------------- --- STEP #3: Create a user account for the *data base*, after switching the database under focus to auxiliary_security. +-- STEP #3: Create a user account for the *database*, after switching the database under focus to auxiliary_security. print 'Step #3 executing....' USE [auxiliary_security] DECLARE @sql_create_user nvarchar(max) @@ -186,11 +185,11 @@ print 'Step #5 executed'; print '' Transfer Credentials ======================================== -Manually transfer tokens to the auxiliary server becomes unmanageable as your institution's collection of API users grows. This script demonstrates how to progamatically transfer all tokens from multiple REDCap instances on your network. The basic steps are: +Manually transferring tokens to the auxiliary server becomes unmanageable as your institution's collection of API users grows. This script demonstrates how to progamatically transfer all tokens from multiple REDCap instances. The basic steps are: -1. Read from the MySQL database underneath each REDCap instance. +1. Read from the MySQL database(s) underneath each REDCap instance on your campus. 1. Combine & groom the credentials. -1. Upload to SQL Server. +1. Upload to SQL Server (called `auxiliary_security` here). ```r rm(list=ls(all=TRUE)) #Clear the memory for any variables set from any previous runs. @@ -198,130 +197,159 @@ rm(list=ls(all=TRUE)) #Clear the memory for any variables set from any previous # ---- load-sources ------------------------------------------------------------ # ---- load-packages ----------------------------------------------------------- +if( !require(OuhscMunge) ) + stop('The `OuhscMunge` package needs to be installed with `devtools::install_github("OuhscBbmc/OuhscMunge")`.') + +testit::assert( + "The `OuhscMunge` package should meet a minimal version.", + compareVersion( as.character(packageVersion("OuhscMunge")), "0.1.9.9009") >= 0L +) + library(magrittr) -requireNamespace("RODBC") +requireNamespace("DBI") +requireNamespace("odbc") requireNamespace("dplyr") requireNamespace("readr") requireNamespace("tibble") +requireNamespace("testit") requireNamespace("checkmate") +requireNamespace("OuhscMunge") # devtools::install_github("OuhscBbmc/OuhscMunge") + # ---- declare-globals --------------------------------------------------------- +# This file assume your campus has two REDCap instances. +# Modify each (a) database name, (b) REDCap URL, and (c) DSN name. + +name_production <- "production" +name_dev <- "dev" + +uri_production <- "https://redcap-production.ouhsc.edu/redcap/api/", +uri_dev <- "https://redcap-dev.ouhsc.edu/redcap/api/" + +dsn_production <- "redcap-production" +dsn_dev <- "redcap-dev" +dsn_source <- "auxiliary_security" # The DSN of the token server. # The Activity Directory name that should precede each username. -# This should correspond with the result of `SYSTEM_USER` +# This should correspond with the result of SQL Server's `SYSTEM_USER` function # (https://msdn.microsoft.com/en-us/library/ms179930.aspx) ldap_prefix <- "OUHSC\\" -#Create a SQL statement for each REDCap instance. Only the `instance` value should change. +#### +# Nothing below this line should need to change, assuming: +# 1. the vignette was followed exactly (https://ouhscbbmc.github.io/REDCapR/articles/SecurityDatabase.html), +# 2. your campus has exactly two REDCap instances. + +# SQL sent to the MySQL database underneath each REDCap instance. sql <- " SELECT username, project_id, api_token FROM redcap_user_rights - WHERE api_token IS NOT NULL" + WHERE api_token IS NOT NULL +" + +# Update this ad-hoc CSV. Each row should represent one REDCap instance. +ds_url <- tibble::tribble( + ~instance , ~redcap_uri, + name_production , uri_production, + name_dev , uri_dev +) -#Update this ad-hoc CSV. Each row should represent one REDCap instance. -# Choose any casual name for the first variable, consistent with the `tweak-data` chunk below. -# Enter the exact URL for the second variable. -ds_url <- readr::read_csv(paste( - "instance,redcap_uri", - "production,https://redcap-production.ouhsc.edu/redcap/api/", - "dev,https://redcap-dev.ouhsc.edu/redcap/api/", -sep="\n")) +# Remove variables that aren't used below. +rm(uri_production, uri_dev) # ---- load-data --------------------------------------------------------------- -# Load the credentials from the first instance. -channel <- RODBC::odbcConnect("redcap-production") # odbcGetInfo(channel) -ds_prod <- RODBC::sqlQuery(channel, query=sql, stringsAsFactors=F) -RODBC::odbcClose(channel); rm(channel) +# Load the credentials from the first/production REDCap instance. +cnn_production <- DBI::dbConnect(odbc::odbc(), dsn=dsn_production) +ds_production <- DBI::dbGetQuery(cnn_production, sql) +DBI::dbDisconnect(cnn_production); rm(cnn_production, dsn_production) + +# Load the credentials from the second/dev REDCap instance. +cnn_dev <- DBI::dbConnect(odbc::odbc(), dsn=dsn_dev) +ds_dev <- DBI::dbGetQuery(cnn_dev, sql) +DBI::dbDisconnect(cnn_dev); rm(cnn_dev, dsn_dev) -# Load the credentials from the second instance. -# Duplicate or remove this block, dependending on the number of instances. -channel <- RODBC::odbcConnect("redcap-dev") # odbcGetInfo(channel) -ds_dev <- RODBC::sqlQuery(channel, query=sql, stringsAsFactors=F) -RODBC::odbcClose(channel); rm(channel, sql) +rm(sql) -# Assert these variables contain valid datasets (instead of a character error message), and -# that at least some rows were returned. -# Adjust this to smaller values if necessary. It's really just to catch blatant retrieval problems. -checkmate::assert_data_frame(ds_bbmc, min.rows=5) -checkmate::assert_data_frame(ds_dev , min.rows=5) +# Assert these are valid datasets and contain at least 5 rows. +# Adjust '5' to smaller value if necessary. It's just to catch blatant retrieval problems. +checkmate::assert_data_frame(ds_production, min.rows=5) +checkmate::assert_data_frame(ds_dev , min.rows=5) # ---- tweak-data -------------------------------------------------------------- -#Label each instance, so they're distinguishable later. Add/remove lines, depending on the number of campus instances -ds_prod$instance <- "production" -ds_dev$instance <- "dev" - -#Combine the token collection from each instance. Then prefix the username and include the URL of each instance. -ds <- ds_prod %>% - dplyr::union(ds_dev) %>% # Add/remove unions, based on the number of REDCap instances. - dplyr::select_( - "username" = "`username`" - , "project_id" = "`project_id`" - , "instance" = "`instance`" - , "token" = "`api_token`" +# Label each instance, so they're distinguishable later. +ds_production$instance <- name_production +ds_dev$instance <- name_dev + +# Stack the token collection from each instance. Then prefix the username and include the URL of each instance. +ds <- ds_production %>% + dplyr::union(ds_dev) %>% # Remove union if the dev instance isn't included. + tibble::as_tibble() %>% + dplyr::select( + username = username, + project_id = project_id, + instance = instance, + token = api_token ) %>% - dplyr::arrange(instance, project_id, username) %>% dplyr::mutate( - username = paste0(ldap_prefix, username), # Qualify for the Active Directory. - id = seq_len(n()) # For the sake of a clustered primary key. + username = paste0(ldap_prefix, username), # Qualify for the Active Directory. ) %>% - dplyr::left_join( ds_url, by="instance") # Include the instance URL. + dplyr::left_join( ds_url, by="instance") %>% # Include the instance URL. + dplyr::arrange(instance, project_id, username) %>% + tibble::rowid_to_column("id") # For the sake of a clustered primary key. -rm(ds_prod, ds_dev, ds_url) +rm(ds_production, ds_dev, ds_url) +rm(name_production, name_dev) +rm(ldap_prefix) # ---- verify-values ----------------------------------------------------------- -# devtools::install_github("OuhscBbmc/OuhscMunge"); OuhscMunge::verify_value_headstart(ds) + # Assert that the dataset is well-behaved. -checkmate::assert_integer( ds$id , any.missing=F , lower=1, upper=2^31-1 , unique=T) -checkmate::assert_character(ds$username , any.missing=F , pattern="^.{1,255}$" ) -checkmate::assert_integer( ds$project_id , any.missing=F , lower=1, upper=2^31-1 ) -checkmate::assert_character(ds$token , any.missing=F , pattern="^.{32}$" , unique=T) -checkmate::assert_character(ds$instance , any.missing=F , pattern="^.{1,255}$" ) -checkmate::assert_character(ds$redcap_uri , any.missing=F , pattern="^.{1,255}$" ) +# OuhscMunge::verify_value_headstart(ds) +checkmate::assert_integer( ds$id , any.missing=F, lower=1, upper=.Machine$integer.max, unique=T) +checkmate::assert_character(ds$username , any.missing=F, pattern="^.{1,255}$" ) +checkmate::assert_integer( ds$project_id , any.missing=F, lower=1, upper=.Machine$integer.max ) +checkmate::assert_character(ds$token , any.missing=F, pattern="^[A-Z0-9]{32}$" , unique=T) +checkmate::assert_character(ds$instance , any.missing=F, pattern="^.{1,255}$" ) +checkmate::assert_character(ds$redcap_uri , any.missing=F, pattern="^.{1,255}$" ) testit::assert( "The `username` x `project_id` x `instance` must be unique.", sum(duplicated(paste0(ds$username, "-", ds$project_id, "-", ds$instance))) == 0L ) -testit::assert("There should be at least 10 tokens written." , 10L <= nrow(ds)) +testit::assert("At least 10 tokens should be ready to write." , 10L <= nrow(ds)) # ---- specify-columns-to-upload ----------------------------------------------- # Dictate the exact columns and order that will be uploaded. columns_to_write <- c("id", "username", "project_id", "instance", "token", "redcap_uri") -ds_slim <- ds[, columns_to_write] - +ds_slim <- ds[, columns_to_write] rm(columns_to_write) -# ---- upload-to-db-credential ------------------------------------------------------------ +# ---- upload-to-db ------------------------------------------------------------------ -#Upload to SQL Server through ODBC. -(start_time <- Sys.time()) - -db_table <- "redcap_private.tbl_credential" -channel <- RODBC::odbcConnect("auxiliary_security") #getSqlTypeInfo("Microsoft SQL Server") #;odbcGetInfo(channel) - -column_info <- RODBC::sqlColumns(channel, db_table) -var_types <- as.character(column_info$TYPE_NAME) -names(var_types) <- as.character(column_info$COLUMN_NAME) #var_types - -RODBC::sqlClear(channel, db_table) -RODBC::sqlSave(channel, ds_slim, db_table, append=TRUE, rownames=FALSE, fast=TRUE, varTypes=var_types) -RODBC::odbcClose(channel); rm(channel) - -(elapsed_duration <- Sys.time() - start_time) #0.6026149 secs 2016-08-29. -rm(db_table, column_info, var_types, start_time, elapsed_duration) +OuhscMunge::upload_sqls_odbc( + d = ds_slim, + schema_name = "redcap_private", + table_name = "tbl_credential", + dsn_name = dsn_source, + create_table = FALSE, + clear_table = TRUE, + transaction = TRUE, + verbose = TRUE +) +# Uploading 252 tokens takes 0.004 minutes. ``` Document Info ======================================== -This document is primarily based on REDCap version 6.11.5, and was last updated 2016-08-30. A development version of the document is available on GitHub: https://ouhscbbmc.github.io/REDCapR/articles/SecurityDatabase.html +This document is primarily based on REDCap version 8.4.0, and was last updated 2018-08-10. A development version of the document is available on GitHub: https://ouhscbbmc.github.io/REDCapR/articles/SecurityDatabase.html diff --git a/inst/doc/SecurityDatabase.html b/inst/doc/SecurityDatabase.html index a24dd751..7dddb6d3 100644 --- a/inst/doc/SecurityDatabase.html +++ b/inst/doc/SecurityDatabase.html @@ -11,7 +11,7 @@ - + Security Database @@ -19,46 +19,256 @@ - + @@ -68,303 +278,333 @@

    Security Database

    -

    2018-07-11

    +

    2018-08-10

    Description

    -

    The SQL code below adds schemas, a table and two stored procedures to an existing Microsoft SQL Database. This second database is not essential to calling the REDCap API, but it helps manage tokens securely.

    +

    The SQL code below adds schemas, a table and two stored procedures to an existing Microsoft SQL Server database. This second database is not essential to calling the REDCap API, but it helps manage tokens securely.

    This database contains the tokens and other sensitive content (such as passwords, API tokens, and file paths) that should not be stored in a Git repository (even a private Git repository). These passwords can be retrieved by REDCapR::retrieve_credential_mssql().

    Create a DSN on each client

    -

    After executing the SQL code in an existing database, create an ODBC DSN on each client machine that calls the database. Download the most recent drivers (as of Aug 2016, the most recent version is 13.1 for Windows and Linux, then run the wizard. Many values in the wizard will remain at the default values. Here are the important ones to change.

    +

    After executing the SQL code in an existing database, create an ODBC DSN on each client machine that calls the database. Download the most recent drivers (as of Aug 2018, the most recent version is 17 for Windows and Linux), then run the wizard. Many values in the wizard will remain at the default values. Here are the important ones to change.

    1. Set the DSN’s name field to whatever is used in the repository’s R code.
    2. Set the authenticity method to Integrated Windows authentication.
      3. -
    3. Set the default database to the name of the database that containing the tokens i.e., corresponding to the SQL code below in the example).
      4. +
    4. Set the default database to the name of the database that containing the tokens
    +

    In our code below, both DSN and database are named auxiliary_security.

    Note

    -

    We use Microsoft SQL Server, because that fits our University’s infrastructure the easiest. But this approach theoretically can work with any LDAP-enabled database server. Please contact us if your institution is using something other than SQL Server (or a different configuration of these components), and would like help adapting this approach to your infrastructure.

    +

    We use Microsoft SQL Server, because that fits our university’s infrastructure most easily. But this approach theoretically can work with any LDAP-enabled database server. Please contact us if your institution is using something other than SQL Server (or a different configuration of these components), and would like help adapting this approach.

    Create Database

    -

    This SQL code is run once inside an existing database to establish the schemas, table, and stored procedure used by REDCapR::retrieve_credential_mssql().

    -
    ------- SQL code to create necessary components in a Microsoft SQL Sever database -------
-
------------------------------------------------------------------------
--- Create two schemas.
--- The first scehma is accessible by all REDCap API users.
--- The second scehma is restricted to administrators.
---
-CREATE SCHEMA [redcap]
-CREATE SCHEMA [redcap_private]
-GO
-
------------------------------------------------------------------------
--- Create a table to contain the token
---
-CREATE TABLE [redcap_private].[tbl_credential](
-  [id]              [smallint]          NOT NULL,
-  [username]        [varchar](30)       NOT NULL,
-  [project_id]      [smallint]          NOT NULL,
-  [instance]        [varchar](30)       NOT NULL,
-  [token]           [char](32)          NOT NULL,
-  [redcap_uri]      [varchar](255)      NOT NULL,
- CONSTRAINT [PK_credential] PRIMARY KEY CLUSTERED
-(
-  [id] ASC
-)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
-) ON [PRIMARY]
-
-CREATE UNIQUE NONCLUSTERED INDEX [IX_tbl_credential_unique] ON [redcap_private].[tbl_credential]
-(
-  [instance]        ASC,
-  [project_id]      ASC,
-  [username]        ASC
-)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
-GO
-
------------------------------------------------------------------------
--- Create a stored procedure for users to call to retrieve the token.
--- Notice it should a different (and more permissive) schema than the table.
---
-CREATE PROCEDURE [redcap].[prc_credential]
-  -- Add the parameters for the stored procedure here
-
-  @project_id smallint,
-  @instance varchar(30)
-AS
-BEGIN
-  -- SET NOCOUNT ON added to prevent extra result sets from
-  -- interfering with SELECT statements.
-  SET NOCOUNT ON;
-
-  SELECT username, project_id, token, redcap_uri FROM [redcap_private].[tbl_credential]
-  WHERE username=system_user AND project_id=@project_id AND instance=@instance
-END
    +

    This SQL code is run once inside an existing database to establish the schemas, table, and stored procedure used by REDCapR::retrieve_credential_mssql(). In this example, we’ve arbitrarily called the database auxiliary_security.

    +
    ------- SQL code to create necessary components in a Microsoft SQL Sever database -------
+
+-----------------------------------------------------------------------
+-- Create two schemas.
+-- The first scehma is accessible by all REDCap API users.
+-- The second scehma is restricted to administrators.
+--
+CREATE SCHEMA [redcap]
+CREATE SCHEMA [redcap_private]
+GO
+
+-----------------------------------------------------------------------
+-- Create a table to contain the token
+--
+CREATE TABLE [redcap_private].[tbl_credential](
+  id            smallint        NOT NULL,
+  username      varchar(30)     NOT NULL,
+  project_id    smallint        NOT NULL,
+  instance      varchar(30)     NOT NULL,
+  token         char(32)        NOT NULL,
+  redcap_uri    varchar(255)    NOT NULL,
+ CONSTRAINT [PK_credential] PRIMARY KEY CLUSTERED
+(
+  [id] ASC
+)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
+) ON [PRIMARY]
+
+CREATE UNIQUE NONCLUSTERED INDEX [IX_tbl_credential_unique] ON [redcap_private].[tbl_credential]
+(
+  instance        ASC,
+  project_id      ASC,
+  username        ASC
+)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
+GO
+
+-----------------------------------------------------------------------
+-- Create a stored procedure for users to call to retrieve the token.
+-- Notice it should a different (and more permissive) schema than the table.
+--
+CREATE PROCEDURE [redcap].[prc_credential]
+  @project_id smallint,
+  @instance   varchar(30)
+AS
+BEGIN
+  SET NOCOUNT ON;
+
+  SELECT username, project_id, token, redcap_uri FROM [redcap_private].[tbl_credential]
+  WHERE
+    username   = system_user -- The username from the server's OS.
+    AND
+    project_id = @project_id -- Restricts to the desired REDCap project.
+    AND
+    instance   = @instance   -- System accommodates multiple REDCap instances.
+END

    Create user credentials to the auxiliary database

    -

    Add a user’s LDAP account to the SecurityAuxiliary database so that they can query the tables to retrieve their API.

    -

    Notice that this only gives the permissions to retrieve the token. You must still: 1. grant them API privileges to each appropriate REDCap project, and 2. copy the API from the REDCap database into the SecurityAuxiliary database.

    -

    In the future REDCapR may expose a function that allows the user to perform the second step (through a stored procedure).

    -

    Also, do not give typical users authorization for the ‘redcap_private’ schema. The current system allows the to view only their own tokens.

    -
    -----------------------------------------------------------------------
--- Add a OUHSC's user account to the auxiliary_security database so that they can query the tables to retrieve their API.
--- Notice that this only gives the permissions to retrieve the token.  You must still:
---   1) grant them API privileges to each appropriate REDCap project, and
---   2) copy the API from the REDCap database into the  auxiliary_security database.
--- Also, do not give typical users authorization for the 'redcap_private' schema.  The current system allows the to view only their own tokens.
------------------------------------------------------------------------
-
--- STEP #1: Declare the user name.  If everything runs correctly, this should be the only piece of code that you need to modify.
-print 'Step #1 executing....'
-USE [master]
-GO
-DECLARE @qualified_user_name varchar(255); SET @qualified_user_name = '[OUHSC\lsuarez3]'
-print 'Resulting login name: ' + @qualified_user_name; print ''
-
---EXEC sp_helplogins @LoginNamePattern=@qualified_user_name
---SELECT * FROM master..syslogins WHERE name = @qualified_user_name
---SELECT * FROM auxiliary_security.sys.sysusers
---SELECT * FROM sys.database_permissions
---SELECT * FROM sys.server_principals
-
------------------------------------------------------------------------
--- STEP #2: Create a login for the *server*.
-print 'Step #2 executing....'
-DECLARE @sql_create_login nvarchar(max)
-SET @sql_create_login = 'CREATE LOGIN ' + @qualified_user_name + ' FROM WINDOWS WITH DEFAULT_DATABASE=[auxiliary_security]'
-EXECUTE sp_executesql @sql_create_login
-DECLARE @login_count AS INT; SET @login_count = (SELECT COUNT(*) AS login_count FROM master..syslogins WHERE '[' + loginname + ']' = @qualified_user_name)
-print 'Logins matching desired name should equal 1.  It equals: ' + CONVERT(varchar, @login_count); print ''
-
------------------------------------------------------------------------
--- STEP #3: Create a user account for the *data base*, after switching the database under focus to auxiliary_security.
-print 'Step #3 executing....'
-USE [auxiliary_security]
-DECLARE @sql_create_user nvarchar(max)
-SET @sql_create_user = 'CREATE USER ' + @qualified_user_name + ' FOR LOGIN ' + @qualified_user_name
-EXECUTE sp_executesql @sql_create_user
-DECLARE @user_count AS INT; SET @user_count = (SELECT COUNT(*) AS user_count FROM auxiliary_security.sys.sysusers WHERE '[' + name + ']' = @qualified_user_name)
-print 'User accounts matching desired name should equal 1.  It equals: ' + CONVERT(varchar, @user_count); print ''
-
------------------------------------------------------------------------
--- STEP #4: Grant appropriate privileges for the 'redcap' schema.
-print 'Step #4 executing....'
-DECLARE @sql_grant_schema_redcap nvarchar(max)
-SET @sql_grant_schema_redcap = 'GRANT EXECUTE ON SCHEMA::[redcap] TO ' + @qualified_user_name
-EXECUTE sp_executesql @sql_grant_schema_redcap
-print 'Step #4 executed'; print ''
-
------------------------------------------------------------------------
--- STEP #5: Grant appropriate privileges for the 'Security' schema.
-print 'Step #5 executing....'
-DECLARE @sql_grant_schema_security nvarchar(max)
-SET @sql_grant_schema_security = 'GRANT EXECUTE ON SCHEMA::[security] TO ' + @qualified_user_name
-EXECUTE sp_executesql @sql_grant_schema_security
-print 'Step #5 executed'; print ''
-
------------------------------------------------------------------------
--- OPTIONAL STEP: Delete the user from the database (the first line) and then the server (the second line).
--- The person's other database user accounts (besides with the auxiliary_security database) will NOT be automatically deleted by these two lines.
---USE [auxiliary_security]; DROP USER [OUHSC\lsuarez3]
---USE [master]; DROP LOGIN [OUHSC\lsuarez3]
-
------------------------------------------------------------------------
--- REFERENCES & NOTES
-  --The @qualified_user_name must have both (a) the 'OUHSC' domain qualification, and (b) the square brackets (to escape the backslash).
-  --Using sp_executesql to add users: http://www.sqlservercentral.com/Forums/Topic497615-359-1.aspx
-  --Check if a server login exists: http://stackoverflow.com/questions/37275/sql-query-for-logins
-  --Retrieve database users: http://stackoverflow.com/questions/2445444/how-to-get-a-list-of-users-for-all-instances-databases
-  --Concatenating strings: http://blog.sqlauthority.com/2010/11/25/sql-server-concat-function-in-sql-server-sql-concatenation/
-  --DROP USER from database: http://msdn.microsoft.com/en-us/library/ms189438.aspx
-  --DROP LOGIN from server: http://msdn.microsoft.com/en-us/library/ms188012.aspx
-  --Declaring variables (eg, the username above): http://technet.microsoft.com/en-us/library/aa258839.aspx
-  --A different (& non-dynamic) way to establish a user: http://pic.dhe.ibm.com/infocenter/dmndhelp/v8r5m0/index.jsp?topic=%2Fcom.ibm.wbpm.imuc.sbpm.doc%2Ftopics%2Fdb_create_users_nd_aix.html
-  --If the variable has to cross a 'GO' (which the current version of the script doesn't need): http://stackoverflow.com/questions/937336/is-there-a-way-to-persist-a-variable-across-a-go
    +

    Add a user’s LDAP account to the auxiliary_security database so that they can query the tables to retrieve their API.

    +

    Notice that this only gives the permissions to retrieve the token. You still must grant them API privileges to each appropriate REDCap project. The automation in the R file below will copy the API token from the MySQL database into the auxiliary_security database (see the ‘Transfer Credentials’ section).

    +

    Only database admins should have authorization for the ‘redcap_private’ schema. Typical users should not be authorized for this schema. The current system allows typical users to view only their own tokens.

    +
    -----------------------------------------------------------------------
+-- Add a user account to the auxiliary_security database so that they can query the tables to retrieve their API.
+-- Notice that this only gives the permissions to retrieve the token.  You must still:
+--   1) grant them API privileges to each appropriate REDCap project, and
+--   2) copy the API from the REDCap database into the  auxiliary_security database.
+-- Also, do not give typical users authorization for the 'redcap_private' schema.  The current system allows them to view only their own tokens.
+-----------------------------------------------------------------------
+
+-- STEP #1: Declare the user name.  If everything runs correctly, this should be the only piece of code that you need to modify.
+print 'Step #1 executing....'
+USE [master]
+GO
+DECLARE @qualified_user_name varchar(255); SET @qualified_user_name = '[OUHSC\lsuarez3]'
+print 'Resulting login name: ' + @qualified_user_name; print ''
+
+--EXEC sp_helplogins @LoginNamePattern=@qualified_user_name
+--SELECT * FROM master..syslogins WHERE name = @qualified_user_name
+--SELECT * FROM auxiliary_security.sys.sysusers
+--SELECT * FROM sys.database_permissions
+--SELECT * FROM sys.server_principals
+
+-----------------------------------------------------------------------
+-- STEP #2: Create a login for the *server*.
+print 'Step #2 executing....'
+DECLARE @sql_create_login nvarchar(max)
+SET @sql_create_login = 'CREATE LOGIN ' + @qualified_user_name + ' FROM WINDOWS WITH DEFAULT_DATABASE=[auxiliary_security]'
+EXECUTE sp_executesql @sql_create_login
+DECLARE @login_count AS INT; SET @login_count = (SELECT COUNT(*) AS login_count FROM master..syslogins WHERE '[' + loginname + ']' = @qualified_user_name)
+print 'Logins matching desired name should equal 1.  It equals: ' + CONVERT(varchar, @login_count); print ''
+
+-----------------------------------------------------------------------
+-- STEP #3: Create a user account for the *database*, after switching the database under focus to auxiliary_security.
+print 'Step #3 executing....'
+USE [auxiliary_security]
+DECLARE @sql_create_user nvarchar(max)
+SET @sql_create_user = 'CREATE USER ' + @qualified_user_name + ' FOR LOGIN ' + @qualified_user_name
+EXECUTE sp_executesql @sql_create_user
+DECLARE @user_count AS INT; SET @user_count = (SELECT COUNT(*) AS user_count FROM auxiliary_security.sys.sysusers WHERE '[' + name + ']' = @qualified_user_name)
+print 'User accounts matching desired name should equal 1.  It equals: ' + CONVERT(varchar, @user_count); print ''
+
+-----------------------------------------------------------------------
+-- STEP #4: Grant appropriate privileges for the 'redcap' schema.
+print 'Step #4 executing....'
+DECLARE @sql_grant_schema_redcap nvarchar(max)
+SET @sql_grant_schema_redcap = 'GRANT EXECUTE ON SCHEMA::[redcap] TO ' + @qualified_user_name
+EXECUTE sp_executesql @sql_grant_schema_redcap
+print 'Step #4 executed'; print ''
+
+-----------------------------------------------------------------------
+-- STEP #5: Grant appropriate privileges for the 'Security' schema.
+print 'Step #5 executing....'
+DECLARE @sql_grant_schema_security nvarchar(max)
+SET @sql_grant_schema_security = 'GRANT EXECUTE ON SCHEMA::[security] TO ' + @qualified_user_name
+EXECUTE sp_executesql @sql_grant_schema_security
+print 'Step #5 executed'; print ''
+
+-----------------------------------------------------------------------
+-- OPTIONAL STEP: Delete the user from the database (the first line) and then the server (the second line).
+-- The person's other database user accounts (besides with the auxiliary_security database) will NOT be automatically deleted by these two lines.
+--USE [auxiliary_security]; DROP USER [OUHSC\lsuarez3]
+--USE [master]; DROP LOGIN [OUHSC\lsuarez3]
+
+-----------------------------------------------------------------------
+-- REFERENCES & NOTES
+  --The @qualified_user_name must have both (a) the 'OUHSC' domain qualification, and (b) the square brackets (to escape the backslash).
+  --Using sp_executesql to add users: http://www.sqlservercentral.com/Forums/Topic497615-359-1.aspx
+  --Check if a server login exists: http://stackoverflow.com/questions/37275/sql-query-for-logins
+  --Retrieve database users: http://stackoverflow.com/questions/2445444/how-to-get-a-list-of-users-for-all-instances-databases
+  --Concatenating strings: http://blog.sqlauthority.com/2010/11/25/sql-server-concat-function-in-sql-server-sql-concatenation/
+  --DROP USER from database: http://msdn.microsoft.com/en-us/library/ms189438.aspx
+  --DROP LOGIN from server: http://msdn.microsoft.com/en-us/library/ms188012.aspx
+  --Declaring variables (eg, the username above): http://technet.microsoft.com/en-us/library/aa258839.aspx
+  --A different (& non-dynamic) way to establish a user: http://pic.dhe.ibm.com/infocenter/dmndhelp/v8r5m0/index.jsp?topic=%2Fcom.ibm.wbpm.imuc.sbpm.doc%2Ftopics%2Fdb_create_users_nd_aix.html
+  --If the variable has to cross a 'GO' (which the current version of the script doesn't need): http://stackoverflow.com/questions/937336/is-there-a-way-to-persist-a-variable-across-a-go

    Transfer Credentials

    -

    Manually transfer tokens to the auxiliary server becomes unmanageable as your institution’s collection of API users grows. This script demonstrates how to progamatically transfer all tokens from multiple REDCap instances on your network. The basic steps are:

    +

    Manually transferring tokens to the auxiliary server becomes unmanageable as your institution’s collection of API users grows. This script demonstrates how to progamatically transfer all tokens from multiple REDCap instances. The basic steps are:

      -
    1. Read from the MySQL database underneath each REDCap instance.
      2. +
    2. Read from the MySQL database(s) underneath each REDCap instance on your campus.
    3. Combine & groom the credentials.
      4. -
    4. Upload to SQL Server.
      5. +
    5. Upload to SQL Server (called auxiliary_security here).
    -
    rm(list=ls(all=TRUE)) #Clear the memory for any variables set from any previous runs.
-
-# ---- load-sources ------------------------------------------------------------
-
-# ---- load-packages -----------------------------------------------------------
-library(magrittr)
-requireNamespace("RODBC")
-requireNamespace("dplyr")
-requireNamespace("readr")
-requireNamespace("tibble")
-requireNamespace("checkmate")
-
-# ---- declare-globals ---------------------------------------------------------
-
-# The Activity Directory name that should precede each username.
-#   This should correspond with the result of `SYSTEM_USER`
-#   (https://msdn.microsoft.com/en-us/library/ms179930.aspx)
-ldap_prefix <- "OUHSC\\"
-
-#Create a SQL statement for each REDCap instance.  Only the `instance` value should change.
-sql <- "
-  SELECT username, project_id, api_token
-  FROM redcap_user_rights
-  WHERE api_token IS NOT NULL"
-
-#Update this ad-hoc CSV.  Each row should represent one REDCap instance.
-#   Choose any casual name for the first variable, consistent with the `tweak-data` chunk below.
-#   Enter the exact URL for the second variable.
-ds_url <- readr::read_csv(paste(
-  "instance,redcap_uri",
-  "production,https://redcap-production.ouhsc.edu/redcap/api/",
-  "dev,https://redcap-dev.ouhsc.edu/redcap/api/",
-sep="\n"))
-
-
-# ---- load-data ---------------------------------------------------------------
-
-# Load the credentials from the first instance.
-channel <- RODBC::odbcConnect("redcap-production") # odbcGetInfo(channel)
-ds_prod <- RODBC::sqlQuery(channel, query=sql, stringsAsFactors=F)
-RODBC::odbcClose(channel); rm(channel)
-
-# Load the credentials from the second instance.
-#   Duplicate or remove this block, dependending on the number of instances.
-channel <- RODBC::odbcConnect("redcap-dev") # odbcGetInfo(channel)
-ds_dev  <- RODBC::sqlQuery(channel, query=sql, stringsAsFactors=F)
-RODBC::odbcClose(channel); rm(channel, sql)
-
-# Assert these variables contain valid datasets (instead of a character error message), and
-#   that at least some rows were returned.
-#   Adjust this to smaller values if necessary.  It's really just to catch blatant retrieval problems.
-checkmate::assert_data_frame(ds_bbmc, min.rows=5)
-checkmate::assert_data_frame(ds_dev , min.rows=5)
-
-
-# ---- tweak-data --------------------------------------------------------------
-
-#Label each instance, so they're distinguishable later.  Add/remove lines, depending on the number of campus instances
-ds_prod$instance <- "production"
-ds_dev$instance  <- "dev"
-
-#Combine the token collection from each instance.  Then prefix the username and include the URL of each instance.
-ds <- ds_prod %>%
-  dplyr::union(ds_dev) %>%                                     # Add/remove unions, based on the number of REDCap instances.
-  dplyr::select_(
-    "username"             = "`username`"
-    , "project_id"         = "`project_id`"
-    , "instance"           = "`instance`"
-    , "token"              = "`api_token`"
-  ) %>%
-  dplyr::arrange(instance, project_id, username) %>%
-  dplyr::mutate(
-    username               = paste0(ldap_prefix, username),    # Qualify for the Active Directory.
-    id                     = seq_len(n())                      # For the sake of a clustered primary key.
-  ) %>%
-  dplyr::left_join( ds_url, by="instance")                     # Include the instance URL.
-
-rm(ds_prod, ds_dev, ds_url)
-
-
-# ---- verify-values -----------------------------------------------------------
-# devtools::install_github("OuhscBbmc/OuhscMunge"); OuhscMunge::verify_value_headstart(ds)
-# Assert that the dataset is well-behaved.
-checkmate::assert_integer(  ds$id         , any.missing=F , lower=1, upper=2^31-1 , unique=T)
-checkmate::assert_character(ds$username   , any.missing=F , pattern="^.{1,255}$"            )
-checkmate::assert_integer(  ds$project_id , any.missing=F , lower=1, upper=2^31-1           )
-checkmate::assert_character(ds$token      , any.missing=F , pattern="^.{32}$"     , unique=T)
-checkmate::assert_character(ds$instance   , any.missing=F , pattern="^.{1,255}$"            )
-checkmate::assert_character(ds$redcap_uri , any.missing=F , pattern="^.{1,255}$"            )
-
-testit::assert(
-  "The `username` x `project_id` x `instance` must be unique.",
-  sum(duplicated(paste0(ds$username, "-", ds$project_id, "-", ds$instance))) == 0L
-)
-
-testit::assert("There should be at least 10 tokens written." , 10L <= nrow(ds))
-
-
-# ---- specify-columns-to-upload -----------------------------------------------
-
-# Dictate the exact columns and order that will be uploaded.
-columns_to_write <- c("id", "username", "project_id", "instance", "token", "redcap_uri")
-ds_slim <- ds[, columns_to_write]
-
-rm(columns_to_write)
-
-
-# ---- upload-to-db-credential ------------------------------------------------------------
-
-#Upload to SQL Server through ODBC.
-(start_time <- Sys.time())
-
-db_table         <- "redcap_private.tbl_credential"
-channel          <- RODBC::odbcConnect("auxiliary_security") #getSqlTypeInfo("Microsoft SQL Server") #;odbcGetInfo(channel)
-
-column_info      <- RODBC::sqlColumns(channel, db_table)
-var_types        <- as.character(column_info$TYPE_NAME)
-names(var_types) <- as.character(column_info$COLUMN_NAME)  #var_types
-
-RODBC::sqlClear(channel, db_table)
-RODBC::sqlSave(channel, ds_slim, db_table, append=TRUE, rownames=FALSE, fast=TRUE, varTypes=var_types)
-RODBC::odbcClose(channel); rm(channel)
-
-(elapsed_duration <-  Sys.time() - start_time) #0.6026149 secs 2016-08-29.
-rm(db_table, column_info, var_types, start_time, elapsed_duration)
    +
    rm(list=ls(all=TRUE)) #Clear the memory for any variables set from any previous runs.
+
+# ---- load-sources ------------------------------------------------------------
+
+# ---- load-packages -----------------------------------------------------------
+if( !require(OuhscMunge) )
+  stop('The `OuhscMunge` package needs to be installed with `devtools::install_github("OuhscBbmc/OuhscMunge")`.')
+
+testit::assert(
+  "The `OuhscMunge` package should meet a minimal version.",
+  compareVersion( as.character(packageVersion("OuhscMunge")), "0.1.9.9009") >= 0L
+)
+
+library(magrittr)
+requireNamespace("DBI")
+requireNamespace("odbc")
+requireNamespace("dplyr")
+requireNamespace("readr")
+requireNamespace("tibble")
+requireNamespace("testit")
+requireNamespace("checkmate")
+requireNamespace("OuhscMunge")  # devtools::install_github("OuhscBbmc/OuhscMunge")
+
+
+# ---- declare-globals ---------------------------------------------------------
+# This file assume your campus has two REDCap instances.
+# Modify each (a) database name, (b) REDCap URL, and (c) DSN name.
+
+name_production <- "production"
+name_dev        <- "dev"
+
+uri_production  <- "https://redcap-production.ouhsc.edu/redcap/api/",
+uri_dev         <- "https://redcap-dev.ouhsc.edu/redcap/api/"
+
+dsn_production  <- "redcap-production"
+dsn_dev         <- "redcap-dev"
+dsn_source      <- "auxiliary_security" # The DSN of the token server.
+
+# The Activity Directory name that should precede each username.
+#   This should correspond with the result of SQL Server's `SYSTEM_USER` function
+#   (https://msdn.microsoft.com/en-us/library/ms179930.aspx)
+ldap_prefix <- "OUHSC\\"
+
+####
+# Nothing below this line should need to change, assuming:
+# 1. the vignette was followed exactly (https://ouhscbbmc.github.io/REDCapR/articles/SecurityDatabase.html),
+# 2. your campus has exactly two REDCap instances.
+
+# SQL sent to the MySQL database underneath each REDCap instance.
+sql <- "
+  SELECT username, project_id, api_token
+  FROM redcap_user_rights
+  WHERE api_token IS NOT NULL
+"
+
+# Update this ad-hoc CSV.  Each row should represent one REDCap instance.
+ds_url <- tibble::tribble(
+  ~instance         , ~redcap_uri,
+  name_production   , uri_production,
+  name_dev          , uri_dev
+)
+
+# Remove variables that aren't used below.
+rm(uri_production, uri_dev)
+
+
+# ---- load-data ---------------------------------------------------------------
+
+# Load the credentials from the first/production REDCap instance.
+cnn_production  <- DBI::dbConnect(odbc::odbc(), dsn=dsn_production)
+ds_production   <- DBI::dbGetQuery(cnn_production, sql)
+DBI::dbDisconnect(cnn_production); rm(cnn_production, dsn_production)
+
+# Load the credentials from the second/dev REDCap instance.
+cnn_dev         <- DBI::dbConnect(odbc::odbc(), dsn=dsn_dev)
+ds_dev          <- DBI::dbGetQuery(cnn_dev, sql)
+DBI::dbDisconnect(cnn_dev); rm(cnn_dev, dsn_dev)
+
+rm(sql)
+
+# Assert these are valid datasets and contain at least 5 rows.
+#   Adjust '5' to smaller value if necessary.  It's just to catch blatant retrieval problems.
+checkmate::assert_data_frame(ds_production, min.rows=5)
+checkmate::assert_data_frame(ds_dev       , min.rows=5)
+
+
+# ---- tweak-data --------------------------------------------------------------
+
+# Label each instance, so they're distinguishable later.
+ds_production$instance <- name_production
+ds_dev$instance        <- name_dev
+
+# Stack the token collection from each instance.  Then prefix the username and include the URL of each instance.
+ds <- ds_production %>%
+  dplyr::union(ds_dev) %>%                                # Remove union if the dev instance isn't included.
+  tibble::as_tibble() %>%
+  dplyr::select(
+    username             = username,
+    project_id           = project_id,
+    instance             = instance,
+    token                = api_token
+  ) %>%
+  dplyr::mutate(
+    username             = paste0(ldap_prefix, username), # Qualify for the Active Directory.
+  ) %>%
+  dplyr::left_join( ds_url, by="instance") %>%            # Include the instance URL.
+  dplyr::arrange(instance, project_id, username) %>%
+  tibble::rowid_to_column("id")                           # For the sake of a clustered primary key.
+
+rm(ds_production, ds_dev, ds_url)
+rm(name_production, name_dev)
+rm(ldap_prefix)
+
+
+# ---- verify-values -----------------------------------------------------------
+
+# Assert that the dataset is well-behaved.
+# OuhscMunge::verify_value_headstart(ds)
+checkmate::assert_integer(  ds$id         , any.missing=F, lower=1, upper=.Machine$integer.max, unique=T)
+checkmate::assert_character(ds$username   , any.missing=F, pattern="^.{1,255}$"                         )
+checkmate::assert_integer(  ds$project_id , any.missing=F, lower=1, upper=.Machine$integer.max          )
+checkmate::assert_character(ds$token      , any.missing=F, pattern="^[A-Z0-9]{32}$"           , unique=T)
+checkmate::assert_character(ds$instance   , any.missing=F, pattern="^.{1,255}$"                         )
+checkmate::assert_character(ds$redcap_uri , any.missing=F, pattern="^.{1,255}$"                         )
+
+testit::assert(
+  "The `username` x `project_id` x `instance` must be unique.",
+  sum(duplicated(paste0(ds$username, "-", ds$project_id, "-", ds$instance))) == 0L
+)
+
+testit::assert("At least 10 tokens should be ready to write." , 10L <= nrow(ds))
+
+
+# ---- specify-columns-to-upload -----------------------------------------------
+
+# Dictate the exact columns and order that will be uploaded.
+columns_to_write <- c("id", "username", "project_id", "instance", "token", "redcap_uri")
+ds_slim          <- ds[, columns_to_write]
+rm(columns_to_write)
+
+
+# ---- upload-to-db ------------------------------------------------------------------
+
+OuhscMunge::upload_sqls_odbc(
+  d               = ds_slim,
+  schema_name     = "redcap_private",
+  table_name      = "tbl_credential",
+  dsn_name        = dsn_source,
+  create_table    = FALSE,
+  clear_table     = TRUE,
+  transaction     = TRUE,
+  verbose         = TRUE
+)
+# Uploading 252 tokens takes 0.004 minutes.

    Document Info

    -

    This document is primarily based on REDCap version 6.11.5, and was last updated 2016-08-30. A development version of the document is available on GitHub: https://ouhscbbmc.github.io/REDCapR/articles/SecurityDatabase.html

    +

    This document is primarily based on REDCap version 8.4.0, and was last updated 2018-08-10. A development version of the document is available on GitHub: https://ouhscbbmc.github.io/REDCapR/articles/SecurityDatabase.html

    diff --git a/inst/doc/TroubleshootingApiCalls.html b/inst/doc/TroubleshootingApiCalls.html index 2c4ffc7f..5773d97b 100644 --- a/inst/doc/TroubleshootingApiCalls.html +++ b/inst/doc/TroubleshootingApiCalls.html @@ -11,7 +11,7 @@ - + Troubleshooting REDCap API Calls @@ -19,46 +19,256 @@ - + @@ -68,7 +278,7 @@

    Troubleshooting REDCap API Calls

    -

    2018-07-11

    +

    2018-08-10

    @@ -115,8 +325,7 @@

    Language Agnostic API

  • Can an administrator query the API successfully with Postman with the admin token? As an administrator, create an account for yourself, and verify that your token works on your server and project.

  • Can an administrator query the API successfully with Postman with the user’s token? Use Postman as before, but replace your token with the user’s token. Once the whole problem is solved, reissue new API tokens to both you and the user.

  • Can an user query the API successfully with Postman with the their own token? The values they enter should be exactly the same as those entered in the previous step. A failure here (assuming the previous step was successful) suggests a network or firewall issue. If the server is behind your institution’s firewall, verify the user is connecting successfully through the VPN.

    • -

  • Can a user query the API with cURL? cURL is a command line tool that’s underneath a lot of libraries. If it’s installed correctly on your location machine, it can be executed from the terminal or command line.

    -
    curl -X POST -H "Cache-Control: no-cache" -F "token=9A81268476645C4E5F03428B8AC3AA7B" -F "content=record" -F "format=csv" "https://bbmc.ouhsc.edu/redcap/api/"
    • +

  • Can a user query the API with cURL? cURL is a command line tool that’s underneath a lot of libraries. If it’s installed correctly on your location machine, it can be executed from the terminal or command line. shell curl -X POST -H "Cache-Control: no-cache" -F "token=9A81268476645C4E5F03428B8AC3AA7B" -F "content=record" -F "format=csv" "https://bbmc.ouhsc.edu/redcap/api/"

    • @@ -124,78 +333,78 @@

    Exporting from REDCap to R

    There are several ways to call REDCap’s API from R. The packages redcapAPI and REDCapR both rely on the httr package, which calls the curl package.

    1. Is httr installed on the user’s local machine? If so, running library(httr) should not produce any error messages if you’re starting with a fresh session of R:

      -
      > library(httr)
      2. +

    2. Does the user have the most recent version of httr? There are several ways to do this, but the easiest is probably to run update.packages(ask=FALSE, repos="http://cran.rstudio.com"). The optional argument ask prevents the user from needing to respond ‘Y’ to each outdated package.

    3. Can the user query a test project using httr? Both the redcapAPI and REDCapR packages employ something similar to the following function in httr. If you’re curious, here is the relevant source code for redcapAPI and REDCapR.

      If this check fails, consider attempting again with the uri and token used above in the Postman example.

      This check avoids checking the SSL certificate in order to simplify the troubleshooting. SSL verification is supported by default in the PyCap, redcapAPI, and REDCapR packages.

      -
      redcap_uri <- "https://bbmc.ouhsc.edu/redcap/api/"
-token      <- "9A81268476645C4E5F03428B8AC3AA7B"
-
-raw_text <- RCurl::postForm(
-  uri                         = redcap_uri
-  , token                     = token
-  , content                   = 'record'
-  , format                    = 'csv'
-  , type                      = 'flat'
-  , rawOrLabel                = 'raw'
-  , exportDataAccessGroups    = 'true'
-  , .opts                     = RCurl::curlOptions(ssl.verifypeer=FALSE)
-)
      +

      Alternatively, you can try using the httr package, which uses RCurl underneath. REDCapR and a recent fork of redcap actually uses httr directly, instead of RCurl. As of 2014-07-06, this works with the Windows 8 version for libcurl (which is underneath `RCurl), but not with some Linux versions; in this case pass the location of the SSL cert file.

      -
      post_body <- list(
-  token                       = token,
-  content                     = 'record',
-  format                      = 'csv',
-  type                        = 'flat',
-  rawOrLabel                  = 'raw',
-  exportDataAccessGroups      = 'true'
-)
-
-raw_text <- httr::POST(
-  url                         = redcap_uri,
-  body                        = post_body,
-  config                      = httr::config(ssl.verifypeer=FALSE),
-  httr::verbose() #Remove this line to suppress the frequent console updates.
-)
      4. +

    4. Can the user query a subset of their project using RCurl? This step is like the previous one, but with two differences. First, it’s using their REDCap project (instead of the test project). Second, it pulls fewer records, and a smaller collection of fields. Subsetting can help troubleshoot by avoiding (and thus identifying) cells with problematic values.

      Notice this call to RCurl::postForm() now passes values to the records and fields parameters. Also notice the value is a single long string, rather a vector of shorter strings (which is more natural to most R users).

      -
      redcap_uri         <- "https://the.urlofyourinsitution.edu/api/"
-token              <- "your-secret-token"
-records_collapsed  <- "1,2,3"                             # Assumes dataset contains ID values of 1-3.
-fields_collapsed   <- "record_id,name_first,name_last"    # Assumes dataset contains these variables.
-
-raw_text <- RCurl::postForm(
-  uri                          = redcap_uri
-  , token                      = token
-  , content                    = 'record'
-  , format                     = 'csv'
-  , type                       = 'flat'
-  , rawOrLabel                 = 'raw'
-  , exportDataAccessGroups     = 'true'
-  , records                    = records_collapsed
-  , fields                     = fields_collapsed
-  , .opts                      = RCurl::curlOptions(ssl.verifypeer=FALSE)
-)
      5. +

    5. Can the user query an entire project using RCurl? There are two advantages of trying a subset of the data. First, small datasets avoid the time-out errors that plague large datasets. Second, it may avoid problematic values being passed through the pipeline. If the current check fails but the previous check succeedes, then experiment with different expanses of records and fields. This should help determine which values are causing the problems, or if there’s simply too much data being pulled in one pass.

      If the desired dataset is too large, consider if you can prune unnecessary records or fields. If not, one solution is to pull smaller, multiple batches using the API, then reassemble them. The redcap_read() function in REDCapR does this automatically, and allows the user to specify a batch_size.

      -
      redcap_uri                  <- "https://the.urlofyourinsitution.edu/api/"
-token                       <- "your-secret-token"
-records_collapsed           <- NULL
-fields_collapsed            <- NULL
-
-raw_text <- RCurl::postForm(
-  uri                        = redcap_uri
-  , token                    = token
-  , content                  = 'record'
-  , format                   = 'csv'
-  , type                     = 'flat'
-  , rawOrLabel               = 'raw'
-  , exportDataAccessGroups   = 'true'
-  , records                  = records_collapsed
-  , fields                   = fields_collapsed
-  , .opts                    = RCurl::curlOptions(ssl.verifypeer=FALSE)
-)
      6. +
    @@ -203,68 +412,21 @@

    Exporting from REDCap to R, using REDCapR

    REDCapR is a package that uses cURL (via httr) to communicate with REDCap, and wraps convenience functions around it to reduce the size and complexity of the user’s code. The package’s basic functions are demonstrated in a vignette and are documented in its reference manual (a downloadable pdf of the functions are also available).

    If you’re not using REDCapR, you can skip this section and proceed to ‘Importing into REDCap from R’ below.

      -

    1. Is REDCapR installed on the user’s machine? Currently the easiest way to install REDCapR is with the devtools. The follow code installs devtools, then installs REDCapR.

      -
      install.packages("devtools", repos="http://cran.rstudio.com")
-devtools::install_github(repo="OuhscBbmc/REDCapR")
      2. -

    2. Does REDCapR load successfully on the user’s machine? If so, running library(REDCapR) should produce the following output if you’re starting with a fresh session of R:

      -
      library(REDCapR)
-## Loading required package: REDCapR
      3. -

    3. Can the user export from an example project? This is the same fake data hosted by the OUHSC BBMC as in the previous section.

      -
      library(REDCapR) #Load the package into the current R session.
-uri   <- "https://bbmc.ouhsc.edu/redcap/api/"
-token <- "9A81268476645C4E5F03428B8AC3AA7B"
-redcap_read(redcap_uri=uri, token=token)$data
      -

      The previous code should produce similar output. Notice there are five rows and the columns will wrap around if your console window is too narrow.

      -
      5 records and 1 columns were read from REDCap in 0.41 seconds.
-Starting to read 5 records  at 2014-06-27 17:19:49
-Reading batch 1 of 1, with ids 1 through 5.
-5 records and 16 columns were read from REDCap in 0.42 seconds.
-
-  record_id name_first name_last                                 address      telephone               email
-1         1     Nutmeg  Nutmouse 14 Rose Cottage St.\nKenning UK, 323232 (432) 456-4848     nutty@mouse.com
-2         2     Tumtum  Nutmouse 14 Rose Cottage Blvd.\nKenning UK 34243 (234) 234-2343    tummy@mouse.comm
-3         3     Marcus      Wood          243 Hill St.\nGuthrie OK 73402 (433) 435-9865        mw@mwood.net
-4         4      Trudy       DAG          342 Elm\nDuncanville TX, 75116 (987) 654-3210 peroxide@blonde.com
-5         5   John Lee    Walker      Hotel Suite\nNew Orleans LA, 70115 (333) 333-4444  left@hippocket.com
-
-         dob age ethnicity race sex height weight   bmi
-1 2003-08-30  10         1    2   0   5.00      1 400.0
-2 2003-03-10  10         1    6   1   6.00      1 277.8
-3 1934-04-09  79         0    4   1 180.00     80  24.7
-4 1952-11-02  61         1    4   0 165.00     54  19.8
-5 1955-04-15  58         1    4   1 193.04    104  27.9
-
-                                                                                                     comments
-1                                                                     Character in a book, with some guessing
-2                                                                          A mouse character from a good book
-3                                                                                          completely made up
-4 This record doesn't have a DAG assigned\n\nSo call up Trudy on the telephone\nSend her a letter in the mail
-5                 Had a hand for trouble and a eye for cash\n\nHe had a gold watch chain and a black mustache
-
-  demographics_complete
-1                     2
-2                     2
-3                     2
-4                     2
-5                     2
      4. -

    4. Can the user export from their own project? The code is similar to the previous check, but the uri and token values will need to be modified.

      -
      library(REDCapR) #Load the package into the current R session, if you haven't already.
-redcap_uri       <- "https://the.urlofyourinsitution.edu/api/"
-token            <- "your-secret-token"
-redcap_read(redcap_uri=uri, token=token)$data
      -

      Alternatively, a redcap_project object can be declared initially, which makes subsequent calls cleaner when the token and url are required only the when the object is declared.

      -
      library(REDCapR) #Load the package into the current R session, if you haven't already.
-uri               <- "https://bbmc.ouhsc.edu/redcap/api/"
-token             <- "9A81268476645C4E5F03428B8AC3AA7B"
-project           <- redcap_project$new(redcap_uri=uri, token=token)
-
-ds_three_columns  <- project$read(fields=c("record_id", "sex", "age"))$data
-
-ids_of_males      <- ds_three_columns$record_id[ds_three_columns$sex==1]
-ids_of_minors     <- ds_three_columns$record_id[ds_three_columns$age < 18]
-
-ds_males          <- project$read(records=ids_of_males, batch_size=2)$data
-ds_minors         <- project$read(records=ids_of_minors)$data
      5. +

    5. Is REDCapR installed on the user’s machine? Currently the easiest way to install REDCapR is with the devtools. The follow code installs devtools, then installs REDCapR. r install.packages("devtools", repos="http://cran.rstudio.com") devtools::install_github(repo="OuhscBbmc/REDCapR")

      6. +

    6. Does REDCapR load successfully on the user’s machine? If so, running library(REDCapR) should produce the following output if you’re starting with a fresh session of R: r library(REDCapR) ## Loading required package: REDCapR

      7. +

    7. Can the user export from an example project? This is the same fake data hosted by the OUHSC BBMC as in the previous section. r library(REDCapR) #Load the package into the current R session. uri <- "https://bbmc.ouhsc.edu/redcap/api/" token <- "9A81268476645C4E5F03428B8AC3AA7B" redcap_read(redcap_uri=uri, token=token)$data

      +

      The previous code should produce similar output. Notice there are five rows and the columns will wrap around if your console window is too narrow. ``` 5 records and 1 columns were read from REDCap in 0.41 seconds. Starting to read 5 records at 2014-06-27 17:19:49 Reading batch 1 of 1, with ids 1 through 5. 5 records and 16 columns were read from REDCap in 0.42 seconds.

      +

      record_id name_first name_last address telephone email 1 1 Nutmeg Nutmouse 14 Rose Cottage St.UK, 323232 (432) 456-4848 nutty@mouse.com 2 2 Tumtum Nutmouse 14 Rose Cottage Blvd.UK 34243 (234) 234-2343 tummy@mouse.comm 3 3 Marcus Wood 243 Hill St.OK 73402 (433) 435-9865 mw@mwood.net 4 4 Trudy DAG 342 ElmTX, 75116 (987) 654-3210 peroxide@blonde.com 5 5 John Lee Walker Hotel SuiteOrleans LA, 70115 (333) 333-4444 left@hippocket.com

      +
            dob age ethnicity race sex height weight   bmi
      +

      1 2003-08-30 10 1 2 0 5.00 1 400.0 2 2003-03-10 10 1 6 1 6.00 1 277.8 3 1934-04-09 79 0 4 1 180.00 80 24.7 4 1952-11-02 61 1 4 0 165.00 54 19.8 5 1955-04-15 58 1 4 1 193.04 104 27.9

      +
                                                                                                        comments
      +

      1 Character in a book, with some guessing 2 A mouse character from a good book 3 completely made up 4 This record doesn’t have a DAG assignedcall up Trudy on the telephoneher a letter in the mail 5 Had a hand for trouble and a eye for cashhad a gold watch chain and a black mustache

      +

      demographics_complete 1 2 2 2 3 2 4 2 5 2 ```

      8. +

    8. Can the user export from their own project? The code is similar to the previous check, but the uri and token values will need to be modified. r library(REDCapR) #Load the package into the current R session, if you haven't already. redcap_uri <- "https://the.urlofyourinsitution.edu/api/" token <- "your-secret-token" redcap_read(redcap_uri=uri, token=token)$data

      +

      Alternatively, a redcap_project object can be declared initially, which makes subsequent calls cleaner when the token and url are required only the when the object is declared. ```r library(REDCapR) #Load the package into the current R session, if you haven’t already. uri <- “https://bbmc.ouhsc.edu/redcap/api/” token <- “9A81268476645C4E5F03428B8AC3AA7B” project <- redcap_project$new(redcap_uri=uri, token=token)

      +

      ds_three_columns <- project\(read(fields=c("record_id", "sex", "age"))\)data

      +

      ids_of_males <- ds_three_columns\(record_id[ds_three_columns\)sex==1] ids_of_minors <- ds_three_columns\(record_id[ds_three_columns\)age < 18]

      +

      ds_males <- project\(read(records=ids_of_males, batch_size=2)\)data ds_minors <- project\(read(records=ids_of_minors)\)data ```

    9. Is the export operation still unsuccessful using REDCapR? If so the “Can the user query a entire REDCap project using RCurl?” check succeeded, but the REDCapR checks did not, consider posting a new GitHub issue to the package developers.

    @@ -273,56 +435,17 @@

    Exporting from REDCap to R, using redcapAPI

    redcapAPI is a package that uses cURL (via httr) to communicate with REDCap, and wraps convenience functions around it to reduce the size and complexity of the user’s code.

    If you’re not using redcapAPI, you can skip this section and proceed to ‘Importing into REDCap from R’ below. More specific discussion about redcapAPI can be found at the package’s wiki.

      -

    1. Is redcapAPI installed on the user’s machine? Currently, the easiest way to install redcapAPI is from CRAN.

      -
      install.packages("redcapAPI")
      -

      Developmental versions may be available on GitHub.

      -
      install.packages("devtools", repos="http://cran.rstudio.com")
-devtools::install_github(repo="nutterb/redcapAPI")
      2. -

    2. Does redcapAPI load successfully on the user’s machine? If so, running library(redcapAPI) should produce the following output if you’re starting with a fresh session of R:

      -
      library(redcapAPI)
-## Loading required package: redcapAPI
      3. -

    3. Can the user export from an example project? This is the same fake data hosted by the OUHSC BBMC as in the previous section.

      -
      library(redcapAPI) #Load the package into the current R session.
-rcon <- redcapConnection(
-  url   = "https://bbmc.ouhsc.edu/redcap/api/",
-  token = "9A81268476645C4E5F03428B8AC3AA7B"
-)
-exportRecords(rcon)
      -

      The previous code should produce similar output. Notice there are five rows and the columns will wrap around if your console window is too narrow.

      -
      record_id name_first name_last                                 address      telephone               email
-1         1     Nutmeg  Nutmouse 14 Rose Cottage St.\nKenning UK, 323232 (432) 456-4848     nutty@mouse.com
-2         2     Tumtum  Nutmouse 14 Rose Cottage Blvd.\nKenning UK 34243 (234) 234-2343    tummy@mouse.comm
-3         3     Marcus      Wood          243 Hill St.\nGuthrie OK 73402 (433) 435-9865        mw@mwood.net
-4         4      Trudy       DAG          342 Elm\nDuncanville TX, 75116 (987) 654-3210 peroxide@blonde.com
-5         5   John Lee    Walker      Hotel Suite\nNew Orleans LA, 70115 (333) 333-4444  left@hippocket.com
-
-       dob age ethnicity race sex height weight   bmi
-1 2003-08-30  10         1    2   0   5.00      1 400.0
-2 2003-03-10  10         1    6   1   6.00      1 277.8
-3 1934-04-09  79         0    4   1 180.00     80  24.7
-4 1952-11-02  61         1    4   0 165.00     54  19.8
-5 1955-04-15  58         1    4   1 193.04    104  27.9
-
-                                                                                                   comments
-1                                                                     Character in a book, with some guessing
-2                                                                          A mouse character from a good book
-3                                                                                          completely made up
-4 This record doesn't have a DAG assigned\n\nSo call up Trudy on the telephone\nSend her a letter in the mail
-5                 Had a hand for trouble and a eye for cash\n\nHe had a gold watch chain and a black mustache
-
-demographics_complete
-1                     2
-2                     2
-3                     2
-4                     2
-5                     2
      4. -

    4. Can the user export from their own project? The code is similar to the previous check, but the uri and token values will need to be modified.

      -
      library(redcapAPI) #Load the package into the current R session, if you haven't already.
-rcon <- redcapConnection(
-  url   = "https://the.urlofyourinsitution.edu/api/", # Adapt this to your server.
-  token = "your-secret-token"                         # Adapt this to your user's token.
-)
-exportRecords(rcon)
      5. +

    5. Is redcapAPI installed on the user’s machine? Currently, the easiest way to install redcapAPI is from CRAN. r install.packages("redcapAPI")

      +

      Developmental versions may be available on GitHub. r install.packages("devtools", repos="http://cran.rstudio.com") devtools::install_github(repo="nutterb/redcapAPI")

      6. +

    6. Does redcapAPI load successfully on the user’s machine? If so, running library(redcapAPI) should produce the following output if you’re starting with a fresh session of R: r library(redcapAPI) ## Loading required package: redcapAPI

      7. +

    7. Can the user export from an example project? This is the same fake data hosted by the OUHSC BBMC as in the previous section. r library(redcapAPI) #Load the package into the current R session. rcon <- redcapConnection( url = "https://bbmc.ouhsc.edu/redcap/api/", token = "9A81268476645C4E5F03428B8AC3AA7B" ) exportRecords(rcon)

      +

      The previous code should produce similar output. Notice there are five rows and the columns will wrap around if your console window is too narrow. ``` record_id name_first name_last address telephone email 1 1 Nutmeg Nutmouse 14 Rose Cottage St.UK, 323232 (432) 456-4848 nutty@mouse.com 2 2 Tumtum Nutmouse 14 Rose Cottage Blvd.UK 34243 (234) 234-2343 tummy@mouse.comm 3 3 Marcus Wood 243 Hill St.OK 73402 (433) 435-9865 mw@mwood.net 4 4 Trudy DAG 342 ElmTX, 75116 (987) 654-3210 peroxide@blonde.com 5 5 John Lee Walker Hotel SuiteOrleans LA, 70115 (333) 333-4444 left@hippocket.com

      +
          dob age ethnicity race sex height weight   bmi
      +

      1 2003-08-30 10 1 2 0 5.00 1 400.0 2 2003-03-10 10 1 6 1 6.00 1 277.8 3 1934-04-09 79 0 4 1 180.00 80 24.7 4 1952-11-02 61 1 4 0 165.00 54 19.8 5 1955-04-15 58 1 4 1 193.04 104 27.9

      +
                                                                                                      comments
      +

      1 Character in a book, with some guessing 2 A mouse character from a good book 3 completely made up 4 This record doesn’t have a DAG assignedcall up Trudy on the telephoneher a letter in the mail 5 Had a hand for trouble and a eye for cashhad a gold watch chain and a black mustache

      +

      demographics_complete 1 2 2 2 3 2 4 2 5 2 ```

      8. +

    8. Can the user export from their own project? The code is similar to the previous check, but the uri and token values will need to be modified. r library(redcapAPI) #Load the package into the current R session, if you haven't already. rcon <- redcapConnection( url = "https://the.urlofyourinsitution.edu/api/", # Adapt this to your server. token = "your-secret-token" # Adapt this to your user's token. ) exportRecords(rcon)

    9. Is the export operation still unsuccessful using redcapAPI? If so the “Can the user query a entire REDCap project using RCurl?” check succeeded, but the redcapAPI checks did not, consider posting a new GitHub issue to the package developers.

    @@ -352,12 +475,11 @@

    Other software for communication and automation with REDCap

    Other good resources

    diff --git a/inst/doc/advanced-redcapr-operations.html b/inst/doc/advanced-redcapr-operations.html index 6f7a9dc0..688f2f2f 100644 --- a/inst/doc/advanced-redcapr-operations.html +++ b/inst/doc/advanced-redcapr-operations.html @@ -11,7 +11,7 @@ - + Advanced REDCapR Operations @@ -19,46 +19,256 @@ - + @@ -68,7 +278,7 @@

    Advanced REDCapR Operations

    -

    2018-07-11

    +

    2018-08-10

    @@ -79,992 +289,209 @@

    Next Steps

    Set project-wide values.

    There is some information that is specific to a REDCap project, as opposed to an individual operation. This includes the (1) uri of the server, and the (2) token for the user’s project. This is hosted on a machine used in REDCapR’s public test suite, so you can run this example from any computer. Unless tests are running.

    -
    library(REDCapR) #Load the package into the current R session.
-uri                   <- "https://bbmc.ouhsc.edu/redcap/api/"
-token_simple          <- "9A81268476645C4E5F03428B8AC3AA7B"
-token_longitudinal    <- "0434F0E9CF53ED0587847AB6E51DE762"
    +

    Converting from tall/long to wide

    Disclaimer: Occasionally we’re asked for a longitudinal dataset to be converted from a “long/tall format” (where typically each row is one observation for a participant) to a “wide format” (where each row is on participant). Usually we advise against it. Besides all the database benefits of a long structure, a wide structure restricts your options with the stat routine. No modern longitudinal analysis procedures (eg, growth curve models or multilevel/hierarchical models) accept wide. You’re pretty much stuck with repeated measures anova, which is very inflexible for real-world medical-ish analyses. It requires a patient to have a measurement at every time point; otherwise the anova excludes the patient entirely.

    However we like going wide to produce visual tables for publications, and here’s one way to do it in R. First retrieve the dataset from REDCap.

    -
    library(magrittr); 
-suppressPackageStartupMessages(requireNamespace("dplyr"))
-suppressPackageStartupMessages(requireNamespace("tidyr"))
-events_to_retain  <- c("dose_1_arm_1", "visit_1_arm_1", "dose_2_arm_1", "visit_2_arm_1")
-
-ds_long <- REDCapR::redcap_read_oneshot(redcap_uri=uri, token=token_longitudinal)$data
    -
    #> 18 records and 125 columns were read from REDCap in 0.7 seconds.  The http status code was 200.
    -
    ds_long %>% 
-  dplyr::select(study_id, redcap_event_name, pmq1, pmq2, pmq3, pmq4)
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -study id - -redcap event name - -pmq1 - -pmq2 - -pmq3 - -pmq4 -
    -100 - -enrollment_arm_1 - -NA - -NA - -NA - -NA -
    -100 - -dose_1_arm_1 - -2 - -2 - -1 - -1 -
    -100 - -visit_1_arm_1 - -1 - -0 - -0 - -0 -
    -100 - -dose_2_arm_1 - -3 - -1 - -0 - -0 -
    -100 - -visit_2_arm_1 - -0 - -1 - -0 - -0 -
    -100 - -final_visit_arm_1 - -NA - -NA - -NA - -NA -
    -220 - -enrollment_arm_1 - -NA - -NA - -NA - -NA -
    -220 - -dose_1_arm_1 - -0 - -1 - -0 - -2 -
    -220 - -visit_1_arm_1 - -0 - -3 - -1 - -0 -
    -220 - -dose_2_arm_1 - -1 - -2 - -0 - -1 -
    -220 - -visit_2_arm_1 - -3 - -4 - -1 - -0 -
    -220 - -final_visit_arm_1 - -NA - -NA - -NA - -NA -
    -304 - -enrollment_arm_2 - -NA - -NA - -NA - -NA -
    -304 - -deadline_to_opt_ou_arm_2 - -NA - -NA - -NA - -NA -
    -304 - -first_dose_arm_2 - -0 - -1 - -0 - -0 -
    -304 - -first_visit_arm_2 - -2 - -0 - -0 - -0 -
    -304 - -final_visit_arm_2 - -NA - -NA - -NA - -NA -
    -304 - -deadline_to_return_arm_2 - -NA - -NA - -NA - -NA -
    + +
    #> 18 records and 125 columns were read from REDCap in 0.5 seconds.  The http status code was 200.
    + +
    #>    study_id        redcap_event_name pmq1 pmq2 pmq3 pmq4
+#> 1       100         enrollment_arm_1   NA   NA   NA   NA
+#> 2       100             dose_1_arm_1    2    2    1    1
+#> 3       100            visit_1_arm_1    1    0    0    0
+#> 4       100             dose_2_arm_1    3    1    0    0
+#> 5       100            visit_2_arm_1    0    1    0    0
+#> 6       100        final_visit_arm_1   NA   NA   NA   NA
+#> 7       220         enrollment_arm_1   NA   NA   NA   NA
+#> 8       220             dose_1_arm_1    0    1    0    2
+#> 9       220            visit_1_arm_1    0    3    1    0
+#> 10      220             dose_2_arm_1    1    2    0    1
+#> 11      220            visit_2_arm_1    3    4    1    0
+#> 12      220        final_visit_arm_1   NA   NA   NA   NA
+#> 13      304         enrollment_arm_2   NA   NA   NA   NA
+#> 14      304 deadline_to_opt_ou_arm_2   NA   NA   NA   NA
+#> 15      304         first_dose_arm_2    0    1    0    0
+#> 16      304        first_visit_arm_2    2    0    0    0
+#> 17      304        final_visit_arm_2   NA   NA   NA   NA
+#> 18      304 deadline_to_return_arm_2   NA   NA   NA   NA

    When widening only one variable (eg, pmq1), the code’s pretty simple:

    -
    ds_wide <- ds_long %>% 
-  dplyr::select(study_id, redcap_event_name, pmq1) %>% 
-  dplyr::filter(redcap_event_name %in% events_to_retain) %>% 
-  tidyr::spread(key=redcap_event_name, value=pmq1)
-ds_wide
    - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -study id - -dose 1 arm 1 - -dose 2 arm 1 - -visit 1 arm 1 - -visit 2 arm 1 -
    -100 - -2 - -3 - -1 - -0 -
    -220 - -0 - -1 - -0 - -3 -
    + +
    #>   study_id dose_1_arm_1 dose_2_arm_1 visit_1_arm_1 visit_2_arm_1
+#> 1      100            2            3             1             0
+#> 2      220            0            1             0             3

    When widening more than one variable (eg, pmq1 - pmq4), it’s usually easiest to go even longer/taller (eg, ds_eav) before reversing direction and going wide:

    -
    pattern <- "^(\\w+?)_arm_(\\d)$"
-
-ds_eav <- ds_long %>% 
-  dplyr::select(study_id, redcap_event_name, pmq1, pmq2, pmq3, pmq4) %>% 
-  dplyr::mutate(
-    event      = sub(pattern, "\\1", redcap_event_name),
-    arm        = as.integer(sub(pattern, "\\2", redcap_event_name))
-  ) %>% 
-  dplyr::select(study_id, event, arm, pmq1, pmq2, pmq3, pmq4) %>% 
-  tidyr::gather(key=key, value=value, pmq1, pmq2, pmq3, pmq4) %>% 
-  dplyr::filter(!(event %in% c(
-    "enrollment", "final_visit", "deadline_to_return", "deadline_to_opt_ou")
-  )) %>% 
-  dplyr::mutate( # Simulate correcting for mismatched names across arms:
-    event = dplyr::recode(event, "first_dose"="dose_1", "first_visit"="visit_1"),
-    key = paste0(event, "_", key)
-  ) %>% 
-  dplyr::select(-event)
-
-# Show the first 10 rows of the EAV table.
-ds_eav %>% 
-  head(10)
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -study id - -arm - -key - -value -
    -100 - -1 - -dose_1_pmq1 - -2 -
    -100 - -1 - -visit_1_pmq1 - -1 -
    -100 - -1 - -dose_2_pmq1 - -3 -
    -100 - -1 - -visit_2_pmq1 - -0 -
    -220 - -1 - -dose_1_pmq1 - -0 -
    -220 - -1 - -visit_1_pmq1 - -0 -
    -220 - -1 - -dose_2_pmq1 - -1 -
    -220 - -1 - -visit_2_pmq1 - -3 -
    -304 - -2 - -dose_1_pmq1 - -0 -
    -304 - -2 - -visit_1_pmq1 - -2 -
    -
    # Spread the EAV to wide.
-ds_wide <- ds_eav %>% 
-  tidyr::spread(key=key, value=value)
-ds_wide
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -study id - -arm - -dose 1 pmq1 - -dose 1 pmq2 - -dose 1 pmq3 - -dose 1 pmq4 - -dose 2 pmq1 - -dose 2 pmq2 - -dose 2 pmq3 - -dose 2 pmq4 - -visit 1 pmq1 - -visit 1 pmq2 - -visit 1 pmq3 - -visit 1 pmq4 - -visit 2 pmq1 - -visit 2 pmq2 - -visit 2 pmq3 - -visit 2 pmq4 -
    -100 - -1 - -2 - -2 - -1 - -1 - -3 - -1 - -0 - -0 - -1 - -0 - -0 - -0 - -0 - -1 - -0 - -0 -
    -220 - -1 - -0 - -1 - -0 - -2 - -1 - -2 - -0 - -1 - -0 - -3 - -1 - -0 - -3 - -4 - -1 - -0 -
    -304 - -2 - -0 - -1 - -0 - -0 - -NA - -NA - -NA - -NA - -2 - -0 - -0 - -0 - -NA - -NA - -NA - -NA -
    + +
    #>    study_id arm          key value
+#> 1       100   1  dose_1_pmq1     2
+#> 2       100   1 visit_1_pmq1     1
+#> 3       100   1  dose_2_pmq1     3
+#> 4       100   1 visit_2_pmq1     0
+#> 5       220   1  dose_1_pmq1     0
+#> 6       220   1 visit_1_pmq1     0
+#> 7       220   1  dose_2_pmq1     1
+#> 8       220   1 visit_2_pmq1     3
+#> 9       304   2  dose_1_pmq1     0
+#> 10      304   2 visit_1_pmq1     2
    + +
    #>   study_id arm dose_1_pmq1 dose_1_pmq2 dose_1_pmq3 dose_1_pmq4 dose_2_pmq1
+#> 1      100   1           2           2           1           1           3
+#> 2      220   1           0           1           0           2           1
+#> 3      304   2           0           1           0           0          NA
+#>   dose_2_pmq2 dose_2_pmq3 dose_2_pmq4 visit_1_pmq1 visit_1_pmq2
+#> 1           1           0           0            1            0
+#> 2           2           0           1            0            3
+#> 3          NA          NA          NA            2            0
+#>   visit_1_pmq3 visit_1_pmq4 visit_2_pmq1 visit_2_pmq2 visit_2_pmq3
+#> 1            0            0            0            1            0
+#> 2            1            0            3            4            1
+#> 3            0            0           NA           NA           NA
+#>   visit_2_pmq4
+#> 1            0
+#> 2            0
+#> 3           NA

    SSL Options

    The official cURL site discusses the process of using SSL to verify the server being connected to.

    Use the SSL cert file that come with the openssl package.

    -
    cert_location <- system.file("cacert.pem", package="openssl")
-if( file.exists(cert_location) ) {
-  config_options         <- list(cainfo=cert_location)
-  ds_different_cert_file <- redcap_read_oneshot(
-    redcap_uri     = uri,
-    token          = token_simple,
-    config_options = config_options
-  )$data
-}
    -
    #> 5 records and 24 columns were read from REDCap in 0.4 seconds.  The http status code was 200.
    + +
    #> 5 records and 24 columns were read from REDCap in 0.2 seconds.  The http status code was 200.

    Force the connection to use SSL=3 (which is not preferred, and possibly insecure).

    -
    config_options <- list(sslversion=3)
-ds_ssl_3 <- redcap_read_oneshot(
-  redcap_uri     = uri,
-  token          = token_simple,
-  config_options = config_options
-)$data
    -
    #> 5 records and 24 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    -
    config_options <- list(ssl.verifypeer=FALSE)
-ds_no_ssl <- redcap_read_oneshot(
-   redcap_uri     = uri,
-   token          = token_simple,
-   config_options = config_options
-)$data
    -
    #> 5 records and 24 columns were read from REDCap in 0.3 seconds.  The http status code was 200.
    + +
    #> 5 records and 24 columns were read from REDCap in 0.2 seconds.  The http status code was 200.
    + +
    #> 5 records and 24 columns were read from REDCap in 0.2 seconds.  The http status code was 200.

    Session Information

    For the sake of documentation and reproducibility, the current report was rendered in the following environment. Click the line below to expand.

    -

    Environment

    -
    #> ─ Session info ──────────────────────────────────────────────────────────
-#>  setting  value                       
-#>  version  R version 3.4.4 (2018-03-15)
-#>  os       Ubuntu 18.04 LTS            
-#>  system   x86_64, linux-gnu           
-#>  ui       RStudio                     
-#>  language (EN)                        
-#>  collate  en_US.UTF-8                 
-#>  tz       America/Chicago             
-#>  date     2018-07-11                  
+
    
+
    Environment 
    
+
    #> - Session info ----------------------------------------------------------
+#>  setting  value                                      
+#>  version  R version 3.5.1 Patched (2018-08-06 r75070)
+#>  os       Windows >= 8 x64                           
+#>  system   x86_64, mingw32                            
+#>  ui       RStudio                                    
+#>  language (EN)                                       
+#>  collate  English_United States.1252                 
+#>  tz       America/Chicago                            
+#>  date     2018-08-10                                 
 #> 
-#> ─ Packages ──────────────────────────────────────────────────────────────
+#> - Packages --------------------------------------------------------------
 #>  package     * version     date       source                          
-#>  assertthat    0.2.0       2017-04-11 cran (@0.2.0)                   
-#>  backports     1.1.2       2017-12-13 cran (@1.1.2)                   
-#>  bindr         0.1.1       2018-03-13 CRAN (R 3.4.3)                  
-#>  bindrcpp    * 0.2.2       2018-03-29 CRAN (R 3.4.3)                  
-#>  checkmate     1.8.6       2018-04-10 Github (mllg/checkmate@489319a) 
-#>  clisymbols    1.2.0       2017-05-21 CRAN (R 3.4.3)                  
-#>  codetools     0.2-15      2016-10-05 CRAN (R 3.4.3)                  
-#>  colorspace    1.3-2       2016-12-14 CRAN (R 3.4.3)                  
-#>  commonmark    1.5         2018-04-28 CRAN (R 3.4.4)                  
-#>  crayon        1.3.4       2017-09-16 CRAN (R 3.4.3)                  
-#>  curl          3.2         2018-03-28 CRAN (R 3.4.4)                  
-#>  desc          1.2.0       2018-05-01 CRAN (R 3.4.4)                  
-#>  devtools      1.13.6      2018-06-27 CRAN (R 3.4.4)                  
-#>  digest        0.6.15      2018-01-28 CRAN (R 3.4.3)                  
-#>  dplyr         0.7.6       2018-06-29 CRAN (R 3.4.4)                  
-#>  evaluate      0.10.1      2017-06-24 CRAN (R 3.4.3)                  
-#>  glue          1.2.0       2017-10-29 cran (@1.2.0)                   
-#>  highr         0.7         2018-06-09 CRAN (R 3.4.4)                  
-#>  hms           0.4.2.9000  2018-05-26 Github (tidyverse/hms@14e74ab)  
-#>  htmltools     0.3.6       2017-04-28 CRAN (R 3.4.3)                  
-#>  httr          1.3.1       2017-08-20 CRAN (R 3.4.3)                  
-#>  kableExtra    0.9.0       2018-05-21 CRAN (R 3.4.4)                  
-#>  knitr       * 1.20        2018-02-20 CRAN (R 3.4.3)                  
-#>  magrittr    * 1.5         2014-11-22 cran (@1.5)                     
-#>  memoise       1.1.0       2017-04-21 CRAN (R 3.4.3)                  
-#>  munsell       0.5.0       2018-06-12 CRAN (R 3.4.4)                  
-#>  pillar        1.2.3       2018-05-25 CRAN (R 3.4.4)                  
-#>  pkgconfig     2.0.1       2017-03-21 cran (@2.0.1)                   
-#>  plyr          1.8.4       2016-06-08 CRAN (R 3.4.3)                  
-#>  purrr         0.2.5       2018-05-29 CRAN (R 3.4.4)                  
-#>  R6            2.2.2       2017-06-17 CRAN (R 3.4.3)                  
-#>  Rcpp          0.12.17     2018-05-18 CRAN (R 3.4.4)                  
-#>  readr         1.2.0       2018-05-26 Github (tidyverse/readr@d6d622b)
+#>  assertthat    0.2.0       2017-04-11 CRAN (R 3.5.0)                  
+#>  backports     1.1.2       2017-12-13 CRAN (R 3.5.0)                  
+#>  bindr         0.1.1       2018-03-13 CRAN (R 3.5.0)                  
+#>  bindrcpp    * 0.2.2       2018-03-29 CRAN (R 3.5.0)                  
+#>  checkmate     1.8.9-9000  2018-08-09 Github (mllg/checkmate@29a1fb9) 
+#>  clisymbols    1.2.0       2017-05-21 CRAN (R 3.5.0)                  
+#>  codetools     0.2-15      2016-10-05 CRAN (R 3.5.1)                  
+#>  colorspace    1.3-2       2016-12-14 CRAN (R 3.5.0)                  
+#>  commonmark    1.5         2018-04-28 CRAN (R 3.5.0)                  
+#>  crayon        1.3.4       2017-09-16 CRAN (R 3.5.0)                  
+#>  curl          3.2         2018-03-28 CRAN (R 3.5.0)                  
+#>  desc          1.2.0       2018-05-01 CRAN (R 3.5.0)                  
+#>  devtools      1.13.6      2018-06-27 CRAN (R 3.5.0)                  
+#>  digest        0.6.15      2018-01-28 CRAN (R 3.5.0)                  
+#>  dplyr         0.7.6       2018-06-29 CRAN (R 3.5.1)                  
+#>  evaluate      0.11        2018-07-17 CRAN (R 3.5.1)                  
+#>  git2r         0.23.0      2018-07-17 CRAN (R 3.5.1)                  
+#>  glue          1.3.0       2018-07-17 CRAN (R 3.5.1)                  
+#>  hms           0.4.2.9001  2018-08-09 Github (tidyverse/hms@979286f)  
+#>  htmltools     0.3.6       2017-04-28 CRAN (R 3.5.0)                  
+#>  httr          1.3.1       2017-08-20 CRAN (R 3.5.0)                  
+#>  kableExtra    0.9.0       2018-05-21 CRAN (R 3.5.0)                  
+#>  knitr       * 1.20        2018-02-20 CRAN (R 3.5.0)                  
+#>  magrittr    * 1.5         2014-11-22 CRAN (R 3.5.0)                  
+#>  memoise       1.1.0       2017-04-21 CRAN (R 3.5.0)                  
+#>  munsell       0.5.0       2018-06-12 CRAN (R 3.5.0)                  
+#>  packrat       0.4.9-3     2018-06-01 CRAN (R 3.5.0)                  
+#>  pillar        1.3.0       2018-07-14 CRAN (R 3.5.1)                  
+#>  pkgconfig     2.0.1       2017-03-21 CRAN (R 3.5.0)                  
+#>  purrr         0.2.5       2018-05-29 CRAN (R 3.5.0)                  
+#>  R6            2.2.2       2017-06-17 CRAN (R 3.5.0)                  
+#>  Rcpp          0.12.18     2018-07-23 CRAN (R 3.5.1)                  
+#>  readr         1.2.0       2018-08-09 Github (tidyverse/readr@4b2e93a)
 #>  REDCapR     * 0.9.10.9001 <NA>       local                           
-#>  rlang         0.2.1       2018-05-30 CRAN (R 3.4.4)                  
-#>  rmarkdown     1.10        2018-06-11 CRAN (R 3.4.4)                  
-#>  roxygen2      6.0.1       2017-02-06 CRAN (R 3.4.4)                  
-#>  rprojroot     1.3-2       2018-01-03 CRAN (R 3.4.3)                  
-#>  rstudioapi    0.7         2017-09-07 CRAN (R 3.4.3)                  
-#>  rvest         0.3.2       2016-06-17 CRAN (R 3.4.3)                  
-#>  scales        0.5.0.9000  2018-03-29 Github (hadley/scales@d767915)  
-#>  sessioninfo   1.0.0       2017-06-21 CRAN (R 3.4.3)                  
-#>  stringi       1.2.3       2018-06-12 CRAN (R 3.4.4)                  
-#>  stringr       1.3.1       2018-05-10 CRAN (R 3.4.4)                  
-#>  testthat      2.0.0       2017-12-13 CRAN (R 3.4.3)                  
-#>  tibble        1.4.2       2018-01-22 CRAN (R 3.4.3)                  
-#>  tidyr         0.8.1       2018-05-18 CRAN (R 3.4.4)                  
-#>  tidyselect    0.2.4       2018-02-26 CRAN (R 3.4.3)                  
-#>  viridisLite   0.3.0       2018-02-01 CRAN (R 3.4.3)                  
-#>  withr         2.1.2       2018-03-29 Github (jimhester/withr@79d7b0d)
-#>  xml2          1.2.0       2018-01-24 CRAN (R 3.4.3)                  
-#>  yaml          2.1.19      2018-05-01 CRAN (R 3.4.4)
    
-
    
-
    Report rendered by wibeasley at 2018-07-11, 11:04 -0500 in 5 seconds.
    
+#>  rlang         0.2.1       2018-05-30 CRAN (R 3.5.0)                  
+#>  rmarkdown     1.10        2018-06-11 CRAN (R 3.5.0)                  
+#>  roxygen2      6.1.0       2018-07-27 CRAN (R 3.5.1)                  
+#>  rprojroot     1.3-2       2018-01-03 CRAN (R 3.5.0)                  
+#>  rstudioapi    0.7         2017-09-07 CRAN (R 3.5.0)                  
+#>  rvest         0.3.2       2016-06-17 CRAN (R 3.5.0)                  
+#>  scales        1.0.0       2018-08-09 CRAN (R 3.5.1)                  
+#>  sessioninfo   1.0.0       2017-06-21 CRAN (R 3.5.0)                  
+#>  stringi       1.2.4       2018-07-20 CRAN (R 3.5.1)                  
+#>  stringr       1.3.1       2018-05-10 CRAN (R 3.5.0)                  
+#>  testthat      2.0.0       2017-12-13 CRAN (R 3.5.0)                  
+#>  tibble        1.4.2       2018-01-22 CRAN (R 3.5.0)                  
+#>  tidyr         0.8.1       2018-05-18 CRAN (R 3.5.0)                  
+#>  tidyselect    0.2.4       2018-02-26 CRAN (R 3.5.0)                  
+#>  viridisLite   0.3.0       2018-02-01 CRAN (R 3.5.0)                  
+#>  withr         2.1.2       2018-03-15 CRAN (R 3.5.0)                  
+#>  xml2          1.2.0       2018-01-24 CRAN (R 3.5.0)                  
+#>  yaml          2.2.0       2018-07-25 CRAN (R 3.5.1)
    +
    +

    Report rendered by Will at 2018-08-10, 20:36 -0500 in 2 seconds.

    diff --git a/man/metadata_utilities.Rd b/man/metadata_utilities.Rd index 1fc83f17..b95e6368 100644 --- a/man/metadata_utilities.Rd +++ b/man/metadata_utilities.Rd @@ -4,7 +4,6 @@ \alias{metadata_utilities} \alias{regex_named_captures} \alias{checkbox_choices} -\alias{checkbox_choices} \title{Manipulate and interpret the metadata of a REDCap project.} \usage{ regex_named_captures(pattern, text, perl = TRUE) diff --git a/man/redcap_metadata_read.Rd b/man/redcap_metadata_read.Rd index 0fa6b440..db83661a 100644 --- a/man/redcap_metadata_read.Rd +++ b/man/redcap_metadata_read.Rd @@ -4,9 +4,9 @@ \alias{redcap_metadata_read} \title{Export the metadata of a REDCap project.} \usage{ -redcap_metadata_read(redcap_uri, token, forms = NULL, forms_collapsed = "", - fields = NULL, fields_collapsed = "", verbose = TRUE, - config_options = NULL) +redcap_metadata_read(redcap_uri, token, forms = NULL, + forms_collapsed = "", fields = NULL, fields_collapsed = "", + verbose = TRUE, config_options = NULL) } \arguments{ \item{redcap_uri}{The URI (uniform resource identifier) of the REDCap project. Required.} diff --git a/man/redcap_variables.Rd b/man/redcap_variables.Rd index 5abae0a4..25143b83 100644 --- a/man/redcap_variables.Rd +++ b/man/redcap_variables.Rd @@ -4,7 +4,8 @@ \alias{redcap_variables} \title{Enumerate the exported variables.} \usage{ -redcap_variables(redcap_uri, token, verbose = TRUE, config_options = NULL) +redcap_variables(redcap_uri, token, verbose = TRUE, + config_options = NULL) } \arguments{ \item{redcap_uri}{The URI (uniform resource identifier) of the REDCap project. Required.} diff --git a/man/redcap_version.Rd b/man/redcap_version.Rd index e1509e06..273ea98f 100644 --- a/man/redcap_version.Rd +++ b/man/redcap_version.Rd @@ -4,7 +4,8 @@ \alias{redcap_version} \title{Determine version of REDCap instance} \usage{ -redcap_version(redcap_uri, token, verbose = TRUE, config_options = NULL) +redcap_version(redcap_uri, token, verbose = TRUE, + config_options = NULL) } \arguments{ \item{redcap_uri}{The URI (uniform resource identifier) of the REDCap project. Required.} diff --git a/vignettes/SecurityDatabase.Rmd b/vignettes/SecurityDatabase.Rmd index f2bfa046..fef69fc6 100644 --- a/vignettes/SecurityDatabase.Rmd +++ b/vignettes/SecurityDatabase.Rmd @@ -11,7 +11,7 @@ vignette: > Description ======================================== -The SQL code below adds schemas, a table and two stored procedures to an existing Microsoft SQL Database. This second database is not essential to calling the REDCap API, but it helps manage tokens securely. +The SQL code below adds schemas, a table and two stored procedures to an existing Microsoft SQL Server database. This second database is not essential to calling the REDCap API, but it helps manage tokens securely. This database contains the tokens and other sensitive content (such as passwords, API tokens, and file paths) that should not be stored in a Git repository (even a private Git repository). These passwords can be retrieved by `REDCapR::retrieve_credential_mssql()`. @@ -19,23 +19,25 @@ This database contains the tokens and other sensitive content (such as passwords Create a DSN on each client ======================================== -After executing the SQL code in an existing database, create an ODBC [DSN](http://en.wikipedia.org/wiki/Data_source_name) on *each* client machine that calls the database. Download the most recent drivers (as of Aug 2016, the [most recent version is 13.1](https://msdn.microsoft.com/library/mt703139.aspx) for Windows and Linux, then run the wizard. Many values in the wizard will remain at the default values. Here are the important ones to change. +After executing the SQL code in an existing database, create an ODBC [DSN](http://en.wikipedia.org/wiki/Data_source_name) on *each* client machine that calls the database. Download the most recent drivers (as of Aug 2018, the [most recent version is 17](https://docs.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server) for Windows and Linux), then run the wizard. Many values in the wizard will remain at the default values. Here are the important ones to change. 1. Set the DSN's `name` field to whatever is used in the repository's R code. 2. Set the authenticity method to `Integrated Windows authentication`. -3. Set the `default database` to the name of the database that containing the tokens *i.e.*, corresponding to the SQL code below in the example). +3. Set the `default database` to the name of the database that containing the tokens + +In our code below, both DSN and database are named `auxiliary_security`. Note ======================================== -We use Microsoft SQL Server, because that fits our University's infrastructure the easiest. But this approach theoretically can work with any LDAP-enabled database server. Please contact us if your institution is using something other than SQL Server (or a different configuration of these components), and would like help adapting this approach to your infrastructure. +We use Microsoft SQL Server, because that fits our university's infrastructure most easily. But this approach theoretically can work with any LDAP-enabled database server. Please contact us if your institution is using something other than SQL Server (or a different configuration of these components), and would like help adapting this approach. Create Database ======================================== -This SQL code is run once inside an existing database to establish the schemas, table, and stored procedure used by `REDCapR::retrieve_credential_mssql()`. +This SQL code is run once inside an existing database to establish the schemas, table, and stored procedure used by `REDCapR::retrieve_credential_mssql()`. In this example, we've arbitrarily called the database `auxiliary_security`. ```sql ------- SQL code to create necessary components in a Microsoft SQL Sever database ------- @@ -53,12 +55,12 @@ GO -- Create a table to contain the token -- CREATE TABLE [redcap_private].[tbl_credential]( - [id] [smallint] NOT NULL, - [username] [varchar](30) NOT NULL, - [project_id] [smallint] NOT NULL, - [instance] [varchar](30) NOT NULL, - [token] [char](32) NOT NULL, - [redcap_uri] [varchar](255) NOT NULL, + id smallint NOT NULL, + username varchar(30) NOT NULL, + project_id smallint NOT NULL, + instance varchar(30) NOT NULL, + token char(32) NOT NULL, + redcap_uri varchar(255) NOT NULL, CONSTRAINT [PK_credential] PRIMARY KEY CLUSTERED ( [id] ASC @@ -67,9 +69,9 @@ CREATE TABLE [redcap_private].[tbl_credential]( CREATE UNIQUE NONCLUSTERED INDEX [IX_tbl_credential_unique] ON [redcap_private].[tbl_credential] ( - [instance] ASC, - [project_id] ASC, - [username] ASC + instance ASC, + project_id ASC, + username ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] GO @@ -78,18 +80,19 @@ GO -- Notice it should a different (and more permissive) schema than the table. -- CREATE PROCEDURE [redcap].[prc_credential] - -- Add the parameters for the stored procedure here - @project_id smallint, - @instance varchar(30) + @instance varchar(30) AS BEGIN - -- SET NOCOUNT ON added to prevent extra result sets from - -- interfering with SELECT statements. SET NOCOUNT ON; SELECT username, project_id, token, redcap_uri FROM [redcap_private].[tbl_credential] - WHERE username=system_user AND project_id=@project_id AND instance=@instance + WHERE + username = system_user -- The username from the server's OS. + AND + project_id = @project_id -- Restricts to the desired REDCap project. + AND + instance = @instance -- System accommodates multiple REDCap instances. END ``` @@ -97,23 +100,19 @@ END Create user credentials to the auxiliary database ======================================== -Add a user's LDAP account to the `SecurityAuxiliary` database so that they can query the tables to retrieve their API. - -Notice that this only gives the permissions to retrieve the token. You must still: - 1. grant them API privileges to each appropriate REDCap project, and - 2. copy the API from the REDCap database into the SecurityAuxiliary database. +Add a user's LDAP account to the `auxiliary_security` database so that they can query the tables to retrieve their API. -In the future `REDCapR` may expose a function that allows the user to perform the second step (through a stored procedure). +Notice that this only gives the permissions to retrieve the token. You still must grant them API privileges to each appropriate REDCap project. The automation in the R file below will copy the API token from the MySQL database into the `auxiliary_security` database (see the 'Transfer Credentials' section). -Also, do not give typical users authorization for the 'redcap_private' schema. The current system allows the to view only their own tokens. +Only database admins should have authorization for the 'redcap_private' schema. Typical users should not be authorized for this schema. The current system allows typical users to view only their own tokens. ```sql ----------------------------------------------------------------------- --- Add a OUHSC's user account to the auxiliary_security database so that they can query the tables to retrieve their API. +-- Add a user account to the auxiliary_security database so that they can query the tables to retrieve their API. -- Notice that this only gives the permissions to retrieve the token. You must still: -- 1) grant them API privileges to each appropriate REDCap project, and -- 2) copy the API from the REDCap database into the auxiliary_security database. --- Also, do not give typical users authorization for the 'redcap_private' schema. The current system allows the to view only their own tokens. +-- Also, do not give typical users authorization for the 'redcap_private' schema. The current system allows them to view only their own tokens. ----------------------------------------------------------------------- -- STEP #1: Declare the user name. If everything runs correctly, this should be the only piece of code that you need to modify. @@ -139,7 +138,7 @@ DECLARE @login_count AS INT; SET @login_count = (SELECT COUNT(*) AS login_count print 'Logins matching desired name should equal 1. It equals: ' + CONVERT(varchar, @login_count); print '' ----------------------------------------------------------------------- --- STEP #3: Create a user account for the *data base*, after switching the database under focus to auxiliary_security. +-- STEP #3: Create a user account for the *database*, after switching the database under focus to auxiliary_security. print 'Step #3 executing....' USE [auxiliary_security] DECLARE @sql_create_user nvarchar(max) @@ -186,11 +185,11 @@ print 'Step #5 executed'; print '' Transfer Credentials ======================================== -Manually transfer tokens to the auxiliary server becomes unmanageable as your institution's collection of API users grows. This script demonstrates how to progamatically transfer all tokens from multiple REDCap instances on your network. The basic steps are: +Manually transferring tokens to the auxiliary server becomes unmanageable as your institution's collection of API users grows. This script demonstrates how to progamatically transfer all tokens from multiple REDCap instances. The basic steps are: -1. Read from the MySQL database underneath each REDCap instance. +1. Read from the MySQL database(s) underneath each REDCap instance on your campus. 1. Combine & groom the credentials. -1. Upload to SQL Server. +1. Upload to SQL Server (called `auxiliary_security` here). ```r rm(list=ls(all=TRUE)) #Clear the memory for any variables set from any previous runs. @@ -198,130 +197,159 @@ rm(list=ls(all=TRUE)) #Clear the memory for any variables set from any previous # ---- load-sources ------------------------------------------------------------ # ---- load-packages ----------------------------------------------------------- +if( !require(OuhscMunge) ) + stop('The `OuhscMunge` package needs to be installed with `devtools::install_github("OuhscBbmc/OuhscMunge")`.') + +testit::assert( + "The `OuhscMunge` package should meet a minimal version.", + compareVersion( as.character(packageVersion("OuhscMunge")), "0.1.9.9009") >= 0L +) + library(magrittr) -requireNamespace("RODBC") +requireNamespace("DBI") +requireNamespace("odbc") requireNamespace("dplyr") requireNamespace("readr") requireNamespace("tibble") +requireNamespace("testit") requireNamespace("checkmate") +requireNamespace("OuhscMunge") # devtools::install_github("OuhscBbmc/OuhscMunge") + # ---- declare-globals --------------------------------------------------------- +# This file assume your campus has two REDCap instances. +# Modify each (a) database name, (b) REDCap URL, and (c) DSN name. + +name_production <- "production" +name_dev <- "dev" + +uri_production <- "https://redcap-production.ouhsc.edu/redcap/api/", +uri_dev <- "https://redcap-dev.ouhsc.edu/redcap/api/" + +dsn_production <- "redcap-production" +dsn_dev <- "redcap-dev" +dsn_source <- "auxiliary_security" # The DSN of the token server. # The Activity Directory name that should precede each username. -# This should correspond with the result of `SYSTEM_USER` +# This should correspond with the result of SQL Server's `SYSTEM_USER` function # (https://msdn.microsoft.com/en-us/library/ms179930.aspx) ldap_prefix <- "OUHSC\\" -#Create a SQL statement for each REDCap instance. Only the `instance` value should change. +#### +# Nothing below this line should need to change, assuming: +# 1. the vignette was followed exactly (https://ouhscbbmc.github.io/REDCapR/articles/SecurityDatabase.html), +# 2. your campus has exactly two REDCap instances. + +# SQL sent to the MySQL database underneath each REDCap instance. sql <- " SELECT username, project_id, api_token FROM redcap_user_rights - WHERE api_token IS NOT NULL" + WHERE api_token IS NOT NULL +" + +# Update this ad-hoc CSV. Each row should represent one REDCap instance. +ds_url <- tibble::tribble( + ~instance , ~redcap_uri, + name_production , uri_production, + name_dev , uri_dev +) -#Update this ad-hoc CSV. Each row should represent one REDCap instance. -# Choose any casual name for the first variable, consistent with the `tweak-data` chunk below. -# Enter the exact URL for the second variable. -ds_url <- readr::read_csv(paste( - "instance,redcap_uri", - "production,https://redcap-production.ouhsc.edu/redcap/api/", - "dev,https://redcap-dev.ouhsc.edu/redcap/api/", -sep="\n")) +# Remove variables that aren't used below. +rm(uri_production, uri_dev) # ---- load-data --------------------------------------------------------------- -# Load the credentials from the first instance. -channel <- RODBC::odbcConnect("redcap-production") # odbcGetInfo(channel) -ds_prod <- RODBC::sqlQuery(channel, query=sql, stringsAsFactors=F) -RODBC::odbcClose(channel); rm(channel) +# Load the credentials from the first/production REDCap instance. +cnn_production <- DBI::dbConnect(odbc::odbc(), dsn=dsn_production) +ds_production <- DBI::dbGetQuery(cnn_production, sql) +DBI::dbDisconnect(cnn_production); rm(cnn_production, dsn_production) + +# Load the credentials from the second/dev REDCap instance. +cnn_dev <- DBI::dbConnect(odbc::odbc(), dsn=dsn_dev) +ds_dev <- DBI::dbGetQuery(cnn_dev, sql) +DBI::dbDisconnect(cnn_dev); rm(cnn_dev, dsn_dev) -# Load the credentials from the second instance. -# Duplicate or remove this block, dependending on the number of instances. -channel <- RODBC::odbcConnect("redcap-dev") # odbcGetInfo(channel) -ds_dev <- RODBC::sqlQuery(channel, query=sql, stringsAsFactors=F) -RODBC::odbcClose(channel); rm(channel, sql) +rm(sql) -# Assert these variables contain valid datasets (instead of a character error message), and -# that at least some rows were returned. -# Adjust this to smaller values if necessary. It's really just to catch blatant retrieval problems. -checkmate::assert_data_frame(ds_bbmc, min.rows=5) -checkmate::assert_data_frame(ds_dev , min.rows=5) +# Assert these are valid datasets and contain at least 5 rows. +# Adjust '5' to smaller value if necessary. It's just to catch blatant retrieval problems. +checkmate::assert_data_frame(ds_production, min.rows=5) +checkmate::assert_data_frame(ds_dev , min.rows=5) # ---- tweak-data -------------------------------------------------------------- -#Label each instance, so they're distinguishable later. Add/remove lines, depending on the number of campus instances -ds_prod$instance <- "production" -ds_dev$instance <- "dev" - -#Combine the token collection from each instance. Then prefix the username and include the URL of each instance. -ds <- ds_prod %>% - dplyr::union(ds_dev) %>% # Add/remove unions, based on the number of REDCap instances. - dplyr::select_( - "username" = "`username`" - , "project_id" = "`project_id`" - , "instance" = "`instance`" - , "token" = "`api_token`" +# Label each instance, so they're distinguishable later. +ds_production$instance <- name_production +ds_dev$instance <- name_dev + +# Stack the token collection from each instance. Then prefix the username and include the URL of each instance. +ds <- ds_production %>% + dplyr::union(ds_dev) %>% # Remove union if the dev instance isn't included. + tibble::as_tibble() %>% + dplyr::select( + username = username, + project_id = project_id, + instance = instance, + token = api_token ) %>% - dplyr::arrange(instance, project_id, username) %>% dplyr::mutate( - username = paste0(ldap_prefix, username), # Qualify for the Active Directory. - id = seq_len(n()) # For the sake of a clustered primary key. + username = paste0(ldap_prefix, username), # Qualify for the Active Directory. ) %>% - dplyr::left_join( ds_url, by="instance") # Include the instance URL. + dplyr::left_join( ds_url, by="instance") %>% # Include the instance URL. + dplyr::arrange(instance, project_id, username) %>% + tibble::rowid_to_column("id") # For the sake of a clustered primary key. -rm(ds_prod, ds_dev, ds_url) +rm(ds_production, ds_dev, ds_url) +rm(name_production, name_dev) +rm(ldap_prefix) # ---- verify-values ----------------------------------------------------------- -# devtools::install_github("OuhscBbmc/OuhscMunge"); OuhscMunge::verify_value_headstart(ds) + # Assert that the dataset is well-behaved. -checkmate::assert_integer( ds$id , any.missing=F , lower=1, upper=2^31-1 , unique=T) -checkmate::assert_character(ds$username , any.missing=F , pattern="^.{1,255}$" ) -checkmate::assert_integer( ds$project_id , any.missing=F , lower=1, upper=2^31-1 ) -checkmate::assert_character(ds$token , any.missing=F , pattern="^.{32}$" , unique=T) -checkmate::assert_character(ds$instance , any.missing=F , pattern="^.{1,255}$" ) -checkmate::assert_character(ds$redcap_uri , any.missing=F , pattern="^.{1,255}$" ) +# OuhscMunge::verify_value_headstart(ds) +checkmate::assert_integer( ds$id , any.missing=F, lower=1, upper=.Machine$integer.max, unique=T) +checkmate::assert_character(ds$username , any.missing=F, pattern="^.{1,255}$" ) +checkmate::assert_integer( ds$project_id , any.missing=F, lower=1, upper=.Machine$integer.max ) +checkmate::assert_character(ds$token , any.missing=F, pattern="^[A-Z0-9]{32}$" , unique=T) +checkmate::assert_character(ds$instance , any.missing=F, pattern="^.{1,255}$" ) +checkmate::assert_character(ds$redcap_uri , any.missing=F, pattern="^.{1,255}$" ) testit::assert( "The `username` x `project_id` x `instance` must be unique.", sum(duplicated(paste0(ds$username, "-", ds$project_id, "-", ds$instance))) == 0L ) -testit::assert("There should be at least 10 tokens written." , 10L <= nrow(ds)) +testit::assert("At least 10 tokens should be ready to write." , 10L <= nrow(ds)) # ---- specify-columns-to-upload ----------------------------------------------- # Dictate the exact columns and order that will be uploaded. columns_to_write <- c("id", "username", "project_id", "instance", "token", "redcap_uri") -ds_slim <- ds[, columns_to_write] - +ds_slim <- ds[, columns_to_write] rm(columns_to_write) -# ---- upload-to-db-credential ------------------------------------------------------------ +# ---- upload-to-db ------------------------------------------------------------------ -#Upload to SQL Server through ODBC. -(start_time <- Sys.time()) - -db_table <- "redcap_private.tbl_credential" -channel <- RODBC::odbcConnect("auxiliary_security") #getSqlTypeInfo("Microsoft SQL Server") #;odbcGetInfo(channel) - -column_info <- RODBC::sqlColumns(channel, db_table) -var_types <- as.character(column_info$TYPE_NAME) -names(var_types) <- as.character(column_info$COLUMN_NAME) #var_types - -RODBC::sqlClear(channel, db_table) -RODBC::sqlSave(channel, ds_slim, db_table, append=TRUE, rownames=FALSE, fast=TRUE, varTypes=var_types) -RODBC::odbcClose(channel); rm(channel) - -(elapsed_duration <- Sys.time() - start_time) #0.6026149 secs 2016-08-29. -rm(db_table, column_info, var_types, start_time, elapsed_duration) +OuhscMunge::upload_sqls_odbc( + d = ds_slim, + schema_name = "redcap_private", + table_name = "tbl_credential", + dsn_name = dsn_source, + create_table = FALSE, + clear_table = TRUE, + transaction = TRUE, + verbose = TRUE +) +# Uploading 252 tokens takes 0.004 minutes. ``` Document Info ======================================== -This document is primarily based on REDCap version 6.11.5, and was last updated 2016-08-30. A development version of the document is available on GitHub: https://ouhscbbmc.github.io/REDCapR/articles/SecurityDatabase.html +This document is primarily based on REDCap version 8.4.0, and was last updated 2018-08-10. A development version of the document is available on GitHub: https://ouhscbbmc.github.io/REDCapR/articles/SecurityDatabase.html