field order matches dictionary

the *_complete boxes are at the end of their form

ref #437

R/redcap-metadata-coltypes.R

@@ -250,11 +250,18 @@ redcap_metadata_internal <- function(
       field_name_base       = .data$original_field_name
     )
 
-  d_complete <-
+  d_inst <-
     d_inst %>%
     dplyr::select(
       form_name   = .data$instrument_name,
     ) %>%
+    dplyr::mutate(
+      form_order  = seq_len(dplyr::n()),
+    )
+
+  # Dataset that holds the *_complete checkboxes
+  d_complete <-
+    d_inst %>%
     dplyr::mutate(
       field_name      = paste0(.data$form_name, "_complete"),
       field_name_base = .data$field_name,  # same for *_complete checkboxes
@@ -269,6 +276,7 @@ redcap_metadata_internal <- function(
       .data$vt,
     )
 
+  # Dataset that holds longitudinal/repeating variables
   d_again <-
     tibble::tibble(
       field_name        = character(0),
@@ -283,9 +291,11 @@ redcap_metadata_internal <- function(
       d_again %>%
       dplyr::union_all(
         tibble::tibble(
-          field_name  = "redcap_event_name",
-          field_type  = "event_name",
-          vt          = NA_character_,
+          field_name      = "redcap_event_name",
+          field_name_base = "redcap_event_name",
+          form_name       = "longitudinal/repeating",
+          field_type      = "event_name",
+          vt              = NA_character_,
         )
       )
   }
@@ -295,14 +305,16 @@ redcap_metadata_internal <- function(
       d_again %>%
       dplyr::union_all(
         tibble::tibble(
-          field_name  = c("redcap_repeat_instrument", "redcap_repeat_instance"),
-          field_type  = c("repeat_instrument"       , "repeat_instance"),
-          vt          = NA_character_,
+          field_name      = c("redcap_repeat_instrument", "redcap_repeat_instance"),
+          field_name_base = c("redcap_repeat_instrument", "redcap_repeat_instance"),
+          form_name       = "longitudinal/repeating",
+          field_type      = c("repeat_instrument"       , "repeat_instance"),
+          vt              = NA_character_,
         )
       )
   }
 
-  # Prepare metadata to be joined
+  # Construct extended metadata
   d_meta <-
     d_meta %>%
     dplyr::select(
@@ -322,14 +334,22 @@ redcap_metadata_internal <- function(
       .data$form_name,
       .data$field_type,
       vt            = .data$text_validation_type_or_show_slider_number,
+    )  %>%
+    dplyr::union_all(d_complete) %>%
+    dplyr::left_join(d_inst, by = "form_name") %>%
+    dplyr::group_by(.data$form_name) %>%
+    dplyr::mutate(
+      field_order_within_form  = seq_len(dplyr::n()),
     ) %>%
-    tibble::add_row(d_again, .after = 1) %>%
-    dplyr::union_all(d_complete)
+    dplyr::ungroup() %>%
+    dplyr::arrange(form_order,  field_order_within_form) %>%
+    dplyr::select(-form_order, -field_order_within_form) %>%
+    tibble::add_row(d_again, .after = 1)
 
   # setdiff(d_meta$field_name_base, d_var$original_field_name)
   # [1] "signature"   "file_upload" "descriptive"
 
-  # Translate the four datasets into a single cohesive dataset.
+  # Determine & notate the likely data type
   d <-
     d_meta %>%
     dplyr::mutate(

inst/test-data/specific-redcapr/metadata-coltypes/longitudinal.R

@@ -43,12 +43,14 @@ structure(list(cols = list(study_id = structure(list(), class = c("collector_cha
 "collector")), weight = structure(list(), class = c("collector_integer", 
 "collector")), bmi = structure(list(), class = c("collector_character", 
 "collector")), comments = structure(list(), class = c("collector_character", 
+"collector")), demographics_complete = structure(list(), class = c("collector_integer", 
 "collector")), ec_phone = structure(list(), class = c("collector_character", 
 "collector")), ec_confirmed = structure(list(), class = c("collector_character", 
 "collector")), next_of_kin_contact_name = structure(list(), class = c("collector_character", 
 "collector")), next_of_kin_contact_address = structure(list(), class = c("collector_character", 
 "collector")), next_of_kin_contact_phone = structure(list(), class = c("collector_character", 
 "collector")), next_of_kin_confirmed = structure(list(), class = c("collector_character", 
+"collector")), contact_info_complete = structure(list(), class = c("collector_integer", 
 "collector")), height2 = structure(list(), class = c("collector_double", 
 "collector")), weight2 = structure(list(), class = c("collector_integer", 
 "collector")), bmi2 = structure(list(), class = c("collector_character", 
@@ -57,15 +59,18 @@ structure(list(cols = list(study_id = structure(list(), class = c("collector_cha
 "collector")), npcr_b = structure(list(), class = c("collector_double", 
 "collector")), chol_b = structure(list(), class = c("collector_double", 
 "collector")), transferrin_b = structure(list(), class = c("collector_double", 
+"collector")), baseline_data_complete = structure(list(), class = c("collector_integer", 
 "collector")), vld1 = structure(list(), class = c("collector_double", 
 "collector")), vld2 = structure(list(), class = c("collector_double", 
 "collector")), vld3 = structure(list(), class = c("collector_double", 
 "collector")), vld4 = structure(list(), class = c("collector_double", 
 "collector")), vld5 = structure(list(), class = c("collector_double", 
+"collector")), visit_lab_data_complete = structure(list(), class = c("collector_integer", 
 "collector")), pmq1 = structure(list(), class = c("collector_character", 
 "collector")), pmq2 = structure(list(), class = c("collector_character", 
 "collector")), pmq3 = structure(list(), class = c("collector_character", 
 "collector")), pmq4 = structure(list(), class = c("collector_character", 
+"collector")), patient_morale_questionnaire_complete = structure(list(), class = c("collector_integer", 
 "collector")), vbw1 = structure(list(), class = c("collector_double", 
 "collector")), vbw2 = structure(list(), class = c("collector_double", 
 "collector")), vbw3 = structure(list(), class = c("collector_double", 
@@ -75,6 +80,7 @@ structure(list(cols = list(study_id = structure(list(), class = c("collector_cha
 "collector")), vbw7 = structure(list(), class = c("collector_character", 
 "collector")), vbw8 = structure(list(), class = c("collector_character", 
 "collector")), vbw9 = structure(list(), class = c("collector_character", 
+"collector")), visit_blood_workup_complete = structure(list(), class = c("collector_integer", 
 "collector")), vob1 = structure(list(), class = c("collector_character", 
 "collector")), vob2 = structure(list(), class = c("collector_character", 
 "collector")), vob3 = structure(list(), class = c("collector_character", 
@@ -89,6 +95,7 @@ structure(list(cols = list(study_id = structure(list(), class = c("collector_cha
 "collector")), vob12 = structure(list(), class = c("collector_character", 
 "collector")), vob13 = structure(list(), class = c("collector_character", 
 "collector")), vob14 = structure(list(), class = c("collector_character", 
+"collector")), visit_observed_behavior_complete = structure(list(), class = c("collector_integer", 
 "collector")), study_comments = structure(list(), class = c("collector_character", 
 "collector")), complete_study = structure(list(), class = c("collector_character", 
 "collector")), withdraw_date = structure(list(format = ""), class = c("collector_date", 
@@ -101,6 +108,7 @@ structure(list(cols = list(study_id = structure(list(), class = c("collector_cha
 "collector")), npcr_4 = structure(list(), class = c("collector_integer", 
 "collector")), chol_4 = structure(list(), class = c("collector_integer", 
 "collector")), withdraw_reason = structure(list(), class = c("collector_character", 
+"collector")), completion_data_complete = structure(list(), class = c("collector_integer", 
 "collector")), cpq1 = structure(list(format = ""), class = c("collector_date", 
 "collector")), cpq2 = structure(list(), class = c("collector_integer", 
 "collector")), cpq3 = structure(list(), class = c("collector_integer", 
@@ -114,14 +122,6 @@ structure(list(cols = list(study_id = structure(list(), class = c("collector_cha
 "collector")), cpq11 = structure(list(), class = c("collector_character", 
 "collector")), cpq12 = structure(list(), class = c("collector_character", 
 "collector")), cpq13 = structure(list(), class = c("collector_character", 
-"collector")), demographics_complete = structure(list(), class = c("collector_character", 
-"collector")), contact_info_complete = structure(list(), class = c("collector_character", 
-"collector")), baseline_data_complete = structure(list(), class = c("collector_character", 
-"collector")), visit_lab_data_complete = structure(list(), class = c("collector_character", 
-"collector")), patient_morale_questionnaire_complete = structure(list(), class = c("collector_character", 
-"collector")), visit_blood_workup_complete = structure(list(), class = c("collector_character", 
-"collector")), visit_observed_behavior_complete = structure(list(), class = c("collector_character", 
-"collector")), completion_data_complete = structure(list(), class = c("collector_character", 
-"collector")), completion_project_questionnaire_complete = structure(list(), class = c("collector_character", 
+"collector")), completion_project_questionnaire_complete = structure(list(), class = c("collector_integer", 
 "collector"))), default = structure(list(), class = c("collector_guess", 
 "collector")), delim = NULL), class = "col_spec")

inst/test-data/specific-redcapr/metadata-coltypes/problematic-dictionary.R

@@ -4,6 +4,6 @@ structure(list(cols = list(record_id = structure(list(), class = c("collector_in
 "collector")), curly_quote_double_left = structure(list(), class = c("collector_character", 
 "collector")), curly_quote_double_right = structure(list(), class = c("collector_character", 
 "collector")), long_dash = structure(list(), class = c("collector_character", 
-"collector")), form_1_complete = structure(list(), class = c("collector_character", 
+"collector")), form_1_complete = structure(list(), class = c("collector_integer", 
 "collector"))), default = structure(list(), class = c("collector_guess", 
 "collector")), delim = NULL), class = "col_spec")

inst/test-data/specific-redcapr/metadata-coltypes/repeating-instruments.R

@@ -8,10 +8,10 @@ structure(list(cols = list(record_id = structure(list(), class = c("collector_in
 "collector")), ethnicity = structure(list(), class = c("collector_character", 
 "collector")), race = structure(list(), class = c("collector_character", 
 "collector")), sex = structure(list(), class = c("collector_character", 
+"collector")), demographics_complete = structure(list(), class = c("collector_integer", 
 "collector")), date_bp = structure(list(format = ""), class = c("collector_date", 
 "collector")), bp_systolic = structure(list(), class = c("collector_integer", 
 "collector")), bp_diastolic = structure(list(), class = c("collector_integer", 
-"collector")), demographics_complete = structure(list(), class = c("collector_character", 
-"collector")), bp_complete = structure(list(), class = c("collector_character", 
+"collector")), bp_complete = structure(list(), class = c("collector_integer", 
 "collector"))), default = structure(list(), class = c("collector_guess", 
 "collector")), delim = NULL), class = "col_spec")

inst/test-data/specific-redcapr/metadata-coltypes/simple.R

@@ -7,11 +7,13 @@ structure(list(cols = list(record_id = structure(list(), class = c("collector_in
 "collector")), dob = structure(list(format = ""), class = c("collector_date", 
 "collector")), age = structure(list(), class = c("collector_character", 
 "collector")), sex = structure(list(), class = c("collector_character", 
+"collector")), demographics_complete = structure(list(), class = c("collector_integer", 
 "collector")), height = structure(list(), class = c("collector_double", 
 "collector")), weight = structure(list(), class = c("collector_integer", 
 "collector")), bmi = structure(list(), class = c("collector_character", 
 "collector")), comments = structure(list(), class = c("collector_character", 
 "collector")), mugshot = structure(list(), class = c("collector_character", 
+"collector")), health_complete = structure(list(), class = c("collector_integer", 
 "collector")), race___1 = structure(list(), class = c("collector_logical", 
 "collector")), race___2 = structure(list(), class = c("collector_logical", 
 "collector")), race___3 = structure(list(), class = c("collector_logical", 
@@ -20,8 +22,6 @@ structure(list(cols = list(record_id = structure(list(), class = c("collector_in
 "collector")), race___6 = structure(list(), class = c("collector_logical", 
 "collector")), ethnicity = structure(list(), class = c("collector_character", 
 "collector")), interpreter_needed = structure(list(), class = c("collector_logical", 
-"collector")), demographics_complete = structure(list(), class = c("collector_character", 
-"collector")), health_complete = structure(list(), class = c("collector_character", 
-"collector")), race_and_ethnicity_complete = structure(list(), class = c("collector_character", 
+"collector")), race_and_ethnicity_complete = structure(list(), class = c("collector_integer", 
 "collector"))), default = structure(list(), class = c("collector_guess", 
 "collector")), delim = NULL), class = "col_spec")

inst/test-data/specific-redcapr/metadata-coltypes/validation-types.R

@@ -49,6 +49,6 @@ structure(list(cols = list(record_id = structure(list(), class = c("collector_in
 "collector")), v_time_mm_ss = structure(list(format = "%M:%S"), class = c("collector_time", 
 "collector")), v_vmrn = structure(list(), class = c("collector_character", 
 "collector")), v_zipcode = structure(list(), class = c("collector_character", 
-"collector")), form_1_complete = structure(list(), class = c("collector_character", 
+"collector")), form_1_complete = structure(list(), class = c("collector_integer", 
 "collector"))), default = structure(list(), class = c("collector_guess", 
 "collector")), delim = NULL), class = "col_spec")

playgrounds/eav-playground-2.R

@@ -16,8 +16,10 @@ token <- "9A81268476645C4E5F03428B8AC3AA7B"  # PHI-free demo: simple static
 # token <- "5C1526186C4D04AE0A0630743E69B53C"  # PHI-free demo: super-wide #3--35,000 columns
 # token <- "56F43A10D01D6578A46393394D76D88F"  # PHI-free demo: Repeating Instruments --Sparse
 
-fields  <- c("record_id", "dob")
-forms   <- "health"
+# fields  <- c("record_id", "dob")
+# forms   <- "health"
+fields  <- NULL
+forms   <- NULL
 
 # ---- load-data ---------------------------------------------------------------
 system.time(
@@ -27,10 +29,12 @@ system.time(
 system.time({
   col_types   <- REDCapR::redcap_metadata_coltypes(redcap_uri, token, print_col_types_to_console = FALSE)
 
+  View(REDCapR::redcap_variables(redcap_uri, token)$data)
+
   ds_metadata <-
     REDCapR:::redcap_metadata_internal(redcap_uri, token)$d_variable %>%
     dplyr::filter(
-      (.data$field_name_base %in% fields) | (.data$form_name %in% forms)
+      # (.data$field_name_base %in% fields) | (.data$form_name %in% forms)
     )
 
   desired_fields <- ds_metadata$field_name