From b5690305d31c7751dd85a2fe0ddc3d25385a3ed0 Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Tue, 23 Nov 2021 16:45:28 -0800 Subject: [PATCH 01/13] read files from minio folder for core-ms --- Reactive_Variables/cloud_revals.R | 31 ++++++ global.R | 52 ++++++++++- python_requirements.txt | 1 + server.R | 150 +++++------------------------- static_objects.R | 23 ----- 5 files changed, 108 insertions(+), 149 deletions(-) create mode 100644 Reactive_Variables/cloud_revals.R create mode 100644 python_requirements.txt delete mode 100644 static_objects.R diff --git a/Reactive_Variables/cloud_revals.R b/Reactive_Variables/cloud_revals.R new file mode 100644 index 0000000..3708f2f --- /dev/null +++ b/Reactive_Variables/cloud_revals.R @@ -0,0 +1,31 @@ +#' @details core-ms files loaded through a header parameter that points to a +#' 'folder' in minio containing all files. +corems_samples <- reactiveValues() + +observe({ + req(!is.null(header_params[['corems-uri']])) + + isolate({ + withProgress(message = "Loading core-ms files...", value = 1, { + uris <- reticulate::iterate( + minio_con$client$list_objects( + minio_con$bucket, + prefix = header_params[['corems-uri']], + recursive = TRUE), + function(x) x$object_name + ) + + fpaths <- lapply(uris, function(uri) { + mapDataAccess::get_file( + minio_con, id = uri, filename = file.path(tempfile(), basename(uri)) + ) + }) + + names(fpaths) <- sapply(fpaths, basename) + + for(name in names(fpaths)) { + corems_samples[[name]] <- read_csv(fpaths[[name]]) + } + }) + }) +}) \ No newline at end of file diff --git a/global.R b/global.R index 5d84913..49a59ba 100644 --- a/global.R +++ b/global.R @@ -18,6 +18,7 @@ library(readr) library(plotly) library(DT) library(shinycssloaders) +library(mapDataAccess) # uncomment either library() or load_all() if you need to load kegg library(KeggData) @@ -25,4 +26,53 @@ library(MetaCycData) # devtools::load_all('~/Documents/git_repos/MetaCycData/') # devtools::load_all('~/Documents/git_repos/KeggData/') -source("static_objects.R", local=TRUE) +# static variables # +dt_checkmark <- '' +dt_minus <- '' + +dbe_opts_info <- 'Semicolon separated strings of elements and their valences, i.e. to calculate the dbe for two sets of valences, input C4H1N3O2S2P3;C3H3N2O2S4P4' +kendrick_opts_info <- 'The base compound(s) used to calculate the Kendrick Mass. See [Foquet and Sato, 2017] for details.' + +dt_checkmark <- '' +dt_minus <- '' + +ttip_text = list("plot_save"="Save the last created plot", "plot_review"="Review saved plots", "page_help"="How do I use this page?") + +#------ Download Example Data ---------# +example_edata <- read_csv('Data/example12T_edata.csv') %>% as.data.frame(stringsAsFactors = FALSE) +example_emeta <- read_csv('Data/example12T_emeta.csv') %>% as.data.frame(stringsAsFactors = FALSE) +calc_opts <- read_csv('calculation_options.csv') %>% as.data.frame(stringsAsFactors = FALSE) +calc_vars <- read_csv('calculation_variables.csv') %>% as.data.frame(stringsAsFactors = FALSE) +# determines when 'large data' options are triggered +max_cells <- 2000000 + +info_text = list( + VALID_LINKED_PLOTS = "Current valid plots to link are Van-Krevelen, Kendrick, single sample density, and custom scatter plots." +) + +# static variables # +dt_checkmark <- '' +dt_minus <- '' + +dbe_opts_info <- 'Semicolon separated strings of elements and their valences, i.e. to calculate the dbe for two sets of valences, input C4H1N3O2S2P3;C3H3N2O2S4P4' +kendrick_opts_info <- 'The base compound(s) used to calculate the Kendrick Mass. See [Foquet and Sato, 2017] for details.' + +dt_checkmark <- '' +dt_minus <- '' + +ttip_text = list("plot_save"="Save the last created plot", "plot_review"="Review saved plots", "page_help"="How do I use this page?") + +#------ Download Example Data ---------# +example_edata <- read_csv('Data/example12T_edata.csv') %>% as.data.frame(stringsAsFactors = FALSE) +example_emeta <- read_csv('Data/example12T_emeta.csv') %>% as.data.frame(stringsAsFactors = FALSE) +calc_opts <- read_csv('calculation_options.csv') %>% as.data.frame(stringsAsFactors = FALSE) +calc_vars <- read_csv('calculation_variables.csv') %>% as.data.frame(stringsAsFactors = FALSE) +# determines when 'large data' options are triggered +max_cells <- 2000000 + +info_text = list( + VALID_LINKED_PLOTS = "Current valid plots to link are Van-Krevelen, Kendrick, single sample density, and custom scatter plots." +) + +# cloud/minio resources +VALID_MINIO_HEADER_PARAMS = c("corems-uri") diff --git a/python_requirements.txt b/python_requirements.txt new file mode 100644 index 0000000..72aba03 --- /dev/null +++ b/python_requirements.txt @@ -0,0 +1 @@ +minio==7.0.2 \ No newline at end of file diff --git a/server.R b/server.R index 16f9af6..085cec6 100644 --- a/server.R +++ b/server.R @@ -23,19 +23,32 @@ shinyServer(function(session, input, output) { source('untracked_resources/store_postmortem_objects.R', local = TRUE) }, error = function(e) message('Not storing postmortem objects')) - # Source various helper functions - source('helper_functions/selection_addons.R') - source('helper_functions/summaryFilter.R') - source('helper_functions/summaryPreprocess.R') - source('helper_functions/database_utils.R') - - # observers and UI elements which operate across tabs - source("Observers/global_observers.R", local = TRUE) - source('srv_ui_elements/global_UI.R', local = TRUE) + # Source all scripts + for (f in Sys.glob("./helper_functions/*.R")) source(f, local = TRUE) + for (f in Sys.glob("./Reactive_variables/*.R")) source(f, local = TRUE) + for (f in Sys.glob("./Observers/*.R")) source(f, local = TRUE) + for (f in Sys.glob("./srv_ui_elements/*.R")) source(f, local = TRUE) + + #### Minio/cloud initialization + header_params = reactiveValues() - # Misc Reactive Values: - # peakData2_dim(), uploaded_data_dim(). The number of cells in e_data of the respective objects - source('Reactive_Variables/misc_revals.R', local = TRUE) + # parse header params + observe({ + query <- parseQueryString(session$clientData$url_search) + + # establish minio connection if we are pulling cloud resources + if(any(names(query) %in% VALID_MINIO_HEADER_PARAMS)) { + minio_con <<- mapDataAccess::map_data_connection("./cfg/minio_config_local.yml") + } + + isolate({ + # store header params in a reactive variable + for(key in names(query)){ + header_params[[key]] <- query[[key]] + message(sprintf("INFO: stored parameter %s: %s", key, query[[key]])) + } + }) + }) revals <- reactiveValues(ntables = 0, makeplot = 1, color_by_choices = NULL, axes_choices = NULL, redraw_largedata = FALSE, react_largedata = FALSE, plot_data_export = NULL, peakData_export = NULL, redraw_filter_plot = TRUE, reac_filter_plot = TRUE, @@ -95,120 +108,7 @@ shinyServer(function(session, input, output) { }, contentType = "application/zip" ) - ############################## - ######## Upload Tab ########## - ############################## - - # Upload Tab Reactive Values: - # Edata(): The uploaded data file - # edata_cnames(): All column names from from Edata() - # Emeta(): The uploaded e_meta file - # emeta_cnames(): All column names from Emeta() - # sample_names(): Sample names from uploaded data (all columns from Edata() minus mass column) - # fdata(): dummy f_data object created from sample_names(). Needed for input to as.peakData. CONTAINS NO GROUPING INFORMATION - source("Reactive_Variables/upload_revals.R", local = TRUE) - ### Upload Observers: Contains conditional dropdown behavior, shinyjs functionality. - source("Observers/upload_observers.R", local = TRUE) - ### - - #### Main Panel (Upload Tab) #### - # Minor upload UI Elements (output$) - # num_peaks, num_samples - # emeta_text, edata_text, success_upload - source('srv_ui_elements/upload_UI_mainpanel.R', local = TRUE) - - #### Sidebar Panel (Upload Tab) #### - # element selection and C13 sidebar elements - source('srv_ui_elements/upload_UI_sidebar.R', local = TRUE) - - ######################### - ####### Groups Tab ###### - ######################### - - source("Observers/groups_observers.R", local = TRUE) - source("srv_ui_elements/groups_UI.R", local = TRUE) - - # Groups tab reactive variables: - # groupstab_df(): reactive table that displays groups and their filtered/non filtered samples - source("Reactive_Variables/groups_revals.R", local = TRUE) - - ############################## - ####### Preprocess Tab ####### - ############################## - - # Preprocess Tab reactive variables: - # emeta_display_choices(): Columns of emeta minus mass column, isotopic information column, and categorical columns with greater than 12 categories - source("Reactive_Variables/preprocess_revals.R", local = TRUE) - - source("Observers/preprocess_observers.R", local = TRUE) - source('srv_ui_elements/preprocess_UI.R', local = TRUE) - - ###################################### - ############## QC tab ################ - ###################################### - - source('Observers/qc_observers.R', local = TRUE) - source('srv_ui_elements/qc_UI.R', local = TRUE) - - ######################################## - ############## Filter tab ############## - ######################################## - - ### Filter Observers. Contains much of the dropdown behavior and helper button functionality. - source("Observers/filter_observers.R", local = TRUE) - ### - - # Filter UI Elements - source('srv_ui_elements/filter_UI.R', local = TRUE) - - ## Filter tab reactive values - source("Reactive_Variables/filter_revals.R", local = TRUE) - - ############################# - ####### Visualize Tab ####### - ############################# - - #Visualize Tab reactive variables: - # plot_data(): Plotting dataframe that is passed to output$FxnPlot. This object triggers an important observer (in server.R) which controls dropdown selection - # plot_defaults(): defaults arguments to plot axes/title values - # numeric_selected(): keeps track of whether the column selected by input$vk_colors is numeric or categorical - # g1_samples() and g2_samples() These store the samples that will be compared during group/sample comparison plots - source("Reactive_Variables/visualize_revals.R", local = TRUE) - - # Viztab observers. Help Button. Dropdown choices, plot clearing, and shinyjs helper functionality### - source("Observers/visualize_observers.R", local = TRUE) - # - - # Minor UI Elements - # Label adjustment and plot download buttons (sidebar) - source('srv_ui_elements/visualize_UI_main_and_plot_opts.R', local = TRUE) - # plot, sample type, and comparison options - source('srv_ui_elements/visualize_UI_sidebar.R', local = TRUE) - # warnings_visualize: displays warning messages in revals$warningmessage_visualize - # chooseplots_icon, axlabs_icon, saveplots_icon: icons for collapse panels - source('srv_ui_elements/visualize_UI_misc.R', local = TRUE) - - ## Linked plots Sub-tab - source('srv_ui_elements/visualize_linked_plots_UI.R', local = TRUE) - source('Observers/linked_plot_observers.R', local = TRUE) - source('Reactive_Variables/linked_plot_revals.R', local = TRUE) - - ############################ - ####### Database Tab ####### - ############################ - - source('./Observers/database_observers.R', local = TRUE) - source('./srv_ui_elements/database_UI.R', local = TRUE) - - ############################ - ####### Download Tab ####### - ############################ - - source("Observers/download_observers.R", local = TRUE) - source("helper_functions/report.R", local = TRUE) - source('srv_ui_elements/download_UI.R', local = TRUE) - #---- processed data download --------# output$download_processed_data <- downloadHandler( filename = paste("FREDA_Output_",proc.time()[1],".zip", sep = ""), diff --git a/static_objects.R b/static_objects.R deleted file mode 100644 index 6439b20..0000000 --- a/static_objects.R +++ /dev/null @@ -1,23 +0,0 @@ -# static variables # -dt_checkmark <- '' -dt_minus <- '' - -dbe_opts_info <- 'Semicolon separated strings of elements and their valences, i.e. to calculate the dbe for two sets of valences, input C4H1N3O2S2P3;C3H3N2O2S4P4' -kendrick_opts_info <- 'The base compound(s) used to calculate the Kendrick Mass. See [Foquet and Sato, 2017] for details.' - -dt_checkmark <- '' -dt_minus <- '' - -ttip_text = list("plot_save"="Save the last created plot", "plot_review"="Review saved plots", "page_help"="How do I use this page?") - -#------ Download Example Data ---------# -example_edata <- read_csv('Data/example12T_edata.csv') %>% as.data.frame(stringsAsFactors = FALSE) -example_emeta <- read_csv('Data/example12T_emeta.csv') %>% as.data.frame(stringsAsFactors = FALSE) -calc_opts <- read_csv('calculation_options.csv') %>% as.data.frame(stringsAsFactors = FALSE) -calc_vars <- read_csv('calculation_variables.csv') %>% as.data.frame(stringsAsFactors = FALSE) -# determines when 'large data' options are triggered -max_cells <- 2000000 - -info_text = list( - VALID_LINKED_PLOTS = "Current valid plots to link are Van-Krevelen, Kendrick, single sample density, and custom scatter plots." -) \ No newline at end of file From e43fdbd29dafc588f438e2d57087ff1401493b4c Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Wed, 24 Nov 2021 14:32:47 -0800 Subject: [PATCH 02/13] refactor intialization of header params and 'upload' UI --- Observers/startup_observers.R | 66 ++++++++++ README.md | 6 +- Reactive_Variables/cloud_revals.R | 31 ----- global.R | 3 + server.R | 112 +++++++++++------ ui.R | 194 ++++-------------------------- www/yeti.css | 5 + 7 files changed, 177 insertions(+), 240 deletions(-) create mode 100644 Observers/startup_observers.R delete mode 100644 Reactive_Variables/cloud_revals.R diff --git a/Observers/startup_observers.R b/Observers/startup_observers.R new file mode 100644 index 0000000..becd7bd --- /dev/null +++ b/Observers/startup_observers.R @@ -0,0 +1,66 @@ +#'@details Parse and store header parameters. If 'corems-uri' is passed, load +#'the files and display a different tab for 'upload'. +observe({ + query <- parseQueryString(session$clientData$url_search) + + # establish minio connection if we are pulling cloud resources + if(any(names(query) %in% VALID_MINIO_HEADER_PARAMS)) { + minio_con <<- mapDataAccess::map_data_connection("./cfg/minio_config_local.yml") + } + + isolate({ + # store header params in a reactive variable + for(key in names(query)){ + header_params[[key]] <- query[[key]] + message(sprintf("INFO: stored parameter %s: %s", key, query[[key]])) + } + + if('corems-uri' %in% names(query)) { + withProgress(message = "Loading core-ms files...", value = 1, { + uris <- reticulate::iterate( + minio_con$client$list_objects( + minio_con$bucket, + prefix = header_params[['corems-uri']], + recursive = TRUE), + function(x) x$object_name + ) + + if(length(uris) > 0) { + tryCatch({ + fpaths <- lapply(uris, function(uri) { + mapDataAccess::get_file( + minio_con, id = uri, filename = file.path(tempfile(), basename(uri)) + ) + }) + + names(fpaths) <- sapply(fpaths, basename) + + for(name in names(fpaths)) { + corems_samples[[name]] <- read_csv(fpaths[[name]]) + } + + modalmessage <- div(class = "column-scroll-sm", + HTML(info_text[["COREMS_UPLOAD_SUCCESS"]]), + HTML(paste(names(fpaths), collapse = "
")) + ) + }, error = function(e) { + modalmessage <<- div(sprintf(info_text[["COREMS_UPLOAD_ERROR"]], e)) + }) + } else { + modalmessage <- div(info_text[["COREMS_UPLOAD_NOSAMPS"]]) + } + + showModal(modalDialog(modalmessage, title = "Core-MS Upload")) + + }) + } + + insertTab( + "top_page", + target = "Welcome", + tab = upload_tab(length(names(corems_samples)) > 0), + position = "after" + ) + + }) +}) \ No newline at end of file diff --git a/README.md b/README.md index ba62a88..725e727 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Then run the docker container: `docker run -p 3838:3838 docker pull docker.arti ### **Development** -#### **1. Dockerfiles** +#### **Dockerfiles** We build a 'base' container which has all the system libraries and R packages installed, and then build a container on top of it that simply copies the app source code and exposes the correct port. There are two Dockerfiles, and two corresponding .dockerignore files. @@ -61,7 +61,7 @@ Simply make sure Dockerfile refers to the correct base container if you have upd If all is well, push new containers to the registry: `docker push :` -#### **2. Dependencies** +#### **Dependencies** We use [renv](https://rstudio.github.io/renv/articles/renv.html) to track dependencies. The renv.lock file contains a list of dependencies and various details about them. We use renv to manage the details about dependencies. When updating the lockfile, we will do the following: @@ -70,7 +70,7 @@ We use [renv](https://rstudio.github.io/renv/articles/renv.html) to track depend Certain dependencies are forced to be recognized by renv without being explicitly loaded in the app by adding `library(somepackage)` to `renv_dependencies.R` -#### **4. Misc** +#### **Misc** **Long text**: Long tooltips or info text should go in the `ttip_text` object or other global objects in `static_objects.R` and then referenced in the app to keep code tidy. diff --git a/Reactive_Variables/cloud_revals.R b/Reactive_Variables/cloud_revals.R deleted file mode 100644 index 3708f2f..0000000 --- a/Reactive_Variables/cloud_revals.R +++ /dev/null @@ -1,31 +0,0 @@ -#' @details core-ms files loaded through a header parameter that points to a -#' 'folder' in minio containing all files. -corems_samples <- reactiveValues() - -observe({ - req(!is.null(header_params[['corems-uri']])) - - isolate({ - withProgress(message = "Loading core-ms files...", value = 1, { - uris <- reticulate::iterate( - minio_con$client$list_objects( - minio_con$bucket, - prefix = header_params[['corems-uri']], - recursive = TRUE), - function(x) x$object_name - ) - - fpaths <- lapply(uris, function(uri) { - mapDataAccess::get_file( - minio_con, id = uri, filename = file.path(tempfile(), basename(uri)) - ) - }) - - names(fpaths) <- sapply(fpaths, basename) - - for(name in names(fpaths)) { - corems_samples[[name]] <- read_csv(fpaths[[name]]) - } - }) - }) -}) \ No newline at end of file diff --git a/global.R b/global.R index 49a59ba..f9c31ce 100644 --- a/global.R +++ b/global.R @@ -71,6 +71,9 @@ calc_vars <- read_csv('calculation_variables.csv') %>% as.data.frame(stringsAsFa max_cells <- 2000000 info_text = list( + COREMS_UPLOAD_ERROR = "There was an error retrieving your Core-MS files: %s", + COREMS_UPLOAD_NOSAMPS = "No files found in the cloud location specified by 'corems-uri' in the URL.", + COREMS_UPLOAD_SUCCESS = "The following files were uploaded from Core-MS:
", VALID_LINKED_PLOTS = "Current valid plots to link are Van-Krevelen, Kendrick, single sample density, and custom scatter plots." ) diff --git a/server.R b/server.R index 085cec6..cbde37c 100644 --- a/server.R +++ b/server.R @@ -14,11 +14,15 @@ shinyServer(function(session, input, output) { # onStop(function() rm(revals$peakData2, pos = 1)) Sys.setenv(R_ZIPCMD="/usr/bin/zip") - # source error handling file if exists, will be a script with observers that store objects that will show up the workspace after disconnect, like so: - # observeEvent(c(objects$omicsData, objects$omicsData_2),{ - # omicsData_postmortem <<- objects$omicsData - # omicsData_2_postmortem <<- objects$omicsData_2 + # source error handling file if exists, will be a script with observers that + # store objects that will show up the workspace after disconnect, like so: + + # reactive values, including peakdata objects + # observeEvent(reactiveValuesToList(revals),{ + # revals$uploaded_emeta <- Emeta() + # revals_postmortem <<- reactiveValuesToList(revals) # }) + tryCatch({ source('untracked_resources/store_postmortem_objects.R', local = TRUE) }, error = function(e) message('Not storing postmortem objects')) @@ -28,42 +32,80 @@ shinyServer(function(session, input, output) { for (f in Sys.glob("./Reactive_variables/*.R")) source(f, local = TRUE) for (f in Sys.glob("./Observers/*.R")) source(f, local = TRUE) for (f in Sys.glob("./srv_ui_elements/*.R")) source(f, local = TRUE) - - #### Minio/cloud initialization + for (f in Sys.glob("./tab_factories/*.R")) source(f, local = TRUE) + + #'@details Store any values passed in the URL header_params = reactiveValues() - # parse header params - observe({ - query <- parseQueryString(session$clientData$url_search) - - # establish minio connection if we are pulling cloud resources - if(any(names(query) %in% VALID_MINIO_HEADER_PARAMS)) { - minio_con <<- mapDataAccess::map_data_connection("./cfg/minio_config_local.yml") - } - - isolate({ - # store header params in a reactive variable - for(key in names(query)){ - header_params[[key]] <- query[[key]] - message(sprintf("INFO: stored parameter %s: %s", key, query[[key]])) - } - }) - }) + #'@details General unorganized reactiveValues + revals <- + reactiveValues( + ntables = 0, + makeplot = 1, + color_by_choices = NULL, + axes_choices = NULL, + redraw_largedata = FALSE, + react_largedata = FALSE, + plot_data_export = NULL, + peakData_export = NULL, + redraw_filter_plot = TRUE, + reac_filter_plot = TRUE, + group_1 = NULL, + group_2 = NULL, + single_group = NULL, + single_sample = NULL, + whichSample1 = NULL, + whichSample2 = NULL, + warningmessage_upload = list(upload = "style = 'color:deepskyblue'>Upload data and molecular identification files described in 'Data Requirements' on the previous page."), + reset_counter = 0, + chooseplots = NULL, + filter_click_disable = list(init = TRUE), + peakData2 = NULL, + groups_list = list(), + removed_samples = list() + ) - revals <- reactiveValues(ntables = 0, makeplot = 1, color_by_choices = NULL, axes_choices = NULL, redraw_largedata = FALSE, react_largedata = FALSE, - plot_data_export = NULL, peakData_export = NULL, redraw_filter_plot = TRUE, reac_filter_plot = TRUE, - group_1 = NULL, group_2 = NULL, single_group = NULL, single_sample = NULL, whichSample1 = NULL, whichSample2 = NULL, - warningmessage_upload = list(upload = "style = 'color:deepskyblue'>Upload data and molecular identification files described in 'Data Requirements' on the previous page."), - reset_counter = 0, chooseplots = NULL, filter_click_disable = list(init = TRUE), peakData2 = NULL, - groups_list = list(), removed_samples = list()) + plots <- + reactiveValues( + last_plot = NULL, + plot_list = list(), + plot_data = list(), + linked_plots = list(), + plot_table = data.frame( + "File Name" = character(0), + 'Download?' = character(0), + "Plot Type" = character(0), + "Sample Type" = character(0), + "Group 1 Samples" = character(0), + "Group 2 Samples" = character(0), + "Boundary Set" = character(0), + "Color By Variable" = character(0), + "X Variable" = character(0), + "Y Variable" = character(0), + "Presence Threshold" = character(0), + "Absence Threshold" = character(0), + "P-Value" = character(0), + "Comparisons Method" = character(0), + check.names = FALSE, + stringsAsFactors = FALSE + ) + ) - plots <- reactiveValues(last_plot = NULL, plot_list = list(), plot_data = list(), linked_plots = list(), - plot_table = data.frame("File Name" = character(0), 'Download?' = character(0), "Plot Type" = character(0), "Sample Type" = character(0), "Group 1 Samples" = character(0), - "Group 2 Samples" = character(0), "Boundary Set" = character(0), "Color By Variable" = character(0), "X Variable" = character(0), - "Y Variable" = character(0), "Presence Threshold" = character(0), "Absence Threshold" = character(0), "P-Value" = character(0), - "Comparisons Method" = character(0), check.names = FALSE, stringsAsFactors = FALSE)) + tables <- + reactiveValues( + mapping_tables = list(), + saved_db_info = data.frame( + 'Tables' = character(0), + 'No. Rows' = character(0), + 'Column Names' = character(0), + check.names = F, + stringsAsFactors = F + ) + ) - tables <- reactiveValues(mapping_tables = list(), saved_db_info = data.frame('Tables' = character(0), 'No. Rows' = character(0), 'Column Names' = character(0), check.names = F, stringsAsFactors = F)) + #' @details core-ms files loaded through a header parameter that points to a + #' 'folder' in minio containing all files. + corems_samples <- reactiveValues() # Reload objects for debugging if they exist observeEvent(input$debug_reload,{ diff --git a/ui.R b/ui.R index faa36ba..91bf976 100644 --- a/ui.R +++ b/ui.R @@ -24,180 +24,32 @@ ui <- tagList(useShinyjs(), navbarPage( includeMarkdown('resources_and_contact.md') ) ), - ################## Upload Panel ####################################### - tabPanel(div("Upload", icon('upload')), value = 'Upload', - fluidRow( - ## Sidebar panel on Upload tab ## - column(width = 4, - bsCollapse(id = 'upload_collapse', open = c('file_upload'), multiple = TRUE, - bsCollapsePanel(div('Upload two linked csv files', - hidden(div(id = 'ok_files', style = 'color:deepskyblue;float:right', icon('ok', lib='glyphicon') - ) - ) - ), value = 'file_upload', - - # Load e_data file - div(id = "js_file_edata", - fileInput("file_edata", "Upload Data File (.csv)", - multiple = TRUE, - accept = c("text/csv", - "text/comma-separated-values,text/plain", - ".csv")) - ), - - ## Get unique identifier column from e_data ## - uiOutput('edata_id'), - - # Load e_meta file - div(id = "js_file_emeta", fileInput("file_emeta", "Upload Molecular Identification File (.csv)", - multiple = TRUE, - accept = c("text/csv", - "text/comma-separated-values,text/plain", - ".csv"))) - ), - bsCollapsePanel(div('Specify data structure', - hidden(div(id = 'ok_idcols', style = 'color:deepskyblue;float:right', icon('ok', lib='glyphicon') - ) - ) - ), value = 'column_info', - # Get which instrument generated the data # - inlineCSS('#js_data_scale .filter-option{text-align:center;}'), - div(id = "js_data_scale", pickerInput('data_scale', - label = 'On what scale are your data?', - choices = list('Log base 2' = 'log2', 'Log base 10'='log10', 'Natural log'='log', - 'Presence/absence' = 'pres', 'Raw intensity'='abundance'), - selected = 'abundance' - ) - ), - - tags$hr(style = "margin:20px 0px 20px 0px"), - - # Get whether formulas or elemental columns are included # - div(id = "js_select", radioGroupButtons('select', - label = 'Does this file have formulas - or elemental columns?', - choices = list('Formulas' = 1, - 'Elemental Columns' = 2), - selected = 'Select an option', justified = TRUE) - ), - - # (Conditional on the above selectInput) Formula: - ## which column contains the formula? # - conditionalPanel( - condition = "input.select == 1", - uiOutput('f_column') - ), - - # (Conditional on the above selectInput) Elemental columns: - ## which columns contain the elements? - - inlineCSS('#element_select button {width:100%}'), - hidden(div(id = "element_select", style = 'width:92.5%;margin-left:2.5%;border-radius:4px', - dropdownButton(inputId = "element_dropdown", circle = FALSE, label = "Specify Elemental Count Columns", width = '100%', - fluidRow( - column(width = 4, - uiOutput("c_column"), - uiOutput("h_column") - ), - column(width = 4, - uiOutput("n_column"), - uiOutput("o_column") - ), - column(width = 4, - uiOutput("s_column"), - uiOutput("p_column") - ) - ) - ) - )), #hidden div - - tags$hr(style = "margin:20px 0px 20px 0px"), - - # Create an option for Isotopic Analysis - div(id = "js_isotope_yn", radioGroupButtons('isotope_yn', - label = 'Were isotopic peaks identified in the molecular assignments file?', - choices = list('Yes' = 1, - 'No' = 2), - selected = 'Select an Option', justified = TRUE) - ), - # Condition on presence of isotope information - conditionalPanel( - condition = "input.isotope_yn == 1", - uiOutput("iso_info_filter_out"), - div(id = "js_iso_info_column", uiOutput('iso_info_column_out')), - div(id = "js_iso_symbol", uiOutput('iso_symbol_out')) - ) - ) - ), - tags$hr(), - - # Action button: pressing this creates the peakData object - div( - actionButton('upload_click', 'Process Data', icon = icon("cog"), lib = "glyphicon"), - hidden(div('Making data object, please wait...', id = 'upload_waiting', class='fadein-out', - style='font-weight:bold;color:deepskyblue;display:inline')) - ), - # Summary panel - hidden(div(id = 'upload_success', style = 'width:75%;margin-top:10px', - wellPanel(style='border-radius:4px', - # Show 'Success' message if peakData created successfully - uiOutput('success_upload'), - - # Number of peaks, samples, and peaks with formulas assigned - textOutput('num_peaks'), - textOutput('num_samples'), - textOutput('num_peaks_formula') - ) - )) - - ), # End sidebar panel - - column(width = 8, - # warnings panel - div(id = "warnings_upload", style = "overflow-y:auto;max-height:250px", uiOutput("warnings_upload")), - - tags$hr(), - - # Show preview of e_data - htmlOutput('edata_text'), - DTOutput("head_edata", width = "90%"), - - tags$hr(), - - # Show preview of e_meta - htmlOutput('emeta_text'), - DTOutput("head_emeta", width = "90%") - - ) # End main panel - - )), # End Upload tab - ################## Groups Panel ############################################### tabPanel(div("Groups", icon('th-large', lib = 'glyphicon')), value = 'Groups', - fluidRow(style = "display:flex;flex-direction:row;align-items:stretch", - column(4, - wellPanel(style = "height:100%", - tags$h4("Define a Group"), - div(id = "js_group_name", textInput("group_name", "Name of this group:")), - fluidRow( - column(6, uiOutput("group_samples")), - column(6, textInput("group_regex", "Search sample names")) - ), - actionButton("add_group", "Add this group"), - br(), - br(), - uiOutput("warnings_groups") - ) + fluidRow(style = "display:flex;flex-direction:row;align-items:stretch", + column(4, + wellPanel(style = "height:100%", + tags$h4("Define a Group"), + div(id = "js_group_name", textInput("group_name", "Name of this group:")), + fluidRow( + column(6, uiOutput("group_samples")), + column(6, textInput("group_regex", "Search sample names")) + ), + actionButton("add_group", "Add this group"), + br(), + br(), + uiOutput("warnings_groups") + ) + ), + column(8, + wellPanel(style = "height:100%", + dataTableOutput("group_table"), + actionButton("remove_group", "Remove selected group") + ) + ) ), - column(8, - wellPanel(style = "height:100%", - dataTableOutput("group_table"), - actionButton("remove_group", "Remove selected group") - ) - ) - ), - hr(), - actionButton("goto_preprocess_main", "Continue to preprocess tab") + hr(), + actionButton("goto_preprocess_main", "Continue to preprocess tab") ), ################## Preprocess Panel ############################################### tabPanel(div("Preprocess", icon('cogs')), value = 'Preprocess', diff --git a/www/yeti.css b/www/yeti.css index 08a2928..a8e52ed 100644 --- a/www/yeti.css +++ b/www/yeti.css @@ -7094,6 +7094,11 @@ misc. style adjustments top:5px; } +.column-scroll-sm { + overflow-y: scroll; + max-height: 500px; +} + .nowrap_scroll { white-space: nowrap; overflow-x:scroll; From 48fd3e660e07a58bc49a2129ba8a957efc4a7e2e Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Wed, 24 Nov 2021 15:24:23 -0800 Subject: [PATCH 03/13] dual github+gitlab access tokens, +lockfile: mapDataAccess and reticulate --- Dockerfile-base | 3 ++- README.md | 9 +++++++-- renv.lock | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/Dockerfile-base b/Dockerfile-base index 934f42b..c619764 100644 --- a/Dockerfile-base +++ b/Dockerfile-base @@ -14,7 +14,8 @@ COPY renv.lock . RUN R -e "install.packages('renv', repos = 'https://cran.rstudio.com')" # install all packages listed in renv.lock -RUN --mount=type=secret,id=github_pat export GITHUB_PAT="$(cat /run/secrets/github_pat)" \ +RUN --mount=type=secret,id=access_tokens set -a \ +&& . /run/secrets/access_tokens && set +a \ && R -e 'renv::restore()' ##### diff --git a/README.md b/README.md index 725e727..ad5b48b 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,15 @@ Then run the docker container: `docker run -p 3838:3838 docker pull docker.arti We build a 'base' container which has all the system libraries and R packages installed, and then build a container on top of it that simply copies the app source code and exposes the correct port. There are two Dockerfiles, and two corresponding .dockerignore files. -**To build the base container**, you must provide a github PAT in order to install package from private git repos; currently these include KeggData and MetaCycData. Generate a personal access token according to: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token. Put this token in a file next to the Dockerfile, say `.mysecret` +**To build the base container**, you must provide a github PAT AND gitlab PAT in order to install packages from private git repos to which you have access; currently these include mapDataAccess, KeggData and MetaCycData. Generate a personal access token according to: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token for github and https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html for gitlab. Put these tokens in a file next to the Dockerfile, say `.mysecrets`. It should look like: + +` +GITLAB_PAT= +GITHUB_PAT= +` Now, replacing <base tag> with whatever version, run: -`docker build -f Dockerfile-base --secret id=gitlab_pat,src=.mysecret -t docker.artifactory.pnnl.gov/mscviz/freda/base: .` +`docker build -f Dockerfile-base --secret id=access_tokens,src=.mysecrets -t docker.artifactory.pnnl.gov/mscviz/freda/base: .` **To build the 'top' container**: Simply make sure Dockerfile refers to the correct base container if you have updated any dependencies (rebuilt the base container) and run: diff --git a/renv.lock b/renv.lock index adf37de..73cd267 100644 --- a/renv.lock +++ b/renv.lock @@ -92,6 +92,18 @@ "Repository": "CRAN", "Hash": "76147821cd3fcd8c4b04e1ef0498e7fb" }, + "mapDataAccess": { + "Package": "mapDataAccess", + "Version": "0.0.0.10", + "Source": "GitLab", + "RemoteType": "gitlab", + "RemoteHost": "code.emsl.pnl.gov", + "RemoteRepo": "mapdataaccess-lib", + "RemoteUsername": "multiomics-analyses", + "RemoteRef": "HEAD", + "RemoteSha": "bca23c536aad7400af52ff1db585f7561d32ce44", + "Hash": "add0bba6dac6effce063a8a9213f335d" + }, "markdown": { "Package": "markdown", "Version": "1.1", @@ -148,6 +160,13 @@ "Repository": "CRAN", "Hash": "bb5996d0bd962d214a11140d77589917" }, + "reticulate": { + "Package": "reticulate", + "Version": "1.22", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "b34a8bb69005168078d1d546a53912b2" + }, "rlang": { "Package": "rlang", "Version": "0.4.11", From 13707a00e657c01452280033234f653f6322e5f0 Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Wed, 24 Nov 2021 15:26:16 -0800 Subject: [PATCH 04/13] track tab_factories --- .dockerignore | 1 + tab_factories/upload_tab.R | 160 +++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 tab_factories/upload_tab.R diff --git a/.dockerignore b/.dockerignore index dbe018b..d69f732 100644 --- a/.dockerignore +++ b/.dockerignore @@ -25,6 +25,7 @@ !Observers !Reactive_Variables !srv_ui_elements +!tab_factories !www !README diff --git a/tab_factories/upload_tab.R b/tab_factories/upload_tab.R new file mode 100644 index 0000000..6402973 --- /dev/null +++ b/tab_factories/upload_tab.R @@ -0,0 +1,160 @@ +#'@details Create the upload tab +#' The upload tab can differ based on whether we are loading data from core-ms +#' If we are loading from core-ms, then we have sub-tabs for processing the +#' core-ms output. +#' +upload_tab <- function(from_corems = FALSE) { + if(from_corems) { + navbarMenu("Core-MS Processing", + tabPanel("tab1", HTML("This is a tab")), + tabPanel("tab2", HTML("This is another tab")) + ) + } else { + tabPanel(div("Upload", icon('upload')), value = 'Upload', + fluidRow( + ## Sidebar panel on Upload tab ## + column(width = 4, + bsCollapse(id = 'upload_collapse', open = c('file_upload'), multiple = TRUE, + bsCollapsePanel(div('Upload two linked csv files', + hidden(div(id = 'ok_files', style = 'color:deepskyblue;float:right', icon('ok', lib='glyphicon') + ) + ) + ), value = 'file_upload', + + # Load e_data file + div(id = "js_file_edata", + fileInput("file_edata", "Upload Data File (.csv)", + multiple = TRUE, + accept = c("text/csv", + "text/comma-separated-values,text/plain", + ".csv")) + ), + + ## Get unique identifier column from e_data ## + uiOutput('edata_id'), + + # Load e_meta file + div(id = "js_file_emeta", fileInput("file_emeta", "Upload Molecular Identification File (.csv)", + multiple = TRUE, + accept = c("text/csv", + "text/comma-separated-values,text/plain", + ".csv"))) + ), + bsCollapsePanel(div('Specify data structure', + hidden(div(id = 'ok_idcols', style = 'color:deepskyblue;float:right', icon('ok', lib='glyphicon') + ) + ) + ), value = 'column_info', + # Get which instrument generated the data # + inlineCSS('#js_data_scale .filter-option{text-align:center;}'), + div(id = "js_data_scale", pickerInput('data_scale', + label = 'On what scale are your data?', + choices = list('Log base 2' = 'log2', 'Log base 10'='log10', 'Natural log'='log', + 'Presence/absence' = 'pres', 'Raw intensity'='abundance'), + selected = 'abundance' + ) + ), + + tags$hr(style = "margin:20px 0px 20px 0px"), + + # Get whether formulas or elemental columns are included # + div(id = "js_select", radioGroupButtons('select', + label = 'Does this file have formulas + or elemental columns?', + choices = list('Formulas' = 1, + 'Elemental Columns' = 2), + selected = 'Select an option', justified = TRUE) + ), + + # (Conditional on the above selectInput) Formula: + ## which column contains the formula? # + conditionalPanel( + condition = "input.select == 1", + uiOutput('f_column') + ), + + # (Conditional on the above selectInput) Elemental columns: + ## which columns contain the elements? + + inlineCSS('#element_select button {width:100%}'), + hidden(div(id = "element_select", style = 'width:92.5%;margin-left:2.5%;border-radius:4px', + dropdownButton(inputId = "element_dropdown", circle = FALSE, label = "Specify Elemental Count Columns", width = '100%', + fluidRow( + column(width = 4, + uiOutput("c_column"), + uiOutput("h_column") + ), + column(width = 4, + uiOutput("n_column"), + uiOutput("o_column") + ), + column(width = 4, + uiOutput("s_column"), + uiOutput("p_column") + ) + ) + ) + )), #hidden div + + tags$hr(style = "margin:20px 0px 20px 0px"), + + # Create an option for Isotopic Analysis + div(id = "js_isotope_yn", radioGroupButtons('isotope_yn', + label = 'Were isotopic peaks identified in the molecular assignments file?', + choices = list('Yes' = 1, + 'No' = 2), + selected = 'Select an Option', justified = TRUE) + ), + # Condition on presence of isotope information + conditionalPanel( + condition = "input.isotope_yn == 1", + uiOutput("iso_info_filter_out"), + div(id = "js_iso_info_column", uiOutput('iso_info_column_out')), + div(id = "js_iso_symbol", uiOutput('iso_symbol_out')) + ) + ) + ), + tags$hr(), + + # Action button: pressing this creates the peakData object + div( + actionButton('upload_click', 'Process Data', icon = icon("cog"), lib = "glyphicon"), + hidden(div('Making data object, please wait...', id = 'upload_waiting', class='fadein-out', + style='font-weight:bold;color:deepskyblue;display:inline')) + ), + # Summary panel + hidden(div(id = 'upload_success', style = 'width:75%;margin-top:10px', + wellPanel(style='border-radius:4px', + # Show 'Success' message if peakData created successfully + uiOutput('success_upload'), + + # Number of peaks, samples, and peaks with formulas assigned + textOutput('num_peaks'), + textOutput('num_samples'), + textOutput('num_peaks_formula') + ) + )) + + ), # End sidebar panel + + column(width = 8, + # warnings panel + div(id = "warnings_upload", style = "overflow-y:auto;max-height:250px", uiOutput("warnings_upload")), + + tags$hr(), + + # Show preview of e_data + htmlOutput('edata_text'), + DTOutput("head_edata", width = "90%"), + + tags$hr(), + + # Show preview of e_meta + htmlOutput('emeta_text'), + DTOutput("head_emeta", width = "90%") + + ) # End main panel + + )) + } +} \ No newline at end of file From 4810f2ca67f29f1e272b96b540c1001e7af0a515 Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Wed, 24 Nov 2021 15:45:26 -0800 Subject: [PATCH 05/13] update README --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ad5b48b..6cf8db1 100644 --- a/README.md +++ b/README.md @@ -52,19 +52,21 @@ We build a 'base' container which has all the system libraries and R packages in **To build the base container**, you must provide a github PAT AND gitlab PAT in order to install packages from private git repos to which you have access; currently these include mapDataAccess, KeggData and MetaCycData. Generate a personal access token according to: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token for github and https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html for gitlab. Put these tokens in a file next to the Dockerfile, say `.mysecrets`. It should look like: -` -GITLAB_PAT= +``` +GITLAB_PAT= GITHUB_PAT= -` +``` Now, replacing <base tag> with whatever version, run: `docker build -f Dockerfile-base --secret id=access_tokens,src=.mysecrets -t docker.artifactory.pnnl.gov/mscviz/freda/base: .` **To build the 'top' container**: Simply make sure Dockerfile refers to the correct base container if you have updated any dependencies (rebuilt the base container) and run: -`docker build -t docker.artifactory.pnnl.gov/mscviz/freda/ .` +`docker build -t docker.artifactory.pnnl.gov/mscviz/freda: .` -If all is well, push new containers to the registry: `docker push :` +If all is well, push new containers to the registry: +`docker push docker.artifactory.pnnl.gov/mscviz/freda/base:` +`docker push docker.artifactory.pnnl.gov/mscviz/freda:` #### **Dependencies** From 02275d11b8cf41048aa135eef810691e05c20f72 Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Thu, 16 Dec 2021 10:41:45 -0800 Subject: [PATCH 06/13] install venv in dockerfile bugfix sourcing reactive variables remove duplicate static objects --- .dockerignore | 1 - Dockerfile-base | 10 ++++++++-- Dockerfile-base.dockerignore | 1 + Observers/startup_observers.R | 10 +++++----- README.md | 13 ++++++++++++- cfg/minio_config_example.yml | 8 ++++++++ global.R | 28 ++-------------------------- server.R | 2 +- 8 files changed, 37 insertions(+), 36 deletions(-) create mode 100644 cfg/minio_config_example.yml diff --git a/.dockerignore b/.dockerignore index d69f732..6006e1d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,7 +19,6 @@ !global.R !server.R !ui.R -!static_objects.R !helper_functions !Observers diff --git a/Dockerfile-base b/Dockerfile-base index c619764..cfae471 100644 --- a/Dockerfile-base +++ b/Dockerfile-base @@ -5,7 +5,8 @@ RUN apt-get update -qq && apt-get install -y \ libssl-dev \ libcurl4-gnutls-dev \ libmagick++-dev \ - libv8-dev + libv8-dev \ + vim python3-venv WORKDIR /srv/shiny-server/ COPY renv.lock . @@ -18,7 +19,12 @@ RUN --mount=type=secret,id=access_tokens set -a \ && . /run/secrets/access_tokens && set +a \ && R -e 'renv::restore()' -##### +## Setup Python venv ## +USER root +COPY python_requirements.txt . +RUN python3 -m venv /venv +RUN /venv/bin/pip install --upgrade pip +RUN /venv/bin/pip install -r python_requirements.txt # Install all plotly's dependencies RUN R -e "install.packages('plotly', dependencies = T)" diff --git a/Dockerfile-base.dockerignore b/Dockerfile-base.dockerignore index cd12d46..b351166 100644 --- a/Dockerfile-base.dockerignore +++ b/Dockerfile-base.dockerignore @@ -1,4 +1,5 @@ * !renv.lock +!python_requirements.txt diff --git a/Observers/startup_observers.R b/Observers/startup_observers.R index becd7bd..cbdfefb 100644 --- a/Observers/startup_observers.R +++ b/Observers/startup_observers.R @@ -1,11 +1,11 @@ -#'@details Parse and store header parameters. If 'corems-uri' is passed, load +#'@details Parse and store header parameters. If 'corems-prefix' is passed, load #'the files and display a different tab for 'upload'. observe({ query <- parseQueryString(session$clientData$url_search) # establish minio connection if we are pulling cloud resources if(any(names(query) %in% VALID_MINIO_HEADER_PARAMS)) { - minio_con <<- mapDataAccess::map_data_connection("./cfg/minio_config_local.yml") + minio_con <<- mapDataAccess::map_data_connection("./cfg/minio_config.yml") } isolate({ @@ -15,12 +15,12 @@ observe({ message(sprintf("INFO: stored parameter %s: %s", key, query[[key]])) } - if('corems-uri' %in% names(query)) { + if('corems-prefix' %in% names(query)) { withProgress(message = "Loading core-ms files...", value = 1, { uris <- reticulate::iterate( minio_con$client$list_objects( minio_con$bucket, - prefix = header_params[['corems-uri']], + prefix = header_params[['corems-prefix']], recursive = TRUE), function(x) x$object_name ) @@ -63,4 +63,4 @@ observe({ ) }) -}) \ No newline at end of file +}) diff --git a/README.md b/README.md index 6cf8db1..9337c94 100644 --- a/README.md +++ b/README.md @@ -33,13 +33,24 @@ Install the required packages. You can do this either by inspecting the global. To install package with `renv`, first `install.packages("renv")`. Then call renv::restore(). This will install all packages contained in the renv.lock file. See the [renv website](https://rstudio.github.io/renv/articles/renv.html) for more details. Then simply call shiny::runApp() +**Pulling from minio** + +FREDA has the ability to pull files from minio, currently only for use in retrieving core-ms output. You must provide a config that contains several connection parameters and put it at cfg/minio_config.yml + +See the example file cfg/minio_config_example.yml + #### 2. Using docker: Either build the container as described in the development section, or pull it from pnnl artifactory if you have access: `docker login docker.artifactory.pnnl.gov` `docker pull docker.artifactory.pnnl.gov/mscviz/freda:latest` -Then run the docker container: `docker run -p 3838:3838 docker pull docker.artifactory.pnnl.gov/mscviz/freda:latest` +Then run the docker container: `docker run -p 3838:3838 docker.artifactory.pnnl.gov/mscviz/freda:latest` + +If you are pulling files from minio, you must mount a minio config to `/srv/shiny-server/FREDA/cfg/minio_config.yml`: + +`docker run -p 3838:3838 -v /path/to/config.yml:/srv/shiny-server/FREDA/cfg/minio_config.yml docker.artifactory.pnnl.gov/mscviz/freda:latest` + ... and navigate to https://127.0.0.1:3838 in your web browser. *** diff --git a/cfg/minio_config_example.yml b/cfg/minio_config_example.yml new file mode 100644 index 0000000..46567fd --- /dev/null +++ b/cfg/minio_config_example.yml @@ -0,0 +1,8 @@ +type: minio +endpoint: localhost:9000 +access_key: minioadmin +secret_key: minioadmin +bucket: map +secure: False +python_venv: /venv +verbose: true \ No newline at end of file diff --git a/global.R b/global.R index f9c31ce..8f99a9d 100644 --- a/global.R +++ b/global.R @@ -46,36 +46,12 @@ calc_vars <- read_csv('calculation_variables.csv') %>% as.data.frame(stringsAsFa # determines when 'large data' options are triggered max_cells <- 2000000 -info_text = list( - VALID_LINKED_PLOTS = "Current valid plots to link are Van-Krevelen, Kendrick, single sample density, and custom scatter plots." -) - -# static variables # -dt_checkmark <- '' -dt_minus <- '' - -dbe_opts_info <- 'Semicolon separated strings of elements and their valences, i.e. to calculate the dbe for two sets of valences, input C4H1N3O2S2P3;C3H3N2O2S4P4' -kendrick_opts_info <- 'The base compound(s) used to calculate the Kendrick Mass. See [Foquet and Sato, 2017] for details.' - -dt_checkmark <- '' -dt_minus <- '' - -ttip_text = list("plot_save"="Save the last created plot", "plot_review"="Review saved plots", "page_help"="How do I use this page?") - -#------ Download Example Data ---------# -example_edata <- read_csv('Data/example12T_edata.csv') %>% as.data.frame(stringsAsFactors = FALSE) -example_emeta <- read_csv('Data/example12T_emeta.csv') %>% as.data.frame(stringsAsFactors = FALSE) -calc_opts <- read_csv('calculation_options.csv') %>% as.data.frame(stringsAsFactors = FALSE) -calc_vars <- read_csv('calculation_variables.csv') %>% as.data.frame(stringsAsFactors = FALSE) -# determines when 'large data' options are triggered -max_cells <- 2000000 - info_text = list( COREMS_UPLOAD_ERROR = "There was an error retrieving your Core-MS files: %s", - COREMS_UPLOAD_NOSAMPS = "No files found in the cloud location specified by 'corems-uri' in the URL.", + COREMS_UPLOAD_NOSAMPS = "No files found in the cloud location specified by 'corems-prefix' in the URL.", COREMS_UPLOAD_SUCCESS = "The following files were uploaded from Core-MS:
", VALID_LINKED_PLOTS = "Current valid plots to link are Van-Krevelen, Kendrick, single sample density, and custom scatter plots." ) # cloud/minio resources -VALID_MINIO_HEADER_PARAMS = c("corems-uri") +VALID_MINIO_HEADER_PARAMS = c("corems-prefix") diff --git a/server.R b/server.R index cbde37c..898e61a 100644 --- a/server.R +++ b/server.R @@ -29,7 +29,7 @@ shinyServer(function(session, input, output) { # Source all scripts for (f in Sys.glob("./helper_functions/*.R")) source(f, local = TRUE) - for (f in Sys.glob("./Reactive_variables/*.R")) source(f, local = TRUE) + for (f in Sys.glob("./Reactive_Variables/*.R")) source(f, local = TRUE) for (f in Sys.glob("./Observers/*.R")) source(f, local = TRUE) for (f in Sys.glob("./srv_ui_elements/*.R")) source(f, local = TRUE) for (f in Sys.glob("./tab_factories/*.R")) source(f, local = TRUE) From d4cc4fce401b5de85a3babfb1dc1d70b05de2849 Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Thu, 13 Jan 2022 11:30:34 -0800 Subject: [PATCH 07/13] dockerfile base tag build arg, readme update for minio testing --- Dockerfile | 10 +++++++++- README.md | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2a77756..ef161ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,12 @@ -FROM docker.artifactory.pnnl.gov/mscviz/freda/base +# Builds off the FREDA base container that contains all dependencies for the +# app. Mostly just copies in app source code and server configuration. + +# Change this when we bump versions, or if you have some test version of the +# base container you can specify --build-arg base_tag= in docker +# build. +ARG base_tag=1.0.4 + +FROM docker.artifactory.pnnl.gov/mscviz/freda/base:$base_tag # All app source/resources COPY . /srv/shiny-server/FREDA diff --git a/README.md b/README.md index 9337c94..3e8fcde 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,22 @@ FREDA has the ability to pull files from minio, currently only for use in retrie See the example file cfg/minio_config_example.yml +To test pulling from minio locally, you must run a local minio docker container. With docker installed on your machine, +run the following: + +`docker pull minio/minio` +`docker run -d -p 9000:9000 --name minio-map minio/minio server /data` + +Now navigate to http://localhost:9000/, which will display a UI where you can create folders and upload files. As an +example, do the following: + +1. Create a folder in the minio UI (call it test_folder, for example) and put a couple csv files in it. +2. Launch FREDA from Rstudio. +3. Nagivate to wherever FREDA is being served at, adding /?corems-prefix=test_folder to the url. + +FREDA will attempt to read all the files in the minio folder `test_folder` and load them into the reactiveValue +corems_samples. + #### 2. Using docker: Either build the container as described in the development section, or pull it from pnnl artifactory if you have access: From a341eee1e23ea9af106c35fc002ee1032b4ecd2b Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Wed, 16 Feb 2022 16:14:49 -0800 Subject: [PATCH 08/13] adding ability to ignore SHINYPROXY_USERNAME (directory) when retrieving corems files --- Observers/startup_observers.R | 3 ++- renv.lock | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Observers/startup_observers.R b/Observers/startup_observers.R index cbdfefb..0e5474e 100644 --- a/Observers/startup_observers.R +++ b/Observers/startup_observers.R @@ -29,7 +29,8 @@ observe({ tryCatch({ fpaths <- lapply(uris, function(uri) { mapDataAccess::get_file( - minio_con, id = uri, filename = file.path(tempfile(), basename(uri)) + minio_con, id = uri, filename = file.path(tempfile(), basename(uri)), + use_dir = FALSE ) }) diff --git a/renv.lock b/renv.lock index 73cd267..cc0ac63 100644 --- a/renv.lock +++ b/renv.lock @@ -101,8 +101,8 @@ "RemoteRepo": "mapdataaccess-lib", "RemoteUsername": "multiomics-analyses", "RemoteRef": "HEAD", - "RemoteSha": "bca23c536aad7400af52ff1db585f7561d32ce44", - "Hash": "add0bba6dac6effce063a8a9213f335d" + "RemoteSha": "12e6bd1e85a406a0d28b636f8c23754d81745ac5", + "Hash": "16970b5f8a17a3d5e2dc74911b59c0b2" }, "markdown": { "Package": "markdown", From 987176880d77cf1af9be9f2f3a4eca5bfd71f4cd Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Fri, 18 Mar 2022 09:39:41 -0700 Subject: [PATCH 09/13] update dependencies for report --- peakData_Report.Rmd | 2 +- renv.lock | 29 +++++++++++++++++++++++++---- renv_dependencies.R | 6 +++++- tag-and-push.sh | 3 +++ 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/peakData_Report.Rmd b/peakData_Report.Rmd index 6c421fb..8b1f78b 100644 --- a/peakData_Report.Rmd +++ b/peakData_Report.Rmd @@ -41,7 +41,7 @@ panderOptions('knitr.auto.asis', FALSE) calc_cols <- read.csv("calculation_variables.csv", stringsAsFactors = FALSE) ``` -###**Input Data**### +### **Input Data** Your data started with **`r nrow(upload$f_data)` samples** and measurements for **`r nrow(upload$e_data)` peaks**. diff --git a/renv.lock b/renv.lock index cc0ac63..26aba47 100644 --- a/renv.lock +++ b/renv.lock @@ -92,6 +92,20 @@ "Repository": "CRAN", "Hash": "76147821cd3fcd8c4b04e1ef0498e7fb" }, + "kableExtra": { + "Package": "kableExtra", + "Version": "1.3.4", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "49b625e6aabe4c5f091f5850aba8ff78" + }, + "knitr": { + "Package": "knitr", + "Version": "1.37", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "a4ec675eb332a33fe7b7fe26f70e1f98" + }, "mapDataAccess": { "Package": "mapDataAccess", "Version": "0.0.0.10", @@ -169,17 +183,17 @@ }, "rlang": { "Package": "rlang", - "Version": "0.4.11", + "Version": "1.0.2", "Source": "Repository", "Repository": "CRAN", - "Hash": "515f341d3affe0de9e4a7f762efb0456" + "Hash": "04884d9a75d778aca22c7154b8333ec9" }, "rmarkdown": { "Package": "rmarkdown", - "Version": "2.11", + "Version": "2.13", "Source": "Repository", "Repository": "CRAN", - "Hash": "320017b52d05a943981272b295750388" + "Hash": "ac78f4d2e0289d4cba73b88af567b8b1" }, "scales": { "Package": "scales", @@ -264,6 +278,13 @@ "Source": "Repository", "Repository": "CRAN", "Hash": "ad03909b44677f930fa156d47d7a3aeb" + }, + "xfun": { + "Package": "xfun", + "Version": "0.30", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "e83f48136b041845e50a6658feffb197" } } } diff --git a/renv_dependencies.R b/renv_dependencies.R index 5778278..2c14138 100644 --- a/renv_dependencies.R +++ b/renv_dependencies.R @@ -3,5 +3,9 @@ #'For example datadr is only suggested by ftmsRanalysis, but is called in the #'app by one of ftmsRanalysis' functions. #' +#'Other libraries need to be included so that the correct version installs. For +#'some reason 'xfun' is not updated to the correct version for rmarkdown. -library(datadr) \ No newline at end of file +library(datadr) +library(xfun) +library(kableExtra) diff --git a/tag-and-push.sh b/tag-and-push.sh index ccdc052..31784ce 100644 --- a/tag-and-push.sh +++ b/tag-and-push.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Usage: +# tag-and-push.sh -n -t (optional -l) + TAG_LATEST="false" while getopts ":n:t:l;" opt; do From cef3d13f17e5807cff563b1c5aec490d5ee22a87 Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Mon, 21 Mar 2022 10:21:53 -0700 Subject: [PATCH 10/13] bump version --- ui.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui.R b/ui.R index 91bf976..3a8b23c 100644 --- a/ui.R +++ b/ui.R @@ -1,7 +1,7 @@ # Define UI and wrap everything in a taglist that first calls useShinyjs() ui <- tagList(useShinyjs(), navbarPage( - title = tags$div("FREDA", tags$span(style = "font-size:small", "v1.0.4")), + title = tags$div("FREDA", tags$span(style = "font-size:small", "v1.0.6")), windowTitle = 'FREDA', id = "top_page", theme = "yeti.css", From 28b19643bc4833925785578d8fe4d807a77334d9 Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Fri, 3 Jun 2022 09:40:52 -0700 Subject: [PATCH 11/13] fix name collision with filter reset reactivevalue --- Observers/filter_observers.R | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Observers/filter_observers.R b/Observers/filter_observers.R index c4d9bab..b8af307 100644 --- a/Observers/filter_observers.R +++ b/Observers/filter_observers.R @@ -384,14 +384,14 @@ observe({ }) # ----- Filter Reset Setup -----# -# f$clearFilters simply allows/denies the destruction of filters and plots, and the rest of data to pre-filtered state. -f <- reactiveValues(clearFilters = FALSE) +# clear_filters$clearFilters simply allows/denies the destruction of filters and plots, and the rest of data to pre-filtered state. +clear_filters <- reactiveValues(clearFilters = FALSE) observeEvent(input$clear_filters_yes, { - f$clearFilters <- TRUE + clear_filters$clearFilters <- TRUE }, priority = 10) observeEvent(input$filter_click, { - f$clearFilters <- FALSE + clear_filters$clearFilters <- FALSE }, priority = 10) #-------- Reset Activity -------# @@ -447,7 +447,7 @@ observeEvent(input$clear_filters_no,{ # if they click yes, reset data and exit modal dialog observeEvent(input$clear_filters_yes, { - if (f$clearFilters) { + if (clear_filters$clearFilters) { updateCheckboxInput(session, inputId = "massfilter", value = FALSE) updateCheckboxInput(session, inputId = "molfilter", value = FALSE) updateCheckboxInput(session, inputId = "formfilter", value = FALSE) From 039b23a66a1cdddff1b14dc63a9aad83f0e1dac8 Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Fri, 3 Jun 2022 10:28:32 -0700 Subject: [PATCH 12/13] Option to NA replace non-NA values --- Observers/upload_observers.R | 4 ++++ srv_ui_elements/upload_UI_sidebar.R | 9 +++++++++ tab_factories/upload_tab.R | 6 +++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Observers/upload_observers.R b/Observers/upload_observers.R index 6933e56..766b5ed 100644 --- a/Observers/upload_observers.R +++ b/Observers/upload_observers.R @@ -136,6 +136,10 @@ observeEvent(input$upload_click, { check_rows = TRUE, data_scale = input$data_scale) } # End C13 / no C13 if statement + + if (input$NA_value != "NA"){ + res <- edata_replace(res, input$NA_value, NA) + } }, error = function(e){ msg = paste0('Error making your peakData: \n System error: ', e) diff --git a/srv_ui_elements/upload_UI_sidebar.R b/srv_ui_elements/upload_UI_sidebar.R index f4507dd..17b157f 100644 --- a/srv_ui_elements/upload_UI_sidebar.R +++ b/srv_ui_elements/upload_UI_sidebar.R @@ -6,6 +6,15 @@ list( choices = c('Select one', edata_cnames())) }), # End edata_id # + output$NA_value_UI <- renderUI({ + validate(need(Edata(), "Upload data file")) + + n_zeros = sum(Edata() == 0) + prop_zeros = n_zeros/(prod(dim(Edata())) - nrow(Edata())) + value = if(prop_zeros > 0.1) "0" else "NA" + textInput("NA_value", "What value specifies missing data?", value = value) + }), + # Drop-down lists: Choose formula column output$f_column <- renderUI({ selectInput("f_column", "Choose formula column", diff --git a/tab_factories/upload_tab.R b/tab_factories/upload_tab.R index 6402973..cc5a783 100644 --- a/tab_factories/upload_tab.R +++ b/tab_factories/upload_tab.R @@ -52,7 +52,11 @@ upload_tab <- function(from_corems = FALSE) { choices = list('Log base 2' = 'log2', 'Log base 10'='log10', 'Natural log'='log', 'Presence/absence' = 'pres', 'Raw intensity'='abundance'), selected = 'abundance' - ) + ) + ), + div( + id = "js_NA_value", + uiOutput("NA_value_UI") ), tags$hr(style = "margin:20px 0px 20px 0px"), From 700fb7f3466768a3eae0c5b3afd9a1b66b976d7e Mon Sep 17 00:00:00 2001 From: Daniel Claborne Date: Fri, 3 Jun 2022 10:31:21 -0700 Subject: [PATCH 13/13] bump v1.0.7 --- ui.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui.R b/ui.R index 3a8b23c..57c1112 100644 --- a/ui.R +++ b/ui.R @@ -1,7 +1,7 @@ # Define UI and wrap everything in a taglist that first calls useShinyjs() ui <- tagList(useShinyjs(), navbarPage( - title = tags$div("FREDA", tags$span(style = "font-size:small", "v1.0.6")), + title = tags$div("FREDA", tags$span(style = "font-size:small", "v1.0.7")), windowTitle = 'FREDA', id = "top_page", theme = "yeti.css",