Merge pull request #4 from EMSL-Computing/develop

Develop into master

.dockerignore

@@ -19,12 +19,12 @@
 !global.R
 !server.R
 !ui.R
-!static_objects.R
 
 !helper_functions
 !Observers
 !Reactive_Variables
 !srv_ui_elements
+!tab_factories
 !www
 !README
 

Dockerfile

@@ -1,4 +1,12 @@
-FROM docker.artifactory.pnnl.gov/mscviz/freda/base
+# Builds off the FREDA base container that contains all dependencies for the
+# app.  Mostly just copies in app source code and server configuration.
+
+# Change this when we bump versions, or if you have some test version of the
+# base container you can specify --build-arg base_tag=<yourtag> in docker 
+# build.
+ARG base_tag=1.0.4
+
+FROM docker.artifactory.pnnl.gov/mscviz/freda/base:$base_tag
 
 # All app source/resources
 COPY . /srv/shiny-server/FREDA

Dockerfile-base

@@ -5,7 +5,8 @@ RUN apt-get update -qq && apt-get install -y \
   libssl-dev \
   libcurl4-gnutls-dev \
   libmagick++-dev \
-  libv8-dev
+  libv8-dev \
+  vim python3-venv
 
 WORKDIR /srv/shiny-server/
 COPY renv.lock .
@@ -14,10 +15,16 @@ COPY renv.lock .
 RUN R -e "install.packages('renv', repos = 'https://cran.rstudio.com')"
 
 # install all packages listed in renv.lock
-RUN --mount=type=secret,id=github_pat export GITHUB_PAT="$(cat /run/secrets/github_pat)" \
+RUN --mount=type=secret,id=access_tokens set -a \
+&& . /run/secrets/access_tokens && set +a \
 && R -e 'renv::restore()'
 
-#####
+## Setup Python venv ##
+USER root
+COPY python_requirements.txt .
+RUN python3 -m venv /venv
+RUN /venv/bin/pip install --upgrade pip
+RUN /venv/bin/pip install -r python_requirements.txt
 
 # Install all plotly's dependencies
 RUN R -e "install.packages('plotly', dependencies = T)"

Dockerfile-base.dockerignore

@@ -1,4 +1,5 @@
 *
 
 !renv.lock
+!python_requirements.txt
 

Observers/filter_observers.R

@@ -384,14 +384,14 @@ observe({
 })
 
 # ----- Filter Reset Setup -----# 
-# f$clearFilters simply allows/denies the destruction of filters and plots, and the rest of data to pre-filtered state.
-f <- reactiveValues(clearFilters = FALSE)
+# clear_filters$clearFilters simply allows/denies the destruction of filters and plots, and the rest of data to pre-filtered state.
+clear_filters <- reactiveValues(clearFilters = FALSE)
 observeEvent(input$clear_filters_yes, {
-  f$clearFilters <- TRUE
+  clear_filters$clearFilters <- TRUE
 }, priority = 10)
 
 observeEvent(input$filter_click, {
-  f$clearFilters <- FALSE
+  clear_filters$clearFilters <- FALSE
 }, priority = 10)
 
 #-------- Reset Activity -------#
@@ -447,7 +447,7 @@ observeEvent(input$clear_filters_no,{
 
 # if they click yes, reset data and exit modal dialog
 observeEvent(input$clear_filters_yes, {
-  if (f$clearFilters) {
+  if (clear_filters$clearFilters) {
     updateCheckboxInput(session, inputId = "massfilter", value = FALSE)
     updateCheckboxInput(session, inputId = "molfilter", value = FALSE)
     updateCheckboxInput(session, inputId = "formfilter", value = FALSE)

Observers/startup_observers.R

@@ -0,0 +1,67 @@
+#'@details Parse and store header parameters.  If 'corems-prefix' is passed, load 
+#'the files and display a different tab for 'upload'.
+observe({
+  query <- parseQueryString(session$clientData$url_search)
+
+  # establish minio connection if we are pulling cloud resources
+  if(any(names(query) %in% VALID_MINIO_HEADER_PARAMS)) {
+    minio_con <<- mapDataAccess::map_data_connection("./cfg/minio_config.yml")
+  }
+
+  isolate({
+    # store header params in a reactive variable
+    for(key in names(query)){
+      header_params[[key]] <- query[[key]]
+      message(sprintf("INFO: stored parameter %s: %s", key, query[[key]]))
+    }
+
+    if('corems-prefix' %in% names(query)) {
+      withProgress(message = "Loading core-ms files...", value = 1, {
+        uris <- reticulate::iterate(
+          minio_con$client$list_objects(
+            minio_con$bucket,
+            prefix = header_params[['corems-prefix']], 
+            recursive = TRUE),
+          function(x) x$object_name
+        )
+
+        if(length(uris) > 0) {
+          tryCatch({
+            fpaths <- lapply(uris, function(uri) {
+              mapDataAccess::get_file(
+                minio_con, id = uri, filename = file.path(tempfile(), basename(uri)), 
+                use_dir = FALSE
+              )
+            })
+
+            names(fpaths) <- sapply(fpaths, basename)
+
+            for(name in names(fpaths)) {
+              corems_samples[[name]] <- read_csv(fpaths[[name]])
+            }
+
+            modalmessage <- div(class = "column-scroll-sm",
+              HTML(info_text[["COREMS_UPLOAD_SUCCESS"]]), 
+              HTML(paste(names(fpaths), collapse = "<br>"))
+            )
+          }, error = function(e) {
+            modalmessage <<- div(sprintf(info_text[["COREMS_UPLOAD_ERROR"]], e))
+          })
+        } else {
+          modalmessage <- div(info_text[["COREMS_UPLOAD_NOSAMPS"]])
+        }
+
+        showModal(modalDialog(modalmessage, title = "Core-MS Upload"))
+
+      })
+    }
+
+    insertTab(
+      "top_page",
+      target = "Welcome",
+      tab = upload_tab(length(names(corems_samples)) > 0),
+      position = "after"  
+    )
+
+  })
+})

Observers/upload_observers.R

@@ -136,6 +136,10 @@ observeEvent(input$upload_click, {
                            check_rows = TRUE, data_scale = input$data_scale)
 
       } # End C13 / no C13 if statement
+
+      if (input$NA_value != "NA"){
+        res <- edata_replace(res, input$NA_value, NA) 
+      }
     },
     error = function(e){
       msg = paste0('Error making your peakData: \n System error: ', e)

README.md

@@ -33,35 +33,69 @@ Install the required packages.  You can do this either by inspecting the global.
 To install package with `renv`, first `install.packages("renv")`.  Then call renv::restore().  This will install all packages contained in the renv.lock file.  See the [renv website](https://rstudio.github.io/renv/articles/renv.html) for more details.
 Then simply call shiny::runApp()
 
+**Pulling from minio**
+
+FREDA has the ability to pull files from minio, currently only for use in retrieving core-ms output.  You must provide a config that contains several connection parameters and put it at cfg/minio_config.yml
+
+See the example file cfg/minio_config_example.yml
+
+To test pulling from minio locally, you must run a local minio docker container.  With docker installed on your machine,
+run the following:
+
+`docker pull minio/minio`  
+`docker run -d -p 9000:9000 --name minio-map minio/minio server /data`
+
+Now navigate to http://localhost:9000/, which will display a UI where you can create folders and upload files.  As an
+example, do the following:
+
+1.  Create a folder in the minio UI (call it test_folder, for example) and put a couple csv files in it.
+2.  Launch FREDA from Rstudio.  
+3.  Nagivate to wherever FREDA is being served at, adding /?corems-prefix=test_folder to the url.
+
+FREDA will attempt to read all the files in the minio folder `test_folder` and load them into the reactiveValue 
+corems_samples.
+
 #### 2.  Using docker:
 
 Either build the container as described in the development section, or pull it from pnnl artifactory if you have access:
 `docker login docker.artifactory.pnnl.gov`
 `docker pull docker.artifactory.pnnl.gov/mscviz/freda:latest`
 
-Then run the docker container:  `docker run -p 3838:3838 docker pull docker.artifactory.pnnl.gov/mscviz/freda:latest`  
+Then run the docker container:  `docker run -p 3838:3838 docker.artifactory.pnnl.gov/mscviz/freda:latest`  
+
+If you are pulling files from minio, you must mount a minio config to `/srv/shiny-server/FREDA/cfg/minio_config.yml`:
+
+`docker run -p 3838:3838 -v /path/to/config.yml:/srv/shiny-server/FREDA/cfg/minio_config.yml docker.artifactory.pnnl.gov/mscviz/freda:latest`  
+
 ... and navigate to https://127.0.0.1:3838 in your web browser.
 
 ***
 
 ### **Development**
 
-#### **1. Dockerfiles**
+#### **Dockerfiles**
 
 We build a 'base' container which has all the system libraries and R packages installed, and then build a container on top of it that simply copies the app source code and exposes the correct port.  There are two Dockerfiles, and two corresponding .dockerignore files.
 
-**To build the base container**, you must provide a github PAT in order to install package from private git repos; currently these include KeggData and MetaCycData. Generate a personal access token according to:  https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token.  Put this token in a file next to the Dockerfile, say `.mysecret`  
+**To build the base container**, you must provide a github PAT AND gitlab PAT in order to install packages from private git repos to which you have access; currently these include mapDataAccess, KeggData and MetaCycData. Generate a personal access token according to:  https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token for github and https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html for gitlab.  Put these tokens in a file next to the Dockerfile, say `.mysecrets`.  It should look like:
+
+```
+GITLAB_PAT=<your gitlab pat>  
+GITHUB_PAT=<your github pat>
+```
 
 Now, replacing &lt;base tag&gt; with whatever version, run:  
-`docker build -f Dockerfile-base --secret id=gitlab_pat,src=.mysecret -t docker.artifactory.pnnl.gov/mscviz/freda/base:<base tag> .`
+`docker build -f Dockerfile-base --secret id=access_tokens,src=.mysecrets -t docker.artifactory.pnnl.gov/mscviz/freda/base:<base tag> .`
 
 **To build the 'top' container**:  
 Simply make sure Dockerfile refers to the correct base container if you have updated any dependencies (rebuilt the base container) and run:  
-`docker build -t docker.artifactory.pnnl.gov/mscviz/freda/<top tag> .`
+`docker build -t docker.artifactory.pnnl.gov/mscviz/freda:<top tag> .`
 
-If all is well, push new containers to the registry:  `docker push <container_name>:<tag>`
+If all is well, push new containers to the registry:  
+`docker push docker.artifactory.pnnl.gov/mscviz/freda/base:<base tag>`  
+`docker push docker.artifactory.pnnl.gov/mscviz/freda:<top tag>`
 
-#### **2. Dependencies**
+#### **Dependencies**
 
 We use [renv](https://rstudio.github.io/renv/articles/renv.html) to track dependencies.  The renv.lock file contains a list of dependencies and various details about them.  We use renv to manage the details about dependencies.  When updating the lockfile, we will do the following:
 
@@ -70,7 +104,7 @@ We use [renv](https://rstudio.github.io/renv/articles/renv.html) to track depend
 
 Certain dependencies are forced to be recognized by renv without being explicitly loaded in the app by adding `library(somepackage)` to `renv_dependencies.R`
 
-#### **4. Misc**
+#### **Misc**
 
 **Long text**:  Long tooltips or info text should go in the `ttip_text` object or other global objects in `static_objects.R` and then referenced in the app to keep code tidy.
 

cfg/minio_config_example.yml

@@ -0,0 +1,8 @@
+type: minio
+endpoint: localhost:9000 
+access_key: minioadmin
+secret_key: minioadmin
+bucket: map
+secure: False
+python_venv: /venv
+verbose: true

global.R

@@ -18,11 +18,40 @@ library(readr)
 library(plotly)
 library(DT)
 library(shinycssloaders)
+library(mapDataAccess)
 
 # uncomment either library() or load_all() if you need to load kegg
 library(KeggData)
 library(MetaCycData)
 # devtools::load_all('~/Documents/git_repos/MetaCycData/')
 # devtools::load_all('~/Documents/git_repos/KeggData/')
 
-source("static_objects.R", local=TRUE)
+# static variables #
+dt_checkmark <- '<span class="glyphicon glyphicon-ok" style="color:deepskyblue"></span>'
+dt_minus <- '<span class="glyphicon glyphicon-minus"></span>'
+
+dbe_opts_info <- 'Semicolon separated strings of elements and their valences, i.e. to calculate the dbe for two sets of valences, input C4H1N3O2S2P3;C3H3N2O2S4P4'
+kendrick_opts_info <- 'The base compound(s) used to calculate the Kendrick Mass.  See <a href = ""https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5447562/"">[Foquet and Sato, 2017]</a> for details.'
+
+dt_checkmark <- '<span class="glyphicon glyphicon-ok" style="color:deepskyblue"></span>'
+dt_minus <- '<span class="glyphicon glyphicon-minus"></span>'
+
+ttip_text = list("plot_save"="Save the last created plot", "plot_review"="Review saved plots", "page_help"="How do I use this page?")
+
+#------ Download Example Data ---------#
+example_edata <- read_csv('Data/example12T_edata.csv') %>% as.data.frame(stringsAsFactors = FALSE)
+example_emeta <- read_csv('Data/example12T_emeta.csv') %>% as.data.frame(stringsAsFactors = FALSE)
+calc_opts <- read_csv('calculation_options.csv') %>% as.data.frame(stringsAsFactors = FALSE)
+calc_vars <- read_csv('calculation_variables.csv') %>% as.data.frame(stringsAsFactors = FALSE)
+# determines when 'large data' options are triggered
+max_cells <- 2000000
+
+info_text = list(
+  COREMS_UPLOAD_ERROR = "There was an error retrieving your Core-MS files: %s",
+  COREMS_UPLOAD_NOSAMPS = "No files found in the cloud location specified by 'corems-prefix' in the URL.",
+  COREMS_UPLOAD_SUCCESS = "The following files were uploaded from Core-MS: <br>",
+  VALID_LINKED_PLOTS = "Current valid plots to link are Van-Krevelen, Kendrick, single sample density, and custom scatter plots."
+)
+
+# cloud/minio resources
+VALID_MINIO_HEADER_PARAMS = c("corems-prefix")

peakData_Report.Rmd

@@ -41,7 +41,7 @@ panderOptions('knitr.auto.asis', FALSE)
 calc_cols <- read.csv("calculation_variables.csv", stringsAsFactors = FALSE)
 ```
 
-###**Input Data**###
+### **Input Data**
 
 Your data started with **`r nrow(upload$f_data)` samples** and measurements for **`r nrow(upload$e_data)` peaks**.
 

python_requirements.txt

@@ -0,0 +1 @@
+minio==7.0.2

renv.lock

@@ -92,6 +92,32 @@
       "Repository": "CRAN",
       "Hash": "76147821cd3fcd8c4b04e1ef0498e7fb"
     },
+    "kableExtra": {
+      "Package": "kableExtra",
+      "Version": "1.3.4",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "49b625e6aabe4c5f091f5850aba8ff78"
+    },
+    "knitr": {
+      "Package": "knitr",
+      "Version": "1.37",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "a4ec675eb332a33fe7b7fe26f70e1f98"
+    },
+    "mapDataAccess": {
+      "Package": "mapDataAccess",
+      "Version": "0.0.0.10",
+      "Source": "GitLab",
+      "RemoteType": "gitlab",
+      "RemoteHost": "code.emsl.pnl.gov",
+      "RemoteRepo": "mapdataaccess-lib",
+      "RemoteUsername": "multiomics-analyses",
+      "RemoteRef": "HEAD",
+      "RemoteSha": "12e6bd1e85a406a0d28b636f8c23754d81745ac5",
+      "Hash": "16970b5f8a17a3d5e2dc74911b59c0b2"
+    },
     "markdown": {
       "Package": "markdown",
       "Version": "1.1",
@@ -148,19 +174,26 @@
       "Repository": "CRAN",
       "Hash": "bb5996d0bd962d214a11140d77589917"
     },
+    "reticulate": {
+      "Package": "reticulate",
+      "Version": "1.22",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "b34a8bb69005168078d1d546a53912b2"
+    },
     "rlang": {
       "Package": "rlang",
-      "Version": "0.4.11",
+      "Version": "1.0.2",
       "Source": "Repository",
       "Repository": "CRAN",
-      "Hash": "515f341d3affe0de9e4a7f762efb0456"
+      "Hash": "04884d9a75d778aca22c7154b8333ec9"
     },
     "rmarkdown": {
       "Package": "rmarkdown",
-      "Version": "2.11",
+      "Version": "2.13",
       "Source": "Repository",
       "Repository": "CRAN",
-      "Hash": "320017b52d05a943981272b295750388"
+      "Hash": "ac78f4d2e0289d4cba73b88af567b8b1"
     },
     "scales": {
       "Package": "scales",
@@ -245,6 +278,13 @@
       "Source": "Repository",
       "Repository": "CRAN",
       "Hash": "ad03909b44677f930fa156d47d7a3aeb"
+    },
+    "xfun": {
+      "Package": "xfun",
+      "Version": "0.30",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "e83f48136b041845e50a6658feffb197"
     }
   }
 }