From d1dc4835e92c669fbbac46e3117ad27b203b169f Mon Sep 17 00:00:00 2001 From: Pol Alvarez Date: Thu, 21 Jul 2022 17:04:50 +0200 Subject: [PATCH] removed v1 references --- .../R/gem2s-1-download_user_files.R | 49 ++----------------- pipeline-runner/data-raw/sysdata.R | 4 +- pipeline-runner/init.R | 21 +++----- 3 files changed, 11 insertions(+), 63 deletions(-) diff --git a/pipeline-runner/R/gem2s-1-download_user_files.R b/pipeline-runner/R/gem2s-1-download_user_files.R index 81f9c2e5..4d566714 100644 --- a/pipeline-runner/R/gem2s-1-download_user_files.R +++ b/pipeline-runner/R/gem2s-1-download_user_files.R @@ -12,45 +12,6 @@ download_and_store <- function(bucket, key, file_path, s3) { writeBin(body, con = file_path) } -#' Download user files from S3 -#' -#' @param input The input params received from the api -#' @param originals_bucket The bucket where the sample files are -#' @param input_dir A string, where the s3 object is going to be stored -#' @param s3 S3 aws client -#' -#' @export -#' -get_gem2s_file_v1 <- function(input, originals_bucket, input_dir, s3) { - project_id <- input$projectId - sample_ids <- unlist(input$sampleIds) - sample_ids <- sort(sample_ids) - - - unlink(input_dir, recursive = TRUE) - - for (sample_id in sample_ids) { - message("\nSample --> ", sample_id) - - res <- s3$list_objects( - Bucket = originals_bucket, - Prefix = file.path(project_id, sample_id) - ) - - keys <- unlist(lapply(res$Contents, `[[`, "Key")) - - for (gem_key in keys) { - message("GEM key: ", gem_key) - fname <- basename(gem_key) - - # Preparing directories - local_fpath <- file.path(input_dir, sample_id, fname) - - download_and_store(originals_bucket, gem_key, local_fpath, s3) - } - } -} - #' Download user files from S3 #' #' @param input The input params received from the api @@ -61,7 +22,7 @@ get_gem2s_file_v1 <- function(input, originals_bucket, input_dir, s3) { #' #' @export #' -get_gem2s_file_v2 <- function(input, originals_bucket, input_dir, s3) { +get_gem2s_file <- function(input, originals_bucket, input_dir, s3) { project_id <- input$projectId sample_ids <- input$sampleIds sample_s3_paths <- input$sampleS3Paths @@ -73,7 +34,7 @@ get_gem2s_file_v2 <- function(input, originals_bucket, input_dir, s3) { for (file_type in file_types_by_technology[[technology]]) { s3_path <- sample_s3_paths[[sample_id]][[file_type]] - local_fpath <- file.path(input_dir, sample_id, file_names_v1[[file_type]]) + local_fpath <- file.path(input_dir, sample_id, file_names[[file_type]]) download_and_store(originals_bucket, s3_path, local_fpath, s3) } } @@ -92,11 +53,7 @@ get_gem2s_file_v2 <- function(input, originals_bucket, input_dir, s3) { download_user_files <- function(input, pipeline_config, prev_out = list(), input_dir= "/input") { s3 <- paws::s3(config = pipeline_config$aws_config) - if (input$apiVersion == "v1") { - get_gem2s_file_v1(input, pipeline_config$originals_bucket, input_dir, s3) - } else if (input$apiVersion == "v2") { - get_gem2s_file_v2(input, pipeline_config$originals_bucket, input_dir, s3) - } + get_gem2s_file(input, pipeline_config$originals_bucket, input_dir, s3) config <- list( name = input$experimentName, diff --git a/pipeline-runner/data-raw/sysdata.R b/pipeline-runner/data-raw/sysdata.R index 5574cce8..0ee07487 100644 --- a/pipeline-runner/data-raw/sysdata.R +++ b/pipeline-runner/data-raw/sysdata.R @@ -16,7 +16,7 @@ file_types_by_technology <- list( "10x" = list("barcodes10x", "features10x", "matrix10x") ) -file_names_v1 <- list( +file_names <- list( barcodes10x = "barcodes.tsv.gz", features10x = "features.tsv.gz", matrix10x = "matrix.mtx.gz" @@ -37,7 +37,7 @@ IDS_IDS <- "ids_ids" usethis::use_data( gem2s, DEBUG_PATH, - file_names_v1, + file_names, file_types_by_technology, SYM_IDS, SYM_SYM, diff --git a/pipeline-runner/init.R b/pipeline-runner/init.R index c22e6668..629f3c5e 100644 --- a/pipeline-runner/init.R +++ b/pipeline-runner/init.R @@ -27,7 +27,7 @@ buildActivityArn <- function(aws_region, aws_account_id, activity_id) { return(activity_arn) } -load_config <- function(development_aws_server, api_version = "v1") { +load_config <- function(development_aws_server) { label_path <- "/etc/podinfo/labels" aws_account_id <- Sys.getenv("AWS_ACCOUNT_ID", unset="242905224710") aws_region <- Sys.getenv("AWS_DEFAULT_REGION", unset="eu-west-1") @@ -107,18 +107,10 @@ load_config <- function(development_aws_server, api_version = "v1") { config[["plot_data_bucket"]] <- paste("plots-tables", config$cluster_env, config$aws_account_id, sep = "-") config[["cell_sets_bucket"]] <- paste("cell-sets", config$cluster_env, config$aws_account_id, sep = "-") config[["debug_bucket"]] <- paste("biomage-pipeline-debug", config$cluster_env, config$aws_account_id, sep = "-") - - if (api_version == "v1") { - config[["sns_topic"]] <- paste( - paste("arn:aws:sns", config$aws_region, config$aws_account_id, "work-results", sep = ":"), - config$cluster_env, config$sandbox_id, sep = "-" - ) - } else if (api_version == "v2") { - config[["sns_topic"]] <- paste( - paste("arn:aws:sns", config$aws_region, config$aws_account_id, "work-results", sep = ":"), - config$cluster_env, config$sandbox_id, "v2", sep = "-" - ) - } + config[["sns_topic"]] <- paste( + paste("arn:aws:sns", config$aws_region, config$aws_account_id, "work-results", sep = ":"), + config$cluster_env, config$sandbox_id, "v2", sep = "-" + ) return(config) } @@ -227,7 +219,6 @@ call_data_processing <- function(task_name, input, pipeline_config) { #need this for embed_and_cluster config$api_url <- pipeline_config$api_url config$auth_JWT <- input$authJWT - config$api_version <- input$apiVersion if (!exists("scdata")) { message("No single-cell data has been loaded, reloading from S3...") @@ -411,7 +402,7 @@ init <- function() { tryCatchLog({ # Refresh pipeline_config with the new task input - pipeline_config <- load_config(input$server, api_version = input$apiVersion) + pipeline_config <- load_config(input$server) wrapper(input, pipeline_config)