qbic-pipelines · WackerO · May 11, 2022 · Apr 15, 2022 · Apr 15, 2022 · Apr 19, 2022
diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml
@@ -42,4 +42,3 @@ jobs:
             Thanks again for your contribution!
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           allow-repeats: false
-#
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -69,7 +69,16 @@ jobs:
       NXF_ANSI_LOG: false
     strategy:
       matrix:
-        config: ["test", "test_contrast_matrix", "test_contrast_list", "test_blacklist", "test_skip_pathway_analysis"]
+        config:
+          [
+            "test",
+            "test_contrast_matrix",
+            "test_contrast_list",
+            "test_skip_pathway_analysis",
+            "test_star_salmon",
+            "test_star_rsem",
+            "test_use_vst",
+          ]
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2
@@ -89,5 +98,3 @@ jobs:
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
-
-#
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -77,5 +77,3 @@ jobs:
             lint_log.txt
             lint_results.md
             PR_number.txt
-
-#
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,8 @@
 
 ### Added
 
+- [#122](https://github.com/qbic-pipelines/rnadeseq/pull/122) Add searchable/sortable tables to report
+- [#118](https://github.com/qbic-pipelines/rnadeseq/pull/118) Add parameter "--input_type" (and change --rawcounts to --gene_counts) to process featurecounts, rsem and salmon output from the new rnaseq; add igenomes.config to process different species
 - [#104](https://github.com/qbic-pipelines/rnadeseq/pull/104) Add parameter "--skip_pathway_analysis"
 - Bump versions to 1.4.0dev
 - Add parameter "--min_DE_genes"
@@ -14,12 +16,14 @@
 
 ### Changed
 
+- [#115](https://github.com/qbic-pipelines/rnadeseq/pull/115) Template update
 - [#117](https://github.com/qbic-pipelines/rnadeseq/pull/117) Turned LabID optional for report output in RNAseq_report.Rmd
 - Removed assets/report_options.yml
 - [#110](https://github.com/qbic-pipelines/rnadeseq/pull/110) Changed report to use rlog normalization by default, vst is used if --skip_rlog is enabled
 
 ### Fixed
 
+- [#118](https://github.com/qbic-pipelines/rnadeseq/pull/118) Removed blacklist parameter and config and instead added trycatch to ignore pathways with errors
 - [#105](https://github.com/qbic-pipelines/rnadeseq/pull/105) Fixed relevel and added test_relevel.config
 - [#106](https://github.com/qbic-pipelines/rnadeseq/pull/106) Fixed `--logFCthreshold` bug
 - [#108](https://github.com/qbic-pipelines/rnadeseq/pull/108) Fixed blacklist file not working

diff --git a/assets/RNAseq_report.Rmd b/assets/RNAseq_report.Rmd
diff --git a/assets/email_template.txt b/assets/email_template.txt
@@ -1,6 +1,6 @@
-========================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  qbic-pipelines/rnadeseq v${version}
-========================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Run Name: $runName
 
 <% if (success){

diff --git a/bin/DESeq2.R b/bin/DESeq2.R
diff --git a/bin/Execute_report.R b/bin/Execute_report.R
@@ -13,6 +13,7 @@ option_list = list(
     make_option(c("-l", "--genelist"), type="character", default=NULL, help="path to gene list file", metavar="character"),
     make_option(c("-q", "--quote"), type="character", default=NULL, help="path to the signed quote PDF file", metavar="character"),
     make_option(c("-g", "--organism"), type="character", default=NULL, help="Organism, e.g. Hsapiens."),
+    make_option(c("-c", "--species_library"), type="character", default=NULL, help="Library name. Example format: org.At.tair.db", metavar="character"),
     make_option(c("-b", "--batch_effect"), action="store_true", default=FALSE, help="Batch effect correction."),
     make_option(c("-f", "--log_FC"), type="double", default=NULL, help="Log Fold Change threshold to consider a gene DE."),
     make_option(c("-x", "--revision"), type="character", default=NULL, help="rnadeseq workflow revision", metavar="character"),
@@ -43,6 +44,7 @@ rmarkdown::render(opt$report, output_file = opt$output, knit_root_dir = wd, outp
                                 path_min_DEG = opt$min_DEG_pathway,
                                 path_quote = opt$quote,
                                 organism = opt$organism,
+                                species_library = opt$species_library,
                                 batch_effect = opt$batch_effect,
                                 log_FC = opt$log_FC,
                                 nsub_genes = opt$nsub_genes,

diff --git a/bin/pathway_analysis.R b/bin/pathway_analysis.R
diff --git a/conf/igenomes.config b/conf/igenomes.config
diff --git a/conf/test.config b/conf/test.config
@@ -5,7 +5,7 @@
     Defines input files and everything required to run a fast and simple pipeline test.
 
     Use as follows:
-        nextflow run nf-core/rnadeseq -profile test,<docker/singularity> --outdir <OUTDIR>
+        nextflow run qbic-pipelines/rnadeseq -profile test,<docker/singularity> --outdir <OUTDIR>
 
 ----------------------------------------------------------------------------------------
 */
@@ -19,7 +19,7 @@ params {
     max_time = 48.h
     // Input data
     metadata = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/Sample_preparations.tsv'
-    rawcounts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
+    gene_counts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
     model = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/design.txt'
     genelist = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/requested_genes.txt'
     logFCthreshold = 2
@@ -28,5 +28,5 @@ params {
     versions = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/software_versions.csv'
     multiqc = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/MultiQC.zip'
     quote = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/offer_example.pdf'
-    species = 'mmusculus'
+    genome = 'GRCm38'
 }
diff --git a/conf/test_contrast_list.config b/conf/test_contrast_list.config
@@ -16,13 +16,14 @@ params {
     max_time = 48.h
     // Input data
     metadata = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/Sample_preparations.tsv'
-    rawcounts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
+    gene_counts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
     model = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/design.txt'
     genelist = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/requested_genes.txt'
     contrast_list = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/contrast_list.tsv'
     project_summary = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/summary.tsv'
     versions = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/software_versions.csv'
     multiqc = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/MultiQC.zip'
-    species = 'mmusculus'
+    genome = 'GRCm38'
     quote = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/offer_example.pdf'
+    logFCthreshold = 2
 }
diff --git a/conf/test_contrast_matrix.config b/conf/test_contrast_matrix.config
@@ -16,13 +16,14 @@ params {
     max_time = 48.h
     // Input data
     metadata = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/Sample_preparations.tsv'
-    rawcounts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
+    gene_counts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
     model = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/design.txt'
     genelist = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/requested_genes.txt'
     contrast_table = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/contrasts.tsv'
     project_summary = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/summary.tsv'
     versions = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/software_versions.csv'
     multiqc = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/MultiQC.zip'
-    species = 'mmusculus'
+    genome = 'GRCm38'
     quote = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/offer_example.pdf'
+    logFCthreshold = 2
 }
diff --git a/conf/test_relevel.config b/conf/test_relevel.config
@@ -16,14 +16,15 @@ params {
     max_time = 48.h
     // Input data
     metadata = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/Sample_preparations.tsv'
-    rawcounts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
+    gene_counts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
     model = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/design.txt'
     relevel = 'https://raw.githubusercontent.com/SusiJo/rnadeseq/dev/testdata/relevel.tsv'
     genelist = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/requested_genes.txt'
     //report_options = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/report_options.yml'
     project_summary = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/summary.tsv'
     versions = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/software_versions.csv'
     multiqc = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/MultiQC.zip'
-    species = 'mmusculus'
-quote = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/offer_example.pdf'
+    genome = 'GRCm38'
+    quote = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/offer_example.pdf'
+    logFCthreshold = 2
 }
diff --git a/conf/test_skip_pathway_analysis.config b/conf/test_skip_pathway_analysis.config
@@ -4,24 +4,26 @@
  * -------------------------------------------------
  * Defines bundled input files and everything required
  * to run a fast and simple test. Use as follows:
- *   nextflow run qbic-pipelines/rnadeseq -profile test
+ *   nextflow run qbic-pipelines/rnadeseq -profile test_skip_pathway_analysis
  */
 
 params {
-    config_profile_name = 'Test profile'
+    config_profile_name = 'Test skip pathway analysis'
     config_profile_description = 'Minimal test dataset to check pipeline function'
     // Limit resources so that this can run on Travis
     max_cpus = 2
     max_memory = 6.GB
     max_time = 48.h
     // Input data
     metadata = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/Sample_preparations.tsv'
-    rawcounts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
+    gene_counts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
     model = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/design.txt'
     genelist = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/requested_genes.txt'
     project_summary = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/summary.tsv'
     versions = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/software_versions.csv'
     multiqc = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/MultiQC.zip'
     quote = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/offer_example.pdf'
     skip_pathway_analysis = true
+    genome = 'GRCm38'
+    logFCthreshold = 2
 }
diff --git a/conf/test_blacklist.config → conf/test_star_rsem.config b/conf/test_blacklist.config → conf/test_star_rsem.config
@@ -4,26 +4,31 @@
  * -------------------------------------------------
  * Defines bundled input files and everything required
  * to run a fast and simple test. Use as follows:
- *   nextflow run qbic-pipelines/rnadeseq -profile test_contrast_list
+ *   nextflow run qbic-pipelines/rnadeseq -profile test_star_rsem
  */
 
 params {
-    config_profile_name = 'Test blacklist'
+    config_profile_name = 'Test star rsem'
     config_profile_description = 'Minimal test dataset to check pipeline function'
     // Limit resources so that this can run on Travis
     max_cpus = 2
     max_memory = 6.GB
     max_time = 48.h
     // Input data
-    metadata = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/Sample_preparations.tsv'
-    rawcounts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
-    model = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/design.txt'
+    metadata = 'testdata/QDESQ/QDESQ_Sample_preparations.tsv'
+    gene_counts = 'testdata/QDESQ/star_rsem/'
+    model = 'testdata/QDESQ/QDESQ_design.txt'
     genelist = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/requested_genes.txt'
-    contrast_list = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/contrast_list.tsv'
+    logFCthreshold = 2
+    //report_options = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/report_options.yml'
     project_summary = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/summary.tsv'
     versions = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/software_versions.csv'
     multiqc = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/MultiQC.zip'
-    species = 'mmusculus'
     quote = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/offer_example.pdf'
-    kegg_blacklist = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/kegg_blacklist.txt'
+    input_type = 'rsem'
+    genome = 'GRCh37'
+    //   library     = "org.Hs.eg.db"
+    // organism    = "hsapiens"
+    //   keytype     = "ENSEMBL"
+    // gtf         = "s3://ngi-igenomes/igenomes/Homo_sapiens/Ensembl/GRCh37/Annotation/Genes/genes.gtf"
 }
diff --git a/conf/test_star_salmon.config b/conf/test_star_salmon.config
@@ -0,0 +1,34 @@
+/*
+ * -------------------------------------------------
+ *  Nextflow config file for running tests
+ * -------------------------------------------------
+ * Defines bundled input files and everything required
+ * to run a fast and simple test. Use as follows:
+ *   nextflow run qbic-pipelines/rnadeseq -profile test_star_salmon
+ */
+
+params {
+    config_profile_name = 'Test star salmon'
+    config_profile_description = 'Minimal test dataset to check pipeline function'
+    // Limit resources so that this can run on Travis
+    max_cpus = 2
+    max_memory = 6.GB
+    max_time = 48.h
+    // Input data
+    metadata = 'testdata/QDESQ/QDESQ_Sample_preparations.tsv'
+    gene_counts = 'testdata/QDESQ/star_salmon/'
+    model = 'testdata/QDESQ/QDESQ_design.txt'
+    genelist = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/requested_genes.txt'
+    logFCthreshold = 2
+    //report_options = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/report_options.yml'
+    project_summary = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/summary.tsv'
+    versions = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/software_versions.csv'
+    multiqc = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/MultiQC.zip'
+    quote = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/offer_example.pdf'
+    input_type = 'salmon'
+    //  genome = 'GRCh37'
+    library     = "org.Hs.eg.db"
+    organism    = "hsapiens"
+    keytype     = "ENSEMBL"
+    gtf         = "s3://ngi-igenomes/igenomes/Homo_sapiens/Ensembl/GRCh37/Annotation/Genes/genes.gtf"
+}
diff --git a/conf/test_use_vst.config b/conf/test_use_vst.config
@@ -4,19 +4,19 @@
  * -------------------------------------------------
  * Defines bundled input files and everything required
  * to run a fast and simple test. Use as follows:
- *   nextflow run qbic-pipelines/rnadeseq -profile test
+ *   nextflow run qbic-pipelines/rnadeseq -profile test_use_vst
  */
 
 params {
-    config_profile_name = 'Test profile'
+    config_profile_name = 'Test use vst'
     config_profile_description = 'Minimal test dataset to check pipeline function'
     // Limit resources so that this can run on Travis
     max_cpus = 2
     max_memory = 6.GB
     max_time = 48.h
     // Input data
     metadata = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/Sample_preparations.tsv'
-    rawcounts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
+    gene_counts = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/merged_gene_counts.txt'
     model = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/design.txt'
     genelist = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/requested_genes.txt'
     //report_options = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/report_options.yml'
@@ -26,6 +26,6 @@ params {
     quote = 'https://raw.githubusercontent.com/qbic-pipelines/rnadeseq/dev/testdata/offer_example.pdf'
     skip_pathway_analysis = false
     use_vst = true
-    species = 'mmusculus'
+    genome = 'GRCm38'
     logFCthreshold = 2
 }
diff --git a/docs/output.md b/docs/output.md
@@ -31,7 +31,7 @@ This directory contains the zipped results. When unzipping them, the following s
   - `rlog_transformed_gene_counts.tsv`: normalized gene counts with the "regularized logarithm" approach. This normalization is used prior to PCA analysis and heatmap plotting of the gene counts.
   - `vst_transformed_gene_counts.tsv`: normalized gene counts with the "variance stabilizing" transformation.
   - `sizeFactor_libraries.tsv`: size factors for each sample.
-- `DE_genes_tables/`: folder containing one tab-separated table for each of the contrasts in the analysis. Each table contains a list of all differentially expressed genes in the contrast, specifying the mean gene expression across all samples (baseMean), and the log2 fold change value and p-adjusted values (padj) for this contrast.
+- `DE_genes_tables/`: folder containing one tab-separated table for each of the contrasts in the analysis. Each table contains a list of all differentially expressed genes in the contrast, specifying the mean gene expression across all samples (baseMean), and the log2 fold change value and p-adjusted values (padj) for this contrast. Files whose name contain "\_allgenes" are NOT filtered and therefore contain every gene investigated, whether significant or not (these are used for plotting and can be ignored while evaluating the results).
 - `final_gene_table/final_gene_list_DESeq2.tsv`: table containing a list of all genes considered in the analysis. Here a summary of the log2 Fold Change and p-adjusted values for all contrasts is displayed. Additionally, the column **filter** shows if this gene was differentially expressed (DE) in any of the contrasts, or not (not_DE). The column **contrast_vector** contains for each contrast considered in the analysis a 1 if the gene was differentially expressed for this contrast or a 0 if it was not.
 - `plots/`:
   - `Heatmaps_of_distances.pdf/.svg`: heatmap of the pairwise euclidean distances among samples, when the rlog normalized gene counts are considered.