Merge branch 'dev' into aws-fix

.nf-core.yml

@@ -1 +1,5 @@
+lint:
+  files_exist:
+    - conf/igenomes.config
+    - lib/WorkflowAirrflow.groovy
 repository_type: pipeline

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - [#150](https://github.com/nf-core/airrflow/pull/150) Added option to search for reverse primers.
 - [#159](https://github.com/nf-core/airrflow/pull/159) Template update to nf-core tools v2.3.1, v2.3.1
 - [#161](https://github.com/nf-core/airrflow/pull/161) Add option to skip clustering sequences in the UMI workflow
+- [#163](https://github.com/nf-core/airrflow/pull/163) Added process labels and software version emitting to all modules. Fixed output folder name for changeo processes.
 
 ### `Fixed`
 

conf/modules.config

@@ -224,19 +224,19 @@ process {
 
         withName: CHANGEO_DEFINECLONES {
             publishDir = [
-                path: { "${params.outdir}/shazam/06-define-clones/${meta.id}" },
+                path: { "${params.outdir}/changeo/06-define-clones/${meta.id}" },
             ]
         }
 
         withName: CHANGEO_CREATEGERMLINES {
             publishDir = [
-                path: { "${params.outdir}/shazam/07-create-germlines/${meta.id}" },
+                path: { "${params.outdir}/changeo/07-create-germlines/${meta.id}" },
             ]
         }
 
         withName: CHANGEO_BUILDTREES {
             publishDir = [
-                path: { "${params.outdir}/shazam/08-build-trees/${meta.id}" },
+                path: { "${params.outdir}/changeo/08-build-trees/${meta.id}" },
             ]
             ext.args = '--igphyml --collapse'
         }

conf/test_reveal_no_cc.config

@@ -17,8 +17,6 @@ params {
     max_time   = 6.h
 
     // Input data
-    // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets
-    // TODO nf-core: Give any required params for the test so that command line flags are not needed
     subworkflow = 'reveal'
     input = 'https://raw.githubusercontent.com/nf-core/test-datasets/airrflow/testdata-reveal/test_reveal_metadata.tsv'
     outdir = 'test-reveal-results-no-cc'

docs/usage.md

@@ -279,15 +279,36 @@ The UMI barcodes are typically read from an index file but sometimes can be prov
 
 - No UMIs in R1 or R2 reads: if no UMIs are present in the samples, specify `--umi_length 0` to use the sans-UMI subworkflow.
 
-### Updating the pipeline
+## Experimental features
+
+We are working on a new subworkflow (`reveal`) to analyze bulk and single cell processed reads. The workflow takes as input assembled reads (`.fasta`) or repertoire `.tsv` (example: 10x `airr.tsv`) files and runs quality controls, and generates reports of clonal analysis and lineage trees. The subworkflow (`--subworkflow reveal`) is under active development, and therefore it is not recommended to use in production. Suggestions and feedback are welcome.
+
+This subworkflow can be tested with this command:
+
+```console
+ nextflow run nf-core/airrflow -profile docker,test_reveal
+```
+
+An example command to run an analysis:
+
+```
+nextflow run nf-core/airrflow --subworkflow reveal \
+--input input_samplesheet.tsv \
+--outdir results \
+--reassign --productive_only --remove_chimeric \
+--collapseby filename \
+--cloneby subject_id
+```
+
+## Updating the pipeline
 
 When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline:
 
 ```console
 nextflow pull nf-core/airrflow
 ```
 
-### Reproducibility
+## Reproducibility
 
 It is a good idea to specify a pipeline version when running the pipeline on your data. This ensures that a specific version of the pipeline code and software are used when you run your pipeline. If you keep using the same tag, you'll be running the same version of the pipeline, even if there have been changes to the code since.
 

modules.json

@@ -7,10 +7,10 @@
                 "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d"
             },
             "fastqc": {
-                "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d"
+                "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe"
             },
             "multiqc": {
-                "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d"
+                "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe"
             }
         }
     }

modules/local/alakazam/alakazam_lineage.nf

@@ -2,7 +2,7 @@ process ALAKAZAM_LINEAGE {
     tag "$meta.id"
     label 'process_high'
 
-    conda (params.enable_conda ? "conda-forge::r-base=4.1.2 bioconda::r-alakazam=1.2.0 bioconda::changeo=1.2.0 bioconda::phylip=3.697" : null)              // Conda package
+    conda (params.enable_conda ? "conda-forge::r-base=4.1.2 bioconda::r-alakazam=1.2.0 bioconda::changeo=1.2.0 bioconda::phylip=3.697" : null)  // Please also update the phylip version manually in the script section below as phylip does not print the version
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/mulled-v2-afe1e5f3879e265b14ec08dd3a1875df9c23630d:d6b54b0fae81ec8e55d41b5ea9cc8f39d75cf2d7-0' :
         'quay.io/biocontainers/mulled-v2-afe1e5f3879e265b14ec08dd3a1875df9c23630d:d6b54b0fae81ec8e55d41b5ea9cc8f39d75cf2d7-0' }"
@@ -12,7 +12,7 @@ process ALAKAZAM_LINEAGE {
 
     output:
     tuple val(meta), path("${tab}"), emit: tab
-    path("versions.yml"), emit: versions
+    path "versions.yml" , emit: versions
     path("*.tsv")
     path("Clone_tree_plots/*.pdf")
     path("Graphml_trees/All_graphs_patient.graphml")
@@ -26,7 +26,10 @@ process ALAKAZAM_LINEAGE {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
+        R: \$(echo \$(R --version 2>&1) | awk -F' '  '{print \$3}')
         alakazam: \$(Rscript -e "library(alakazam); cat(paste(packageVersion('alakazam'), collapse='.'))")
+        changeo: \$( AssignGenes.py --version | awk -F' '  '{print \$2}' )
+        pyhlip: 3.697
     END_VERSIONS
     """
 

modules/local/alakazam/alakazam_shazam_repertoires.nf

@@ -2,7 +2,7 @@ process ALAKAZAM_SHAZAM_REPERTOIRES {
     tag "report"
     label 'process_high'
 
-    conda (params.enable_conda ? "conda-forge::r-base=4.1.2 bioconda::r-alakazam=1.2.0 bioconda::r-shazam=1.1.0 conda-forge::r-kableextra=1.3.4 conda-forge::r-knitr=1.33 conda-forge::r-stringr=1.4.0 conda-forge::r-dplyr=1.0.6 conda-forge::r-optparse=1.7.1" : null)              // Conda package
+    conda (params.enable_conda ? "conda-forge::r-base=4.1.2 bioconda::r-alakazam=1.2.0 bioconda::r-shazam=1.1.0 conda-forge::r-kableextra=1.3.4 conda-forge::r-knitr=1.33 conda-forge::r-stringr=1.4.0 conda-forge::r-dplyr=1.0.6 conda-forge::r-optparse=1.7.1" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/mulled-v2-7da73314bcc47157b442d16c3dcfbe81e75a404f:9bb35f8114dffcd97b3afb5de8587355aca16b66-0' :
         'quay.io/biocontainers/mulled-v2-7da73314bcc47157b442d16c3dcfbe81e75a404f:9bb35f8114dffcd97b3afb5de8587355aca16b66-0' }"
@@ -13,7 +13,7 @@ process ALAKAZAM_SHAZAM_REPERTOIRES {
     tuple path(repertoire_report), path(css), path(logo)
 
     output:
-    path("versions.yml"), emit: versions
+    path "versions.yml" , emit: versions
     path("repertoire_comparison")
     path("Bcellmagic_report.html")
 
@@ -25,6 +25,9 @@ process ALAKAZAM_SHAZAM_REPERTOIRES {
     "${task.process}":
         alakazam: \$(Rscript -e "library(alakazam); cat(paste(packageVersion('alakazam'), collapse='.'))")
         shazam: \$(Rscript -e "library(shazam); cat(paste(packageVersion('shazam'), collapse='.'))")
+        stringr: \$(Rscript -e "library(stringr); cat(paste(packageVersion('stringr'), collapse='.'))")
+        dplyr: \$(Rscript -e "library(dplyr); cat(paste(packageVersion('dplyr'), collapse='.'))")
+        knitr: \$(Rscript -e "library(knitr); cat(paste(packageVersion('knitr'), collapse='.'))")
         R: \$(echo \$(R --version 2>&1) | awk -F' '  '{print \$3}')
     END_VERSIONS
     """

modules/local/changeo/changeo_assigngenes.nf

@@ -2,7 +2,7 @@ process CHANGEO_ASSIGNGENES {
     tag "$meta.id"
     label 'process_low'
 
-    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)              // Conda package
+    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' :
         'quay.io/biocontainers/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' }"

modules/local/changeo/changeo_buildtrees.nf

@@ -2,7 +2,7 @@ process CHANGEO_BUILDTREES {
     tag "$meta.id"
     label 'process_medium'
 
-    conda (params.enable_conda ? "conda-forge::r-base=4.1.2 bioconda:r-alakazam=1.2.0 bioconda::changeo=1.2.0 bioconda::igphyml=1.1.3" : null)              // Conda package
+    conda (params.enable_conda ? "conda-forge::r-base=4.1.2 bioconda:r-alakazam=1.2.0 bioconda::changeo=1.2.0 bioconda::igphyml=1.1.3" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/mulled-v2-d432bd3f78aaba1be2f7eb105c18998acb64d739:2c83ca89e577c8839f746f0fe4a6c63ef5984b99-0' :
         'quay.io/biocontainers/mulled-v2-d432bd3f78aaba1be2f7eb105c18998acb64d739:2c83ca89e577c8839f746f0fe4a6c63ef5984b99-0' }"
@@ -12,10 +12,19 @@ process CHANGEO_BUILDTREES {
 
     output:
     tuple val(meta), path("*_lineages.tsv")
+    path "versions.yml" , emit: versions
 
     script:
     def args = task.ext.args ?: ''
     """
     BuildTrees.py -d ${tab} --outname ${meta.id} --log ${meta.id}.log --nproc $task.cpus $args
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        R: \$(echo \$(R --version 2>&1) | awk -F' '  '{print \$3}')
+        alakazam: \$(Rscript -e "library(alakazam); cat(paste(packageVersion('alakazam'), collapse='.'))")
+        changeo: \$(AssignGenes.py --version | awk -F' '  '{print \$2}')
+        igphyml: \$(igphyml --version | grep -o "IgPhyML [0-9\\. ]\\+" | grep -o "[0-9\\. ]\\+")
+    END_VERSIONS
     """
 }

modules/local/changeo/changeo_convertdb_fasta.nf

@@ -3,7 +3,7 @@ process CHANGEO_CONVERTDB_FASTA {
     label 'process_low'
     label 'immcantation'
 
-    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)              // Conda package
+    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' :
         'quay.io/biocontainers/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' }"
@@ -23,7 +23,8 @@ process CHANGEO_CONVERTDB_FASTA {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        changeo: \$( ConvertDb.py --version | awk -F' ' '{print \$2}' )
+        igblastn: \$( igblastn -version | grep -o "igblast[0-9\\. ]\\+" | grep -o "[0-9\\. ]\\+" )
+        changeo: \$( ConvertDb.py --version | awk -F' '  '{print \$2}' )
     END_VERSIONS
     """
 }

modules/local/changeo/changeo_creategermlines.nf

@@ -2,7 +2,7 @@ process CHANGEO_CREATEGERMLINES {
     tag "$meta.id"
     label 'process_low'
 
-    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)              // Conda package
+    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' :
         'quay.io/biocontainers/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' }"
@@ -14,6 +14,7 @@ process CHANGEO_CREATEGERMLINES {
     output:
     tuple val(meta), path("*germ-pass.tsv"), emit: tab
     path("*_command_log.txt"), emit: logs
+    path "versions.yml" , emit: versions
 
     script:
     """
@@ -22,5 +23,11 @@ process CHANGEO_CREATEGERMLINES {
     --format airr \\
     --log ${meta.id}.log --outname ${meta.id} > ${meta.id}_command_log.txt
     ParseLog.py -l ${meta.id}.log -f ID V_CALL D_CALL J_CALL
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        igblastn: \$( igblastn -version | grep -o "igblast[0-9\\. ]\\+" | grep -o "[0-9\\. ]\\+" )
+        changeo: \$( CreateGermlines.py --version | awk -F' '  '{print \$2}' )
+    END_VERSIONS
     """
 }

modules/local/changeo/changeo_defineclones.nf

@@ -2,7 +2,7 @@ process CHANGEO_DEFINECLONES {
     tag "$meta.id"
     label 'process_medium'
 
-    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)              // Conda package
+    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' :
         'quay.io/biocontainers/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' }"
@@ -14,6 +14,7 @@ process CHANGEO_DEFINECLONES {
     output:
     tuple val(meta), path("*clone-pass.tsv"), emit: tab // sequence tsv table in AIRR format
     path "*_command_log.txt" , emit: logs
+    path "versions.yml" , emit: versions
 
     script:
     if (params.set_cluster_threshold) {
@@ -25,5 +26,11 @@ process CHANGEO_DEFINECLONES {
     """
     DefineClones.py -d $tab --act set --model ham --norm len --nproc $task.cpus --dist $thr --outname ${meta.id} --log ${meta.id}.log > "${meta.id}_command_log.txt"
     ParseLog.py -l "${meta.id}.log" -f id v_call j_call junction_length cloned filtered clones
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        igblastn: \$( igblastn -version | grep -o "igblast[0-9\\. ]\\+" | grep -o "[0-9\\. ]\\+" )
+        changeo: \$( DefineClones.py --version | awk -F' '  '{print \$2}' )
+    END_VERSIONS
     """
 }

modules/local/changeo/changeo_makedb.nf

@@ -2,7 +2,7 @@ process CHANGEO_MAKEDB {
     tag "$meta.id"
     label 'process_low'
 
-    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)              // Conda package
+    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' :
         'quay.io/biocontainers/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' }"
@@ -16,6 +16,7 @@ process CHANGEO_MAKEDB {
     output:
     tuple val(meta), path("*db-pass.tsv"), emit: tab //sequence table in AIRR format
     path("*_command_log.txt"), emit: logs //process logs
+    path "versions.yml" , emit: versions
 
     script:
     def args = task.ext.args ?: ''
@@ -24,5 +25,11 @@ process CHANGEO_MAKEDB {
     ${imgt_base}/${meta.species.toLowerCase()}/vdj/ \\
     $args \\
     --outname "${meta.id}" > "${meta.id}_command_log.txt"
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        igblastn: \$( igblastn -version | grep -o "igblast[0-9\\. ]\\+" | grep -o "[0-9\\. ]\\+" )
+        changeo: \$( MakeDb.py --version | awk -F' '  '{print \$2}' )
+    END_VERSIONS
     """
 }

modules/local/changeo/changeo_parsedb_select.nf

@@ -2,7 +2,7 @@ process CHANGEO_PARSEDB_SELECT {
     tag "$meta.id"
     label 'process_low'
 
-    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)              // Conda package
+    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' :
         'quay.io/biocontainers/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' }"
@@ -13,17 +13,30 @@ process CHANGEO_PARSEDB_SELECT {
     output:
     tuple val(meta), path("*parse-select.tsv"), emit: tab // sequence tsv in AIRR format
     path("*_command_log.txt"), emit: logs //process logs
+    path "versions.yml" , emit: versions
 
     script:
     def args = task.ext.args ?: ''
     def args2 = task.ext.args2 ?: ''
     if (meta.locus == 'IG'){
         """
         ParseDb.py select -d $tab $args --outname ${meta.id} > "${meta.id}_command_log.txt"
+
+        cat <<-END_VERSIONS > versions.yml
+        "${task.process}":
+            igblastn: \$( igblastn -version | grep -o "igblast[0-9\\. ]\\+" | grep -o "[0-9\\. ]\\+" )
+            changeo: \$( ParseDb.py --version | awk -F' '  '{print \$2}' )
+        END_VERSIONS
         """
     } else if (meta.locus == 'TR'){
         """
         ParseDb.py select -d $tab $args2 --outname ${meta.id} > "${meta.id}_command_log.txt"
+
+        cat <<-END_VERSIONS > versions.yml
+        "${task.process}":
+            igblastn: \$( igblastn -version | grep -o "igblast[0-9\\. ]\\+" | grep -o "[0-9\\. ]\\+" )
+            changeo: \$( ParseDb.py --version | awk -F' '  '{print \$2}' )
+        END_VERSIONS
         """
     }
 }

modules/local/changeo/changeo_parsedb_split.nf

@@ -3,7 +3,7 @@ process CHANGEO_PARSEDB_SPLIT {
     label 'process_low'
     label 'immcantation'
 
-    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)              // Conda package
+    conda (params.enable_conda ? "bioconda::changeo=1.2.0 bioconda::igblast=1.17.1" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' :
         'quay.io/biocontainers/mulled-v2-2665a8a48fa054ad1fcccf53e711669939b3eac1:f479475bceae84156e57e303cfe804ab5629d62b-0' }"
@@ -19,6 +19,7 @@ process CHANGEO_PARSEDB_SPLIT {
     script:
     """
     ParseDb.py split -d $tab -f productive --outname ${meta.id} > "${meta.id}_split_command_log.txt"
+
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         changeo: \$( ParseDb.py --version | awk -F' '  '{print \$2}' )

modules/local/fetch_databases.nf

@@ -20,6 +20,8 @@ process FETCH_DATABASES {
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         IMGT download date: \$( echo \$(date "+%F") )
+        igblastn: \$( igblastn -version | grep -o "igblast[0-9\\. ]\\+" | grep -o "[0-9\\. ]\\+" )
+        changeo: \$( AssignGenes.py --version | awk -F' '  '{print \$2}' )
     END_VERSIONS
     """
 }

modules/local/merge_UMI.nf

@@ -13,11 +13,18 @@ process MERGE_UMI {
 
     output:
     tuple val(meta), path('*_R1.fastq.gz'), path('*_R2.fastq.gz')   , emit: reads
+    path "versions.yml" , emit: versions
 
     script:
     """
     merge_R1_umi.py -R1 "${R1}" -I1 "${I1}" -o UMI_R1.fastq.gz --umi_start $params.umi_start --umi_length $params.umi_length
     mv "UMI_R1.fastq.gz" "${meta.id}_UMI_R1.fastq.gz"
     mv "${R2}" "${meta.id}_R2.fastq.gz"
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        python: \$( echo \$(python --version | grep -o "[0-9\\. ]\\+") )
+        biopython: \$(echo \$(python -c "import pkg_resources; print(pkg_resources.get_distribution('biopython').version)"))
+    END_VERSIONS
     """
 }