Subworkflow Infrastructure

.github/workflows/nf-core-linting.yml

@@ -71,6 +71,8 @@ jobs:
 
       - name: Lint ${{ matrix.tags }}
         run: nf-core modules lint ${{ matrix.tags }}
+        # HACK
+        if: startsWith( matrix.tags, 'subworkflow' ) != true
 
       - uses: actions/cache@v2
         with:

subworkflows/nf-core/align_bowtie2/main.nf

@@ -0,0 +1,47 @@
+//
+// Alignment with Bowtie2
+//
+
+params.align_options    = [:]
+params.samtools_sort_options  = [:]
+params.samtools_index_options = [:]
+params.samtools_stats_options = [:]
+
+include { BOWTIE2_ALIGN     } from '../../../modules/bowtie2/align/main' addParams( options: params.align_options    )
+include { BAM_SORT_SAMTOOLS } from '../bam_sort_samtools/main'           addParams( sort_options: params.samtools_sort_options, index_options: params.samtools_index_options, stats_options: params.samtools_stats_options )
+
+workflow ALIGN_BOWTIE2 {
+    take:
+    reads // channel: [ val(meta), [ reads ] ]
+    index // channel: /path/to/bowtie2/index/
+
+    main:
+
+    ch_versions = Channel.empty()
+
+    //
+    // Map reads with Bowtie2
+    //
+    BOWTIE2_ALIGN ( reads, index )
+    ch_versions = ch_versions.mix(BOWTIE2_ALIGN.out.versions.first())
+
+    //
+    // Sort, index BAM file and run samtools stats, flagstat and idxstats
+    //
+    BAM_SORT_SAMTOOLS ( BOWTIE2_ALIGN.out.bam )
+    ch_versions = ch_versions.mix(BAM_SORT_SAMTOOLS.out.versions)
+
+    emit:
+    bam_orig          = BOWTIE2_ALIGN.out.bam          // channel: [ val(meta), bam   ]
+    log_out           = BOWTIE2_ALIGN.out.log          // channel: [ val(meta), log   ]
+    fastq             = BOWTIE2_ALIGN.out.fastq        // channel: [ val(meta), fastq ]
+
+    bam              = BAM_SORT_SAMTOOLS.out.bam      // channel: [ val(meta), [ bam ] ]
+    bai              = BAM_SORT_SAMTOOLS.out.bai      // channel: [ val(meta), [ bai ] ]
+    csi              = BAM_SORT_SAMTOOLS.out.csi      // channel: [ val(meta), [ csi ] ]
+    stats            = BAM_SORT_SAMTOOLS.out.stats    // channel: [ val(meta), [ stats ] ]
+    flagstat         = BAM_SORT_SAMTOOLS.out.flagstat // channel: [ val(meta), [ flagstat ] ]
+    idxstats         = BAM_SORT_SAMTOOLS.out.idxstats // channel: [ val(meta), [ idxstats ] ]
+
+    versions       = ch_versions                      // channel: [ versions.yml ]
+}

subworkflows/nf-core/align_bowtie2/meta.yml

@@ -0,0 +1,50 @@
+name: align_bowtie2
+description: Align reads to a reference genome using bowtie2 then sort with samtools
+keywords:
+  - align
+  - fasta
+  - genome
+  - reference
+modules:
+  - bowtie2/align
+  - samtools/sort
+  - samtools/index
+  - samtools/stats
+  - samtools/idxstats
+  - samtools/flagstat
+input:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - reads:
+      type: file
+      description: |
+        List of input FastQ files of size 1 and 2 for single-end and paired-end data,
+        respectively.
+  - index:
+      type: file
+      description: Bowtie2 genome index files
+      pattern: '*.ebwt'
+# TODO
+output:
+  - bam:
+      type: file
+      description: Output BAM file containing read alignments
+      pattern: '*.{bam}'
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: 'versions.yml'
+  - fastq:
+      type: file
+      description: Unaligned FastQ files
+      pattern: '*.fastq.gz'
+  - log:
+      type: file
+      description: Alignment log
+      pattern: '*.log'
+  # TODO Add samtools outputs
+authors:
+  - '@drpatelh'

subworkflows/nf-core/align_bowtie2/nextflow.config

@@ -0,0 +1,2 @@
+params.align_options    = [:]
+params.samtools_options = [:]

subworkflows/nf-core/bam_sort_samtools/main.nf

@@ -0,0 +1,53 @@
+//
+// Sort, index BAM file and run samtools stats, flagstat and idxstats
+//
+
+params.sort_options  = [:]
+params.index_options = [:]
+params.stats_options = [:]
+
+include { SAMTOOLS_SORT      } from '../../../modules/samtools/sort/main'  addParams( options: params.sort_options  )
+include { SAMTOOLS_INDEX     } from '../../../modules/samtools/index/main' addParams( options: params.index_options )
+include { BAM_STATS_SAMTOOLS } from '../bam_stats_samtools/main'           addParams( options: params.stats_options )
+
+workflow BAM_SORT_SAMTOOLS {
+    take:
+    ch_bam // channel: [ val(meta), [ bam ] ]
+
+    main:
+
+    ch_versions = Channel.empty()
+
+    SAMTOOLS_SORT ( ch_bam )
+    ch_versions = ch_versions.mix(SAMTOOLS_SORT.out.versions.first())
+
+    SAMTOOLS_INDEX ( SAMTOOLS_SORT.out.bam )
+    ch_versions = ch_versions.mix(SAMTOOLS_INDEX.out.versions.first())
+
+    SAMTOOLS_SORT.out.bam
+        .join(SAMTOOLS_INDEX.out.bai, by: [0], remainder: true)
+        .join(SAMTOOLS_INDEX.out.csi, by: [0], remainder: true)
+        .map {
+            meta, bam, bai, csi ->
+                if (bai) {
+                    [ meta, bam, bai ]
+                } else {
+                    [ meta, bam, csi ]
+                }
+        }
+        .set { ch_bam_bai }
+
+    BAM_STATS_SAMTOOLS ( ch_bam_bai )
+    ch_versions = ch_versions.mix(BAM_STATS_SAMTOOLS.out.versions)
+
+    emit:
+    bam      = SAMTOOLS_SORT.out.bam           // channel: [ val(meta), [ bam ] ]
+    bai      = SAMTOOLS_INDEX.out.bai          // channel: [ val(meta), [ bai ] ]
+    csi      = SAMTOOLS_INDEX.out.csi          // channel: [ val(meta), [ csi ] ]
+
+    stats    = BAM_STATS_SAMTOOLS.out.stats    // channel: [ val(meta), [ stats ] ]
+    flagstat = BAM_STATS_SAMTOOLS.out.flagstat // channel: [ val(meta), [ flagstat ] ]
+    idxstats = BAM_STATS_SAMTOOLS.out.idxstats // channel: [ val(meta), [ idxstats ] ]
+
+    versions = ch_versions                     // channel: [ versions.yml ]
+}

subworkflows/nf-core/bam_sort_samtools/meta.yml

@@ -0,0 +1,41 @@
+name: bam_sort_samtools
+description: Sort SAM/BAM/CRAM file
+keywords:
+  - sort
+  - bam
+  - sam
+  - cram
+modules:
+  - samtools/sort
+  - samtools/index
+  - samtools/stats
+  - samtools/idxstats
+  - samtools/flagstat
+input:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - bam:
+      type: file
+      description: BAM/CRAM/SAM file
+      pattern: '*.{bam,cram,sam}'
+# TODO
+output:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - bam:
+      type: file
+      description: Sorted BAM/CRAM/SAM file
+      pattern: '*.{bam,cram,sam}'
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: 'versions.yml'
+authors:
+  - '@drpatelh'
+  - '@ewels'

subworkflows/nf-core/bam_sort_samtools/nextflow.config

@@ -0,0 +1 @@
+params.options = [:]

subworkflows/nf-core/bam_stats_samtools/main.nf

@@ -0,0 +1,33 @@
+//
+// Run SAMtools stats, flagstat and idxstats
+//
+
+params.options = [:]
+
+include { SAMTOOLS_STATS    } from '../../../modules/samtools/stats/main'    addParams( options: params.options )
+include { SAMTOOLS_IDXSTATS } from '../../../modules/samtools/idxstats/main' addParams( options: params.options )
+include { SAMTOOLS_FLAGSTAT } from '../../../modules/samtools/flagstat/main' addParams( options: params.options )
+
+workflow BAM_STATS_SAMTOOLS {
+    take:
+    ch_bam_bai // channel: [ val(meta), [ bam ], [bai/csi] ]
+
+    main:
+    ch_versions = Channel.empty()
+
+    SAMTOOLS_STATS ( ch_bam_bai )
+    ch_versions = ch_versions.mix(SAMTOOLS_STATS.out.versions.first())
+
+    SAMTOOLS_FLAGSTAT ( ch_bam_bai )
+    ch_versions = ch_versions.mix(SAMTOOLS_FLAGSTAT.out.versions.first())
+
+    SAMTOOLS_IDXSTATS ( ch_bam_bai )
+    ch_versions = ch_versions.mix(SAMTOOLS_IDXSTATS.out.versions.first())
+
+    emit:
+    stats    = SAMTOOLS_STATS.out.stats       // channel: [ val(meta), [ stats ] ]
+    flagstat = SAMTOOLS_FLAGSTAT.out.flagstat // channel: [ val(meta), [ flagstat ] ]
+    idxstats = SAMTOOLS_IDXSTATS.out.idxstats // channel: [ val(meta), [ idxstats ] ]
+
+    versions = ch_versions                    // channel: [ versions.yml ]
+}

subworkflows/nf-core/bam_stats_samtools/meta.yml

@@ -0,0 +1,43 @@
+name: samtools_stats
+description: Produces comprehensive statistics from SAM/BAM/CRAM file
+keywords:
+  - statistics
+  - counts
+  - bam
+  - sam
+  - cram
+modules:
+  - samtools/stats
+  - samtools/idxstats
+  - samtools/flagstat
+input:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - bam:
+      type: file
+      description: BAM/CRAM/SAM file
+      pattern: '*.{bam,cram,sam}'
+  - bai:
+      type: file
+      description: Index for BAM/CRAM/SAM file
+      pattern: '*.{bai,crai,sai}'
+# TODO
+output:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - stats:
+      type: file
+      description: File containing samtools stats output
+      pattern: '*.{stats}'
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: 'versions.yml'
+authors:
+  - '@drpatelh'

subworkflows/nf-core/bam_stats_samtools/nextflow.config

@@ -0,0 +1 @@
+params.options = [:]

tests/config/pytest_modules.yml

@@ -1074,3 +1074,15 @@ yara/index:
 yara/mapper:
   - modules/yara/mapper/**
   - tests/modules/yara/mapper/**
+
+subworkflows/align_bowtie2:
+  - subworkflows/nf-core/align_bowtie2/**
+  - tests/subworkflows/nf-core/align_bowtie2/**
+
+subworkflows/bam_stats_samtools:
+  - subworkflows/nf-core/bam_stats_samtools/**
+  - tests/subworkflows/nf-core/bam_stats_samtools/**
+
+subworkflows/bam_sort_samtools:
+  - subworkflows/nf-core/bam_sort_samtools/**
+  - tests/subworkflows/nf-core/bam_sort_samtools/**

tests/modules/samtools/sort/main.nf

@@ -2,7 +2,7 @@
 
 nextflow.enable.dsl = 2
 
-include { SAMTOOLS_SORT } from '../../../../modules/samtools/sort/main.nf' addParams( options: [:] )
+include { SAMTOOLS_SORT } from '../../../../modules/samtools/sort/main.nf' addParams( options: ['suffix': '.sorted'] )
 
 workflow test_samtools_sort  {
     input = [ [ id:'test', single_end:false ], // meta map

tests/modules/samtools/sort/test.yml

@@ -4,5 +4,5 @@
     - samtools
     - samtools/sort
   files:
-    - path: output/samtools/test.bam
-      md5sum: bdc2d9e3f579f84df1e242207b627f89
+    - path: output/samtools/test.sorted.bam
+      md5sum: bbb2db225f140e69a4ac577f74ccc90f

tests/subworkflows/nf-core/align_bowtie2/main.nf

@@ -0,0 +1,27 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { BOWTIE2_BUILD } from '../../../../modules/bowtie2/build/main.nf' addParams( options: [:] )
+include { ALIGN_BOWTIE2 } from '../../../../subworkflows/nf-core/align_bowtie2/main.nf' addParams( 'samtools_sort_options': ['suffix': '.sorted'] )
+
+workflow test_align_bowtie2_single_end {
+    input = [ [ id:'test', single_end:true ], // meta map
+              [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ]
+            ]
+    fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+
+    BOWTIE2_BUILD ( fasta )
+    ALIGN_BOWTIE2 ( input, BOWTIE2_BUILD.out.index )
+}
+
+workflow test_align_bowtie2_paired_end {
+    input = [ [ id:'test', single_end:false ], // meta map
+              [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true),
+                file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) ]
+            ]
+    fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+
+    BOWTIE2_BUILD ( fasta )
+    ALIGN_BOWTIE2 ( input, BOWTIE2_BUILD.out.index )
+}