From 606a388ec31743f47848c511e8dc1099cf87fe42 Mon Sep 17 00:00:00 2001 From: CMGG Cloud Team Date: Tue, 19 Mar 2024 20:26:09 +0100 Subject: [PATCH 1/7] migrate picard_addorreplacereadgroups to nf-testand and add cram support --- .../picard/addorreplacereadgroups/main.nf | 34 +++- .../picard/addorreplacereadgroups/meta.yml | 18 +- .../addorreplacereadgroups/tests/bam.config | 4 +- .../addorreplacereadgroups/tests/cram.config | 13 ++ .../addorreplacereadgroups/tests/main.nf.test | 86 +++++++++ .../tests/main.nf.test.snap | 165 ++++++++++++++++++ .../addorreplacereadgroups/tests/tags.yml | 2 + tests/config/pytest_modules.yml | 3 - .../picard/addorreplacereadgroups/main.nf | 15 -- .../picard/addorreplacereadgroups/test.yml | 17 -- 10 files changed, 310 insertions(+), 47 deletions(-) rename tests/modules/nf-core/picard/addorreplacereadgroups/nextflow.config => modules/nf-core/picard/addorreplacereadgroups/tests/bam.config (71%) create mode 100644 modules/nf-core/picard/addorreplacereadgroups/tests/cram.config create mode 100644 modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test create mode 100644 modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test.snap create mode 100644 modules/nf-core/picard/addorreplacereadgroups/tests/tags.yml delete mode 100644 tests/modules/nf-core/picard/addorreplacereadgroups/main.nf delete mode 100644 tests/modules/nf-core/picard/addorreplacereadgroups/test.yml diff --git a/modules/nf-core/picard/addorreplacereadgroups/main.nf b/modules/nf-core/picard/addorreplacereadgroups/main.nf index dc1a387d3e6..3e83c489622 100644 --- a/modules/nf-core/picard/addorreplacereadgroups/main.nf +++ b/modules/nf-core/picard/addorreplacereadgroups/main.nf @@ -9,11 +9,15 @@ process PICARD_ADDORREPLACEREADGROUPS { input: tuple val(meta), path(bam) + tuple val(meta2), path(fasta) + tuple val(meta3), path(fasta_index) output: - tuple val(meta), path("*.bam"), emit: bam - tuple val(meta), path("*.bai"), emit: bai, optional: true - path "versions.yml" , emit: versions + tuple val(meta), path("*.bam") , emit: bam, optional: true + tuple val(meta), path("*.bai") , emit: bai, optional: true + tuple val(meta), path("*.cram"), emit: cram, optional: true + tuple val(meta), path("*.crai"), emit: crai, optional: true + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -21,6 +25,13 @@ process PICARD_ADDORREPLACEREADGROUPS { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + def suffix = task.ext.suffix ?: "${bam.getExtension()}" + def reference = fasta ? "--REFERENCE_SEQUENCE ${fasta}" : "" + def create_index = ( suffix == "bam" )? "--CREATE_INDEX" : "" + //def create_index = "" + //if (suffix == "BAM") { + // create_index = "--CREATE_INDEX" + //} def avail_mem = 3072 if (!task.memory) { log.info '[Picard AddOrReplaceReadGroups] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' @@ -28,15 +39,17 @@ process PICARD_ADDORREPLACEREADGROUPS { avail_mem = (task.memory.mega*0.8).intValue() } - if ("$bam" == "${prefix}.bam") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" + if ("$bam" == "${prefix}.${suffix}") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" """ picard \\ -Xmx${avail_mem}M \\ AddOrReplaceReadGroups \\ $args \\ + $reference \\ + $create_index \\ --INPUT ${bam} \\ - --OUTPUT ${prefix}.bam + --OUTPUT ${prefix}.${suffix} cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -46,10 +59,15 @@ process PICARD_ADDORREPLACEREADGROUPS { stub: def prefix = task.ext.prefix ?: "${meta.id}" - - if ("$bam" == "${prefix}.bam") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" + def suffix = task.ext.suffix ?: "${bam.getExtension()}" + if ("$bam" == "${prefix}.${suffix}") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" + def create_index = "" + if (suffix == "bam") { + create_index = "touch ${prefix}.${suffix}.bai" + } """ - touch ${prefix}.bam + touch ${prefix}.${suffix} + ${create_index} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/picard/addorreplacereadgroups/meta.yml b/modules/nf-core/picard/addorreplacereadgroups/meta.yml index ab573ac850c..64939fd8f23 100644 --- a/modules/nf-core/picard/addorreplacereadgroups/meta.yml +++ b/modules/nf-core/picard/addorreplacereadgroups/meta.yml @@ -24,6 +24,14 @@ input: type: file description: Input BAM file pattern: "*.{bam}" + - fasta: + type: file + description: Reference genome file + pattern: "*.{fasta,fa,fasta.gz,fa.gz}" + - fasta_index: + type: file + description: Reference genome index file + pattern: "*.{fai,fasta.fai,fa.fai,fasta.gz.fai,fa.gz.fai}" output: - meta: type: map @@ -40,8 +48,16 @@ output: pattern: "*.{bam}" - bai: type: file - description: BAM index file + description: An optional BAM index file pattern: "*.{bai}" + - cram: + type: file + description: Output CRAM file + pattern: "*.{cram}" + - crai: + type: file + description: An optional CRAM index file + pattern: "*.{crai}" authors: - "@sateeshperi" - "@mjcipriano" diff --git a/tests/modules/nf-core/picard/addorreplacereadgroups/nextflow.config b/modules/nf-core/picard/addorreplacereadgroups/tests/bam.config similarity index 71% rename from tests/modules/nf-core/picard/addorreplacereadgroups/nextflow.config rename to modules/nf-core/picard/addorreplacereadgroups/tests/bam.config index 4dc97d09e2f..8ce837e53f8 100644 --- a/tests/modules/nf-core/picard/addorreplacereadgroups/nextflow.config +++ b/modules/nf-core/picard/addorreplacereadgroups/tests/bam.config @@ -1,7 +1,4 @@ process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - withName: 'PICARD_ADDORREPLACEREADGROUPS'{ ext.prefix = { "${meta.id}.replaced"} ext.args = {[ @@ -11,4 +8,5 @@ process { "-SM ${meta.id}" ].join(' ').trim()} } + } diff --git a/modules/nf-core/picard/addorreplacereadgroups/tests/cram.config b/modules/nf-core/picard/addorreplacereadgroups/tests/cram.config new file mode 100644 index 00000000000..966c14d7955 --- /dev/null +++ b/modules/nf-core/picard/addorreplacereadgroups/tests/cram.config @@ -0,0 +1,13 @@ +process { + withName: 'PICARD_ADDORREPLACEREADGROUPS'{ + ext.prefix = { "${meta.id}.replaced"} + ext.args = {[ + "-LB ${meta.id}", + "-PL ILLUMINA", + "-PU bc1", + "-SM ${meta.id}" + ].join(' ').trim()} + ext.suffix = { "cram" } + } + +} diff --git a/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test b/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test new file mode 100644 index 00000000000..d4dfaf70771 --- /dev/null +++ b/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test @@ -0,0 +1,86 @@ + +nextflow_process { + + name "Test Process PICARD_ADDORREPLACEREADGROUPS" + script "../main.nf" + process "PICARD_ADDORREPLACEREADGROUPS" + + tag "modules" + tag "modules_nfcore" + tag "picard" + tag "picard/addorreplacereadgroups" + + test("sarscov2 - bam") { + config "./bam.config" + + when { + process { + """ + input[0] = [ [:], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) ] + input[1] = [ [:], [] ] + input[2] = [ [:], [] ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("homo_sapiens - cram") { + config "./cram.config" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram', checkIfExists: true), ] + ] + input[1] = [ [:], file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) ] + input[2] = [ [:], file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta.fai', checkIfExists: true) ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.cram).match("cram") }, + { assert snapshot(process.out.crai).match("crai") }, + { assert snapshot(process.out.versions).match("versions") }, + ) + } + + } + + test("sarscov2 - bam - stub") { + config "./bam.config" + options "-stub" + + when { + process { + """ + input[0] = [ [:], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) ] + input[1] = [ [:], [] ] + input[2] = [ [:], [] ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test.snap b/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test.snap new file mode 100644 index 00000000000..a9eab987453 --- /dev/null +++ b/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test.snap @@ -0,0 +1,165 @@ +{ + "sarscov2 - bam - stub": { + "content": [ + { + "0": [ + [ + { + + }, + "null.replaced.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + + }, + "null.replaced.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + "versions.yml:md5,0a6f049f94501dcf23aabfbc0e272891" + ], + "bai": [ + [ + { + + }, + "null.replaced.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "bam": [ + [ + { + + }, + "null.replaced.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "crai": [ + + ], + "cram": [ + + ], + "versions": [ + "versions.yml:md5,0a6f049f94501dcf23aabfbc0e272891" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T20:13:15.246897" + }, + "cram": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.replaced.cram:md5,cdaa8c1705c46b905c483c578b47106b" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T17:32:24.87662" + }, + "versions": { + "content": [ + [ + "versions.yml:md5,0a6f049f94501dcf23aabfbc0e272891" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T17:32:24.906639" + }, + "crai": { + "content": [ + [ + + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T17:29:35.062295" + }, + "sarscov2 - bam": { + "content": [ + { + "0": [ + [ + { + + }, + "null.replaced.bam:md5,4f962d8dd2d1c5cc65d0a10ecac8123d" + ] + ], + "1": [ + [ + { + + }, + "null.replaced.bai:md5,cf080f52063ce7f6e28b43f22d52afd9" + ] + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + "versions.yml:md5,0a6f049f94501dcf23aabfbc0e272891" + ], + "bai": [ + [ + { + + }, + "null.replaced.bai:md5,cf080f52063ce7f6e28b43f22d52afd9" + ] + ], + "bam": [ + [ + { + + }, + "null.replaced.bam:md5,4f962d8dd2d1c5cc65d0a10ecac8123d" + ] + ], + "crai": [ + + ], + "cram": [ + + ], + "versions": [ + "versions.yml:md5,0a6f049f94501dcf23aabfbc0e272891" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T20:09:37.278254" + } +} \ No newline at end of file diff --git a/modules/nf-core/picard/addorreplacereadgroups/tests/tags.yml b/modules/nf-core/picard/addorreplacereadgroups/tests/tags.yml new file mode 100644 index 00000000000..733010d625d --- /dev/null +++ b/modules/nf-core/picard/addorreplacereadgroups/tests/tags.yml @@ -0,0 +1,2 @@ +picard/addorreplacereadgroups: + - "modules/nf-core/picard/addorreplacereadgroups/**" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 4f9637b251e..87f77fd3776 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -1595,9 +1595,6 @@ phispy: phyloflash: - modules/nf-core/phyloflash/** - tests/modules/nf-core/phyloflash/** -picard/addorreplacereadgroups: - - modules/nf-core/picard/addorreplacereadgroups/** - - tests/modules/nf-core/picard/addorreplacereadgroups/** picard/bedtointervallist: - modules/nf-core/picard/bedtointervallist/** - tests/modules/nf-core/picard/bedtointervallist/** diff --git a/tests/modules/nf-core/picard/addorreplacereadgroups/main.nf b/tests/modules/nf-core/picard/addorreplacereadgroups/main.nf deleted file mode 100644 index e0196f34016..00000000000 --- a/tests/modules/nf-core/picard/addorreplacereadgroups/main.nf +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { PICARD_ADDORREPLACEREADGROUPS } from '../../../../../modules/nf-core/picard/addorreplacereadgroups/main.nf' - -workflow test_picard_addorreplacereadgroups { - - input = [ - [ id:'test', single_end:false ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) - ] - - PICARD_ADDORREPLACEREADGROUPS ( input ) -} diff --git a/tests/modules/nf-core/picard/addorreplacereadgroups/test.yml b/tests/modules/nf-core/picard/addorreplacereadgroups/test.yml deleted file mode 100644 index 4ffe7234bd7..00000000000 --- a/tests/modules/nf-core/picard/addorreplacereadgroups/test.yml +++ /dev/null @@ -1,17 +0,0 @@ -- name: picard addorreplacereadgroups test_picard_addorreplacereadgroups - command: nextflow run ./tests/modules/nf-core/picard/addorreplacereadgroups -entry test_picard_addorreplacereadgroups -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/picard/addorreplacereadgroups/nextflow.config - tags: - - picard/addorreplacereadgroups - - picard - files: - - path: output/picard/test.replaced.bam - md5sum: 9068e93d318470f8ab34ccbbbbb15b2b - - path: output/picard/versions.yml -- name: picard addorreplacereadgroups test_picard_addorreplacereadgroups stub - command: nextflow run ./tests/modules/nf-core/picard/addorreplacereadgroups -entry test_picard_addorreplacereadgroups -c ./tests/config/nextflow.config -stub-run - tags: - - picard - - picard/addorreplacereadgroups - files: - - path: output/picard/test.replaced.bam - - path: output/picard/versions.yml From 1c3cf1e76f2edeeb7c0f719bbc9532e523a6020e Mon Sep 17 00:00:00 2001 From: CMGG Cloud Team Date: Tue, 19 Mar 2024 20:29:39 +0100 Subject: [PATCH 2/7] remove unused code --- modules/nf-core/picard/addorreplacereadgroups/main.nf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/nf-core/picard/addorreplacereadgroups/main.nf b/modules/nf-core/picard/addorreplacereadgroups/main.nf index 3e83c489622..9670a66a460 100644 --- a/modules/nf-core/picard/addorreplacereadgroups/main.nf +++ b/modules/nf-core/picard/addorreplacereadgroups/main.nf @@ -28,10 +28,6 @@ process PICARD_ADDORREPLACEREADGROUPS { def suffix = task.ext.suffix ?: "${bam.getExtension()}" def reference = fasta ? "--REFERENCE_SEQUENCE ${fasta}" : "" def create_index = ( suffix == "bam" )? "--CREATE_INDEX" : "" - //def create_index = "" - //if (suffix == "BAM") { - // create_index = "--CREATE_INDEX" - //} def avail_mem = 3072 if (!task.memory) { log.info '[Picard AddOrReplaceReadGroups] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' From 8385ce2181b7dc4417e5fe7e843df4899eecb7db Mon Sep 17 00:00:00 2001 From: CMGG Cloud Team Date: Wed, 20 Mar 2024 12:19:43 +0100 Subject: [PATCH 3/7] fix unstable checksums in tests --- .../addorreplacereadgroups/tests/main.nf.test | 6 +- .../tests/main.nf.test.snap | 92 ++++--------------- 2 files changed, 19 insertions(+), 79 deletions(-) diff --git a/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test b/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test index d4dfaf70771..6d50cfe2689 100644 --- a/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test +++ b/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test @@ -26,7 +26,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(file(process.out.bam[0][1]).name).match("bam_name") }, + { assert snapshot(file(process.out.bai[0][1]).name).match("bai_name") }, ) } @@ -52,8 +53,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.cram).match("cram") }, - { assert snapshot(process.out.crai).match("crai") }, + { assert snapshot(file(process.out.cram[0][1]).name).match("cram_name") }, { assert snapshot(process.out.versions).match("versions") }, ) } diff --git a/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test.snap b/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test.snap index a9eab987453..0f43f33218f 100644 --- a/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test.snap +++ b/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test.snap @@ -1,4 +1,14 @@ { + "bam_name": { + "content": [ + "null.replaced.bam" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T12:05:21.587215" + }, "sarscov2 - bam - stub": { "content": [ { @@ -60,23 +70,6 @@ }, "timestamp": "2024-03-19T20:13:15.246897" }, - "cram": { - "content": [ - [ - [ - { - "id": "test" - }, - "test.replaced.cram:md5,cdaa8c1705c46b905c483c578b47106b" - ] - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-03-19T17:32:24.87662" - }, "versions": { "content": [ [ @@ -89,77 +82,24 @@ }, "timestamp": "2024-03-19T17:32:24.906639" }, - "crai": { + "cram_name": { "content": [ - [ - - ] + "test.replaced.cram" ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-19T17:29:35.062295" + "timestamp": "2024-03-20T12:06:05.10743" }, - "sarscov2 - bam": { + "bai_name": { "content": [ - { - "0": [ - [ - { - - }, - "null.replaced.bam:md5,4f962d8dd2d1c5cc65d0a10ecac8123d" - ] - ], - "1": [ - [ - { - - }, - "null.replaced.bai:md5,cf080f52063ce7f6e28b43f22d52afd9" - ] - ], - "2": [ - - ], - "3": [ - - ], - "4": [ - "versions.yml:md5,0a6f049f94501dcf23aabfbc0e272891" - ], - "bai": [ - [ - { - - }, - "null.replaced.bai:md5,cf080f52063ce7f6e28b43f22d52afd9" - ] - ], - "bam": [ - [ - { - - }, - "null.replaced.bam:md5,4f962d8dd2d1c5cc65d0a10ecac8123d" - ] - ], - "crai": [ - - ], - "cram": [ - - ], - "versions": [ - "versions.yml:md5,0a6f049f94501dcf23aabfbc0e272891" - ] - } + "null.replaced.bai" ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-19T20:09:37.278254" + "timestamp": "2024-03-20T12:05:21.604866" } } \ No newline at end of file From b115804b2476fc57c1ffcf33062924fcacd64b2b Mon Sep 17 00:00:00 2001 From: CMGG Cloud Team Date: Wed, 20 Mar 2024 12:52:20 +0100 Subject: [PATCH 4/7] remove crai and add meta 2 and 3 --- .../nf-core/picard/addorreplacereadgroups/main.nf | 1 - .../nf-core/picard/addorreplacereadgroups/meta.yml | 14 ++++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/nf-core/picard/addorreplacereadgroups/main.nf b/modules/nf-core/picard/addorreplacereadgroups/main.nf index 9670a66a460..4ea018d56bd 100644 --- a/modules/nf-core/picard/addorreplacereadgroups/main.nf +++ b/modules/nf-core/picard/addorreplacereadgroups/main.nf @@ -16,7 +16,6 @@ process PICARD_ADDORREPLACEREADGROUPS { tuple val(meta), path("*.bam") , emit: bam, optional: true tuple val(meta), path("*.bai") , emit: bai, optional: true tuple val(meta), path("*.cram"), emit: cram, optional: true - tuple val(meta), path("*.crai"), emit: crai, optional: true path "versions.yml" , emit: versions when: diff --git a/modules/nf-core/picard/addorreplacereadgroups/meta.yml b/modules/nf-core/picard/addorreplacereadgroups/meta.yml index 64939fd8f23..17110b7875f 100644 --- a/modules/nf-core/picard/addorreplacereadgroups/meta.yml +++ b/modules/nf-core/picard/addorreplacereadgroups/meta.yml @@ -20,6 +20,16 @@ input: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] + - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - bam: type: file description: Input BAM file @@ -54,10 +64,6 @@ output: type: file description: Output CRAM file pattern: "*.{cram}" - - crai: - type: file - description: An optional CRAM index file - pattern: "*.{crai}" authors: - "@sateeshperi" - "@mjcipriano" From 0fb97ce73603090c2e63d6853ffec9eb07db6fa8 Mon Sep 17 00:00:00 2001 From: CMGG Cloud Team Date: Wed, 20 Mar 2024 12:58:55 +0100 Subject: [PATCH 5/7] update snap --- .../picard/addorreplacereadgroups/tests/main.nf.test.snap | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test.snap b/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test.snap index 0f43f33218f..f80eaba047d 100644 --- a/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test.snap +++ b/modules/nf-core/picard/addorreplacereadgroups/tests/main.nf.test.snap @@ -32,9 +32,6 @@ ], "3": [ - - ], - "4": [ "versions.yml:md5,0a6f049f94501dcf23aabfbc0e272891" ], "bai": [ @@ -52,9 +49,6 @@ }, "null.replaced.bam:md5,d41d8cd98f00b204e9800998ecf8427e" ] - ], - "crai": [ - ], "cram": [ @@ -68,7 +62,7 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-19T20:13:15.246897" + "timestamp": "2024-03-20T12:54:03.639457" }, "versions": { "content": [ From 352aaaaa2c59bb8cee65a5368f093544da0e2420 Mon Sep 17 00:00:00 2001 From: CMGG Cloud Team Date: Wed, 20 Mar 2024 13:27:00 +0100 Subject: [PATCH 6/7] Duplicate as gatk4 module --- .../addorreplacereadgroups/environment.yml | 7 ++ .../gatk4/addorreplacereadgroups/main.nf | 71 +++++++++++++ .../gatk4/addorreplacereadgroups/meta.yml | 80 +++++++++++++++ .../addorreplacereadgroups/tests/bam.config | 12 +++ .../addorreplacereadgroups/tests/cram.config | 13 +++ .../addorreplacereadgroups/tests/main.nf.test | 85 ++++++++++++++++ .../tests/main.nf.test.snap | 99 +++++++++++++++++++ .../addorreplacereadgroups/tests/tags.yml | 2 + 8 files changed, 369 insertions(+) create mode 100644 modules/nf-core/gatk4/addorreplacereadgroups/environment.yml create mode 100644 modules/nf-core/gatk4/addorreplacereadgroups/main.nf create mode 100644 modules/nf-core/gatk4/addorreplacereadgroups/meta.yml create mode 100644 modules/nf-core/gatk4/addorreplacereadgroups/tests/bam.config create mode 100644 modules/nf-core/gatk4/addorreplacereadgroups/tests/cram.config create mode 100644 modules/nf-core/gatk4/addorreplacereadgroups/tests/main.nf.test create mode 100644 modules/nf-core/gatk4/addorreplacereadgroups/tests/main.nf.test.snap create mode 100644 modules/nf-core/gatk4/addorreplacereadgroups/tests/tags.yml diff --git a/modules/nf-core/gatk4/addorreplacereadgroups/environment.yml b/modules/nf-core/gatk4/addorreplacereadgroups/environment.yml new file mode 100644 index 00000000000..6a38f44e9f3 --- /dev/null +++ b/modules/nf-core/gatk4/addorreplacereadgroups/environment.yml @@ -0,0 +1,7 @@ +name: picard_addorreplacereadgroups +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::gatk4=4.5.0.0 diff --git a/modules/nf-core/gatk4/addorreplacereadgroups/main.nf b/modules/nf-core/gatk4/addorreplacereadgroups/main.nf new file mode 100644 index 00000000000..861f7045a14 --- /dev/null +++ b/modules/nf-core/gatk4/addorreplacereadgroups/main.nf @@ -0,0 +1,71 @@ +process GATK4_ADDORREPLACEREADGROUPS { + tag "$meta.id" + label 'process_low' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/gatk4:4.5.0.0--py36hdfd78af_0': + 'biocontainers/gatk4:4.5.0.0--py36hdfd78af_0' }" + + input: + tuple val(meta), path(bam) + tuple val(meta2), path(fasta) + tuple val(meta3), path(fasta_index) + + output: + tuple val(meta), path("*.bam") , emit: bam, optional: true + tuple val(meta), path("*.bai") , emit: bai, optional: true + tuple val(meta), path("*.cram"), emit: cram, optional: true + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def suffix = task.ext.suffix ?: "${bam.getExtension()}" + def reference = fasta ? "--REFERENCE_SEQUENCE ${fasta}" : "" + def create_index = ( suffix == "bam" )? "--CREATE_INDEX" : "" + def avail_mem = 3072 + if (!task.memory) { + log.info '[GATK AddOrReplaceReadGroups] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' + } else { + avail_mem = (task.memory.mega*0.8).intValue() + } + + if ("$bam" == "${prefix}.${suffix}") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" + + """ + gatk --java-options "-Xmx${avail_mem}M -XX:-UsePerfData" \\ + AddOrReplaceReadGroups \\ + $args \\ + $reference \\ + $create_index \\ + --INPUT ${bam} \\ + --OUTPUT ${prefix}.${suffix} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + gatk4: \$(echo \$(gatk AddOrReplaceReadGroups --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + def suffix = task.ext.suffix ?: "${bam.getExtension()}" + if ("$bam" == "${prefix}.${suffix}") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" + def create_index = "" + if (suffix == "bam") { + create_index = "touch ${prefix}.${suffix}.bai" + } + """ + touch ${prefix}.${suffix} + ${create_index} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + gatk4: \$(echo \$(gatk AddOrReplaceReadGroups --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS + """ +} diff --git a/modules/nf-core/gatk4/addorreplacereadgroups/meta.yml b/modules/nf-core/gatk4/addorreplacereadgroups/meta.yml new file mode 100644 index 00000000000..d78081d5db9 --- /dev/null +++ b/modules/nf-core/gatk4/addorreplacereadgroups/meta.yml @@ -0,0 +1,80 @@ +name: gatk4_addorreplacereadgroups +description: Assigns all the reads in a file to a single new read-group +keywords: + - add + - replace + - read-group + - picard + - gatk +tools: + - gatk4: + description: | + Developed in the Data Sciences Platform at the Broad Institute, the toolkit offers a wide variety of tools + with a primary focus on variant discovery and genotyping. Its powerful processing engine + and high-performance computing features make it capable of taking on projects of any size. + homepage: https://gatk.broadinstitute.org/hc/en-us + documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s + doi: 10.1158/1538-7445.AM2017-3590 + licence: ["Apache-2.0"] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - bam: + type: file + description: Input BAM file + pattern: "*.{bam}" + - fasta: + type: file + description: Reference genome file + pattern: "*.{fasta,fa,fasta.gz,fa.gz}" + - fasta_index: + type: file + description: Reference genome index file + pattern: "*.{fai,fasta.fai,fa.fai,fasta.gz.fai,fa.gz.fai}" +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - bam: + type: file + description: Output BAM file + pattern: "*.{bam}" + - bai: + type: file + description: An optional BAM index file + pattern: "*.{bai}" + - cram: + type: file + description: Output CRAM file + pattern: "*.{cram}" +authors: + - "@sateeshperi" + - "@mjcipriano" + - "@hseabolt" + - "@cmatKhan" + - "@muffato" +maintainers: + - "@sateeshperi" + - "@mjcipriano" + - "@hseabolt" + - "@cmatKhan" + - "@muffato" diff --git a/modules/nf-core/gatk4/addorreplacereadgroups/tests/bam.config b/modules/nf-core/gatk4/addorreplacereadgroups/tests/bam.config new file mode 100644 index 00000000000..21573297604 --- /dev/null +++ b/modules/nf-core/gatk4/addorreplacereadgroups/tests/bam.config @@ -0,0 +1,12 @@ +process { + withName: 'GATK4_ADDORREPLACEREADGROUPS'{ + ext.prefix = { "${meta.id}.replaced"} + ext.args = {[ + "-LB ${meta.id}", + "-PL ILLUMINA", + "-PU bc1", + "-SM ${meta.id}" + ].join(' ').trim()} + } + +} diff --git a/modules/nf-core/gatk4/addorreplacereadgroups/tests/cram.config b/modules/nf-core/gatk4/addorreplacereadgroups/tests/cram.config new file mode 100644 index 00000000000..3885afca292 --- /dev/null +++ b/modules/nf-core/gatk4/addorreplacereadgroups/tests/cram.config @@ -0,0 +1,13 @@ +process { + withName: 'GATK4_ADDORREPLACEREADGROUPS'{ + ext.prefix = { "${meta.id}.replaced"} + ext.args = {[ + "-LB ${meta.id}", + "-PL ILLUMINA", + "-PU bc1", + "-SM ${meta.id}" + ].join(' ').trim()} + ext.suffix = { "cram" } + } + +} diff --git a/modules/nf-core/gatk4/addorreplacereadgroups/tests/main.nf.test b/modules/nf-core/gatk4/addorreplacereadgroups/tests/main.nf.test new file mode 100644 index 00000000000..cd140e82776 --- /dev/null +++ b/modules/nf-core/gatk4/addorreplacereadgroups/tests/main.nf.test @@ -0,0 +1,85 @@ +nextflow_process { + + name "Test Process GATK4_ADDORREPLACEREADGROUPS" + script "../main.nf" + process "GATK4_ADDORREPLACEREADGROUPS" + + tag "modules" + tag "modules_nfcore" + tag "gatk4" + tag "gatk4/addorreplacereadgroups" + + test("sarscov2 - bam") { + config "./bam.config" + + when { + process { + """ + input[0] = [ [:], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) ] + input[1] = [ [:], [] ] + input[2] = [ [:], [] ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(file(process.out.bam[0][1]).name).match("bam_name") }, + { assert snapshot(file(process.out.bai[0][1]).name).match("bai_name") }, + ) + } + + } + + test("homo_sapiens - cram") { + config "./cram.config" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram', checkIfExists: true), ] + ] + input[1] = [ [:], file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) ] + input[2] = [ [:], file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta.fai', checkIfExists: true) ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(file(process.out.cram[0][1]).name).match("cram_name") }, + { assert snapshot(process.out.versions).match("versions") }, + ) + } + + } + + test("sarscov2 - bam - stub") { + config "./bam.config" + options "-stub" + + when { + process { + """ + input[0] = [ [:], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) ] + input[1] = [ [:], [] ] + input[2] = [ [:], [] ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/nf-core/gatk4/addorreplacereadgroups/tests/main.nf.test.snap b/modules/nf-core/gatk4/addorreplacereadgroups/tests/main.nf.test.snap new file mode 100644 index 00000000000..93a87cc39eb --- /dev/null +++ b/modules/nf-core/gatk4/addorreplacereadgroups/tests/main.nf.test.snap @@ -0,0 +1,99 @@ +{ + "bam_name": { + "content": [ + "null.replaced.bam" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T12:05:21.587215" + }, + "sarscov2 - bam - stub": { + "content": [ + { + "0": [ + [ + { + + }, + "null.replaced.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + + }, + "null.replaced.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + + ], + "3": [ + "versions.yml:md5,da55295b55986cfcf65d41745c7bda4c" + ], + "bai": [ + [ + { + + }, + "null.replaced.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "bam": [ + [ + { + + }, + "null.replaced.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "cram": [ + + ], + "versions": [ + "versions.yml:md5,da55295b55986cfcf65d41745c7bda4c" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T13:19:33.815262" + }, + "versions": { + "content": [ + [ + "versions.yml:md5,da55295b55986cfcf65d41745c7bda4c" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T13:19:18.103325" + }, + "cram_name": { + "content": [ + "test.replaced.cram" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T12:06:05.10743" + }, + "bai_name": { + "content": [ + "null.replaced.bai" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T12:05:21.604866" + } +} \ No newline at end of file diff --git a/modules/nf-core/gatk4/addorreplacereadgroups/tests/tags.yml b/modules/nf-core/gatk4/addorreplacereadgroups/tests/tags.yml new file mode 100644 index 00000000000..246ed5573bc --- /dev/null +++ b/modules/nf-core/gatk4/addorreplacereadgroups/tests/tags.yml @@ -0,0 +1,2 @@ +gatk4/addorreplacereadgroups: + - "modules/nf-core/gatk4/addorreplacereadgroups/**" From 736ce06cffa5c444fddbd4fcc7ca16e973d11e14 Mon Sep 17 00:00:00 2001 From: CMGG Cloud Team Date: Wed, 20 Mar 2024 13:34:13 +0100 Subject: [PATCH 7/7] name conflict --- modules/nf-core/gatk4/addorreplacereadgroups/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/gatk4/addorreplacereadgroups/environment.yml b/modules/nf-core/gatk4/addorreplacereadgroups/environment.yml index 6a38f44e9f3..82f9173806e 100644 --- a/modules/nf-core/gatk4/addorreplacereadgroups/environment.yml +++ b/modules/nf-core/gatk4/addorreplacereadgroups/environment.yml @@ -1,4 +1,4 @@ -name: picard_addorreplacereadgroups +name: gatk4_addorreplacereadgroups channels: - conda-forge - bioconda