From ff7fea5524b9180b30cd512c1cd06c0b89de185f Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Mon, 18 Mar 2024 11:55:15 +0100 Subject: [PATCH 01/37] feat: add wittyer as module --- modules/nf-core/wittyer/environment.yml | 9 ++ modules/nf-core/wittyer/main.nf | 95 ++++++++++++++++++++++ modules/nf-core/wittyer/meta.yml | 57 +++++++++++++ modules/nf-core/wittyer/tests/main.nf.test | 73 +++++++++++++++++ modules/nf-core/wittyer/tests/tags.yml | 2 + 5 files changed, 236 insertions(+) create mode 100644 modules/nf-core/wittyer/environment.yml create mode 100644 modules/nf-core/wittyer/main.nf create mode 100644 modules/nf-core/wittyer/meta.yml create mode 100644 modules/nf-core/wittyer/tests/main.nf.test create mode 100644 modules/nf-core/wittyer/tests/tags.yml diff --git a/modules/nf-core/wittyer/environment.yml b/modules/nf-core/wittyer/environment.yml new file mode 100644 index 00000000000..6783e3c68a8 --- /dev/null +++ b/modules/nf-core/wittyer/environment.yml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +name: "wittyer" +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - "YOUR-TOOL-HERE" diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf new file mode 100644 index 00000000000..c3abd23a07d --- /dev/null +++ b/modules/nf-core/wittyer/main.nf @@ -0,0 +1,95 @@ +// TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) +// https://github.com/nf-core/modules/tree/master/modules/nf-core/ +// You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: +// https://nf-co.re/join +// TODO nf-core: A module file SHOULD only define input and output files as command-line parameters. +// All other parameters MUST be provided using the "task.ext" directive, see here: +// https://www.nextflow.io/docs/latest/process.html#ext +// where "task.ext" is a string. +// Any parameters that need to be evaluated in the context of a particular sample +// e.g. single-end/paired-end data MUST also be defined and evaluated appropriately. +// TODO nf-core: Software that can be piped together SHOULD be added to separate module files +// unless there is a run-time, storage advantage in implementing in this way +// e.g. it's ok to have a single module for bwa to output BAM instead of SAM: +// bwa mem | samtools view -B -T ref.fasta +// TODO nf-core: Optional inputs are not currently supported by Nextflow. However, using an empty +// list (`[]`) instead of a file can be used to work around this issue. + +process WITTYER { + tag "$meta.id" + label 'process_single' + + // TODO nf-core: List required Conda package(s). + // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). + // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. + // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE': + 'biocontainers/YOUR-TOOL-HERE' }" + + input: + // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" + // MUST be provided as an input via a Groovy Map called "meta". + // This information may not be required in some instances e.g. indexing reference genome files: + // https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf + // TODO nf-core: Where applicable please provide/convert compressed files as input/output + // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. + tuple val(meta), path(bam) + + output: + // TODO nf-core: Named file extensions MUST be emitted for ALL output channels + tuple val(meta), path("*.bam"), emit: bam + // TODO nf-core: List additional required output channels/values here + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "BCL2FASTQ module does not support Conda. Please use Docker / Singularity / Podman instead." + } + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 + // If the software is unable to output a version number on the command-line then it can be manually specified + // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf + // Each software used MUST provide the software name and version number in the YAML version file (versions.yml) + // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "task.ext.args" directive + // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter + // using the Nextflow "task" variable e.g. "--threads $task.cpus" + // TODO nf-core: Please replace the example samtools command below with your module's command + // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) + """ + samtools \\ + sort \\ + $args \\ + -@ $task.cpus \\ + -o ${prefix}.bam \\ + -T $prefix \\ + $bam + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + wittyer: \$(samtools --version |& sed '1!d ; s/samtools //') + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + // TODO nf-core: A stub section should mimic the execution of the original module as best as possible + // Have a look at the following examples: + // Simple example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bcftools/annotate/main.nf#L47-L63 + // Complex example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bedtools/split/main.nf#L38-L54 + """ + touch ${prefix}.bam + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + wittyer: \$(samtools --version |& sed '1!d ; s/samtools //') + END_VERSIONS + """ +} diff --git a/modules/nf-core/wittyer/meta.yml b/modules/nf-core/wittyer/meta.yml new file mode 100644 index 00000000000..9b2a1e6ef3e --- /dev/null +++ b/modules/nf-core/wittyer/meta.yml @@ -0,0 +1,57 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "wittyer" +## TODO nf-core: Add a description of the module and list keywords +description: write your description here +keywords: + - sort + - example + - genomics +tools: + - "wittyer": + ## TODO nf-core: Add a description and other details for the software below + description: "" + homepage: "" + documentation: "" + tool_dev_url: "" + doi: "" + licence: + +## TODO nf-core: Add a description of all of the variables used as input +input: + # Only when we have meta + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + + ## TODO nf-core: Delete / customise this example input + - bam: + type: file + description: Sorted BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + +## TODO nf-core: Add a description of all of the variables used as output +output: + #Only when we have meta + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + ## TODO nf-core: Delete / customise this example output + - bam: + type: file + description: Sorted BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + +authors: + - "@famosab" +maintainers: + - "@famosab" diff --git a/modules/nf-core/wittyer/tests/main.nf.test b/modules/nf-core/wittyer/tests/main.nf.test new file mode 100644 index 00000000000..3234fda9c49 --- /dev/null +++ b/modules/nf-core/wittyer/tests/main.nf.test @@ -0,0 +1,73 @@ +// TODO nf-core: Once you have added the required tests, please run the following command to build this file: +// nf-core modules test wittyer +nextflow_process { + + name "Test Process WITTYER" + script "../main.nf" + process "WITTYER" + + tag "modules" + tag "modules_nfcore" + tag "wittyer" + + // TODO nf-core: Change the test name preferably indicating the test-data and file-format used + test("sarscov2 - bam") { + + // TODO nf-core: If you are created a test for a chained module + // (the module requires running more than one process to generate the required output) + // add the 'setup' method here. + // You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules). + + when { + process { + """ + // TODO nf-core: define inputs of the process here. Example: + + input[0] = [ + [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + //TODO nf-core: Add all required assertions to verify the test output. + // See https://nf-co.re/docs/contributing/tutorials/nf-test_assertions for more information and examples. + ) + } + + } + + // TODO nf-core: Change the test name preferably indicating the test-data and file-format used but keep the " - stub" suffix. + test("sarscov2 - bam - stub") { + + options "-stub" + + when { + process { + """ + // TODO nf-core: define inputs of the process here. Example: + + input[0] = [ + [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + //TODO nf-core: Add all required assertions to verify the test output. + ) + } + + } + +} diff --git a/modules/nf-core/wittyer/tests/tags.yml b/modules/nf-core/wittyer/tests/tags.yml new file mode 100644 index 00000000000..177db940a55 --- /dev/null +++ b/modules/nf-core/wittyer/tests/tags.yml @@ -0,0 +1,2 @@ +wittyer: + - "modules/nf-core/wittyer/**" From 03da21d07abac29d11a8a45d9b859d3f31a90abb Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Mon, 18 Mar 2024 14:08:58 +0100 Subject: [PATCH 02/37] fix: adjust output and version --- modules/nf-core/wittyer/main.nf | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index c3abd23a07d..a1f7cfda326 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -23,10 +23,11 @@ process WITTYER { // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. - conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE': - 'biocontainers/YOUR-TOOL-HERE' }" + // conda "${moduleDir}/environment.yml" + container "nf-core/wittier:0.3.3" + // "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + // 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE': + // 'biocontainers/YOUR-TOOL-HERE' }" input: // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" @@ -39,9 +40,13 @@ process WITTYER { output: // TODO nf-core: Named file extensions MUST be emitted for ALL output channels - tuple val(meta), path("*.bam"), emit: bam + // directory where all files will be located + // overall per-sample-pair stats summary + // Wittyer.stats.json + // Additional vcf file for each sample pair + tuple val(meta), path("${prefix}/"), emit: wittyer_results // TODO nf-core: List additional required output channels/values here - path "versions.yml" , emit: versions + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -71,9 +76,15 @@ process WITTYER { -T $prefix \\ $bam + wittyer \\ + -i input.vcf \\ + -t truth.vcf \\ + -o outputdir \\ + --bpDistance= + cat <<-END_VERSIONS > versions.yml "${task.process}": - wittyer: \$(samtools --version |& sed '1!d ; s/samtools //') + wittyer: \$(wittyer --version |& sed '1!d ; s/witty.er //') END_VERSIONS """ @@ -89,7 +100,7 @@ process WITTYER { cat <<-END_VERSIONS > versions.yml "${task.process}": - wittyer: \$(samtools --version |& sed '1!d ; s/samtools //') + wittyer: \$(wittyer --version |& sed '1!d ; s/witty.er //') END_VERSIONS """ } From 699666c1db2383fdeed89b22d40e5427bc510fb4 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Mon, 18 Mar 2024 14:32:44 +0100 Subject: [PATCH 03/37] fix: copy output and script from local --- modules/nf-core/wittyer/main.nf | 52 ++++++++++++++------------------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index a1f7cfda326..71e964c6f37 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -39,14 +39,11 @@ process WITTYER { tuple val(meta), path(bam) output: - // TODO nf-core: Named file extensions MUST be emitted for ALL output channels - // directory where all files will be located - // overall per-sample-pair stats summary - // Wittyer.stats.json - // Additional vcf file for each sample pair - tuple val(meta), path("${prefix}/"), emit: wittyer_results - // TODO nf-core: List additional required output channels/values here - path "versions.yml" , emit: versions + tuple val(meta), path("*ConfigFileUsed.json") , emit: config + tuple val(meta), path("*.Stats.json") , emit: report + tuple val(meta), path("*eval.vcf.gz") , emit: bench_vcf + tuple val(meta), path("*eval.vcf.gz.tbi") , emit: bench_vcf_gzi + path "versions.yml" when: task.ext.when == null || task.ext.when @@ -56,31 +53,26 @@ process WITTYER { if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { error "BCL2FASTQ module does not support Conda. Please use Docker / Singularity / Podman instead." } - def args = task.ext.args ?: '' + def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 - // If the software is unable to output a version number on the command-line then it can be manually specified - // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf - // Each software used MUST provide the software name and version number in the YAML version file (versions.yml) - // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "task.ext.args" directive - // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter - // using the Nextflow "task" variable e.g. "--threads $task.cpus" - // TODO nf-core: Please replace the example samtools command below with your module's command - // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) + def regions = bed ? "--includeBed=$bed" : "" + def config = config ? "--configFile=$config" : "" """ - samtools \\ - sort \\ - $args \\ - -@ $task.cpus \\ - -o ${prefix}.bam \\ - -T $prefix \\ - $bam + mkdir bench + dotnet /opt/Wittyer/Wittyer.dll \\ + --truthVcf=${truth_vcf} \\ + --inputVcf=${vcf} \\ + --outputDirectory=bench \\ + ${regions} \\ + ${config} \\ + ${args} + + mv bench/Wittyer.ConfigFileUsed.json ${prefix}.ConfigFileUsed.json + mv bench/Wittyer.Stats.json ${prefix}.Stats.json + mv bench/*.vcf.gz ${prefix}.eval.vcf.gz + mv bench/*.vcf.gz.tbi ${prefix}.eval.vcf.gz.tbi - wittyer \\ - -i input.vcf \\ - -t truth.vcf \\ - -o outputdir \\ - --bpDistance= + rm -rf bench cat <<-END_VERSIONS > versions.yml "${task.process}": From 1efcfe32a2ac36ef2c36a34f189bf235da80834c Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Mon, 18 Mar 2024 15:04:19 +0100 Subject: [PATCH 04/37] feat: fill in in meta file --- modules/nf-core/wittyer/meta.yml | 107 +++++++++++++++++-------------- 1 file changed, 58 insertions(+), 49 deletions(-) diff --git a/modules/nf-core/wittyer/meta.yml b/modules/nf-core/wittyer/meta.yml index 9b2a1e6ef3e..4414f2c5808 100644 --- a/modules/nf-core/wittyer/meta.yml +++ b/modules/nf-core/wittyer/meta.yml @@ -1,57 +1,66 @@ ---- # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json name: "wittyer" -## TODO nf-core: Add a description of the module and list keywords -description: write your description here +description: A large variant benchmarking tool analogous to hap.py for small variants. keywords: - - sort - - example - - genomics +- structural-variants +- benchmarking +- vcf tools: - - "wittyer": - ## TODO nf-core: Add a description and other details for the software below - description: "" - homepage: "" - documentation: "" - tool_dev_url: "" - doi: "" - licence: - -## TODO nf-core: Add a description of all of the variables used as input +- "wittyer": + description: "Illumina tool for large variant benchmarking" + homepage: "https://github.com/Illumina/witty.er" + documentation: "https://github.com/Illumina/witty.er" + tool_dev_url: "https://github.com/Illumina/witty.er" + licence: "['BSD-2']" input: - # Only when we have meta - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. `[ id:'sample1', single_end:false ]` - - ## TODO nf-core: Delete / customise this example input - - bam: - type: file - description: Sorted BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" - -## TODO nf-core: Add a description of all of the variables used as output +- meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` +- query_vcf: + type: file + description: A VCF with called variants to benchmark against the standard + pattern: "*.{vcf,vcf.gz}" +- query_vcf_index: + type: file + description: The index of the called VCF (optional) + pattern: "*.tbi" +- truth_vcf: + type: file + description: A standard VCF to compare against + pattern: "*.{vcf,vcf.gz}" +- truth_vcf_index: + type: file + description: The index of the standard VCF (optional) + pattern: "*.tbi" +- bed: + type: file + description: A BED file specifying regions to be included in the analysis (optional) + pattern: "*.bed" output: - #Only when we have meta - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. `[ id:'sample1', single_end:false ]` - - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" - ## TODO nf-core: Delete / customise this example output - - bam: - type: file - description: Sorted BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" - +- meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` +- versions: + type: file + description: File containing software versions + pattern: "versions.yml" +- report: + type: file + description: Detailed per-sample-pair, per-svtype, per-bin stats + pattern: "*.Stats.json" +- bench_vcf: + type: file + description: Updated query and truth entries merged into one file + pattern: "*eval.vcf.gz" +- bench_vcf_tbi: + type: file + description: Index of merged query and truth entries VCF file + pattern: "*eval.vcf.gz.tbi" authors: - - "@famosab" +- "@famosab" maintainers: - - "@famosab" +- "@famosab" From 9b5c0c730e7e9edc44fee764c8f53b1a020b58e1 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Mon, 18 Mar 2024 15:05:04 +0100 Subject: [PATCH 05/37] fix: remove config file input --- modules/nf-core/wittyer/main.nf | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index 71e964c6f37..666e9b27fd5 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -36,13 +36,12 @@ process WITTYER { // https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf // TODO nf-core: Where applicable please provide/convert compressed files as input/output // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. - tuple val(meta), path(bam) + tuple val(meta), path(query_vcf), path(query_vcf_tbi), path(truth_vcf), path(truth_vcf_tbi), path(bed) output: - tuple val(meta), path("*ConfigFileUsed.json") , emit: config tuple val(meta), path("*.Stats.json") , emit: report tuple val(meta), path("*eval.vcf.gz") , emit: bench_vcf - tuple val(meta), path("*eval.vcf.gz.tbi") , emit: bench_vcf_gzi + tuple val(meta), path("*eval.vcf.gz.tbi") , emit: bench_vcf_tbi path "versions.yml" when: @@ -51,23 +50,20 @@ process WITTYER { script: // Exit if running this module with -profile conda / -profile mamba if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - error "BCL2FASTQ module does not support Conda. Please use Docker / Singularity / Podman instead." + error "WHITTYER module does not support Conda. Please use Docker / Singularity / Podman instead." } def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def regions = bed ? "--includeBed=$bed" : "" - def config = config ? "--configFile=$config" : "" """ mkdir bench - dotnet /opt/Wittyer/Wittyer.dll \\ + wittyer \\ --truthVcf=${truth_vcf} \\ - --inputVcf=${vcf} \\ + --inputVcf=${query_vcf} \\ --outputDirectory=bench \\ ${regions} \\ - ${config} \\ ${args} - mv bench/Wittyer.ConfigFileUsed.json ${prefix}.ConfigFileUsed.json mv bench/Wittyer.Stats.json ${prefix}.Stats.json mv bench/*.vcf.gz ${prefix}.eval.vcf.gz mv bench/*.vcf.gz.tbi ${prefix}.eval.vcf.gz.tbi From b79ee744fc25e8f8b988f600df94da4136ebd150 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Mon, 18 Mar 2024 15:13:29 +0100 Subject: [PATCH 06/37] fix: run prettier --- modules/nf-core/wittyer/meta.yml | 114 +++++++++++++++---------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/modules/nf-core/wittyer/meta.yml b/modules/nf-core/wittyer/meta.yml index 4414f2c5808..784d14c33ed 100644 --- a/modules/nf-core/wittyer/meta.yml +++ b/modules/nf-core/wittyer/meta.yml @@ -2,65 +2,65 @@ name: "wittyer" description: A large variant benchmarking tool analogous to hap.py for small variants. keywords: -- structural-variants -- benchmarking -- vcf + - structural-variants + - benchmarking + - vcf tools: -- "wittyer": - description: "Illumina tool for large variant benchmarking" - homepage: "https://github.com/Illumina/witty.er" - documentation: "https://github.com/Illumina/witty.er" - tool_dev_url: "https://github.com/Illumina/witty.er" - licence: "['BSD-2']" + - "wittyer": + description: "Illumina tool for large variant benchmarking" + homepage: "https://github.com/Illumina/witty.er" + documentation: "https://github.com/Illumina/witty.er" + tool_dev_url: "https://github.com/Illumina/witty.er" + licence: "['BSD-2']" input: -- meta: - type: map - description: | - Groovy Map containing sample information - e.g. `[ id:'sample1', single_end:false ]` -- query_vcf: - type: file - description: A VCF with called variants to benchmark against the standard - pattern: "*.{vcf,vcf.gz}" -- query_vcf_index: - type: file - description: The index of the called VCF (optional) - pattern: "*.tbi" -- truth_vcf: - type: file - description: A standard VCF to compare against - pattern: "*.{vcf,vcf.gz}" -- truth_vcf_index: - type: file - description: The index of the standard VCF (optional) - pattern: "*.tbi" -- bed: - type: file - description: A BED file specifying regions to be included in the analysis (optional) - pattern: "*.bed" + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - query_vcf: + type: file + description: A VCF with called variants to benchmark against the standard + pattern: "*.{vcf,vcf.gz}" + - query_vcf_index: + type: file + description: The index of the called VCF (optional) + pattern: "*.tbi" + - truth_vcf: + type: file + description: A standard VCF to compare against + pattern: "*.{vcf,vcf.gz}" + - truth_vcf_index: + type: file + description: The index of the standard VCF (optional) + pattern: "*.tbi" + - bed: + type: file + description: A BED file specifying regions to be included in the analysis (optional) + pattern: "*.bed" output: -- meta: - type: map - description: | - Groovy Map containing sample information - e.g. `[ id:'sample1', single_end:false ]` -- versions: - type: file - description: File containing software versions - pattern: "versions.yml" -- report: - type: file - description: Detailed per-sample-pair, per-svtype, per-bin stats - pattern: "*.Stats.json" -- bench_vcf: - type: file - description: Updated query and truth entries merged into one file - pattern: "*eval.vcf.gz" -- bench_vcf_tbi: - type: file - description: Index of merged query and truth entries VCF file - pattern: "*eval.vcf.gz.tbi" + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - report: + type: file + description: Detailed per-sample-pair, per-svtype, per-bin stats + pattern: "*.Stats.json" + - bench_vcf: + type: file + description: Updated query and truth entries merged into one file + pattern: "*eval.vcf.gz" + - bench_vcf_tbi: + type: file + description: Index of merged query and truth entries VCF file + pattern: "*eval.vcf.gz.tbi" authors: -- "@famosab" + - "@famosab" maintainers: -- "@famosab" + - "@famosab" From cbb2e0b07f06c3fa19752b0cd673efee62f991f1 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Mon, 18 Mar 2024 17:08:57 +0100 Subject: [PATCH 07/37] feat: explain how to create docker image --- modules/nf-core/wittyer/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 modules/nf-core/wittyer/README.md diff --git a/modules/nf-core/wittyer/README.md b/modules/nf-core/wittyer/README.md new file mode 100644 index 00000000000..8a9e8b004bf --- /dev/null +++ b/modules/nf-core/wittyer/README.md @@ -0,0 +1,26 @@ +# Updating the docker container and making a new module release + +witty.er is a commercial tool from Illumina. The container provided for the witty.er nf-core module is not provided nor supported by Illumina. Updating the witty.er versions in the container and pushing the update to Dockerhub needs to be done manually. + +1. Navigate to the witty.er github repository. - [witty.er](https://github.com/Illumina/witty.er) +2. Download the latest release. + ```bash + wget https://github.com/Illumina/witty.er/archive/refs/tags/.tar.gz + curl https://github.com/Illumina/witty.er/archive/refs/tags/.tar.gz -o .tar.gz + ``` +3. Uncompress the released package. + ```bash + tar -xvf .tar.gz + ``` +4. Change to the uncompressed directory. +5. Build docker image using provided Dockerfile. + + ```bash + docker build -t wittyer: --platform linux/amd64 . + ``` +6. Access rights are needed to push the container to the Dockerhub nfcore organization, please ask a core team member to do so. + + ```bash + docker tag wittyer: quay.io/nf-core/wittyer: + docker push quay.io/nf-core/wittyer: + ``` \ No newline at end of file From c974cbd4dea450c867fc60a468d5601f216ba1af Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 09:19:35 +0100 Subject: [PATCH 08/37] fix: README for Docker image --- modules/nf-core/wittyer/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/nf-core/wittyer/README.md b/modules/nf-core/wittyer/README.md index 8a9e8b004bf..5d084dcf986 100644 --- a/modules/nf-core/wittyer/README.md +++ b/modules/nf-core/wittyer/README.md @@ -6,7 +6,6 @@ witty.er is a commercial tool from Illumina. The container provided for the witt 2. Download the latest release. ```bash wget https://github.com/Illumina/witty.er/archive/refs/tags/.tar.gz - curl https://github.com/Illumina/witty.er/archive/refs/tags/.tar.gz -o .tar.gz ``` 3. Uncompress the released package. ```bash From e0f5f6928f1fa8b1d82fdbbfa3b75b8c204a74d1 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 09:21:32 +0100 Subject: [PATCH 09/37] fix: modify environment file --- modules/nf-core/wittyer/environment.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/nf-core/wittyer/environment.yml b/modules/nf-core/wittyer/environment.yml index 6783e3c68a8..bdcd0d84b73 100644 --- a/modules/nf-core/wittyer/environment.yml +++ b/modules/nf-core/wittyer/environment.yml @@ -1,9 +1,5 @@ ---- -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json -name: "wittyer" +name: wittyer channels: - conda-forge - bioconda - defaults -dependencies: - - "YOUR-TOOL-HERE" From 213d6fa63f25da47926c0eb55686eecb77e0d577 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 09:38:58 +0100 Subject: [PATCH 10/37] fix: stub and file extensions --- modules/nf-core/wittyer/main.nf | 49 ++++++--------------------------- 1 file changed, 8 insertions(+), 41 deletions(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index 666e9b27fd5..80dd770e748 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -1,48 +1,17 @@ -// TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) -// https://github.com/nf-core/modules/tree/master/modules/nf-core/ -// You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: -// https://nf-co.re/join -// TODO nf-core: A module file SHOULD only define input and output files as command-line parameters. -// All other parameters MUST be provided using the "task.ext" directive, see here: -// https://www.nextflow.io/docs/latest/process.html#ext -// where "task.ext" is a string. -// Any parameters that need to be evaluated in the context of a particular sample -// e.g. single-end/paired-end data MUST also be defined and evaluated appropriately. -// TODO nf-core: Software that can be piped together SHOULD be added to separate module files -// unless there is a run-time, storage advantage in implementing in this way -// e.g. it's ok to have a single module for bwa to output BAM instead of SAM: -// bwa mem | samtools view -B -T ref.fasta -// TODO nf-core: Optional inputs are not currently supported by Nextflow. However, using an empty -// list (`[]`) instead of a file can be used to work around this issue. - process WITTYER { tag "$meta.id" label 'process_single' - // TODO nf-core: List required Conda package(s). - // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). - // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. - // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. - // conda "${moduleDir}/environment.yml" container "nf-core/wittier:0.3.3" - // "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - // 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE': - // 'biocontainers/YOUR-TOOL-HERE' }" input: - // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" - // MUST be provided as an input via a Groovy Map called "meta". - // This information may not be required in some instances e.g. indexing reference genome files: - // https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf - // TODO nf-core: Where applicable please provide/convert compressed files as input/output - // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. tuple val(meta), path(query_vcf), path(query_vcf_tbi), path(truth_vcf), path(truth_vcf_tbi), path(bed) output: tuple val(meta), path("*.Stats.json") , emit: report tuple val(meta), path("*eval.vcf.gz") , emit: bench_vcf tuple val(meta), path("*eval.vcf.gz.tbi") , emit: bench_vcf_tbi - path "versions.yml" + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -50,7 +19,7 @@ process WITTYER { script: // Exit if running this module with -profile conda / -profile mamba if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - error "WHITTYER module does not support Conda. Please use Docker / Singularity / Podman instead." + error "WITTYER module does not support Conda. Please use Docker / Singularity / Podman instead." } def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" @@ -64,9 +33,9 @@ process WITTYER { ${regions} \\ ${args} - mv bench/Wittyer.Stats.json ${prefix}.Stats.json - mv bench/*.vcf.gz ${prefix}.eval.vcf.gz - mv bench/*.vcf.gz.tbi ${prefix}.eval.vcf.gz.tbi + mv bench/Wittyer.Stats.json ${prefix}.json + mv bench/*.vcf.gz ${prefix}.vcf.gz + mv bench/*.vcf.gz.tbi ${prefix}.vcf.gz.tbi rm -rf bench @@ -79,12 +48,10 @@ process WITTYER { stub: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - // TODO nf-core: A stub section should mimic the execution of the original module as best as possible - // Have a look at the following examples: - // Simple example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bcftools/annotate/main.nf#L47-L63 - // Complex example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bedtools/split/main.nf#L38-L54 """ - touch ${prefix}.bam + touch ${prefix}.json + touch ${prefix}.vcf.gz + touch ${prefix}.vcf.gz.tbi cat <<-END_VERSIONS > versions.yml "${task.process}": From 80e352eee8ebb51573b0fb4e453b314025ebfec7 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 10:42:18 +0100 Subject: [PATCH 11/37] fix: remove custom extensions --- modules/nf-core/wittyer/main.nf | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index 80dd770e748..cf974901070 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -2,7 +2,12 @@ process WITTYER { tag "$meta.id" label 'process_single' - container "nf-core/wittier:0.3.3" + container "quay.io/nf-core/wittier:0.3.3" + + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "WITTYER module does not support Conda. Please use Docker / Singularity / Podman instead." + } input: tuple val(meta), path(query_vcf), path(query_vcf_tbi), path(truth_vcf), path(truth_vcf_tbi), path(bed) @@ -17,15 +22,18 @@ process WITTYER { task.ext.when == null || task.ext.when script: - // Exit if running this module with -profile conda / -profile mamba - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - error "WITTYER module does not support Conda. Please use Docker / Singularity / Podman instead." - } def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def regions = bed ? "--includeBed=$bed" : "" """ + script: + if ("$truth_vcf" == "${prefix}.vcf.gz") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + if ("$query_vcf" == "${prefix}.vcf.gz") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + if ("$truth_vcf_tbi" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + if ("$query_vcf_tbi" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + mkdir bench + wittyer \\ --truthVcf=${truth_vcf} \\ --inputVcf=${query_vcf} \\ From 7babe7abdd8fdb95f1ecd644c45d1fe144ce318e Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 10:43:03 +0100 Subject: [PATCH 12/37] feat: add test file - failing --- modules/nf-core/wittyer/tests/main.nf.test | 26 ++----- .../nf-core/wittyer/tests/main.nf.test.snap | 72 +++++++++++++++++++ 2 files changed, 77 insertions(+), 21 deletions(-) create mode 100644 modules/nf-core/wittyer/tests/main.nf.test.snap diff --git a/modules/nf-core/wittyer/tests/main.nf.test b/modules/nf-core/wittyer/tests/main.nf.test index 3234fda9c49..b5e0308e38a 100644 --- a/modules/nf-core/wittyer/tests/main.nf.test +++ b/modules/nf-core/wittyer/tests/main.nf.test @@ -1,5 +1,3 @@ -// TODO nf-core: Once you have added the required tests, please run the following command to build this file: -// nf-core modules test wittyer nextflow_process { name "Test Process WITTYER" @@ -10,22 +8,14 @@ nextflow_process { tag "modules_nfcore" tag "wittyer" - // TODO nf-core: Change the test name preferably indicating the test-data and file-format used - test("sarscov2 - bam") { - - // TODO nf-core: If you are created a test for a chained module - // (the module requires running more than one process to generate the required output) - // add the 'setup' method here. - // You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules). + test("human - simulatedSV - vcf_gz") { when { process { """ - // TODO nf-core: define inputs of the process here. Example: - input[0] = [ [ id:'test', single_end:false ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) + file(params.test_data['homo_sapiens']['illumina']['simulated_sv'], checkIfExists: true) ] """ } @@ -35,26 +25,21 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot(process.out).match() } - //TODO nf-core: Add all required assertions to verify the test output. - // See https://nf-co.re/docs/contributing/tutorials/nf-test_assertions for more information and examples. ) } } - // TODO nf-core: Change the test name preferably indicating the test-data and file-format used but keep the " - stub" suffix. - test("sarscov2 - bam - stub") { + test("human - simulatedSV - vcf_gz - stub") { options "-stub" when { process { - """ - // TODO nf-core: define inputs of the process here. Example: - + """ input[0] = [ [ id:'test', single_end:false ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) + file(params.test_data['homo_sapiens']['illumina']['simulated_sv'], checkIfExists: true) ] """ } @@ -64,7 +49,6 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot(process.out).match() } - //TODO nf-core: Add all required assertions to verify the test output. ) } diff --git a/modules/nf-core/wittyer/tests/main.nf.test.snap b/modules/nf-core/wittyer/tests/main.nf.test.snap new file mode 100644 index 00000000000..a715230484f --- /dev/null +++ b/modules/nf-core/wittyer/tests/main.nf.test.snap @@ -0,0 +1,72 @@ +{ + "human - simulatedSV - vcf_gz": { + "content": [ + { + "0": [ + + ], + "1": [ + + ], + "2": [ + + ], + "3": [ + + ], + "bench_vcf": [ + + ], + "bench_vcf_tbi": [ + + ], + "report": [ + + ], + "versions": [ + + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T10:39:34.53397108" + }, + "human - simulatedSV - vcf_gz - stub": { + "content": [ + { + "0": [ + + ], + "1": [ + + ], + "2": [ + + ], + "3": [ + + ], + "bench_vcf": [ + + ], + "bench_vcf_tbi": [ + + ], + "report": [ + + ], + "versions": [ + + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T10:39:37.734751136" + } +} \ No newline at end of file From e702a2056d265f647112195de9c4b0baa36f8e6e Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 11:16:28 +0100 Subject: [PATCH 13/37] fix: correct input for tests --- modules/nf-core/wittyer/tests/main.nf.test | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/wittyer/tests/main.nf.test b/modules/nf-core/wittyer/tests/main.nf.test index b5e0308e38a..765358c0a1c 100644 --- a/modules/nf-core/wittyer/tests/main.nf.test +++ b/modules/nf-core/wittyer/tests/main.nf.test @@ -15,7 +15,11 @@ nextflow_process { """ input[0] = [ [ id:'test', single_end:false ], // meta map - file(params.test_data['homo_sapiens']['illumina']['simulated_sv'], checkIfExists: true) + file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['na24385_chr22_sv_vcf'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['na24385_chr22_sv_vcf_tbi'], checkIfExists: true), + [] ] """ } @@ -39,7 +43,11 @@ nextflow_process { """ input[0] = [ [ id:'test', single_end:false ], // meta map - file(params.test_data['homo_sapiens']['illumina']['simulated_sv'], checkIfExists: true) + file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['na24385_chr22_sv_vcf'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['na24385_chr22_sv_vcf_tbi'], checkIfExists: true), + [] ] """ } From 522f72e03a165012698399fc4f3e8c91e4599b7f Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 11:23:05 +0100 Subject: [PATCH 14/37] fix: licence --- modules/nf-core/wittyer/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/wittyer/meta.yml b/modules/nf-core/wittyer/meta.yml index 784d14c33ed..1bb2f35130a 100644 --- a/modules/nf-core/wittyer/meta.yml +++ b/modules/nf-core/wittyer/meta.yml @@ -11,7 +11,7 @@ tools: homepage: "https://github.com/Illumina/witty.er" documentation: "https://github.com/Illumina/witty.er" tool_dev_url: "https://github.com/Illumina/witty.er" - licence: "['BSD-2']" + licence: ["BSD-2"] input: - meta: type: map From 7d06402d7cd1bf3603c760e7c5439cace08bae5d Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 11:27:40 +0100 Subject: [PATCH 15/37] fix: container name --- modules/nf-core/wittyer/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index cf974901070..f532a37c5a1 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -2,7 +2,7 @@ process WITTYER { tag "$meta.id" label 'process_single' - container "quay.io/nf-core/wittier:0.3.3" + container "nf-core/wittyer:0.3.3" // Exit if running this module with -profile conda / -profile mamba if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { From fdf993d55e00e995211090ea0c8761f65ce892eb Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 11:33:56 +0100 Subject: [PATCH 16/37] fix: finally correct container --- modules/nf-core/wittyer/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index f532a37c5a1..cfb575e0984 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -2,7 +2,7 @@ process WITTYER { tag "$meta.id" label 'process_single' - container "nf-core/wittyer:0.3.3" + container "nf-core/wittyer:0.3.3.0" // Exit if running this module with -profile conda / -profile mamba if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { From c07bd111a029f01211aab1ab7e563be87dd1b3bf Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 11:48:58 +0100 Subject: [PATCH 17/37] fix syntax errors --- modules/nf-core/wittyer/main.nf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index cfb575e0984..cd69d20312b 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -25,13 +25,11 @@ process WITTYER { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def regions = bed ? "--includeBed=$bed" : "" - """ - script: if ("$truth_vcf" == "${prefix}.vcf.gz") error "Input and output names are the same, set prefix in module configuration to disambiguate!" if ("$query_vcf" == "${prefix}.vcf.gz") error "Input and output names are the same, set prefix in module configuration to disambiguate!" if ("$truth_vcf_tbi" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" if ("$query_vcf_tbi" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" - + """ mkdir bench wittyer \\ From d6e428b820b8166cf35df4f1e71dc2809ca2168f Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 13:00:42 +0100 Subject: [PATCH 18/37] normal test works, stub fails --- modules/nf-core/wittyer/tests/main.nf.test | 12 ++-- .../nf-core/wittyer/tests/main.nf.test.snap | 68 +------------------ 2 files changed, 9 insertions(+), 71 deletions(-) diff --git a/modules/nf-core/wittyer/tests/main.nf.test b/modules/nf-core/wittyer/tests/main.nf.test index 765358c0a1c..9d18ae37755 100644 --- a/modules/nf-core/wittyer/tests/main.nf.test +++ b/modules/nf-core/wittyer/tests/main.nf.test @@ -17,8 +17,8 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['na24385_chr22_sv_vcf'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['na24385_chr22_sv_vcf_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), [] ] """ @@ -28,7 +28,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(process.out.version).match("version") } ) } @@ -45,8 +45,8 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['na24385_chr22_sv_vcf'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['na24385_chr22_sv_vcf_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), [] ] """ @@ -56,7 +56,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(process.out.version).match("version") } ) } diff --git a/modules/nf-core/wittyer/tests/main.nf.test.snap b/modules/nf-core/wittyer/tests/main.nf.test.snap index a715230484f..b8a7b44d396 100644 --- a/modules/nf-core/wittyer/tests/main.nf.test.snap +++ b/modules/nf-core/wittyer/tests/main.nf.test.snap @@ -1,72 +1,10 @@ { - "human - simulatedSV - vcf_gz": { - "content": [ - { - "0": [ - - ], - "1": [ - - ], - "2": [ - - ], - "3": [ - - ], - "bench_vcf": [ - - ], - "bench_vcf_tbi": [ - - ], - "report": [ - - ], - "versions": [ - - ] - } - ], + "version": { + "content": null, "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-19T10:39:34.53397108" - }, - "human - simulatedSV - vcf_gz - stub": { - "content": [ - { - "0": [ - - ], - "1": [ - - ], - "2": [ - - ], - "3": [ - - ], - "bench_vcf": [ - - ], - "bench_vcf_tbi": [ - - ], - "report": [ - - ], - "versions": [ - - ] - } - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-03-19T10:39:37.734751136" + "timestamp": "2024-03-19T12:42:05.574419458" } } \ No newline at end of file From 3f7e0167d2dc3075f489ae3f6c70d88217e141f8 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 13:01:09 +0100 Subject: [PATCH 19/37] fix expected outputs --- modules/nf-core/wittyer/meta.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/wittyer/meta.yml b/modules/nf-core/wittyer/meta.yml index 1bb2f35130a..490b50ae867 100644 --- a/modules/nf-core/wittyer/meta.yml +++ b/modules/nf-core/wittyer/meta.yml @@ -51,15 +51,15 @@ output: - report: type: file description: Detailed per-sample-pair, per-svtype, per-bin stats - pattern: "*.Stats.json" + pattern: "*.json" - bench_vcf: type: file description: Updated query and truth entries merged into one file - pattern: "*eval.vcf.gz" + pattern: "*.vcf.gz" - bench_vcf_tbi: type: file description: Index of merged query and truth entries VCF file - pattern: "*eval.vcf.gz.tbi" + pattern: "*.vcf.gz.tbi" authors: - "@famosab" maintainers: From bf138d67f856e33a5380af427a674313a94bb643 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 13:01:31 +0100 Subject: [PATCH 20/37] fix: mention making image public --- modules/nf-core/wittyer/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/nf-core/wittyer/README.md b/modules/nf-core/wittyer/README.md index 5d084dcf986..ec2e485b62e 100644 --- a/modules/nf-core/wittyer/README.md +++ b/modules/nf-core/wittyer/README.md @@ -22,4 +22,6 @@ witty.er is a commercial tool from Illumina. The container provided for the witt ```bash docker tag wittyer: quay.io/nf-core/wittyer: docker push quay.io/nf-core/wittyer: - ``` \ No newline at end of file + ``` + +7. Make the image public. \ No newline at end of file From d205ddd8b4c8683dfb0345167e6f67de32b164cc Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 13:02:31 +0100 Subject: [PATCH 21/37] fix add stageAs, use dotnet --- modules/nf-core/wittyer/main.nf | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index cd69d20312b..0b0c876ebc3 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -10,13 +10,14 @@ process WITTYER { } input: - tuple val(meta), path(query_vcf), path(query_vcf_tbi), path(truth_vcf), path(truth_vcf_tbi), path(bed) + // use stageAs to allow comparing the same vcfs against each other + tuple val(meta), path(query_vcf, stageAs: "query.vcf.gz"), path(query_vcf_tbi, stageAs: "query.vcf.gz.tbi"), path(truth_vcf, stageAs: "truth.vcf.gz"), path(truth_vcf_tbi, stageAs: "truth.vcf.gz.tbi"), path(bed) output: - tuple val(meta), path("*.Stats.json") , emit: report - tuple val(meta), path("*eval.vcf.gz") , emit: bench_vcf - tuple val(meta), path("*eval.vcf.gz.tbi") , emit: bench_vcf_tbi - path "versions.yml" , emit: versions + tuple val(meta), path("*.json") , emit: report + tuple val(meta), path("*.vcf.gz") , emit: bench_vcf + tuple val(meta), path("*.vcf.gz.tbi") , emit: bench_vcf_tbi + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -32,7 +33,7 @@ process WITTYER { """ mkdir bench - wittyer \\ + dotnet /opt/Wittyer/Wittyer.dll \\ --truthVcf=${truth_vcf} \\ --inputVcf=${query_vcf} \\ --outputDirectory=bench \\ @@ -47,7 +48,7 @@ process WITTYER { cat <<-END_VERSIONS > versions.yml "${task.process}": - wittyer: \$(wittyer --version |& sed '1!d ; s/witty.er //') + wittyer: \$(dotnet /opt/Wittyer/Wittyer.dll --version |& sed '1!d ; s/witty.er //') END_VERSIONS """ @@ -61,7 +62,7 @@ process WITTYER { cat <<-END_VERSIONS > versions.yml "${task.process}": - wittyer: \$(wittyer --version |& sed '1!d ; s/witty.er //') + wittyer: \$(dotnet /opt/Wittyer/Wittyer.dll --version |& sed '1!d ; s/witty.er //') END_VERSIONS """ } From e4164581d8736824876d77b8cea4ffbf87344a97 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 13:04:56 +0100 Subject: [PATCH 22/37] run prettier --- modules/nf-core/wittyer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/wittyer/README.md b/modules/nf-core/wittyer/README.md index ec2e485b62e..5c294965c42 100644 --- a/modules/nf-core/wittyer/README.md +++ b/modules/nf-core/wittyer/README.md @@ -9,7 +9,7 @@ witty.er is a commercial tool from Illumina. The container provided for the witt ``` 3. Uncompress the released package. ```bash - tar -xvf .tar.gz + tar -xvf .tar.gz ``` 4. Change to the uncompressed directory. 5. Build docker image using provided Dockerfile. From a6564b625bc02d4fc3af269b5e0daf09397458b7 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 19 Mar 2024 12:08:07 +0000 Subject: [PATCH 23/37] [automated] Fix linting with Prettier --- modules/nf-core/wittyer/README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/nf-core/wittyer/README.md b/modules/nf-core/wittyer/README.md index 5c294965c42..d2ba0e09b41 100644 --- a/modules/nf-core/wittyer/README.md +++ b/modules/nf-core/wittyer/README.md @@ -4,19 +4,20 @@ witty.er is a commercial tool from Illumina. The container provided for the witt 1. Navigate to the witty.er github repository. - [witty.er](https://github.com/Illumina/witty.er) 2. Download the latest release. - ```bash - wget https://github.com/Illumina/witty.er/archive/refs/tags/.tar.gz - ``` + ```bash + wget https://github.com/Illumina/witty.er/archive/refs/tags/.tar.gz + ``` 3. Uncompress the released package. - ```bash - tar -xvf .tar.gz - ``` + ```bash + tar -xvf .tar.gz + ``` 4. Change to the uncompressed directory. 5. Build docker image using provided Dockerfile. ```bash docker build -t wittyer: --platform linux/amd64 . ``` + 6. Access rights are needed to push the container to the Dockerhub nfcore organization, please ask a core team member to do so. ```bash @@ -24,4 +25,4 @@ witty.er is a commercial tool from Illumina. The container provided for the witt docker push quay.io/nf-core/wittyer: ``` -7. Make the image public. \ No newline at end of file +7. Make the image public. From a205762715dcc3da4dc42dd2542b20d64c0be7ac Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 13:10:43 +0100 Subject: [PATCH 24/37] add comment for run --- modules/nf-core/wittyer/main.nf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index 0b0c876ebc3..f17740218dc 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -30,6 +30,8 @@ process WITTYER { if ("$query_vcf" == "${prefix}.vcf.gz") error "Input and output names are the same, set prefix in module configuration to disambiguate!" if ("$truth_vcf_tbi" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" if ("$query_vcf_tbi" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + + // dotnet /opt/Wittyer/Wittyer.dll might need to be replaced with new docker image """ mkdir bench From ae11718d877094ef54eba105982e86a39cb8413e Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 13:16:09 +0100 Subject: [PATCH 25/37] fix trailing whitespace --- modules/nf-core/wittyer/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index f17740218dc..a2bf382cf3d 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -30,7 +30,7 @@ process WITTYER { if ("$query_vcf" == "${prefix}.vcf.gz") error "Input and output names are the same, set prefix in module configuration to disambiguate!" if ("$truth_vcf_tbi" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" if ("$query_vcf_tbi" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" - + // dotnet /opt/Wittyer/Wittyer.dll might need to be replaced with new docker image """ mkdir bench From 32081bc2efa2350a93b8a142dc01472e72871141 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 13:17:17 +0100 Subject: [PATCH 26/37] add dependencies to fix linting --- modules/nf-core/wittyer/environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/nf-core/wittyer/environment.yml b/modules/nf-core/wittyer/environment.yml index bdcd0d84b73..012536110a6 100644 --- a/modules/nf-core/wittyer/environment.yml +++ b/modules/nf-core/wittyer/environment.yml @@ -3,3 +3,5 @@ channels: - conda-forge - bioconda - defaults +dependencies: + - tabix \ No newline at end of file From 1e55171f010e5839a0204eeed645cd9725a1e318 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 13:18:45 +0100 Subject: [PATCH 27/37] fix harmonize inputs in meta and main --- modules/nf-core/wittyer/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index a2bf382cf3d..9b8a4f51c16 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -11,7 +11,7 @@ process WITTYER { input: // use stageAs to allow comparing the same vcfs against each other - tuple val(meta), path(query_vcf, stageAs: "query.vcf.gz"), path(query_vcf_tbi, stageAs: "query.vcf.gz.tbi"), path(truth_vcf, stageAs: "truth.vcf.gz"), path(truth_vcf_tbi, stageAs: "truth.vcf.gz.tbi"), path(bed) + tuple val(meta), path(query_vcf, stageAs: "query.vcf.gz"), path(query_vcf_index, stageAs: "query.vcf.gz.tbi"), path(truth_vcf, stageAs: "truth.vcf.gz"), path(truth_vcf_index, stageAs: "truth.vcf.gz.tbi"), path(bed) output: tuple val(meta), path("*.json") , emit: report From 1eb7526e7a437ecbf6d6457d96a82b46dfbd7cbd Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 13:28:32 +0100 Subject: [PATCH 28/37] fix: run prettier --- modules/nf-core/wittyer/environment.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/wittyer/environment.yml b/modules/nf-core/wittyer/environment.yml index 012536110a6..3df7931d6eb 100644 --- a/modules/nf-core/wittyer/environment.yml +++ b/modules/nf-core/wittyer/environment.yml @@ -1,7 +1,7 @@ name: wittyer channels: - - conda-forge - - bioconda - - defaults +- conda-forge +- bioconda +- defaults dependencies: - - tabix \ No newline at end of file +- tabix From fe25500320ecf58f85a9a3c8384866476fe990a2 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 19 Mar 2024 12:33:30 +0000 Subject: [PATCH 29/37] [automated] Fix linting with Prettier --- modules/nf-core/wittyer/environment.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/wittyer/environment.yml b/modules/nf-core/wittyer/environment.yml index 3df7931d6eb..f8378df2479 100644 --- a/modules/nf-core/wittyer/environment.yml +++ b/modules/nf-core/wittyer/environment.yml @@ -1,7 +1,7 @@ name: wittyer channels: -- conda-forge -- bioconda -- defaults + - conda-forge + - bioconda + - defaults dependencies: -- tabix + - tabix From 1903c8d6d71ec37b45eaef5c2a91cedc459a5b1c Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 13:57:40 +0100 Subject: [PATCH 30/37] fix index names --- modules/nf-core/wittyer/main.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index 9b8a4f51c16..188c38f91fe 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -28,8 +28,8 @@ process WITTYER { def regions = bed ? "--includeBed=$bed" : "" if ("$truth_vcf" == "${prefix}.vcf.gz") error "Input and output names are the same, set prefix in module configuration to disambiguate!" if ("$query_vcf" == "${prefix}.vcf.gz") error "Input and output names are the same, set prefix in module configuration to disambiguate!" - if ("$truth_vcf_tbi" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" - if ("$query_vcf_tbi" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + if ("$query_vcf_index" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + if ("$query_vcf_index" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" // dotnet /opt/Wittyer/Wittyer.dll might need to be replaced with new docker image """ @@ -59,7 +59,7 @@ process WITTYER { def prefix = task.ext.prefix ?: "${meta.id}" """ touch ${prefix}.json - touch ${prefix}.vcf.gz + echo "" | gzip > ${prefix}.vcf.gz touch ${prefix}.vcf.gz.tbi cat <<-END_VERSIONS > versions.yml From b2a8c10b6a7b96414b8e59debac55096573300d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20B=C3=A4uerle?= <45968370+famosab@users.noreply.github.com> Date: Tue, 19 Mar 2024 14:09:36 +0100 Subject: [PATCH 31/37] fix version to stub_version in stub test Co-authored-by: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> --- modules/nf-core/wittyer/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/wittyer/tests/main.nf.test b/modules/nf-core/wittyer/tests/main.nf.test index 9d18ae37755..4b595c99558 100644 --- a/modules/nf-core/wittyer/tests/main.nf.test +++ b/modules/nf-core/wittyer/tests/main.nf.test @@ -56,7 +56,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.version).match("version") } + { assert snapshot(process.out.version).match("stub_version") } ) } From 9f908f817effb966af58fc48ff7dc41d78a001d8 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 14:20:21 +0100 Subject: [PATCH 32/37] extend tests to capture as much as possible --- modules/nf-core/wittyer/tests/main.nf.test | 2 + .../nf-core/wittyer/tests/main.nf.test.snap | 150 ++++++++++++++++++ 2 files changed, 152 insertions(+) diff --git a/modules/nf-core/wittyer/tests/main.nf.test b/modules/nf-core/wittyer/tests/main.nf.test index 4b595c99558..9b9947d3801 100644 --- a/modules/nf-core/wittyer/tests/main.nf.test +++ b/modules/nf-core/wittyer/tests/main.nf.test @@ -28,6 +28,7 @@ nextflow_process { then { assertAll( { assert process.success }, + { assert snapshot(process.out).match() }, { assert snapshot(process.out.version).match("version") } ) } @@ -56,6 +57,7 @@ nextflow_process { then { assertAll( { assert process.success }, + { assert snapshot(process.out).match() }, { assert snapshot(process.out.version).match("stub_version") } ) } diff --git a/modules/nf-core/wittyer/tests/main.nf.test.snap b/modules/nf-core/wittyer/tests/main.nf.test.snap index b8a7b44d396..17e3e71f337 100644 --- a/modules/nf-core/wittyer/tests/main.nf.test.snap +++ b/modules/nf-core/wittyer/tests/main.nf.test.snap @@ -1,4 +1,83 @@ { + "human - simulatedSV - vcf_gz": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.json:md5,29f926a59e33766692b74649d0e1dc0a" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz:md5,fe678339e77eb1bda9458df709ad5b9c" + ] + ], + "2": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz.tbi:md5,e8d861ed64c6958b1ffa20bc74df722c" + ] + ], + "3": [ + "versions.yml:md5,4a5148f206a3b12f0ebe87e81cedc31a" + ], + "bench_vcf": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz:md5,fe678339e77eb1bda9458df709ad5b9c" + ] + ], + "bench_vcf_tbi": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz.tbi:md5,e8d861ed64c6958b1ffa20bc74df722c" + ] + ], + "report": [ + [ + { + "id": "test", + "single_end": false + }, + "test.json:md5,29f926a59e33766692b74649d0e1dc0a" + ] + ], + "versions": [ + "versions.yml:md5,4a5148f206a3b12f0ebe87e81cedc31a" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T14:17:20.291512122" + }, + "stub_version": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T14:18:12.974741386" + }, "version": { "content": null, "meta": { @@ -6,5 +85,76 @@ "nextflow": "23.10.1" }, "timestamp": "2024-03-19T12:42:05.574419458" + }, + "human - simulatedSV - vcf_gz - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.json:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "2": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "3": [ + "versions.yml:md5,4a5148f206a3b12f0ebe87e81cedc31a" + ], + "bench_vcf": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "bench_vcf_tbi": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "report": [ + [ + { + "id": "test", + "single_end": false + }, + "test.json:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,4a5148f206a3b12f0ebe87e81cedc31a" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T14:19:30.017309341" } } \ No newline at end of file From fd4c3076a1d9fbdede552287f6946c2fefb9f20f Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Tue, 19 Mar 2024 14:36:11 +0100 Subject: [PATCH 33/37] test with bed file --- modules/nf-core/wittyer/tests/main.nf.test | 28 +++++++- .../nf-core/wittyer/tests/main.nf.test.snap | 71 +++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/modules/nf-core/wittyer/tests/main.nf.test b/modules/nf-core/wittyer/tests/main.nf.test index 9b9947d3801..ab3ce3208ea 100644 --- a/modules/nf-core/wittyer/tests/main.nf.test +++ b/modules/nf-core/wittyer/tests/main.nf.test @@ -35,7 +35,33 @@ nextflow_process { } - test("human - simulatedSV - vcf_gz - stub") { + test("human - simulatedSV - vcf_gz - bed") { + + when { + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_21_multi_interval_bed'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + + } + + test("human - simulatedSV - vcf_gz - stub") { options "-stub" diff --git a/modules/nf-core/wittyer/tests/main.nf.test.snap b/modules/nf-core/wittyer/tests/main.nf.test.snap index 17e3e71f337..70212dc47ca 100644 --- a/modules/nf-core/wittyer/tests/main.nf.test.snap +++ b/modules/nf-core/wittyer/tests/main.nf.test.snap @@ -1,4 +1,75 @@ { + "human - simulatedSV - vcf_gz - bed": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.json:md5,f56d91127e70fc5f4a10240b93626868" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz:md5,bbb01fc92c21f36cf38cbd247ba05d89" + ] + ], + "2": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz.tbi:md5,b3ab34abd96cfe220211feb3199db97e" + ] + ], + "3": [ + "versions.yml:md5,4a5148f206a3b12f0ebe87e81cedc31a" + ], + "bench_vcf": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz:md5,bbb01fc92c21f36cf38cbd247ba05d89" + ] + ], + "bench_vcf_tbi": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz.tbi:md5,b3ab34abd96cfe220211feb3199db97e" + ] + ], + "report": [ + [ + { + "id": "test", + "single_end": false + }, + "test.json:md5,f56d91127e70fc5f4a10240b93626868" + ] + ], + "versions": [ + "versions.yml:md5,4a5148f206a3b12f0ebe87e81cedc31a" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T14:33:26.99336282" + }, "human - simulatedSV - vcf_gz": { "content": [ { From 6aac9e8fac27f32e4aa82e07808ccf29b528d6f9 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Wed, 20 Mar 2024 09:35:31 +0100 Subject: [PATCH 34/37] fix: remove stageAs --- modules/nf-core/wittyer/main.nf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index 188c38f91fe..c2b943f393b 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -10,8 +10,7 @@ process WITTYER { } input: - // use stageAs to allow comparing the same vcfs against each other - tuple val(meta), path(query_vcf, stageAs: "query.vcf.gz"), path(query_vcf_index, stageAs: "query.vcf.gz.tbi"), path(truth_vcf, stageAs: "truth.vcf.gz"), path(truth_vcf_index, stageAs: "truth.vcf.gz.tbi"), path(bed) + tuple val(meta), path(query_vcf), path(query_vcf_index), path(truth_vcf), path(truth_vcf_index), path(bed) output: tuple val(meta), path("*.json") , emit: report From 67fa0228e54c60e5d4c5184d00a882cae016d665 Mon Sep 17 00:00:00 2001 From: Famke Baeuerle Date: Wed, 20 Mar 2024 09:44:40 +0100 Subject: [PATCH 35/37] fix tests for stageAs removal --- modules/nf-core/wittyer/tests/main.nf.test | 35 ++-- .../nf-core/wittyer/tests/main.nf.test.snap | 188 ++++++++++++++---- 2 files changed, 173 insertions(+), 50 deletions(-) diff --git a/modules/nf-core/wittyer/tests/main.nf.test b/modules/nf-core/wittyer/tests/main.nf.test index ab3ce3208ea..3c23ffe66eb 100644 --- a/modules/nf-core/wittyer/tests/main.nf.test +++ b/modules/nf-core/wittyer/tests/main.nf.test @@ -15,10 +15,10 @@ nextflow_process { """ input[0] = [ [ id:'test', single_end:false ], // meta map - file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv2'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv2_tbi'], checkIfExists: true), [] ] """ @@ -29,6 +29,9 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot(process.out).match() }, + { assert snapshot(process.out.bench_vcf).match("bench_vcf") }, + { assert snapshot(process.out.bench_vcf_tbi).match("bench_vcf_tbi") }, + { assert snapshot(process.out.report).match("report") }, { assert snapshot(process.out.version).match("version") } ) } @@ -41,11 +44,11 @@ nextflow_process { process { """ input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), + [ id:'test_bed', single_end:false ], // meta map + file(params.test_data['homo_sapiens']['illumina']['simulated_sv'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv2'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv2_tbi'], checkIfExists: true), file(params.test_data['homo_sapiens']['genome']['genome_21_multi_interval_bed'], checkIfExists: true) ] """ @@ -56,6 +59,10 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot(process.out).match() }, + { assert snapshot(process.out.bench_vcf).match("bed_bench_vcf") }, + { assert snapshot(process.out.bench_vcf_tbi).match("bed_bench_vcf_tbi") }, + { assert snapshot(process.out.report).match("bed_report") }, + { assert snapshot(process.out.version).match("bed_version") } ) } @@ -69,11 +76,11 @@ nextflow_process { process { """ input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_sv_vcf_tbi'], checkIfExists: true), + [ id:'test_stub', single_end:false ], // meta map + file(params.test_data['homo_sapiens']['illumina']['simulated_sv'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv2'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv2_tbi'], checkIfExists: true), [] ] """ diff --git a/modules/nf-core/wittyer/tests/main.nf.test.snap b/modules/nf-core/wittyer/tests/main.nf.test.snap index 70212dc47ca..a25d824420d 100644 --- a/modules/nf-core/wittyer/tests/main.nf.test.snap +++ b/modules/nf-core/wittyer/tests/main.nf.test.snap @@ -5,28 +5,28 @@ "0": [ [ { - "id": "test", + "id": "test_bed", "single_end": false }, - "test.json:md5,f56d91127e70fc5f4a10240b93626868" + "test_bed.json:md5,c6515ada81b5ccf5aa5b4f1268da2800" ] ], "1": [ [ { - "id": "test", + "id": "test_bed", "single_end": false }, - "test.vcf.gz:md5,bbb01fc92c21f36cf38cbd247ba05d89" + "test_bed.vcf.gz:md5,7e5f24415c80ca986e81be90f831e000" ] ], "2": [ [ { - "id": "test", + "id": "test_bed", "single_end": false }, - "test.vcf.gz.tbi:md5,b3ab34abd96cfe220211feb3199db97e" + "test_bed.vcf.gz.tbi:md5,e4de1e1d27208b56f5a7bfbe31542240" ] ], "3": [ @@ -35,28 +35,28 @@ "bench_vcf": [ [ { - "id": "test", + "id": "test_bed", "single_end": false }, - "test.vcf.gz:md5,bbb01fc92c21f36cf38cbd247ba05d89" + "test_bed.vcf.gz:md5,7e5f24415c80ca986e81be90f831e000" ] ], "bench_vcf_tbi": [ [ { - "id": "test", + "id": "test_bed", "single_end": false }, - "test.vcf.gz.tbi:md5,b3ab34abd96cfe220211feb3199db97e" + "test_bed.vcf.gz.tbi:md5,e4de1e1d27208b56f5a7bfbe31542240" ] ], "report": [ [ { - "id": "test", + "id": "test_bed", "single_end": false }, - "test.json:md5,f56d91127e70fc5f4a10240b93626868" + "test_bed.json:md5,c6515ada81b5ccf5aa5b4f1268da2800" ] ], "versions": [ @@ -68,7 +68,15 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-19T14:33:26.99336282" + "timestamp": "2024-03-20T09:41:49.782336703" + }, + "stub_version": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:57.223103062" }, "human - simulatedSV - vcf_gz": { "content": [ @@ -79,7 +87,7 @@ "id": "test", "single_end": false }, - "test.json:md5,29f926a59e33766692b74649d0e1dc0a" + "test.json:md5,fb70eac691c1067167091ab2d3b12de3" ] ], "1": [ @@ -88,7 +96,7 @@ "id": "test", "single_end": false }, - "test.vcf.gz:md5,fe678339e77eb1bda9458df709ad5b9c" + "test.vcf.gz:md5,ff56c3084a59507362f6b7b7dc46ffdc" ] ], "2": [ @@ -97,7 +105,7 @@ "id": "test", "single_end": false }, - "test.vcf.gz.tbi:md5,e8d861ed64c6958b1ffa20bc74df722c" + "test.vcf.gz.tbi:md5,b9b448e5f11eebbcfeb9a123e838caa4" ] ], "3": [ @@ -109,7 +117,7 @@ "id": "test", "single_end": false }, - "test.vcf.gz:md5,fe678339e77eb1bda9458df709ad5b9c" + "test.vcf.gz:md5,ff56c3084a59507362f6b7b7dc46ffdc" ] ], "bench_vcf_tbi": [ @@ -118,7 +126,7 @@ "id": "test", "single_end": false }, - "test.vcf.gz.tbi:md5,e8d861ed64c6958b1ffa20bc74df722c" + "test.vcf.gz.tbi:md5,b9b448e5f11eebbcfeb9a123e838caa4" ] ], "report": [ @@ -127,7 +135,7 @@ "id": "test", "single_end": false }, - "test.json:md5,29f926a59e33766692b74649d0e1dc0a" + "test.json:md5,fb70eac691c1067167091ab2d3b12de3" ] ], "versions": [ @@ -139,15 +147,87 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-19T14:17:20.291512122" + "timestamp": "2024-03-20T09:41:44.466462714" }, - "stub_version": { + "bench_vcf": { + "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz:md5,ff56c3084a59507362f6b7b7dc46ffdc" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:47.523573536" + }, + "bench_vcf_tbi": { + "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz.tbi:md5,b9b448e5f11eebbcfeb9a123e838caa4" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:47.532504108" + }, + "report": { + "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + "test.json:md5,fb70eac691c1067167091ab2d3b12de3" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:47.537692046" + }, + "bed_report": { + "content": [ + [ + [ + { + "id": "test_bed", + "single_end": false + }, + "test_bed.json:md5,c6515ada81b5ccf5aa5b4f1268da2800" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:52.863563564" + }, + "bed_version": { "content": null, "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-19T14:18:12.974741386" + "timestamp": "2024-03-20T09:36:52.868467701" }, "version": { "content": null, @@ -155,7 +235,7 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-19T12:42:05.574419458" + "timestamp": "2024-03-20T09:36:47.542666285" }, "human - simulatedSV - vcf_gz - stub": { "content": [ @@ -163,28 +243,28 @@ "0": [ [ { - "id": "test", + "id": "test_stub", "single_end": false }, - "test.json:md5,d41d8cd98f00b204e9800998ecf8427e" + "test_stub.json:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "1": [ [ { - "id": "test", + "id": "test_stub", "single_end": false }, - "test.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + "test_stub.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" ] ], "2": [ [ { - "id": "test", + "id": "test_stub", "single_end": false }, - "test.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + "test_stub.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "3": [ @@ -193,28 +273,28 @@ "bench_vcf": [ [ { - "id": "test", + "id": "test_stub", "single_end": false }, - "test.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + "test_stub.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" ] ], "bench_vcf_tbi": [ [ { - "id": "test", + "id": "test_stub", "single_end": false }, - "test.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + "test_stub.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "report": [ [ { - "id": "test", + "id": "test_stub", "single_end": false }, - "test.json:md5,d41d8cd98f00b204e9800998ecf8427e" + "test_stub.json:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "versions": [ @@ -226,6 +306,42 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-19T14:19:30.017309341" + "timestamp": "2024-03-20T09:36:57.215084162" + }, + "bed_bench_vcf": { + "content": [ + [ + [ + { + "id": "test_bed", + "single_end": false + }, + "test_bed.vcf.gz:md5,7e5f24415c80ca986e81be90f831e000" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:52.852045028" + }, + "bed_bench_vcf_tbi": { + "content": [ + [ + [ + { + "id": "test_bed", + "single_end": false + }, + "test_bed.vcf.gz.tbi:md5,e4de1e1d27208b56f5a7bfbe31542240" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:52.857651771" } } \ No newline at end of file From e46dbd318a6d62d3d62af1d72b80ebea8bd3dea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 20 Mar 2024 10:00:08 +0100 Subject: [PATCH 36/37] remove '' in meta file --- modules/nf-core/wittyer/meta.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/wittyer/meta.yml b/modules/nf-core/wittyer/meta.yml index 490b50ae867..097f90fbbad 100644 --- a/modules/nf-core/wittyer/meta.yml +++ b/modules/nf-core/wittyer/meta.yml @@ -1,12 +1,12 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json -name: "wittyer" +name: wittyer description: A large variant benchmarking tool analogous to hap.py for small variants. keywords: - structural-variants - benchmarking - vcf tools: - - "wittyer": + - wittyer: description: "Illumina tool for large variant benchmarking" homepage: "https://github.com/Illumina/witty.er" documentation: "https://github.com/Illumina/witty.er" From 682c539bf5dd63f6a9e551c87a985c7a3ff6ce73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 20 Mar 2024 10:00:37 +0100 Subject: [PATCH 37/37] exclude conda test for wittyer --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d2a616ca48c..30e6a86a700 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -583,6 +583,8 @@ jobs: tags: subworkflows/vcf_annotate_ensemblvep - profile: singularity tags: bases2fastq + - profile: conda + tags: wittyer env: NXF_ANSI_LOG: false SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}