From 26808304b6c0eeedaca391532f6d36030237814b Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:06:55 +0100 Subject: [PATCH 01/66] add pigz to clustalo --- modules/nf-core/clustalo/align/environment.yml | 1 + modules/nf-core/clustalo/align/main.nf | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/clustalo/align/environment.yml b/modules/nf-core/clustalo/align/environment.yml index 1e08e574e37..be1eef95ebf 100644 --- a/modules/nf-core/clustalo/align/environment.yml +++ b/modules/nf-core/clustalo/align/environment.yml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - bioconda::clustalo=1.2.4 + - conda-forge::pigz=2.8 diff --git a/modules/nf-core/clustalo/align/main.nf b/modules/nf-core/clustalo/align/main.nf index f35ed407aad..92266e5cc51 100644 --- a/modules/nf-core/clustalo/align/main.nf +++ b/modules/nf-core/clustalo/align/main.nf @@ -12,7 +12,7 @@ process CLUSTALO_ALIGN { tuple val(meta2), path(tree) output: - tuple val(meta), path("*.aln"), emit: alignment + tuple val(meta), path("*.aln.gz"), emit: alignment path "versions.yml" , emit: versions when: @@ -26,11 +26,12 @@ process CLUSTALO_ALIGN { -i ${fasta} \\ --threads=${task.cpus} \\ $args \\ - -o ${prefix}.aln + | pigz -p ${task.cpus} -c > ${prefix}.aln.gz cat <<-END_VERSIONS > versions.yml "${task.process}": clustalo: \$( clustalo --version ) + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ @@ -38,11 +39,12 @@ process CLUSTALO_ALIGN { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix}.aln + touch ${prefix}.aln.gz cat <<-END_VERSIONS > versions.yml "${task.process}": clustalo: \$( clustalo --version ) + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ } From 23f496e96aa3f33ff96755e2b14370ee700194e8 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:33:48 +0100 Subject: [PATCH 02/66] add compression to muscle5 --- modules/nf-core/muscle5/super5/environment.yml | 1 + modules/nf-core/muscle5/super5/main.nf | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/muscle5/super5/environment.yml b/modules/nf-core/muscle5/super5/environment.yml index f2c60f16d00..fbaf4a2ef9f 100644 --- a/modules/nf-core/muscle5/super5/environment.yml +++ b/modules/nf-core/muscle5/super5/environment.yml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - bioconda::muscle=5.1 + - conda-forge::pigz=2.8 diff --git a/modules/nf-core/muscle5/super5/main.nf b/modules/nf-core/muscle5/super5/main.nf index 0f58be9461c..4386c2ccdff 100644 --- a/modules/nf-core/muscle5/super5/main.nf +++ b/modules/nf-core/muscle5/super5/main.nf @@ -10,7 +10,7 @@ process MUSCLE5_SUPER5 { tuple val(meta), path(fasta) output: - tuple val(meta), path("*.aln"), emit: alignment + tuple val(meta), path("*.aln.gz"), emit: alignment path "versions.yml" , emit: versions when: @@ -27,9 +27,12 @@ process MUSCLE5_SUPER5 { ${args} \\ -threads ${task.cpus} + pigz -p ${task.cpus} ${prefix}.aln + cat <<-END_VERSIONS > versions.yml "${task.process}": muscle: \$(muscle -version | head -n 1 | cut -d ' ' -f 2 | sed 's/.linux64//') + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ @@ -37,11 +40,12 @@ process MUSCLE5_SUPER5 { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix}.aln + touch ${prefix}.aln.gz cat <<-END_VERSIONS > versions.yml "${task.process}": muscle: \$(muscle -version | head -n 1 | cut -d ' ' -f 2 | sed 's/.linux64//') + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ } From 2d704a71e78d7ced9803cb12bda45538a1c44610 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:34:19 +0100 Subject: [PATCH 03/66] enabled compression flag for famsa --- modules/nf-core/famsa/align/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/famsa/align/main.nf b/modules/nf-core/famsa/align/main.nf index 388b133951e..9a48dc9edd2 100644 --- a/modules/nf-core/famsa/align/main.nf +++ b/modules/nf-core/famsa/align/main.nf @@ -21,7 +21,7 @@ process FAMSA_ALIGN { task.ext.when == null || task.ext.when script: - def args = task.ext.args ?: '' + def args = task.ext.args ?: '-gz' def prefix = task.ext.prefix ?: "${meta.id}" def options_tree = tree ? "-gt import $tree" : "" """ @@ -29,7 +29,7 @@ process FAMSA_ALIGN { $args \\ -t ${task.cpus} \\ ${fasta} \\ - ${prefix}.aln + ${prefix}.aln.gz cat <<-END_VERSIONS > versions.yml "${task.process}": From f8ddf9f77cec49c5264a4d97d3823eca054cd9df Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:35:23 +0100 Subject: [PATCH 04/66] added compression to mafft --- modules/nf-core/mafft/environment.yml | 1 + modules/nf-core/mafft/main.nf | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/mafft/environment.yml b/modules/nf-core/mafft/environment.yml index fff5c9f3599..595252e08c0 100644 --- a/modules/nf-core/mafft/environment.yml +++ b/modules/nf-core/mafft/environment.yml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - bioconda::mafft=7.520 + - conda-forge::pigz=2.8 diff --git a/modules/nf-core/mafft/main.nf b/modules/nf-core/mafft/main.nf index 43c32c98ba0..1385b9989df 100644 --- a/modules/nf-core/mafft/main.nf +++ b/modules/nf-core/mafft/main.nf @@ -16,7 +16,7 @@ process MAFFT { tuple val(meta6), path(addlong) output: - tuple val(meta), path("*.fas"), emit: fas + tuple val(meta), path("*.fas.gz"), emit: fas path "versions.yml" , emit: versions when: @@ -41,11 +41,12 @@ process MAFFT { ${addlong} \\ ${args} \\ ${fasta} \\ - > ${prefix}.fas + | pigz -p ${task.cpus} -c > ${prefix}.fas.gz cat <<-END_VERSIONS > versions.yml "${task.process}": mafft: \$(mafft --version 2>&1 | sed 's/^v//' | sed 's/ (.*)//') + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ @@ -59,11 +60,12 @@ process MAFFT { def addlong = addlong ? "--addlong ${addlong}" : '' if ("$fasta" == "${prefix}.fas" ) error "Input and output names are the same, set prefix in module configuration to disambiguate!" """ - touch ${prefix}.fas + touch ${prefix}.fas.gz cat <<-END_VERSIONS > versions.yml "${task.process}": mafft: \$(mafft --version 2>&1 | sed 's/^v//' | sed 's/ (.*)//') + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ From c0d1d507b9c7ebebd1dbd563c46e9a7d2bf8cf9e Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Thu, 21 Dec 2023 17:12:29 +0100 Subject: [PATCH 05/66] compression for mtmalign --- modules/nf-core/mtmalign/align/environment.yml | 1 + modules/nf-core/mtmalign/align/main.nf | 11 +++++++---- modules/nf-core/mtmalign/align/meta.yml | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/modules/nf-core/mtmalign/align/environment.yml b/modules/nf-core/mtmalign/align/environment.yml index 1cc49c894d5..59d426bb597 100644 --- a/modules/nf-core/mtmalign/align/environment.yml +++ b/modules/nf-core/mtmalign/align/environment.yml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - bioconda::mtm-align=20220104 + - conda-forge::pigz=2.8 diff --git a/modules/nf-core/mtmalign/align/main.nf b/modules/nf-core/mtmalign/align/main.nf index 40254dea85d..0faba8c4fca 100644 --- a/modules/nf-core/mtmalign/align/main.nf +++ b/modules/nf-core/mtmalign/align/main.nf @@ -13,8 +13,8 @@ process MTMALIGN_ALIGN { tuple val(meta), path('*.pdb', arity: '2..*') output: - tuple val(meta), path("./mTM_result/${prefix}.aln") , emit: alignment - tuple val(meta), path("./mTM_result/${prefix}.pdb") , emit: structure + tuple val(meta), path("./mTM_result/${prefix}.aln.gz") , emit: alignment + tuple val(meta), path("./mTM_result/${prefix}.pdb.gz") , emit: structure path "versions.yml" , emit: versions when: @@ -29,6 +29,9 @@ process MTMALIGN_ALIGN { # -o does not affect the fasta naming, so move it to the new name mv ./mTM_result/result.fasta ./mTM_result/${prefix}.aln + # compress both output files + pigz -p ${task.cpus} ./mTM_result/${prefix}.aln ./mTM_result/${prefix}.pdb + # mtm-align -v prints the wrong version 20180725, so extract it from the cosmetic output in the help message cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -41,8 +44,8 @@ process MTMALIGN_ALIGN { prefix = task.ext.prefix ?: "${meta.id}" """ mkdir mTM_result - touch mTM_result/${prefix}.aln - touch mTM_result/${prefix}.pdb + touch mTM_result/${prefix}.aln.gz + touch mTM_result/${prefix}.pdb.gz # mtm-align -v prints the wrong version 20180725, so extract it from the cosmetic output in the help message cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/mtmalign/align/meta.yml b/modules/nf-core/mtmalign/align/meta.yml index 9c4b783649e..d3cb8dbd475 100644 --- a/modules/nf-core/mtmalign/align/meta.yml +++ b/modules/nf-core/mtmalign/align/meta.yml @@ -35,11 +35,11 @@ output: - alignment: type: file description: Alignment in FASTA format. - pattern: "*.aln" + pattern: "*.aln.gz" - structure: type: file description: Overlaid structures in PDB format. - pattern: "${prefix}.pdb" + pattern: "${prefix}.pdb.gz" - versions: type: file description: File containing software versions From 3eeca26eb864565b88da2c5f2e454e0911c414cc Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 16 Jan 2024 11:42:16 +0100 Subject: [PATCH 06/66] set to mulled containers --- modules/nf-core/clustalo/align/main.nf | 4 ++-- modules/nf-core/mafft/main.nf | 4 ++-- modules/nf-core/mtmalign/align/main.nf | 4 ++-- modules/nf-core/muscle5/super5/main.nf | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/nf-core/clustalo/align/main.nf b/modules/nf-core/clustalo/align/main.nf index 92266e5cc51..5f179d483e9 100644 --- a/modules/nf-core/clustalo/align/main.nf +++ b/modules/nf-core/clustalo/align/main.nf @@ -4,8 +4,8 @@ process CLUSTALO_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/clustalo:1.2.4--h87f3376_5': - 'biocontainers/clustalo:1.2.4--h87f3376_5' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-4cefc38542f86c17596c29b35a059de10387c6a7: + 'biocontainers/mulled-v2-4cefc38542f86c17596c29b35a059de10387c6a7' }" input: tuple val(meta), path(fasta) diff --git a/modules/nf-core/mafft/main.nf b/modules/nf-core/mafft/main.nf index 1385b9989df..ef34b0d83e7 100644 --- a/modules/nf-core/mafft/main.nf +++ b/modules/nf-core/mafft/main.nf @@ -4,8 +4,8 @@ process MAFFT { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mafft:7.520--h031d066_3': - 'biocontainers/mafft:7.520--h031d066_3' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-12eba4a074f913c639117640936668f5a6a01da6': + 'biocontainers/mulled-v2-12eba4a074f913c639117640936668f5a6a01da6' }" input: tuple val(meta), path(fasta) diff --git a/modules/nf-core/mtmalign/align/main.nf b/modules/nf-core/mtmalign/align/main.nf index 0faba8c4fca..5981df70756 100644 --- a/modules/nf-core/mtmalign/align/main.nf +++ b/modules/nf-core/mtmalign/align/main.nf @@ -6,8 +6,8 @@ process MTMALIGN_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mtm-align:20220104--h4ac6f70_0': - 'biocontainers/mtm-align:20220104--h4ac6f70_0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-5bcf71dc66dac33d8e003c5e78043b80f5c7f269': + 'biocontainers/mulled-v2-5bcf71dc66dac33d8e003c5e78043b80f5c7f269' }" input: tuple val(meta), path('*.pdb', arity: '2..*') diff --git a/modules/nf-core/muscle5/super5/main.nf b/modules/nf-core/muscle5/super5/main.nf index 4386c2ccdff..c4c48fad281 100644 --- a/modules/nf-core/muscle5/super5/main.nf +++ b/modules/nf-core/muscle5/super5/main.nf @@ -3,8 +3,8 @@ process MUSCLE5_SUPER5 { label 'process_medium' conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/muscle:5.1--h9f5acd7_1': - 'biocontainers/muscle:5.1--h9f5acd7_1' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-8eb01a3c2755c935d070dd03ff2dee698eeb4466': + 'biocontainers/mulled-v2-8eb01a3c2755c935d070dd03ff2dee698eeb4466' }" input: tuple val(meta), path(fasta) From d9666d79913fab5c894ac700ebd160481cf9a657 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 16 Jan 2024 11:52:46 +0100 Subject: [PATCH 07/66] more informative test name --- modules/nf-core/mtmalign/align/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test b/modules/nf-core/mtmalign/align/tests/main.nf.test index d3bd9d444f8..ffed25fa01d 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test @@ -8,7 +8,7 @@ nextflow_process { tag "mtmalign" tag "mtmalign/align" - test("Should run without failures") { + test("Test on seatoxin dataset") { setup { run("UNTAR") { From 21f5f816b68141f973b27491ac5909897aa3370c Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 16 Jan 2024 12:12:31 +0100 Subject: [PATCH 08/66] change mtmalign test to search after unzipping --- modules/nf-core/mtmalign/align/tests/main.nf.test | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test b/modules/nf-core/mtmalign/align/tests/main.nf.test index ffed25fa01d..9bbf21695f9 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test @@ -37,11 +37,11 @@ nextflow_process { { assert process.success }, // mTMalign may be nondeterministic, just check if the pdbs are all in there //assert snapshot(process.out).match() - { assert file(process.out.alignment[0][1]).text.contains(">1.pdb") }, - { assert file(process.out.alignment[0][1]).text.contains(">2.pdb") }, - { assert file(process.out.alignment[0][1]).text.contains(">3.pdb") }, - { assert file(process.out.alignment[0][1]).text.contains(">4.pdb") }, - { assert file(process.out.alignment[0][1]).text.contains(">5.pdb") }, + { assert file(process.out.alignment[0][1]).getTextGzip().contains(">1.pdb") }, + { assert file(process.out.alignment[0][1]).getTextGzip().contains(">2.pdb") }, + { assert file(process.out.alignment[0][1]).getTextGzip().contains(">3.pdb") }, + { assert file(process.out.alignment[0][1]).getTextGzip().contains(">4.pdb") }, + { assert file(process.out.alignment[0][1]).getTextGzip().contains(">5.pdb") }, { assert snapshot(process.out.versions).match("versions") } ) } From c1e26f39bb6ea54da8e475706cacec724da1da02 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 16 Jan 2024 15:07:17 +0100 Subject: [PATCH 09/66] update mtmalign tests to work with gzip, fix typo --- modules/nf-core/clustalo/align/main.nf | 2 +- modules/nf-core/mtmalign/align/tests/main.nf.test | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/nf-core/clustalo/align/main.nf b/modules/nf-core/clustalo/align/main.nf index 5f179d483e9..c8a546094ec 100644 --- a/modules/nf-core/clustalo/align/main.nf +++ b/modules/nf-core/clustalo/align/main.nf @@ -4,7 +4,7 @@ process CLUSTALO_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-4cefc38542f86c17596c29b35a059de10387c6a7: + 'https://depot.galaxyproject.org/singularity/mulled-v2-4cefc38542f86c17596c29b35a059de10387c6a7': 'biocontainers/mulled-v2-4cefc38542f86c17596c29b35a059de10387c6a7' }" input: diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test b/modules/nf-core/mtmalign/align/tests/main.nf.test index 9bbf21695f9..40569353223 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test @@ -37,11 +37,11 @@ nextflow_process { { assert process.success }, // mTMalign may be nondeterministic, just check if the pdbs are all in there //assert snapshot(process.out).match() - { assert file(process.out.alignment[0][1]).getTextGzip().contains(">1.pdb") }, - { assert file(process.out.alignment[0][1]).getTextGzip().contains(">2.pdb") }, - { assert file(process.out.alignment[0][1]).getTextGzip().contains(">3.pdb") }, - { assert file(process.out.alignment[0][1]).getTextGzip().contains(">4.pdb") }, - { assert file(process.out.alignment[0][1]).getTextGzip().contains(">5.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">2.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">3.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">4.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">5.pdb") }, { assert snapshot(process.out.versions).match("versions") } ) } From 8eb641e138f2b4466afdf4a43d828a6d0931257c Mon Sep 17 00:00:00 2001 From: Leon Rauschning Date: Tue, 16 Jan 2024 16:05:52 +0100 Subject: [PATCH 10/66] regenerate test snaps --- .../clustalo/align/tests/main.nf.test.snap | 16 +- modules/nf-core/mafft/tests/main.nf.test.snap | 239 +++--------------- .../mtmalign/align/tests/main.nf.test.snap | 47 +--- 3 files changed, 55 insertions(+), 247 deletions(-) diff --git a/modules/nf-core/clustalo/align/tests/main.nf.test.snap b/modules/nf-core/clustalo/align/tests/main.nf.test.snap index 976cb5085c3..685ab7aac1c 100644 --- a/modules/nf-core/clustalo/align/tests/main.nf.test.snap +++ b/modules/nf-core/clustalo/align/tests/main.nf.test.snap @@ -2,10 +2,10 @@ "versions": { "content": [ [ - "versions.yml:md5,67b67f513195c4f7ab9ddadc309135fe" + "versions.yml:md5,327da6a4250a6b7c4e45cddaa1f56280" ] ], - "timestamp": "2023-11-27T14:50:24.517805" + "timestamp": "2024-01-16T15:13:48.820697337" }, "with_guide_tree_alignment": { "content": [ @@ -14,11 +14,11 @@ { "id": "test" }, - "test.aln:md5,74bb9a2820a91cf68db94dbd46787722" + "test.aln.gz:md5,877274cf8a4acc8cf1c4fb19f8d0cf4c" ] ] ], - "timestamp": "2023-11-28T16:28:59.20367" + "timestamp": "2024-01-16T15:19:47.200211384" }, "alignment": { "content": [ @@ -27,18 +27,18 @@ { "id": "test" }, - "test.aln:md5,74bb9a2820a91cf68db94dbd46787722" + "test.aln.gz:md5,877274cf8a4acc8cf1c4fb19f8d0cf4c" ] ] ], - "timestamp": "2023-11-27T14:50:24.471198" + "timestamp": "2024-01-16T15:13:48.789129957" }, "with_guide_tree_versions": { "content": [ [ - "versions.yml:md5,67b67f513195c4f7ab9ddadc309135fe" + "versions.yml:md5,327da6a4250a6b7c4e45cddaa1f56280" ] ], - "timestamp": "2023-11-28T16:28:59.225509" + "timestamp": "2024-01-16T15:19:47.225320481" } } \ No newline at end of file diff --git a/modules/nf-core/mafft/tests/main.nf.test.snap b/modules/nf-core/mafft/tests/main.nf.test.snap index ebf00f7d393..771d6f0929c 100644 --- a/modules/nf-core/mafft/tests/main.nf.test.snap +++ b/modules/nf-core/mafft/tests/main.nf.test.snap @@ -8,11 +8,11 @@ "id": "test", "single_end": false }, - "test.fas:md5,23426611f4a0df532b6708f072bd445b" + "test.fas.gz:md5,d1f6be5d8ab412f582ee090256670fca" ] ], "1": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ], "fas": [ [ @@ -20,172 +20,17 @@ "id": "test", "single_end": false }, - "test.fas:md5,23426611f4a0df532b6708f072bd445b" + "test.fas.gz:md5,d1f6be5d8ab412f582ee090256670fca" ] ], "versions": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ] } ], - "timestamp": "2023-11-17T15:31:46.463322875" + "timestamp": "2024-01-16T14:28:01.433708229" }, - "SARS-CoV-2 scaffolds fasta - add informative sites fasta normal": { - "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": false - }, - "test.fas:md5,a57a34f1c566dea114dc1b13416536d4" - ] - ], - "1": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" - ], - "fas": [ - [ - { - "id": "test", - "single_end": false - }, - "test.fas:md5,a57a34f1c566dea114dc1b13416536d4" - ] - ], - "versions": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" - ] - } - ], - "timestamp": "2023-11-17T15:39:41.739290654" - }, - "SARS-CoV-2 scaffolds fasta - add informative sites fasta long": { - "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": false - }, - "test.fas:md5,e8868da70d1f3050a8daaee0e53b2fd9" - ] - ], - "1": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" - ], - "fas": [ - [ - { - "id": "test", - "single_end": false - }, - "test.fas:md5,e8868da70d1f3050a8daaee0e53b2fd9" - ] - ], - "versions": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" - ] - } - ], - "timestamp": "2023-11-17T15:40:47.75559372" - }, - "SARS-CoV-2 scaffolds fasta - add informative sites fasta profile": { - "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": false - }, - "test.fas:md5,c2b5caf39beff4473878e6aa4036ad43" - ] - ], - "1": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" - ], - "fas": [ - [ - { - "id": "test", - "single_end": false - }, - "test.fas:md5,c2b5caf39beff4473878e6aa4036ad43" - ] - ], - "versions": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" - ] - } - ], - "timestamp": "2023-11-17T15:40:33.934616343" - }, - "SARS-CoV-2 scaffolds fasta - add informative sites fasta fragments": { - "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": false - }, - "test.fas:md5,aed7f866c3a20dc9d2f2b4ad73515961" - ] - ], - "1": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" - ], - "fas": [ - [ - { - "id": "test", - "single_end": false - }, - "test.fas:md5,aed7f866c3a20dc9d2f2b4ad73515961" - ] - ], - "versions": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" - ] - } - ], - "timestamp": "2023-11-17T15:40:05.528308049" - }, - "SARS-CoV-2 scaffolds fasta - add informative sites fasta full": { - "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": false - }, - "test.fas:md5,611cb0a65195a282f110f7f56e310c66" - ] - ], - "1": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" - ], - "fas": [ - [ - { - "id": "test", - "single_end": false - }, - "test.fas:md5,611cb0a65195a282f110f7f56e310c66" - ] - ], - "versions": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" - ] - } - ], - "timestamp": "2023-11-17T15:40:20.088737531" - }, - "SARS-CoV-2 scaffolds fasta - add full": { + "SARS-CoV-2 scaffolds fasta - add informative sites fasta multiple": { "content": [ { "0": [ @@ -194,11 +39,11 @@ "id": "test", "single_end": false }, - "test.fas:md5,611cb0a65195a282f110f7f56e310c66" + "test.fas.gz:md5,33ed0bb9ebd994797774012877558d52" ] ], "1": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ], "fas": [ [ @@ -206,17 +51,17 @@ "id": "test", "single_end": false }, - "test.fas:md5,611cb0a65195a282f110f7f56e310c66" + "test.fas.gz:md5,33ed0bb9ebd994797774012877558d52" ] ], "versions": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ] } ], - "timestamp": "2023-11-17T15:36:41.023632311" + "timestamp": "2024-01-16T14:45:14.79664521" }, - "SARS-CoV-2 scaffolds fasta - add fragments": { + "SARS-CoV-2 scaffolds fasta - add informative sites fasta normal": { "content": [ { "0": [ @@ -225,11 +70,11 @@ "id": "test", "single_end": false }, - "test.fas:md5,aed7f866c3a20dc9d2f2b4ad73515961" + "test.fas.gz:md5,0e940fa705333ea38b3c0354aa053bff" ] ], "1": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ], "fas": [ [ @@ -237,17 +82,17 @@ "id": "test", "single_end": false }, - "test.fas:md5,aed7f866c3a20dc9d2f2b4ad73515961" + "test.fas.gz:md5,0e940fa705333ea38b3c0354aa053bff" ] ], "versions": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ] } ], - "timestamp": "2023-11-17T15:36:20.211074168" + "timestamp": "2024-01-16T14:30:54.033549017" }, - "SARS-CoV-2 scaffolds fasta - add normal": { + "SARS-CoV-2 scaffolds fasta - add informative sites fasta long": { "content": [ { "0": [ @@ -256,11 +101,11 @@ "id": "test", "single_end": false }, - "test.fas:md5,a57a34f1c566dea114dc1b13416536d4" + "test.fas.gz:md5,ffe1f669f170833bd2cbb1d2f9fbcc48" ] ], "1": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ], "fas": [ [ @@ -268,17 +113,17 @@ "id": "test", "single_end": false }, - "test.fas:md5,a57a34f1c566dea114dc1b13416536d4" + "test.fas.gz:md5,ffe1f669f170833bd2cbb1d2f9fbcc48" ] ], "versions": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ] } ], - "timestamp": "2023-11-17T15:36:06.708296829" + "timestamp": "2024-01-16T14:42:21.276116074" }, - "SARS-CoV-2 scaffolds fasta - add informative sites fasta multiple": { + "SARS-CoV-2 scaffolds fasta - add informative sites fasta profile": { "content": [ { "0": [ @@ -287,11 +132,11 @@ "id": "test", "single_end": false }, - "test.fas:md5,aed7f866c3a20dc9d2f2b4ad73515961" + "test.fas.gz:md5,c6a7794437c55e9e5ec3fe62c036b77b" ] ], "1": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ], "fas": [ [ @@ -299,17 +144,17 @@ "id": "test", "single_end": false }, - "test.fas:md5,aed7f866c3a20dc9d2f2b4ad73515961" + "test.fas.gz:md5,c6a7794437c55e9e5ec3fe62c036b77b" ] ], "versions": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ] } ], - "timestamp": "2023-11-17T15:44:19.543603564" + "timestamp": "2024-01-16T14:39:28.070017656" }, - "SARS-CoV-2 scaffolds fasta - add profile": { + "SARS-CoV-2 scaffolds fasta - add informative sites fasta fragments": { "content": [ { "0": [ @@ -318,11 +163,11 @@ "id": "test", "single_end": false }, - "test.fas:md5,c2b5caf39beff4473878e6aa4036ad43" + "test.fas.gz:md5,33ed0bb9ebd994797774012877558d52" ] ], "1": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ], "fas": [ [ @@ -330,17 +175,17 @@ "id": "test", "single_end": false }, - "test.fas:md5,c2b5caf39beff4473878e6aa4036ad43" + "test.fas.gz:md5,33ed0bb9ebd994797774012877558d52" ] ], "versions": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ] } ], - "timestamp": "2023-11-17T15:36:53.336352899" + "timestamp": "2024-01-16T14:33:44.579206218" }, - "SARS-CoV-2 scaffolds fasta - add long": { + "SARS-CoV-2 scaffolds fasta - add informative sites fasta full": { "content": [ { "0": [ @@ -349,11 +194,11 @@ "id": "test", "single_end": false }, - "test.fas:md5,e8868da70d1f3050a8daaee0e53b2fd9" + "test.fas.gz:md5,bc0e5750801ceb7466174f0ea943f590" ] ], "1": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ], "fas": [ [ @@ -361,14 +206,14 @@ "id": "test", "single_end": false }, - "test.fas:md5,e8868da70d1f3050a8daaee0e53b2fd9" + "test.fas.gz:md5,bc0e5750801ceb7466174f0ea943f590" ] ], "versions": [ - "versions.yml:md5,b505d3a7cf3cd5d0e6a5e4d2944df9b7" + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" ] } ], - "timestamp": "2023-11-17T15:37:15.058221937" + "timestamp": "2024-01-16T14:36:37.15084246" } -} +} \ No newline at end of file diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test.snap b/modules/nf-core/mtmalign/align/tests/main.nf.test.snap index d9d6c9bed5b..6870f7d0a45 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test.snap +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test.snap @@ -1,47 +1,10 @@ { - "Should run without failures": { + "versions": { "content": [ - { - "0": [ - [ - { - "id": "test" - }, - "result.fasta:md5,8400950485c61b741f06c9b936bd2863" - ] - ], - "1": [ - [ - { - "id": "test" - }, - "result.pdb:md5,53aeaeae0bd1c0c2c887c4215ec44f5e" - ] - ], - "2": [ - "versions.yml:md5,4b618e4e98abf72feabb3778d3c39105" - ], - "alignment": [ - [ - { - "id": "test" - }, - "result.fasta:md5,8400950485c61b741f06c9b936bd2863" - ] - ], - "structure": [ - [ - { - "id": "test" - }, - "result.pdb:md5,53aeaeae0bd1c0c2c887c4215ec44f5e" - ] - ], - "versions": [ - "versions.yml:md5,4b618e4e98abf72feabb3778d3c39105" - ] - } + [ + "versions.yml:md5,4b618e4e98abf72feabb3778d3c39105" + ] ], - "timestamp": "2023-12-04T17:24:24.975306332" + "timestamp": "2024-01-16T12:20:11.211768299" } } \ No newline at end of file From 82ea41ac0662c1798460265737c5c577e0c0f297 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 16 Jan 2024 16:02:35 +0100 Subject: [PATCH 11/66] muscle5: zip multiple output files, if present --- modules/nf-core/muscle5/super5/main.nf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/nf-core/muscle5/super5/main.nf b/modules/nf-core/muscle5/super5/main.nf index c4c48fad281..73a5444eef6 100644 --- a/modules/nf-core/muscle5/super5/main.nf +++ b/modules/nf-core/muscle5/super5/main.nf @@ -27,7 +27,10 @@ process MUSCLE5_SUPER5 { ${args} \\ -threads ${task.cpus} - pigz -p ${task.cpus} ${prefix}.aln + # output may be multiple files if -perm all is set + for f in *.aln; do + pigz -p ${task.cpus} \$f + done cat <<-END_VERSIONS > versions.yml "${task.process}": From 15f5102dba25d6bb2226a9f3617cc73cccadb56b Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 16 Jan 2024 17:14:53 +0100 Subject: [PATCH 12/66] Change MUSCLE5 tests to the same testcase TCOFFEE is using, also fix it --- .../nf-core/muscle5/super5/tests/main.nf.test | 56 ++++++++++--------- .../muscle5/super5/tests/nextflow.config | 3 - 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/modules/nf-core/muscle5/super5/tests/main.nf.test b/modules/nf-core/muscle5/super5/tests/main.nf.test index 50821a7f86c..2b1afd8394a 100644 --- a/modules/nf-core/muscle5/super5/tests/main.nf.test +++ b/modules/nf-core/muscle5/super5/tests/main.nf.test @@ -1,36 +1,38 @@ nextflow_process { name "Test Process MUSCLE5_SUPER5" - script "../main.nf" - process "MUSCLE5_SUPER5" - config "./nextflow.config" + script "../main.nf" + process "MUSCLE5_SUPER5" + config "./nextflow.config" - tag "modules" - tag "modules_nfcore" - tag "muscle5" - tag "muscle5/super5" + tag "modules" + tag "modules_nfcore" + tag "muscle5" + tag "muscle5/super5" - test("sarscov2 - fasta") { - - when { - process { - """ - input[0] = [[ id:'test' ], - fasta = [ file(params.test_data['sarscov2']['genome']['proteome_fasta'], checkIfExists: true) ] - ] - """ + test("fasta - align_sequence") { + when { + process { + """ + input[0] = [ [ id:'test' ], + file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/setoxin-ref.fa", checkIfExists: true) + ] + """ + } } - } - then { - assertAll( - { assert process.success }, - { assert snapshot(process.out.alignment[0][1][0]).match("aln0")}, - { assert snapshot(process.out.alignment[0][1][1]).match("aln1")}, - { assert snapshot(process.out.alignment[0][1][2]).match("aln2")}, - { assert snapshot(process.out.alignment[0][1][3]).match("aln3")}, - { assert snapshot(process.out.versions).match("versions") } - ) + then { + assertAll( + { assert process.success }, + // although MUSCLE5 is deterministic, for some reason nf-test still computes the checksum before unzipping, which is not deterministic + //{ assert snapshot(process.out.alignment).match("alignment")}, + // check for correct entries instead + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1bds") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1ahl") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1atx") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1sh1") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1apf") } + ) + } } - } } diff --git a/modules/nf-core/muscle5/super5/tests/nextflow.config b/modules/nf-core/muscle5/super5/tests/nextflow.config index f6192f874cd..e69de29bb2d 100644 --- a/modules/nf-core/muscle5/super5/tests/nextflow.config +++ b/modules/nf-core/muscle5/super5/tests/nextflow.config @@ -1,3 +0,0 @@ -process { - ext.args = '-perturb 3 -perm all' -} From 1915cc3a0840fb506c4719729b8296bb817fcd97 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Wed, 17 Jan 2024 17:00:05 +0100 Subject: [PATCH 13/66] add tags requested by nf-core-lint --- modules/nf-core/clustalo/align/tests/main.nf.test | 5 +++-- modules/nf-core/famsa/align/tests/main.nf.test | 3 ++- modules/nf-core/mtmalign/align/tests/main.nf.test | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/clustalo/align/tests/main.nf.test b/modules/nf-core/clustalo/align/tests/main.nf.test index 849dd5fa127..a0e415e3b54 100644 --- a/modules/nf-core/clustalo/align/tests/main.nf.test +++ b/modules/nf-core/clustalo/align/tests/main.nf.test @@ -4,11 +4,12 @@ nextflow_process { script "../main.nf" process "CLUSTALO_ALIGN" config "./nextflow.config" - + tag "modules" tag "modules_nfcore" tag "clustalo" tag "clustalo/align" + tag "clustalo/guidetree" test("sarscov2 - contigs-fasta") { @@ -68,4 +69,4 @@ nextflow_process { ) } } -} +} \ No newline at end of file diff --git a/modules/nf-core/famsa/align/tests/main.nf.test b/modules/nf-core/famsa/align/tests/main.nf.test index 45bfee267fb..adf31b7ebc3 100644 --- a/modules/nf-core/famsa/align/tests/main.nf.test +++ b/modules/nf-core/famsa/align/tests/main.nf.test @@ -8,6 +8,7 @@ nextflow_process { tag "modules_nfcore" tag "famsa" tag "famsa/align" + tag "famsa/guidetree" test("sarscov2 - fasta") { @@ -66,4 +67,4 @@ nextflow_process { ) } } -} +} \ No newline at end of file diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test b/modules/nf-core/mtmalign/align/tests/main.nf.test index 40569353223..94d1a2beee2 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test @@ -7,6 +7,7 @@ nextflow_process { tag "modules_nfcore" tag "mtmalign" tag "mtmalign/align" + tag "untar" test("Test on seatoxin dataset") { setup { From 0839b98d6fbbbfb5015143d43ae411a4b5b6928d Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Wed, 17 Jan 2024 17:24:40 +0100 Subject: [PATCH 14/66] add full url to singularity/biocontainers --- modules/nf-core/clustalo/align/main.nf | 4 ++-- modules/nf-core/mafft/main.nf | 4 ++-- modules/nf-core/mtmalign/align/main.nf | 4 ++-- modules/nf-core/muscle5/super5/main.nf | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/nf-core/clustalo/align/main.nf b/modules/nf-core/clustalo/align/main.nf index c8a546094ec..e2d3514f8f4 100644 --- a/modules/nf-core/clustalo/align/main.nf +++ b/modules/nf-core/clustalo/align/main.nf @@ -4,8 +4,8 @@ process CLUSTALO_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-4cefc38542f86c17596c29b35a059de10387c6a7': - 'biocontainers/mulled-v2-4cefc38542f86c17596c29b35a059de10387c6a7' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-4cefc38542f86c17596c29b35a059de10387c6a7:adbe4fbad680f9beb083956d79128039a727e7b3-0': + 'biocontainers/mulled-v2-4cefc38542f86c17596c29b35a059de10387c6a7:adbe4fbad680f9beb083956d79128039a727e7b3-0' }" input: tuple val(meta), path(fasta) diff --git a/modules/nf-core/mafft/main.nf b/modules/nf-core/mafft/main.nf index ef34b0d83e7..725c2fce64e 100644 --- a/modules/nf-core/mafft/main.nf +++ b/modules/nf-core/mafft/main.nf @@ -4,8 +4,8 @@ process MAFFT { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-12eba4a074f913c639117640936668f5a6a01da6': - 'biocontainers/mulled-v2-12eba4a074f913c639117640936668f5a6a01da6' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-12eba4a074f913c639117640936668f5a6a01da6:425707898cf4f85051b77848be253b88f1d2298a-0': + 'biocontainers/mulled-v2-12eba4a074f913c639117640936668f5a6a01da6:425707898cf4f85051b77848be253b88f1d2298a-0' }" input: tuple val(meta), path(fasta) diff --git a/modules/nf-core/mtmalign/align/main.nf b/modules/nf-core/mtmalign/align/main.nf index 5981df70756..51c83beea4b 100644 --- a/modules/nf-core/mtmalign/align/main.nf +++ b/modules/nf-core/mtmalign/align/main.nf @@ -6,8 +6,8 @@ process MTMALIGN_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-5bcf71dc66dac33d8e003c5e78043b80f5c7f269': - 'biocontainers/mulled-v2-5bcf71dc66dac33d8e003c5e78043b80f5c7f269' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-5bcf71dc66dac33d8e003c5e78043b80f5c7f269:8f0e486d46f7ab38892c1a8f78d2894a549d03b5-0': + 'biocontainers/mulled-v2-5bcf71dc66dac33d8e003c5e78043b80f5c7f269:8f0e486d46f7ab38892c1a8f78d2894a549d03b5-0' }" input: tuple val(meta), path('*.pdb', arity: '2..*') diff --git a/modules/nf-core/muscle5/super5/main.nf b/modules/nf-core/muscle5/super5/main.nf index 73a5444eef6..4c18440a90a 100644 --- a/modules/nf-core/muscle5/super5/main.nf +++ b/modules/nf-core/muscle5/super5/main.nf @@ -3,8 +3,8 @@ process MUSCLE5_SUPER5 { label 'process_medium' conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-8eb01a3c2755c935d070dd03ff2dee698eeb4466': - 'biocontainers/mulled-v2-8eb01a3c2755c935d070dd03ff2dee698eeb4466' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-8eb01a3c2755c935d070dd03ff2dee698eeb4466:ceb6e65e00346ed20d0d8078dddf9858a7af0fe2-0': + 'biocontainers/mulled-v2-8eb01a3c2755c935d070dd03ff2dee698eeb4466:ceb6e65e00346ed20d0d8078dddf9858a7af0fe2-0' }" input: tuple val(meta), path(fasta) From b21623e54fc27d17bfe71d5e2133764e153cc3cd Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Wed, 17 Jan 2024 17:37:41 +0100 Subject: [PATCH 15/66] fix famsa --- modules/nf-core/famsa/align/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/famsa/align/main.nf b/modules/nf-core/famsa/align/main.nf index 9a48dc9edd2..9e1abc17413 100644 --- a/modules/nf-core/famsa/align/main.nf +++ b/modules/nf-core/famsa/align/main.nf @@ -14,7 +14,7 @@ process FAMSA_ALIGN { tuple val(meta2), path(tree) output: - tuple val(meta), path("*.aln"), emit: alignment + tuple val(meta), path("*.aln.gz"), emit: alignment path "versions.yml" , emit: versions when: @@ -40,7 +40,7 @@ process FAMSA_ALIGN { stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix}.aln + touch ${prefix}.aln.gz cat <<-END_VERSIONS > versions.yml "${task.process}": From c790e444c71f3c3a8425b800e6db7f55ace50750 Mon Sep 17 00:00:00 2001 From: Leon Rauschning Date: Wed, 17 Jan 2024 19:02:34 +0100 Subject: [PATCH 16/66] regenerated snapshots with nf-test 0.8.3. Reenabled snapshots for muscle5 and mtmalign --- .../clustalo/align/tests/main.nf.test.snap | 8 ++-- .../famsa/align/tests/main.nf.test.snap | 12 ++--- .../nf-core/mtmalign/align/tests/main.nf.test | 7 +-- .../mtmalign/align/tests/main.nf.test.snap | 45 +++++++++++++++++++ .../nf-core/muscle5/super5/tests/main.nf.test | 11 +---- .../muscle5/super5/tests/main.nf.test.snap | 35 ++++----------- 6 files changed, 66 insertions(+), 52 deletions(-) diff --git a/modules/nf-core/clustalo/align/tests/main.nf.test.snap b/modules/nf-core/clustalo/align/tests/main.nf.test.snap index 685ab7aac1c..a73607c4801 100644 --- a/modules/nf-core/clustalo/align/tests/main.nf.test.snap +++ b/modules/nf-core/clustalo/align/tests/main.nf.test.snap @@ -14,11 +14,11 @@ { "id": "test" }, - "test.aln.gz:md5,877274cf8a4acc8cf1c4fb19f8d0cf4c" + "test.aln.gz:md5,74bb9a2820a91cf68db94dbd46787722" ] ] ], - "timestamp": "2024-01-16T15:19:47.200211384" + "timestamp": "2024-01-17T18:24:51.474838687" }, "alignment": { "content": [ @@ -27,11 +27,11 @@ { "id": "test" }, - "test.aln.gz:md5,877274cf8a4acc8cf1c4fb19f8d0cf4c" + "test.aln.gz:md5,74bb9a2820a91cf68db94dbd46787722" ] ] ], - "timestamp": "2024-01-16T15:13:48.789129957" + "timestamp": "2024-01-17T18:18:36.468757251" }, "with_guide_tree_versions": { "content": [ diff --git a/modules/nf-core/famsa/align/tests/main.nf.test.snap b/modules/nf-core/famsa/align/tests/main.nf.test.snap index f5795e7363e..a281f981fb4 100644 --- a/modules/nf-core/famsa/align/tests/main.nf.test.snap +++ b/modules/nf-core/famsa/align/tests/main.nf.test.snap @@ -5,7 +5,7 @@ "versions.yml:md5,7d9e0a8c263fa6d9017075fe88c9e9dc" ] ], - "timestamp": "2023-11-29T11:45:52.435457374" + "timestamp": "2024-01-17T17:51:57.390391023" }, "with_guide_tree_alignment": { "content": [ @@ -14,11 +14,11 @@ { "id": "test" }, - "test.aln:md5,7cf7375f2ba360814ea978731838b972" + "test.aln.gz:md5,7cf7375f2ba360814ea978731838b972" ] ] ], - "timestamp": "2023-11-29T11:46:17.140198556" + "timestamp": "2024-01-17T18:22:38.944328729" }, "alignment": { "content": [ @@ -27,11 +27,11 @@ { "id": "test" }, - "test.aln:md5,7cf7375f2ba360814ea978731838b972" + "test.aln.gz:md5,7cf7375f2ba360814ea978731838b972" ] ] ], - "timestamp": "2023-11-29T11:45:52.421966018" + "timestamp": "2024-01-17T18:12:17.496026351" }, "with_guide_tree_versions": { "content": [ @@ -39,6 +39,6 @@ "versions.yml:md5,7d9e0a8c263fa6d9017075fe88c9e9dc" ] ], - "timestamp": "2023-11-29T11:46:17.147997548" + "timestamp": "2024-01-17T18:02:24.755216456" } } \ No newline at end of file diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test b/modules/nf-core/mtmalign/align/tests/main.nf.test index 94d1a2beee2..319f7c32723 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test @@ -37,12 +37,7 @@ nextflow_process { assertAll( { assert process.success }, // mTMalign may be nondeterministic, just check if the pdbs are all in there - //assert snapshot(process.out).match() - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1.pdb") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">2.pdb") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">3.pdb") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">4.pdb") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">5.pdb") }, + { assert snapshot(process.out).match() }, { assert snapshot(process.out.versions).match("versions") } ) } diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test.snap b/modules/nf-core/mtmalign/align/tests/main.nf.test.snap index 6870f7d0a45..b4b9ee6888c 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test.snap +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test.snap @@ -6,5 +6,50 @@ ] ], "timestamp": "2024-01-16T12:20:11.211768299" + }, + "Test on seatoxin dataset": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.aln.gz:md5,8400950485c61b741f06c9b936bd2863" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.pdb.gz:md5,53aeaeae0bd1c0c2c887c4215ec44f5e" + ] + ], + "2": [ + "versions.yml:md5,4b618e4e98abf72feabb3778d3c39105" + ], + "alignment": [ + [ + { + "id": "test" + }, + "test.aln.gz:md5,8400950485c61b741f06c9b936bd2863" + ] + ], + "structure": [ + [ + { + "id": "test" + }, + "test.pdb.gz:md5,53aeaeae0bd1c0c2c887c4215ec44f5e" + ] + ], + "versions": [ + "versions.yml:md5,4b618e4e98abf72feabb3778d3c39105" + ] + } + ], + "timestamp": "2024-01-17T18:55:19.617543477" } } \ No newline at end of file diff --git a/modules/nf-core/muscle5/super5/tests/main.nf.test b/modules/nf-core/muscle5/super5/tests/main.nf.test index 2b1afd8394a..af0a08d729a 100644 --- a/modules/nf-core/muscle5/super5/tests/main.nf.test +++ b/modules/nf-core/muscle5/super5/tests/main.nf.test @@ -24,15 +24,8 @@ nextflow_process { then { assertAll( { assert process.success }, - // although MUSCLE5 is deterministic, for some reason nf-test still computes the checksum before unzipping, which is not deterministic - //{ assert snapshot(process.out.alignment).match("alignment")}, - // check for correct entries instead - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1bds") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1ahl") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1atx") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1sh1") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1apf") } + { assert snapshot(process.out.alignment).match("alignment")}, ) } } -} +} \ No newline at end of file diff --git a/modules/nf-core/muscle5/super5/tests/main.nf.test.snap b/modules/nf-core/muscle5/super5/tests/main.nf.test.snap index ba665414273..a810255062b 100644 --- a/modules/nf-core/muscle5/super5/tests/main.nf.test.snap +++ b/modules/nf-core/muscle5/super5/tests/main.nf.test.snap @@ -1,34 +1,15 @@ { - "aln0": { - "content": [ - "testabc.3.aln:md5,006a173438952c838215cd6cc9b42a3d" - ], - "timestamp": "2023-11-28T18:01:32.513414439" - }, - "aln2": { - "content": [ - "testbca.3.aln:md5,00496f951481d58bf23bb7690057c59f" - ], - "timestamp": "2023-11-28T18:01:32.536803426" - }, - "aln1": { - "content": [ - "testacb.3.aln:md5,fc94bb7d0b0308537fa0997c06c5a86f" - ], - "timestamp": "2023-11-28T18:01:32.52911204" - }, - "versions": { + "alignment": { "content": [ [ - "versions.yml:md5,eccf3d4f3b7ffc7d1bacb74c4d20304e" + [ + { + "id": "test" + }, + "test.aln.gz:md5,46ba556df08f7aabbe5e1ba31d226b6a" + ] ] ], - "timestamp": "2023-11-28T17:57:05.227355467" - }, - "aln3": { - "content": [ - "testnone.3.aln:md5,958666a45eabba897417fc2ade08a21b" - ], - "timestamp": "2023-11-28T18:01:32.544996959" + "timestamp": "2024-01-17T18:38:46.432230995" } } \ No newline at end of file From 5a6e78e5a4141be35a0843d48f0112131d2db376 Mon Sep 17 00:00:00 2001 From: Leon Rauschning Date: Wed, 17 Jan 2024 19:32:05 +0100 Subject: [PATCH 17/66] forgot to regenerate mafft, also mtmalign seems to still be nondeterministic --- modules/nf-core/mafft/tests/main.nf.test.snap | 42 +++++++++---------- .../nf-core/mtmalign/align/tests/main.nf.test | 7 +++- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/modules/nf-core/mafft/tests/main.nf.test.snap b/modules/nf-core/mafft/tests/main.nf.test.snap index 771d6f0929c..941fa3c3534 100644 --- a/modules/nf-core/mafft/tests/main.nf.test.snap +++ b/modules/nf-core/mafft/tests/main.nf.test.snap @@ -8,7 +8,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,d1f6be5d8ab412f582ee090256670fca" + "test.fas.gz:md5,23426611f4a0df532b6708f072bd445b" ] ], "1": [ @@ -20,7 +20,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,d1f6be5d8ab412f582ee090256670fca" + "test.fas.gz:md5,23426611f4a0df532b6708f072bd445b" ] ], "versions": [ @@ -28,7 +28,7 @@ ] } ], - "timestamp": "2024-01-16T14:28:01.433708229" + "timestamp": "2024-01-17T19:10:44.597244231" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta multiple": { "content": [ @@ -39,7 +39,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,33ed0bb9ebd994797774012877558d52" + "test.fas.gz:md5,aed7f866c3a20dc9d2f2b4ad73515961" ] ], "1": [ @@ -51,7 +51,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,33ed0bb9ebd994797774012877558d52" + "test.fas.gz:md5,aed7f866c3a20dc9d2f2b4ad73515961" ] ], "versions": [ @@ -59,7 +59,7 @@ ] } ], - "timestamp": "2024-01-16T14:45:14.79664521" + "timestamp": "2024-01-17T19:29:32.199853244" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta normal": { "content": [ @@ -70,7 +70,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,0e940fa705333ea38b3c0354aa053bff" + "test.fas.gz:md5,a57a34f1c566dea114dc1b13416536d4" ] ], "1": [ @@ -82,7 +82,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,0e940fa705333ea38b3c0354aa053bff" + "test.fas.gz:md5,a57a34f1c566dea114dc1b13416536d4" ] ], "versions": [ @@ -90,7 +90,7 @@ ] } ], - "timestamp": "2024-01-16T14:30:54.033549017" + "timestamp": "2024-01-17T19:14:09.623223095" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta long": { "content": [ @@ -101,7 +101,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,ffe1f669f170833bd2cbb1d2f9fbcc48" + "test.fas.gz:md5,e8868da70d1f3050a8daaee0e53b2fd9" ] ], "1": [ @@ -113,7 +113,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,ffe1f669f170833bd2cbb1d2f9fbcc48" + "test.fas.gz:md5,e8868da70d1f3050a8daaee0e53b2fd9" ] ], "versions": [ @@ -121,7 +121,7 @@ ] } ], - "timestamp": "2024-01-16T14:42:21.276116074" + "timestamp": "2024-01-17T19:26:27.263653083" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta profile": { "content": [ @@ -132,7 +132,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,c6a7794437c55e9e5ec3fe62c036b77b" + "test.fas.gz:md5,c2b5caf39beff4473878e6aa4036ad43" ] ], "1": [ @@ -144,7 +144,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,c6a7794437c55e9e5ec3fe62c036b77b" + "test.fas.gz:md5,c2b5caf39beff4473878e6aa4036ad43" ] ], "versions": [ @@ -152,7 +152,7 @@ ] } ], - "timestamp": "2024-01-16T14:39:28.070017656" + "timestamp": "2024-01-17T19:23:20.904278207" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta fragments": { "content": [ @@ -163,7 +163,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,33ed0bb9ebd994797774012877558d52" + "test.fas.gz:md5,aed7f866c3a20dc9d2f2b4ad73515961" ] ], "1": [ @@ -175,7 +175,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,33ed0bb9ebd994797774012877558d52" + "test.fas.gz:md5,aed7f866c3a20dc9d2f2b4ad73515961" ] ], "versions": [ @@ -183,7 +183,7 @@ ] } ], - "timestamp": "2024-01-16T14:33:44.579206218" + "timestamp": "2024-01-17T19:17:15.613863351" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta full": { "content": [ @@ -194,7 +194,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,bc0e5750801ceb7466174f0ea943f590" + "test.fas.gz:md5,611cb0a65195a282f110f7f56e310c66" ] ], "1": [ @@ -206,7 +206,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,bc0e5750801ceb7466174f0ea943f590" + "test.fas.gz:md5,611cb0a65195a282f110f7f56e310c66" ] ], "versions": [ @@ -214,6 +214,6 @@ ] } ], - "timestamp": "2024-01-16T14:36:37.15084246" + "timestamp": "2024-01-17T19:20:19.23954473" } } \ No newline at end of file diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test b/modules/nf-core/mtmalign/align/tests/main.nf.test index 319f7c32723..120a32ac81b 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test @@ -37,7 +37,12 @@ nextflow_process { assertAll( { assert process.success }, // mTMalign may be nondeterministic, just check if the pdbs are all in there - { assert snapshot(process.out).match() }, + //{ assert snapshot(process.out).match() } + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">2.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">3.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">4.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">5.pdb") }, { assert snapshot(process.out.versions).match("versions") } ) } From d26dae7351bbec52a76fd3e2ffe4bc78e26ee4da Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Thu, 18 Jan 2024 11:01:38 +0100 Subject: [PATCH 18/66] update metas --- modules/nf-core/clustalo/align/meta.yml | 4 ++-- modules/nf-core/famsa/align/meta.yml | 4 ++-- modules/nf-core/mafft/meta.yml | 4 ++-- modules/nf-core/mtmalign/align/meta.yml | 2 +- modules/nf-core/muscle5/super5/meta.yml | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/nf-core/clustalo/align/meta.yml b/modules/nf-core/clustalo/align/meta.yml index 639013676bc..2b559abab22 100644 --- a/modules/nf-core/clustalo/align/meta.yml +++ b/modules/nf-core/clustalo/align/meta.yml @@ -39,8 +39,8 @@ output: e.g. `[ id:'test']` - alignment: type: file - description: Alignment file. - pattern: "*.{aln}" + description: Alignment file, in gzipped fasta format + pattern: "*.{aln.gz}" - versions: type: file description: File containing software versions diff --git a/modules/nf-core/famsa/align/meta.yml b/modules/nf-core/famsa/align/meta.yml index 06a66c274f3..c853a7cf839 100644 --- a/modules/nf-core/famsa/align/meta.yml +++ b/modules/nf-core/famsa/align/meta.yml @@ -41,8 +41,8 @@ output: e.g. `[ id:'test']` - alignment: type: file - description: Alignment file. - pattern: "*.{aln}" + description: Alignment file, in gzipped FASTA format + pattern: "*.{aln.z}" - versions: type: file description: File containing software versions diff --git a/modules/nf-core/mafft/meta.yml b/modules/nf-core/mafft/meta.yml index fbe121aa11e..ab4415f7d54 100644 --- a/modules/nf-core/mafft/meta.yml +++ b/modules/nf-core/mafft/meta.yml @@ -79,8 +79,8 @@ output: pattern: "versions.yml" - fas: type: file - description: Aligned sequences in FASTA format - pattern: "*.{fas}" + description: Aligned sequences in gzipped FASTA format + pattern: "*.{fas.gz}" authors: - "@MillironX" maintainers: diff --git a/modules/nf-core/mtmalign/align/meta.yml b/modules/nf-core/mtmalign/align/meta.yml index d3cb8dbd475..753eb838062 100644 --- a/modules/nf-core/mtmalign/align/meta.yml +++ b/modules/nf-core/mtmalign/align/meta.yml @@ -34,7 +34,7 @@ output: e.g. `[ id:'test']` - alignment: type: file - description: Alignment in FASTA format. + description: Alignment in gzipped FASTA format. pattern: "*.aln.gz" - structure: type: file diff --git a/modules/nf-core/muscle5/super5/meta.yml b/modules/nf-core/muscle5/super5/meta.yml index a789aab0951..a158720f24c 100644 --- a/modules/nf-core/muscle5/super5/meta.yml +++ b/modules/nf-core/muscle5/super5/meta.yml @@ -33,8 +33,8 @@ output: pattern: "versions.yml" - alignment: type: file - description: Multiple sequence alignment produced in FASTA format - pattern: "*.{aln}" + description: Multiple sequence alignment produced in gzipped FASTA format + pattern: "*.{aln.gz}" authors: - "@alessiovignoli" - "@JoseEspinosa" From 083fc72b3759060bbbfde5bfaaef20b2b222c573 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 19 Jan 2024 10:55:08 +0100 Subject: [PATCH 19/66] compression support for tcoffee modules --- modules/nf-core/tcoffee/align/environment.yml | 1 + modules/nf-core/tcoffee/align/main.nf | 7 +++++-- modules/nf-core/tcoffee/align/meta.yml | 4 ++-- modules/nf-core/tcoffee/alncompare/environment.yml | 1 + modules/nf-core/tcoffee/alncompare/main.nf | 6 ++++-- modules/nf-core/tcoffee/alncompare/meta.yml | 8 ++++---- 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/modules/nf-core/tcoffee/align/environment.yml b/modules/nf-core/tcoffee/align/environment.yml index 6f63405d47e..28f159fd459 100644 --- a/modules/nf-core/tcoffee/align/environment.yml +++ b/modules/nf-core/tcoffee/align/environment.yml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - bioconda::t-coffee=13.46.0.919e8c6b + - conda-forge::pigz=2.8 diff --git a/modules/nf-core/tcoffee/align/main.nf b/modules/nf-core/tcoffee/align/main.nf index b09a819fb4d..01c3c42f95a 100644 --- a/modules/nf-core/tcoffee/align/main.nf +++ b/modules/nf-core/tcoffee/align/main.nf @@ -13,7 +13,7 @@ process TCOFFEE_ALIGN { tuple val(meta3), path(template), path(accessory_informations) output: - tuple val (meta), path ("*.aln"), emit: alignment + tuple val(meta), path("*.aln.gz"), emit: alignment path "versions.yml" , emit: versions when: @@ -32,22 +32,25 @@ process TCOFFEE_ALIGN { $args \ -thread ${task.cpus} \ -outfile ${prefix}.aln + pigz -p ${task.cpus} ${prefix}.aln cat <<-END_VERSIONS > versions.yml "${task.process}": tcoffee: \$( t_coffee -version | awk '{gsub("Version_", ""); print \$3}') + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix}.aln + touch ${prefix}.aln.gz cat <<-END_VERSIONS > versions.yml "${task.process}": tcoffee: \$( t_coffee -version | awk '{gsub("Version_", ""); print \$3}') + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ } diff --git a/modules/nf-core/tcoffee/align/meta.yml b/modules/nf-core/tcoffee/align/meta.yml index 531aa3e4b04..5ee012ae593 100644 --- a/modules/nf-core/tcoffee/align/meta.yml +++ b/modules/nf-core/tcoffee/align/meta.yml @@ -52,8 +52,8 @@ output: e.g. `[ id:'test']` - alignment: type: file - description: Alignment file. - pattern: "*.{aln}" + description: Alignment file in gzipped FASTA format + pattern: "*.{aln.gz}" - versions: type: file description: File containing software versions diff --git a/modules/nf-core/tcoffee/alncompare/environment.yml b/modules/nf-core/tcoffee/alncompare/environment.yml index 099528eccf4..dfa14141528 100644 --- a/modules/nf-core/tcoffee/alncompare/environment.yml +++ b/modules/nf-core/tcoffee/alncompare/environment.yml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - bioconda::t-coffee=13.46.0.919e8c6b + - conda-forge::pigz=2.8 diff --git a/modules/nf-core/tcoffee/alncompare/main.nf b/modules/nf-core/tcoffee/alncompare/main.nf index 75c008af5bb..03180e9f8df 100644 --- a/modules/nf-core/tcoffee/alncompare/main.nf +++ b/modules/nf-core/tcoffee/alncompare/main.nf @@ -27,8 +27,8 @@ process TCOFFEE_ALNCOMPARE { """ export TEMP='./' t_coffee -other_pg aln_compare \ - -al1 ${ref_msa} \ - -al2 ${msa} \ + -al1 <(unpigz -cdf ${ref_msa})\ + -al2 <(unpigz -cdf ${msa})\ ${args} \ | grep -v "seq1" | grep -v '*' | \ awk '{ print \$4}' ORS="\t" \ @@ -44,6 +44,7 @@ process TCOFFEE_ALNCOMPARE { cat <<-END_VERSIONS > versions.yml "${task.process}": tcoffee: \$( t_coffee -version | awk '{gsub("Version_", ""); print \$3}') + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ stub: @@ -55,6 +56,7 @@ process TCOFFEE_ALNCOMPARE { cat <<-END_VERSIONS > versions.yml "${task.process}": tcoffee: \$( t_coffee -version | awk '{gsub("Version_", ""); print \$3}') + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ } diff --git a/modules/nf-core/tcoffee/alncompare/meta.yml b/modules/nf-core/tcoffee/alncompare/meta.yml index 9f85247692b..2276ef578c6 100644 --- a/modules/nf-core/tcoffee/alncompare/meta.yml +++ b/modules/nf-core/tcoffee/alncompare/meta.yml @@ -21,12 +21,12 @@ input: e.g. [ id:'test', ... ] - msa: type: file - description: fasta file containing the alignment to be evaluated - pattern: "*.{aln,fa,fasta,fas}" + description: fasta file containing the alignment to be evaluated. Can be gzipped or uncompressed + pattern: "*.{aln,fa,fasta,fas}{.gz}" - ref_msa: type: file - description: fasta file containing the reference alignment used for the evaluation - pattern: "*.{aln,fa,fasta,fas}" + description: fasta file containing the reference alignment used for the evaluation. Can be gzipped or uncompressed + pattern: "*.{aln,fa,fasta,fas}{.gz}" output: - meta: From 2497b9fb7cd1bd3bb72f2fdfcd21613e88597bae Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 19 Jan 2024 11:53:07 +0100 Subject: [PATCH 20/66] added pigz to tools in meta --- modules/nf-core/clustalo/align/meta.yml | 4 ++++ modules/nf-core/mafft/meta.yml | 6 +++++- modules/nf-core/mtmalign/align/meta.yml | 4 ++++ modules/nf-core/muscle5/super5/meta.yml | 4 ++++ modules/nf-core/tcoffee/align/meta.yml | 4 ++++ modules/nf-core/tcoffee/alncompare/meta.yml | 4 ++++ 6 files changed, 25 insertions(+), 1 deletion(-) diff --git a/modules/nf-core/clustalo/align/meta.yml b/modules/nf-core/clustalo/align/meta.yml index 2b559abab22..f0718037d55 100644 --- a/modules/nf-core/clustalo/align/meta.yml +++ b/modules/nf-core/clustalo/align/meta.yml @@ -12,6 +12,10 @@ tools: tool_dev_url: "http://www.clustal.org/omega/" doi: "10.1038/msb.2011.75" licence: ["GPL v2"] + - "pigz": + description: "Parallel implementation of the gzip algorithm." + homepage: "https://zlib.net/pigz/" + documentation: "https://zlib.net/pigz/pigz.pdf" input: - meta: type: map diff --git a/modules/nf-core/mafft/meta.yml b/modules/nf-core/mafft/meta.yml index ab4415f7d54..fad68669361 100644 --- a/modules/nf-core/mafft/meta.yml +++ b/modules/nf-core/mafft/meta.yml @@ -5,13 +5,17 @@ keywords: - msa - multiple sequence alignment tools: - - mafft: + - "mafft": description: Multiple alignment program for amino acid or nucleotide sequences based on fast Fourier transform homepage: https://mafft.cbrc.jp/alignment/software/ documentation: https://mafft.cbrc.jp/alignment/software/manual/manual.html tool_dev_url: https://mafft.cbrc.jp/alignment/software/source.html doi: "10.1093/nar/gkf436" licence: ["BSD"] + - "pigz": + description: "Parallel implementation of the gzip algorithm." + homepage: "https://zlib.net/pigz/" + documentation: "https://zlib.net/pigz/pigz.pdf" input: - meta: type: map diff --git a/modules/nf-core/mtmalign/align/meta.yml b/modules/nf-core/mtmalign/align/meta.yml index 753eb838062..fad55afd790 100644 --- a/modules/nf-core/mtmalign/align/meta.yml +++ b/modules/nf-core/mtmalign/align/meta.yml @@ -15,6 +15,10 @@ tools: tool_dev_url: "http://yanglab.nankai.edu.cn/mTM-align/" doi: "10.1093/bioinformatics/btx828" licence: ["None"] + - "pigz": + description: "Parallel implementation of the gzip algorithm." + homepage: "https://zlib.net/pigz/" + documentation: "https://zlib.net/pigz/pigz.pdf" input: - meta: type: map diff --git a/modules/nf-core/muscle5/super5/meta.yml b/modules/nf-core/muscle5/super5/meta.yml index a158720f24c..2950a500bb9 100644 --- a/modules/nf-core/muscle5/super5/meta.yml +++ b/modules/nf-core/muscle5/super5/meta.yml @@ -11,6 +11,10 @@ tools: documentation: "https://drive5.com/muscle5/manual/" doi: "10.1101/2021.06.20.449169" licence: ["Public Domain"] + - "pigz": + description: "Parallel implementation of the gzip algorithm." + homepage: "https://zlib.net/pigz/" + documentation: "https://zlib.net/pigz/pigz.pdf" input: - meta: type: map diff --git a/modules/nf-core/tcoffee/align/meta.yml b/modules/nf-core/tcoffee/align/meta.yml index 5ee012ae593..270de91be7a 100644 --- a/modules/nf-core/tcoffee/align/meta.yml +++ b/modules/nf-core/tcoffee/align/meta.yml @@ -12,6 +12,10 @@ tools: tool_dev_url: "https://github.com/cbcrg/tcoffee" doi: "10.1006/jmbi.2000.4042" licence: ["GPL v3"] + - "pigz": + description: "Parallel implementation of the gzip algorithm." + homepage: "https://zlib.net/pigz/" + documentation: "https://zlib.net/pigz/pigz.pdf" input: - meta: type: map diff --git a/modules/nf-core/tcoffee/alncompare/meta.yml b/modules/nf-core/tcoffee/alncompare/meta.yml index 2276ef578c6..6267960b8ce 100644 --- a/modules/nf-core/tcoffee/alncompare/meta.yml +++ b/modules/nf-core/tcoffee/alncompare/meta.yml @@ -12,6 +12,10 @@ tools: tool_dev_url: "https://github.com/cbcrg/tcoffee" doi: "10.1006/jmbi.2000.4042" licence: ["GPL v3"] + - "pigz": + description: "Parallel implementation of the gzip algorithm." + homepage: "https://zlib.net/pigz/" + documentation: "https://zlib.net/pigz/pigz.pdf" input: - meta: From f6cfc4a0363518589a332d7ab3426bb3de46ec47 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 19 Jan 2024 17:51:25 +0100 Subject: [PATCH 21/66] fix typo --- modules/nf-core/tcoffee/align/tests/main.nf.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test b/modules/nf-core/tcoffee/align/tests/main.nf.test index bb17194c568..1330f82d3f4 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test @@ -69,7 +69,7 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot(process.out.alignment).match("alignment_guidetree")}, - { assert snapshot(process.out.versions).match("versions_guidtree") } + { assert snapshot(process.out.versions).match("versions_guidetree") } ) } @@ -116,4 +116,4 @@ nextflow_process { } } -} +} \ No newline at end of file From 9c6481d9a4aa3825945a3052e57b9acd51978015 Mon Sep 17 00:00:00 2001 From: Leon Rauschning Date: Fri, 19 Jan 2024 17:58:57 +0100 Subject: [PATCH 22/66] regenerate snaps, adjust test to gzip --- .../nf-core/tcoffee/align/tests/main.nf.test | 2 +- .../tcoffee/align/tests/main.nf.test.snap | 26 +++++++++---------- .../alncompare/tests/main.nf.test.snap | 6 ++--- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test b/modules/nf-core/tcoffee/align/tests/main.nf.test index 1330f82d3f4..97184136988 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test @@ -110,7 +110,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert path(process.out.alignment.get(0).get(1)).getText().contains("1ahl") }, + { assert path(process.out.alignment.get(0).get(1)).getTextGzip().contains("1ahl") }, { assert snapshot(process.out.versions).match("versions_structure") } ) } diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test.snap b/modules/nf-core/tcoffee/align/tests/main.nf.test.snap index 79dd44e5707..3e931aa5eb0 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test.snap +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test.snap @@ -2,51 +2,51 @@ "versions_structure": { "content": [ [ - "versions.yml:md5,4d38a2c2211ee21b5c1b181e8260f5f7" + "versions.yml:md5,cc94990c99a9b72c0ad1e22026aaee72" ] ], - "timestamp": "2023-11-28T12:26:47.937090039" + "timestamp": "2024-01-19T17:56:36.791833358" }, "versions": { "content": [ [ - "versions.yml:md5,4d38a2c2211ee21b5c1b181e8260f5f7" + "versions.yml:md5,cc94990c99a9b72c0ad1e22026aaee72" ] ], - "timestamp": "2023-11-28T12:26:26.949694794" + "timestamp": "2024-01-19T17:56:17.01424266" }, - "versions_guidtree": { + "versions_guidetree": { "content": [ [ - "versions.yml:md5,4d38a2c2211ee21b5c1b181e8260f5f7" + "versions.yml:md5,cc94990c99a9b72c0ad1e22026aaee72" ] ], - "timestamp": "2023-11-28T12:26:36.766599176" + "timestamp": "2024-01-19T17:56:26.914325234" }, - "msa": { + "alignment": { "content": [ [ [ { "id": "test" }, - "test.aln:md5,bd1db08ad04514cc6d1334598c1a6ef0" + "test.aln.gz:md5,bd1db08ad04514cc6d1334598c1a6ef0" ] ] ], - "timestamp": "2023-11-28T12:26:26.904709139" + "timestamp": "2024-01-19T17:56:16.987455592" }, - "msa_guidetree": { + "alignment_guidetree": { "content": [ [ [ { "id": "test" }, - "test.aln:md5,93bc8adfcd88f7913718eacc13da8e4a" + "test.aln.gz:md5,93bc8adfcd88f7913718eacc13da8e4a" ] ] ], - "timestamp": "2023-11-28T12:26:36.702189365" + "timestamp": "2024-01-19T17:56:26.885044145" } } \ No newline at end of file diff --git a/modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap b/modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap index 7524515a848..95595a784a0 100644 --- a/modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap +++ b/modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap @@ -11,7 +11,7 @@ ] ], "1": [ - "versions.yml:md5,8a30677771f2dc3b61b60702622fdfda" + "versions.yml:md5,2e5d1ba11377635c98d33aae95b5a330" ], "scores": [ [ @@ -22,10 +22,10 @@ ] ], "versions": [ - "versions.yml:md5,8a30677771f2dc3b61b60702622fdfda" + "versions.yml:md5,2e5d1ba11377635c98d33aae95b5a330" ] } ], - "timestamp": "2023-12-13T10:50:50.701336" + "timestamp": "2024-01-19T17:58:18.966168883" } } \ No newline at end of file From bd443c19e272b7bd2f60a8e97c73fe7ea5f61628 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:54:41 +0100 Subject: [PATCH 23/66] added mulled containers for tcoffee --- modules/nf-core/tcoffee/align/main.nf | 4 ++-- modules/nf-core/tcoffee/alncompare/main.nf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/tcoffee/align/main.nf b/modules/nf-core/tcoffee/align/main.nf index 01c3c42f95a..adee945adba 100644 --- a/modules/nf-core/tcoffee/align/main.nf +++ b/modules/nf-core/tcoffee/align/main.nf @@ -4,8 +4,8 @@ process TCOFFEE_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/t-coffee:13.46.0.919e8c6b--hfc96bf3_0': - 'biocontainers/t-coffee:13.46.0.919e8c6b--hfc96bf3_0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-a76a981c07359a31ff55b9dc13bd3da5ce1909c1:84c8f17f1259b49e2f7783b95b7a89c6f2cb199e-0': + 'biocontainers/mulled-v2-a76a981c07359a31ff55b9dc13bd3da5ce1909c1:84c8f17f1259b49e2f7783b95b7a89c6f2cb199e-0' }" input: tuple val(meta) , path(fasta) diff --git a/modules/nf-core/tcoffee/alncompare/main.nf b/modules/nf-core/tcoffee/alncompare/main.nf index 03180e9f8df..c3f7d42ab2a 100644 --- a/modules/nf-core/tcoffee/alncompare/main.nf +++ b/modules/nf-core/tcoffee/alncompare/main.nf @@ -4,8 +4,8 @@ process TCOFFEE_ALNCOMPARE { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/t-coffee:13.46.0.919e8c6b--hfc96bf3_0': - 'biocontainers/t-coffee:13.46.0.919e8c6b--hfc96bf3_0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-a76a981c07359a31ff55b9dc13bd3da5ce1909c1:84c8f17f1259b49e2f7783b95b7a89c6f2cb199e-0': + 'biocontainers/mulled-v2-a76a981c07359a31ff55b9dc13bd3da5ce1909c1:84c8f17f1259b49e2f7783b95b7a89c6f2cb199e-0' }" input: tuple val(meta), path(msa), path(ref_msa) From ea6d06ee694725b3f826a26238c9ce0d6ce0885e Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:48:17 +0100 Subject: [PATCH 24/66] implement compression switching with channel --- modules/nf-core/tcoffee/align/main.nf | 11 ++++--- .../nf-core/tcoffee/align/tests/main.nf.test | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/tcoffee/align/main.nf b/modules/nf-core/tcoffee/align/main.nf index adee945adba..32c23efd682 100644 --- a/modules/nf-core/tcoffee/align/main.nf +++ b/modules/nf-core/tcoffee/align/main.nf @@ -11,9 +11,10 @@ process TCOFFEE_ALIGN { tuple val(meta) , path(fasta) tuple val(meta2), path(tree) tuple val(meta3), path(template), path(accessory_informations) + val(compress) output: - tuple val(meta), path("*.aln.gz"), emit: alignment + tuple val(meta), path("*.aln{.gz,}"), emit: alignment path "versions.yml" , emit: versions when: @@ -21,9 +22,10 @@ process TCOFFEE_ALIGN { script: def args = task.ext.args ?: '' - prefix = task.ext.prefix ?: "${meta.id}" + def prefix = task.ext.prefix ?: "${meta.id}" def tree_args = tree ? "-usetree $tree" : "" def template_args = template ? "-template_file $template" : "" + def pigz_call = compress ? "pigz -p ${task.cpus} ${prefix}.aln" : "" """ export TEMP='./' t_coffee -seq ${fasta} \ @@ -32,7 +34,7 @@ process TCOFFEE_ALIGN { $args \ -thread ${task.cpus} \ -outfile ${prefix}.aln - pigz -p ${task.cpus} ${prefix}.aln + ${pigz_call} cat <<-END_VERSIONS > versions.yml @@ -44,8 +46,9 @@ process TCOFFEE_ALIGN { stub: def prefix = task.ext.prefix ?: "${meta.id}" + def pigz_call = compressed ? ".gz": "" """ - touch ${prefix}.aln.gz + touch ${prefix}.aln${pigz_call} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test b/modules/nf-core/tcoffee/align/tests/main.nf.test index 97184136988..ad53451a3cb 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test @@ -21,6 +21,33 @@ nextflow_process { ] input[1] = [[:],[]] input[2] = [[:],[],[]] + input[3] = [true] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.alignment).match("alignment")}, + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("fasta - align_sequence - uncompressed") { + + config "./sequence.config" + + when { + process { + """ + input[0] = [ [ id:'test' ], + file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/setoxin-ref.fa", checkIfExists: true) + ] + input[1] = [[:],[]] + input[2] = [[:],[],[]] + input[3] = [false] """ } } @@ -61,6 +88,7 @@ nextflow_process { ] input[1] = FAMSA_GUIDETREE.out.tree.collect{ meta, tree -> tree }.map{ tree -> [[ id: 'test'], tree]} input[2] = [ [:], [], [] ] + input[3] = [true] """ } } @@ -102,6 +130,7 @@ nextflow_process { ] input[1] = [ [:], [] ] input[2] = UNTAR.out.untar.map { meta,dir -> [[ id:'test' ], [] ,file(dir).listFiles().collect()]} + input[3] = [true] """ } From 746d60181dc139efe833c4e79cb62b1c2d5cffb1 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:48:25 +0100 Subject: [PATCH 25/66] add tags wanted by lint --- modules/nf-core/tcoffee/align/tests/main.nf.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test b/modules/nf-core/tcoffee/align/tests/main.nf.test index ad53451a3cb..0d28c2f3304 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test @@ -8,6 +8,8 @@ nextflow_process { tag "modules_nfcore" tag "tcoffee" tag "tcoffee/align" + tag "famsa/guidetree" + tag "untar" test("fasta - align_sequence") { From 93f2943bae610ea43b24e60abb88204901355806 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:56:25 +0100 Subject: [PATCH 26/66] regenerate snapshots --- modules/nf-core/tcoffee/align/tests/main.nf.test.snap | 8 ++++---- .../nf-core/tcoffee/alncompare/tests/main.nf.test.snap | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test.snap b/modules/nf-core/tcoffee/align/tests/main.nf.test.snap index 3e931aa5eb0..0dc5c127da0 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test.snap +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test.snap @@ -2,7 +2,7 @@ "versions_structure": { "content": [ [ - "versions.yml:md5,cc94990c99a9b72c0ad1e22026aaee72" + "versions.yml:md5,fb187c9186b50a8076d08cd3be3c1b70" ] ], "timestamp": "2024-01-19T17:56:36.791833358" @@ -10,7 +10,7 @@ "versions": { "content": [ [ - "versions.yml:md5,cc94990c99a9b72c0ad1e22026aaee72" + "versions.yml:md5,fb187c9186b50a8076d08cd3be3c1b70" ] ], "timestamp": "2024-01-19T17:56:17.01424266" @@ -18,7 +18,7 @@ "versions_guidetree": { "content": [ [ - "versions.yml:md5,cc94990c99a9b72c0ad1e22026aaee72" + "versions.yml:md5,fb187c9186b50a8076d08cd3be3c1b70" ] ], "timestamp": "2024-01-19T17:56:26.914325234" @@ -49,4 +49,4 @@ ], "timestamp": "2024-01-19T17:56:26.885044145" } -} \ No newline at end of file +} diff --git a/modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap b/modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap index 95595a784a0..3d0f4d99dd8 100644 --- a/modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap +++ b/modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap @@ -11,7 +11,7 @@ ] ], "1": [ - "versions.yml:md5,2e5d1ba11377635c98d33aae95b5a330" + "versions.yml:md5,438507517a1a831c7b7a1571b1fdd98d" ], "scores": [ [ @@ -28,4 +28,4 @@ ], "timestamp": "2024-01-19T17:58:18.966168883" } -} \ No newline at end of file +} From a3e62057df626cb23a04e62947006845e8081a09 Mon Sep 17 00:00:00 2001 From: Leon Rauschning Date: Mon, 22 Jan 2024 17:09:39 +0100 Subject: [PATCH 27/66] whoops, regenerated using container this time --- modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap b/modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap index 3d0f4d99dd8..d1fd92e4304 100644 --- a/modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap +++ b/modules/nf-core/tcoffee/alncompare/tests/main.nf.test.snap @@ -22,10 +22,10 @@ ] ], "versions": [ - "versions.yml:md5,2e5d1ba11377635c98d33aae95b5a330" + "versions.yml:md5,438507517a1a831c7b7a1571b1fdd98d" ] } ], - "timestamp": "2024-01-19T17:58:18.966168883" + "timestamp": "2024-01-22T17:08:59.494237269" } -} +} \ No newline at end of file From 7a5bb1ec374dc10c5535be3cbc25c10b27b42f67 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:13:56 +0100 Subject: [PATCH 28/66] update meta.yml --- modules/nf-core/tcoffee/align/meta.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/tcoffee/align/meta.yml b/modules/nf-core/tcoffee/align/meta.yml index 270de91be7a..e1a42dc1e8a 100644 --- a/modules/nf-core/tcoffee/align/meta.yml +++ b/modules/nf-core/tcoffee/align/meta.yml @@ -48,6 +48,9 @@ input: type: file description: Accessory files to be used in the alignment. For example, it could be protein structures or secondary structures. pattern: "*" + - compress: + type: val + description: Flag representing whether the output MSA should be compressed. Set to true to enable/false to disable compression. Compression is done using pigz, and is multithreaded. output: - meta: type: map @@ -56,8 +59,8 @@ output: e.g. `[ id:'test']` - alignment: type: file - description: Alignment file in gzipped FASTA format - pattern: "*.{aln.gz}" + description: Alignment file in FASTA format. May be gzipped. + pattern: "*.aln{.gz,}" - versions: type: file description: File containing software versions @@ -68,3 +71,4 @@ authors: maintainers: - "@luisas" - "@JoseEspinosa" + - "@lrauschning" From 32e3508042fdbf3e2313efbbd8c47cb957b75e6a Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:30:27 +0100 Subject: [PATCH 29/66] update glob in meta.yml --- modules/nf-core/tcoffee/alncompare/meta.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/tcoffee/alncompare/meta.yml b/modules/nf-core/tcoffee/alncompare/meta.yml index 6267960b8ce..31502555d9e 100644 --- a/modules/nf-core/tcoffee/alncompare/meta.yml +++ b/modules/nf-core/tcoffee/alncompare/meta.yml @@ -26,11 +26,11 @@ input: - msa: type: file description: fasta file containing the alignment to be evaluated. Can be gzipped or uncompressed - pattern: "*.{aln,fa,fasta,fas}{.gz}" + pattern: "*.{aln,fa,fasta,fas}{.gz,}" - ref_msa: type: file description: fasta file containing the reference alignment used for the evaluation. Can be gzipped or uncompressed - pattern: "*.{aln,fa,fasta,fas}{.gz}" + pattern: "*.{aln,fa,fasta,fas}{.gz,}" output: - meta: From b29d8023426e972b9025a9bc61e1d3bea7b4ba4e Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:30:35 +0100 Subject: [PATCH 30/66] support compressed input in irmsd --- modules/nf-core/tcoffee/irmsd/environment.yml | 1 + modules/nf-core/tcoffee/irmsd/main.nf | 6 ++++-- modules/nf-core/tcoffee/irmsd/meta.yml | 14 +++++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/modules/nf-core/tcoffee/irmsd/environment.yml b/modules/nf-core/tcoffee/irmsd/environment.yml index 635de77940d..4065fe70bd6 100644 --- a/modules/nf-core/tcoffee/irmsd/environment.yml +++ b/modules/nf-core/tcoffee/irmsd/environment.yml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - bioconda::t-coffee=13.46.0.919e8c6b + - conda-forge::pigz=2.8 diff --git a/modules/nf-core/tcoffee/irmsd/main.nf b/modules/nf-core/tcoffee/irmsd/main.nf index 0c60312ea78..8ce53345db1 100644 --- a/modules/nf-core/tcoffee/irmsd/main.nf +++ b/modules/nf-core/tcoffee/irmsd/main.nf @@ -25,13 +25,14 @@ process TCOFFEE_IRMSD { export TEMP='./' t_coffee -other_pg irmsd \ - $msa \ + <( unpigz -cdf $msa ) \ $args \ - -template_file $template > ${prefix}.irmsd + -template_file <( unpigz -cdf $template ) > ${prefix}.irmsd cat <<-END_VERSIONS > versions.yml "${task.process}": tcoffee: \$( t_coffee -version | awk '{gsub("Version_", ""); print \$3}') + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ @@ -44,6 +45,7 @@ process TCOFFEE_IRMSD { cat <<-END_VERSIONS > versions.yml "${task.process}": tcoffee: \$( t_coffee -version | awk '{gsub("Version_", ""); print \$3}') + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ } diff --git a/modules/nf-core/tcoffee/irmsd/meta.yml b/modules/nf-core/tcoffee/irmsd/meta.yml index abee1a3a5d4..9b4aae56556 100644 --- a/modules/nf-core/tcoffee/irmsd/meta.yml +++ b/modules/nf-core/tcoffee/irmsd/meta.yml @@ -1,5 +1,5 @@ name: "tcoffee_irmsd" -description: Computes irmsd score for a given alignment and the structures. +description: Computes the irmsd score for a given alignment and the structures. keywords: - alignment - MSA @@ -12,6 +12,10 @@ tools: tool_dev_url: "https://github.com/cbcrg/tcoffee" doi: "10.1006/jmbi.2000.4042" licence: ["GPL v3"] + - "pigz": + description: "Parallel implementation of the gzip algorithm." + homepage: "https://zlib.net/pigz/" + documentation: "https://zlib.net/pigz/pigz.pdf" input: - meta: @@ -21,12 +25,12 @@ input: e.g. [ id:'test', ... ] - msa: type: file - description: fasta file containing the alignment to be evaluated - pattern: "*.{aln,fa,fasta,fas}" + description: fasta file containing the alignment to be evaluated. May be gzipped or uncompressed. + pattern: "*.{aln,fa,fasta,fas}{.gz,}" - template: type: file - description: Template file matching the structures to the sequences in the alignment - pattern: "*" + description: Template file matching the structures to the sequences in the alignment. May be gzipped or uncompressed. + pattern: "*{.gz,}" - structures: type: directory description: Directory containing the structures file matching the sequences in the alignment in PDB format From ad33963eec44d097b4e12b61d1d0e4b4f9d3bdc9 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Mon, 22 Jan 2024 18:37:18 +0100 Subject: [PATCH 31/66] assign more precise type in meta.yml --- modules/nf-core/tcoffee/align/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/tcoffee/align/meta.yml b/modules/nf-core/tcoffee/align/meta.yml index e1a42dc1e8a..6cfcc72d46a 100644 --- a/modules/nf-core/tcoffee/align/meta.yml +++ b/modules/nf-core/tcoffee/align/meta.yml @@ -49,7 +49,7 @@ input: description: Accessory files to be used in the alignment. For example, it could be protein structures or secondary structures. pattern: "*" - compress: - type: val + type: boolean description: Flag representing whether the output MSA should be compressed. Set to true to enable/false to disable compression. Compression is done using pigz, and is multithreaded. output: - meta: From 21ba7d4a5d904ee102b91b4a2ce977b7d0bf4e61 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Mon, 22 Jan 2024 18:38:45 +0100 Subject: [PATCH 32/66] add tag flagged by lint to tcoffee/irmsd --- modules/nf-core/tcoffee/irmsd/tests/main.nf.test | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/tcoffee/irmsd/tests/main.nf.test b/modules/nf-core/tcoffee/irmsd/tests/main.nf.test index 1519a66da89..954bed7befe 100644 --- a/modules/nf-core/tcoffee/irmsd/tests/main.nf.test +++ b/modules/nf-core/tcoffee/irmsd/tests/main.nf.test @@ -6,7 +6,8 @@ nextflow_process { tag "modules" tag "modules_nfcore" tag "tcoffee" - tag "tcoffee/irmsd" + tag "tcoffee/irmsd" + tag "untar" test("seatoxin") { @@ -24,7 +25,7 @@ nextflow_process { } } } - + when { process { """ @@ -35,7 +36,7 @@ nextflow_process { input[1] = UNTAR.out.untar.map { meta,dir -> [[ id:'test' ], file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/templates/seatoxin-ref_template.txt", checkIfExists: true) ,file(dir).listFiles().collect()]} """ } - + } then { @@ -47,4 +48,4 @@ nextflow_process { } -} +} \ No newline at end of file From e9617ba98f437be76d224dc271987b852fe5086c Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Mon, 22 Jan 2024 18:47:19 +0100 Subject: [PATCH 33/66] set tcoffee/irmsd to use mulled container --- modules/nf-core/tcoffee/align/main.nf | 7 +++++-- modules/nf-core/tcoffee/irmsd/main.nf | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/tcoffee/align/main.nf b/modules/nf-core/tcoffee/align/main.nf index 32c23efd682..555aacbe4ee 100644 --- a/modules/nf-core/tcoffee/align/main.nf +++ b/modules/nf-core/tcoffee/align/main.nf @@ -28,13 +28,16 @@ process TCOFFEE_ALIGN { def pigz_call = compress ? "pigz -p ${task.cpus} ${prefix}.aln" : "" """ export TEMP='./' + mkfifo named_pipe t_coffee -seq ${fasta} \ $tree_args \ $template_args \ $args \ -thread ${task.cpus} \ - -outfile ${prefix}.aln - ${pigz_call} + -outfile named_pipe & + cat named_pipe | pigz -cp ${task.cpus} > ${prefix}.aln.gz + cat named_pipe + #${pigz_call} cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/tcoffee/irmsd/main.nf b/modules/nf-core/tcoffee/irmsd/main.nf index 8ce53345db1..3d11112ea37 100644 --- a/modules/nf-core/tcoffee/irmsd/main.nf +++ b/modules/nf-core/tcoffee/irmsd/main.nf @@ -4,8 +4,8 @@ process TCOFFEE_IRMSD { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/t-coffee:13.46.0.919e8c6b--hfc96bf3_0': - 'biocontainers/t-coffee:13.46.0.919e8c6b--hfc96bf3_0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-a76a981c07359a31ff55b9dc13bd3da5ce1909c1:84c8f17f1259b49e2f7783b95b7a89c6f2cb199e-0': + 'biocontainers/mulled-v2-a76a981c07359a31ff55b9dc13bd3da5ce1909c1:84c8f17f1259b49e2f7783b95b7a89c6f2cb199e-0' }" input: tuple val(meta), file (msa) From 4fd533d5fd52e53eef4ce5063cb7e6c4076d6401 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Mon, 22 Jan 2024 20:10:14 +0100 Subject: [PATCH 34/66] tcoffee/irmsd: do not compress template file, and correctly uncompress for irmsd --- modules/nf-core/tcoffee/irmsd/main.nf | 5 +++-- modules/nf-core/tcoffee/irmsd/meta.yml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/tcoffee/irmsd/main.nf b/modules/nf-core/tcoffee/irmsd/main.nf index 3d11112ea37..2dbf9db58c4 100644 --- a/modules/nf-core/tcoffee/irmsd/main.nf +++ b/modules/nf-core/tcoffee/irmsd/main.nf @@ -24,10 +24,11 @@ process TCOFFEE_IRMSD { """ export TEMP='./' + unpigz -cdf $msa > msafile & t_coffee -other_pg irmsd \ - <( unpigz -cdf $msa ) \ + msafile \ $args \ - -template_file <( unpigz -cdf $template ) > ${prefix}.irmsd + -template_file $template > ${prefix}.irmsd cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/tcoffee/irmsd/meta.yml b/modules/nf-core/tcoffee/irmsd/meta.yml index 9b4aae56556..f55fca748cf 100644 --- a/modules/nf-core/tcoffee/irmsd/meta.yml +++ b/modules/nf-core/tcoffee/irmsd/meta.yml @@ -29,8 +29,8 @@ input: pattern: "*.{aln,fa,fasta,fas}{.gz,}" - template: type: file - description: Template file matching the structures to the sequences in the alignment. May be gzipped or uncompressed. - pattern: "*{.gz,}" + description: Template file matching the structures to the sequences in the alignment. + pattern: "*" - structures: type: directory description: Directory containing the structures file matching the sequences in the alignment in PDB format From 1353118e181a5227cc94e54870c3ee3390bb9f6f Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 23 Jan 2024 12:26:47 +0100 Subject: [PATCH 35/66] tcoffee/align: reimplement toggling compression --- modules/nf-core/tcoffee/align/main.nf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/nf-core/tcoffee/align/main.nf b/modules/nf-core/tcoffee/align/main.nf index 555aacbe4ee..da5b5325c05 100644 --- a/modules/nf-core/tcoffee/align/main.nf +++ b/modules/nf-core/tcoffee/align/main.nf @@ -25,19 +25,19 @@ process TCOFFEE_ALIGN { def prefix = task.ext.prefix ?: "${meta.id}" def tree_args = tree ? "-usetree $tree" : "" def template_args = template ? "-template_file $template" : "" - def pigz_call = compress ? "pigz -p ${task.cpus} ${prefix}.aln" : "" + def write_output = compress ? " | pigz -cp ${task.cpus} > ${prefix}.aln.gz" : "> ${prefix}.aln" """ export TEMP='./' - mkfifo named_pipe + mkfifo out_pipe t_coffee -seq ${fasta} \ + #-output fasta_aln \ # suppress html output $tree_args \ $template_args \ $args \ -thread ${task.cpus} \ - -outfile named_pipe & - cat named_pipe | pigz -cp ${task.cpus} > ${prefix}.aln.gz - cat named_pipe - #${pigz_call} + -outfile out_pipe & + cat named_pipe ${write_output} + cat named_pipe # opening a file handle again is necessary for tcoffee to end execution cat <<-END_VERSIONS > versions.yml From 9e4485943877180636a7d086e30c71f1d6f88f51 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 23 Jan 2024 12:32:58 +0100 Subject: [PATCH 36/66] tcoffee/align: use new pipe name everywhere --- modules/nf-core/tcoffee/align/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/tcoffee/align/main.nf b/modules/nf-core/tcoffee/align/main.nf index da5b5325c05..1c52f6ead09 100644 --- a/modules/nf-core/tcoffee/align/main.nf +++ b/modules/nf-core/tcoffee/align/main.nf @@ -36,8 +36,8 @@ process TCOFFEE_ALIGN { $args \ -thread ${task.cpus} \ -outfile out_pipe & - cat named_pipe ${write_output} - cat named_pipe # opening a file handle again is necessary for tcoffee to end execution + cat out_pipe ${write_output} + cat out_pipe # opening a file handle again is necessary for tcoffee to end execution cat <<-END_VERSIONS > versions.yml From ad9516ec2a161a582c41b292fe9181dbd96381c4 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:10:18 +0100 Subject: [PATCH 37/66] tcoffee/align: reenable default html output, add comment --- modules/nf-core/tcoffee/align/main.nf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/tcoffee/align/main.nf b/modules/nf-core/tcoffee/align/main.nf index 1c52f6ead09..5e10cf9e672 100644 --- a/modules/nf-core/tcoffee/align/main.nf +++ b/modules/nf-core/tcoffee/align/main.nf @@ -30,14 +30,15 @@ process TCOFFEE_ALIGN { export TEMP='./' mkfifo out_pipe t_coffee -seq ${fasta} \ - #-output fasta_aln \ # suppress html output + # enable to suppress default html output that is not passed on by the module + #-output fasta_aln\ $tree_args \ $template_args \ $args \ -thread ${task.cpus} \ -outfile out_pipe & cat out_pipe ${write_output} - cat out_pipe # opening a file handle again is necessary for tcoffee to end execution + cat out_pipe > /dev/null # opening a file handle again is necessary for tcoffee to end execution cat <<-END_VERSIONS > versions.yml From a181b030f452362c6155e2c9b3d5171fdfc237ea Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:13:31 +0100 Subject: [PATCH 38/66] fix escaped line at end of comment... --- modules/nf-core/tcoffee/align/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/tcoffee/align/main.nf b/modules/nf-core/tcoffee/align/main.nf index 5e10cf9e672..5ef6f79b2fb 100644 --- a/modules/nf-core/tcoffee/align/main.nf +++ b/modules/nf-core/tcoffee/align/main.nf @@ -30,13 +30,13 @@ process TCOFFEE_ALIGN { export TEMP='./' mkfifo out_pipe t_coffee -seq ${fasta} \ - # enable to suppress default html output that is not passed on by the module - #-output fasta_aln\ $tree_args \ $template_args \ $args \ -thread ${task.cpus} \ -outfile out_pipe & + # enable to suppress default html output that is not passed on by the module + #-output fasta_aln cat out_pipe ${write_output} cat out_pipe > /dev/null # opening a file handle again is necessary for tcoffee to end execution From c047a6da4a58ae5996e4f26693a41ba580264c71 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 23 Jan 2024 17:40:43 +0100 Subject: [PATCH 39/66] tcoffee/align: make tcoffee write to stdout, avoid using fifo --- modules/nf-core/tcoffee/align/main.nf | 13 +++++-------- .../nf-core/tcoffee/align/tests/structure.config | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/modules/nf-core/tcoffee/align/main.nf b/modules/nf-core/tcoffee/align/main.nf index 5ef6f79b2fb..c9e072f0ef5 100644 --- a/modules/nf-core/tcoffee/align/main.nf +++ b/modules/nf-core/tcoffee/align/main.nf @@ -25,21 +25,18 @@ process TCOFFEE_ALIGN { def prefix = task.ext.prefix ?: "${meta.id}" def tree_args = tree ? "-usetree $tree" : "" def template_args = template ? "-template_file $template" : "" - def write_output = compress ? " | pigz -cp ${task.cpus} > ${prefix}.aln.gz" : "> ${prefix}.aln" + def write_output = compress ? " >(pigz -cp ${task.cpus} > ${prefix}.aln.gz)" : "> ${prefix}.aln" + // using >() is necessary to preserve the tcoffee return value, + // so nextflow knows to display an error when it failed """ export TEMP='./' - mkfifo out_pipe t_coffee -seq ${fasta} \ $tree_args \ $template_args \ $args \ -thread ${task.cpus} \ - -outfile out_pipe & - # enable to suppress default html output that is not passed on by the module - #-output fasta_aln - cat out_pipe ${write_output} - cat out_pipe > /dev/null # opening a file handle again is necessary for tcoffee to end execution - + -outfile stdout \ + $write_output cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/tcoffee/align/tests/structure.config b/modules/nf-core/tcoffee/align/tests/structure.config index 4ae9d7acc1a..1cbd9c9c8e1 100644 --- a/modules/nf-core/tcoffee/align/tests/structure.config +++ b/modules/nf-core/tcoffee/align/tests/structure.config @@ -1,5 +1,5 @@ process { withName: "TCOFFEE_ALIGN" { - ext.args = { "-method TMalign_pair" } + ext.args = { "-method TMalign_pair -output fasta_aln" } } } From 6251e0aca3915be8372a5abd84b130adf57bf298 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:43:44 +0100 Subject: [PATCH 40/66] clustalo/align: add optional compression --- modules/nf-core/clustalo/align/main.nf | 21 +++++++++----- modules/nf-core/clustalo/align/meta.yml | 6 +++- .../nf-core/clustalo/align/tests/main.nf.test | 28 ++++++++++++++++++- modules/nf-core/muscle5/super5/main.nf | 3 +- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/modules/nf-core/clustalo/align/main.nf b/modules/nf-core/clustalo/align/main.nf index e2d3514f8f4..5632069deb5 100644 --- a/modules/nf-core/clustalo/align/main.nf +++ b/modules/nf-core/clustalo/align/main.nf @@ -10,10 +10,11 @@ process CLUSTALO_ALIGN { input: tuple val(meta), path(fasta) tuple val(meta2), path(tree) + val(compress) output: - tuple val(meta), path("*.aln.gz"), emit: alignment - path "versions.yml" , emit: versions + tuple val(meta), path("*.aln{.gz,}"), emit: alignment + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -21,12 +22,18 @@ process CLUSTALO_ALIGN { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + def write_output = compress ? " >(pigz -cp ${task.cpus} > ${prefix}.aln.gz)" : "> ${prefix}.aln" + // using >() is necessary to preserve the return value, + // so nextflow knows to display an error when it failed """ - clustalo \\ - -i ${fasta} \\ - --threads=${task.cpus} \\ - $args \\ - | pigz -p ${task.cpus} -c > ${prefix}.aln.gz + clustalo \ + -i ${fasta} \ + --threads=${task.cpus} \ + $args --force -o \ + $write_output + # the --force -o is necessary, as clustalo expands the commandline input, + # causing it to treat the pipe as a parameter and fail + # this way, the command expands to /dev/fd/, and --force allows writing output to an already existing file cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/clustalo/align/meta.yml b/modules/nf-core/clustalo/align/meta.yml index f0718037d55..469b3a37691 100644 --- a/modules/nf-core/clustalo/align/meta.yml +++ b/modules/nf-core/clustalo/align/meta.yml @@ -35,6 +35,9 @@ input: type: file description: Input guide tree in Newick format pattern: "*.{dnd}" + - compress: + type: boolean + description: Flag representing whether the output MSA should be compressed. Set to true to enable/false to disable compression. Compression is done using pigz, and is multithreaded. output: - meta: type: map @@ -44,7 +47,7 @@ output: - alignment: type: file description: Alignment file, in gzipped fasta format - pattern: "*.{aln.gz}" + pattern: "*.aln{.gz,}" - versions: type: file description: File containing software versions @@ -55,3 +58,4 @@ authors: maintainers: - "@luisas" - "@joseespinosa" + - "@lrauschning" diff --git a/modules/nf-core/clustalo/align/tests/main.nf.test b/modules/nf-core/clustalo/align/tests/main.nf.test index a0e415e3b54..05acb3e272f 100644 --- a/modules/nf-core/clustalo/align/tests/main.nf.test +++ b/modules/nf-core/clustalo/align/tests/main.nf.test @@ -11,7 +11,7 @@ nextflow_process { tag "clustalo/align" tag "clustalo/guidetree" - test("sarscov2 - contigs-fasta") { + test("sarscov2 - contigs-fasta - uncompressed") { when { process { @@ -20,6 +20,31 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['contigs_fasta'], checkIfExists: true) ] input[1] = [[:],[]] + input[2] = [false] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.alignment).match("alignment")}, + { assert snapshot(process.out.versions).match("versions") } + ) + } + + } + + test("sarscov2 - contigs-fasta - compressed") { + + when { + process { + """ + input[0] = [ [ id:'test' ], // meta map + file(params.test_data['sarscov2']['illumina']['contigs_fasta'], checkIfExists: true) + ] + input[1] = [[:],[]] + input[2] = [true] """ } } @@ -57,6 +82,7 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['contigs_fasta'], checkIfExists: true) ] input[1] = CLUSTALO_GUIDETREE.out.tree.collect{ meta, tree -> tree }.map{ tree -> [[ id: 'test_summary'], tree]} + input[2] = [true] """ } } diff --git a/modules/nf-core/muscle5/super5/main.nf b/modules/nf-core/muscle5/super5/main.nf index 4c18440a90a..7af8fcd8634 100644 --- a/modules/nf-core/muscle5/super5/main.nf +++ b/modules/nf-core/muscle5/super5/main.nf @@ -8,9 +8,10 @@ process MUSCLE5_SUPER5 { input: tuple val(meta), path(fasta) + val(compress) output: - tuple val(meta), path("*.aln.gz"), emit: alignment + tuple val(meta), path("*.aln{.gz,}"), emit: alignment path "versions.yml" , emit: versions when: From ac15c921a151edaef5ef51e89b23e7e8b312ba7b Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:56:05 +0100 Subject: [PATCH 41/66] muscle5/super5: add optional compression, also expand tests --- modules/nf-core/muscle5/super5/main.nf | 20 ++++++--- modules/nf-core/muscle5/super5/meta.yml | 6 ++- .../nf-core/muscle5/super5/tests/main.nf.test | 43 ++++++++++++++++++- .../muscle5/super5/tests/nextflow.config | 0 .../muscle5/super5/tests/perm_all.config | 3 ++ 5 files changed, 64 insertions(+), 8 deletions(-) delete mode 100644 modules/nf-core/muscle5/super5/tests/nextflow.config create mode 100644 modules/nf-core/muscle5/super5/tests/perm_all.config diff --git a/modules/nf-core/muscle5/super5/main.nf b/modules/nf-core/muscle5/super5/main.nf index 7af8fcd8634..e10d3e570cc 100644 --- a/modules/nf-core/muscle5/super5/main.nf +++ b/modules/nf-core/muscle5/super5/main.nf @@ -12,7 +12,7 @@ process MUSCLE5_SUPER5 { output: tuple val(meta), path("*.aln{.gz,}"), emit: alignment - path "versions.yml" , emit: versions + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -21,17 +21,25 @@ process MUSCLE5_SUPER5 { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" prefix = args.contains('-perm all') ? "${prefix}@" : "${prefix}" + // if -perm all is set, mafft will produce multiple files + // compress these individually in a loop + def compress_loop = args.contains('-perm all') ? "for f in *.aln; do pigz -p ${task.cpus} \$f; done" : "" + def write_output = (compress && !args.contains('-perm all')) ? " -output >(pigz -cp ${task.cpus} > ${prefix}.aln.gz)" : "-output ${prefix}.aln" + // muscle internally expands the shell pipe to a file descriptor of the form /dev/fd/ + // this causes it to fail, unless -output is left at the end of the call + // see also clustalo/align + // using >() is necessary to preserve the return value, + // so nextflow knows to display an error when it failed """ muscle \\ -super5 ${fasta} \\ - -output ${prefix}.aln \\ ${args} \\ - -threads ${task.cpus} + -threads ${task.cpus} \\ + $write_output + # output may be multiple files if -perm all is set - for f in *.aln; do - pigz -p ${task.cpus} \$f - done + $compress_loop cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/muscle5/super5/meta.yml b/modules/nf-core/muscle5/super5/meta.yml index 2950a500bb9..057128dc855 100644 --- a/modules/nf-core/muscle5/super5/meta.yml +++ b/modules/nf-core/muscle5/super5/meta.yml @@ -25,6 +25,9 @@ input: type: file description: Input sequences for alignment must be in FASTA format pattern: "*.{fasta,fa,fna}" + - compress: + type: boolean + description: Flag representing whether the output MSA should be compressed. Set to true to enable/false to disable compression. Compression is done using pigz, and is multithreaded. output: - meta: type: map @@ -37,7 +40,7 @@ output: pattern: "versions.yml" - alignment: type: file - description: Multiple sequence alignment produced in gzipped FASTA format + description: Multiple sequence alignment produced in gzipped FASTA format. If '-perm all' is passed in ext.args, this will be multiple files per input! pattern: "*.{aln.gz}" authors: - "@alessiovignoli" @@ -45,3 +48,4 @@ authors: maintainers: - "@alessiovignoli" - "@JoseEspinosa" + - "@lrauschning" diff --git a/modules/nf-core/muscle5/super5/tests/main.nf.test b/modules/nf-core/muscle5/super5/tests/main.nf.test index af0a08d729a..d057993969b 100644 --- a/modules/nf-core/muscle5/super5/tests/main.nf.test +++ b/modules/nf-core/muscle5/super5/tests/main.nf.test @@ -10,13 +10,14 @@ nextflow_process { tag "muscle5" tag "muscle5/super5" - test("fasta - align_sequence") { + test("fasta - align_sequence - uncompressed") { when { process { """ input[0] = [ [ id:'test' ], file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/setoxin-ref.fa", checkIfExists: true) ] + input[1] = [ false ] """ } } @@ -28,4 +29,44 @@ nextflow_process { ) } } + + test("fasta - align_sequence - compressed") { + when { + process { + """ + input[0] = [ [ id:'test' ], + file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/setoxin-ref.fa", checkIfExists: true) + ] + input[1] = [ true ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.alignment).match("alignment")}, + ) + } + } + test("fasta - align_sequence - compressed - perm_all") { + config "./perm_all.config" + when { + process { + """ + input[0] = [ [ id:'test' ], + file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/setoxin-ref.fa", checkIfExists: true) + ] + input[1] = [ true ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.alignment).match("perm-all")}, + ) + } + } } \ No newline at end of file diff --git a/modules/nf-core/muscle5/super5/tests/nextflow.config b/modules/nf-core/muscle5/super5/tests/nextflow.config deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/modules/nf-core/muscle5/super5/tests/perm_all.config b/modules/nf-core/muscle5/super5/tests/perm_all.config new file mode 100644 index 00000000000..d3502716003 --- /dev/null +++ b/modules/nf-core/muscle5/super5/tests/perm_all.config @@ -0,0 +1,3 @@ +process { + ext.args = { "-perm all" } +} From 2ee7bedbe36a7358eb244bec8047119695bae595 Mon Sep 17 00:00:00 2001 From: Leon Rauschning Date: Wed, 24 Jan 2024 17:59:55 +0100 Subject: [PATCH 42/66] update snapshot --- .../muscle5/super5/tests/main.nf.test.snap | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/modules/nf-core/muscle5/super5/tests/main.nf.test.snap b/modules/nf-core/muscle5/super5/tests/main.nf.test.snap index a810255062b..275e1cf5621 100644 --- a/modules/nf-core/muscle5/super5/tests/main.nf.test.snap +++ b/modules/nf-core/muscle5/super5/tests/main.nf.test.snap @@ -1,4 +1,22 @@ { + "perm-all": { + "content": [ + [ + [ + { + "id": "test" + }, + [ + "testabc.0.aln.gz:md5,46ba556df08f7aabbe5e1ba31d226b6a", + "testacb.0.aln.gz:md5,46ba556df08f7aabbe5e1ba31d226b6a", + "testbca.0.aln.gz:md5,46ba556df08f7aabbe5e1ba31d226b6a", + "testnone.0.aln.gz:md5,46ba556df08f7aabbe5e1ba31d226b6a" + ] + ] + ] + ], + "timestamp": "2024-01-24T17:53:57.627560248" + }, "alignment": { "content": [ [ From 86568d9050a7ff9015de16c5f9ec69ed4e0a9ea0 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Wed, 24 Jan 2024 18:05:59 +0100 Subject: [PATCH 43/66] muscle5/super5: re-add empty config file --- modules/nf-core/muscle5/super5/tests/nextflow.config | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 modules/nf-core/muscle5/super5/tests/nextflow.config diff --git a/modules/nf-core/muscle5/super5/tests/nextflow.config b/modules/nf-core/muscle5/super5/tests/nextflow.config new file mode 100644 index 00000000000..e69de29bb2d From b0dbaa44897530a61256a405dece2bd721a13a87 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Wed, 24 Jan 2024 18:56:39 +0100 Subject: [PATCH 44/66] mafft: implement optional output compression, handle compressed input --- modules/nf-core/mafft/main.nf | 19 +++++++----- modules/nf-core/mafft/meta.yml | 31 ++++++++++--------- modules/nf-core/mafft/tests/main.nf.test | 38 ++++++++++++++++++++++-- 3 files changed, 64 insertions(+), 24 deletions(-) diff --git a/modules/nf-core/mafft/main.nf b/modules/nf-core/mafft/main.nf index 725c2fce64e..506bfd10a45 100644 --- a/modules/nf-core/mafft/main.nf +++ b/modules/nf-core/mafft/main.nf @@ -14,10 +14,11 @@ process MAFFT { tuple val(meta4), path(addfull) tuple val(meta5), path(addprofile) tuple val(meta6), path(addlong) + val(compress) output: - tuple val(meta), path("*.fas.gz"), emit: fas - path "versions.yml" , emit: versions + tuple val(meta), path("*.fas{.gz,}"), emit: fas + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -25,11 +26,13 @@ process MAFFT { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def add = add ? "--add ${add}" : '' - def addfragments = addfragments ? "--addfragments ${addfragments}" : '' - def addfull = addfull ? "--addfull ${addfull}" : '' - def addprofile = addprofile ? "--addprofile ${addprofile}" : '' - def addlong = addlong ? "--addlong ${addlong}" : '' + def add = add ? "--add <(unpigz -cdf ${add})" : '' + def addfragments = addfragments ? "--addfragments <(unpigz -cdf ${addfragments})" : '' + def addfull = addfull ? "--addfull <(unpigz -cdf ${addfull})" : '' + def addprofile = addprofile ? "--addprofile <(unpigz -cdf ${addprofile})" : '' + def addlong = addlong ? "--addlong <(unpigz -cdf ${addlong})" : '' + def write_output = compress ? " | pigz -cp ${task.cpus} > ${prefix}.fas.gz" : "> ${prefix}.fas" + // this will not preserve MAFFTs return value, but mafft crashes when it receives a process substitution if ("$fasta" == "${prefix}.fas" ) error "Input and output names are the same, set prefix in module configuration to disambiguate!" """ mafft \\ @@ -41,7 +44,7 @@ process MAFFT { ${addlong} \\ ${args} \\ ${fasta} \\ - | pigz -p ${task.cpus} -c > ${prefix}.fas.gz + ${write_output} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/mafft/meta.yml b/modules/nf-core/mafft/meta.yml index fad68669361..90b9ed39c32 100644 --- a/modules/nf-core/mafft/meta.yml +++ b/modules/nf-core/mafft/meta.yml @@ -24,8 +24,8 @@ input: e.g. [ id:'test', single_end:false ] - fasta: type: file - description: FASTA file containing the sequences to align - pattern: "*.{fa,fasta}" + description: FASTA file containing the sequences to align. May be gzipped or uncompressed. + pattern: "*.{fa,fasta}{.gz,}" - meta2: type: map description: | @@ -33,8 +33,8 @@ input: e.g. [ id:'test', single_end:false ] - add: type: file - description: FASTA file containing sequences to align to the sequences in `fasta` using `--add` - pattern: "*.{fa,fasta}" + description: FASTA file containing sequences to align to the sequences in `fasta` using `--add`. May be gzipped or uncompressed. + pattern: "*.{fa,fasta}{.gz,}" - meta3: type: map description: | @@ -42,8 +42,8 @@ input: e.g. [ id:'test', single_end:false ] - addfragments: type: file - description: FASTA file containing sequences to align to the sequences in `fasta` using `--addfragments` - pattern: "*.{fa,fasta}" + description: FASTA file containing sequences to align to the sequences in `fasta` using `--addfragments`. May be gzipped or uncompressed. + pattern: "*.{fa,fasta}{.gz,}" - meta4: type: map description: | @@ -51,8 +51,8 @@ input: e.g. [ id:'test', single_end:false ] - addfull: type: file - description: FASTA file containing sequences to align to the sequences in `fasta` using `--addfull` - pattern: "*.{fa,fasta}" + description: FASTA file containing sequences to align to the sequences in `fasta` using `--addfull`. May be gzipped or uncompressed. + pattern: "*.{fa,fasta}{.gz,}" - meta5: type: map description: | @@ -60,8 +60,8 @@ input: e.g. [ id:'test', single_end:false ] - addprofile: type: file - description: FASTA file containing sequences to align to the sequences in `fasta` using `--addprofile` - pattern: "*.{fa,fasta}" + description: FASTA file containing sequences to align to the sequences in `fasta` using `--addprofile`. May be gzipped or uncompressed. + pattern: "*.{fa,fasta}{.gz,}" - meta6: type: map description: | @@ -69,8 +69,11 @@ input: e.g. [ id:'test', single_end:false ] - addlong: type: file - description: FASTA file containing sequences to align to the sequences in `fasta` using `--addlong` - pattern: "*.{fa,fasta}" + description: FASTA file containing sequences to align to the sequences in `fasta` using `--addlong`. May be gzipped or uncompressed. + pattern: "*.{fa,fasta}{.gz,}" + - compress: + type: boolean + description: Flag representing whether the output MSA should be compressed. Set to true to enable/false to disable compression. Compression is done using pigz, and is multithreaded. output: - meta: type: map @@ -83,8 +86,8 @@ output: pattern: "versions.yml" - fas: type: file - description: Aligned sequences in gzipped FASTA format - pattern: "*.{fas.gz}" + description: Aligned sequences in FASTA format. May be gzipped or uncompressed. + pattern: "*.fas{.gz,}" authors: - "@MillironX" maintainers: diff --git a/modules/nf-core/mafft/tests/main.nf.test b/modules/nf-core/mafft/tests/main.nf.test index f622724e20e..957e867824e 100644 --- a/modules/nf-core/mafft/tests/main.nf.test +++ b/modules/nf-core/mafft/tests/main.nf.test @@ -7,7 +7,7 @@ nextflow_process { tag "modules_nfcore" tag "mafft" - test("SARS-CoV-2 scaffolds fasta") { + test("SARS-CoV-2 scaffolds fasta - uncompressed") { when { process { """ @@ -20,6 +20,34 @@ nextflow_process { input[3] = [[:], []] input[4] = [[:], []] input[5] = [[:], []] + input[6] = [ false ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta")} + ) + } + + } + + test("SARS-CoV-2 scaffolds fasta - compressed") { + when { + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['scaffolds_fasta'], checkIfExists: true) + ] + input[1] = [[:], []] + input[2] = [[:], []] + input[3] = [[:], []] + input[4] = [[:], []] + input[5] = [[:], []] + input[6] = [ true ] """ } } @@ -49,6 +77,7 @@ nextflow_process { input[3] = [[:], []] input[4] = [[:], []] input[5] = [[:], []] + input[6] = [ true ] """ } } @@ -78,6 +107,7 @@ nextflow_process { input[3] = [[:], []] input[4] = [[:], []] input[5] = [[:], []] + input[6] = [ true ] """ } } @@ -106,6 +136,7 @@ nextflow_process { ] input[4] = [[:], []] input[5] = [[:], []] + input[6] = [ true ] """ } } @@ -135,6 +166,7 @@ nextflow_process { file(params.test_data['sarscov2']['genome']['informative_sites_fas'], checkIfExists: true) ] input[5] = [[:], []] + input[6] = [ true ] """ } } @@ -165,6 +197,7 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.test_data['sarscov2']['genome']['informative_sites_fas'], checkIfExists: true) ] + input[6] = [ true ] """ } } @@ -198,6 +231,7 @@ nextflow_process { input[3] = [[:], []] input[4] = [[:], []] input[5] = [[:], []] + input[6] = [ true ] """ } } @@ -211,4 +245,4 @@ nextflow_process { } -} +} \ No newline at end of file From 222b02929769dab1a0dbe0af35217dea8266f02d Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Wed, 24 Jan 2024 19:31:09 +0100 Subject: [PATCH 45/66] muscle5/super5: better parallelization for compressed -perm all --- modules/nf-core/muscle5/super5/main.nf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/muscle5/super5/main.nf b/modules/nf-core/muscle5/super5/main.nf index e10d3e570cc..faf56864ec6 100644 --- a/modules/nf-core/muscle5/super5/main.nf +++ b/modules/nf-core/muscle5/super5/main.nf @@ -21,9 +21,6 @@ process MUSCLE5_SUPER5 { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" prefix = args.contains('-perm all') ? "${prefix}@" : "${prefix}" - // if -perm all is set, mafft will produce multiple files - // compress these individually in a loop - def compress_loop = args.contains('-perm all') ? "for f in *.aln; do pigz -p ${task.cpus} \$f; done" : "" def write_output = (compress && !args.contains('-perm all')) ? " -output >(pigz -cp ${task.cpus} > ${prefix}.aln.gz)" : "-output ${prefix}.aln" // muscle internally expands the shell pipe to a file descriptor of the form /dev/fd/ // this causes it to fail, unless -output is left at the end of the call @@ -39,7 +36,10 @@ process MUSCLE5_SUPER5 { # output may be multiple files if -perm all is set - $compress_loop + # compress these individually if set to compress output + if ${args.contains('-perm all') && compress}; then + pigz -p ${task.cpus} *.aln + fi cat <<-END_VERSIONS > versions.yml "${task.process}": From 81ced391b24422d95778773cf72d46c7dfcc3225 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:57:15 +0100 Subject: [PATCH 46/66] mtmalign/align: implement optional compression --- modules/nf-core/mtmalign/align/main.nf | 20 +++++++-- modules/nf-core/mtmalign/align/meta.yml | 13 +++--- .../nf-core/mtmalign/align/tests/main.nf.test | 43 ++++++++++++++++++- 3 files changed, 66 insertions(+), 10 deletions(-) diff --git a/modules/nf-core/mtmalign/align/main.nf b/modules/nf-core/mtmalign/align/main.nf index 51c83beea4b..79bf6b65e85 100644 --- a/modules/nf-core/mtmalign/align/main.nf +++ b/modules/nf-core/mtmalign/align/main.nf @@ -11,11 +11,12 @@ process MTMALIGN_ALIGN { input: tuple val(meta), path('*.pdb', arity: '2..*') + val(compress) output: - tuple val(meta), path("./mTM_result/${prefix}.aln.gz") , emit: alignment - tuple val(meta), path("./mTM_result/${prefix}.pdb.gz") , emit: structure - path "versions.yml" , emit: versions + tuple val(meta), path("./mTM_result/${prefix}.aln{.gz,}"), emit: alignment + tuple val(meta), path("./mTM_result/${prefix}.pdb{.gz,}"), emit: structure + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -23,14 +24,25 @@ process MTMALIGN_ALIGN { script: def args = task.ext.args ?: '' prefix = task.ext.prefix ?: "${meta.id}" + // mTMalign is not capable of writing to stdout + // if -o /dev/stdout is specified, the output file will be polluted with debug messages emitted by mTMalign """ + # decompress input files if required + if ls ./*.pdb.gz 2&> /dev/null; then # check if any files are compressed; calling unpigz with an empty arg will cause it to panic + unpigz -d ./*.pdb.gz + fi + + # construct input file for mtmalign ls *.pdb | sed s/\\ /\\n/ > input_list.txt + mtm-align -i input_list.txt -o ${prefix}.pdb # -o does not affect the fasta naming, so move it to the new name mv ./mTM_result/result.fasta ./mTM_result/${prefix}.aln # compress both output files - pigz -p ${task.cpus} ./mTM_result/${prefix}.aln ./mTM_result/${prefix}.pdb + if ${compress}; then + pigz -p ${task.cpus} ./mTM_result/${prefix}.aln ./mTM_result/${prefix}.pdb + fi # mtm-align -v prints the wrong version 20180725, so extract it from the cosmetic output in the help message cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/mtmalign/align/meta.yml b/modules/nf-core/mtmalign/align/meta.yml index fad55afd790..1e444e1c6ba 100644 --- a/modules/nf-core/mtmalign/align/meta.yml +++ b/modules/nf-core/mtmalign/align/meta.yml @@ -27,9 +27,12 @@ input: e.g. `[ id:'test']` - pdbs: type: file - description: Input protein structures in PDB format. + description: Input protein structures in PDB format. Files may be gzipped or uncompressed. They should contain exactly one chain! pattern: "*.{pdb}" + - compress: + type: boolean + description: Flag representing whether the output MSA should be compressed. Set to true to enable/false to disable compression. Compression is done using pigz, and is multithreaded. output: - meta: type: map @@ -38,12 +41,12 @@ output: e.g. `[ id:'test']` - alignment: type: file - description: Alignment in gzipped FASTA format. - pattern: "*.aln.gz" + description: Alignment in FASTA format. May be gzipped or uncompressed. + pattern: "*.aln{.gz,}" - structure: type: file - description: Overlaid structures in PDB format. - pattern: "${prefix}.pdb.gz" + description: Overlaid structures in PDB format. May be gzipped or uncompressed. + pattern: "${prefix}.pdb{.gz,}" - versions: type: file description: File containing software versions diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test b/modules/nf-core/mtmalign/align/tests/main.nf.test index 120a32ac81b..de4b71a2b81 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test @@ -9,7 +9,7 @@ nextflow_process { tag "mtmalign/align" tag "untar" - test("Test on seatoxin dataset") { + test("Test on seatoxin dataset - uncompressed") { setup { run("UNTAR") { @@ -29,6 +29,47 @@ nextflow_process { process { """ input[0] = UNTAR.out.untar.map { meta,dir -> [[ id:'test' ], file(dir).listFiles().collect()]} + input[1] = Channel.of(false) + """ + } + } + + then { + assertAll( + { assert process.success }, + // mTMalign may be nondeterministic, just check if the pdbs are all in there + //{ assert snapshot(process.out).match() } + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">2.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">3.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">4.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">5.pdb") }, + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("Test on seatoxin dataset - compressed") { + setup { + + run("UNTAR") { + script "../../../../../modules/nf-core/untar/main.nf" + process { + """ + archive = file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/structures/seatoxin-ref.tar.gz", checkIfExists: true) + input[0] = Channel.of(tuple([id:'test'], archive)) + """ + } + } + } + + when { + params { + } + process { + """ + input[0] = UNTAR.out.untar.map { meta,dir -> [[ id:'test' ], file(dir).listFiles().collect()]} + input[1] = true """ } } From 3e82387b84b668d2aa87b14b23ac459a6cab1eca Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Thu, 25 Jan 2024 17:07:09 +0100 Subject: [PATCH 47/66] mtmalign/align: add pigz to versions.yml --- modules/nf-core/mtmalign/align/main.nf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/nf-core/mtmalign/align/main.nf b/modules/nf-core/mtmalign/align/main.nf index 79bf6b65e85..0592474844a 100644 --- a/modules/nf-core/mtmalign/align/main.nf +++ b/modules/nf-core/mtmalign/align/main.nf @@ -48,6 +48,7 @@ process MTMALIGN_ALIGN { cat <<-END_VERSIONS > versions.yml "${task.process}": mTM-align: \$( mtm-align -h | grep -e "\\(Version [[:digit:]]*\\)" | grep -oe "[[:digit:]]*" ) + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ @@ -63,6 +64,7 @@ process MTMALIGN_ALIGN { cat <<-END_VERSIONS > versions.yml "${task.process}": mTM-align: \$( mtm-align -h | grep -e "\\(Version [[:digit:]]*\\)" | grep -oe "[[:digit:]]*" ) + pigz: \$(echo \$(pigz --version 2>&1) | sed 's/^.*pigz\\w*//' )) END_VERSIONS """ } From 2df6c38d4e97d15f9eea5fb4bf80270369f1983e Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Thu, 25 Jan 2024 18:16:14 +0100 Subject: [PATCH 48/66] mtmalign/align: fix --- modules/nf-core/mtmalign/align/main.nf | 9 +++++---- modules/nf-core/mtmalign/align/tests/main.nf.test | 12 ++++++------ modules/nf-core/tcoffee/align/main.nf | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/modules/nf-core/mtmalign/align/main.nf b/modules/nf-core/mtmalign/align/main.nf index 0592474844a..83420ed07e9 100644 --- a/modules/nf-core/mtmalign/align/main.nf +++ b/modules/nf-core/mtmalign/align/main.nf @@ -14,8 +14,8 @@ process MTMALIGN_ALIGN { val(compress) output: - tuple val(meta), path("./mTM_result/${prefix}.aln{.gz,}"), emit: alignment - tuple val(meta), path("./mTM_result/${prefix}.pdb{.gz,}"), emit: structure + tuple val(meta), path("./mTM_result/${prefix}.aln${compress ? '.gz' : ''}"), emit: alignment + tuple val(meta), path("./mTM_result/${prefix}.pdb${compress ? '.gz' : ''}"), emit: structure path "versions.yml" , emit: versions when: @@ -43,6 +43,7 @@ process MTMALIGN_ALIGN { if ${compress}; then pigz -p ${task.cpus} ./mTM_result/${prefix}.aln ./mTM_result/${prefix}.pdb fi + tree # mtm-align -v prints the wrong version 20180725, so extract it from the cosmetic output in the help message cat <<-END_VERSIONS > versions.yml @@ -57,8 +58,8 @@ process MTMALIGN_ALIGN { prefix = task.ext.prefix ?: "${meta.id}" """ mkdir mTM_result - touch mTM_result/${prefix}.aln.gz - touch mTM_result/${prefix}.pdb.gz + touch mTM_result/${prefix}.aln${compress ? '.gz' : ''} + touch mTM_result/${prefix}.pdb${compress ? '.gz' : ''} # mtm-align -v prints the wrong version 20180725, so extract it from the cosmetic output in the help message cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test b/modules/nf-core/mtmalign/align/tests/main.nf.test index de4b71a2b81..a0fa3a33066 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test @@ -29,7 +29,7 @@ nextflow_process { process { """ input[0] = UNTAR.out.untar.map { meta,dir -> [[ id:'test' ], file(dir).listFiles().collect()]} - input[1] = Channel.of(false) + input[1] = false """ } } @@ -39,11 +39,11 @@ nextflow_process { { assert process.success }, // mTMalign may be nondeterministic, just check if the pdbs are all in there //{ assert snapshot(process.out).match() } - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1.pdb") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">2.pdb") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">3.pdb") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">4.pdb") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">5.pdb") }, + { assert path(process.out.alignment[0][1]).getText().contains(">1.pdb") }, + { assert path(process.out.alignment[0][1]).getText().contains(">2.pdb") }, + { assert path(process.out.alignment[0][1]).getText().contains(">3.pdb") }, + { assert path(process.out.alignment[0][1]).getText().contains(">4.pdb") }, + { assert path(process.out.alignment[0][1]).getText().contains(">5.pdb") }, { assert snapshot(process.out.versions).match("versions") } ) } diff --git a/modules/nf-core/tcoffee/align/main.nf b/modules/nf-core/tcoffee/align/main.nf index c9e072f0ef5..3e073cdd5ac 100644 --- a/modules/nf-core/tcoffee/align/main.nf +++ b/modules/nf-core/tcoffee/align/main.nf @@ -47,7 +47,7 @@ process TCOFFEE_ALIGN { stub: def prefix = task.ext.prefix ?: "${meta.id}" - def pigz_call = compressed ? ".gz": "" + def pigz_call = compress ? ".gz": "" """ touch ${prefix}.aln${pigz_call} From 6c141abfe054d72789d34942c01d69118153429b Mon Sep 17 00:00:00 2001 From: Leon Rauschning Date: Thu, 25 Jan 2024 18:24:02 +0100 Subject: [PATCH 49/66] regenerate snapshot --- .../mtmalign/align/tests/main.nf.test.snap | 49 +------------------ 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test.snap b/modules/nf-core/mtmalign/align/tests/main.nf.test.snap index b4b9ee6888c..eb321457474 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test.snap +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test.snap @@ -2,54 +2,9 @@ "versions": { "content": [ [ - "versions.yml:md5,4b618e4e98abf72feabb3778d3c39105" + "versions.yml:md5,7cbacec15bb9e0c8cbb27610bde74c10" ] ], - "timestamp": "2024-01-16T12:20:11.211768299" - }, - "Test on seatoxin dataset": { - "content": [ - { - "0": [ - [ - { - "id": "test" - }, - "test.aln.gz:md5,8400950485c61b741f06c9b936bd2863" - ] - ], - "1": [ - [ - { - "id": "test" - }, - "test.pdb.gz:md5,53aeaeae0bd1c0c2c887c4215ec44f5e" - ] - ], - "2": [ - "versions.yml:md5,4b618e4e98abf72feabb3778d3c39105" - ], - "alignment": [ - [ - { - "id": "test" - }, - "test.aln.gz:md5,8400950485c61b741f06c9b936bd2863" - ] - ], - "structure": [ - [ - { - "id": "test" - }, - "test.pdb.gz:md5,53aeaeae0bd1c0c2c887c4215ec44f5e" - ] - ], - "versions": [ - "versions.yml:md5,4b618e4e98abf72feabb3778d3c39105" - ] - } - ], - "timestamp": "2024-01-17T18:55:19.617543477" + "timestamp": "2024-01-25T18:21:22.385207003" } } \ No newline at end of file From 6338cf26169bea6fb3ee126dd99891beb3c01c84 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Thu, 25 Jan 2024 19:46:09 +0100 Subject: [PATCH 50/66] famsa/align: implement optional compression --- modules/nf-core/famsa/align/main.nf | 11 +++++--- modules/nf-core/famsa/align/meta.yml | 7 +++-- .../nf-core/famsa/align/tests/main.nf.test | 28 ++++++++++++++++++- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/modules/nf-core/famsa/align/main.nf b/modules/nf-core/famsa/align/main.nf index 9e1abc17413..5789c23c217 100644 --- a/modules/nf-core/famsa/align/main.nf +++ b/modules/nf-core/famsa/align/main.nf @@ -12,24 +12,27 @@ process FAMSA_ALIGN { input: tuple val(meta), path(fasta) tuple val(meta2), path(tree) + val(compress) output: - tuple val(meta), path("*.aln.gz"), emit: alignment - path "versions.yml" , emit: versions + tuple val(meta), path("*.aln{.gz,}"), emit: alignment + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when script: - def args = task.ext.args ?: '-gz' + def args = task.ext.args ?: '' + def compress_args = compress ? '-gz' : '' def prefix = task.ext.prefix ?: "${meta.id}" def options_tree = tree ? "-gt import $tree" : "" """ famsa $options_tree \\ + $compress_args \\ $args \\ -t ${task.cpus} \\ ${fasta} \\ - ${prefix}.aln.gz + ${prefix}.aln${compress ? '.gz':''} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/famsa/align/meta.yml b/modules/nf-core/famsa/align/meta.yml index c853a7cf839..6acf3c21010 100644 --- a/modules/nf-core/famsa/align/meta.yml +++ b/modules/nf-core/famsa/align/meta.yml @@ -33,6 +33,9 @@ input: type: file description: Input guide tree in Newick format pattern: "*.{dnd}" + - compress: + type: boolean + description: Flag representing whether the output MSA should be compressed. Set to true to enable/false to disable compression. Compression is handled by passing '-gz' to FAMSA along with any other options specified in task.ext.args. output: - meta: type: map @@ -41,8 +44,8 @@ output: e.g. `[ id:'test']` - alignment: type: file - description: Alignment file, in gzipped FASTA format - pattern: "*.{aln.z}" + description: Alignment file, in FASTA format. May be gzipped or uncompressed, depending on if compress is set to true or false + pattern: "*.aln{.gz,}" - versions: type: file description: File containing software versions diff --git a/modules/nf-core/famsa/align/tests/main.nf.test b/modules/nf-core/famsa/align/tests/main.nf.test index adf31b7ebc3..0f0d3742f85 100644 --- a/modules/nf-core/famsa/align/tests/main.nf.test +++ b/modules/nf-core/famsa/align/tests/main.nf.test @@ -10,7 +10,7 @@ nextflow_process { tag "famsa/align" tag "famsa/guidetree" - test("sarscov2 - fasta") { + test("sarscov2 - fasta - uncompressed") { when { process { @@ -19,6 +19,31 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['contigs_fasta'], checkIfExists: true) ] input[1] = [[:],[]] + input[2] = false + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.alignment).match("alignment")}, + { assert snapshot(process.out.versions).match("versions") } + ) + } + + } + + test("sarscov2 - fasta - compressed") { + + when { + process { + """ + input[0] = [ [ id:'test' ], // meta map + file(params.test_data['sarscov2']['illumina']['contigs_fasta'], checkIfExists: true) + ] + input[1] = [[:],[]] + input[2] = true """ } } @@ -55,6 +80,7 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['contigs_fasta'], checkIfExists: true) ] input[1] = FAMSA_GUIDETREE.out.tree.collect{ meta, tree -> tree }.map{ tree -> [[ id: 'test_summary'], tree]} + input[2] = true """ } } From 984ee517df2badfd3a7f15c11077d0f010289573 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Thu, 25 Jan 2024 19:50:47 +0100 Subject: [PATCH 51/66] whoops, fix tests --- .../nf-core/clustalo/align/tests/main.nf.test | 6 +++--- modules/nf-core/mafft/tests/main.nf.test | 16 ++++++++-------- .../nf-core/muscle5/super5/tests/main.nf.test | 6 +++--- modules/nf-core/tcoffee/align/tests/main.nf.test | 8 ++++---- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/modules/nf-core/clustalo/align/tests/main.nf.test b/modules/nf-core/clustalo/align/tests/main.nf.test index 05acb3e272f..c43e2f7dc3e 100644 --- a/modules/nf-core/clustalo/align/tests/main.nf.test +++ b/modules/nf-core/clustalo/align/tests/main.nf.test @@ -20,7 +20,7 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['contigs_fasta'], checkIfExists: true) ] input[1] = [[:],[]] - input[2] = [false] + input[2] = false """ } } @@ -44,7 +44,7 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['contigs_fasta'], checkIfExists: true) ] input[1] = [[:],[]] - input[2] = [true] + input[2] = true """ } } @@ -82,7 +82,7 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['contigs_fasta'], checkIfExists: true) ] input[1] = CLUSTALO_GUIDETREE.out.tree.collect{ meta, tree -> tree }.map{ tree -> [[ id: 'test_summary'], tree]} - input[2] = [true] + input[2] = true """ } } diff --git a/modules/nf-core/mafft/tests/main.nf.test b/modules/nf-core/mafft/tests/main.nf.test index 957e867824e..b2e0ff45db0 100644 --- a/modules/nf-core/mafft/tests/main.nf.test +++ b/modules/nf-core/mafft/tests/main.nf.test @@ -20,7 +20,7 @@ nextflow_process { input[3] = [[:], []] input[4] = [[:], []] input[5] = [[:], []] - input[6] = [ false ] + input[6] = false """ } } @@ -47,7 +47,7 @@ nextflow_process { input[3] = [[:], []] input[4] = [[:], []] input[5] = [[:], []] - input[6] = [ true ] + input[6] = true """ } } @@ -77,7 +77,7 @@ nextflow_process { input[3] = [[:], []] input[4] = [[:], []] input[5] = [[:], []] - input[6] = [ true ] + input[6] = true """ } } @@ -107,7 +107,7 @@ nextflow_process { input[3] = [[:], []] input[4] = [[:], []] input[5] = [[:], []] - input[6] = [ true ] + input[6] = true """ } } @@ -136,7 +136,7 @@ nextflow_process { ] input[4] = [[:], []] input[5] = [[:], []] - input[6] = [ true ] + input[6] = true """ } } @@ -166,7 +166,7 @@ nextflow_process { file(params.test_data['sarscov2']['genome']['informative_sites_fas'], checkIfExists: true) ] input[5] = [[:], []] - input[6] = [ true ] + input[6] = true """ } } @@ -197,7 +197,7 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.test_data['sarscov2']['genome']['informative_sites_fas'], checkIfExists: true) ] - input[6] = [ true ] + input[6] = true """ } } @@ -231,7 +231,7 @@ nextflow_process { input[3] = [[:], []] input[4] = [[:], []] input[5] = [[:], []] - input[6] = [ true ] + input[6] = true """ } } diff --git a/modules/nf-core/muscle5/super5/tests/main.nf.test b/modules/nf-core/muscle5/super5/tests/main.nf.test index d057993969b..a701c4912ca 100644 --- a/modules/nf-core/muscle5/super5/tests/main.nf.test +++ b/modules/nf-core/muscle5/super5/tests/main.nf.test @@ -17,7 +17,7 @@ nextflow_process { input[0] = [ [ id:'test' ], file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/setoxin-ref.fa", checkIfExists: true) ] - input[1] = [ false ] + input[1] = false """ } } @@ -37,7 +37,7 @@ nextflow_process { input[0] = [ [ id:'test' ], file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/setoxin-ref.fa", checkIfExists: true) ] - input[1] = [ true ] + input[1] = true """ } } @@ -57,7 +57,7 @@ nextflow_process { input[0] = [ [ id:'test' ], file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/setoxin-ref.fa", checkIfExists: true) ] - input[1] = [ true ] + input[1] = true """ } } diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test b/modules/nf-core/tcoffee/align/tests/main.nf.test index 0d28c2f3304..b744897639c 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test @@ -23,7 +23,7 @@ nextflow_process { ] input[1] = [[:],[]] input[2] = [[:],[],[]] - input[3] = [true] + input[3] = true """ } } @@ -49,7 +49,7 @@ nextflow_process { ] input[1] = [[:],[]] input[2] = [[:],[],[]] - input[3] = [false] + input[3] = false """ } } @@ -90,7 +90,7 @@ nextflow_process { ] input[1] = FAMSA_GUIDETREE.out.tree.collect{ meta, tree -> tree }.map{ tree -> [[ id: 'test'], tree]} input[2] = [ [:], [], [] ] - input[3] = [true] + input[3] = true """ } } @@ -132,7 +132,7 @@ nextflow_process { ] input[1] = [ [:], [] ] input[2] = UNTAR.out.untar.map { meta,dir -> [[ id:'test' ], [] ,file(dir).listFiles().collect()]} - input[3] = [true] + input[3] = true """ } From a5330ef5efa7018a790ecfd395c0e11d617a5705 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Thu, 25 Jan 2024 20:02:10 +0100 Subject: [PATCH 52/66] clustalo/align: fix --- modules/nf-core/clustalo/align/main.nf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/nf-core/clustalo/align/main.nf b/modules/nf-core/clustalo/align/main.nf index 5632069deb5..0c5d7dcf29a 100644 --- a/modules/nf-core/clustalo/align/main.nf +++ b/modules/nf-core/clustalo/align/main.nf @@ -22,18 +22,18 @@ process CLUSTALO_ALIGN { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def write_output = compress ? " >(pigz -cp ${task.cpus} > ${prefix}.aln.gz)" : "> ${prefix}.aln" + def write_output = compress ? "--force -o >(pigz -cp ${task.cpus} > ${prefix}.aln.gz)" : "> ${prefix}.aln" // using >() is necessary to preserve the return value, // so nextflow knows to display an error when it failed + // the --force -o is necessary, as clustalo expands the commandline input, + // causing it to treat the pipe as a parameter and fail + // this way, the command expands to /dev/fd/, and --force allows writing output to an already existing file """ clustalo \ -i ${fasta} \ --threads=${task.cpus} \ - $args --force -o \ + $args \ $write_output - # the --force -o is necessary, as clustalo expands the commandline input, - # causing it to treat the pipe as a parameter and fail - # this way, the command expands to /dev/fd/, and --force allows writing output to an already existing file cat <<-END_VERSIONS > versions.yml "${task.process}": From 1f7791bb3d6de5b8930972ce16e1228ad7405ee6 Mon Sep 17 00:00:00 2001 From: Leon Rauschning Date: Thu, 25 Jan 2024 20:10:09 +0100 Subject: [PATCH 53/66] update snapshots --- modules/nf-core/clustalo/align/tests/main.nf.test.snap | 4 ++-- modules/nf-core/famsa/align/tests/main.nf.test.snap | 2 +- modules/nf-core/mafft/tests/main.nf.test.snap | 2 +- modules/nf-core/muscle5/super5/tests/main.nf.test.snap | 2 +- modules/nf-core/tcoffee/align/tests/main.nf.test.snap | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/nf-core/clustalo/align/tests/main.nf.test.snap b/modules/nf-core/clustalo/align/tests/main.nf.test.snap index a73607c4801..52e0ec82e50 100644 --- a/modules/nf-core/clustalo/align/tests/main.nf.test.snap +++ b/modules/nf-core/clustalo/align/tests/main.nf.test.snap @@ -5,7 +5,7 @@ "versions.yml:md5,327da6a4250a6b7c4e45cddaa1f56280" ] ], - "timestamp": "2024-01-16T15:13:48.820697337" + "timestamp": "2024-01-25T19:59:54.545448144" }, "with_guide_tree_alignment": { "content": [ @@ -31,7 +31,7 @@ ] ] ], - "timestamp": "2024-01-17T18:18:36.468757251" + "timestamp": "2024-01-25T20:07:37.297437293" }, "with_guide_tree_versions": { "content": [ diff --git a/modules/nf-core/famsa/align/tests/main.nf.test.snap b/modules/nf-core/famsa/align/tests/main.nf.test.snap index a281f981fb4..7be2dbfd28c 100644 --- a/modules/nf-core/famsa/align/tests/main.nf.test.snap +++ b/modules/nf-core/famsa/align/tests/main.nf.test.snap @@ -31,7 +31,7 @@ ] ] ], - "timestamp": "2024-01-17T18:12:17.496026351" + "timestamp": "2024-01-25T20:03:31.159492883" }, "with_guide_tree_versions": { "content": [ diff --git a/modules/nf-core/mafft/tests/main.nf.test.snap b/modules/nf-core/mafft/tests/main.nf.test.snap index 941fa3c3534..6879fa61c72 100644 --- a/modules/nf-core/mafft/tests/main.nf.test.snap +++ b/modules/nf-core/mafft/tests/main.nf.test.snap @@ -28,7 +28,7 @@ ] } ], - "timestamp": "2024-01-17T19:10:44.597244231" + "timestamp": "2024-01-25T20:05:43.568947954" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta multiple": { "content": [ diff --git a/modules/nf-core/muscle5/super5/tests/main.nf.test.snap b/modules/nf-core/muscle5/super5/tests/main.nf.test.snap index 275e1cf5621..e120260de53 100644 --- a/modules/nf-core/muscle5/super5/tests/main.nf.test.snap +++ b/modules/nf-core/muscle5/super5/tests/main.nf.test.snap @@ -28,6 +28,6 @@ ] ] ], - "timestamp": "2024-01-17T18:38:46.432230995" + "timestamp": "2024-01-25T20:08:47.802983412" } } \ No newline at end of file diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test.snap b/modules/nf-core/tcoffee/align/tests/main.nf.test.snap index 0dc5c127da0..7bc86f435c7 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test.snap +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test.snap @@ -30,11 +30,11 @@ { "id": "test" }, - "test.aln.gz:md5,bd1db08ad04514cc6d1334598c1a6ef0" + "test.aln:md5,bd1db08ad04514cc6d1334598c1a6ef0" ] ] ], - "timestamp": "2024-01-19T17:56:16.987455592" + "timestamp": "2024-01-25T20:09:19.122281854" }, "alignment_guidetree": { "content": [ @@ -49,4 +49,4 @@ ], "timestamp": "2024-01-19T17:56:26.885044145" } -} +} \ No newline at end of file From 838ec1bfc1ca19ac5643210e77f26e94b44e1d97 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Thu, 25 Jan 2024 20:59:45 +0100 Subject: [PATCH 54/66] generate different snapshots for compressed & uncompressed tests, prettify code --- modules/nf-core/clustalo/align/main.nf | 2 +- modules/nf-core/clustalo/align/tests/main.nf.test | 4 ++-- modules/nf-core/famsa/align/main.nf | 2 +- modules/nf-core/famsa/align/tests/main.nf.test | 4 ++-- modules/nf-core/mafft/main.nf | 2 +- modules/nf-core/mafft/tests/main.nf.test | 4 ++-- modules/nf-core/mtmalign/align/main.nf | 2 +- modules/nf-core/mtmalign/align/tests/main.nf.test | 8 ++++---- modules/nf-core/muscle5/super5/tests/main.nf.test | 4 ++-- modules/nf-core/tcoffee/align/main.nf | 2 +- modules/nf-core/tcoffee/align/tests/main.nf.test | 2 +- 11 files changed, 18 insertions(+), 18 deletions(-) diff --git a/modules/nf-core/clustalo/align/main.nf b/modules/nf-core/clustalo/align/main.nf index 0c5d7dcf29a..da35609fc58 100644 --- a/modules/nf-core/clustalo/align/main.nf +++ b/modules/nf-core/clustalo/align/main.nf @@ -8,7 +8,7 @@ process CLUSTALO_ALIGN { 'biocontainers/mulled-v2-4cefc38542f86c17596c29b35a059de10387c6a7:adbe4fbad680f9beb083956d79128039a727e7b3-0' }" input: - tuple val(meta), path(fasta) + tuple val(meta) , path(fasta) tuple val(meta2), path(tree) val(compress) diff --git a/modules/nf-core/clustalo/align/tests/main.nf.test b/modules/nf-core/clustalo/align/tests/main.nf.test index c43e2f7dc3e..4d7c97081a3 100644 --- a/modules/nf-core/clustalo/align/tests/main.nf.test +++ b/modules/nf-core/clustalo/align/tests/main.nf.test @@ -28,7 +28,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment")}, + { assert snapshot(process.out.alignment).match("alignment - uncompressed")}, { assert snapshot(process.out.versions).match("versions") } ) } @@ -52,7 +52,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment")}, + { assert snapshot(process.out.alignment).match("alignment - compressed")}, { assert snapshot(process.out.versions).match("versions") } ) } diff --git a/modules/nf-core/famsa/align/main.nf b/modules/nf-core/famsa/align/main.nf index 5789c23c217..1f1452a65a4 100644 --- a/modules/nf-core/famsa/align/main.nf +++ b/modules/nf-core/famsa/align/main.nf @@ -10,7 +10,7 @@ process FAMSA_ALIGN { 'biocontainers/famsa:2.2.2--h9f5acd7_0' }" input: - tuple val(meta), path(fasta) + tuple val(meta) , path(fasta) tuple val(meta2), path(tree) val(compress) diff --git a/modules/nf-core/famsa/align/tests/main.nf.test b/modules/nf-core/famsa/align/tests/main.nf.test index 0f0d3742f85..2d8e1482a6d 100644 --- a/modules/nf-core/famsa/align/tests/main.nf.test +++ b/modules/nf-core/famsa/align/tests/main.nf.test @@ -27,7 +27,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment")}, + { assert snapshot(process.out.alignment).match("alignment_uncompressed")}, { assert snapshot(process.out.versions).match("versions") } ) } @@ -51,7 +51,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment")}, + { assert snapshot(process.out.alignment).match("alignment_compressed")}, { assert snapshot(process.out.versions).match("versions") } ) } diff --git a/modules/nf-core/mafft/main.nf b/modules/nf-core/mafft/main.nf index 506bfd10a45..a586ca8e11a 100644 --- a/modules/nf-core/mafft/main.nf +++ b/modules/nf-core/mafft/main.nf @@ -8,7 +8,7 @@ process MAFFT { 'biocontainers/mulled-v2-12eba4a074f913c639117640936668f5a6a01da6:425707898cf4f85051b77848be253b88f1d2298a-0' }" input: - tuple val(meta), path(fasta) + tuple val(meta) , path(fasta) tuple val(meta2), path(add) tuple val(meta3), path(addfragments) tuple val(meta4), path(addfull) diff --git a/modules/nf-core/mafft/tests/main.nf.test b/modules/nf-core/mafft/tests/main.nf.test index b2e0ff45db0..f57ab4965da 100644 --- a/modules/nf-core/mafft/tests/main.nf.test +++ b/modules/nf-core/mafft/tests/main.nf.test @@ -28,7 +28,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta")} + { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - uncompressed")} ) } @@ -55,7 +55,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta")} + { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - compressed")} ) } diff --git a/modules/nf-core/mtmalign/align/main.nf b/modules/nf-core/mtmalign/align/main.nf index 83420ed07e9..c6ad07b7c6d 100644 --- a/modules/nf-core/mtmalign/align/main.nf +++ b/modules/nf-core/mtmalign/align/main.nf @@ -16,7 +16,7 @@ process MTMALIGN_ALIGN { output: tuple val(meta), path("./mTM_result/${prefix}.aln${compress ? '.gz' : ''}"), emit: alignment tuple val(meta), path("./mTM_result/${prefix}.pdb${compress ? '.gz' : ''}"), emit: structure - path "versions.yml" , emit: versions + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test b/modules/nf-core/mtmalign/align/tests/main.nf.test index a0fa3a33066..e12ef76a6a7 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test @@ -39,8 +39,8 @@ nextflow_process { { assert process.success }, // mTMalign may be nondeterministic, just check if the pdbs are all in there //{ assert snapshot(process.out).match() } - { assert path(process.out.alignment[0][1]).getText().contains(">1.pdb") }, - { assert path(process.out.alignment[0][1]).getText().contains(">2.pdb") }, + { assert path(process.out.alignment[0][1]).getText().contains(">1.pdb") }, + { assert path(process.out.alignment[0][1]).getText().contains(">2.pdb") }, { assert path(process.out.alignment[0][1]).getText().contains(">3.pdb") }, { assert path(process.out.alignment[0][1]).getText().contains(">4.pdb") }, { assert path(process.out.alignment[0][1]).getText().contains(">5.pdb") }, @@ -79,8 +79,8 @@ nextflow_process { { assert process.success }, // mTMalign may be nondeterministic, just check if the pdbs are all in there //{ assert snapshot(process.out).match() } - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1.pdb") }, - { assert path(process.out.alignment[0][1]).getTextGzip().contains(">2.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">1.pdb") }, + { assert path(process.out.alignment[0][1]).getTextGzip().contains(">2.pdb") }, { assert path(process.out.alignment[0][1]).getTextGzip().contains(">3.pdb") }, { assert path(process.out.alignment[0][1]).getTextGzip().contains(">4.pdb") }, { assert path(process.out.alignment[0][1]).getTextGzip().contains(">5.pdb") }, diff --git a/modules/nf-core/muscle5/super5/tests/main.nf.test b/modules/nf-core/muscle5/super5/tests/main.nf.test index a701c4912ca..c1541208bcd 100644 --- a/modules/nf-core/muscle5/super5/tests/main.nf.test +++ b/modules/nf-core/muscle5/super5/tests/main.nf.test @@ -25,7 +25,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment")}, + { assert snapshot(process.out.alignment).match("alignment - uncompressed")}, ) } } @@ -45,7 +45,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment")}, + { assert snapshot(process.out.alignment).match("alignment - compressed")}, ) } } diff --git a/modules/nf-core/tcoffee/align/main.nf b/modules/nf-core/tcoffee/align/main.nf index 3e073cdd5ac..08acdef4aae 100644 --- a/modules/nf-core/tcoffee/align/main.nf +++ b/modules/nf-core/tcoffee/align/main.nf @@ -15,7 +15,7 @@ process TCOFFEE_ALIGN { output: tuple val(meta), path("*.aln{.gz,}"), emit: alignment - path "versions.yml" , emit: versions + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test b/modules/nf-core/tcoffee/align/tests/main.nf.test index b744897639c..d31d387205e 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test @@ -57,7 +57,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment")}, + { assert snapshot(process.out.alignment).match("alignment - uncompressed")}, { assert snapshot(process.out.versions).match("versions") } ) } From d2e05daccb7855d5a60fdc2e165e2fe275d289f6 Mon Sep 17 00:00:00 2001 From: Leon Rauschning Date: Thu, 25 Jan 2024 21:23:45 +0100 Subject: [PATCH 55/66] updated snapshots --- .../clustalo/align/tests/main.nf.test.snap | 23 ++++++++--- .../famsa/align/tests/main.nf.test.snap | 33 +++++++++++----- modules/nf-core/mafft/tests/main.nf.test.snap | 39 +++++++++++++++++-- .../muscle5/super5/tests/main.nf.test.snap | 19 +++++++-- .../tcoffee/align/tests/main.nf.test.snap | 17 +++++++- 5 files changed, 107 insertions(+), 24 deletions(-) diff --git a/modules/nf-core/clustalo/align/tests/main.nf.test.snap b/modules/nf-core/clustalo/align/tests/main.nf.test.snap index 52e0ec82e50..f034d96b8c4 100644 --- a/modules/nf-core/clustalo/align/tests/main.nf.test.snap +++ b/modules/nf-core/clustalo/align/tests/main.nf.test.snap @@ -1,4 +1,17 @@ { + "alignment - compressed": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.aln.gz:md5,74bb9a2820a91cf68db94dbd46787722" + ] + ] + ], + "timestamp": "2024-01-25T21:04:52.662283926" + }, "versions": { "content": [ [ @@ -7,20 +20,20 @@ ], "timestamp": "2024-01-25T19:59:54.545448144" }, - "with_guide_tree_alignment": { + "alignment - uncompressed": { "content": [ [ [ { "id": "test" }, - "test.aln.gz:md5,74bb9a2820a91cf68db94dbd46787722" + "test.aln:md5,74bb9a2820a91cf68db94dbd46787722" ] ] ], - "timestamp": "2024-01-17T18:24:51.474838687" + "timestamp": "2024-01-25T21:04:26.896932354" }, - "alignment": { + "with_guide_tree_alignment": { "content": [ [ [ @@ -31,7 +44,7 @@ ] ] ], - "timestamp": "2024-01-25T20:07:37.297437293" + "timestamp": "2024-01-17T18:24:51.474838687" }, "with_guide_tree_versions": { "content": [ diff --git a/modules/nf-core/famsa/align/tests/main.nf.test.snap b/modules/nf-core/famsa/align/tests/main.nf.test.snap index 7be2dbfd28c..2fd253843cd 100644 --- a/modules/nf-core/famsa/align/tests/main.nf.test.snap +++ b/modules/nf-core/famsa/align/tests/main.nf.test.snap @@ -1,4 +1,17 @@ { + "alignment_uncompressed": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.aln:md5,7cf7375f2ba360814ea978731838b972" + ] + ] + ], + "timestamp": "2024-01-25T21:00:36.525346869" + }, "versions": { "content": [ [ @@ -20,7 +33,15 @@ ], "timestamp": "2024-01-17T18:22:38.944328729" }, - "alignment": { + "with_guide_tree_versions": { + "content": [ + [ + "versions.yml:md5,7d9e0a8c263fa6d9017075fe88c9e9dc" + ] + ], + "timestamp": "2024-01-17T18:02:24.755216456" + }, + "alignment_compressed": { "content": [ [ [ @@ -31,14 +52,6 @@ ] ] ], - "timestamp": "2024-01-25T20:03:31.159492883" - }, - "with_guide_tree_versions": { - "content": [ - [ - "versions.yml:md5,7d9e0a8c263fa6d9017075fe88c9e9dc" - ] - ], - "timestamp": "2024-01-17T18:02:24.755216456" + "timestamp": "2024-01-25T21:01:03.727336388" } } \ No newline at end of file diff --git a/modules/nf-core/mafft/tests/main.nf.test.snap b/modules/nf-core/mafft/tests/main.nf.test.snap index 6879fa61c72..64911a41e07 100644 --- a/modules/nf-core/mafft/tests/main.nf.test.snap +++ b/modules/nf-core/mafft/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "SARS-CoV-2 scaffolds fasta": { + "SARS-CoV-2 scaffolds fasta - uncompressed": { "content": [ { "0": [ @@ -8,7 +8,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,23426611f4a0df532b6708f072bd445b" + "test.fas:md5,23426611f4a0df532b6708f072bd445b" ] ], "1": [ @@ -20,7 +20,7 @@ "id": "test", "single_end": false }, - "test.fas.gz:md5,23426611f4a0df532b6708f072bd445b" + "test.fas:md5,23426611f4a0df532b6708f072bd445b" ] ], "versions": [ @@ -28,7 +28,7 @@ ] } ], - "timestamp": "2024-01-25T20:05:43.568947954" + "timestamp": "2024-01-25T21:02:43.951149407" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta multiple": { "content": [ @@ -215,5 +215,36 @@ } ], "timestamp": "2024-01-17T19:20:19.23954473" + }, + "SARS-CoV-2 scaffolds fasta - compressed": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.fas.gz:md5,23426611f4a0df532b6708f072bd445b" + ] + ], + "1": [ + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" + ], + "fas": [ + [ + { + "id": "test", + "single_end": false + }, + "test.fas.gz:md5,23426611f4a0df532b6708f072bd445b" + ] + ], + "versions": [ + "versions.yml:md5,6e930f6a5acc19ff3a7849536a9fd0ee" + ] + } + ], + "timestamp": "2024-01-25T21:03:06.058030723" } } \ No newline at end of file diff --git a/modules/nf-core/muscle5/super5/tests/main.nf.test.snap b/modules/nf-core/muscle5/super5/tests/main.nf.test.snap index e120260de53..4503d229bd4 100644 --- a/modules/nf-core/muscle5/super5/tests/main.nf.test.snap +++ b/modules/nf-core/muscle5/super5/tests/main.nf.test.snap @@ -1,4 +1,17 @@ { + "alignment - compressed": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.aln.gz:md5,46ba556df08f7aabbe5e1ba31d226b6a" + ] + ] + ], + "timestamp": "2024-01-25T21:05:57.940317215" + }, "perm-all": { "content": [ [ @@ -17,17 +30,17 @@ ], "timestamp": "2024-01-24T17:53:57.627560248" }, - "alignment": { + "alignment - uncompressed": { "content": [ [ [ { "id": "test" }, - "test.aln.gz:md5,46ba556df08f7aabbe5e1ba31d226b6a" + "test.aln:md5,46ba556df08f7aabbe5e1ba31d226b6a" ] ] ], - "timestamp": "2024-01-25T20:08:47.802983412" + "timestamp": "2024-01-25T21:05:50.236788075" } } \ No newline at end of file diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test.snap b/modules/nf-core/tcoffee/align/tests/main.nf.test.snap index 7bc86f435c7..2a702cbe1f0 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test.snap +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test.snap @@ -15,6 +15,19 @@ ], "timestamp": "2024-01-19T17:56:17.01424266" }, + "alignment - uncompressed": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.aln:md5,bd1db08ad04514cc6d1334598c1a6ef0" + ] + ] + ], + "timestamp": "2024-01-25T21:06:29.387718149" + }, "versions_guidetree": { "content": [ [ @@ -30,11 +43,11 @@ { "id": "test" }, - "test.aln:md5,bd1db08ad04514cc6d1334598c1a6ef0" + "test.aln.gz:md5,bd1db08ad04514cc6d1334598c1a6ef0" ] ] ], - "timestamp": "2024-01-25T20:09:19.122281854" + "timestamp": "2024-01-25T21:06:19.529059609" }, "alignment_guidetree": { "content": [ From 7a0e78d821b8bb19d2b56d7cbf23af505f8f4725 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 26 Jan 2024 16:53:07 +0100 Subject: [PATCH 56/66] mtmalign/align: update input pattern --- modules/nf-core/mtmalign/align/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/mtmalign/align/main.nf b/modules/nf-core/mtmalign/align/main.nf index c6ad07b7c6d..2d46506b28b 100644 --- a/modules/nf-core/mtmalign/align/main.nf +++ b/modules/nf-core/mtmalign/align/main.nf @@ -10,7 +10,7 @@ process MTMALIGN_ALIGN { 'biocontainers/mulled-v2-5bcf71dc66dac33d8e003c5e78043b80f5c7f269:8f0e486d46f7ab38892c1a8f78d2894a549d03b5-0' }" input: - tuple val(meta), path('*.pdb', arity: '2..*') + tuple val(meta), path('*.pdb{.gz,}', arity: '2..*') val(compress) output: From 103736ab9f3c5f15cd91d2d720993ad93c1fc3fb Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 26 Jan 2024 16:53:50 +0100 Subject: [PATCH 57/66] tcoffee/alncompare,irmsd: implement jose's suggestion --- modules/nf-core/tcoffee/alncompare/main.nf | 6 ++++-- modules/nf-core/tcoffee/irmsd/main.nf | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/tcoffee/alncompare/main.nf b/modules/nf-core/tcoffee/alncompare/main.nf index c3f7d42ab2a..043158a75ad 100644 --- a/modules/nf-core/tcoffee/alncompare/main.nf +++ b/modules/nf-core/tcoffee/alncompare/main.nf @@ -23,12 +23,14 @@ process TCOFFEE_ALNCOMPARE { def metric_name = args.split('compare_mode ')[1].split(' ')[0] def header = meta.keySet().join(",") def values = meta.values().join(",") + def read_msa = msa.getName().endsWith(".gz") ? "<(unpigz -cdf ${msa})" : msa + def read_ref = ref_msa.getName().endsWith(".gz") ? "<(unpigz -cdf ${ref_msa})" : ref_msa """ export TEMP='./' t_coffee -other_pg aln_compare \ - -al1 <(unpigz -cdf ${ref_msa})\ - -al2 <(unpigz -cdf ${msa})\ + -al1 ${read_ref} \ + -al2 ${read_msa} \ ${args} \ | grep -v "seq1" | grep -v '*' | \ awk '{ print \$4}' ORS="\t" \ diff --git a/modules/nf-core/tcoffee/irmsd/main.nf b/modules/nf-core/tcoffee/irmsd/main.nf index 2dbf9db58c4..95e6b61efb4 100644 --- a/modules/nf-core/tcoffee/irmsd/main.nf +++ b/modules/nf-core/tcoffee/irmsd/main.nf @@ -24,9 +24,12 @@ process TCOFFEE_IRMSD { """ export TEMP='./' - unpigz -cdf $msa > msafile & + if [[ \$(basename $msa) == *.gz ]] ; then + unpigz -f $msa + fi + t_coffee -other_pg irmsd \ - msafile \ + \$(basename $msa .gz) \ $args \ -template_file $template > ${prefix}.irmsd From d870e1dc8c1bdeb576b4c2f9ce4f1be29cb22d4d Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 26 Jan 2024 16:54:28 +0100 Subject: [PATCH 58/66] tcoffee/irmsd: additional test for compressed input --- .../nf-core/tcoffee/irmsd/tests/main.nf.test | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/modules/nf-core/tcoffee/irmsd/tests/main.nf.test b/modules/nf-core/tcoffee/irmsd/tests/main.nf.test index 954bed7befe..60165788649 100644 --- a/modules/nf-core/tcoffee/irmsd/tests/main.nf.test +++ b/modules/nf-core/tcoffee/irmsd/tests/main.nf.test @@ -48,4 +48,51 @@ nextflow_process { } + test("seatoxin - compressed") { + + setup { + + run("UNTAR") { + script "../../../untar/main.nf" + process { + """ + input[0] = [ [ id:'test' ], + file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/structures/seatoxin-ref.tar.gz", checkIfExists: true) + ] + + """ + } + } + + run("PIGZ_COMPRESS") { + script "../../../pigz/compress/main.nf" + process { + """ + input[0] = [ [ id:'test' ], + file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/setoxin.ref", checkIfExists: true) + ] + + """ + } + } + } + + when { + process { + """ + input[0] = PIGZ_COMPRESS.out.archive + input[1] = UNTAR.out.untar.map { meta,dir -> [[ id:'test' ], file("https://raw.githubusercontent.com/nf-core/test-datasets/multiplesequencealign/testdata/templates/seatoxin-ref_template.txt", checkIfExists: true) ,file(dir).listFiles().collect()]} + """ + } + + } + + then { + assertAll( + { assert process.success }, + { assert path(process.out.irmsd.get(0).get(1)).getText().contains("1ahl") } + ) + } + + } } \ No newline at end of file From ddc67250eca9a7c675affe05e24692709a72b204 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 26 Jan 2024 16:57:27 +0100 Subject: [PATCH 59/66] tcoffee/irmsd: add tag required by lint --- modules/nf-core/tcoffee/irmsd/tests/main.nf.test | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/nf-core/tcoffee/irmsd/tests/main.nf.test b/modules/nf-core/tcoffee/irmsd/tests/main.nf.test index 60165788649..55a574a4616 100644 --- a/modules/nf-core/tcoffee/irmsd/tests/main.nf.test +++ b/modules/nf-core/tcoffee/irmsd/tests/main.nf.test @@ -8,6 +8,7 @@ nextflow_process { tag "tcoffee" tag "tcoffee/irmsd" tag "untar" + tag "pigz/compress" test("seatoxin") { From e8ae1615483c054dc3980e4f61486f0357b626f7 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 26 Jan 2024 17:04:55 +0100 Subject: [PATCH 60/66] Revert "mtmalign/align: update input pattern" This reverts commit 7a0e78d821b8bb19d2b56d7cbf23af505f8f4725. --- modules/nf-core/mtmalign/align/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/mtmalign/align/main.nf b/modules/nf-core/mtmalign/align/main.nf index 2d46506b28b..c6ad07b7c6d 100644 --- a/modules/nf-core/mtmalign/align/main.nf +++ b/modules/nf-core/mtmalign/align/main.nf @@ -10,7 +10,7 @@ process MTMALIGN_ALIGN { 'biocontainers/mulled-v2-5bcf71dc66dac33d8e003c5e78043b80f5c7f269:8f0e486d46f7ab38892c1a8f78d2894a549d03b5-0' }" input: - tuple val(meta), path('*.pdb{.gz,}', arity: '2..*') + tuple val(meta), path('*.pdb', arity: '2..*') val(compress) output: From 8184055b2a0c38244ebecd51aca0f4db88342139 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Fri, 9 Feb 2024 18:03:07 +0100 Subject: [PATCH 61/66] incorporate adams suggestion, fix stub filename extensions --- modules/nf-core/clustalo/align/main.nf | 2 +- modules/nf-core/famsa/align/main.nf | 2 +- modules/nf-core/mafft/main.nf | 2 +- modules/nf-core/muscle5/super5/main.nf | 2 +- modules/nf-core/tcoffee/align/main.nf | 3 +-- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/modules/nf-core/clustalo/align/main.nf b/modules/nf-core/clustalo/align/main.nf index da35609fc58..eb230cad269 100644 --- a/modules/nf-core/clustalo/align/main.nf +++ b/modules/nf-core/clustalo/align/main.nf @@ -46,7 +46,7 @@ process CLUSTALO_ALIGN { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix}.aln.gz + touch ${prefix}.aln${compress ? '.gz' : ''} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/famsa/align/main.nf b/modules/nf-core/famsa/align/main.nf index 1f1452a65a4..096d8ff3890 100644 --- a/modules/nf-core/famsa/align/main.nf +++ b/modules/nf-core/famsa/align/main.nf @@ -43,7 +43,7 @@ process FAMSA_ALIGN { stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix}.aln.gz + touch ${prefix}.aln${compress ? '.gz' : ''} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/mafft/main.nf b/modules/nf-core/mafft/main.nf index a586ca8e11a..f09a0c96246 100644 --- a/modules/nf-core/mafft/main.nf +++ b/modules/nf-core/mafft/main.nf @@ -63,7 +63,7 @@ process MAFFT { def addlong = addlong ? "--addlong ${addlong}" : '' if ("$fasta" == "${prefix}.fas" ) error "Input and output names are the same, set prefix in module configuration to disambiguate!" """ - touch ${prefix}.fas.gz + touch ${prefix}.fas${compress ? '.gz' : ''} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/muscle5/super5/main.nf b/modules/nf-core/muscle5/super5/main.nf index faf56864ec6..87af149b605 100644 --- a/modules/nf-core/muscle5/super5/main.nf +++ b/modules/nf-core/muscle5/super5/main.nf @@ -52,7 +52,7 @@ process MUSCLE5_SUPER5 { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix}.aln.gz + touch ${prefix}.aln${compress ? '.gz' : ''} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/tcoffee/align/main.nf b/modules/nf-core/tcoffee/align/main.nf index 08acdef4aae..671aca8eacc 100644 --- a/modules/nf-core/tcoffee/align/main.nf +++ b/modules/nf-core/tcoffee/align/main.nf @@ -47,9 +47,8 @@ process TCOFFEE_ALIGN { stub: def prefix = task.ext.prefix ?: "${meta.id}" - def pigz_call = compress ? ".gz": "" """ - touch ${prefix}.aln${pigz_call} + touch ${prefix}.aln${compress ? '.gz':''} cat <<-END_VERSIONS > versions.yml "${task.process}": From 1d35d3e34601447ee39c252a0c9ea32a19642279 Mon Sep 17 00:00:00 2001 From: Leon Rauschning Date: Fri, 9 Feb 2024 19:44:17 +0100 Subject: [PATCH 62/66] apparently this requires regenerating the snapshots? --- .../clustalo/align/tests/main.nf.test.snap | 10 ++++---- .../famsa/align/tests/main.nf.test.snap | 24 +++++++++---------- modules/nf-core/mafft/tests/main.nf.test.snap | 16 ++++++------- .../muscle5/super5/tests/main.nf.test.snap | 6 ++--- .../tcoffee/align/tests/main.nf.test.snap | 12 +++++----- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/modules/nf-core/clustalo/align/tests/main.nf.test.snap b/modules/nf-core/clustalo/align/tests/main.nf.test.snap index f034d96b8c4..d7d69870c87 100644 --- a/modules/nf-core/clustalo/align/tests/main.nf.test.snap +++ b/modules/nf-core/clustalo/align/tests/main.nf.test.snap @@ -10,7 +10,7 @@ ] ] ], - "timestamp": "2024-01-25T21:04:52.662283926" + "timestamp": "2024-02-09T19:39:46.647351958" }, "versions": { "content": [ @@ -18,7 +18,7 @@ "versions.yml:md5,327da6a4250a6b7c4e45cddaa1f56280" ] ], - "timestamp": "2024-01-25T19:59:54.545448144" + "timestamp": "2024-02-09T19:39:14.826528498" }, "alignment - uncompressed": { "content": [ @@ -31,7 +31,7 @@ ] ] ], - "timestamp": "2024-01-25T21:04:26.896932354" + "timestamp": "2024-02-09T19:39:14.786480272" }, "with_guide_tree_alignment": { "content": [ @@ -44,7 +44,7 @@ ] ] ], - "timestamp": "2024-01-17T18:24:51.474838687" + "timestamp": "2024-02-09T19:40:45.057777867" }, "with_guide_tree_versions": { "content": [ @@ -52,6 +52,6 @@ "versions.yml:md5,327da6a4250a6b7c4e45cddaa1f56280" ] ], - "timestamp": "2024-01-16T15:19:47.225320481" + "timestamp": "2024-02-09T19:40:45.122824595" } } \ No newline at end of file diff --git a/modules/nf-core/famsa/align/tests/main.nf.test.snap b/modules/nf-core/famsa/align/tests/main.nf.test.snap index 2fd253843cd..95bbbf17750 100644 --- a/modules/nf-core/famsa/align/tests/main.nf.test.snap +++ b/modules/nf-core/famsa/align/tests/main.nf.test.snap @@ -10,7 +10,7 @@ ] ] ], - "timestamp": "2024-01-25T21:00:36.525346869" + "timestamp": "2024-02-09T19:08:43.577982822" }, "versions": { "content": [ @@ -18,7 +18,7 @@ "versions.yml:md5,7d9e0a8c263fa6d9017075fe88c9e9dc" ] ], - "timestamp": "2024-01-17T17:51:57.390391023" + "timestamp": "2024-02-09T19:08:43.670136799" }, "with_guide_tree_alignment": { "content": [ @@ -31,15 +31,7 @@ ] ] ], - "timestamp": "2024-01-17T18:22:38.944328729" - }, - "with_guide_tree_versions": { - "content": [ - [ - "versions.yml:md5,7d9e0a8c263fa6d9017075fe88c9e9dc" - ] - ], - "timestamp": "2024-01-17T18:02:24.755216456" + "timestamp": "2024-02-09T19:10:05.167368314" }, "alignment_compressed": { "content": [ @@ -52,6 +44,14 @@ ] ] ], - "timestamp": "2024-01-25T21:01:03.727336388" + "timestamp": "2024-02-09T19:09:25.819156831" + }, + "with_guide_tree_versions": { + "content": [ + [ + "versions.yml:md5,7d9e0a8c263fa6d9017075fe88c9e9dc" + ] + ], + "timestamp": "2024-02-09T19:10:05.231995851" } } \ No newline at end of file diff --git a/modules/nf-core/mafft/tests/main.nf.test.snap b/modules/nf-core/mafft/tests/main.nf.test.snap index 64911a41e07..c14ad086fa1 100644 --- a/modules/nf-core/mafft/tests/main.nf.test.snap +++ b/modules/nf-core/mafft/tests/main.nf.test.snap @@ -28,7 +28,7 @@ ] } ], - "timestamp": "2024-01-25T21:02:43.951149407" + "timestamp": "2024-02-09T19:08:41.735774847" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta multiple": { "content": [ @@ -59,7 +59,7 @@ ] } ], - "timestamp": "2024-01-17T19:29:32.199853244" + "timestamp": "2024-02-09T19:10:38.940555785" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta normal": { "content": [ @@ -90,7 +90,7 @@ ] } ], - "timestamp": "2024-01-17T19:14:09.623223095" + "timestamp": "2024-02-09T19:09:35.656248409" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta long": { "content": [ @@ -121,7 +121,7 @@ ] } ], - "timestamp": "2024-01-17T19:26:27.263653083" + "timestamp": "2024-02-09T19:10:26.372655394" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta profile": { "content": [ @@ -152,7 +152,7 @@ ] } ], - "timestamp": "2024-01-17T19:23:20.904278207" + "timestamp": "2024-02-09T19:10:14.039053212" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta fragments": { "content": [ @@ -183,7 +183,7 @@ ] } ], - "timestamp": "2024-01-17T19:17:15.613863351" + "timestamp": "2024-02-09T19:09:49.737364197" }, "SARS-CoV-2 scaffolds fasta - add informative sites fasta full": { "content": [ @@ -214,7 +214,7 @@ ] } ], - "timestamp": "2024-01-17T19:20:19.23954473" + "timestamp": "2024-02-09T19:10:02.952480822" }, "SARS-CoV-2 scaffolds fasta - compressed": { "content": [ @@ -245,6 +245,6 @@ ] } ], - "timestamp": "2024-01-25T21:03:06.058030723" + "timestamp": "2024-02-09T19:09:21.096197597" } } \ No newline at end of file diff --git a/modules/nf-core/muscle5/super5/tests/main.nf.test.snap b/modules/nf-core/muscle5/super5/tests/main.nf.test.snap index 4503d229bd4..ce7aadf0575 100644 --- a/modules/nf-core/muscle5/super5/tests/main.nf.test.snap +++ b/modules/nf-core/muscle5/super5/tests/main.nf.test.snap @@ -10,7 +10,7 @@ ] ] ], - "timestamp": "2024-01-25T21:05:57.940317215" + "timestamp": "2024-02-09T19:08:23.498404397" }, "perm-all": { "content": [ @@ -28,7 +28,7 @@ ] ] ], - "timestamp": "2024-01-24T17:53:57.627560248" + "timestamp": "2024-02-09T19:08:37.386512953" }, "alignment - uncompressed": { "content": [ @@ -41,6 +41,6 @@ ] ] ], - "timestamp": "2024-01-25T21:05:50.236788075" + "timestamp": "2024-02-09T19:16:25.330353817" } } \ No newline at end of file diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test.snap b/modules/nf-core/tcoffee/align/tests/main.nf.test.snap index 2a702cbe1f0..94b9d4e3cce 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test.snap +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test.snap @@ -5,7 +5,7 @@ "versions.yml:md5,fb187c9186b50a8076d08cd3be3c1b70" ] ], - "timestamp": "2024-01-19T17:56:36.791833358" + "timestamp": "2024-02-09T19:43:36.590833926" }, "versions": { "content": [ @@ -13,7 +13,7 @@ "versions.yml:md5,fb187c9186b50a8076d08cd3be3c1b70" ] ], - "timestamp": "2024-01-19T17:56:17.01424266" + "timestamp": "2024-02-09T19:42:52.802861054" }, "alignment - uncompressed": { "content": [ @@ -26,7 +26,7 @@ ] ] ], - "timestamp": "2024-01-25T21:06:29.387718149" + "timestamp": "2024-02-09T19:43:05.936032398" }, "versions_guidetree": { "content": [ @@ -34,7 +34,7 @@ "versions.yml:md5,fb187c9186b50a8076d08cd3be3c1b70" ] ], - "timestamp": "2024-01-19T17:56:26.914325234" + "timestamp": "2024-02-09T19:43:20.741062719" }, "alignment": { "content": [ @@ -47,7 +47,7 @@ ] ] ], - "timestamp": "2024-01-25T21:06:19.529059609" + "timestamp": "2024-02-09T19:42:52.774882703" }, "alignment_guidetree": { "content": [ @@ -60,6 +60,6 @@ ] ] ], - "timestamp": "2024-01-19T17:56:26.885044145" + "timestamp": "2024-02-09T19:43:20.688201033" } } \ No newline at end of file From 706d05fc4fd850ed4399cafa0e9e575f6b4f7743 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 13 Feb 2024 11:59:40 +0100 Subject: [PATCH 63/66] try removing test match names, as per sateesh's suggestion --- .../nf-core/clustalo/align/tests/main.nf.test | 12 ++++++------ modules/nf-core/famsa/align/tests/main.nf.test | 12 ++++++------ modules/nf-core/mafft/tests/main.nf.test | 16 ++++++++-------- .../nf-core/mtmalign/align/tests/main.nf.test | 4 ++-- modules/nf-core/tcoffee/align/tests/main.nf.test | 14 +++++++------- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/modules/nf-core/clustalo/align/tests/main.nf.test b/modules/nf-core/clustalo/align/tests/main.nf.test index 4d7c97081a3..c04a682b078 100644 --- a/modules/nf-core/clustalo/align/tests/main.nf.test +++ b/modules/nf-core/clustalo/align/tests/main.nf.test @@ -28,8 +28,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment - uncompressed")}, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.alignment).match()}, + { assert snapshot(process.out.versions).match() } ) } @@ -52,8 +52,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment - compressed")}, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.alignment).match()}, + { assert snapshot(process.out.versions).match() } ) } @@ -90,8 +90,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("with_guide_tree_alignment")}, - { assert snapshot(process.out.versions).match("with_guide_tree_versions") } + { assert snapshot(process.out.alignment).match()}, + { assert snapshot(process.out.versions).match() } ) } } diff --git a/modules/nf-core/famsa/align/tests/main.nf.test b/modules/nf-core/famsa/align/tests/main.nf.test index 2d8e1482a6d..5b5c263ce3c 100644 --- a/modules/nf-core/famsa/align/tests/main.nf.test +++ b/modules/nf-core/famsa/align/tests/main.nf.test @@ -27,8 +27,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment_uncompressed")}, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.alignment).match()}, + { assert snapshot(process.out.versions).match() } ) } @@ -51,8 +51,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment_compressed")}, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.alignment).match()}, + { assert snapshot(process.out.versions).match() } ) } @@ -88,8 +88,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("with_guide_tree_alignment")}, - { assert snapshot(process.out.versions).match("with_guide_tree_versions") } + { assert snapshot(process.out.alignment).match()}, + { assert snapshot(process.out.versions).match()} ) } } diff --git a/modules/nf-core/mafft/tests/main.nf.test b/modules/nf-core/mafft/tests/main.nf.test index f57ab4965da..97a07221424 100644 --- a/modules/nf-core/mafft/tests/main.nf.test +++ b/modules/nf-core/mafft/tests/main.nf.test @@ -28,7 +28,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - uncompressed")} + { assert snapshot(process.out).match()} ) } @@ -55,7 +55,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - compressed")} + { assert snapshot(process.out).match()} ) } @@ -85,7 +85,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - add informative sites fasta normal") } + { assert snapshot(process.out).match() } ) } } @@ -115,7 +115,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - add informative sites fasta fragments") } + { assert snapshot(process.out).match() } ) } } @@ -144,7 +144,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - add informative sites fasta full") } + { assert snapshot(process.out).match() } ) } @@ -174,7 +174,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - add informative sites fasta profile") } + { assert snapshot(process.out).match() } ) } @@ -205,7 +205,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - add informative sites fasta long") } + { assert snapshot(process.out).match() } ) } @@ -239,7 +239,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - add informative sites fasta multiple") } + { assert snapshot(process.out).match() } ) } diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test b/modules/nf-core/mtmalign/align/tests/main.nf.test index e12ef76a6a7..0ce52020654 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test @@ -44,7 +44,7 @@ nextflow_process { { assert path(process.out.alignment[0][1]).getText().contains(">3.pdb") }, { assert path(process.out.alignment[0][1]).getText().contains(">4.pdb") }, { assert path(process.out.alignment[0][1]).getText().contains(">5.pdb") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match() } ) } } @@ -84,7 +84,7 @@ nextflow_process { { assert path(process.out.alignment[0][1]).getTextGzip().contains(">3.pdb") }, { assert path(process.out.alignment[0][1]).getTextGzip().contains(">4.pdb") }, { assert path(process.out.alignment[0][1]).getTextGzip().contains(">5.pdb") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match() } ) } } diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test b/modules/nf-core/tcoffee/align/tests/main.nf.test index d31d387205e..67588ab042b 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test @@ -31,8 +31,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment")}, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.alignment).match()}, + { assert snapshot(process.out.versions).match() } ) } } @@ -57,8 +57,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment - uncompressed")}, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.alignment).match()}, + { assert snapshot(process.out.versions).match()} ) } } @@ -98,8 +98,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match("alignment_guidetree")}, - { assert snapshot(process.out.versions).match("versions_guidetree") } + { assert snapshot(process.out.alignment).match()}, + { assert snapshot(process.out.versions).match() } ) } @@ -142,7 +142,7 @@ nextflow_process { assertAll( { assert process.success }, { assert path(process.out.alignment.get(0).get(1)).getTextGzip().contains("1ahl") }, - { assert snapshot(process.out.versions).match("versions_structure") } + { assert snapshot(process.out.versions).match() } ) } From b338910c2a84ca71d146ff6fbc6d195ee2095cd7 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Tue, 13 Feb 2024 15:22:01 +0100 Subject: [PATCH 64/66] Revert "try removing test match names, as per sateesh's suggestion" This reverts commit 706d05fc4fd850ed4399cafa0e9e575f6b4f7743. --- .../nf-core/clustalo/align/tests/main.nf.test | 12 ++++++------ modules/nf-core/famsa/align/tests/main.nf.test | 12 ++++++------ modules/nf-core/mafft/tests/main.nf.test | 16 ++++++++-------- .../nf-core/mtmalign/align/tests/main.nf.test | 4 ++-- modules/nf-core/tcoffee/align/tests/main.nf.test | 14 +++++++------- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/modules/nf-core/clustalo/align/tests/main.nf.test b/modules/nf-core/clustalo/align/tests/main.nf.test index c04a682b078..4d7c97081a3 100644 --- a/modules/nf-core/clustalo/align/tests/main.nf.test +++ b/modules/nf-core/clustalo/align/tests/main.nf.test @@ -28,8 +28,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match()}, - { assert snapshot(process.out.versions).match() } + { assert snapshot(process.out.alignment).match("alignment - uncompressed")}, + { assert snapshot(process.out.versions).match("versions") } ) } @@ -52,8 +52,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match()}, - { assert snapshot(process.out.versions).match() } + { assert snapshot(process.out.alignment).match("alignment - compressed")}, + { assert snapshot(process.out.versions).match("versions") } ) } @@ -90,8 +90,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match()}, - { assert snapshot(process.out.versions).match() } + { assert snapshot(process.out.alignment).match("with_guide_tree_alignment")}, + { assert snapshot(process.out.versions).match("with_guide_tree_versions") } ) } } diff --git a/modules/nf-core/famsa/align/tests/main.nf.test b/modules/nf-core/famsa/align/tests/main.nf.test index 5b5c263ce3c..2d8e1482a6d 100644 --- a/modules/nf-core/famsa/align/tests/main.nf.test +++ b/modules/nf-core/famsa/align/tests/main.nf.test @@ -27,8 +27,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match()}, - { assert snapshot(process.out.versions).match() } + { assert snapshot(process.out.alignment).match("alignment_uncompressed")}, + { assert snapshot(process.out.versions).match("versions") } ) } @@ -51,8 +51,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match()}, - { assert snapshot(process.out.versions).match() } + { assert snapshot(process.out.alignment).match("alignment_compressed")}, + { assert snapshot(process.out.versions).match("versions") } ) } @@ -88,8 +88,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match()}, - { assert snapshot(process.out.versions).match()} + { assert snapshot(process.out.alignment).match("with_guide_tree_alignment")}, + { assert snapshot(process.out.versions).match("with_guide_tree_versions") } ) } } diff --git a/modules/nf-core/mafft/tests/main.nf.test b/modules/nf-core/mafft/tests/main.nf.test index 97a07221424..f57ab4965da 100644 --- a/modules/nf-core/mafft/tests/main.nf.test +++ b/modules/nf-core/mafft/tests/main.nf.test @@ -28,7 +28,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match()} + { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - uncompressed")} ) } @@ -55,7 +55,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match()} + { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - compressed")} ) } @@ -85,7 +85,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - add informative sites fasta normal") } ) } } @@ -115,7 +115,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - add informative sites fasta fragments") } ) } } @@ -144,7 +144,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - add informative sites fasta full") } ) } @@ -174,7 +174,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - add informative sites fasta profile") } ) } @@ -205,7 +205,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - add informative sites fasta long") } ) } @@ -239,7 +239,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(process.out).match("SARS-CoV-2 scaffolds fasta - add informative sites fasta multiple") } ) } diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test b/modules/nf-core/mtmalign/align/tests/main.nf.test index 0ce52020654..e12ef76a6a7 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test @@ -44,7 +44,7 @@ nextflow_process { { assert path(process.out.alignment[0][1]).getText().contains(">3.pdb") }, { assert path(process.out.alignment[0][1]).getText().contains(">4.pdb") }, { assert path(process.out.alignment[0][1]).getText().contains(">5.pdb") }, - { assert snapshot(process.out.versions).match() } + { assert snapshot(process.out.versions).match("versions") } ) } } @@ -84,7 +84,7 @@ nextflow_process { { assert path(process.out.alignment[0][1]).getTextGzip().contains(">3.pdb") }, { assert path(process.out.alignment[0][1]).getTextGzip().contains(">4.pdb") }, { assert path(process.out.alignment[0][1]).getTextGzip().contains(">5.pdb") }, - { assert snapshot(process.out.versions).match() } + { assert snapshot(process.out.versions).match("versions") } ) } } diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test b/modules/nf-core/tcoffee/align/tests/main.nf.test index 67588ab042b..d31d387205e 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test @@ -31,8 +31,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match()}, - { assert snapshot(process.out.versions).match() } + { assert snapshot(process.out.alignment).match("alignment")}, + { assert snapshot(process.out.versions).match("versions") } ) } } @@ -57,8 +57,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match()}, - { assert snapshot(process.out.versions).match()} + { assert snapshot(process.out.alignment).match("alignment - uncompressed")}, + { assert snapshot(process.out.versions).match("versions") } ) } } @@ -98,8 +98,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.alignment).match()}, - { assert snapshot(process.out.versions).match() } + { assert snapshot(process.out.alignment).match("alignment_guidetree")}, + { assert snapshot(process.out.versions).match("versions_guidetree") } ) } @@ -142,7 +142,7 @@ nextflow_process { assertAll( { assert process.success }, { assert path(process.out.alignment.get(0).get(1)).getTextGzip().contains("1ahl") }, - { assert snapshot(process.out.versions).match() } + { assert snapshot(process.out.versions).match("versions_structure") } ) } From 1f89b9294b8525321b97509e19b3cd6bf3fb703a Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Thu, 15 Feb 2024 10:50:22 +0100 Subject: [PATCH 65/66] tcoffee/align change snapshot names --- modules/nf-core/tcoffee/align/tests/main.nf.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/tcoffee/align/tests/main.nf.test b/modules/nf-core/tcoffee/align/tests/main.nf.test index d31d387205e..9d66f86bf39 100644 --- a/modules/nf-core/tcoffee/align/tests/main.nf.test +++ b/modules/nf-core/tcoffee/align/tests/main.nf.test @@ -32,7 +32,7 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot(process.out.alignment).match("alignment")}, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions_uncomp") } ) } } @@ -58,7 +58,7 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot(process.out.alignment).match("alignment - uncompressed")}, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions_comp") } ) } } From a624c589fe597a6c58fcaebcdb393b7e804b58f1 Mon Sep 17 00:00:00 2001 From: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> Date: Thu, 15 Feb 2024 11:04:07 +0100 Subject: [PATCH 66/66] make snapshot names unique for nf-test 0.8.4 --- modules/nf-core/clustalo/align/tests/main.nf.test | 4 ++-- modules/nf-core/famsa/align/tests/main.nf.test | 4 ++-- modules/nf-core/mtmalign/align/tests/main.nf.test | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/nf-core/clustalo/align/tests/main.nf.test b/modules/nf-core/clustalo/align/tests/main.nf.test index 4d7c97081a3..3edd36a21c0 100644 --- a/modules/nf-core/clustalo/align/tests/main.nf.test +++ b/modules/nf-core/clustalo/align/tests/main.nf.test @@ -29,7 +29,7 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot(process.out.alignment).match("alignment - uncompressed")}, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions0") } ) } @@ -53,7 +53,7 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot(process.out.alignment).match("alignment - compressed")}, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions1") } ) } diff --git a/modules/nf-core/famsa/align/tests/main.nf.test b/modules/nf-core/famsa/align/tests/main.nf.test index 2d8e1482a6d..2d7cac3c4e0 100644 --- a/modules/nf-core/famsa/align/tests/main.nf.test +++ b/modules/nf-core/famsa/align/tests/main.nf.test @@ -28,7 +28,7 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot(process.out.alignment).match("alignment_uncompressed")}, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions0") } ) } @@ -52,7 +52,7 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot(process.out.alignment).match("alignment_compressed")}, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions1") } ) } diff --git a/modules/nf-core/mtmalign/align/tests/main.nf.test b/modules/nf-core/mtmalign/align/tests/main.nf.test index e12ef76a6a7..cb3f3885b67 100644 --- a/modules/nf-core/mtmalign/align/tests/main.nf.test +++ b/modules/nf-core/mtmalign/align/tests/main.nf.test @@ -44,7 +44,7 @@ nextflow_process { { assert path(process.out.alignment[0][1]).getText().contains(">3.pdb") }, { assert path(process.out.alignment[0][1]).getText().contains(">4.pdb") }, { assert path(process.out.alignment[0][1]).getText().contains(">5.pdb") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions0") } ) } } @@ -84,7 +84,7 @@ nextflow_process { { assert path(process.out.alignment[0][1]).getTextGzip().contains(">3.pdb") }, { assert path(process.out.alignment[0][1]).getTextGzip().contains(">4.pdb") }, { assert path(process.out.alignment[0][1]).getTextGzip().contains(">5.pdb") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions1") } ) } }