From 8d54c9f0527b6209790861983128e2c53498e9d4 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Mar 2024 09:44:06 +0000 Subject: [PATCH 1/4] Migrate to nf-test and only execute mv when dirs actually present --- .../amrfinderplus/run/tests/main.nf.test | 1 + modules/nf-core/gtdbtk/classifywf/main.nf | 20 +- .../gtdbtk/classifywf/tests/main.nf.test | 43 ++++ .../gtdbtk/classifywf/tests/main.nf.test.snap | 199 ++++++++++++++++++ .../gtdbtk/classifywf/tests/nextflow.config | 2 + .../nf-core/gtdbtk/classifywf/tests/tags.yml | 2 + tests/config/pytest_modules.yml | 3 - .../modules/nf-core/gtdbtk/classifywf/main.nf | 43 ---- .../nf-core/gtdbtk/classifywf/nextflow.config | 5 - .../nf-core/gtdbtk/classifywf/test.yml | 7 - 10 files changed, 261 insertions(+), 64 deletions(-) create mode 100644 modules/nf-core/gtdbtk/classifywf/tests/main.nf.test create mode 100644 modules/nf-core/gtdbtk/classifywf/tests/main.nf.test.snap create mode 100644 modules/nf-core/gtdbtk/classifywf/tests/nextflow.config create mode 100644 modules/nf-core/gtdbtk/classifywf/tests/tags.yml delete mode 100644 tests/modules/nf-core/gtdbtk/classifywf/main.nf delete mode 100644 tests/modules/nf-core/gtdbtk/classifywf/nextflow.config delete mode 100644 tests/modules/nf-core/gtdbtk/classifywf/test.yml diff --git a/modules/nf-core/amrfinderplus/run/tests/main.nf.test b/modules/nf-core/amrfinderplus/run/tests/main.nf.test index 8103bb0fb9e..ba09dda9bdf 100644 --- a/modules/nf-core/amrfinderplus/run/tests/main.nf.test +++ b/modules/nf-core/amrfinderplus/run/tests/main.nf.test @@ -30,6 +30,7 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/prokaryotes/haemophilus_influenzae/genome/genome.fna.gz', checkIfExists: true) ] + def fake_mash input[1] = AMRFINDERPLUS_UPDATE.out.db """ } diff --git a/modules/nf-core/gtdbtk/classifywf/main.nf b/modules/nf-core/gtdbtk/classifywf/main.nf index 6d9733ba205..d5cb73236e9 100644 --- a/modules/nf-core/gtdbtk/classifywf/main.nf +++ b/modules/nf-core/gtdbtk/classifywf/main.nf @@ -51,11 +51,19 @@ process GTDBTK_CLASSIFYWF { --min_perc_aa $params.gtdbtk_min_perc_aa \\ --min_af $params.gtdbtk_min_af - mv classify/* . + ## If mash db given, classify/ and identify/ directories won't be created + if [[ -d classify/ ]]; then + mv classify/* . + fi - mv identify/* . + if [[ -d identify/ ]]; then + mv identify/* . + fi - mv align/* .\ + ## If nothing aligns, no output, so only run + if [[ -d align/ ]]; then + mv align/* .\ + fi mv gtdbtk.log "gtdbtk.${prefix}.log" @@ -74,10 +82,10 @@ process GTDBTK_CLASSIFYWF { prefix = task.ext.prefix ?: "${meta.id}" """ touch gtdbtk.${prefix}.stub.summary.tsv - touch gtdbtk.${prefix}.stub.classify.tree.gz + echo "" | gzip > gtdbtk.${prefix}.stub.classify.tree.gz touch gtdbtk.${prefix}.stub.markers_summary.tsv - touch gtdbtk.${prefix}.stub.msa.fasta.gz - touch gtdbtk.${prefix}.stub.user_msa.fasta.gz + echo "" | gzip > gtdbtk.${prefix}.stub.msa.fasta.gz + echo "" | gzip > gtdbtk.${prefix}.stub.user_msa.fasta.gz touch gtdbtk.${prefix}.stub.filtered.tsv touch gtdbtk.${prefix}.log touch gtdbtk.${prefix}.warnings.log diff --git a/modules/nf-core/gtdbtk/classifywf/tests/main.nf.test b/modules/nf-core/gtdbtk/classifywf/tests/main.nf.test new file mode 100644 index 00000000000..074670ea4a4 --- /dev/null +++ b/modules/nf-core/gtdbtk/classifywf/tests/main.nf.test @@ -0,0 +1,43 @@ +nextflow_process { + + name "Test Process GTDBTK_CLASSIFYWF" + script "../main.nf" + process "GTDBTK_CLASSIFYWF" + + tag "modules" + tag "modules_nfcore" + tag "gtdbtk" + tag "gtdbtk/classifywf" + + // Only stub test is possible due to very large required database (>70GB) + test("sarscov2 - genome fasta - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test', single_end:false, assembler:'SPADES' ], + [ + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['contigs_fasta'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['scaffolds_fasta'], checkIfExists: true) + ] + ] + input[1] = [[], []] + input[2] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/nf-core/gtdbtk/classifywf/tests/main.nf.test.snap b/modules/nf-core/gtdbtk/classifywf/tests/main.nf.test.snap new file mode 100644 index 00000000000..e821084cfaa --- /dev/null +++ b/modules/nf-core/gtdbtk/classifywf/tests/main.nf.test.snap @@ -0,0 +1,199 @@ +{ + "sarscov2 - genome fasta - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.stub.summary.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.stub.classify.tree.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "2": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.stub.markers_summary.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "3": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.stub.msa.fasta.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "4": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.stub.user_msa.fasta.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "5": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.stub.filtered.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "6": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.failed_genomes.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "7": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "8": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.warnings.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "9": [ + "versions.yml:md5,a8ab755bce9f17684f235d49ab99f6d2" + ], + "failed": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.failed_genomes.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "filtered": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.stub.filtered.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "log": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "markers": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.stub.markers_summary.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "msa": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.stub.msa.fasta.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "summary": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.stub.summary.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "tree": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.stub.classify.tree.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "user_msa": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.stub.user_msa.fasta.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "versions": [ + "versions.yml:md5,a8ab755bce9f17684f235d49ab99f6d2" + ], + "warnings": [ + [ + { + "id": "test", + "single_end": false, + "assembler": "SPADES" + }, + "gtdbtk.test.warnings.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-26T09:39:21.632259941" + } +} \ No newline at end of file diff --git a/modules/nf-core/gtdbtk/classifywf/tests/nextflow.config b/modules/nf-core/gtdbtk/classifywf/tests/nextflow.config new file mode 100644 index 00000000000..e177a144f42 --- /dev/null +++ b/modules/nf-core/gtdbtk/classifywf/tests/nextflow.config @@ -0,0 +1,2 @@ +process { +} diff --git a/modules/nf-core/gtdbtk/classifywf/tests/tags.yml b/modules/nf-core/gtdbtk/classifywf/tests/tags.yml new file mode 100644 index 00000000000..5d8badacbef --- /dev/null +++ b/modules/nf-core/gtdbtk/classifywf/tests/tags.yml @@ -0,0 +1,2 @@ +gtdbtk/classifywf: + - "modules/nf-core/gtdbtk/classifywf/**" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 226136354b1..1c8393a6f3a 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -919,9 +919,6 @@ gstama/merge: gstama/polyacleanup: - modules/nf-core/gstama/polyacleanup/** - tests/modules/nf-core/gstama/polyacleanup/** -gtdbtk/classifywf: - - modules/nf-core/gtdbtk/classifywf/** - - tests/modules/nf-core/gtdbtk/classifywf/** gubbins: - modules/nf-core/gubbins/** - tests/modules/nf-core/gubbins/** diff --git a/tests/modules/nf-core/gtdbtk/classifywf/main.nf b/tests/modules/nf-core/gtdbtk/classifywf/main.nf deleted file mode 100644 index e0fa8131407..00000000000 --- a/tests/modules/nf-core/gtdbtk/classifywf/main.nf +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { GTDBTK_CLASSIFYWF } from '../../../../../modules/nf-core/gtdbtk/classifywf/main.nf' - -process STUB_GTDBTK_DATABASE { - output: - tuple val("gtdbtk_r207_v2_data"), path("database/*"), emit: database - - stub: - """ - mkdir database - touch database/gtdbtk_r207_v2_data - """ -} - -process STUB_MASH_DATABASE { - output: - path("db.msh"), emit: mash_db - - stub: - """ - touch db.msh - """ -} - -workflow test_gtdbtk_classifywf { - - STUB_GTDBTK_DATABASE() - STUB_MASH_DATABASE() - - input = [ - [ id:'test', single_end:false, assembler:'SPADES' ], - [ - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['contigs_fasta'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['scaffolds_fasta'], checkIfExists: true) - ] - ] - - GTDBTK_CLASSIFYWF ( input, STUB_GTDBTK_DATABASE.out.database, STUB_MASH_DATABASE.out.mash_db ) -} diff --git a/tests/modules/nf-core/gtdbtk/classifywf/nextflow.config b/tests/modules/nf-core/gtdbtk/classifywf/nextflow.config deleted file mode 100644 index 8730f1c4b93..00000000000 --- a/tests/modules/nf-core/gtdbtk/classifywf/nextflow.config +++ /dev/null @@ -1,5 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - -} diff --git a/tests/modules/nf-core/gtdbtk/classifywf/test.yml b/tests/modules/nf-core/gtdbtk/classifywf/test.yml deleted file mode 100644 index 70b9e842bde..00000000000 --- a/tests/modules/nf-core/gtdbtk/classifywf/test.yml +++ /dev/null @@ -1,7 +0,0 @@ -- name: gtdbtk classifywf - command: nextflow run ./tests/modules/nf-core/gtdbtk/classifywf -entry test_gtdbtk_classifywf -c ./tests/config/nextflow.config -stub-run -c ./tests/modules/nf-core/gtdbtk/classifywf/nextflow.config - tags: - - gtdbtk - - gtdbtk/classifywf - files: - - path: output/gtdbtk/gtdbtk.test.stub.summary.tsv From ec2388f0f676206305c03ec6aad2de9409ebd905 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Mar 2024 11:03:02 +0100 Subject: [PATCH 2/4] Update modules/nf-core/gtdbtk/classifywf/main.nf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- modules/nf-core/gtdbtk/classifywf/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/gtdbtk/classifywf/main.nf b/modules/nf-core/gtdbtk/classifywf/main.nf index d5cb73236e9..14dd28a4fd0 100644 --- a/modules/nf-core/gtdbtk/classifywf/main.nf +++ b/modules/nf-core/gtdbtk/classifywf/main.nf @@ -62,7 +62,7 @@ process GTDBTK_CLASSIFYWF { ## If nothing aligns, no output, so only run if [[ -d align/ ]]; then - mv align/* .\ + mv align/* . fi mv gtdbtk.log "gtdbtk.${prefix}.log" From adecf81724c1a745825cb65fa43c68cf481fd613 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Mar 2024 10:06:48 +0000 Subject: [PATCH 3/4] Remove uesless file and use new test data notation --- modules/nf-core/gtdbtk/classifywf/tests/main.nf.test | 6 +++--- modules/nf-core/gtdbtk/classifywf/tests/nextflow.config | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 modules/nf-core/gtdbtk/classifywf/tests/nextflow.config diff --git a/modules/nf-core/gtdbtk/classifywf/tests/main.nf.test b/modules/nf-core/gtdbtk/classifywf/tests/main.nf.test index 074670ea4a4..60bedb684b4 100644 --- a/modules/nf-core/gtdbtk/classifywf/tests/main.nf.test +++ b/modules/nf-core/gtdbtk/classifywf/tests/main.nf.test @@ -20,9 +20,9 @@ nextflow_process { input[0] = [ [ id:'test', single_end:false, assembler:'SPADES' ], [ - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['contigs_fasta'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['scaffolds_fasta'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fasta/contigs.fasta', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fasta/scaffolds.fasta', checkIfExists: true), ] ] input[1] = [[], []] diff --git a/modules/nf-core/gtdbtk/classifywf/tests/nextflow.config b/modules/nf-core/gtdbtk/classifywf/tests/nextflow.config deleted file mode 100644 index e177a144f42..00000000000 --- a/modules/nf-core/gtdbtk/classifywf/tests/nextflow.config +++ /dev/null @@ -1,2 +0,0 @@ -process { -} From 3dce329dd2a6826034620c9cdc46c74c345db738 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Mar 2024 11:10:30 +0100 Subject: [PATCH 4/4] Apply suggestions from code review --- modules/nf-core/amrfinderplus/run/tests/main.nf.test | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/nf-core/amrfinderplus/run/tests/main.nf.test b/modules/nf-core/amrfinderplus/run/tests/main.nf.test index ba09dda9bdf..8103bb0fb9e 100644 --- a/modules/nf-core/amrfinderplus/run/tests/main.nf.test +++ b/modules/nf-core/amrfinderplus/run/tests/main.nf.test @@ -30,7 +30,6 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/prokaryotes/haemophilus_influenzae/genome/genome.fna.gz', checkIfExists: true) ] - def fake_mash input[1] = AMRFINDERPLUS_UPDATE.out.db """ }