Fixing sentieon - code and tests

CHANGELOG.md

@@ -57,7 +57,7 @@ Balgattjåhkkå is the other top peak (over 2k m) in the Pårte massif, the othe
 - [#1422](https://github.com/nf-core/sarek/pull/1422) - Fix `Cannot serialize context map` warning
 - [#1462](https://github.com/nf-core/sarek/pull/1462) - Fix ascat input channels
 - [#1463](https://github.com/nf-core/sarek/pull/1463) - Add `spark` profile to all gatk4spark tests
-- [#1465](https://github.com/nf-core/sarek/pull/1465) - Fix input channels for `SENTIEON_DEDUP`
+- [#1465](https://github.com/nf-core/sarek/pull/1465), [#1469](https://github.com/nf-core/sarek/pull/1469) - Fix input channels and tests of Sentieon workflows
 
 ### Removed
 

subworkflows/local/bam_joint_calling_germline_sentieon/main.nf

@@ -36,7 +36,7 @@ workflow BAM_JOINT_CALLING_GERMLINE_SENTIEON {
         .map{ meta, gvcf, tbi, intervals -> [ [ id:'joint_variant_calling', intervals_name:intervals.baseName, num_intervals:meta.num_intervals ], gvcf, tbi, intervals ] }
         .groupTuple(by:[0, 3])
 
-    SENTIEON_GVCFTYPER(sentieon_input, fasta, fai, dbsnp, dbsnp_tbi)
+    SENTIEON_GVCFTYPER(sentieon_input, fasta.map{meta, it -> [ it ]}, fai.map{meta, it -> [ it ]}, dbsnp, dbsnp_tbi)
 
     BCFTOOLS_SORT(SENTIEON_GVCFTYPER.out.vcf_gz)
 
@@ -68,16 +68,16 @@ workflow BAM_JOINT_CALLING_GERMLINE_SENTIEON {
             resource_indels_vcf,
             resource_indels_tbi,
             indels_resource_label,
-            fasta,
-            fai)
+            fasta.map{meta, it -> [ it ]},
+            fai.map{meta, it -> [ it ]})
 
         SENTIEON_VARCAL_SNP(
             vqsr_input,
             resource_snps_vcf,
             resource_snps_tbi,
             snps_resource_label,
-            fasta,
-            fai)
+            fasta.map{meta, it -> [ it ]},
+            fai.map{meta, it -> [ it ]})
 
         //Prepare SNPs and INDELs for Sentieon's applyvarcal
         // Step 1. : applyvarcal to SNPs
@@ -90,10 +90,7 @@ workflow BAM_JOINT_CALLING_GERMLINE_SENTIEON {
             .join(SENTIEON_VARCAL_SNP.out.tranches, failOnDuplicate: true)
             .map{ meta, vcf, tbi, recal, index, tranche -> [ meta + [ id:'recalibrated_joint_variant_calling' ], vcf, tbi, recal, index, tranche ] }
 
-        SENTIEON_APPLYVARCAL_SNP(
-            vqsr_input_snp,
-            fasta.map{ fasta -> [ [ id:fasta.baseName ], fasta ] },
-            fai.map{ fai -> [ [ id:fai.baseName ], fai ] })
+        SENTIEON_APPLYVARCAL_SNP(vqsr_input_snp, fasta, fai)
 
         // Join results of SENTIEON_APPLYVARCAL_SNP and use as input for SENTIEON_APPLYVARCAL_INDEL to avoid duplicate entries in the result
         // Rework meta for variantscalled.csv and annotation tools
@@ -103,10 +100,7 @@ workflow BAM_JOINT_CALLING_GERMLINE_SENTIEON {
             .join(SENTIEON_VARCAL_INDEL.out.tranches, failOnDuplicate: true)
             .map{ meta, vcf, tbi, recal, index, tranche -> [ meta + [ id:'recalibrated_joint_variant_calling' ], vcf, tbi, recal, index, tranche ] }
 
-        SENTIEON_APPLYVARCAL_INDEL(
-            vqsr_input_indel,
-            fasta.map{ fasta -> [ [ id:fasta.baseName ], fasta ] },
-            fai.map{ fai -> [ [ id:fai.baseName ], fai ] })
+        SENTIEON_APPLYVARCAL_INDEL(vqsr_input_indel, fasta, fai)
 
         // The following is an ugly monster to achieve the following:
         // When MERGE_GENOTYPEGVCFS and SENTIEON_APPLYVARCAL are run, then use output from SENTIEON_APPLYVARCAL

subworkflows/local/bam_variant_calling_germline_all/main.nf

@@ -239,7 +239,7 @@ workflow BAM_VARIANT_CALLING_GERMLINE_ALL {
                 SENTIEON_DNAMODELAPPLY(
                     vcf_sentieon_dnascope.join(vcf_tbi_sentieon_dnascope, failOnDuplicate: true, failOnMismatch: true),
                     fasta,
-                    fasta_fai.map{ fai -> [ [ id:fai.baseName ], fai ] },
+                    fasta_fai,
                     sentieon_dnascope_model.map{ model -> [ [ id:model.baseName ], model ] })
 
                 vcf_sentieon_dnascope = SENTIEON_DNAMODELAPPLY.out.vcf
@@ -297,8 +297,8 @@ workflow BAM_VARIANT_CALLING_GERMLINE_ALL {
 
                 SENTIEON_HAPLOTYPER_VCF_VARIANT_FILTERING_GATK(
                     vcf_sentieon_haplotyper.join(vcf_tbi_sentieon_haplotyper, failOnDuplicate: true, failOnMismatch: true),
-                    fasta,
-                    fasta_fai,
+                    fasta.map{ meta, it -> [ it ] },
+                    fasta_fai.map{ meta, it -> [ it ] },
                     dict.map{ meta, dict -> [ dict ] },
                     intervals_bed_combined_haplotypec,
                     known_sites_indels.concat(known_sites_snps).flatten().unique().collect(),

subworkflows/local/bam_variant_calling_sentieon_dnascope/main.nf

@@ -50,8 +50,8 @@ workflow BAM_VARIANT_CALLING_SENTIEON_DNASCOPE {
 
     SENTIEON_DNASCOPE(
         cram_intervals_for_sentieon,
-        fasta.map{it -> [[:], it]},
-        fasta_fai.map{it -> [[:], it]},
+        fasta,
+        fasta_fai,
         dbsnp.map{it -> [[:], it]},
         dbsnp_tbi.map{it -> [[:], it]},
         sentieon_dnascope_model.map{it -> [[:], it]},

subworkflows/local/bam_variant_calling_sentieon_haplotyper/main.nf

@@ -49,8 +49,8 @@ workflow BAM_VARIANT_CALLING_SENTIEON_HAPLOTYPER {
 
     SENTIEON_HAPLOTYPER(
         cram_intervals_for_sentieon,
-        fasta,
-        fasta_fai,
+        fasta.map{ meta, it -> it },
+        fasta_fai.map{ meta, it ->  it },
         dbsnp,
         dbsnp_tbi,
         emit_vcf,

subworkflows/local/samplesheet_to_channel/main.nf

@@ -224,12 +224,8 @@ workflow  SAMPLESHEET_TO_CHANNEL{
         joint_germline &&
             ( !dbsnp || !known_indels || !known_snps || no_intervals )
     ) {
-        log.warn("""If GATK's Haplotypecaller, Sentieon's Dnascope and/or Sentieon's Haplotyper is specified, \
-    but without `--dbsnp`, `--known_snps`, `--known_indels` or the associated resource labels (ie `known_snps_vqsr`), \
-    no variant recalibration will be done. For recalibration you must provide all of these resources.\nFor more information \
-    see VariantRecalibration: https://gatk.broadinstitute.org/hc/en-us/articles/5358906115227-VariantRecalibrator \n\
-    Joint germline variant calling also requires intervals in order to genotype the samples. \
-    As a result, if `--no_intervals` is set to `true` the joint germline variant calling will not be performed.""")
+        log.warn("""If GATK's Haplotypecaller, Sentieon's Dnascope and/or Sentieon's Haplotyper is specified, but without `--dbsnp`, `--known_snps`, `--known_indels` or the associated resource labels (ie `known_snps_vqsr`), no variant recalibration will be done. For recalibration you must provide all of these resources.\nFor more information see VariantRecalibration: https://gatk.broadinstitute.org/hc/en-us/articles/5358906115227-VariantRecalibrator \n\
+Joint germline variant calling also requires intervals in order to genotype the samples. As a result, if `--no_intervals` is set to `true` the joint germline variant calling will not be performed.""")
     }
 
     if (tools &&

tests/test_sentieon_dnascope.yml

@@ -17,7 +17,6 @@
     - path: results/preprocessing/recalibrated/test/test.recal.cram.crai
       should_exist: false
     - path: results/reports/bcftools/sentieon_dnascope/test/test.dnascope.filtered.bcftools_stats.txt
-      md5sum: fb3923060b59b7dc18705cac5704caba
     - path: results/reports/vcftools/sentieon_dnascope/test/test.dnascope.filtered.FILTER.summary
       md5sum: e67b24d296810a075378e5864bcea0fa
     - path: results/reports/vcftools/sentieon_dnascope/test/test.dnascope.filtered.TsTv.count
@@ -71,7 +70,6 @@
     - path: results/preprocessing/recalibrated/test/test.recal.cram.crai
       should_exist: false
     - path: results/reports/bcftools/sentieon_dnascope/test/test.dnascope.filtered.bcftools_stats.txt
-      md5sum: fb3923060b59b7dc18705cac5704caba
     - path: results/reports/vcftools/sentieon_dnascope/test/test.dnascope.filtered.FILTER.summary
       md5sum: e67b24d296810a075378e5864bcea0fa
     - path: results/reports/vcftools/sentieon_dnascope/test/test.dnascope.filtered.TsTv.count

tests/test_sentieon_dnascope_joint_germline.yml

@@ -39,26 +39,16 @@
     - path: results/variant_calling/sentieon_dnascope/joint_variant_calling/joint_germline_recalibrated.vcf.gz.tbi
       should_exist: false
     - path: results/reports/mosdepth/testN/testN.recal.mosdepth.global.dist.txt
-      md5sum: 69e29702ef01fd8f6c7a5468fc35a16a
     - path: results/reports/mosdepth/testN/testN.recal.mosdepth.region.dist.txt
-      md5sum: 6ec49cd7d510c2eb3d9d90fdb79b783a
     - path: results/reports/mosdepth/testN/testN.recal.mosdepth.summary.txt
-      md5sum: 103098d0bf76ed82d2b87d5f242b099a
     - path: results/reports/mosdepth/testN/testN.recal.regions.bed.gz
-      md5sum: b5888cf7395c57d39879a5faa6159eb3
     - path: results/reports/mosdepth/testN/testN.recal.regions.bed.gz.csi
-      md5sum: 9cb0ad7039a3b703d16ca7d5b835c0ee
     - path: results/reports/samtools/testN/testN.recal.cram.stats
     - path: results/reports/mosdepth/testT/testT.recal.mosdepth.global.dist.txt
-      md5sum: 69e29702ef01fd8f6c7a5468fc35a16a
     - path: results/reports/mosdepth/testT/testT.recal.mosdepth.region.dist.txt
-      md5sum: 6ec49cd7d510c2eb3d9d90fdb79b783a
     - path: results/reports/mosdepth/testT/testT.recal.mosdepth.summary.txt
-      md5sum: 103098d0bf76ed82d2b87d5f242b099a
     - path: results/reports/mosdepth/testT/testT.recal.regions.bed.gz
-      md5sum: b5888cf7395c57d39879a5faa6159eb3
     - path: results/reports/mosdepth/testT/testT.recal.regions.bed.gz.csi
-      md5sum: 9cb0ad7039a3b703d16ca7d5b835c0ee
     - path: results/reports/samtools/testT/testT.recal.cram.stats
 - name: Run joint germline variant calling with sentieon dnascope all intervals at once
   command: nextflow run main.nf -profile test_cache,targeted,software_license --sentieon_extension --input ./tests/csv/3.0/mapped_joint_bam.csv --tools sentieon_dnascope --step variant_calling --joint_germline --outdir results --sentieon_dnascope_emit_mode gvcf --nucleotides_per_second 100
@@ -87,24 +77,14 @@
     - path: results/dnascope
       should_exist: false
     - path: results/reports/mosdepth/testN/testN.recal.mosdepth.global.dist.txt
-      md5sum: 69e29702ef01fd8f6c7a5468fc35a16a
     - path: results/reports/mosdepth/testN/testN.recal.mosdepth.region.dist.txt
-      md5sum: 6ec49cd7d510c2eb3d9d90fdb79b783a
     - path: results/reports/mosdepth/testN/testN.recal.mosdepth.summary.txt
-      md5sum: 103098d0bf76ed82d2b87d5f242b099a
     - path: results/reports/mosdepth/testN/testN.recal.regions.bed.gz
-      md5sum: b5888cf7395c57d39879a5faa6159eb3
     - path: results/reports/mosdepth/testN/testN.recal.regions.bed.gz.csi
-      md5sum: 9cb0ad7039a3b703d16ca7d5b835c0ee
     - path: results/reports/samtools/testN/testN.recal.cram.stats
     - path: results/reports/mosdepth/testT/testT.recal.mosdepth.global.dist.txt
-      md5sum: 69e29702ef01fd8f6c7a5468fc35a16a
     - path: results/reports/mosdepth/testT/testT.recal.mosdepth.region.dist.txt
-      md5sum: 6ec49cd7d510c2eb3d9d90fdb79b783a
     - path: results/reports/mosdepth/testT/testT.recal.mosdepth.summary.txt
-      md5sum: 103098d0bf76ed82d2b87d5f242b099a
     - path: results/reports/mosdepth/testT/testT.recal.regions.bed.gz
-      md5sum: b5888cf7395c57d39879a5faa6159eb3
     - path: results/reports/mosdepth/testT/testT.recal.regions.bed.gz.csi
-      md5sum: 9cb0ad7039a3b703d16ca7d5b835c0ee
     - path: results/reports/samtools/testT/testT.recal.cram.stats

tests/test_sentieon_haplotyper.yml

@@ -17,7 +17,6 @@
     - path: results/preprocessing/recalibrated/test/test.recal.cram.crai
       should_exist: false
     - path: results/reports/bcftools/sentieon_haplotyper/test/test.haplotyper.filtered.bcftools_stats.txt
-      md5sum: 66be03d4e6535175514f54a1a031d49f
     - path: results/reports/vcftools/sentieon_haplotyper/test/test.haplotyper.filtered.FILTER.summary
       md5sum: d501a93356f3c91c743f51104e24514a
     - path: results/reports/vcftools/sentieon_haplotyper/test/test.haplotyper.filtered.TsTv.count
@@ -39,7 +38,6 @@
     - path: results/reports/mosdepth/test/test.recal.mosdepth.region.dist.txt
       md5sum: 3a2030e5e8af7bc12720c3a5592bf921
     - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
-      md5sum: 615c5c5019d88045a9ff5bbe6e63d270
     - path: results/reports/mosdepth/test/test.recal.regions.bed.gz
       md5sum: 9f1ea20e7461db948ba21f70c4d1b3ba
     - path: results/reports/mosdepth/test/test.recal.regions.bed.gz.csi
@@ -71,7 +69,6 @@
     - path: results/preprocessing/recalibrated/test/test.recal.cram.crai
       should_exist: false
     - path: results/reports/bcftools/sentieon_haplotyper/test/test.haplotyper.filtered.bcftools_stats.txt
-      md5sum: 66be03d4e6535175514f54a1a031d49f
     - path: results/reports/vcftools/sentieon_haplotyper/test/test.haplotyper.filtered.FILTER.summary
       md5sum: d501a93356f3c91c743f51104e24514a
     - path: results/reports/vcftools/sentieon_haplotyper/test/test.haplotyper.filtered.TsTv.count
@@ -91,7 +88,6 @@
     - path: results/reports/mosdepth/test/test.recal.mosdepth.global.dist.txt
       md5sum: e82e90c7d508a135b5a8a7cd6933452e
     - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
-      md5sum: 615c5c5019d88045a9ff5bbe6e63d270
     - path: results/reports/mosdepth/test/test.recal.per-base.bed.gz
       md5sum: bc1df47d46f818fee5275975925d769a
     - path: results/reports/mosdepth/test/test.recal.per-base.bed.gz.csi
@@ -140,7 +136,6 @@
     - path: results/reports/mosdepth/test/test.recal.mosdepth.region.dist.txt
       md5sum: 3a2030e5e8af7bc12720c3a5592bf921
     - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
-      md5sum: 615c5c5019d88045a9ff5bbe6e63d270
     - path: results/reports/mosdepth/test/test.recal.regions.bed.gz
       md5sum: 9f1ea20e7461db948ba21f70c4d1b3ba
     - path: results/reports/mosdepth/test/test.recal.regions.bed.gz.csi
@@ -181,7 +176,6 @@
     - path: results/reports/mosdepth/test/test.recal.mosdepth.region.dist.txt
       md5sum: 3a2030e5e8af7bc12720c3a5592bf921
     - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
-      md5sum: 615c5c5019d88045a9ff5bbe6e63d270
     - path: results/reports/mosdepth/test/test.recal.regions.bed.gz
       md5sum: 9f1ea20e7461db948ba21f70c4d1b3ba
     - path: results/reports/mosdepth/test/test.recal.regions.bed.gz.csi
@@ -206,7 +200,6 @@
     - path: results/preprocessing/recalibrated/test/test.recal.cram.crai
       should_exist: false
     - path: results/reports/bcftools/sentieon_haplotyper/test/test.haplotyper.filtered.bcftools_stats.txt
-      md5sum: 66be03d4e6535175514f54a1a031d49f
     - path: results/reports/vcftools/sentieon_haplotyper/test/test.haplotyper.filtered.FILTER.summary
       md5sum: d501a93356f3c91c743f51104e24514a
     - path: results/reports/vcftools/sentieon_haplotyper/test/test.haplotyper.filtered.TsTv.count
@@ -236,7 +229,6 @@
     - path: results/reports/mosdepth/test/test.recal.mosdepth.region.dist.txt
       md5sum: 3a2030e5e8af7bc12720c3a5592bf921
     - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
-      md5sum: 615c5c5019d88045a9ff5bbe6e63d270
     - path: results/reports/mosdepth/test/test.recal.regions.bed.gz
       md5sum: 9f1ea20e7461db948ba21f70c4d1b3ba
     - path: results/reports/mosdepth/test/test.recal.regions.bed.gz.csi

tests/test_sentieon_haplotyper_joint_germline.yml

@@ -112,9 +112,6 @@
     - path: results/preprocessing/recalibrated/test/test.recal.cram.crai
       should_exist: false
     - path: results/reports/bcftools/sentieon_haplotyper/joint_variant_calling/joint_germline_recalibrated.bcftools_stats.txt
-    - path: results/reports/vcftools/sentieon_haplotyper/joint_variant_calling/joint_germline_recalibrated.FILTER.summary
-    - path: results/reports/vcftools/sentieon_haplotyper/joint_variant_calling/joint_germline_recalibrated.TsTv.count
-    - path: results/reports/vcftools/sentieon_haplotyper/joint_variant_calling/joint_germline_recalibrated.TsTv.qual
     - path: results/variant_calling/sentieon_haplotyper/joint_variant_calling/joint_germline.vcf.gz
     - path: results/variant_calling/sentieon_haplotyper/joint_variant_calling/joint_germline.vcf.gz.tbi
     - path: results/variant_calling/sentieon_haplotyper/joint_variant_calling/joint_germline_recalibrated.vcf.gz
@@ -132,7 +129,7 @@
     - path: results/reports/mosdepth/testN/testN.recal.summary.txt
       md5sum: d41d8cd98f00b204e9800998ecf8427e
     - path: results/reports/mosdepth/testN/testN.recal.regions.bed.gz
-      md5sum: d41d8cd98f00b204e9800998ecf8427e
+      md5sum: 1a60c330fb42841e8dcf3cd507a70bfc
     - path: results/reports/mosdepth/testN/testN.recal.regions.bed.gz.csi
       md5sum: d41d8cd98f00b204e9800998ecf8427e
     - path: results/reports/samtools/testN/testN.recal.cram.stats
@@ -143,7 +140,7 @@
     - path: results/reports/mosdepth/testT/testT.recal.summary.txt
       md5sum: d41d8cd98f00b204e9800998ecf8427e
     - path: results/reports/mosdepth/testT/testT.recal.regions.bed.gz
-      md5sum: d41d8cd98f00b204e9800998ecf8427e
+      md5sum: 1a60c330fb42841e8dcf3cd507a70bfc
     - path: results/reports/mosdepth/testT/testT.recal.regions.bed.gz.csi
       md5sum: d41d8cd98f00b204e9800998ecf8427e
     - path: results/reports/samtools/testT/testT.recal.cram.stats

tests/test_sentieon_haplotyper_skip_filter.yml

@@ -17,7 +17,6 @@
     - path: results/preprocessing/recalibrated/test/test.recal.cram.crai
       should_exist: false
     - path: results/reports/bcftools/sentieon_haplotyper/test/test.haplotyper.unfiltered.bcftools_stats.txt
-      md5sum: d2660f9f6074b4bf18756c42ee656b8f
     - path: results/reports/vcftools/sentieon_haplotyper/test/test.haplotyper.unfiltered.FILTER.summary
       md5sum: 01b3d10464a3ac86f90ee82cdda23f68
     - path: results/reports/vcftools/sentieon_haplotyper/test/test.haplotyper.unfiltered.TsTv.count
@@ -37,7 +36,6 @@
     - path: results/reports/mosdepth/test/test.recal.mosdepth.global.dist.txt
       md5sum: e82e90c7d508a135b5a8a7cd6933452e
     - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
-      md5sum: 4f0d231060cbde4efdd673863bd2fb59
     - path: results/reports/mosdepth/test/test.recal.per-base.bed.gz
       md5sum: bc1df47d46f818fee5275975925d769a
     - path: results/reports/mosdepth/test/test.recal.per-base.bed.gz.csi
@@ -69,7 +67,6 @@
     - path: results/preprocessing/recalibrated/test/test.recal.cram.crai
       should_exist: false
     - path: results/reports/bcftools/sentieon_haplotyper/test/test.haplotyper.unfiltered.bcftools_stats.txt
-      md5sum: d2660f9f6074b4bf18756c42ee656b8f
     - path: results/reports/vcftools/sentieon_haplotyper/test/test.haplotyper.unfiltered.FILTER.summary
       md5sum: 01b3d10464a3ac86f90ee82cdda23f68
     - path: results/reports/vcftools/sentieon_haplotyper/test/test.haplotyper.unfiltered.TsTv.count
@@ -89,7 +86,6 @@
     - path: results/reports/mosdepth/test/test.recal.mosdepth.global.dist.txt
       md5sum: e82e90c7d508a135b5a8a7cd6933452e
     - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
-      md5sum: 4f0d231060cbde4efdd673863bd2fb59
     - path: results/reports/mosdepth/test/test.recal.per-base.bed.gz
       md5sum: bc1df47d46f818fee5275975925d769a
     - path: results/reports/mosdepth/test/test.recal.per-base.bed.gz.csi

workflows/sarek/main.nf

@@ -878,6 +878,7 @@ workflow SAREK {
         ch_methods_description                = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description))
         ch_multiqc_files                      = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml'))
         ch_multiqc_files                      = ch_multiqc_files.mix(version_yaml)
+        ch_multiqc_files                      = ch_multiqc_files.mix(reports)
         ch_multiqc_files                      = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: false))
 
         MULTIQC (