Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Make it possible to skip MultiQC #140

Merged
merged 10 commits into from
Aug 26, 2023
Merged

Make it possible to skip MultiQC #140

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7bf7d14
Make it possible to skip multiqc with `--skip_tools`
Aratz Aug 23, 2023
4df539f
Check multiqc report exists in other tests
Aratz Aug 24, 2023
9c92771
Fix old test
Aratz Aug 24, 2023
6664f01
Add test to check MultiQC can be skipped
Aratz Aug 24, 2023
e4a1403
Update snapshot
Aratz Aug 24, 2023
1a62427
Add description in schema for `skip_tools`
Aratz Aug 24, 2023
c74d5b7
Fix error raising
Aratz Aug 24, 2023
5ba20d8
Add contribution to changelog
Aratz Aug 24, 2023
14f0de7
Fix tests
Aratz Aug 24, 2023
c09b873
Merge branch 'dev' into dev
edmundmiller Aug 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### `Fixed`

- [#127](https://github.com/nf-core/demultiplex/pull/127) Add `singularity.registry = 'quay.io'` and bump NF version to 23.04.0
- [#140](https://github.com/nf-core/demultiplex/pull/140) Make it possible to skip MultiQC, fix error raising

## `Removed`

Expand Down
3 changes: 2 additions & 1 deletion nextflow_schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
},
"skip_tools": {
"type": "string",
"default": null
"default": null,
"description": "Comma-separated list of tools to skip (fastp,falco,multiqc)"
}
}
},
Expand Down
35 changes: 33 additions & 2 deletions tests/pipeline/skip_tools.nf.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ nextflow_pipeline {
{ assert workflow.success },
{ assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions_skip_trimming") },
{ assert workflow.trace.succeeded().size() == 6 },
{ assert path("$outputDir/multiqc/multiqc_report.html").exists() },
{ assert snapshot(
path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz.md5"),
Expand Down Expand Up @@ -49,6 +50,7 @@ nextflow_pipeline {
{ assert workflow.success },
{ assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions_skip_fastp") },
{ assert workflow.trace.succeeded().size() == 5 },
{ assert path("$outputDir/multiqc/multiqc_report.html").exists() },
{ assert snapshot(
path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz.md5"),
Expand Down Expand Up @@ -76,10 +78,10 @@ nextflow_pipeline {
{ assert workflow.success },
{ assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions_skip_fastqc") },
{ assert workflow.trace.succeeded().size() == 6 },
{ assert path("$outputDir/multiqc/multiqc_report.html").exists() },
{ assert snapshot(
path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
// FIXME
// path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz.md5"),
path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001.fastp.fastq.gz.md5"),
path("$outputDir/220422_M11111_0222_000000000-K9H97/Undetermined_S0_L001_R1_001.fastq.gz"),
path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
).match("skip_fastqc") }
Expand All @@ -104,6 +106,7 @@ nextflow_pipeline {
{ assert workflow.success },
{ assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions_skip_fastp_fastqc") },
{ assert workflow.trace.succeeded().size() == 5 },
{ assert path("$outputDir/multiqc/multiqc_report.html").exists() },
{ assert snapshot(
path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz.md5"),
Expand All @@ -115,4 +118,32 @@ nextflow_pipeline {

}

test("Skip MultiQC") {

when {
params {
input = 'https://raw.githubusercontent.com/nf-core/test-datasets/demultiplex/samplesheet/1.3.0/flowcell_input.csv'
demultiplexer = 'bclconvert'
outdir = "$outputDir"
skip_tools = "multiqc"
}
}

then {
assertAll(
{ assert workflow.success },
{ assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions_skip_multiqc") },
{ assert workflow.trace.succeeded().size() == 5 },
{ assert !path("$outputDir/multiqc/multiqc_report.html").exists() },
{ assert snapshot(
path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001.fastp.fastq.gz.md5"),
path("$outputDir/220422_M11111_0222_000000000-K9H97/Undetermined_S0_L001_R1_001.fastq.gz"),
path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
).match("skip_multiqc") }
)
}

}

}
45 changes: 37 additions & 8 deletions tests/pipeline/skip_tools.nf.test.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@
"content": [
"{BCLCONVERT={bclconvert=00.000.000.4.0.3}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FALCO={falco=1.2.1}, FASTP={fastp=0.23.4}, Workflow={nf-core/demultiplex=1.4.0dev}}"
],
"timestamp": "2023-06-15T00:02:11+0000"
"timestamp": "2023-08-24T07:43:09+0000"
},
"skip_fastqc": {
"content": [
"Sample1_S1_L001_R1_001.fastq.gz:md5,0a0341e2990b4fa1d9ad4b4c603144c1",
"Sample1_S1_L001.fastp.fastq.gz.md5:md5,c3cd96f3a22fb6afbaa1df324814d54c",
"Undetermined_S0_L001_R1_001.fastq.gz:md5,febef808ae5397ea4ee7ee40e5fd39e0",
[
"Adapter_Cycle_Metrics.csv:md5,5a0c88793b4a0885fe3dda16609b576e",
Expand All @@ -24,7 +25,7 @@
"fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
]
],
"timestamp": "2023-06-15T00:02:11+0000"
"timestamp": "2023-08-24T07:43:09+0000"
},
"skip_fastp": {
"content": [
Expand All @@ -46,7 +47,29 @@
"fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
]
],
"timestamp": "2023-06-15T00:02:11+0000"
"timestamp": "2023-08-24T07:43:09+0000"
},
"skip_multiqc": {
"content": [
"Sample1_S1_L001_R1_001.fastq.gz:md5,0a0341e2990b4fa1d9ad4b4c603144c1",
"Sample1_S1_L001.fastp.fastq.gz.md5:md5,c3cd96f3a22fb6afbaa1df324814d54c",
"Undetermined_S0_L001_R1_001.fastq.gz:md5,febef808ae5397ea4ee7ee40e5fd39e0",
[
"Adapter_Cycle_Metrics.csv:md5,5a0c88793b4a0885fe3dda16609b576e",
"Adapter_Metrics.csv:md5,989240b8840b2169ac1061f952c90f6c",
"Demultiplex_Stats.csv:md5,93949a8cd96f907d83e0808c1ec2a04b",
"Demultiplex_Tile_Stats.csv:md5,83120160b0f22a1303fa1db31c19f6e9",
"IndexMetricsOut.bin:md5,9e688c58a5487b8eaf69c9e1005ad0bf",
"Index_Hopping_Counts.csv:md5,1059369e375fd8f8423c0f6c934be978",
"Quality_Metrics.csv:md5,6614accb1bb414fe312b17b81f5521f7",
"Quality_Tile_Metrics.csv:md5,cdc89fd2962bdd4a24f71e186112118a",
"RunInfo.xml:md5,03038959f4dd181c86bc97ae71fe270a",
"SampleSheet.csv:md5,2df2e405991814571c021dc8749c2a89",
"Top_Unknown_Barcodes.csv:md5,2e2faba761137f228e56bd3428453ccc",
"fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
]
],
"timestamp": "2023-08-24T07:43:09+0000"
},
"skip_fastp_fastqc": {
"content": [
Expand All @@ -68,25 +91,31 @@
"fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
]
],
"timestamp": "2023-06-15T00:02:11+0000"
"timestamp": "2023-08-24T07:43:09+0000"
},
"software_versions_skip_fastqc": {
"content": [
"{BCLCONVERT={bclconvert=00.000.000.4.0.3}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FALCO={falco=1.2.1}, FASTP={fastp=0.23.4}, Workflow={nf-core/demultiplex=1.4.0dev}}"
],
"timestamp": "2023-06-15T00:02:11+0000"
"timestamp": "2023-08-24T07:43:09+0000"
},
"software_versions_skip_fastp_fastqc": {
"content": [
"{BCLCONVERT={bclconvert=00.000.000.4.0.3}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FALCO={falco=1.2.1}, Workflow={nf-core/demultiplex=1.4.0dev}}"
],
"timestamp": "2023-06-15T00:02:11+0000"
"timestamp": "2023-08-24T07:43:09+0000"
},
"software_versions_skip_multiqc": {
"content": [
"{BCLCONVERT={bclconvert=00.000.000.4.0.3}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FALCO={falco=1.2.1}, FASTP={fastp=0.23.4}, Workflow={nf-core/demultiplex=1.4.0dev}}"
],
"timestamp": "2023-08-24T07:43:09+0000"
},
"software_versions_skip_fastp": {
"content": [
"{BCLCONVERT={bclconvert=00.000.000.4.0.3}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FALCO={falco=1.2.1}, Workflow={nf-core/demultiplex=1.4.0dev}}"
],
"timestamp": "2023-06-15T00:02:11+0000"
"timestamp": "2023-08-24T07:43:09+0000"
},
"skip_trimming": {
"content": [
Expand All @@ -108,6 +137,6 @@
"fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
]
],
"timestamp": "2023-06-15T00:02:11+0000"
"timestamp": "2023-08-24T07:43:09+0000"
}
}
52 changes: 27 additions & 25 deletions workflows/demultiplex.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def checkPathParamList = [
for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true) } }

// Check mandatory parameters
if (params.input) { ch_input = file(params.input) } else { Nextflow.error 'Input samplesheet not specified!' }
if (params.input) { ch_input = file(params.input) } else { error 'Input samplesheet not specified!' }
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interested why you removed the Nextflow.errors?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good question, when I run it (e.g. by removing a column in the input samplesheet), I get No such variable: Nextflow, when I remove Nextflow I get the correct error message. Maybe you could try it on your side to confirm this is not because of some glitch in my setup?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What Nextflow version are you on? Here's the commit that changed it: d271419

@adamrtalbot Could you explain it?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm on 23.04.3.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@adamrtalbot Could you explain it?

Honestly, no! If it doesn't import Nextflow and error works then go for it! Could be a good chance to add an explicit test for correctly raising an error.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Other way around, @Aratz was finding Nextflow.error didn't work outside of a workflow context.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That would make sense, we have to import the Nextflow class in the groovy files I think...

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see the misunderstanding.

  • nextflow.Nextflow.error: Already imported into the namespace by Nextflow
  • nextflow.Nextflow: Not in namespace, but can be manually imported if you really want to (but really, don't).
  • nextflow: Not in namespace, but can be manually imported if you really want to (but really, don't).

For those that would prefer examples:

This will not work

Here because Nextflow is not in the namespace:

def testError() { 
    Nextflow.error "Exiting due to an error" 
}

workflow {
    testError()
}

The following will all work

Here we give the full namespace:

def testError() { 
    nextflow.Nextflow.error "Exiting due to an error" 
}

workflow {
    testError()
}

Here because we import Nextflow into the namespace:

import nextflow.Nextflow
def testError() { 
   Nextflow.error "Exiting due to an error" 
}

workflow {
    testError()
}

The two "working" examples are only provided for pedagogical purposes. They should always be abandoned in favour of the example below, where we rely on Nextflow importing nextflow.Nextflow.error into namespace for us:

and of course, the best and correct way:

def testError() { 
    error "Exiting due to an error" 
}

workflow {
    testError()
}

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Awesome explanation! Thanks for going through that!

Now do you want to PR it to the official Nextflow docs or Nextflow gotchas? 😁

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Haha, there's always a catch. Not quite sure if there's an obvious place in the Nextflow docs. Maybe a good home would be the "Groovy imports" module in the advanced training which is currently being merged into training.nextflow.io.

/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
CONFIG FILES
Expand Down Expand Up @@ -192,7 +192,7 @@ workflow DEMULTIPLEX {
ch_versions = ch_versions.mix(SINGULAR_DEMULTIPLEX.out.versions)
break
default:
Nextflow.error "Unknown demultiplexer: ${demultiplexer}"
error "Unknown demultiplexer: ${demultiplexer}"
}
ch_raw_fastq.dump(tag: "DEMULTIPLEX::Demultiplexed Fastq",{FormattingService.prettyFormat(it)})

Expand Down Expand Up @@ -229,24 +229,26 @@ workflow DEMULTIPLEX {
)

// MODULE: MultiQC
workflow_summary = WorkflowDemultiplex.paramsSummaryMultiqc(workflow, summary_params)
ch_workflow_summary = Channel.value(workflow_summary)

methods_description = WorkflowDemultiplex.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description)
ch_methods_description = Channel.value(methods_description)

ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml'))
ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml'))
ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect())
ch_multiqc_files.collect().dump(tag: "DEMULTIPLEX::MultiQC files",{FormattingService.prettyFormat(it)})

MULTIQC (
ch_multiqc_files.collect(),
ch_multiqc_config.toList(),
ch_multiqc_custom_config.toList(),
ch_multiqc_logo.toList()
)
multiqc_report = MULTIQC.out.report.toList()
if (!("multiqc" in skip_tools)){
workflow_summary = WorkflowDemultiplex.paramsSummaryMultiqc(workflow, summary_params)
ch_workflow_summary = Channel.value(workflow_summary)

methods_description = WorkflowDemultiplex.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description)
ch_methods_description = Channel.value(methods_description)

ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml'))
ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml'))
ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect())
ch_multiqc_files.collect().dump(tag: "DEMULTIPLEX::MultiQC files",{FormattingService.prettyFormat(it)})

MULTIQC (
ch_multiqc_files.collect(),
ch_multiqc_config.toList(),
ch_multiqc_custom_config.toList(),
ch_multiqc_logo.toList()
)
multiqc_report = MULTIQC.out.report.toList()
}
}

/*
Expand Down Expand Up @@ -321,14 +323,14 @@ def extract_csv(input_csv, input_schema=null) {
diff in all_columns ? missing_columns.add(diff) : wrong_columns.add(diff)
}
if(missing_columns.size() > 0){
Nextflow.error "[Samplesheet Error] The column(s) $missing_columns is/are not present. The header should look like: $all_columns"
error "[Samplesheet Error] The column(s) $missing_columns is/are not present. The header should look like: $all_columns"
}
else {
Nextflow.error "[Samplesheet Error] The column(s) $wrong_columns should not be in the header. The header should look like: $all_columns"
error "[Samplesheet Error] The column(s) $wrong_columns should not be in the header. The header should look like: $all_columns"
}
}
else {
Nextflow.error "[Samplesheet Error] The columns $row are not in the right order. The header should look like: $all_columns"
error "[Samplesheet Error] The columns $row are not in the right order. The header should look like: $all_columns"
}

}
Expand All @@ -345,7 +347,7 @@ def extract_csv(input_csv, input_schema=null) {
row[column] ?: missing_mandatory_columns.add(column)
}
if(missing_mandatory_columns.size > 0){
Nextflow.error "[Samplesheet Error] The mandatory column(s) $missing_mandatory_columns is/are empty on line $row_count"
error "[Samplesheet Error] The mandatory column(s) $missing_mandatory_columns is/are empty on line $row_count"
}

def output = []
Expand All @@ -355,7 +357,7 @@ def extract_csv(input_csv, input_schema=null) {
content = row[key]

if(!(content ==~ col.value['pattern']) && col.value['pattern'] != '' && content != '') {
Nextflow.error "[Samplesheet Error] The content of column '$key' on line $row_count does not match the pattern '${col.value['pattern']}'"
error "[Samplesheet Error] The content of column '$key' on line $row_count does not match the pattern '${col.value['pattern']}'"
}

if(col.value['content'] == 'path'){
Expand Down
Loading