Merge pull request #157 from Aratz/fix_publishdirs

Fix `bcl2fastq` and `bclconvert` publishDir
nf-core · Dec 11, 2023 · a766eb0 · a766eb0
2 parents ae1ec69 + 4553be6
commit a766eb0
Show file tree

Hide file tree

Showing 11 changed files with 113 additions and 66 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,7 +11,7 @@ on:
 env:
   NXF_ANSI_LOG: false
   CAPSULE_LOG: none
-  NFTEST_VER: "0.8.1"
+  NFTEST_VER: "0.8.2"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#140](https://github.com/nf-core/demultiplex/pull/140) Make it possible to skip MultiQC, fix error raising
 - [#145](https://github.com/nf-core/demultiplex/pull/145) Fix MultiQC report generation
 - [#152](https://github.com/nf-core/demultiplex/pull/152) Close [#150](https://github.com/nf-core/demultiplex/issues/150)
+- [#157](https://github.com/nf-core/demultiplex/pull/157) Fix bcl2fastq and bclconvert publishDir
 
 ## `Removed`
 

diff --git a/conf/modules.config b/conf/modules.config
@@ -38,14 +38,6 @@ process {
                 pattern: "**.bin",
                 saveAs: {filename -> filename.split("/")[-1] }
             ],
-            [
-                // Fetch RunInfo.xml
-                path: { "${params.outdir}/${meta.id}" },
-                mode: params.publish_dir_mode,
-                pattern: "RunInfo.xml",
-                overwrite: true,
-                saveAs: {filename -> filename.split("/")[-1] }
-            ],
             [
                 // Gather and write Reports
                 path: { meta.lane ? "${params.outdir}/${meta.id}/L00${meta.lane}" : "${params.outdir}/${meta.id}" },
@@ -72,19 +64,11 @@ process {
                 pattern: "**.bin",
                 saveAs: {filename -> filename.split("/")[-1] }
             ],
-            [
-                // Fetch RunInfo.xml
-                path: { "${params.outdir}/${meta.id}" },
-                mode: params.publish_dir_mode,
-                pattern: "RunInfo.xml",
-                overwrite: true,
-                saveAs: {filename -> filename.split("/")[-1] }
-            ],
             [
                 // Gather and write Reports and Stats
                 path: { meta.lane ? "${params.outdir}/${meta.id}/L00${meta.lane}" : "${params.outdir}/${meta.id}" },
                 mode: params.publish_dir_mode,
-                pattern: "{Reports, Stats}",
+                pattern: "{Reports,Stats}",
                 saveAs: {filename -> filename.split("/")[-1] }
             ],
             [

diff --git a/conf/test.config b/conf/test.config
@@ -40,14 +40,6 @@ process {
                 pattern: "**.bin",
                 saveAs: {filename -> filename.split("/")[-1] }
             ],
-            [
-                // Fetch RunInfo.xml
-                path: { "${params.outdir}/${meta.id}" },
-                mode: params.publish_dir_mode,
-                pattern: "RunInfo.xml",
-                overwrite: true,
-                saveAs: {filename -> filename.split("/")[-1] }
-            ],
             [
                 // Gather and write Reports
                 path: { meta.lane ? "${params.outdir}/${meta.id}/L00${meta.lane}" : "${params.outdir}/${meta.id}" },
@@ -66,5 +58,25 @@ process {
         ext.args = {[
             "--tiles s_1_1101",
         ].join(" ").trim()}
+        publishDir = [
+            [
+                // Gather and write InterOp files
+                path: { "${params.outdir}/${meta.id}/InterOp" },
+                mode: params.publish_dir_mode,
+                pattern: "**.bin",
+                saveAs: {filename -> filename.split("/")[-1] }
+            ],
+            [
+                // Gather and write Reports and Stats
+                path: { meta.lane ? "${params.outdir}/${meta.id}/L00${meta.lane}" : "${params.outdir}/${meta.id}" },
+                mode: params.publish_dir_mode,
+                pattern: "{Reports,Stats}",
+                saveAs: {filename -> filename.split("/")[-1] }
+            ],
+            [
+                path: { "${params.outdir}/${meta.id}" },
+                pattern: "**.fastq.gz",
+            ]
+        ]
     }
 }
diff --git a/conf/test_bcl2fastq.config b/conf/test_bcl2fastq.config
@@ -38,19 +38,11 @@ process {
                 pattern: "**.bin",
                 saveAs: {filename -> filename.split("/")[-1] }
             ],
-            [
-                // Fetch RunInfo.xml
-                path: { "${params.outdir}/${meta.id}" },
-                mode: params.publish_dir_mode,
-                pattern: "RunInfo.xml",
-                overwrite: true,
-                saveAs: {filename -> filename.split("/")[-1] }
-            ],
             [
                 // Gather and write Reports and Stats
                 path: { meta.lane ? "${params.outdir}/${meta.id}/L00${meta.lane}" : "${params.outdir}/${meta.id}" },
                 mode: params.publish_dir_mode,
-                pattern: "{Reports, Stats}",
+                pattern: "{Reports,Stats}",
                 saveAs: {filename -> filename.split("/")[-1] }
             ],
             [

diff --git a/tests/pipeline/bases2fastq.nf.test.snap b/tests/pipeline/bases2fastq.nf.test.snap
@@ -10,9 +10,9 @@
             "Metrics.csv:md5,87702e562aa1091e70e5d6e2e19f10b0",
             "RunManifest.json:md5,d4ecdb19e5fbcffab3e6e770c34023fb",
             "UnassignedSequences.csv:md5,11c1693830ce941b8cfb8d2431a59097",
-            "DefaultSample_R1.fastq.gz:md5,5b631c9bec4fe8227a467bfcc382654c",
-            "DefaultSample_R2.fastq.gz:md5,7318b86370fdbc87f989c45c73331e38"
+            "DefaultSample_R1.fastq.gz:md5,0cc292ca98e126f915a6c0ea9588b752",
+            "DefaultSample_R2.fastq.gz:md5,069b0ccb3069a47abd97c39377b10c34"
         ],
-        "timestamp": "2023-10-17T07:59:51+0000"
+        "timestamp": "2023-12-11T16:40:05.942646"
     }
 }
diff --git a/tests/pipeline/bcl2fastq.nf.test b/tests/pipeline/bcl2fastq.nf.test
@@ -21,11 +21,16 @@ nextflow_pipeline {
                 { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions") },
                 { assert workflow.trace.succeeded().size() == 6 },
                 { assert snapshot(
-                        // FIXME
-                        // path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz.md5"),
-
-                        path("$outputDir/bcl2fastq/Sample1_S1_L001_R1_001.fastq.gz"),
-                        path("$outputDir/bcl2fastq/Stats").list(),
+                        path("$outputDir/multiqc/multiqc_data/multiqc_bcl2fastq_bylane.txt"),
+                        path("$outputDir/multiqc/multiqc_data/multiqc_fastp.txt"),
+                        path("$outputDir/multiqc/multiqc_data/multiqc_bcl2fastq_bysample.txt")
+                    ).match("multiqc") },
+                { assert snapshot(
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001.fastp.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_summary.txt"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Stats/").list(),
                     ).match("bcl2fastq") }
             )
         }

diff --git a/tests/pipeline/bcl2fastq.nf.test.snap b/tests/pipeline/bcl2fastq.nf.test.snap
@@ -7,7 +7,50 @@
     },
     "bcl2fastq": {
         "content": [
-            "Sample1_S1_L001_R1_001.fastq.gz:md5,e92fce7b86c6447b053d72c5cb4be89c",
+            "Sample1_S1_L001.fastp.fastq.gz:md5,64639ff33d4b07d62a9079f88cbbd373",
+            "Sample1_S1_L001_R1_001.fastq.gz:md5,0675fb6365322eaafb33c0f8e862b54b",
+            "Sample1_S1_L001_summary.txt:md5,510c6b8a26ff8d530950c05b3ca5a1c6",
+            [
+                [
+                    [
+                        [
+                            [
+                                [
+                                    "lane.html:md5,794e48287f47a9f22dcb6b6d0c22c3eb",
+                                    "laneBarcode.html:md5,2bbdae3ee57151eab520c966597d7438"
+                                ],
+                                [
+
+                                ]
+                            ]
+                        ],
+                        [
+                            [
+                                [
+                                    "lane.html:md5,f741870307050dcea79a838f5971770f",
+                                    "laneBarcode.html:md5,ffe2e863811c76cb9da27d5d124e0611"
+                                ],
+                                [
+                                    "lane.html:md5,cca97e771d3491dbc8cd3fe389595b09",
+                                    "laneBarcode.html:md5,cca97e771d3491dbc8cd3fe389595b09"
+                                ]
+                            ],
+                            [
+                                [
+                                    "lane.html:md5,c383b0768d9978733d3f5d3b91c15af0",
+                                    "laneBarcode.html:md5,48842c23b9a2816aec540177df870968"
+                                ],
+                                [
+
+                                ]
+                            ]
+                        ]
+                    ],
+                    "Report.css:md5,eb7d3eb68fc1539f411404987246b59b",
+                    "index.html:md5,5747c407854ae2c358d0ec201ce622d8",
+                    "tree.html:md5,a1b9bf592973ca829ec69ddf888b7e34"
+                ]
+            ],
             [
                 "AdapterTrimming.txt:md5,48ed2b914b1246c0b5d8667525550946",
                 "ConversionStats.xml:md5,8fe0f57f3f5d256a0762dba75ac62d05",
@@ -17,6 +60,14 @@
                 "Stats.json:md5,8e5f038b8aa9e465599d3575f930e604"
             ]
         ],
-        "timestamp": "2023-10-17T07:58:06+0000"
+        "timestamp": "2023-12-11T16:37:27.684322"
+    },
+    "multiqc": {
+        "content": [
+            "multiqc_bcl2fastq_bylane.txt:md5,3c90d50fec9eb01e32c0f9d7e8295d85",
+            "multiqc_fastp.txt:md5,76b067a76f4f58759526ca2aa4d6679d",
+            "multiqc_bcl2fastq_bysample.txt:md5,1865f27f430d153085b26c86320718fe"
+        ],
+        "timestamp": "2023-12-11T16:24:41.682929"
     }
 }
diff --git a/tests/pipeline/bclconvert.nf.test b/tests/pipeline/bclconvert.nf.test
@@ -20,11 +20,13 @@ nextflow_pipeline {
                 { assert workflow.success },
                 { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions") },
                 { assert workflow.trace.succeeded().size() == 6 },
-                { assert snapshot(path("$outputDir/multiqc/multiqc_data/multiqc_bclconvert_bylane.txt"),
+                { assert snapshot(
+                        path("$outputDir/multiqc/multiqc_data/multiqc_bclconvert_bylane.txt"),
                         path("$outputDir/multiqc/multiqc_data/multiqc_fastp.txt"),
                         path("$outputDir/multiqc/multiqc_data/multiqc_bclconvert_bysample.txt")
                     ).match("multiqc") },
-                { assert snapshot(path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001.fastp.fastq.gz"),
+                { assert snapshot(
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001.fastp.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_summary.txt"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Undetermined_S0_L001_R1_001.fastq.gz"),

diff --git a/tests/pipeline/bclconvert.nf.test.snap b/tests/pipeline/bclconvert.nf.test.snap
@@ -1,10 +1,10 @@
 {
     "bclconvert": {
         "content": [
-            "Sample1_S1_L001.fastp.fastq.gz:md5,8a723787b4a0c0fbf5b81d3a0b989c69",
-            "Sample1_S1_L001_R1_001.fastq.gz:md5,a0c1359a2c922967660524a6e77db494",
+            "Sample1_S1_L001.fastp.fastq.gz:md5,64639ff33d4b07d62a9079f88cbbd373",
+            "Sample1_S1_L001_R1_001.fastq.gz:md5,0675fb6365322eaafb33c0f8e862b54b",
             "Sample1_S1_L001_summary.txt:md5,510c6b8a26ff8d530950c05b3ca5a1c6",
-            "Undetermined_S0_L001_R1_001.fastq.gz:md5,163be0a88c70ca629fd516dbaadad96a",
+            "Undetermined_S0_L001_R1_001.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e",
             [
                 "Adapter_Cycle_Metrics.csv:md5,5a0c88793b4a0885fe3dda16609b576e",
                 "Adapter_Metrics.csv:md5,989240b8840b2169ac1061f952c90f6c",
@@ -20,7 +20,7 @@
                 "fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
             ]
         ],
-        "timestamp": "2023-10-31T12:33:16.768754"
+        "timestamp": "2023-12-11T16:38:54.135338"
     },
     "software_versions": {
         "content": [

diff --git a/tests/pipeline/skip_tools.nf.test.snap b/tests/pipeline/skip_tools.nf.test.snap
@@ -7,9 +7,9 @@
     },
     "skip_fastqc": {
         "content": [
-            "Sample1_S1_L001_R1_001.fastq.gz:md5,a0c1359a2c922967660524a6e77db494",
+            "Sample1_S1_L001_R1_001.fastq.gz:md5,0675fb6365322eaafb33c0f8e862b54b",
             "Sample1_S1_L001.fastp.fastq.gz.md5:md5,c3cd96f3a22fb6afbaa1df324814d54c",
-            "Undetermined_S0_L001_R1_001.fastq.gz:md5,163be0a88c70ca629fd516dbaadad96a",
+            "Undetermined_S0_L001_R1_001.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e",
             [
                 "Adapter_Cycle_Metrics.csv:md5,5a0c88793b4a0885fe3dda16609b576e",
                 "Adapter_Metrics.csv:md5,989240b8840b2169ac1061f952c90f6c",
@@ -25,13 +25,13 @@
                 "fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
             ]
         ],
-        "timestamp": "2023-10-31T12:41:59.969786"
+        "timestamp": "2023-12-11T16:45:17.736105"
     },
     "skip_fastp": {
         "content": [
-            "Sample1_S1_L001_R1_001.fastq.gz:md5,a0c1359a2c922967660524a6e77db494",
+            "Sample1_S1_L001_R1_001.fastq.gz:md5,0675fb6365322eaafb33c0f8e862b54b",
             "Sample1_S1_L001_R1_001.fastq.gz.md5:md5,10295d4e5ca66dae7788bbf24886cded",
-            "Undetermined_S0_L001_R1_001.fastq.gz:md5,163be0a88c70ca629fd516dbaadad96a",
+            "Undetermined_S0_L001_R1_001.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e",
             [
                 "Adapter_Cycle_Metrics.csv:md5,5a0c88793b4a0885fe3dda16609b576e",
                 "Adapter_Metrics.csv:md5,989240b8840b2169ac1061f952c90f6c",
@@ -47,13 +47,13 @@
                 "fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
             ]
         ],
-        "timestamp": "2023-10-31T12:40:44.801689"
+        "timestamp": "2023-12-11T16:43:54.005346"
     },
     "skip_multiqc": {
         "content": [
-            "Sample1_S1_L001_R1_001.fastq.gz:md5,a0c1359a2c922967660524a6e77db494",
+            "Sample1_S1_L001_R1_001.fastq.gz:md5,0675fb6365322eaafb33c0f8e862b54b",
             "Sample1_S1_L001.fastp.fastq.gz.md5:md5,c3cd96f3a22fb6afbaa1df324814d54c",
-            "Undetermined_S0_L001_R1_001.fastq.gz:md5,163be0a88c70ca629fd516dbaadad96a",
+            "Undetermined_S0_L001_R1_001.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e",
             [
                 "Adapter_Cycle_Metrics.csv:md5,5a0c88793b4a0885fe3dda16609b576e",
                 "Adapter_Metrics.csv:md5,989240b8840b2169ac1061f952c90f6c",
@@ -69,13 +69,13 @@
                 "fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
             ]
         ],
-        "timestamp": "2023-10-31T12:43:37.155748"
+        "timestamp": "2023-12-11T16:47:17.03681"
     },
     "skip_fastp_fastqc": {
         "content": [
-            "Sample1_S1_L001_R1_001.fastq.gz:md5,a0c1359a2c922967660524a6e77db494",
+            "Sample1_S1_L001_R1_001.fastq.gz:md5,0675fb6365322eaafb33c0f8e862b54b",
             "Sample1_S1_L001_R1_001.fastq.gz.md5:md5,10295d4e5ca66dae7788bbf24886cded",
-            "Undetermined_S0_L001_R1_001.fastq.gz:md5,163be0a88c70ca629fd516dbaadad96a",
+            "Undetermined_S0_L001_R1_001.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e",
             [
                 "Adapter_Cycle_Metrics.csv:md5,5a0c88793b4a0885fe3dda16609b576e",
                 "Adapter_Metrics.csv:md5,989240b8840b2169ac1061f952c90f6c",
@@ -91,7 +91,7 @@
                 "fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
             ]
         ],
-        "timestamp": "2023-10-31T12:42:35.089011"
+        "timestamp": "2023-12-11T16:46:00.460287"
     },
     "software_versions_skip_fastqc": {
         "content": [
@@ -119,9 +119,9 @@
     },
     "skip_trimming": {
         "content": [
-            "Sample1_S1_L001_R1_001.fastq.gz:md5,a0c1359a2c922967660524a6e77db494",
+            "Sample1_S1_L001_R1_001.fastq.gz:md5,0675fb6365322eaafb33c0f8e862b54b",
             "Sample1_S1_L001_R1_001.fastq.gz.md5:md5,10295d4e5ca66dae7788bbf24886cded",
-            "Undetermined_S0_L001_R1_001.fastq.gz:md5,163be0a88c70ca629fd516dbaadad96a",
+            "Undetermined_S0_L001_R1_001.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e",
             [
                 "Adapter_Cycle_Metrics.csv:md5,5a0c88793b4a0885fe3dda16609b576e",
                 "Adapter_Metrics.csv:md5,989240b8840b2169ac1061f952c90f6c",
@@ -137,6 +137,6 @@
                 "fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
             ]
         ],
-        "timestamp": "2023-10-31T12:40:12.565447"
+        "timestamp": "2023-12-11T16:43:15.513271"
     }
 }