Np pd 2505 demultiplexing file existence (#1352)

* pin all latest docker version * warning for batch size * unbound var * unbound var again * echo total files * remove batch number from input json and default it to 6 * remove batch number from input json and default it to 6 * remove batch number from input json and default it to 6 * Updated pipeline_versions.txt with all pipeline version information * Updated pipeline_versions.txt with all pipeline version information * changelogs * Updated pipeline_versions.txt with all pipeline version information * Update pipelines/skylab/snm3C/snm3C.changelog.md Co-authored-by: ekiernan <[email protected]> * Updated pipeline_versions.txt with all pipeline version information * Updated pipeline_versions.txt with all pipeline version information --------- Co-authored-by: GitHub Action <[email protected]> Co-authored-by: ekiernan <[email protected]>
broadinstitute · Aug 7, 2024 · 9b5e498 · 9b5e498
1 parent 2b75a2e
commit 9b5e498
Show file tree

Hide file tree

Showing 6 changed files with 46 additions and 36 deletions.
diff --git a/pipeline_versions.txt b/pipeline_versions.txt
@@ -1,42 +1,42 @@
 Pipeline Name	Version	Date of Last Commit
+Optimus	 7.6.0	2024-08-06 
 Multiome	 5.5.0	2024-08-06 
+PairedTag	 1.5.0	2024-08-06 
+atac	 2.2.3	2024-08-02 
+SlideSeq	 3.4.0	2024-08-06 
+snm3C	 4.0.3	2024-08-05 
 MultiSampleSmartSeq2SingleNucleus	 1.4.2	2024-08-25-02 
+scATAC	 1.3.2	2023-08-03 
+SmartSeq2SingleSample	 5.1.20	2023-04-19 
 BuildIndices	 3.0.0	2023-12-06 
 MultiSampleSmartSeq2	 2.2.21	2023-04-19 
-SmartSeq2SingleSample	 5.1.20	2023-04-19 
-scATAC	 1.3.2	2023-08-03 
-SlideSeq	 3.4.0	2024-08-06 
-snm3C	 4.0.2	2024-07-09 
-PairedTag	 1.5.0	2024-08-06 
-Optimus	 7.6.0	2024-08-06 
-atac	 2.2.3	2024-08-02 
-ExomeReprocessing	 3.2.2	2024-08-02 
-ExternalExomeReprocessing	 3.2.2	2024-08-02 
-ExternalWholeGenomeReprocessing	 2.2.2	2024-08-02 
-WholeGenomeReprocessing	 3.2.2	2024-08-02 
-CramToUnmappedBams	 1.1.3	2024-08-02 
-GDCWholeGenomeSomaticSingleSample	 1.3.2	2024-08-02  
+CEMBA	 1.1.6	2023-12-18 
+BuildCembaReferences	 1.0.0	2020-11-15 
 UltimaGenomicsWholeGenomeCramOnly	 1.0.20	2024-08-02 
-ReblockGVCF	 2.2.1	2024-06-12 
-JointGenotypingByChromosomePartTwo	 1.4.11	2023-12-18 
-JointGenotypingByChromosomePartOne	 1.4.12	2023-12-18 
-JointGenotyping	 1.6.10	2023-12-18 
-UltimaGenomicsJointGenotyping	 1.1.7	2023-12-18 
+GDCWholeGenomeSomaticSingleSample	 1.3.2	2024-08-02  
 ExomeGermlineSingleSample	 3.1.22	2024-06-12 
-WholeGenomeGermlineSingleSample	 3.2.1	2024-06-12 
 UltimaGenomicsWholeGenomeGermline	 1.0.20	2024-08-02 
+WholeGenomeGermlineSingleSample	 3.2.1	2024-06-12 
 VariantCalling	 2.2.1	2024-06-12 
-CheckFingerprint	 1.0.20	2024-08-02 
+UltimaGenomicsJointGenotyping	 1.1.7	2023-12-18 
+JointGenotyping	 1.6.10	2023-12-18 
+ReblockGVCF	 2.2.1	2024-06-12 
+JointGenotypingByChromosomePartTwo	 1.4.11	2023-12-18 
+JointGenotypingByChromosomePartOne	 1.4.12	2023-12-18 
+ExternalExomeReprocessing	 3.2.2	2024-08-02 
+ExternalWholeGenomeReprocessing	 2.2.2	2024-08-02 
+ExomeReprocessing	 3.2.2	2024-08-02 
+CramToUnmappedBams	 1.1.3	2024-08-02 
+WholeGenomeReprocessing	 3.2.2	2024-08-02 
 IlluminaGenotypingArray	 1.12.21	2024-08-02 
-RNAWithUMIsPipeline	 1.0.16	2023-12-18 
-Imputation	 1.1.13	2024-05-21 
-MultiSampleArrays	 1.6.2	2024-08-02 
 Arrays	 2.6.27	2024-08-02 
+MultiSampleArrays	 1.6.2	2024-08-02 
 ValidateChip	 1.16.5	2024-08-02 
+Imputation	 1.1.13	2024-05-21 
+RNAWithUMIsPipeline	 1.0.16	2023-12-18 
 BroadInternalUltimaGenomics	 1.0.21	2024-08-02 
-BroadInternalRNAWithUMIs	 1.0.33	2024-08-02 
-BroadInternalImputation	 1.1.12	2024-08-02 
 BroadInternalArrays	 1.1.11	2024-08-02 
+BroadInternalImputation	 1.1.12	2024-08-02 
+BroadInternalRNAWithUMIs	 1.0.33	2024-08-02 
+CheckFingerprint	 1.0.20	2024-08-02 
 AnnotationFiltration	 1.2.5	2023-12-18 
-CEMBA	 1.1.6	2023-12-18 
-BuildCembaReferences	 1.0.0	2020-11-15 
diff --git a/pipelines/skylab/snm3C/snm3C.changelog.md b/pipelines/skylab/snm3C/snm3C.changelog.md
@@ -1,3 +1,8 @@
+# 4.0.3
+2024-08-05 (Date of Last Commit)
+
+* Updated the demultiplexing task in snm3C wdl to dynamically update the batch number based on the number of fastq files present
+
 # 4.0.2
 2024-07-09 (Date of Last Commit)
 

diff --git a/pipelines/skylab/snm3C/snm3C.wdl b/pipelines/skylab/snm3C/snm3C.wdl
@@ -25,7 +25,7 @@ workflow snm3C {
         Int num_upstr_bases = 0
         Int num_downstr_bases = 2
         Int compress_level = 5
-        Int batch_number
+        Int batch_number = 6
     }
     #docker images
     String m3c_yap_hisat_docker = "m3c-yap-hisat:2.4"
@@ -44,7 +44,7 @@ workflow snm3C {
     }
 
     # version of the pipeline
-    String pipeline_version = "4.0.2"
+    String pipeline_version = "4.0.3"
 
     call Demultiplexing {
         input:
@@ -213,7 +213,18 @@ task Demultiplexing {
     CODE
 
     # Batch the fastq files into folders of batch_number size
+    R1_files=($(ls $WORKING_DIR | grep "\-R1.fq.gz"))
+    R2_files=($(ls $WORKING_DIR | grep "\-R2.fq.gz"))
+    total_files=${#R1_files[@]}
+    echo "Total files: $total_files"
+
     batch_number=~{batch_number}
+
+    if [[ $total_files -lt $batch_number ]]; then
+        echo "Warning: Number of files is less than the batch number. Updating batch number to $total_files."
+        batch_number=$total_files
+    fi
+
     for i in $(seq 1 "${batch_number}"); do  # Use seq for reliable brace expansion
         mkdir -p "batch${i}"  # Combine batch and i, use -p to create parent dirs
     done
@@ -222,10 +233,6 @@ task Demultiplexing {
     folder_index=1
     WORKING_DIR=`pwd`
 
-    # Define lists of r1 and r2 fq files
-    R1_files=($(ls $WORKING_DIR | grep "\-R1.fq.gz"))
-    R2_files=($(ls $WORKING_DIR | grep "\-R2.fq.gz"))
-
     # Distribute the FASTQ files and create TAR files
     for file in "${R1_files[@]}"; do
         sample_id=$(basename "$file" "-R1.fq.gz")

diff --git a/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json b/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json
@@ -16,7 +16,6 @@
   "snm3C.tarred_index_files":"gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38_index_files.tar.gz",
   "snm3C.chromosome_sizes": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.chrom.sizes",
   "snm3C.genome_fa": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.fa",
-  "snm3C.batch_number": 2,
   "snm3C.Hisat_paired_end.cpu_platform" : "Intel Cascade Lake",
   "snm3C.Hisat_single_end.cpu_platform" : "Intel Cascade Lake",
   "snm3C.Merge_sort_analyze.cpu_platform" : "Intel Cascade Lake",

diff --git a/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json b/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json
@@ -16,7 +16,6 @@
   "snm3C.tarred_index_files":"gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38_index_files.tar.gz",
   "snm3C.chromosome_sizes": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.chrom.sizes",
   "snm3C.genome_fa": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.fa",
-  "snm3C.batch_number": 2,
   "snm3C.Hisat_paired_end.cpu_platform" : "Intel Cascade Lake",
   "snm3C.Hisat_single_end.cpu_platform" : "Intel Cascade Lake",
   "snm3C.Merge_sort_analyze.cpu_platform" : "Intel Cascade Lake",

diff --git a/verification/test-wdls/Testsnm3C.wdl b/verification/test-wdls/Testsnm3C.wdl
@@ -27,7 +27,7 @@ workflow Testsnm3C {
       Int num_upstr_bases = 0
       Int num_downstr_bases = 2
       Int compress_level = 5
-      Int batch_number
+      Int batch_number = 6
 
       # These values will be determined and injected into the inputs by the scala test framework
       String truth_path