Merge branch 'master' into ct-add-delphy-task-and-workflow

broadinstitute · Nov 5, 2024 · 12347ab · 12347ab
2 parents 53579e1 + d655726
commit 12347ab
Show file tree

Hide file tree

Showing 17 changed files with 113 additions and 110 deletions.
diff --git a/pipes/WDL/tasks/tasks_assembly.wdl b/pipes/WDL/tasks/tasks_assembly.wdl
@@ -15,7 +15,7 @@ task assemble {
       String   sample_name = basename(basename(reads_unmapped_bam, ".bam"), ".taxfilt")
 
       Int?     machine_mem_gb
-      String   docker = "quay.io/broadinstitute/viral-assemble:2.3.2.0"
+      String   docker = "quay.io/broadinstitute/viral-assemble:2.3.6.1"
     }
     parameter_meta{
       reads_unmapped_bam: {
@@ -115,7 +115,7 @@ task select_references {
     Int?          skani_s
     Int?          skani_c
 
-    String        docker = "quay.io/broadinstitute/viral-assemble:2.3.2.0"
+    String        docker = "quay.io/broadinstitute/viral-assemble:2.3.6.1"
     Int           machine_mem_gb = 4
     Int           cpu = 2
     Int           disk_size = 100
@@ -206,7 +206,7 @@ task scaffold {
       Float?       scaffold_min_pct_contig_aligned
 
       Int?         machine_mem_gb
-      String       docker="quay.io/broadinstitute/viral-assemble:2.3.2.0"
+      String       docker="quay.io/broadinstitute/viral-assemble:2.3.6.1"
 
       # do this in multiple steps in case the input doesn't actually have "assembly1-x" in the name
       String       sample_name = basename(basename(contigs_fasta, ".fasta"), ".assembly1-spades")
@@ -583,7 +583,7 @@ task align_reads {
     Boolean  skip_mark_dupes = false
 
     Int?     machine_mem_gb
-    String   docker = "quay.io/broadinstitute/viral-core:2.3.2"
+    String   docker = "quay.io/broadinstitute/viral-core:2.4.0"
 
     String   sample_name = basename(basename(basename(reads_unmapped_bam, ".bam"), ".taxfilt"), ".clean")
   }
@@ -720,7 +720,7 @@ task refine_assembly_with_aligned_reads {
       Int      min_coverage = 3
 
       Int      machine_mem_gb = 15
-      String   docker = "quay.io/broadinstitute/viral-assemble:2.3.2.0"
+      String   docker = "quay.io/broadinstitute/viral-assemble:2.3.6.1"
     }
 
     Int disk_size = 375
@@ -845,7 +845,7 @@ task refine_2x_and_plot {
       String? plot_coverage_novoalign_options = "-r Random -l 40 -g 40 -x 20 -t 100 -k"
 
       Int?    machine_mem_gb
-      String  docker = "quay.io/broadinstitute/viral-assemble:2.3.2.0"
+      String  docker = "quay.io/broadinstitute/viral-assemble:2.3.6.1"
 
       # do this in two steps in case the input doesn't actually have "cleaned" in the name
       String  sample_name = basename(basename(reads_unmapped_bam, ".bam"), ".cleaned")
@@ -981,7 +981,7 @@ task run_discordance {
       String out_basename = "run"
       Int    min_coverage = 4
 
-      String docker = "quay.io/broadinstitute/viral-core:2.3.2"
+      String docker = "quay.io/broadinstitute/viral-core:2.4.0"
     }
     parameter_meta {
       reads_aligned_bam: {

diff --git a/pipes/WDL/tasks/tasks_demux.wdl b/pipes/WDL/tasks/tasks_demux.wdl
@@ -6,7 +6,7 @@ task merge_tarballs {
     String       out_filename
 
     Int?         machine_mem_gb
-    String       docker = "quay.io/broadinstitute/viral-core:2.3.2"
+    String       docker = "quay.io/broadinstitute/viral-core:2.4.0"
   }
 
   Int disk_size = 2625
@@ -163,7 +163,7 @@ task illumina_demux {
 
     Int?    machine_mem_gb
     Int     disk_size = 2625
-    String  docker = "quay.io/broadinstitute/viral-core:2.3.2"
+    String  docker = "quay.io/broadinstitute/viral-core:2.4.0"
   }
 
   parameter_meta {

diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl
@@ -160,7 +160,7 @@ task multi_align_mafft_ref {
     Float?       mafft_gapOpeningPenalty
 
     Int?         machine_mem_gb
-    String       docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+    String       docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
   }
 
   String         fasta_basename = basename(reference_fasta, '.fasta')
@@ -207,7 +207,7 @@ task multi_align_mafft {
     Float?       mafft_gapOpeningPenalty
 
     Int?         machine_mem_gb
-    String       docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+    String       docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
   }
 
   Int disk_size = 200
@@ -351,7 +351,7 @@ task index_ref {
     File?  novocraft_license
 
     Int?   machine_mem_gb
-    String docker = "quay.io/broadinstitute/viral-core:2.3.2"
+    String docker = "quay.io/broadinstitute/viral-core:2.4.0"
   }
 
   Int disk_size = 100
@@ -476,7 +476,7 @@ task merge_vcfs_gatk {
     File        ref_fasta
 
     Int?        machine_mem_gb
-    String      docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+    String      docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
 
     String      output_prefix = "merged"
   }

diff --git a/pipes/WDL/tasks/tasks_intrahost.wdl b/pipes/WDL/tasks/tasks_intrahost.wdl
@@ -136,7 +136,7 @@ task lofreq {
     File      reference_fasta
 
     String    out_basename = basename(aligned_bam, '.bam')
-    String    docker = "quay.io/biocontainers/lofreq:2.1.5--py38h588ecb2_4"
+    String    docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
   }
   Int disk_size = 200
   command <<<
@@ -145,8 +145,13 @@ task lofreq {
     lofreq version | grep version | sed 's/.* \(.*\)/\1/g' | tee LOFREQ_VERSION
 
     # make local copies because CWD is writeable but localization dir isn't always
-    cp "~{reference_fasta}" reference.fasta
     cp "~{aligned_bam}" aligned.bam
+    python3<<CODE
+    import shutil
+    import util.file
+    with util.file.fastas_with_sanitized_ids("~{reference_fasta}", use_tmp=True) as sanitized_fastas:
+        shutil.copyfile(sanitized_fastas[0], 'reference.fasta')
+    CODE
 
     # samtools faidx fails if fasta is empty
     if [ $(grep -v '^>' reference.fasta | tr -d '\nNn' | wc -c) == "0" ]; then
@@ -191,7 +196,7 @@ task isnvs_per_sample {
     Boolean removeDoublyMappedReads = true
 
     Int?    machine_mem_gb
-    String  docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+    String  docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
 
     String  sample_name = basename(basename(basename(mapped_bam, ".bam"), ".all"), ".mapped")
   }
@@ -234,7 +239,7 @@ task isnvs_vcf {
     Boolean        naiveFilter = false
 
     Int?           machine_mem_gb
-    String         docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+    String         docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
   }
 
   parameter_meta {
@@ -308,7 +313,7 @@ task annotate_vcf_snpeff {
     String?        emailAddress
 
     Int?           machine_mem_gb
-    String         docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+    String         docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
 
     String         output_basename = basename(basename(in_vcf, ".gz"), ".vcf")
   }

diff --git a/pipes/WDL/tasks/tasks_megablast.wdl b/pipes/WDL/tasks/tasks_megablast.wdl
@@ -15,7 +15,7 @@ task trim_rmdup_subsamp {
         Int cpu            = 16
         Int disk_size_gb   = 100 
 
-        String docker      = "quay.io/broadinstitute/viral-assemble:2.3.2.0"
+        String docker      = "quay.io/broadinstitute/viral-assemble:2.3.6.1"
     }
 
     parameter_meta {
@@ -36,7 +36,7 @@ task trim_rmdup_subsamp {
     command <<<
         set -ex o pipefail
         assembly.py --version | tee VERSION
-        #BAM ->FASTQ-> OutBam? https://github.com/broadinstitute/viral-assemble/blob/80bcc1da5c6a0174362ca9fd8bc0b49ee0b4103b/assembly.py#L91
+        #BAM ->FASTQ-> OutBam? https://github.com/broadinstitute/viral-assemble:2.3.6.1
         assembly.py trim_rmdup_subsamp \
         "~{inBam}" \
         "~{clipDb}" \

diff --git a/pipes/WDL/tasks/tasks_ncbi.wdl b/pipes/WDL/tasks/tasks_ncbi.wdl
@@ -6,7 +6,7 @@ task download_fasta {
     Array[String]+ accessions
     String         emailAddress
 
-    String         docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+    String         docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
   }
 
   command {
@@ -38,7 +38,7 @@ task download_annotations {
     String         emailAddress
     String         combined_out_prefix
 
-    String         docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+    String         docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
   }
 
   command <<<
@@ -85,7 +85,7 @@ task annot_transfer {
     File         reference_fasta
     Array[File]+ reference_feature_table
 
-    String       docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+    String       docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
   }
 
   parameter_meta {
@@ -139,7 +139,7 @@ task align_and_annot_transfer_single {
     Array[File]+ reference_fastas
     Array[File]+ reference_feature_tables
 
-    String       docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+    String       docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
   }
 
   parameter_meta {
@@ -192,7 +192,7 @@ task structured_comments {
 
     File?  filter_to_ids
 
-    String docker = "quay.io/broadinstitute/viral-core:2.3.2"
+    String docker = "quay.io/broadinstitute/viral-core:2.4.0"
   }
   String out_base = basename(assembly_stats_tsv, '.txt')
   command <<<
@@ -272,7 +272,7 @@ task rename_fasta_header {
 
     String out_basename = basename(genome_fasta, ".fasta")
 
-    String docker = "quay.io/broadinstitute/viral-core:2.3.2"
+    String docker = "quay.io/broadinstitute/viral-core:2.4.0"
   }
   command {
     set -e
@@ -437,7 +437,7 @@ task sra_meta_prep {
     Boolean     paired
 
     String      out_name = "sra_metadata.tsv"
-    String      docker="quay.io/broadinstitute/viral-core:2.3.2"
+    String      docker="quay.io/broadinstitute/viral-core:2.4.0"
   }
   Int disk_size = 100
   parameter_meta {
@@ -653,7 +653,7 @@ task biosample_to_genbank {
     File?   filter_to_ids
 
     Boolean s_dropout_note = true
-    String  docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+    String  docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
   }
   String base = basename(biosample_attributes, ".txt")
   command {
@@ -851,7 +851,7 @@ task prepare_genbank {
     String?      assembly_method_version
 
     Int?         machine_mem_gb
-    String       docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+    String       docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
   }
 
   parameter_meta {

diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl
@@ -11,9 +11,15 @@ task taxid_to_nextclade_dataset_name {
             2697049 : 'sars-cov-2',
             641809  : 'flu_h1n1pdm_ha',
             335341  : 'flu_h3n2_ha',
+            119210  : 'flu_h3n2_ha',
             518987  : 'flu_vic_ha',
             208893  : 'rsv_a',
             208895  : 'rsv_b',
+            11234   : 'nextstrain/measles/N450/WHO-2012',
+            11053   : 'nextstrain/dengue/all',
+            11060   : 'nextstrain/dengue/all',
+            11069   : 'nextstrain/dengue/all',
+            11070   : 'nextstrain/dengue/all',
             10244   : 'MPXV',
             619591  : 'hMPXV'
         }
@@ -43,13 +49,11 @@ task nextclade_one_sample {
         File  genome_fasta
         File? root_sequence
         File? auspice_reference_tree_json
-        File? qc_config_json
+        File? pathogen_json
         File? gene_annotations_json
-        File? pcr_primers_csv
-        File? virus_properties
         String? dataset_name
         Int    disk_size = 50
-        String docker = "nextstrain/nextclade:2.14.0"
+        String docker = "nextstrain/nextclade:3.9.1"
     }
     String basename = basename(genome_fasta, ".fasta")
     command <<<
@@ -67,21 +71,19 @@ task nextclade_one_sample {
             DATASET_ARG="--input-dataset ."
         python3<<CODE1
         import json, os
-        with open('tag.json', 'rt') as inf:
+        with open('pathogen.json', 'rt') as inf:
             datasetinfo = json.load(inf)
         with open('VERSION', 'wt') as outf:
-            outf.write(os.environ['NEXTCLADE_VERSION'] + "; name=" + datasetinfo['name'] + "; tag=" + datasetinfo['tag'] + "\n")
+            outf.write(os.environ['NEXTCLADE_VERSION'] + "; name=" + datasetinfo['attributes']['name'] + "; tag=" + datasetinfo['version']['tag'] + "\n")
         CODE1
         fi
 
         nextclade run \
             $DATASET_ARG \
-            ~{"--input-root-seq " + root_sequence} \
+            ~{"--input-ref " + root_sequence} \
             ~{"--input-tree " + auspice_reference_tree_json} \
-            ~{"--input-qc-config " + qc_config_json} \
-            ~{"--input-gene-map " + gene_annotations_json} \
-            ~{"--input-pcr-primers " + pcr_primers_csv} \
-            ~{"--input-virus-properties " + virus_properties}  \
+            ~{"--input-annotation " + gene_annotations_json} \
+            ~{"--input-pathogen-json " + pathogen_json}  \
             --output-all=. \
             --output-basename "~{basename}" \
             --output-json "~{basename}".nextclade.json \
@@ -136,15 +138,13 @@ task nextclade_many_samples {
         Array[File]+ genome_fastas
         File?        root_sequence
         File?        auspice_reference_tree_json
-        File?        qc_config_json
+        File?        pathogen_json
         File?        gene_annotations_json
-        File?        pcr_primers_csv
-        File?        virus_properties
         String?      dataset_name
         String       basename
         File?        genome_ids_setdefault_blank
         Int          disk_size = 150
-        String       docker = "nextstrain/nextclade:2.14.0"
+        String       docker = "nextstrain/nextclade:3.9.1"
     }
     command <<<
         set -e
@@ -170,12 +170,10 @@ task nextclade_many_samples {
 
         nextclade run \
             $DATASET_ARG \
-            ~{"--input-root-seq " + root_sequence} \
+            ~{"--input-ref " + root_sequence} \
             ~{"--input-tree " + auspice_reference_tree_json} \
-            ~{"--input-qc-config " + qc_config_json} \
-            ~{"--input-gene-map " + gene_annotations_json} \
-            ~{"--input-pcr-primers " + pcr_primers_csv} \
-            ~{"--input-virus-properties " + virus_properties}  \
+            ~{"--input-annotation " + gene_annotations_json} \
+            ~{"--input-pathogen-json " + pathogen_json}  \
             --output-all=. \
             --output-basename "~{basename}" \
             --output-json "~{basename}".nextclade.json \
@@ -323,7 +321,7 @@ task derived_cols {
         String?       lab_highlight_loc
         Array[File]   table_map = []
 
-        String        docker = "quay.io/broadinstitute/viral-core:2.3.2"
+        String        docker = "quay.io/broadinstitute/viral-core:2.4.0"
         Int           disk_size = 50
     }
     parameter_meta {
@@ -891,7 +889,7 @@ task filter_sequences_to_list {
 
         String       out_fname = sub(sub(basename(sequences, ".zst"), ".vcf", ".filtered.vcf"), ".fasta$", ".filtered.fasta")
         # Prior docker image: "nextstrain/base:build-20240318T173028Z"
-        String       docker = "quay.io/broadinstitute/viral-core:2.3.2"
+        String       docker = "quay.io/broadinstitute/viral-core:2.4.0"
         Int          disk_size = 750
     }
     parameter_meta {
@@ -992,7 +990,7 @@ task mafft_one_chr {
         Boolean  large = false
         Boolean  memsavetree = false
 
-        String   docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+        String   docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
         Int      mem_size = 500
         Int      cpus = 64
         Int      disk_size = 750
@@ -1080,7 +1078,7 @@ task mafft_one_chr_chunked {
         Int      batch_chunk_size = 2000
         Int      threads_per_job = 2
 
-        String   docker = "quay.io/broadinstitute/viral-phylo:2.1.20.2"
+        String   docker = "quay.io/broadinstitute/viral-phylo:2.3.6.0"
         Int      mem_size = 32
         Int      cpus = 64
         Int      disk_size = 750