P4 stage2 init options

Dockerfile

@@ -1,8 +1,9 @@
 ARG BASE_IMAGE=ubuntu:22.04
 
-ARG BAMBI_VERSION="0.17.1"
+ARG BAMBI_VERSION="0.18.0"
 ARG BIOBAMBAM2_VERSION="2.0.185-release-20221211202123"
 ARG BWA_VERSION="0.7.18"
+ARG BWA_MEM2_VERSION="2.2.1"
 ARG DEFLATE_VERSION="1.20"
 ARG HTSLIB_VERSION="1.20"
 ARG IO_LIB_VERSION="1.15.0"
@@ -127,6 +128,17 @@ RUN curl -sSL -O "https://github.com/lh3/bwa/archive/refs/tags/v${BWA_VERSION}.t
     chmod +x /usr/local/bin/bwa && \
     ln -s /usr/local/bin/bwa /usr/local/bin/bwa0_6
 
+ARG BWA_MEM2_VERSION
+RUN curl -sSL -O "https://github.com/bwa-mem2/bwa-mem2/releases/download/v${BWA_MEM2_VERSION}/bwa-mem2-${BWA_MEM2_VERSION}_x64-linux.tar.bz2" && \
+    tar xfj ./bwa-mem2-${BWA_MEM2_VERSION}_x64-linux.tar.bz2 && \
+    cd ./bwa-mem2-${BWA_MEM2_VERSION}_x64-linux && \
+    cp ./bwa-mem2 /usr/local/bin/ && \
+    cp ./bwa-mem2.avx /usr/local/bin/ && \
+    cp ./bwa-mem2.avx2 /usr/local/bin/ && \
+    cp ./bwa-mem2.avx512bw /usr/local/bin/ && \
+    cp ./bwa-mem2.sse41 /usr/local/bin/ && \
+    cp ./bwa-mem2.sse42 /usr/local/bin/
+
 ARG BAMBI_VERSION
 RUN git clone --single-branch --branch="$BAMBI_VERSION" --depth=1 "https://github.com/wtsi-npg/bambi.git" && \
     cd bambi && \

bin/vtfp.pl

@@ -451,6 +451,12 @@ sub apply_subst {
 		$ewi->{removelabel}->();
 	}
 
+	if($cfg->{subgraph_io}) {
+		$ewi->{addlabel}->(q{subgraph_io});
+		$cfg->{subgraph_io} = subst_walk($cfg->{subgraph_io}, $params, $ewi);
+		$ewi->{removelabel}->();
+	}
+
 	return;
 }
 
@@ -1412,9 +1418,9 @@ sub validate_splice_candidates {
 		}
 	}
 
-	#  all edge termini must be unique (over replacement and pruning edges) except for STDIN/STDOUT
+	#  all edge termini must be unique (over replacement edges) except for STDIN/STDOUT
 	my %endpoints;
-	for my $edge (@{$splice_candidates->{replacement_edges}}, @{$prune_edges}) {
+	for my $edge (@{$splice_candidates->{replacement_edges}}) {
 		my $from_end = $edge->{from};
 		if($from_end and $from_end !~ /:/) { $from_end .= q[:STDOUT] };
 
@@ -1472,15 +1478,33 @@ sub final_splice {
 	# add new edges
 	push @{$flat_graph->{edges}}, @{$splice_candidates->{replacement_edges}};
 
-	# remove pruned ports - prune edges are not required to be two-ended; just disregard undefined to/from attributes
+	# remove pruned ports - prune edges are not required to be two-ended; just disregard undefined to/from attributes; only remove ports
+	#   that do not appear in splice edges (aka replacement edges)
 	for my $prune_edge (@{$splice_candidates->{prune_edges}}) {
-		if($prune_edge->{from}) { remove_port($prune_edge->{from}, $SRC, $flat_graph); }
-		if($prune_edge->{to}) { remove_port($prune_edge->{to}, $DST, $flat_graph); }
+		if($prune_edge->{from} and not _in_replacement_edges($prune_edge->{from}, $splice_candidates, $SRC)) { remove_port($prune_edge->{from}, $SRC, $flat_graph); }
+		if($prune_edge->{to} and not _in_replacement_edges($prune_edge->{to}, $splice_candidates, $DST)) { remove_port($prune_edge->{to}, $DST, $flat_graph); }
 	}
 
 	return $flat_graph;
 }
 
+sub _in_replacement_edges {
+	my ($port_spec, $splice_candidates, $type) = @_;
+
+	my $direction = ($type == $SRC)? q[from]: q[to];
+	my $std_port = ($type == $SRC)? q[STDIN]: q[STDOUT];
+
+	for my $edge (@{$splice_candidates->{replacement_edges}}) {
+		my $end = $edge->{$direction};
+		if($end and $end !~ /:/) { $end .= qq[:$std_port] };
+
+		if($end eq $port_spec) { return 1; }
+	}
+
+	return 0;
+
+}
+
 ################################################################################################
 # resolve_ports:
 #  given a splice_pair specification, fully determine the [set of] source and destination ports

data/static_params/stage2_reanalysis/align_bwa_mem2.json

@@ -0,0 +1,13 @@
+{
+  "assign": [
+    {
+      "alignment_method": "bwa_mem",
+      "bwa_executable": "bwa-mem2"
+    }
+  ],
+  "assign_local": {},
+  "ops": {
+    "splice": [],
+    "prune": []
+  }
+}

data/static_params/stage2_reanalysis/base_params_duplexseq_cram.json

@@ -0,0 +1,37 @@
+{
+  "assign": [
+    {
+      "spatial_filter_switch":"off",
+      "markdup_optical_distance_value": "100",
+      "s2_se_pe": "pe",
+      "samtools_executable": "samtools",
+      "s2_input_format": "cram",
+      "markdup_method": "duplexseq",
+      "s2_ppi_switch":"s2_ppi",
+      "pp_read2tags":"on",
+      "pp_import_method":"crammerge",
+      "fastq_s2_pi_fq1": "DUMMY",
+      "fastq_s2_pi_fq2": "DUMMY",
+      "fastq_s2_pi_RG_ID": "DUMMY",
+      "s2_filter_files": "DUMMY",
+      "spatial_filter_file": "DUMMY",
+      "phix_reference_genome_fasta":"DUMMY",
+      "realignment_switch":1
+    }
+  ],
+  "assign_local": {},
+  "ops": {
+    "splice": [
+      "aln_bam12auxmerge:-foptgt_000_fixmate:",
+      "foptgt_seqchksum_file:-scs_cmp_seqchksum:outputchk"
+    ],
+    "prune": [
+      "foptgt.*_bmd_multiway:calibration_pu-",
+      "foptgt_cram_tee:c2a-",
+      "foptgt.*samtools_stats_F0.*_target.*-",
+      "foptgt.*samtools_stats_F0.*00_bait.*-",
+      "aln_tee3_tee3:to_phix_aln-scs_cmp_seqchksum:outputchk",
+      "ssfqc_tee_ssfqc:subsample-"
+    ]
+  }
+}

data/static_params/stage2_reanalysis/base_params_duplexseq_fastq.json

@@ -0,0 +1,35 @@
+{
+  "assign": [
+    {
+      "spatial_filter_switch":"off",
+      "markdup_optical_distance_value": "100",
+      "s2_se_pe": "pe",
+      "samtools_executable": "samtools",
+      "s2_input_format": "cram",
+      "markdup_method": "duplexseq",
+      "s2_ppi_switch":"s2_ppi",
+      "pp_read2tags":"on",
+      "pp_import_method":"fastq",
+      "incrams": "DUMMY",
+      "s2_filter_files": "DUMMY",
+      "spatial_filter_file": "DUMMY",
+      "phix_reference_genome_fasta":"DUMMY",
+      "realignment_switch":1
+    }
+  ],
+  "assign_local": {},
+  "ops": {
+    "splice": [
+      "aln_bam12auxmerge:-foptgt_000_fixmate:",
+      "foptgt_seqchksum_file:-scs_cmp_seqchksum:outputchk"
+    ],
+    "prune": [
+      "foptgt.*_bmd_multiway:calibration_pu-",
+      "foptgt_cram_tee:c2a-",
+      "foptgt.*samtools_stats_F0.*_target.*-",
+      "foptgt.*samtools_stats_F0.*00_bait.*-",
+      "aln_tee3_tee3:to_phix_aln-scs_cmp_seqchksum:outputchk",
+      "ssfqc_tee_ssfqc:subsample-"
+    ]
+  }
+}

data/vtlib/alignment_wtsi_stage2_template.json

@@ -95,24 +95,20 @@
 ],
 "nodes":[
 	{
-		"id":"crammerge",
-		"type":"EXEC",
-		"use_STDIN": false,
-		"use_STDOUT": true,
-		"cmd": [
-			"samtools",
-			"merge",
-			"-n",
-			"-O", "BAM",
-			"-l", "0",
-			{"select":"s2_input_format", "default":"cram", "select_range":[1], "cases":{
-				"cram":["--input-fmt-option", "no_ref=1"],
-				"bam":["--input-fmt", "bam"]
+		"id":"preprocess_inputs",
+		"type":"VTFILE",
+		"name":{"subst":"s2_preprocess_inputs_method", "required":true,
+			"ifnull":{
+				"select":"s2_ppi_switch", "default":"crammerge","select_range":[1],
+				"cases":{
+					"crammerge":"crammerge.json",
+					"s2_ppi":"stage2_preprocess_inputs.json"
+				}
 			}},
-			"-",
-			{"subst":"incrams"}
-		],
-		"description":"merge individual cram files from a sample into one bam file"
+                "subst_map":{"input_format":{"subst":"s2_input_format"}},
+		"comment":"inputs: NONE; outputs: _stdout_ (bam), subst_map_parameters:[input_format]",
+		"node_prefix":"ppi_",
+		"description":"subgraph to preprocess inputs. Default: merge individual cram files from a sample into one bam file"
 	},
 	{
 		"id":"spatial_filter",
@@ -264,7 +260,7 @@
 	}
 ],
 "edges":[
-	{ "id":"src_to_bc2", "from":"crammerge", "to":{"subst":"post_cm","required":true} },
+	{ "id":"src_to_bc2", "from":"preprocess_inputs", "to":{"subst":"post_cm","required":true} },
 	{"select":"spatial_filter_switch", "required":true, "select_range":[1], "default":"on", "allow_unspec_keys":true,
 	"cases":{
 		"on": [

data/vtlib/crammerge.json

@@ -0,0 +1,37 @@
+{
+"version":"2.0",
+"description":"run bwa mem to to align input bam to supplied reference genome",
+"subgraph_io":{
+	"ports":{
+		"inputs":{
+		},
+		"outputs":{
+			"_stdout_":"crammerge"
+		}
+	}
+},
+"subst_params":[],
+"nodes":[
+	{
+		"id":"crammerge",
+		"type":"EXEC",
+		"use_STDIN": false,
+		"use_STDOUT": true,
+		"cmd": [
+			"samtools",
+			"merge",
+			"-n",
+			"-O", "BAM",
+			"-l", "0",
+			{"select":"input_format", "default":"cram", "select_range":[1], "cases":{
+				"cram":["--input-fmt-option", "no_ref=1"],
+				"bam":["--input-fmt", "bam"]
+			}},
+			"-",
+			{"subst":"incrams", "required":true}
+		],
+		"description":"merge individual cram files from a sample into one bam file"
+	}
+],
+"edges":[]
+}

data/vtlib/elembio_nanoseq_s2_pi.json

@@ -0,0 +1,71 @@
+{
+"version":"2.0",
+"description":"alternate pre-processing method for stage2 inputs (Elembio NanoSeq)",
+"subgraph_io":{
+	"ports":{
+		"inputs":{},
+		"outputs":{ "_stdout_":"ubam_convert" }
+	}
+},
+"nodes":[
+	{
+		"id":"import",
+		"type":"EXEC",
+		"use_STDIN": false,
+		"use_STDOUT": true,
+		"cmd": [
+			{"subst":"samtools_executable", "required":true, "ifnull":"samtools"}, "import",
+			"-R", {"subst":"elembio_nanoseq_RG","required":true, "comment":"readgroup"},
+			"-1", {"subst":"elembio_nanoseq_fq1","required":true, "comment":"FASTQ read 1"},
+			"-2", {"subst":"elembio_nanoseq_fq2","required":true, "comment":"FASTQ read 2"},
+			"-i",
+			"-u",
+			"-O", "bam"
+		]
+	},
+	{
+		"id":"read2tags",
+		"type":{
+			"select":"pp_read2tags",
+			"required":true,
+			"select_range":[1],
+			"default":"off",
+			"cases":{
+				"on":"EXEC",
+				"off":"INACTIVE"
+			}
+		},
+		"use_STDIN": true,
+		"use_STDOUT": true,
+		"cmd": [
+			{"subst":"bambi_executable", "required":true, "ifnull":"bambi"}, "read2tags",
+			"--tags", "rb,mb,br,rb,mb,br",
+			"--qtags", "rq,mq,bq,rq,mq,bq",
+			"--positions", "1:1:1:3,1:2:1:3,1:1:4:7,2:2:1:3,2:1:1:3,2:2:4:7",
+			"--output-fmt", "sam"
+		]
+	},
+        {
+                "id":"add_nanoseq_mate_tags",
+                "type":"INACTIVE",
+                "use_STDIN": true,
+                "use_STDOUT": true,
+                "cmd": [ "add_NanoSeq_mate_tags.pl" ]
+        },
+	{
+		"id":"ubam_convert",
+		"type":"EXEC",
+		"use_STDIN": true,
+		"use_STDOUT": true,
+		"cmd": [
+			{"subst":"samtools_executable", "required":true, "ifnull":"samtools"}, "view",
+			"-u",
+			"-"
+		]
+	}
+],
+"edges":[
+	{ "id":"import_to_read2tags", "from":"import", "to":"read2tags" },
+        { "id":"read2tags_to_add_nanoseq_mate_tags", "from":"read2tags", "to":"ubam_convert" }
+]
+}

data/vtlib/read2tags.json

@@ -0,0 +1,54 @@
+{
+"version":"2.0",
+"description":"read2tags for NanoSeq processing, including preparatory collation and reset",
+"subgraph_io":{
+	"ports":{
+		"inputs":{"_stdin_":"collate"},
+		"outputs":{ "_stdout_":"read2tags" }
+		}
+},
+"nodes":[
+	{
+		"id":"collate",
+		"type": "EXEC",
+		"use_STDIN": true,
+		"use_STDOUT": true,
+		"cmd": [
+			{"subst":"samtools_executable", "required":true, "ifnull":"samtools"}, "collate",
+			"--threads", {"subst":"s2_r2t_coll_threads","required":true,"ifnull":2},
+			"-u",
+			"-O",
+			"-"
+		]
+	},
+	{
+		"id":"reset",
+		"type": "EXEC",
+		"use_STDIN": true,
+		"use_STDOUT": true,
+		"cmd": [
+			{"subst":"samtools_executable", "required":true, "ifnull":"samtools"}, "reset",
+			"--threads", {"subst":"s2_r2t_rs_threads","required":true,"ifnull":4},
+			"--output-fmt", "BAM,level=0"
+		]
+	},
+	{
+		"id":"read2tags",
+		"type": "EXEC",
+		"use_STDIN": true,
+		"use_STDOUT": true,
+		"cmd": [
+			{"subst":"bambi_executable", "required":true, "ifnull":"bambi"}, "read2tags",
+			"--tags", "rb,mb,br,rb,mb,br",
+			"--qtags", "rq,mq,bq,rq,mq,bq",
+			"--positions", "1:1:1:3,1:2:1:3,1:1:4:7,2:2:1:3,2:1:1:3,2:2:4:7",
+			"--compression-level", 0,
+			"--output-fmt", "bam"
+		]
+	}
+],
+"edges":[
+	{ "id":"collate_to_reset", "from":"collate","to":"reset" },
+	{ "id":"reset_to_read2tags", "from":"reset", "to":"read2tags" }
+]
+}