Merge pull request #27 from pinellolab/trace_errors_search

Trace errors search

PostProcess/analisi_indels_NNN.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e  # trace all errors 
+
 # Script per l'analisi dei targets della ricerca REF e ENR con PAM NNN
 # Il file dei targets della ricerca sul genoma reference si chiama $REFtargets  -> INPUT $1
 # Il file dei targets della ricerca sul genoma enriched si chiama $ENRtargets   -> INPUT $2
@@ -45,7 +47,10 @@ touch $REFtargets.corrected
 
 # 1) Rimozione duplicati, estrazione semicommon e unique e creazione file total
 #echo 'Creazione file .total.txt'
-./extraction.sh "$REFtargets.corrected" "$ENRtargets" "$jobid" # OUTPUT    $jobid.common_targets.txt -> Non usato
+./extraction.sh "$REFtargets.corrected" "$ENRtargets" "$jobid" || {
+	echo "CRISPRme ERROR: indels analysis failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+} # OUTPUT    $jobid.common_targets.txt -> Non usato
 #           $jobid.semi_common_targets.txt
 #           $jobid.unique_targets.txt
 
@@ -68,7 +73,10 @@ rm "$jobid.semi_common_targets.minmaxdisr.txt"
 
 #echo 'Creazione cluster del file .total.txt'
 # 3) Clustering
-./cluster.dict.py "$jobid.total.txt" 'no' 'True' 'True' "$guide_file" 'total' 'orderChr' # OUTPUT     $jobid.total.cluster.txt
+./cluster.dict.py "$jobid.total.txt" 'no' 'True' 'True' "$guide_file" 'total' 'orderChr' || {
+	echo "CRISPRme ERROR: indels clustering failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+} # OUTPUT     $jobid.total.cluster.txt
 
 #sed -i ':a;N;$!ba;s/\n/\tn\tn\tn\n/g' $jobid.total.cluster.txt
 #sed -i '$s/$/\tn\tn\tn/g' $jobid.total.cluster.txt
@@ -96,7 +104,10 @@ rm "$jobid.total.txt"
 
 #echo 'Estrazione sample dal file .total.cluster.txt'
 
-./analisi_indels_NNN.py "$annotationfile" "$jobid.total.cluster.txt" "$jobid" "$dictionaries" "$pam_file" "$mismatch" "$referencegenome" "$guide_file" $bulgesDNA $bulgesRNA
+./analisi_indels_NNN.py "$annotationfile" "$jobid.total.cluster.txt" "$jobid" "$dictionaries" "$pam_file" "$mismatch" "$referencegenome" "$guide_file" $bulgesDNA $bulgesRNA || {
+	echo "CRISPRme ERROR: indels analysis failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
 # OUTPUT    $jobid.bestCFD_INDEL.txt
 #           $jobid.CFDGraph.txt     (per fare l'area graph dei CFD REF vs ENR)
 # NOTA AnnotatorAllTargets.py salva su disco SOLO il target con CFD più alto nel cluster e tra le scomposizioni esistenti
@@ -122,19 +133,37 @@ echo 'Sorting and adjusting results'
 # #tail file w/o header and sort for realguide,chr,cluster_pos,score
 # tail -n +2 $jobid.bestCRISTA_INDEL.txt | LC_ALL=C sort -k15,15 -k4,4 -k6,6n -k21,21rg -T ./ >>$jobid.tmp && mv $jobid.tmp $jobid.bestCRISTA_INDEL.txt
 
-./adjust_cols.py "$jobid.bestCFD_INDEL.txt"
-./adjust_cols.py "$jobid.bestCRISTA_INDEL.txt"
-./adjust_cols.py "$jobid.bestmmblg_INDEL.txt"
+./adjust_cols.py "$jobid.bestCFD_INDEL.txt" || {
+	echo "CRISPRme ERROR: CFD indels report failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
+./adjust_cols.py "$jobid.bestCRISTA_INDEL.txt" || {
+	echo "CRISPRme ERROR: CRISTA indels report failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
+./adjust_cols.py "$jobid.bestmmblg_INDEL.txt" || {
+	echo "CRISPRme ERROR: mismatch+bulges indels report failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
 
 # sed -i '1s/.*/MMBLG_#Bulge_type\tMMBLG_crRNA\tMMBLG_DNA\tMMBLG_Reference\tMMBLG_Chromosome\tMMBLG_Position\tMMBLG_Cluster_Position\tMMBLG_Direction\tMMBLG_Mismatches\tMMBLG_Bulge_Size\tMMBLG_Total\tMMBLG_PAM_gen\tMMBLG_Var_uniq\tMMBLG_Samples\tMMBLG_Annotation_Type\tMMBLG_Real_Guide\tMMBLG_rsID\tMMBLG_AF\tMMBLG_SNP\tMMBLG_#Seq_in_cluster\tMMBLG_CFD\tMMBLG_CFD_ref/' $jobid.bestmmblg_INDEL.txt
 # sed -i '1s/.*/MMBLG_#Bulge_type\tMMBLG_crRNA\tMMBLG_DNA\tMMBLG_Reference\tMMBLG_Chromosome\tMMBLG_Position\tMMBLG_Cluster_Position\tMMBLG_Direction\tMMBLG_Mismatches\tMMBLG_Bulge_Size\tMMBLG_Total\tMMBLG_PAM_gen\tMMBLG_Var_uniq\tMMBLG_Samples\tMMBLG_Annotation_Type\tMMBLG_Real_Guide\tMMBLG_rsID\tMMBLG_AF\tMMBLG_SNP\tMMBLG_#Seq_in_cluster\tMMBLG_CFD\tMMBLG_CFD_ref/' $jobid.altmmblg.txt
 
 # pr -m -t -J $jobid.bestCFD_INDEL.txt $jobid.bestmmblg_INDEL.txt >$jobid.bestMerge.txt
 # pr -m -t -J $jobid.altCFD.txt $jobid.altmmblg.txt >$jobid.altMerge.txt
 
-./remove_bad_indel_targets.py "$jobid.bestCFD_INDEL.txt"
-./remove_bad_indel_targets.py "$jobid.bestCRISTA_INDEL.txt"
-./remove_bad_indel_targets.py "$jobid.bestmmblg_INDEL.txt"
+./remove_bad_indel_targets.py "$jobid.bestCFD_INDEL.txt" || {
+	echo "CRISPRme ERROR: CFD indels report cleaning failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
+./remove_bad_indel_targets.py "$jobid.bestCRISTA_INDEL.txt" || {
+	echo "CRISPRme ERROR: CRISTA indels report cleaning failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
+./remove_bad_indel_targets.py "$jobid.bestmmblg_INDEL.txt" || {
+	echo "CRISPRme ERROR: mismatch+bulges indels report cleaning failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
 
 #merge targets in same chr when they are at distance 3 from each other (inclusive) preserving the highest scoring one
 # ./merge_close_targets_cfd.sh $jobid.bestCFD_INDEL.txt $jobid.bestCFD_INDEL.txt.trimmed 3 'score'

PostProcess/extraction.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e  # trace all errors
+
 #PARAM $1 is ref targets file
 #PARAM $2 is var targets file
 #PARAM $3 is job_id

PostProcess/merge_alt_chr.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e  # trace all failures
+
 dir=$(dirname $1)
 fileIn=$1
 fileOut=$2

PostProcess/merge_close_targets_cfd.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e  # capture any failure
+
 fileIn=$1
 fileOut=$2
 thresh=$3 #threshold to use in order to merge near targets
@@ -24,5 +26,8 @@ echo "Sorting done in $(($ENDTIME - $STARTTIME)) seconds"
 # rm $fileIn.sorted.tmp
 echo "Merging targets"
 # python remove_contiguous_samples_cfd.py $fileIn.sorted $fileOut $thresh $chrom $position $total $true_guide $snp_info $cfd
-python remove_contiguous_samples_cfd.py $fileIn $fileOut $thresh $chrom $position $total $true_guide $snp_info $cfd $sort_criteria
+python remove_contiguous_samples_cfd.py $fileIn $fileOut $thresh $chrom $position $total $true_guide $snp_info $cfd $sort_criteria || {
+    echo "CRISPRme ERROR: contigous SNP removal failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
 # rm $fileIn.sorted

PostProcess/post_analisi_indel.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e 
+
 output_folder=$1
 ref_folder=$2
 ref_name=$(basename $2)
@@ -31,7 +33,10 @@ LC_ALL=C grep -F -w $fake_chr "$output_folder/crispritz_targets/indels_${ref_nam
 header=$(head -1 $output_folder/crispritz_targets/indels_${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt)
 sed -i 1i"$header" "$output_folder/crispritz_targets/indels_${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$true_chr"
 
-./analisi_indels_NNN.sh "$output_folder/crispritz_targets/${ref_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$true_chr" "$output_folder/crispritz_targets/indels_${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$true_chr" "$output_folder/${fake_chr}_${pam_name}_${guide_name}_${annotation_name}_${mm}_${bDNA}_${bRNA}" "$annotation_file" "$dict_folder/log_indels_$vcf_name" "$ref_folder/$true_chr.fa" $mm $bDNA $bRNA "$guide_file" "$pam_file" "$output_folder"
+./analisi_indels_NNN.sh "$output_folder/crispritz_targets/${ref_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$true_chr" "$output_folder/crispritz_targets/indels_${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$true_chr" "$output_folder/${fake_chr}_${pam_name}_${guide_name}_${annotation_name}_${mm}_${bDNA}_${bRNA}" "$annotation_file" "$dict_folder/log_indels_$vcf_name" "$ref_folder/$true_chr.fa" $mm $bDNA $bRNA "$guide_file" "$pam_file" "$output_folder" || {
+    echo "CRISPRme ERROR: indels analysis failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
 rm "$output_folder/crispritz_targets/${ref_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$true_chr"
 rm "$output_folder/crispritz_targets/indels_${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$true_chr"
 # tail -n +2 "$output_folder/${fake_chr}_${pam_name}_${guide_name}_${annotation_name}_${mm}_${bDNA}_${bRNA}_$key.bestMerge.txt" >> "$final_res" #"$output_folder/${fake_chr}_${guide_name}_${mm}_${bDNA}_${bRNA}.bestCFD.txt.tmp"

PostProcess/post_analisi_snp.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e  # trace all command failures
+
 output_folder=$1
 ref_folder=$2
 ref_name=$(basename $2)
@@ -28,7 +30,10 @@ fi
 echo "Processing SNPs for $key"
 LC_ALL=C grep -F -w $key "$output_folder/crispritz_targets/${ref_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt" >"$output_folder/crispritz_targets/${ref_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key"
 LC_ALL=C grep -F -w $key "$output_folder/crispritz_targets/${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt" >"$output_folder/crispritz_targets/${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key"
-./scriptAnalisiNNN_v3.sh "$output_folder/crispritz_targets/${ref_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key" "$output_folder/crispritz_targets/${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key" "$output_folder/${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${annotation_name}_${mm}_${bDNA}_${bRNA}_$key" "$annotation_file" "${dict_folder}/my_dict_${key}.json" "${ref_folder}/${key}.fa" $mm $bDNA $bRNA "$guide_file" "$pam_file" "$output_folder"
+./scriptAnalisiNNN_v3.sh "$output_folder/crispritz_targets/${ref_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key" "$output_folder/crispritz_targets/${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key" "$output_folder/${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${annotation_name}_${mm}_${bDNA}_${bRNA}_$key" "$annotation_file" "${dict_folder}/my_dict_${key}.json" "${ref_folder}/${key}.fa" $mm $bDNA $bRNA "$guide_file" "$pam_file" "$output_folder" || {
+	echo "CRISPRme ERROR: SNP analysis failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
 rm "$output_folder/crispritz_targets/${ref_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key"
 rm "$output_folder/crispritz_targets/${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key"
 # header=$(head -1 "$output_folder/${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${annotation_name}_${mm}_${bDNA}_${bRNA}_$key.bestMerge.txt")

PostProcess/post_analysis_only.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e  # trace all failures
+
 #file for automated search of guide+pam in reference and variant genomes
 
 ref_folder=$(realpath $1)
@@ -119,7 +121,10 @@ if [ "$vcf_name" != "_" ]; then
 		exit
 	fi
 
-	./pool_post_analisi_snp.py $output_folder $ref_folder $vcf_name $guide_file $mm $bDNA $bRNA $annotation_file $pam_file $sampleID $dict_folder $final_res $final_res_alt $ncpus
+	./pool_post_analisi_snp.py $output_folder $ref_folder $vcf_name $guide_file $mm $bDNA $bRNA $annotation_file $pam_file $sampleID $dict_folder $final_res $final_res_alt $ncpus || {
+		echo "CRISPRme ERROR: indels postprocessing failed (script: ${0} line $((LINENO-1)))" >&2
+		exit 1
+	}
 
 	echo "Post-analysis SNPs End: "$(date +%F-%T) >>$output_folder/$log
 
@@ -150,7 +155,10 @@ else
 		echo "Processing $key"
 		LC_ALL=C grep -F -w $key "$output_folder/crispritz_targets/${ref_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt" >"$output_folder/crispritz_targets/${ref_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key"
 		touch "$output_folder/crispritz_targets/${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key"
-		./scriptAnalisiNNN_v3.sh "$output_folder/crispritz_targets/${ref_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key" "$output_folder/crispritz_targets/${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key" "${ref_name}_${pam_name}_${guide_name}_${annotation_name}_${mm}_${bDNA}_${bRNA}_$key" "$annotation_file" "_" "$ref_folder" $mm $bDNA $bRNA "$guide_file" "$pam_file" "$sampleID" "$output_folder"
+		./scriptAnalisiNNN_v3.sh "$output_folder/crispritz_targets/${ref_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key" "$output_folder/crispritz_targets/${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key" "${ref_name}_${pam_name}_${guide_name}_${annotation_name}_${mm}_${bDNA}_${bRNA}_$key" "$annotation_file" "_" "$ref_folder" $mm $bDNA $bRNA "$guide_file" "$pam_file" "$sampleID" "$output_folder" || {
+			echo "CRISPRme ERROR: analysis failed (script: ${0} line $((LINENO-1)))" >&2
+			exit 1
+		}
 		rm "$output_folder/crispritz_targets/${ref_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key"
 		rm "$output_folder/crispritz_targets/${ref_name}+${vcf_name}_${pam_name}_${guide_name}_${mm}_${bDNA}_${bRNA}.targets.txt.$key"
 		header=$(head -1 "$output_folder/${ref_name}_${pam_name}_${guide_name}_${annotation_name}_${mm}_${bDNA}_${bRNA}.bestMerge.txt")
@@ -172,7 +180,10 @@ if [ "$vcf_name" != "_" ]; then
 	cd "$starting_dir"
 
 	echo "Post-analysis INDELs Start: "$(date +%F-%T) >>$output_folder/$log
-	./pool_post_analisi_indel.py $output_folder $ref_folder $vcf_folder $guide_file $mm $bDNA $bRNA $annotation_file $pam_file $sampleID "$output_folder/log_indels_$vcf_name" $final_res $final_res_alt $ncpus
+	./pool_post_analisi_indel.py $output_folder $ref_folder $vcf_folder $guide_file $mm $bDNA $bRNA $annotation_file $pam_file $sampleID "$output_folder/log_indels_$vcf_name" $final_res $final_res_alt $ncpus || {
+		echo "CRISPRme ERROR:indels analysis failed (script: ${0} line $((LINENO-1)))" >&2
+		exit 1
+	}
 	echo "Post-analysis INDELs End: "$(date +%F-%T) >>$output_folder/$log
 	for key in "${array_fake_chroms[@]}"; do
 		tail -n +2 "$output_folder/${key}_${pam_name}_${guide_name}_${annotation_name}_${mm}_${bDNA}_${bRNA}.bestMerge.txt" >>"$final_res" #"$output_folder/${fake_chr}_${guide_name}_${mm}_${bDNA}_${bRNA}.bestCFD.txt.tmp"
@@ -186,7 +197,10 @@ fi
 cd "$starting_dir"
 
 echo "Merging Close Targets Start: "$(date +%F-%T) >>$output_folder/$log
-./merge_close_targets_cfd.sh $final_res $final_res.trimmed $merge_t
+./merge_close_targets_cfd.sh $final_res $final_res.trimmed $merge_t || {
+	echo "CRISPRme ERROR: CFD targets merge failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
 mv $final_res.trimmed $final_res
 mv $final_res.trimmed.discarded_samples $final_res_alt
 
@@ -195,7 +209,10 @@ mv $final_res.trimmed.discarded_samples $final_res_alt
 echo "Merging Close Targets End: "$(date +%F-%T) >>$output_folder/$log
 
 echo "Merging Alternative Chromosomes Start: "$(date +%F-%T) >>$output_folder/$log
-./merge_alt_chr.sh $final_res $final_res.chr_merged
+./merge_alt_chr.sh $final_res $final_res.chr_merged  || {
+	echo "CRISPRme ERROR: alternative targets merge failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
 #rm $final_res.trimmed
 
 #./merge_alt_chr.sh $final_res_alt.trimmed $final_res_alt.trimmed.chr_merged
@@ -213,7 +230,10 @@ echo "Cleaning directory"
 if ! [ -d "$output_folder/cfd_graphs" ]; then
 	mkdir $output_folder/cfd_graphs
 fi
-./assemble_cfd_graphs.py $output_folder
+./assemble_cfd_graphs.py $output_folder || {
+	echo "CRISPRme ERROR: CFD graph creation failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
 mv $output_folder/snps.CFDGraph.txt $output_folder/cfd_graphs
 mv $output_folder/indels.CFDGraph.txt $output_folder/cfd_graphs
 

PostProcess/post_process.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 #$1 is CFD targets file
 #$2 is genecode annotation
 #$3 is empirical data for the guide
@@ -40,7 +42,10 @@ echo 'Sorting final annotation results using original NR to correspond with orig
 # LC_ALL=C sort -T $dir -k4,4rg $1.found.bed -o $1.found.bed
 LC_ALL=C sort -T $dir -k4,4n $1.found.bed -o $1.found.bed
 echo 'Starting integration with empirical data (this may take a while)'
-"$starting_dir"./resultIntegrator.py $1 $3 $1.found.bed $4 $6/ true $5 $7 $9 ${10} ${11}
+"$starting_dir"./resultIntegrator.py $1 $3 $1.found.bed $4 $6/ true $5 $7 $9 ${10} ${11} || {
+    echo "CRISPRme ERROR: result integration failed (script: ${0} line $((LINENO-1)))" >&2
+	exit 1
+}
 echo 'Removing unnecessary files'
 rm -f $1.bed $1.found.bed $1.redirectFile.out $1.temp.bed
 # sed -i 1i"#Bulge_type\tcrRNA\tDNA\tReference\tChromosome\tPosition\tCluster_Position\tDirection\tMismatches\tBulge_Size\tTotal\tPAM_gen\tVar_uniq\tSamples\tAnnotation_Type\tReal_Guide\trsID\tAF\tSNP\t#Seq_in_cluster\tCFD\tCFD_ref\tHighest_CFD_Risk_Score\tHighest_CFD_Absolute_Risk_Score\tMMBLG_#Bulge_type\tMMBLG_crRNA\tMMBLG_DNA\tMMBLG_Reference\tMMBLG_Chromosome\tMMBLG_Position\tMMBLG_Cluster_Position\tMMBLG_Direction\tMMBLG_Mismatches\tMMBLG_Bulge_Size\tMMBLG_Total\tMMBLG_PAM_gen\tMMBLG_Var_uniq\tMMBLG_Samples\tMMBLG_Annotation_Type\tMMBLG_Real_Guide\tMMBLG_rsID\tMMBLG_AF\tMMBLG_SNP\tMMBLG_#Seq_in_cluster\tMMBLG_CFD\tMMBLG_CFD_ref\tMMBLG_CFD_Risk_Score\tMMBLG_CFD_Absolute_Risk_Score" "$1"