Add ngs to cohort extract Dockerfile; remove exception catching in ex…

…tract python script (#7113) * add ngs to cohort extract docker * throw error if there's an exception * add back try block to print job stats * whitespace changes * remove unused var
broadinstitute · Mar 9, 2021 · 359acd4 · 359acd4
1 parent 04a9b44
commit 359acd4
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 9 deletions.
diff --git a/scripts/variantstore/wdl/extract/Dockerfile b/scripts/variantstore/wdl/extract/Dockerfile
@@ -6,6 +6,7 @@ RUN pip install -r /app/requirements.txt
 
 # Add the application source code.
 ADD raw_array_cohort_extract.py /app
+ADD ngs_cohort_extract.py /app
 
 # install google SDK
 RUN curl -sSL https://sdk.cloud.google.com | bash

diff --git a/scripts/variantstore/wdl/extract/ngs_cohort_extract.py b/scripts/variantstore/wdl/extract/ngs_cohort_extract.py
@@ -32,7 +32,6 @@
 VET_DISTINCT_POS_TABLE = f"{output_table_prefix}_vet_distinct_pos"
 PET_NEW_TABLE = f"{output_table_prefix}_pet_new"
 VET_NEW_TABLE = f"{output_table_prefix}_vet_new"
-COHORT_EXTRACT_TABLE = f"{output_table_prefix}_cohort_extract"
 
 def utf8len(s):
     return len(s.encode('utf-8'))
@@ -229,10 +228,9 @@ def do_extract(fq_pet_vet_dataset,
                min_variant_samples,
                fq_sample_mapping_table
               ):
-  try:  
-
+  try:
     global client
-    client = bigquery.Client(project=query_project, 
+    client = bigquery.Client(project=query_project,
                              default_query_job_config=QueryJobConfig(labels={ "id" : f"test_cohort_export_{output_table_prefix}"}, priority="INTERACTIVE", use_query_cache=False ))
 
     ## TODO -- provide a cmdline arg to override this (so we can simulat smaller datasets)
@@ -245,13 +243,12 @@ def do_extract(fq_pet_vet_dataset,
 
     make_new_vet_union_all(fq_pet_vet_dataset, fq_temp_table_dataset, cohort)
 
-    create_position_table(fq_temp_table_dataset, min_variant_samples)  
+    create_position_table(fq_temp_table_dataset, min_variant_samples)
     make_new_pet_union_all(fq_pet_vet_dataset, fq_temp_table_dataset, cohort)
     populate_final_extract_table(fq_temp_table_dataset, fq_destination_dataset, destination_table, fq_sample_mapping_table)
-  except Exception as err:
-    print(err)
+  finally:
+    dump_job_stats()
 
-  dump_job_stats()
   print(f"\nFinal cohort extract written to {fq_destination_dataset}.{destination_table}\n")
 
 if __name__ == '__main__':
@@ -280,4 +277,4 @@ def do_extract(fq_pet_vet_dataset,
              args.fq_destination_dataset,
              args.destination_table,
              args.min_variant_samples,
-             args.fq_sample_mapping_table)
+             args.fq_sample_mapping_table)