Merge pull request #641 from wtsi-npg/devel

merge devel to master to create release 67.1.0
wtsi-npg · Sep 5, 2019 · 14d5529 · 14d5529
2 parents 0ffe1fd + 280a72d
commit 14d5529
Show file tree

Hide file tree

Showing 9 changed files with 121 additions and 72 deletions.
diff --git a/Changes b/Changes
@@ -1,5 +1,12 @@
 LIST OF CHANGES FOR NPG-QC PACKAGE
 
+release 67.1.0
+  - if DO_NOT_USE reference selected don't run VerifyBamID but provide
+      explanatory comment in json output file
+  - in qc store, capture error inferring a path to the archive dir,
+      which we need to retrieve autoqc data, rather from inferring
+      some other path
+
  release 67.0.0
   - fix regular expression used in collapser after change in template
   - adapter check switched to use cram files; fixed an old bug in

diff --git a/lib/npg_qc/Schema/Result/VerifyBamId.pm b/lib/npg_qc/Schema/Result/VerifyBamId.pm
@@ -101,29 +101,25 @@ A foreign key referencing the id_seq_composition column of the seq_composition t
 =head2 avg_depth
 
   data_type: 'decimal'
-  default_value: 0.00
-  is_nullable: 0
+  is_nullable: 1
   size: [11,2]
 
 =head2 freemix
 
   data_type: 'decimal'
-  default_value: 0.00000
-  is_nullable: 0
+  is_nullable: 1
   size: [11,5]
 
 =head2 freeLK0
 
   data_type: 'decimal'
-  default_value: 0.00
-  is_nullable: 0
+  is_nullable: 1
   size: [11,2]
 
 =head2 freeLK1
 
   data_type: 'decimal'
-  default_value: 0.00
-  is_nullable: 0
+  is_nullable: 1
   size: [11,2]
 
 =head2 pass
@@ -174,33 +170,13 @@ __PACKAGE__->add_columns(
   'number_of_reads',
   { data_type => 'integer', is_nullable => 1 },
   'avg_depth',
-  {
-    data_type => 'decimal',
-    default_value => '0.00',
-    is_nullable => 0,
-    size => [11, 2],
-  },
+  { data_type => 'decimal', is_nullable => 1, size => [11, 2] },
   'freemix',
-  {
-    data_type => 'decimal',
-    default_value => '0.00000',
-    is_nullable => 0,
-    size => [11, 5],
-  },
+  { data_type => 'decimal', is_nullable => 1, size => [11, 5] },
   'freeLK0',
-  {
-    data_type => 'decimal',
-    default_value => '0.00',
-    is_nullable => 0,
-    size => [11, 2],
-  },
+  { data_type => 'decimal', is_nullable => 1, size => [11, 2] },
   'freeLK1',
-  {
-    data_type => 'decimal',
-    default_value => '0.00',
-    is_nullable => 0,
-    size => [11, 2],
-  },
+  { data_type => 'decimal', is_nullable => 1, size => [11, 2] },
   'pass',
   { data_type => 'tinyint', is_nullable => 1 },
   'comments',
@@ -274,8 +250,8 @@ __PACKAGE__->belongs_to(
 with 'npg_qc::Schema::Composition', 'npg_qc::Schema::Flators', 'npg_qc::autoqc::role::result', 'npg_qc::autoqc::role::verify_bam_id';
 
 
-# Created by DBIx::Class::Schema::Loader v0.07047 @ 2017-09-14 16:25:18
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fj9PpvtAIizcpJ0BvIj7hw
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2019-08-29 10:31:30
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:g2285gY2V+24Dah34YMHvQ
 
 our $VERSION = '0';
 

diff --git a/lib/npg_qc/autoqc/checks/check.pm b/lib/npg_qc/autoqc/checks/check.pm
@@ -27,6 +27,7 @@ our $VERSION = '0';
 
 Readonly::Scalar my $FILE_EXTENSION  => 'fastq';
 Readonly::Scalar my $HUMAN           => q[Homo_sapiens];
+Readonly::Scalar my $DO_NOT_USE      => q[DO_NOT_USE];
 Readonly::Scalar my $FORWARD_READ_FILE_NAME_SUFFIX => q[1];
 Readonly::Scalar my $REVERSE_READ_FILE_NAME_SUFFIX => q[2];
 
@@ -299,6 +300,26 @@ sub _build_result {
   return $result;
 }
 
+has '_lims_reference'  => (isa         => 'Maybe[Str]',
+                           is          => 'ro',
+                           required    => 0,
+                           lazy_build  => 1,
+                           );
+
+sub _build__lims_reference {
+  my $self = shift;
+
+  if (!$self->can('lims')) {
+    $self->result->add_comment('lims accessor is not defined');
+    return;
+  }
+  if(!$self->lims->reference_genome) {
+    $self->result->add_comment('No reference genome specified');
+    return;
+  }
+  return $self->lims->reference_genome;
+}
+
 =head2 run
 
 Creates an object that can perform the requested test, calls test
@@ -467,29 +488,42 @@ sub create_filename {
 =head2 entity_has_human_reference
 
 Returns true if the reference_genome attribute is defined
-for the entiry and the value of the attribute indicates
+for the entity and the value of the attribute indicates
 that the reference is for Homo Sapiens.
 
 =cut
 
 sub entity_has_human_reference {
   my $self = shift;
-
-  if (!$self->can('lims')) {
-    $self->result->add_comment('lim saccessor is not defined');
-    return 0;
-  }
-
-  my $ref = $self->lims->reference_genome;
-  if(!$ref) {
-    $self->result->add_comment('No reference genome specified');
+  if( !$self->_lims_reference ) {
     return 0;
   }
+  my $ref = $self->_lims_reference;
   if($ref !~ /\A$HUMAN/smx) {
     $self->result->add_comment("Non-human reference genome '$ref'");
     return 0;
   }
+  return 1;
+}
 
+=head2 entity_has_active_reference
+
+Returns true if the reference_genome attribute is defined for
+the entity and the value of the attribute indicates that the
+reference is not marked as do not use.
+
+=cut
+
+sub entity_has_active_reference {
+  my $self = shift;
+  if( !$self->_lims_reference ) {
+    return 0;
+  }
+  my $ref = $self->_lims_reference;
+  if($ref =~ /$DO_NOT_USE/smx) {
+    $self->result->add_comment("Non active reference genome used '$ref'");
+    return 0;
+  }
   return 1;
 }
 

diff --git a/lib/npg_qc/autoqc/checks/verify_bam_id.pm b/lib/npg_qc/autoqc/checks/verify_bam_id.pm
@@ -81,6 +81,9 @@ override 'execute' => sub {
   if(!$self->can_run()) {
     return 1;
   }
+  if (!$self->entity_has_active_reference()) {
+    return 1;
+  }
   if (!$self->snv_file) {
     croak q(Can't find snv file);
   }

diff --git a/lib/npg_qc/autoqc/qc_store.pm b/lib/npg_qc/autoqc/qc_store.pm
@@ -226,10 +226,24 @@ sub load_from_staging {
   (ref $query eq $expected_type)
     or croak qq[Query object should be of type $expected_type];
 
-  my $rfo = $self->_runfolder_obj($query);
+  my $archive_path;
+  try {
+    $archive_path = npg_tracking::illumina::runfolder->new(
+      id_run              => $query->id_run,
+      npg_tracking_schema => $query->npg_tracking_schema
+    )->archive_path;
+  } catch {
+    carp sprintf 'Failed to load data from staging for query "%s" : "%s"',
+      $query->to_string, $_;
+  };
 
-  return $rfo ? $self->load_from_staging_archive($query, $rfo->archive_path)
-              : npg_qc::autoqc::results::collection->new();
+  #####
+  # Potential error retrieving data from the archive directory is
+  # not captured; this is deliberate.
+  #
+  return $archive_path ?
+    $self->load_from_staging_archive($query, $archive_path) :
+    npg_qc::autoqc::results::collection->new();
 }
 
 =head2 load_from_staging_archive
@@ -518,25 +532,6 @@ sub _db_collection4compositions {
   return npg_qc::autoqc::results::collection->new(results => \@rows);
 }
 
-sub _runfolder_obj {
-  my ($self, $query) = @_;
-
-  my $rfs;
-  try {
-    $rfs = npg_tracking::illumina::runfolder->new(
-      id_run              => $query->id_run,
-      npg_tracking_schema => $query->npg_tracking_schema
-    );
-    $rfs->analysis_path; # Might fail to find the run folder analysis directory.
-  } catch {
-    undef $rfs;
-    carp sprintf 'Failed to load data from staging for query "%s" : "%s"',
-      $query->to_string, $_;
-  };
-
-  return $rfs;
-}
-
 sub _query_obj {
   my ($self, $id_run, $lanes, $what, $db_lookup, $npg_schema) = @_;
 

diff --git a/npg_qc_viewer/root/src/ui_checks/verify_bam_id.tt2 b/npg_qc_viewer/root/src/ui_checks/verify_bam_id.tt2
@@ -3,6 +3,14 @@
 
 [% has_reverse = check.reverse_read_gc_percent.defined -%]
 
+[% IF check.comments -%]
+<div class="data_table">
+<table title="verify comments">
+<tr><th class="laligned">Comments</th><td>[% check.comments %]</td></tr>
+</table>
+</div>
+[%- END %]
+
 <div class="data_table">
 
 <table title="verify_bam_id">
@@ -17,6 +25,4 @@
  [%- END -%]
 </tr>
 </table>
-
 </div>
-
diff --git a/scripts/upgrade_schema/upgrade_schema-67.0.1 b/scripts/upgrade_schema/upgrade_schema-67.0.1
@@ -0,0 +1,10 @@
+--
+--  allow verify_bam_id decimal columns to be NULL
+--
+
+
+ALTER TABLE `verify_bam_id` 
+MODIFY COLUMN `avg_depth` decimal(11,2) DEFAULT NULL, 
+MODIFY COLUMN `freemix` decimal(11,5) DEFAULT NULL, 
+MODIFY COLUMN `freeLK0` decimal(11,2) DEFAULT NULL, 
+MODIFY COLUMN `freeLK1` decimal(11,2) DEFAULT NULL;
diff --git a/t/60-autoqc-checks-verify_bam_id.t b/t/60-autoqc-checks-verify_bam_id.t
@@ -2,7 +2,7 @@ use strict;
 use warnings;
 use Cwd;
 use File::Temp qw(tempdir);
-use Test::More tests => 59;
+use Test::More tests => 64;
 use Test::Exception;
 use npg_tracking::util::abs_path qw(abs_path);
 
@@ -14,8 +14,9 @@ $ENV{NPG_WEBSERVICE_CACHE_DIR} = q[t/data/autoqc];
 
 my $tempdir = tempdir( CLEANUP => 1);
 my $tool_path = "$tempdir/verifyBamID";
-my $bam_path =  "$tempdir/13940_8.bam";
-for my $file (($tool_path, $bam_path)) {
+my $bam_path  =  "$tempdir/13940_8.bam";
+my $bam_path2 =  "$tempdir/27483_8#6.bam";
+for my $file (($tool_path, $bam_path, $bam_path2)) {
   open my $fh, '>', $file or die 'cannot open file for writing';
   print $fh '#verifyBamID mock-up';
   close $fh;
@@ -154,4 +155,21 @@ use_ok ('npg_qc::autoqc::checks::verify_bam_id');
     ok($r->can_run, 'Can run on RNA library') or diag $r->result->comments;
 }
 
+{
+    local $ENV{'NPG_CACHED_SAMPLESHEET_FILE'} = q[t/data/autoqc/verify_bam_id/samplesheet_27483.csv];
+    # Tests if
+    my $r = npg_qc::autoqc::checks::verify_bam_id->new(
+        rpt_list       => '27483:8:6',
+        qc_in          => $tempdir,
+        tmp_path       => $tempdir,
+        repository     => $repos,
+        snv_repository => $snv_repository_with_vcf);
+    lives_ok {$r->result;} 'No error creating result object';
+    ok(defined $r->ref_repository(), 'A default reference repository is set');
+    like($r->lims->reference_genome, qr/DO_NOT_USE/, 'human DO_NOT_USE reference genome');
+    ok($r->can_run, 'Can run on human DO_NOT_USE reference') or diag $r->result->comments;
+    $r->execute();
+    like($r->result->comments, qr/Non active reference genome/, 'Inappropriate ref comment recorded');
+}
+
 1;
diff --git a/t/data/autoqc/verify_bam_id/samplesheet_27483.csv b/t/data/autoqc/verify_bam_id/samplesheet_27483.csv
@@ -54,5 +54,5 @@ Lane,Sample_ID,Sample_Name,GenomeFolder,Index,Index2,bait_name,default_library_t
 8,22629979,CTTV0197621369,,GCTTTGGC,GGAAGACA,,RNA PolyA,GCTTTGGC,GGAAGACA,[email protected] [email protected],,[email protected] [email protected],,,0,0,23584446,0,22629979,,9606,S2263,,,standard,,,from:100 to:300,,,Homo sapiens,0,,CTTV0197621369,3840366,CTTV0197621369,,,188 IL22 8,888,EGAS00001001680,1,0,0,The purpose of this study is to establish whether intestinal organoid cultures from IBD patients are a good model of disease by studying various disease phenomena including%2C epithelial repair%2C wound healing%2C fibrotic changes%2C granulomas and inflammatory processes. Organoids will be used for target pertubation and validation studies. %0D%0AThis data is part of a pre-publication release. For information on the proper use of pre-publication data shared by the Wellcome Trust Sanger Institute (including details of any publication moratoria)%2C please see http%3A%2F%2Fwww.sanger.ac.uk%2Fdatasharing%2F,4042,CTTV019 RNAseq,Homo_sapiens (GRCh38_15 %2B ensembl_78_transcriptome),0,Transcriptomes of intestinal organoids,3,
 8,22629991,CTTV0197621370,,AACGACTG,GAAATTAT,,RNA PolyA,AACGACTG,GAAATTAT,[email protected] [email protected],,[email protected] [email protected],,,0,0,23584446,0,22629991,,9606,S2263,,,standard,,,from:100 to:300,,,Homo sapiens,0,,CTTV0197621370,3840367,CTTV0197621370,,,188 IL22 9,888,EGAS00001001680,1,0,0,The purpose of this study is to establish whether intestinal organoid cultures from IBD patients are a good model of disease by studying various disease phenomena including%2C epithelial repair%2C wound healing%2C fibrotic changes%2C granulomas and inflammatory processes. Organoids will be used for target pertubation and validation studies. %0D%0AThis data is part of a pre-publication release. For information on the proper use of pre-publication data shared by the Wellcome Trust Sanger Institute (including details of any publication moratoria)%2C please see http%3A%2F%2Fwww.sanger.ac.uk%2Fdatasharing%2F,4042,CTTV019 RNAseq,Homo_sapiens (GRCh38_15 %2B ensembl_78_transcriptome),0,Transcriptomes of intestinal organoids,4,
 8,22629908,CTTV0197621371,,GACCGTCA,CTTCTTGG,,RNA PolyA,GACCGTCA,CTTCTTGG,[email protected] [email protected],,[email protected] [email protected],,,0,0,23584446,0,22629908,,9606,S2263,,,standard,,,from:100 to:300,,,Homo sapiens,0,,CTTV0197621371,3840368,CTTV0197621371,,,Fut2 KO control 1,888,EGAS00001001680,1,0,0,The purpose of this study is to establish whether intestinal organoid cultures from IBD patients are a good model of disease by studying various disease phenomena including%2C epithelial repair%2C wound healing%2C fibrotic changes%2C granulomas and inflammatory processes. Organoids will be used for target pertubation and validation studies. %0D%0AThis data is part of a pre-publication release. For information on the proper use of pre-publication data shared by the Wellcome Trust Sanger Institute (including details of any publication moratoria)%2C please see http%3A%2F%2Fwww.sanger.ac.uk%2Fdatasharing%2F,4042,CTTV019 RNAseq,Homo_sapiens (GRCh38_15 %2B ensembl_78_transcriptome),0,Transcriptomes of intestinal organoids,5,
-8,22629920,CTTV0197621372,,CTCTGGGT,ACTTCTCG,,RNA PolyA,CTCTGGGT,ACTTCTCG,[email protected] [email protected],,[email protected] [email protected],,,0,0,23584446,0,22629920,,9606,S2263,,,standard,,,from:100 to:300,,,Homo sapiens,0,,CTTV0197621372,3840369,CTTV0197621372,,,Fut2 KO control 2,888,EGAS00001001680,1,0,0,The purpose of this study is to establish whether intestinal organoid cultures from IBD patients are a good model of disease by studying various disease phenomena including%2C epithelial repair%2C wound healing%2C fibrotic changes%2C granulomas and inflammatory processes. Organoids will be used for target pertubation and validation studies. %0D%0AThis data is part of a pre-publication release. For information on the proper use of pre-publication data shared by the Wellcome Trust Sanger Institute (including details of any publication moratoria)%2C please see http%3A%2F%2Fwww.sanger.ac.uk%2Fdatasharing%2F,4042,CTTV019 RNAseq,Homo_sapiens (GRCh38_15 %2B ensembl_78_transcriptome),0,Transcriptomes of intestinal organoids,6,
+8,22629920,CTTV0197621372,,CTCTGGGT,ACTTCTCG,,RNA PolyA,CTCTGGGT,ACTTCTCG,[email protected] [email protected],,[email protected] [email protected],,,0,0,23584446,0,22629920,,9606,S2263,,,standard,,,from:100 to:300,,,Homo sapiens,0,,CTTV0197621372,3840369,CTTV0197621372,,,Fut2 KO control 2,888,EGAS00001001680,1,0,0,The purpose of this study is to establish whether intestinal organoid cultures from IBD patients are a good model of disease by studying various disease phenomena including%2C epithelial repair%2C wound healing%2C fibrotic changes%2C granulomas and inflammatory processes. Organoids will be used for target pertubation and validation studies. %0D%0AThis data is part of a pre-publication release. For information on the proper use of pre-publication data shared by the Wellcome Trust Sanger Institute (including details of any publication moratoria)%2C please see http%3A%2F%2Fwww.sanger.ac.uk%2Fdatasharing%2F,4042,CTTV019 RNAseq,Homo_sapiens (GRCh37_53) DO_NOT_USE,0,Transcriptomes of intestinal organoids,6,
 8,22766063,phiX_for_spiked_buffers,,ACAACGCAATC,,,,ACAACGCAATC,,,,,,,1,0,23584446,0,22766063,,10847,,,,standard,,,,,,,0,,,1255141,phiX_for_spiked_buffers,,PhiX (Sanger-SNPs),,888,,1,0,0,None,198,Illumina Controls, ,0,,888,