merge from devel to master to create release 69.1.0

Build.PL

@@ -42,6 +42,7 @@ my $requires = {
                 'lib'                             => 0,
                 'List::MoreUtils'                 => 0,
                 'List::Util'                      => 0,
+                'Log::Log4perl'                   => 0,
                 'LWP::UserAgent'                  => 0,
                 'Math::Round'                     => 0,
                 'MIME::Base64'                    => 0,
@@ -77,6 +78,8 @@ my $requires = {
                 'Vcf'                             => 0,
 
                 'WTSI::DNAP::Utilities::Timestamp'         => 0,
+    		'WTSI::DNAP::Warehouse::Schema'            => 0,
+                'WTSI::DNAP::Warehouse::Schema::Query::IseqFlowcell' => 0,
                 'npg_common::Alignment'                    => 0,
                 'npg_common::extractor::fastq'             => 0,
                 'npg_common::roles::software_location'     => 0,
@@ -105,7 +108,6 @@ my $requires = {
                 'npg::util'                                => 0,
                 'st::api::base'                            => 0,
                 'st::api::lims'                            => 0,
-                'WTSI::DNAP::Warehouse::Schema'            => 0,
                 'WTSI::NPG::iRODS'                         => 0,
                 'WTSI::NPG::iRODS::DataObject'             => 0
 };

Changes

@@ -1,5 +1,11 @@
 LIST OF CHANGES FOR NPG-QC PACKAGE
 
+release 69.1.0
+ - an extension for the npg_mqc_skipper script to make it work for the
+   Heron study, ie to take into consideration provisional manual QC
+   outcomes from the review autoqc resuls and finalise QC outcomes
+   for sequencing and libraries for runs that are expedited to archival
+
 release 69.0.1
  - npg_qc::autoqc::checks::generic::artic - bug fix in a parser for
    artic QC summary for cases when the summary is empty

MANIFEST

@@ -104,6 +104,7 @@ lib/npg_qc/autoqc/role/upstream_tags.pm
 lib/npg_qc/autoqc/role/verify_bam_id.pm
 lib/npg_qc/autoqc/types.pm
 lib/npg_qc/mqc/reporter.pm
+lib/npg_qc/mqc/skipper.pm
 lib/npg_qc/mqc/outcomes.pm
 lib/npg_qc/mqc/outcomes/keys.pm
 lib/npg_qc/report/genotype_call.pm
@@ -269,6 +270,7 @@ t/60-autoqc-results-tag_metrics.t
 t/60-autoqc-results-verify_bam_id.t
 t/70-mqc-reporter.t
 t/70-mqc-outcomes.t
+t/70-mqc-skipper.t
 t/70-report-genotype_call.t
 t/80-autoqc-qc-script.t
 t/data/autoqc/bam_flagstats/16960_1_0.tar.gz

bin/npg_mqc_skipper

@@ -10,7 +10,7 @@ use Log::Log4perl qw(:levels);
 use DBI;
 use Readonly;
 use Try::Tiny;
-use Carp;
+use Class::Load qw(load_class);
 
 use npg_tracking::Schema;
 use WTSI::DNAP::Warehouse::Schema;
@@ -23,12 +23,17 @@ Readonly::Scalar my $RUN_STATUS_FROM      => 'qc review pending';
 Readonly::Scalar my $RUN_STATUS_TO        => 'archival pending';
 Readonly::Scalar my $INSTRUMENT_MODEL     => 'NovaSeq';
 Readonly::Scalar my $DEPLEXING_PERCENT_THRESHOLD => 93;
+Readonly::Scalar my $QC_FAILS_PERCENT_THRESHOLD  => 10;
 
 my $dry_run = 1;
 my $study_name;
+my $deplexing_threshold = $DEPLEXING_PERCENT_THRESHOLD;
+my $qc_fails_threshold  = $QC_FAILS_PERCENT_THRESHOLD;
 
 GetOptions('dry_run|dry-run!' => \$dry_run,
            'study_name=s'     => \$study_name,
+           'deplexing_percent_threshold=i' => \$deplexing_threshold,
+           'qc_fails_percent_threshold=i'  => \$qc_fails_threshold,
            'help'             => sub {
              pod2usage(-verbose => 2,
                        -exitval => 0)
@@ -44,6 +49,10 @@ my $logger = Log::Log4perl->get_logger();
 if (not $study_name) {
   $logger->fatal('Study name should be given, use --study_name option');
   exit 1;
+} else {
+  $logger->info("Study name: $study_name");
+  $logger->info("Deplexing percent threshold: $deplexing_threshold");
+  $dry_run and $logger->info('DRY RUN');
 }
 
 my $tracking_schema = npg_tracking::Schema->connect();
@@ -75,15 +84,17 @@ my $get_run_ids = sub {
 my @run_ids = map { $_->id_run } @rows;
 my $placeholders = $run_list->(@run_ids);
 
-my $dbh = WTSI::DNAP::Warehouse::Schema->connect->storage->dbh;
+my $mlwh_schema = WTSI::DNAP::Warehouse::Schema->connect();
+my $dbh = $mlwh_schema->storage->dbh;
 my $query =
    q[select p.id_run, count(distinct s.id_study_lims) as study_count ] .
    q[from iseq_product_metrics p ] .
    q[join iseq_flowcell f on p.id_iseq_flowcell_tmp=f.id_iseq_flowcell_tmp ] .
    q[join study s on s.id_study_tmp=f.id_study_tmp ] .
   qq[where s.name != ? and p.id_run in (${placeholders}) ] .
    q[group by p.id_run having study_count = ?];
-my $sth = $dbh->prepare($query) or croak "Failed to prepare statement: $DBI::errstr";
+my $sth = $dbh->prepare($query) or
+  ($logger->fatal("Failed to prepare statement: $DBI::errstr") and exit 1);
 # Run time database errors are thrown by the execute method, no need to
 # do anything special.
 $sth->execute($EXCLUDE_STUDY_NAME, @run_ids, $NUM_DISTINCT_STUDIES);
@@ -96,7 +107,8 @@ if (@run_ids) {
      q[on p.id_iseq_flowcell_tmp=f.id_iseq_flowcell_tmp ] .
      q[join study s on s.id_study_tmp=f.id_study_tmp ] .
     qq[where s.name = ? and p.id_run in (${placeholders})];
-  $sth = $dbh->prepare($query) or croak "Failed to prepare statement: $DBI::errstr";
+  $sth = $dbh->prepare($query) or
+    ($logger->fatal("Failed to prepare statement: $DBI::errstr") and exit 1);
   $sth->execute($study_name, @run_ids);
   @run_ids = $get_run_ids->($sth);
 }
@@ -106,8 +118,9 @@ if (@run_ids) {
   $query =  q[select distinct(id_run) from iseq_run_lane_metrics ] .
             q[where (tags_decode_percent is null or tags_decode_percent < ?) ] .
            qq[and id_run in (${placeholders})];
-  $sth = $dbh->prepare($query) or croak "Failed to prepare statement: $DBI::errstr";
-  $sth->execute($DEPLEXING_PERCENT_THRESHOLD, @run_ids);
+  $sth = $dbh->prepare($query) or
+    ($logger->fatal("Failed to prepare statement: $DBI::errstr") and exit 1);
+  $sth->execute($deplexing_threshold, @run_ids);
   my $temp = {};
   while (my @data = $sth->fetchrow_array()) {
      $temp->{$data[0]} = 1;
@@ -117,6 +130,28 @@ if (@run_ids) {
   }
 }
 
+if (@run_ids and ($study_name =~ /Heron/xms)) {
+  $logger->info("QC fails percent threshold: $qc_fails_threshold");
+  $logger->info('Candidate runs: ' . join q[, ], @run_ids);
+  load_class 'npg_qc::mqc::skipper';
+  load_class 'npg_qc::Schema';
+  my $skipper = npg_qc::mqc::skipper->new(
+          npg_tracking_schema    => $tracking_schema,
+          mlwh_schema            => $mlwh_schema,
+          qc_schema              => npg_qc::Schema->connect(),
+          qc_fails_threshold     => $qc_fails_threshold,
+          id_runs                => \@run_ids,
+          logger                 => $logger);
+  @run_ids = $skipper->select_runs();
+  $logger->info('Candidate runs that satisfy QC criteria: ' . join q[, ], @run_ids);
+  if (!$dry_run && @run_ids) {
+    @run_ids = $skipper->save_review_results(@run_ids);
+  }
+  if (!$dry_run && @run_ids) {
+    @run_ids = $skipper->set_final_seq_outcomes(@run_ids);
+  }
+}
+
 if (@run_ids) {
   $logger->info(join qq[\n\t], q[],
                 qq[Study '$study_name'],
@@ -201,12 +236,19 @@ npg_mqc_skipper
 
 =item  --dry_run or --dry-run and --no-dry_run and --no-dry-run
 
+=item  --deplexing_percent_threshold 
+
+=item  --qc_fails_percent_threshold
+
+=item  --help
+
 =back
 
 =head1 EXIT STATUS
 
-0 if all runs were processed succesfully, 1 if there are problems while Updating
-any record and the transaction fails to rollback
+0 if all runs were processed succesfully, 1 if there are problems
+getting an initial list of runs or while updating any record and the
+transaction fails to rollback.
 
 =head1 CONFIGURATION
 
@@ -226,6 +268,14 @@ In non-dry run mode the status of this runs is changed to
 'archival pending', ie the manual QC stage of the run life cycle
 is skipped.
 
+Additionally, for studies containing string 'Heron', before the run
+status is changed, an assessment of automatic Robo QC results is
+performed to decide whether the run can be expedited to archival. If
+a run can be moved to archival, preliminary autoqc review results are
+saved to a database, which saves preliminary QC outcomes to the
+database. Following this, sequencing and library QC outcomes are
+finalised for all lanes of this run.  
+
 =head1 SUBROUTINES/METHODS
 
 =head1 DIAGNOSTICS
@@ -244,8 +294,6 @@ is skipped.
 
 =item FindBin
 
-=item Carp
-
 =item Log::Log4perl
 
 =item Getopt::Long
@@ -258,6 +306,8 @@ is skipped.
 
 =item Try::Tiny
 
+=item Class::Load
+
 =item npg_tracking::Schema
 
 =item WTSI::DNAP::Warehouse::Schema
@@ -274,7 +324,7 @@ Marina Gourtovaia
 
 =head1 LICENSE AND COPYRIGHT
 
-Copyright (C) 2019 Genome Research Limited
+Copyright (C) 2019,2020 Genome Research Ltd.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

lib/npg_qc/Schema/Mqc/OutcomeDict.pm

@@ -17,6 +17,11 @@ sub is_final_outcome_description {
   return $desc =~ /$FINAL\Z/smx; #The short description includes the word final.
 }
 
+sub is_rejected_outcome_description {
+  my ($self, $desc) = @_;
+  return $desc =~ /\A$REJECTED/smx;
+}
+
 sub is_final_outcome {
   my $self = shift;
   return $self->is_final_outcome_description($self->short_desc);
@@ -29,7 +34,7 @@ sub is_accepted {
 
 sub is_rejected {
   my $self = shift;
-  return $self->short_desc =~ /\A $REJECTED/smx; #The short description includes the word rejected.
+  return $self->is_rejected_outcome_description($self->short_desc);
 }
 
 sub is_final_accepted {
@@ -114,6 +119,15 @@ __END__
   __PACKAGE__->is_final_outcome_description('Accepted final'); # returns true
   $row->is_final_outcome_description('Accepted preliminary'); # returns false
 
+=head2 is_rejected_outcome_description
+
+  Argument - short qc outcome description.
+  Returns true if the argument describes a rejected outcome, false otherwise.
+  Can be used as both class and instance method.
+
+  __PACKAGE__->is_rejected_outcome_description('Rejected final'); # returns true
+  $row->is_rejected_outcome_description('Accepted preliminary'); # returns false
+
 =head2 is_final_outcome
 
   Utility method to check if the outcome is considered final.