Merge branch 'devel' of https://github.com/wtsi-npg/npg_irods into 'm…

…aster' to create release 2.31.0
wtsi-npg · Aug 11, 2021 · 7204f8a · 7204f8a
2 parents e2eb72c + 090169f
commit 7204f8a
Show file tree

Hide file tree

Showing 45 changed files with 1,348 additions and 50 deletions.
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -19,21 +19,16 @@ jobs:
 
     strategy:
       matrix:
-        perl: [ "5.22.4" ]
-        baton: [ "2.1.0" ]
-        experimental: [ false ]
         include:
           - perl: "5.22.4"
-            baton: "2.1.0"
-            plugins: "201712+irods_4.2.7"
             irods: "4.2.7"
             server_image: "wsinpg/ub-16.04-irods-4.2.7:latest"
+            baton: "2.1.0"
             experimental: false
           - perl: "5.22.4"
-            baton: "3.0.0"
-            plugins: "201712+irods_4.2.8"
             irods: "4.2.8"
             server_image: "wsinpg/ub-18.04-irods-4.2.8:latest"
+            baton: "3.0.1"
             experimental: true
 
     services:
@@ -73,15 +68,14 @@ jobs:
 
           conda config --set auto_update_conda False
           conda config --prepend channels "$WSI_CONDA_CHANNEL"
-          conda config --append channels conda-forge
           conda info
 
       - name: "Install iRODS clients"
         run: |
           conda create -qy -n "$CONDA_TEST_ENVIRONMENT"
           conda install -qy -n "$CONDA_TEST_ENVIRONMENT" "irods-icommands ==${{ matrix.irods }}"
           conda install -qy -n "$CONDA_TEST_ENVIRONMENT" "baton ==${{ matrix.baton }}"
-          conda install -qy -n "$CONDA_TEST_ENVIRONMENT" "libhts-plugins ==${{ matrix.plugins }}"
+          conda install -qy -n "$CONDA_TEST_ENVIRONMENT" libhts-plugins
           conda install -qy -n "$CONDA_TEST_ENVIRONMENT" samtools
           conda install -qy -n "$CONDA_TEST_ENVIRONMENT" tears
 
@@ -179,4 +173,4 @@ jobs:
 
           export WTSI_NPG_iRODS_Test_irodsEnvFile=NULL
           export WTSI_NPG_iRODS_Test_IRODS_ENVIRONMENT_FILE="$HOME/.irods/irods_environment.json"
-          ./Build test
+          ./Build test --verbose
diff --git a/Changes b/Changes
@@ -1,5 +1,13 @@
 Upcoming
 
+Release 2.31.0
+
+ - PacBio - ensure runfolders are writable for md5 files prior to iRODS
+   loading. 
+ - Allow another BAM file prefix in PacBio analysis loading (IsoSeq).
+ - PacBio code to check and change unix permissions on runfolders.
+ - GitHub Actions: verbose testing and update baton clients
+
 Release 2.30.0
 
  - Add substitution_analyis and substition_metrics ancillary files.

diff --git a/MANIFEST b/MANIFEST
@@ -4,7 +4,7 @@
 bin/npg_add_illumina_composition_metadata.pl
 bin/npg_bam_deletion.pl
 bin/npg_pacbio_analysis_monitor.pl
-bin/npg_pacbio_rundelete.pl
+bin/npg_pacbio_run_auxiliary.pl
 bin/npg_pacbio_runmonitor.pl
 bin/npg_process_consent_withdrawn.pl
 bin/npg_publish_bionano_run.pl
@@ -73,6 +73,7 @@ lib/WTSI/NPG/HTS/PacBio/Sequel/ImageArchive.pm
 lib/WTSI/NPG/HTS/PacBio/Sequel/MetaXMLParser.pm
 lib/WTSI/NPG/HTS/PacBio/Sequel/MonitorBase.pm
 lib/WTSI/NPG/HTS/PacBio/Sequel/Reportdata.pm
+lib/WTSI/NPG/HTS/PacBio/Sequel/RunAuditor.pm
 lib/WTSI/NPG/HTS/PacBio/Sequel/RunDeleteMonitor.pm
 lib/WTSI/NPG/HTS/PacBio/Sequel/RunDelete.pm
 lib/WTSI/NPG/HTS/PacBio/Sequel/RunMonitor.pm
@@ -16353,6 +16354,32 @@ t/data/pacbio/sequel_analysis/0000003499/cromwell-job/call-export_bam/execution/
 t/data/pacbio/sequel_analysis/0000003499/cromwell-job/call-export_bam/execution/m64089e_210503_164858.hifi_reads.bam.bai
 t/data/pacbio/sequel_analysis/0000003499/cromwell-job/call-export_bam/execution/m64089e_210503_164858.hifi_reads.bam.pbi
 t/data/pacbio/sequel_analysis/0000003499/cromwell-job/call-export_bam/execution/merged_analysis_report.json
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1001_5p--bc1001_3p.bam
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1001_5p--bc1001_3p.bam.pbi
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1001_5p--bc1001_3p.consensusreadset.xml
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1002_5p--bc1002_3p.bam
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1002_5p--bc1002_3p.bam.pbi
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1002_5p--bc1002_3p.consensusreadset.xml
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1003_5p--bc1003_3p.bam
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1003_5p--bc1003_3p.bam.pbi
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1003_5p--bc1003_3p.consensusreadset.xml
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1004_5p--bc1004_3p.bam
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1004_5p--bc1004_3p.bam.pbi
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1004_5p--bc1004_3p.consensusreadset.xml
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1005_5p--bc1005_3p.bam
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1005_5p--bc1005_3p.bam.pbi
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1005_5p--bc1005_3p.consensusreadset.xml
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1006_5p--bc1006_3p.bam
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1006_5p--bc1006_3p.bam.pbi
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1006_5p--bc1006_3p.consensusreadset.xml
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1008_5p--bc1008_3p.bam
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1008_5p--bc1008_3p.bam.pbi
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1008_5p--bc1008_3p.consensusreadset.xml
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1012_5p--bc1012_3p.bam
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1012_5p--bc1012_3p.bam.pbi
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/fl_transcripts.bc1012_5p--bc1012_3p.consensusreadset.xml
+t/data/pacbio/sequel_analysis/0000004117/cromwell-job/call-lima_isoseq/execution/merged_analysis_report.json
+t/data/pacbio/sequel_analysis/0000004117/entry-points/222a1be3-85b1-12b3-b3bc-9742f8e22f95.consensusreadset.xml
 t/data/pacbio/sequel_analysis/001185/entry-points/2c677e30-8918-4ef1-8649-f44adaac75d3.subreadset.xml
 t/data/pacbio/sequel_analysis/001185/tasks/barcoding.tasks.lima-0/lima.bc1022_BAK8B_OA--bc1022_BAK8B_OA.bam
 t/data/pacbio/sequel_analysis/001185/tasks/barcoding.tasks.lima-0/lima.bc1022_BAK8B_OA--bc1022_BAK8B_OA.bam.pbi
@@ -16449,6 +16476,7 @@ t/lib/WTSI/NPG/HTS/PacBio/Sequel/AnalysisPublisherTest.pm
 t/lib/WTSI/NPG/HTS/PacBio/Sequel/AnalysisReportTest.pm
 t/lib/WTSI/NPG/HTS/PacBio/Sequel/ApiClientTest.pm
 t/lib/WTSI/NPG/HTS/PacBio/Sequel/ImageArchiveTest.pm
+t/lib/WTSI/NPG/HTS/PacBio/Sequel/RunAuditorTest.pm
 t/lib/WTSI/NPG/HTS/PacBio/Sequel/RunDeleteMonitorTest.pm
 t/lib/WTSI/NPG/HTS/PacBio/Sequel/RunDeleteTest.pm
 t/lib/WTSI/NPG/HTS/PacBio/Sequel/RunMonitorTest.pm
@@ -16470,6 +16498,7 @@ t/pacbio_sequel_analysis_publisher.t
 t/pacbio_sequel_analysis_report.t
 t/pacbio_sequel_api_client.t
 t/pacbio_sequel_image_archive.t
+t/pacbio_sequel_run_auditor.t
 t/pacbio_sequel_run_delete_monitor.t
 t/pacbio_sequel_run_delete.t
 t/pacbio_sequel_run_monitor.t

diff --git a/bin/npg_pacbio_rundelete.pl → bin/npg_pacbio_run_auxiliary.pl b/bin/npg_pacbio_rundelete.pl → bin/npg_pacbio_run_auxiliary.pl
@@ -6,18 +6,21 @@
 use FindBin qw[$Bin];
 use lib (-d "$Bin/../lib/perl5" ? "$Bin/../lib/perl5" : "$Bin/../lib");
 use Getopt::Long;
+use List::Util qw[any];
 use Log::Log4perl qw[:levels];
 use Pod::Usage;
 use Readonly;
 
 use WTSI::DNAP::Warehouse::Schema;
 use WTSI::NPG::DriRODS;
 use WTSI::NPG::HTS::PacBio::Sequel::RunDeleteMonitor;
+use WTSI::NPG::HTS::PacBio::Sequel::RunAuditor;
 
 our $VERSION = '';
 
 Readonly::Scalar my $DEFAULT_INTERVAL_DAYS   => 14;
-Readonly::Scalar my $DEFAULT_OLDER_THAN_DAYS => 180;
+Readonly::Scalar my $DEFAULT_OLDER_THAN_DAYS => 60;
+Readonly::Array  my @TYPES => qw(delete audit);
 
 my $api_uri;
 my $collection;
@@ -28,6 +31,7 @@
 my $local_path;
 my $log4perl_config;
 my $older_than = $DEFAULT_OLDER_THAN_DAYS;
+my $task;
 my $verbose;
 
 GetOptions('api-uri|api_uri=s'          => \$api_uri,
@@ -42,6 +46,7 @@
            'local-path|local_path=s'    => \$local_path,
            'logconf=s'                  => \$log4perl_config,
            'older-than|older_than=i'    => \$older_than,
+           'task=s'                     => \$task,
            'verbose'                    => \$verbose);
 
 
@@ -60,6 +65,11 @@
             -exitval => 2);
 }
 
+if (! $task || !(any { /$task/smx } @TYPES)) {
+  pod2usage(-msg => 'A valid task type argument must be defined',
+            -exitval => 2);
+}
+
 my $irods     = WTSI::NPG::DriRODS->new;
 my $wh_schema = WTSI::DNAP::Warehouse::Schema->connect;
 
@@ -80,28 +90,34 @@
   push @init_args, api_uri => $api_uri;
 }
 
-my $deletable = WTSI::NPG::HTS::PacBio::Sequel::RunDeleteMonitor->new(@init_args);
+my ($num_runs, $num_processed, $num_actioned, $num_errors);
 
-my ($num_runs, $num_processed, $num_deleted, $num_errors) = $deletable->delete_runs;
+if ($task eq 'delete') {
+  my $delete = WTSI::NPG::HTS::PacBio::Sequel::RunDeleteMonitor->new(@init_args);
+  ($num_runs, $num_processed, $num_actioned, $num_errors) = $delete->delete_runs;
+} elsif ($task eq 'audit') {
+  my $audit = WTSI::NPG::HTS::PacBio::Sequel::RunAuditor->new(@init_args);
+  ($num_runs, $num_processed, $num_actioned, $num_errors) = $audit->check_runs;
+}
 
 my $log = Log::Log4perl->get_logger('main');
 $log->level($ALL);
 
 my $output = qq{Considered $num_runs, processed $num_processed,}.
-    qq{ deleted $num_deleted with $num_errors errors};
+    qq{ actioned $num_actioned with $num_errors errors};
 
 ($num_errors == 0) ? $log->info($output) : $log->logcroak($output);
 
 
 =head1 NAME
 
-npg_pacbio_rundelete
+npg_pacbio_run_auxiliary
 
 =head1 SYNOPSIS
 
-npg_pacbio_rundelete --local-path </path/to/staging/area>
-  [--api-uri] [--check-format] [--debug] [--dry-run] [--interval days] 
-  [--logconf file] [--older-than days] [--verbose]
+npg_pacbio_run_auxiliary --local-path </path/to/staging/area>
+   --task <task_type>  [--api-uri] [--check-format] [--debug] [--dry-run]
+  [--interval days] [--logconf file] [--older-than days] [--verbose]
 
  Options:
   --api-uri
@@ -125,15 +141,23 @@ =head1 SYNOPSIS
   --logconf       A log4perl configuration file. Optional.
   --older-than
   --older_than    Only consider runs older than a specified number of 
-                  days. Optional defaults to 180 days. 
+                  days. Optional defaults to 60 days.
+  --task          Required. Current permitted options are delete and
+                  audit. 
   --verbose       Print messages while processing. Optional.
 
 
 =head1 DESCRIPTION
 
 Looks for runs between an interval (default 14 days) a specified number of 
-days ago (default 180 days) and deletes run folders which are determined 
-to have been correctly uploaded to iRODS. Only handles run folders from Sequel (I or II) instruments.
+days ago (default 60 days) and carries out the request task. Only handles 
+run folders from Sequel (I or II) instruments.
+
+If the task is 'delete' run folders which are determined to have been correctly 
+uploaded to iRODS are deleted.
+
+If the task is 'audit' run folders are checked - primarily to check correct 
+permissions are set on the directory and correct as necessary.
 
 =head1 SYNOPSIS
 
@@ -142,7 +166,7 @@ =head1 AUTHOR
 
 =head1 COPYRIGHT AND DISCLAIMER
 
-Copyright (C) 2020 Genome Research Limited. All Rights Reserved.
+Copyright (C) 2020, 2021 Genome Research Limited. All Rights Reserved.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the Perl Artistic License or the GNU General

diff --git a/lib/WTSI/NPG/HTS/PacBio/Sequel/AnalysisPublisher.pm b/lib/WTSI/NPG/HTS/PacBio/Sequel/AnalysisPublisher.pm
@@ -33,7 +33,7 @@ our $ENTRY_DIR       = 'entry-points';
 our $WELL_DIRECTORY_PATTERN = '\d+_[A-Z]\d+$';
 
 # Additional sequence filenames permitted for loading 
-our @FNAME_PERMITTED    = qw[removed ccs hifi_reads];
+our @FNAME_PERMITTED    = qw[removed ccs hifi_reads fl_transcripts];
 our @FNAME_NON_DEPLEXED = qw[removed];
 
 # Data processing level