Merge pull request #782 from wtsi-npg/devel

Pull from devel to master to create release 69.9.0
wtsi-npg · Aug 31, 2021 · 048e7a7 · 048e7a7
2 parents 676f773 + 4d22a34
commit 048e7a7
Show file tree

Hide file tree

Showing 14 changed files with 218 additions and 131 deletions.
diff --git a/Changes b/Changes
@@ -1,5 +1,16 @@
 LIST OF CHANGES FOR NPG-QC PACKAGE
 
+release 69.9.0
+ - Change database column type from unsigned int to insigned bigint
+   for columns where we store number of reads and similar values and
+   where we do not already set type to bigint. Some NovaSeq runs produce
+   more data than we envisaged when these tables were set up.
+ - Update pulldown_metrics autoqc check to use the up-to-date Picard
+   metrics via GATK, use CRAM files instead of BAM files as input.  
+ - Drop redundant tag_index column from the tag_metrics QC database
+   table. The results of this QC check are always per lane. Only two
+   distinct values in this culumn - NULL and -1, the latter from pre-
+   composition times.
 
 release 69.8.1
  - bug fix npg_substitution_metrics.pl to not bail on single read data

diff --git a/MANIFEST b/MANIFEST
@@ -1,4 +1,4 @@
-.github/workflows/testing_and_building_repos.yml 
+.github/workflows/testing_and_building_repos.yml
 bin/gtck_combine_gt_data.sh
 bin/gtck_extract_fluidigm_data_from_irods.sh
 bin/gtck_genplexlist.sh
@@ -219,7 +219,7 @@ t/60-autoqc-checks-af_metrics.t
 t/60-autoqc-checks-bam_flagstats.t
 t/60-autoqc-checks-bcfstats.t
 t/60-autoqc-checks-check.t
-t/60-autoqc-checks-interop.t 
+t/60-autoqc-checks-interop.t
 t/60-autoqc-checks-gc_bias.t
 t/60-autoqc-checks-gc_fraction.t
 t/60-autoqc-checks-generic.t
@@ -413,7 +413,7 @@ t/data/autoqc/alignment.bam
 t/data/autoqc/alignment.sam
 t/data/autoqc/alignment_small.sam
 t/data/autoqc/fixtures/Adapter.yml
-t/data/autoqc/gbs_plex_repository/Hs_MajorQC/default/all/bcftools/Hs_MajorQC.annotation.vcf 
+t/data/autoqc/gbs_plex_repository/Hs_MajorQC/default/all/bcftools/Hs_MajorQC.annotation.vcf
 t/data/autoqc/gbs_plex_repository/Hs_MajorQC/default/all/bcftools/Hs_MajorQC.info.json
 t/data/autoqc/gbs_plex_repository/Hs_MajorQC/default/all/bcftools/Hs_MajorQC.ploidy
 t/data/autoqc/gbs_plex_repository/Hs_MajorQC/default/all/fasta/Hs_MajorQC.fa
@@ -477,6 +477,7 @@ t/data/autoqc/npg/run/2222.xml
 t/data/autoqc/npg/run/2549.xml
 t/data/autoqc/npg/run/3612.xml
 t/data/autoqc/pulldown_metrics/pdm.out
+t/data/autoqc/pulldown_metrics/samplesheet_pulldown.csv
 t/data/autoqc/qc/1937_3.map
 t/data/autoqc/qX_yield/npg/instrument/15.xml
 t/data/autoqc/qX_yield/npg/run/2549.xml

diff --git a/lib/npg_qc/Schema/Result/Adapter.pm b/lib/npg_qc/Schema/Result/Adapter.pm
@@ -102,13 +102,13 @@ A foreign key referencing the id_seq_composition column of the seq_composition t
 
 =head2 forward_fasta_read_count
 
-  data_type: 'integer'
+  data_type: 'bigint'
   extra: {unsigned => 1}
   is_nullable: 1
 
 =head2 forward_contaminated_read_count
 
-  data_type: 'integer'
+  data_type: 'bigint'
   extra: {unsigned => 1}
   is_nullable: 1
 
@@ -124,13 +124,13 @@ A foreign key referencing the id_seq_composition column of the seq_composition t
 
 =head2 reverse_fasta_read_count
 
-  data_type: 'integer'
+  data_type: 'bigint'
   extra: {unsigned => 1}
   is_nullable: 1
 
 =head2 reverse_contaminated_read_count
 
-  data_type: 'integer'
+  data_type: 'bigint'
   extra: {unsigned => 1}
   is_nullable: 1
 
@@ -192,17 +192,17 @@ __PACKAGE__->add_columns(
   'reverse_read_filename',
   { data_type => 'varchar', is_nullable => 1, size => 256 },
   'forward_fasta_read_count',
-  { data_type => 'integer', extra => { unsigned => 1 }, is_nullable => 1 },
+  { data_type => 'bigint', extra => { unsigned => 1 }, is_nullable => 1 },
   'forward_contaminated_read_count',
-  { data_type => 'integer', extra => { unsigned => 1 }, is_nullable => 1 },
+  { data_type => 'bigint', extra => { unsigned => 1 }, is_nullable => 1 },
   'forward_blat_hash',
   { data_type => 'text', is_nullable => 1 },
   'forward_start_counts',
   { data_type => 'text', is_nullable => 1 },
   'reverse_fasta_read_count',
-  { data_type => 'integer', extra => { unsigned => 1 }, is_nullable => 1 },
+  { data_type => 'bigint', extra => { unsigned => 1 }, is_nullable => 1 },
   'reverse_contaminated_read_count',
-  { data_type => 'integer', extra => { unsigned => 1 }, is_nullable => 1 },
+  { data_type => 'bigint', extra => { unsigned => 1 }, is_nullable => 1 },
   'reverse_blat_hash',
   { data_type => 'text', is_nullable => 1 },
   'reverse_start_counts',
@@ -280,8 +280,8 @@ __PACKAGE__->belongs_to(
 with 'npg_qc::Schema::Composition', 'npg_qc::Schema::Flators', 'npg_qc::autoqc::role::result', 'npg_qc::autoqc::role::adapter';
 
 
-# Created by DBIx::Class::Schema::Loader v0.07047 @ 2017-09-14 16:25:18
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qBi6HgQ59WjoyG18q6NmVA
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-19 12:16:50
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:jxMC400VgMoWLi7VZ1BQvg
 
 our $VERSION = '0';
 

diff --git a/lib/npg_qc/Schema/Result/PulldownMetrics.pm b/lib/npg_qc/Schema/Result/PulldownMetrics.pm
@@ -423,7 +423,7 @@ Marina Gourtovaia E<lt>[email protected]<gt>
 
 =head1 LICENSE AND COPYRIGHT
 
-Copyright (C) 2017 GRL
+Copyright (C) 2017, 2021 Genome Research Ltd.
 
 This file is part of NPG.
 

diff --git a/lib/npg_qc/Schema/Result/QXYield.pm b/lib/npg_qc/Schema/Result/QXYield.pm
@@ -120,37 +120,37 @@ A foreign key referencing the id_seq_composition column of the seq_composition t
 
 =head2 yield1
 
-  data_type: 'integer'
+  data_type: 'bigint'
   extra: {unsigned => 1}
   is_nullable: 1
 
 =head2 yield2
 
-  data_type: 'integer'
+  data_type: 'bigint'
   extra: {unsigned => 1}
   is_nullable: 1
 
 =head2 yield1_q30
 
-  data_type: 'integer'
+  data_type: 'bigint'
   extra: {unsigned => 1}
   is_nullable: 1
 
 =head2 yield2_q30
 
-  data_type: 'integer'
+  data_type: 'bigint'
   extra: {unsigned => 1}
   is_nullable: 1
 
 =head2 yield1_q40
 
-  data_type: 'integer'
+  data_type: 'bigint'
   extra: {unsigned => 1}
   is_nullable: 1
 
 =head2 yield2_q40
 
-  data_type: 'integer'
+  data_type: 'bigint'
   extra: {unsigned => 1}
   is_nullable: 1
 
@@ -208,17 +208,17 @@ __PACKAGE__->add_columns(
   'threshold_yield2',
   { data_type => 'integer', extra => { unsigned => 1 }, is_nullable => 1 },
   'yield1',
-  { data_type => 'integer', extra => { unsigned => 1 }, is_nullable => 1 },
+  { data_type => 'bigint', extra => { unsigned => 1 }, is_nullable => 1 },
   'yield2',
-  { data_type => 'integer', extra => { unsigned => 1 }, is_nullable => 1 },
+  { data_type => 'bigint', extra => { unsigned => 1 }, is_nullable => 1 },
   'yield1_q30',
-  { data_type => 'integer', extra => { unsigned => 1 }, is_nullable => 1 },
+  { data_type => 'bigint', extra => { unsigned => 1 }, is_nullable => 1 },
   'yield2_q30',
-  { data_type => 'integer', extra => { unsigned => 1 }, is_nullable => 1 },
+  { data_type => 'bigint', extra => { unsigned => 1 }, is_nullable => 1 },
   'yield1_q40',
-  { data_type => 'integer', extra => { unsigned => 1 }, is_nullable => 1 },
+  { data_type => 'bigint', extra => { unsigned => 1 }, is_nullable => 1 },
   'yield2_q40',
-  { data_type => 'integer', extra => { unsigned => 1 }, is_nullable => 1 },
+  { data_type => 'bigint', extra => { unsigned => 1 }, is_nullable => 1 },
   'pass',
   { data_type => 'tinyint', is_nullable => 1 },
   'comments',
@@ -292,8 +292,8 @@ __PACKAGE__->belongs_to(
 with 'npg_qc::Schema::Composition', 'npg_qc::Schema::Flators', 'npg_qc::autoqc::role::result', 'npg_qc::autoqc::role::qX_yield';
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2018-08-29 13:44:29
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:NBYWaBVvG2S3Etgm+Jj4sA
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-19 12:16:50
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LHzBwbDWKgAAeb8bUrM8og
 
 our $VERSION = '0';
 

diff --git a/lib/npg_qc/Schema/Result/TagMetrics.pm b/lib/npg_qc/Schema/Result/TagMetrics.pm
@@ -183,11 +183,6 @@ A foreign key referencing the id_seq_composition column of the seq_composition t
   data_type: 'text'
   is_nullable: 1
 
-=head2 tag_index
-
-  data_type: 'bigint'
-  is_nullable: 1
-
 =head2 tag_hops_percent
 
   data_type: 'float'
@@ -259,8 +254,6 @@ __PACKAGE__->add_columns(
   { data_type => 'text', is_nullable => 1 },
   'info',
   { data_type => 'text', is_nullable => 1 },
-  'tag_index',
-  { data_type => 'bigint', is_nullable => 1 },
   'tag_hops_percent',
   { data_type => 'float', extra => { unsigned => 1 }, is_nullable => 1 },
   'tag_hops_power',
@@ -330,8 +323,8 @@ __PACKAGE__->belongs_to(
 with 'npg_qc::Schema::Composition', 'npg_qc::Schema::Flators', 'npg_qc::autoqc::role::result', 'npg_qc::autoqc::role::tag_metrics';
 
 
-# Created by DBIx::Class::Schema::Loader v0.07048 @ 2018-04-10 14:09:37
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:01BT+8rbCPfUeB3ItyFNjQ
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-19 13:25:57
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:X85s6Sx84dZtx0r2CpnPQA
 
 our $VERSION = '0';