Merge pull request #396 from wtsi-npg/devel

merge devel to master, preparation for release 62.6

.travis.yml

@@ -1,26 +1,49 @@
+
 # This file was adapted from work by Keith James (keithj). The original source
 # can be found as part of the wtsi-npg/data_handling project here:
 #
 #   https://github.com/wtsi-npg/data_handling
+#
+# iRODS setup added by Iain Bancarz (ib5), adapted from baton:
+#   https://github.com/wtsi-npg/baton
 
 sudo: required
 
 language: perl
 
 perl:
   - "5.16"
+  - "5.22-shrplib"
+
+addons:
+  postgresql: "9.3"
 
 env:
   global:
-    - DNAP_UTILITIES_VERSION="0.5.1"
-  matrix:
-    - TRAVIS_NODE_VERSION="0.12.9"
+    - secure: cDfB188ECmloGfScZQezqwiFef7l+gXgn2RiTOxINriy9wYS6RmZxuZBHGuR36u7QV3QEJtMdihyQ+XBN2eQPf5jULQXV15t7gArXEVPzzF8i+f8MTgVHugU3TqmPLQkY94wBBbpzvRD9xCAC/uNiQcLLwuD2SjPfTXkqgqqtd0=
+    - PGVERSION="9.3"
+    - DISPOSABLE_IRODS_VERSION="1.2"
+    - JANSSON_VERSION="2.7"
+    - BATON_VERSION="0.17.0"
     - TRAVIS_NODE_VERSION="4.5.0"
+    - RENCI_URL=ftp://ftp.renci.org
+    - WTSI_NPG_GITHUB_URL=https://github.com/wtsi-npg
+
+  matrix:
+    - IRODS_VERSION=3.3.1 IRODS_RIP_DIR=/usr/local/irods
+    - IRODS_VERSION=4.1.10 PG_PLUGIN_VERSION=1.10 PLATFORM=ubuntu12
 
 before_install:
- - ./.travis/before_install.sh
+  - ./.travis/before_install_mysql57.sh
+  - ./.travis/before_install.sh
+
 install:
- - ./.travis/install.sh
+  - ./.travis/install.sh
+
 script:
- - ./.travis/script.sh
+  - export PATH=$IRODS_RIP_DIR/iRODS/clients/icommands/bin:$PATH
+  - ienv
+  - ils
+  - ilsresc -l
+  - ./.travis/script.sh
 

.travis/before_install.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -e -x
+# set -u fails cause npm install to fail
 
 sudo apt-get update -qq
 

.travis/before_install_mysql57.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -e -x
+set -e -u -x
 
 sudo apt-key add ./.travis/mysql_pubkey.asc # Import public key for MySQL
 sudo apt-get remove --purge "^mysql.*"
@@ -11,20 +11,30 @@ sudo apt-get autoclean
 sudo rm -rf /var/lib/mysql
 sudo rm -rf /var/log/mysql
 
-# Specify which version we want
-echo "deb http://repo.mysql.com/apt/ubuntu/ precise mysql-5.7" | sudo tee /etc/apt/sources.list.d/mysql.list
-sudo -E apt-get update -q
-sudo -E apt-get -q -y install mysql-server libmysqlclient-dev libdbd-mysql-perl
+# Get packages from oracle
+sudo -E apt-get install libaio1 apparmor
+wget http://downloads.mysql.com/archives/get/file/mysql-common_5.7.13-1ubuntu12.04_amd64.deb
+wget http://downloads.mysql.com/archives/get/file/libmysqlclient20_5.7.13-1ubuntu12.04_amd64.deb
+wget http://downloads.mysql.com/archives/get/file/mysql-community-client_5.7.13-1ubuntu12.04_amd64.deb
+wget http://downloads.mysql.com/archives/get/file/mysql-client_5.7.13-1ubuntu12.04_amd64.deb
+wget http://downloads.mysql.com/archives/get/file/mysql-community-server_5.7.13-1ubuntu12.04_amd64.deb
+wget http://downloads.mysql.com/archives/get/file/mysql-server_5.7.13-1ubuntu12.04_amd64.deb
+sudo debconf-set-selections <<< "mysql-server mysql-server/root_password password \"''\""
+sudo debconf-set-selections <<< "mysql-server mysql-server/root_password_again password \"''\""
+sudo dpkg-preconfigure mysql-community-server_5.7.13-1ubuntu12.04_amd64.deb  
+sudo dpkg -i mysql-{common,community-client,client,community-server,server}_*.deb
+sudo dpkg -i libmysqlclient20_5.7.13-1ubuntu12.04_amd64.deb
+sudo -E apt-get -q -y install libmysqlclient-dev libdbd-mysql-perl
 
 sudo /etc/init.d/mysql stop
 sleep 5
 # Start the server without password
-sudo mysqld --skip-grant-tables --explicit_defaults_for_timestamp=false --sql-mode="STRICT_TRANS_TABLES" &
+sudo mysqld --user=root --skip-grant-tables --explicit_defaults_for_timestamp=false --sql-mode="STRICT_TRANS_TABLES" &
 
 #Give some time for server to start
 sleep 10
 
 # print versions
 mysql --version
-mysql -e "SELECT @@GLOBAL.sql_mode; SELECT @@SESSION.sql_mode; SHOW GLOBAL VARIABLES LIKE '%version%'; show variables like '%time%';" -uroot
+mysql -h localhost -e "SELECT @@GLOBAL.sql_mode; SELECT @@SESSION.sql_mode; SHOW GLOBAL VARIABLES LIKE '%version%'; show variables like '%time%';" -uroot
 

.travis/install.sh

@@ -4,8 +4,16 @@
 # can be found as part of the wtsi-npg/data_handling project here:
 #
 #   https://github.com/wtsi-npg/data_handling
+#
+# iRODS setup added by Iain Bancarz (ib5), adapted from baton:
+#   https://github.com/wtsi-npg/baton
+
+set -e -u -x
 
-set -e -x
+# The default build branch for all repositories. This defaults to
+# TRAVIS_BRANCH unless set in the Travis build environment.
+WTSI_NPG_BUILD_BRANCH=${WTSI_NPG_BUILD_BRANCH:=$TRAVIS_BRANCH}
+IRODS_RIP_DIR=${IRODS_RIP_DIR:+$IRODS_RIP_DIR}
 
 sudo apt-get install libgd2-xpm-dev # For npg_tracking
 sudo apt-get install liblzma-dev # For npg_qc
@@ -16,24 +24,61 @@ cpanm --notest --reinstall App::cpanminus
 cpanm --quiet --notest --reinstall ExtUtils::ParseXS
 cpanm --quiet --notest --reinstall MooseX::Role::Parameterized
 cpanm --quiet --notest Alien::Tidyp
-cpanm --no-lwp --notest https://github.com/wtsi-npg/perl-dnap-utilities/releases/download/${DNAP_UTILITIES_VERSION}/WTSI-DNAP-Utilities-${DNAP_UTILITIES_VERSION}.tar.gz
 
-# WTSI NPG Perl repo dependencies
-cd /tmp
-git clone --branch devel --depth 1 https://github.com/wtsi-npg/ml_warehouse.git ml_warehouse.git
-#git clone --branch devel --depth 1 https://github.com/wtsi-npg/npg_tracking.git npg_tracking.git
-git clone --branch subset_in_factories --depth 1 https://github.com/mgcam/npg_tracking.git npg_tracking.git
-git clone --branch devel --depth 1 https://github.com/wtsi-npg/npg_seq_common.git npg_seq_common.git
+# iRODS
+wget -q https://github.com/wtsi-npg/disposable-irods/releases/download/${DISPOSABLE_IRODS_VERSION}/disposable-irods-${DISPOSABLE_IRODS_VERSION}.tar.gz -O /tmp/disposable-irods-${DISPOSABLE_IRODS_VERSION}.tar.gz
+tar xfz /tmp/disposable-irods-${DISPOSABLE_IRODS_VERSION}.tar.gz -C /tmp
+cd /tmp/disposable-irods-${DISPOSABLE_IRODS_VERSION}
+./scripts/download_and_verify_irods.sh
+./scripts/install_irods.sh
+./scripts/configure_irods.sh
+
+# Jansson
+wget -q https://github.com/akheron/jansson/archive/v${JANSSON_VERSION}.tar.gz -O /tmp/jansson-${JANSSON_VERSION}.tar.gz
+tar xfz /tmp/jansson-${JANSSON_VERSION}.tar.gz -C /tmp
+cd /tmp/jansson-${JANSSON_VERSION}
+autoreconf -fi
+./configure ; make ; sudo make install
+sudo ldconfig
+
+# baton
+wget -q https://github.com/wtsi-npg/baton/releases/download/${BATON_VERSION}/baton-${BATON_VERSION}.tar.gz -O /tmp/baton-${BATON_VERSION}.tar.gz
+tar xfz /tmp/baton-${BATON_VERSION}.tar.gz -C /tmp
+cd /tmp/baton-${BATON_VERSION}
 
-repos="/tmp/ml_warehouse.git /tmp/npg_tracking.git /tmp/npg_seq_common.git"
+IRODS_HOME=
+baton_irods_conf="--with-irods"
+
+if [ -n "$IRODS_RIP_DIR" ]
+then
+    export IRODS_HOME="$IRODS_RIP_DIR/iRODS"
+    baton_irods_conf="--with-irods=$IRODS_HOME"
+fi
+
+./configure ${baton_irods_conf} ; make ; sudo make install
+sudo ldconfig
+
+# WTSI NPG Perl repo dependencies
+repos=""
+for repo in perl-dnap-utilities ml_warehouse npg_tracking npg_seq_common perl-irods-wrap; do
+    cd /tmp
+    # Always clone master when using depth 1 to get current tag
+    git clone --branch master --depth 1 ${WTSI_NPG_GITHUB_URL}/${repo}.git ${repo}.git
+    cd /tmp/${repo}.git
+    # Shift off master to appropriate branch (if possible)
+    git ls-remote --heads --exit-code origin ${WTSI_NPG_BUILD_BRANCH} && git pull origin ${WTSI_NPG_BUILD_BRANCH} && echo "Switched to branch ${WTSI_NPG_BUILD_BRANCH}"
+    repos=$repos" /tmp/${repo}.git"
+done
 
+# Finally, bring any common dependencies up to the latest version and
+# install
 for repo in $repos
 do
-  cd "$repo"
-  cpanm --quiet --notest --installdeps . || find /home/travis/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20  {} \;
-  perl Build.PL
-  ./Build
-  ./Build install
+    cd $repo
+    cpanm --quiet --notest --installdeps . || find /home/travis/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20  {} \;		
+    perl Build.PL
+    ./Build		
+    ./Build install
 done
 
-cd "$TRAVIS_BUILD_DIR"
+cd $TRAVIS_BUILD_DIR

.travis/script.sh

@@ -1,10 +1,14 @@
 #!/bin/bash
 
-set -e -x
+set -e -u -x
 
 unset PERL5LIB
 
 export PATH=/home/travis/.nvm/versions/node/v${TRAVIS_NODE_VERSION}/bin:$PATH
+export TEST_AUTHOR=1
+export WTSI_NPG_iRODS_Test_irodsEnvFile=$HOME/.irods/.irodsEnv
+export WTSI_NPG_iRODS_Test_IRODS_ENVIRONMENT_FILE=$HOME/.irods/irods_environment.json
+export WTSI_NPG_iRODS_Test_Resource=testResc
 
 cpanm --notest --installdeps . || find /home/travis/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20  {} \;
 perl Build.PL

Build.PL

@@ -244,7 +244,9 @@ my $extended_requires = {
                 'npg_tracking::data::bait::find'           => 0,
                 'npg_tracking::data::snv::find'            => 0,
                 'npg_tracking::glossary::composition'      => 0,
+                'npg_tracking::glossary::composition::factory' => 0,
                 'npg_tracking::glossary::composition::factory::attributes' => 0,
+                'npg_tracking::glossary::composition::component'           => 0,
                 'npg_tracking::glossary::composition::component::illumina' => 0,
                 'npg_tracking::glossary::rpt'              => 0,
                 'npg_tracking::glossary::run'              => 0,
@@ -259,6 +261,7 @@ my $extended_requires = {
                 'st::api::base'                            => 0,
                 'st::api::lims'                            => 0,
                 'WTSI::DNAP::Warehouse::Schema'            => 0,
+                'WTSI::NPG::iRODS'                         => 0,
 };
 
 my $build_requires = {

Changes

@@ -1,5 +1,24 @@
 LIST OF CHANGES FOR NPG-QC PACKAGE
 
+ - make building the iRODS handle of npg_qc::autoqc::checks::genotype
+   lazy to avoid calling baton until required at runtime
+ - translation from a database composition representation to the
+     npg_tracking::glossary::composition type object
+ - db query for compisition-based tables should include a condition
+     on tag_index if only lanes or plexes are being retrieved
+ - Fix in npg_qc::utils::bam_genotype for rare out of order mpileup failures 
+ - Travis CI build: when building dependencies, us the same
+   branch as the one the pull request is made to.
+ - back to testing with mysql 5.7.13 in travis
+ - drop node 0.12 from node travis matrix
+ - add perl 5.22-shrplib to travis matrix
+ - to reduce uncertainty in ranking close results, increase precision
+   used for ranking contamination results
+ - call_gtck_composite_rpt.pl - supply rpt list to genotype check
+ - Code and tests changes to reduce number of warnings under Perl 5.22.2
+ - sequence mismatch - do not return PDL special values, treat NaN as
+   a bad value and return undefined instead
+
 release 62.5
  - a very concise replacement for NPG_QC pages
  - no templates rna_seqc results - we should not try rendering

MANIFEST

@@ -291,7 +291,6 @@ lib/npg_qc/Schema/ResultSet.pm
 lib/npg_qc/Schema/ResultSet/MqcOutcomeEnt.pm
 lib/npg_qc/util.pm
 lib/npg_qc/utils/bam_genotype.pm
-lib/npg_qc/utils/iRODS.pm
 lib/npg_qc/view/analysis.pm
 lib/npg_qc/view/cache_query.pm
 lib/npg_qc/view/create_xml.pm

bin/call_gtck_composite_rpt.pl

@@ -49,8 +49,9 @@
 ##########
 my @bam_file_list;
 my $ext = $opts{c}? q[cram]: q[bam];
-if($opts{r}) {
-	@bam_file_list = map { my ($r, $p, $t) = (split ":", $_); find_runlanefolder($r, $p, $t, $ext) or sprintf "irods:/seq/%d/%d_%d%s.%s", $r, $r, $p, $t? "#$t": "", $ext; } (split ";", $opts{r});
+my $rpt_list = $opts{r};
+if($rpt_list) {
+	@bam_file_list = map { my ($r, $p, $t) = (split ":", $_); find_runlanefolder($r, $p, $t, $ext) or sprintf "irods:/seq/%d/%d_%d%s.%s", $r, $r, $p, $t? "#$t": "", $ext; } (split ";", $rpt_list);
 
 	carp qq[bam_file_list:\n\t], join("\n\t", @bam_file_list), "\n";
 }
@@ -92,7 +93,6 @@
 	alignments_in_bam => 1,
 	reference_fasta => $reference_genome,
 	input_files => [ (@bam_file_list) ],
-	path => q[.],
 );
 if(defined $plex_name) {
 	$attribs{sequenom_plex} = $plex_name;
@@ -109,6 +109,8 @@
 	$attribs{genotype_executables_path} = $gt_exec_path;
 }
 
+$attribs{rpt_list} = $rpt_list;
+
 my $gtck= npg_qc::autoqc::checks::genotype->new(%attribs);
 
 exit 0 unless($gtck->can_run);

lib/npg_qc/Schema/Flators.pm

@@ -134,7 +134,7 @@ setting the inflator for compressed scalar data.
 =cut
 
 sub compress_xz {
-  my ($package, $data) = @_;
+  my ($package_name, $data) = @_;
   my $out;
   if (defined $data) {
     xz \$data => \$out or croak "xz failed: $XzError\n";
@@ -169,6 +169,41 @@ sub set_inflator4scalar {
   return;
 }
 
+=head2 create_composition_attribute
+
+Adds a lazy-build 'composition' attribute and a build method for it
+for tables that have 'iseq_composition' column - an attempt to 
+mirror the functionality of the result objects and hide the complexity
+of database implementation for composition.
+
+ __PACKAGE__->create_composition_attribute();
+
+=cut
+
+sub create_composition_attribute {
+  my $package_name = shift;
+
+  if (!$package_name->result_source_instance()->has_column('id_seq_composition')) {
+    croak "Cannot create composition attribute for $package_name";
+  }
+
+  my $meta = $package_name->meta();
+  $meta->add_attribute(
+    'composition' => ( is         => 'ro',
+                       isa        => 'npg_tracking::glossary::composition',
+                       lazy_build => 1,
+                     )
+  );
+  $meta->add_method(
+    '_build_composition' => sub {
+      my $self = shift;
+      return $self->seq_composition()->create_composition();
+    }
+  );
+
+  return;
+}
+
 1;
 
 __END__
@@ -207,7 +242,7 @@ Marina Gourtovaia
 
 =head1 LICENSE AND COPYRIGHT
 
-Copyright (C) 2015 GRL
+Copyright (C) 2016 GRL
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

lib/npg_qc/Schema/Result/SamtoolsStats.pm

@@ -160,7 +160,8 @@ __PACKAGE__->belongs_to(
 
 # You can replace this text with custom code or comments, and it will be preserved on regeneration
 
-with 'npg_qc::Schema::Flators';
+with qw/ npg_qc::Schema::Flators
+         npg_qc::autoqc::role::result /;
 
 our $VERSION = '0';
 
@@ -215,6 +216,15 @@ around 'set_inflated_columns' => sub {
 
 __PACKAGE__->set_inflator4xz_compressed_scalar(qw(stats));
 
+=head2 composition
+
+An lazy-build attribute representing a composition this result
+corresponds to. 
+
+=cut
+
+__PACKAGE__->create_composition_attribute();
+
 __PACKAGE__->meta->make_immutable;
 1;
 __END__