Merge pull request #388 from iainrb/irods_travis_and_interface_2

Use disposable-irods in Travis build; use perl-irods-wrap

.travis.yml

@@ -1,4 +1,11 @@
-# This file was adapted from work by Keith James (keithj)
+
+# This file was adapted from work by Keith James (keithj). The original source
+# can be found as part of the wtsi-npg/data_handling project here:
+#
+#   https://github.com/wtsi-npg/data_handling
+#
+# iRODS setup added by Iain Bancarz (ib5), adapted from baton:
+#   https://github.com/wtsi-npg/baton
 
 sudo: required
 
@@ -8,17 +15,35 @@ perl:
   - "5.16"
   - "5.22-shrplib"
 
+addons:
+  postgresql: "9.3"
+
 env:
   global:
+    - secure: cDfB188ECmloGfScZQezqwiFef7l+gXgn2RiTOxINriy9wYS6RmZxuZBHGuR36u7QV3QEJtMdihyQ+XBN2eQPf5jULQXV15t7gArXEVPzzF8i+f8MTgVHugU3TqmPLQkY94wBBbpzvRD9xCAC/uNiQcLLwuD2SjPfTXkqgqqtd0=
+    - PGVERSION="9.3"
+    - DISPOSABLE_IRODS_VERSION="1.2"
+    - JANSSON_VERSION="2.7"
+    - BATON_VERSION="0.17.0"
+    - TRAVIS_NODE_VERSION="4.5.0"
+    - RENCI_URL=ftp://ftp.renci.org
     - WTSI_NPG_GITHUB_URL=https://github.com/wtsi-npg
+
   matrix:
-    - TRAVIS_NODE_VERSION="4.5.0"
+    - IRODS_VERSION=3.3.1 IRODS_RIP_DIR=/usr/local/irods
+    - IRODS_VERSION=4.1.10 PG_PLUGIN_VERSION=1.10 PLATFORM=ubuntu12
 
 before_install:
- - ./.travis/before_install_mysql57.sh
- - ./.travis/before_install.sh
+  - ./.travis/before_install_mysql57.sh
+  - ./.travis/before_install.sh
+
 install:
- - ./.travis/install.sh
+  - ./.travis/install.sh
+
 script:
- - ./.travis/script.sh
+  - export PATH=$IRODS_RIP_DIR/iRODS/clients/icommands/bin:$PATH
+  - ienv
+  - ils
+  - ilsresc -l
+  - ./.travis/script.sh
 

.travis/install.sh

@@ -4,9 +4,16 @@
 # can be found as part of the wtsi-npg/data_handling project here:
 #
 #   https://github.com/wtsi-npg/data_handling
+#
+# iRODS setup added by Iain Bancarz (ib5), adapted from baton:
+#   https://github.com/wtsi-npg/baton
 
 set -e -x
 
+# The default build branch for all repositories. This defaults to
+# TRAVIS_BRANCH unless set in the Travis build environment.
+WTSI_NPG_BUILD_BRANCH=${WTSI_NPG_BUILD_BRANCH:=$TRAVIS_BRANCH}
+
 sudo apt-get install libgd2-xpm-dev # For npg_tracking
 sudo apt-get install liblzma-dev # For npg_qc
 sudo apt-get install --yes nodejs
@@ -17,24 +24,59 @@ cpanm --quiet --notest --reinstall ExtUtils::ParseXS
 cpanm --quiet --notest --reinstall MooseX::Role::Parameterized
 cpanm --quiet --notest Alien::Tidyp
 
-# Git branch to merge to or custom branch
-WTSI_NPG_BUILD_BRANCH=${WTSI_NPG_BUILD_BRANCH:=$TRAVIS_BRANCH}
+# iRODS
+wget -q https://github.com/wtsi-npg/disposable-irods/releases/download/${DISPOSABLE_IRODS_VERSION}/disposable-irods-${DISPOSABLE_IRODS_VERSION}.tar.gz -O /tmp/disposable-irods-${DISPOSABLE_IRODS_VERSION}.tar.gz
+tar xfz /tmp/disposable-irods-${DISPOSABLE_IRODS_VERSION}.tar.gz -C /tmp
+cd /tmp/disposable-irods-${DISPOSABLE_IRODS_VERSION}
+./scripts/download_and_verify_irods.sh
+./scripts/install_irods.sh
+./scripts/configure_irods.sh
+
+# Jansson
+wget -q https://github.com/akheron/jansson/archive/v${JANSSON_VERSION}.tar.gz -O /tmp/jansson-${JANSSON_VERSION}.tar.gz
+tar xfz /tmp/jansson-${JANSSON_VERSION}.tar.gz -C /tmp
+cd /tmp/jansson-${JANSSON_VERSION}
+autoreconf -fi
+./configure ; make ; sudo make install
+sudo ldconfig
+
+# baton
+wget -q https://github.com/wtsi-npg/baton/releases/download/${BATON_VERSION}/baton-${BATON_VERSION}.tar.gz -O /tmp/baton-${BATON_VERSION}.tar.gz
+tar xfz /tmp/baton-${BATON_VERSION}.tar.gz -C /tmp
+cd /tmp/baton-${BATON_VERSION}
+
+IRODS_HOME=
+baton_irods_conf="--with-irods"
+
+if [ -n "$IRODS_RIP_DIR" ]
+then
+    export IRODS_HOME="$IRODS_RIP_DIR/iRODS"
+    baton_irods_conf="--with-irods=$IRODS_HOME"
+fi
+
+./configure ${baton_irods_conf} ; make ; sudo make install
+sudo ldconfig
+
 # WTSI NPG Perl repo dependencies
-repos="perl-dnap-utilities ml_warehouse npg_tracking npg_seq_common"
+for repo in perl-dnap-utilities ml_warehouse npg_tracking npg_seq_common perl-irods-wrap; do
+    cd /tmp
+    # Always clone master when using depth 1 to get current tag
+    git clone --branch master --depth 1 ${WTSI_NPG_GITHUB_URL}/${repo}.git ${repo}.git
+    cd /tmp/${repo}.git
+    # Shift off master to appropriate branch (if possible)
+    git ls-remote --heads --exit-code origin ${WTSI_NPG_BUILD_BRANCH} && git pull origin ${WTSI_NPG_BUILD_BRANCH} && echo "Switched to branch ${WTSI_NPG_BUILD_BRANCH}"
+    repos=$repos" /tmp/${repo}.git"
+done
+
+# Finally, bring any common dependencies up to the latest version and
+# install
 for repo in $repos
 do
-  # Logic of keeping branch consistent was taken from @dkj
-  # contribution to https://github.com/wtsi-npg/npg_irods
-  cd /tmp
-  # Always clone master when using depth 1 to get current tag
-  git clone --branch master --depth 1 ${WTSI_NPG_GITHUB_URL}/${repo}.git ${repo}.git
-  cd /tmp/${repo}.git
-  # Shift off master to appropriate branch (if possible)
-  git ls-remote --heads --exit-code origin ${WTSI_NPG_BUILD_BRANCH} && git pull origin ${WTSI_NPG_BUILD_BRANCH} && echo "Switched to branch ${WTSI_NPG_BUILD_BRANCH}"
-  cpanm --quiet --notest --installdeps . || find /home/travis/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20  {} \;
-  perl Build.PL
-  ./Build
-  ./Build install
+    cd $repo
+    cpanm --quiet --notest --installdeps . || find /home/travis/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20  {} \;		
+    perl Build.PL
+    ./Build		
+    ./Build install
 done
 
-cd "$TRAVIS_BUILD_DIR"
+cd $TRAVIS_BUILD_DIR

.travis/script.sh

@@ -5,6 +5,10 @@ set -e -x
 unset PERL5LIB
 
 export PATH=/home/travis/.nvm/versions/node/v${TRAVIS_NODE_VERSION}/bin:$PATH
+export TEST_AUTHOR=1
+export WTSI_NPG_iRODS_Test_irodsEnvFile=$HOME/.irods/.irodsEnv
+export WTSI_NPG_iRODS_Test_IRODS_ENVIRONMENT_FILE=$HOME/.irods/irods_environment.json
+export WTSI_NPG_iRODS_Test_Resource=testResc
 
 cpanm --notest --installdeps . || find /home/travis/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20  {} \;
 perl Build.PL

MANIFEST

@@ -291,7 +291,6 @@ lib/npg_qc/Schema/ResultSet.pm
 lib/npg_qc/Schema/ResultSet/MqcOutcomeEnt.pm
 lib/npg_qc/util.pm
 lib/npg_qc/utils/bam_genotype.pm
-lib/npg_qc/utils/iRODS.pm
 lib/npg_qc/view/analysis.pm
 lib/npg_qc/view/cache_query.pm
 lib/npg_qc/view/create_xml.pm

lib/npg_qc/autoqc/checks/genotype.pm

@@ -9,10 +9,11 @@ use List::MoreUtils qw { any };
 use File::Slurp;
 use JSON;
 use npg_qc::utils::bam_genotype;
-use npg_qc::utils::iRODS;
 use npg_qc::autoqc::types;
 use Readonly;
 use FindBin qw($Bin);
+use Try::Tiny;
+use WTSI::NPG::iRODS::DataObject;
 
 extends qw(npg_qc::autoqc::checks::check);
 with qw(npg_tracking::data::reference::find
@@ -21,6 +22,14 @@ with qw(npg_tracking::data::reference::find
 
 our $VERSION = '0';
 
+has 'irods' =>
+  (is       => 'ro',
+   isa      => 'WTSI::NPG::iRODS',
+   required => 1,
+   default  => sub {
+       return WTSI::NPG::iRODS->new;
+   },);
+
 Readonly::Scalar our $HUMAN_REFERENCES_DIR => q[Homo_sapiens];
 Readonly::Scalar my $GENOTYPE_DATA => 'sgd';
 Readonly::Scalar my $SAMTOOLS_NAME => q[samtools_irods];
@@ -331,40 +340,44 @@ has 'input_files_md5' => (
 	lazy_build => 1,
 );
 sub _build_input_files_md5 {
-	my ($self) = @_;
-	my $md5 = q{};
-	Readonly::Scalar my $IRODS_PREFIX_LEN_IS_THIS_READABLE_ENOUGH => 6;
-
-	my @md5_vals = ();
-	for my $input_file (@{$self->input_files}) {
-		if($input_file =~ /^irods:/smx) {
-			my $irods_filename = substr $input_file, $IRODS_PREFIX_LEN_IS_THIS_READABLE_ENOUGH;  # strip leading "irods:"
-
-			$md5 = npg_qc::utils::iRODS->new->get_file_md5($irods_filename);
-
-			$md5 ||= '0000000000000000';
-
-			push @md5_vals, $md5;
-		}
-		else {
-			my $md5_file = "${input_file}.md5";
-
-			$md5 = q{};
-			if(-r $md5_file) {
-				open my $f, '<', $md5_file or croak "$md5_file readable, but open fails";
-
-				$md5 = <$f>;
-
-				close $f or croak "Failed to close $md5_file";
-
-			}
-			$md5 ||= '0000000000000000';
-			push @md5_vals, $md5;
-		}
-
-	}
+    my ($self) = @_;
+    my $md5;
+    Readonly::Scalar my $IRODS_PREFIX_LEN_IS_THIS_READABLE_ENOUGH => 6;
+
+    my @md5_vals = ();
+    for my $input_file (@{$self->input_files}) {
+        $md5 = q{};
+        if ($input_file =~ /^irods:/smx) {
+            my $irods_filename = substr $input_file, $IRODS_PREFIX_LEN_IS_THIS_READABLE_ENOUGH;  # strip leading "irods:"
+            try {
+                my $data_obj = WTSI::NPG::iRODS::DataObject->new(
+                    $self->irods, $irods_filename
+                );
+                $md5 = $data_obj->checksum;
+            } catch {
+                my $msg = 'Unable to find md5 checksum for '.
+                    "iRODS file '$irods_filename': $_";
+                carp($msg);
+                $md5 = '0000000000000000';
+            }
+        } else {
+            my $md5_file = "${input_file}.md5";
+            if(-r $md5_file) {
+                open my $f, '<', $md5_file or croak "$md5_file readable, but open fails";
+                $md5 = <$f>;
+                chomp $md5;
+                close $f or croak "Failed to close $md5_file";
+            }
+            if (! $md5) {
+                my $msg = "Unable to read md5 checksum from file '$md5_file'";
+                carp($msg);
+                $md5 = '0000000000000000';
+            }
+        }
+        push @md5_vals, $md5;
+    }
 
-	return join q[;], @md5_vals;
+    return join q[;], @md5_vals;
 }
 
 #####################################################################################################################

lib/npg_qc/autoqc/checks/tags_reporters.pm

@@ -8,7 +8,6 @@ use File::Slurp;
 use File::Spec::Functions qw(catfile catdir);
 use List::MoreUtils qw(any);
 use JSON;
-use npg_qc::utils::iRODS;
 use npg_qc::autoqc::types;
 use Readonly;
 

lib/npg_qc/autoqc/checks/upstream_tags.pm

@@ -14,7 +14,6 @@ use FindBin qw($Bin);
 
 use npg_qc::autoqc::checks::tag_metrics;
 use npg_qc::autoqc::qc_store;
-use npg_qc::utils::iRODS;
 use npg_qc::autoqc::types;
 
 extends qw(npg_qc::autoqc::checks::check);