Merge pull request #764 from wtsi-npg/devel

merge from devel to master to create release 69.7.0

.github/workflows/testing_and_building_repos.yml

@@ -0,0 +1,103 @@
+name: testing_and_building_repo
+on: [push, pull_request]
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ['ubuntu-18.04']
+        perl: ['5.26']
+        node: ['14.8.0']
+
+    runs-on: ${{ matrix.os }}
+
+    defaults:
+      run:
+        shell: bash -l -e -o pipefail {0}
+
+    env:
+      WSI_CONDA_CHANNEL: "https://dnap.cog.sanger.ac.uk/npg/conda/devel/generic"
+      NPM_CONFIG_PREFIX: "$HOME/.npm-global"
+
+    name: Perl ${{ matrix.perl }} and Node ${{ matrix.node }} on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+
+    # Caching cpanm external modules
+    - name: Cache cpanm external modules
+      id: cpanmCache
+      uses: actions/cache@v2
+      with:
+        path: ~/perl5ext
+        key: ${{ matrix.os }}-build-cpanm-external
+
+    - name: Initialize Miniconda
+      run: |
+          echo 'source $CONDA/etc/profile.d/conda.sh' >> "$HOME/.bash_profile"
+
+    # Caching conda
+    - name: Cache conda
+      id: minicondaCache
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/conda/pkgs
+          ~/conda/envs
+        key: ${{ matrix.os }}-build-miniconda
+
+    - name: install libgd-dev and uuid-dev
+      run:
+          sudo apt-get install -y libgd-dev uuid-dev
+
+    - name: install conda client, baton,samtools, set up conda environment
+      run: |
+          conda config --prepend pkgs_dirs ~/conda/pkgs
+          conda config --prepend envs_dirs ~/conda/envs
+
+          conda config --set auto_update_conda False
+          conda config --prepend channels "$WSI_CONDA_CHANNEL"
+          conda config --append channels conda-forge
+          conda info
+
+          conda create -y -n "$CONDA_TEST_ENV"
+          conda install -y -n "$CONDA_TEST_ENV" nodejs=="${{ matrix.node }}"
+          conda install -y -n "$CONDA_TEST_ENV" npg_qc_utils
+          conda install -y -n "$CONDA_TEST_ENV" baton
+      env:
+        CONDA_TEST_ENV: test-environment
+
+    - name: install cpanm
+      run: |
+          wget -qO - https://cpanmin.us | /usr/bin/perl - --sudo App::cpanminus
+
+    - name: Run install scripts
+      run: |
+          mkdir -p "$NPM_CONFIG_PREFIX"
+          ${GITHUB_WORKSPACE}/scripts/before_install.sh
+
+          cpanm --local-lib=~/perl5 local::lib && eval $(perl -I ~/perl5/lib/perl5/ -Mlocal::lib)
+          ${GITHUB_WORKSPACE}/scripts/install.sh $WTSI_NPG_BUILD_BRANCH $WTSI_NPG_GITHUB_URL
+      env:
+        NPM_VERSION: "${{ matrix.npm }}"
+        WTSI_NPG_BUILD_BRANCH: ${GITHUB_HEAD_REF}
+        WTSI_NPG_GITHUB_URL: https://github.com/wtsi-npg
+
+    - name: run script
+      run: |
+          conda activate "$CONDA_TEST_ENV"
+          conda info --envs
+          cpanm --local-lib=~/perl5 local::lib && eval $(perl -I ~/perl5/lib/perl5/ -Mlocal::lib)
+          ${GITHUB_WORKSPACE}/scripts/script.sh "${{ matrix.node }}" $WTSI_NPG_BUILD_BRANCH
+      env:
+        CONDA_TEST_ENV: test-environment
+        NODE_VERSION: ${{ matrix.node }}
+        WTSI_NPG_BUILD_BRANCH: ${GITHUB_HEAD_REF}
+
+    # Archive logs if failure
+    - name: Archive CPAN logs
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: cpan_log
+        path: /home/runner/.cpanm/work/*/build.log
+        retention-days: 5

Build.PL

@@ -83,7 +83,6 @@ my $requires = {
                 'npg_common::Alignment'                    => 0,
                 'npg_common::extractor::fastq'             => 0,
                 'npg_common::roles::software_location'     => 0,
-                'npg_common::sequence::reference::base_count'  => 0,
                 'npg_tracking::data::reference::list'      => 0,
                 'npg_tracking::data::bait::find'           => 0,
                 'npg_tracking::data::geno_refset::find'    => 0,

Changes

@@ -1,5 +1,17 @@
 LIST OF CHANGES FOR NPG-QC PACKAGE
 
+release 69.7.0
+ - added new npg_subtitution_metrics.pl script
+ - move from Travis CI to GitHub Actions
+ - switch to node 14 for CI, using npm from default conda pkg
+ - remove dependency on npg_common::sequence::reference::base_count, which
+   has been moved to an internal repository and is no longer available for
+   the CI builds
+ - make labels for number of reads in SeqQC generic artic display
+   consistent with display for bam flagstats data, ie use 'reads'
+   instead of 'fragments'
+ - do not generate autoqc generic artic results for spiked-in control
+
 release 69.6.1
  - mqc skipper bug fix - inspect qc outcomes for review autoqc results
    only

MANIFEST

@@ -1,3 +1,4 @@
+.github/workflows/testing_and_building_repos.yml 
 bin/gtck_combine_gt_data.sh
 bin/gtck_extract_fluidigm_data_from_irods.sh
 bin/gtck_genplexlist.sh
@@ -10,6 +11,7 @@ bin/mqc_outcome_reporter
 bin/npg_mqc_skipper
 bin/npg_qc_autoqc_data.pl
 bin/npg_qc_tag_sniff.pl
+bin/npg_substitution_metrics.pl
 bin/call_gtck_composite_rpt.pl
 bin/genotype_call_results_reporter
 Build.PL
@@ -161,7 +163,10 @@ lib/npg_qc/utils/bam_genotype.pm
 lib/npg_qc/utils/genotype_calling.pm
 MANIFEST
 README
+scripts/before_install.sh
+scripts/install.sh
 scripts/npgqc_dbix_schema_loader.pl
+scripts/script.sh
 t/00-critic.t
 t/00-distribution.t
 t/00-podcoverage.t
@@ -349,6 +354,8 @@ t/data/autoqc/generic/ampliconstats/34719_1.ampstats
 t/data/autoqc/generic/ampliconstats/samplesheet_34719.csv
 t/data/autoqc/generic/artic/35177_2.tag_metrics.json
 t/data/autoqc/generic/artic/samplesheet_35177.csv
+t/data/autoqc/generic/artic/samplesheet_36214.csv
+t/data/autoqc/generic/artic/36214_2.with_spike.tag_metrics.json
 t/data/autoqc/generic/artic/lane2/plex1/35177.qc.csv
 t/data/autoqc/generic/artic/lane2/plex137/35177.qc.csv
 t/data/autoqc/generic/artic/lane2/plex140/35177.qc.csv

bin/npg_qc_tag_sniff.pl

@@ -191,7 +191,7 @@ sub showTags{
         foreach my $i (0..$#subtags) {
             my $subtag = $subtags[$i];
             my @matches = ();
-            if ($clips->[$i]) {
+            if (@{$clips} && $clips->[$i]) {
                 if ($clips->[$i] < 0) {
                     @matches = grep {m/$subtag$/} keys %db_tags;
                 } elsif ($clips->[$i]) {
@@ -205,12 +205,16 @@ sub showTags{
             foreach my $sequence (@matches) {
                 foreach my $match (@{$db_tags{$sequence}}) {
                     my ($name,$id,$map_id,$revcomp) = @{$match};
-                    if ($revcomp && $revcomps->[$i]) {
-                        $matches{$subtag}->{$id} = [$map_id,1];
-                    } elsif ($revcomp) {
-                        # not looking for revcomp matches on this subtag
+                    if ($revcomp) {
+                        # revcomp match AND we are looking for revcomp matches AND we are looking for revcomp matches on this tag
+                        if (@{$revcomps} && $revcomps->[$i]) {
+                            $matches{$subtag}->{$id} = [$map_id,1];
+                        }
                     } else {
-                        $matches{$subtag}->{$id} = [$map_id,0];
+                        # not a revcomp match AND we are not looking for revcomp matches or we are not looking for revcomp matches on this tag
+                        if (!@{$revcomps} || !$revcomps->[$i]) {
+                            $matches{$subtag}->{$id} = [$map_id,0];
+                        }
                     }
                     $groups[$i]->{$id}++;
                     $names{$id} = $name;