updated tests to reflect build-taxonomy (#40)

* fix path to taxonomy files * download and build taxonomy * merge Listeria into Yersinia matrix * m * updated output directory as matrix.GENUS * kraken1 tests patches * m
lskatz · May 7, 2024 · 6b662ba · 6b662ba
1 parent f22510d
commit 6b662ba
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 104 deletions.
diff --git a/.github/workflows/unit-testing.Listeria.Kraken1.yml b/.github/workflows/unit-testing.Listeria.Kraken1.yml
@@ -1,7 +1,7 @@
 # This is a subsampling unit test to get early results
 on: 
   push:
-    branches: [master, dev]
+    branches: [master, dev, build-taxonomy]
 name: Listeria-with-Kraken1
 
 env:
@@ -23,7 +23,7 @@ jobs:
           perl-version: ${{ matrix.perl }}
           multi-thread: "true"
       - name: checkout my repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           path: Kalamari
 
@@ -48,7 +48,9 @@ jobs:
           perl -MNet::FTP -e '$ftp = new Net::FTP("ftp.ncbi.nlm.nih.gov", Passive => 1); $ftp->login; $ftp->binary; $ftp->get("/entrez/entrezdirect/edirect.tar.gz");'
           gunzip -cv edirect.tar.gz | tar xf -
           rm -v edirect.tar.gz
-          export PATH=${PATH}:$HOME/edirect >& /dev/null || setenv PATH "${PATH}:$HOME/edirect"
+          echo $GITHUB_WORKSPACE/edirect >> $GITHUB_PATH
+          echo $GITHUB_WORKSPACE/Kalamari/bin >> $GITHUB_PATH
+          #export PATH=${PATH}:$HOME/edirect >& /dev/null || setenv PATH "${PATH}:$HOME/edirect"
           yes Y | ./edirect/setup.sh
           tree edirect
       - name: check-env
@@ -64,10 +66,6 @@ jobs:
         run:  perl Kalamari/bin/downloadKalamari.pl --outdir ${{ env.OUTDIR }} ${{ env.TSV }}
       - name: check-results
         run:  tree ${{ env.OUTDIR }}
-      #- name: download-more
-      #  run:  perl Kalamari/bin/downloadKalamari.pl --outdir ${{ env.OUTDIR }} ${{ env.TSV }} --and protein --and nucleotide
-      #- name: check-results
-      #  run:  tree ${{ env.OUTDIR }}
       - name: install kraken
         run:  |
           wget https://github.com/DerrickWood/kraken/archive/refs/tags/v1.1.1.tar.gz -O kraken-v1.1.1.tar.gz
@@ -76,12 +74,17 @@ jobs:
           chmod -v +x kraken-1.1.1/kraken-src/*
           echo $(realpath kraken-1.1.1/kraken-src) >> $GITHUB_PATH
           tree $(realpath) kraken-1.1.1
+      - name: build taxonomy
+        run: |
+          export PATH=$PATH:Kalamari/bin
+          buildTaxonomy.sh
+          ls -lh Kalamari/share
       - name: Kraken1 database
         run:  |
           echo $PATH
           which kraken-build
           mkdir -pv kraken
-          cp -rv Kalamari/src/taxonomy kraken/taxonomy
+          cp -rv Kalamari/share/kalamari-*/taxonomy kraken/taxonomy
           find ${{ env.OUTDIR }} -name '*.fasta' -exec kraken-build --db kraken --add-to-library {} \;
           tree kraken
           # Some super debugging here with -x

diff --git a/.github/workflows/unit-testing.Listeria.Kraken2.yml b/.github/workflows/unit-testing.Listeria.Kraken2.yml
diff --git a/.github/workflows/unit-testing.Yersinia.Kraken2.yml b/.github/workflows/unit-testing.Yersinia.Kraken2.yml
@@ -1,14 +1,12 @@
 # This is a subsampling unit test to get early results
 on: 
   push:
-    branches: [master, dev]
-name: Yersinia-with-Kraken2
+    branches: [master, dev, build-taxonomy]
+name: Genera-with-Kraken2
 
 env:
   TSV: "Kalamari/src/genus.tsv"
-  OUTDIR: "Yersinia.out"
   DB: "kraken2"
-  SRC_TAX: "Kalamari/src/taxonomy"
   SRC_CHR: "Kalamari/src/chromosomes.tsv"
   SRC_PLD: "Kalamari/src/plasmids.tsv"
   GENUS: Yersinia
@@ -20,15 +18,16 @@ jobs:
       matrix:
         os: ['ubuntu-20.04' ]
         perl: [ '5.32' ]
-    name: Perl ${{ matrix.perl }} on ${{ matrix.os }}
+        GENUS: [ 'Yersinia', 'Listeria']
+    name: ${{ matrix.GENUS }} Perl ${{ matrix.perl }} on ${{ matrix.os }}
     steps:
       - name: Set up perl
         uses: shogo82148/actions-setup-perl@v1
         with:
           perl-version: ${{ matrix.perl }}
           multi-thread: "true"
       - name: checkout my repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           path: Kalamari
 
@@ -40,29 +39,34 @@ jobs:
       - name: select for only for this genus
         run:  |
           head -n 1 ${{ env.SRC_CHR }} > ${{ env.TSV }}
-          grep -m 2 ${{ env.GENUS }} ${{ env.SRC_CHR }} >> ${{ env.TSV }}
-          grep -m 2 ${{ env.GENUS }} ${{ env.SRC_PLD }} >> ${{ env.TSV }}
-          echo "These are the ${{ env.GENUS }} genomes for downstream tests"
+          grep -m 2 ${{ matrix.GENUS }} ${{ env.SRC_CHR }} >> ${{ env.TSV }}
+          grep -m 2 ${{ matrix.GENUS }} ${{ env.SRC_PLD }} >> ${{ env.TSV }}
+          echo "These are the ${{ matrix.GENUS }} genomes for downstream tests"
           column -ts $'\t' ${{ env.TSV }}
           hexdump -c ${{ env.TSV }}
       - name: download
-        run:  perl Kalamari/bin/downloadKalamari.pl --outdir ${{ env.OUTDIR }} ${{ env.TSV }}
+        run:  perl Kalamari/bin/downloadKalamari.pl --outdir ${{ matrix.GENUS }} ${{ env.TSV }}
       - name: check-results
-        run:  tree ${{ env.OUTDIR }}
+        run:  tree ${{ matrix.GENUS }}
       - name: install kraken
         run:  |
           wget https://github.com/DerrickWood/kraken2/archive/refs/tags/v2.1.2.tar.gz -O kraken-v2.1.2.tar.gz
           tar zxvf kraken-v2.1.2.tar.gz
           cd kraken2-2.1.2 && bash install_kraken2.sh target && cd -
           ls -lhS kraken2-2.1.2/target
           chmod +x kraken2-2.1.2/target/*
+      - name: build taxonomy
+        run: |
+          export PATH=$PATH:Kalamari/bin
+          buildTaxonomy.sh
+          ls -lh Kalamari/share
       - name: Kraken2 database
         run:  |
           export PATH=$PATH:kraken2-2.1.2/target
           which kraken2-build
           mkdir -pv ${{ env.DB }}
-          cp -rv ${{ env.SRC_TAX }} ${{ env.DB }}/taxonomy
-          find ${{ env.OUTDIR }} -name '*.fasta' -exec kraken2-build --db ${{ env.DB }} --add-to-library {} \;
+          cp -rv Kalamari/share/kalamari-*/taxonomy ${{ env.DB }}/taxonomy
+          find ${{ matrix.GENUS }} -name '*.fasta' -exec kraken2-build --db ${{ env.DB }} --add-to-library {} \;
           tree ${{ env.DB }}
           echo ".....Building the database....."
           kraken2-build --build --db ${{ env.DB }} --threads 2
@@ -71,7 +75,7 @@ jobs:
           export PATH=$PATH:kraken2-2.1.2/target
           tree ${{ env.DB }}
           ls -lhSR ${{ env.DB }}
-          QUERY=$(find ${{ env.OUTDIR }} -name '*.fasta' | head -n 1)
+          QUERY=$(find ${{ matrix.GENUS }} -name '*.fasta' | head -n 1)
           echo "QUERY is $QUERY"
           head -n 2 $QUERY
           kraken2 --db ${{ env.DB }} --report kraken2.report --use-mpa-style --output kraken2.raw $QUERY