Merge pull request #82 from sanger-tol/test_ci

Test ci

.editorconfig

@@ -23,11 +23,3 @@ indent_size = unset
 [/assets/email*]
 indent_size = unset
 
-# To prevent errors for these test diamond databases
-[/assets/test*/*.dmnd]
-charset = unset
-end_of_line = unset
-insert_final_newline = unset
-trim_trailing_whitespace = unset
-indent_style = unset
-indent_size = unset

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -38,7 +38,7 @@ body:
     id: system
     attributes:
       label: System information
-      description: "* Nextflow version _(eg. 23.04.1)_
+      description: "* Nextflow version _(eg. 22.10.1)_
 
         * Hardware _(eg. HPC, Desktop, Cloud)_
 

.github/workflows/ci.yml

@@ -1,7 +1,12 @@
 name: nf-core CI
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
 on:
-  workflow_dispatch:
+  push:
+    branches:
+      - dev
+  pull_request:
+  release:
+    types: [published]
 
 env:
   NXF_ANSI_LOG: false
@@ -19,7 +24,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.1"
+          - "22.10.1"
           - "latest-everything"
     steps:
       - name: Check out pipeline code
@@ -30,9 +35,19 @@ jobs:
         with:
           version: "${{ matrix.NXF_VER }}"
 
+      - name: Download the NCBI taxdump database
+        run: |
+          mkdir ncbi_taxdump
+          curl -L https://ftp.ncbi.nih.gov/pub/taxonomy/new_taxdump/new_taxdump.tar.gz | tar -C ncbi_taxdump -xzf -
+
+      - name: Download the BUSCO lineage database
+        run: |
+          mkdir busco_database
+          curl -L https://tolit.cog.sanger.ac.uk/test-data/resources/busco/blobtoolkit.GCA_922984935.2.2023-08-03.lineages.tar.gz | tar -C busco_database -xzf -
+
       - name: Run pipeline with test data
         # You can customise CI pipeline run tests as required
         # For example: adding multiple test runs with different parameters
         # Remember that you can parallelise this by using strategy.matrix
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --taxdump $PWD/ncbi_taxdump --busco $PWD/busco_database --outdir ./results

.github/workflows/fix-linting.yml

@@ -8,21 +8,21 @@ jobs:
     # Only run if comment is on a PR with the main repo, and if it contains the magic keywords
     if: >
       contains(github.event.comment.html_url, '/pull/') &&
-      contains(github.event.comment.body, '@nf-core-bot fix linting') &&
+      contains(github.event.comment.body, '@sanger-tolsoft fix linting') &&
       github.repository == 'sanger-tol/blobtoolkit'
     runs-on: ubuntu-latest
     steps:
-      # Use the @nf-core-bot token to check out so we can push later
+      # Use the @sanger-tolsoft token to check out so we can push later
       - uses: actions/checkout@v3
         with:
-          token: ${{ secrets.nf_core_bot_auth_token }}
+          token: ${{ secrets.sangertolsoft_access_token }}
 
       # Action runs on the issue comment, so we don't get the PR by default
       # Use the gh cli to check out the PR
       - name: Checkout Pull Request
         run: gh pr checkout ${{ github.event.issue.number }}
         env:
-          GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
+          GITHUB_TOKEN: ${{ secrets.sangertolsoft_access_token }}
 
       - uses: actions/setup-node@v3
 
@@ -46,8 +46,8 @@ jobs:
       - name: Commit & push changes
         if: steps.prettier_status.outputs.result == 'fail'
         run: |
-          git config user.email "[email protected]"
-          git config user.name "nf-core-bot"
+          git config user.email "[email protected]"
+          git config user.name "sanger-tolsoft"
           git config push.default upstream
           git add .
           git status

.github/workflows/sangertest.yml → .github/workflows/sanger_test.yml

@@ -1,4 +1,4 @@
-name: nf-core Sanger LSF tests
+name: sanger-tol LSF tests
 
 on:
   workflow_dispatch:
@@ -13,16 +13,21 @@ jobs:
         if: github.event_name == 'workflow_dispatch'
 
       - name: Launch workflow via tower
-        uses: nf-core/tower-action@v2
+        uses: seqeralabs/action-tower-launch@v2
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
           compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
-          pipeline: ${{ github.repository }}
           revision: ${{ env.REVISION }}
           workdir: ${{ secrets.TOWER_WORKDIR_PARENT }}/work/${{ github.repository }}/work-${{ env.REVISION }}
           parameters: |
             {
               "outdir": "${{ secrets.TOWER_WORKDIR_PARENT }}/results/${{ github.repository }}/results-${{ env.REVISION }}",
             }
           profiles: test,sanger,singularity,cleanup
+      - uses: actions/upload-artifact@v3
+        with:
+          name: Tower debug log file
+          path: |
+            tower_action_*.log
+            tower_action_*.json

.github/workflows/sangerfulltest.yml → .github/workflows/sanger_test_full.yml

@@ -1,4 +1,4 @@
-name: nf-core Sanger LSF full size tests
+name: sanger-tol LSF full size tests
 
 on:
   workflow_dispatch:
@@ -18,16 +18,21 @@ jobs:
         if: github.event_name == 'workflow_dispatch'
 
       - name: Launch workflow via tower
-        uses: nf-core/tower-action@v2
+        uses: seqeralabs/action-tower-launch@v2
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
           compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
-          pipeline: ${{ github.repository }}
           revision: ${{ env.REVISION }}
           workdir: ${{ secrets.TOWER_WORKDIR_PARENT }}/work/${{ github.repository }}/work-${{ env.REVISION }}
           parameters: |
             {
               "outdir": "${{ secrets.TOWER_WORKDIR_PARENT }}/results/${{ github.repository }}/results-${{ env.REVISION }}",
             }
           profiles: test_full,sanger,singularity,cleanup
+      - uses: actions/upload-artifact@v3
+        with:
+          name: Tower debug log file
+          path: |
+            tower_action_*.log
+            tower_action_*.json

.nf-core.yml

@@ -8,6 +8,7 @@ lint:
   files_unchanged:
     - LICENSE
     - .github/ISSUE_TEMPLATE/bug_report.yml
+    - .github/workflows/linting.yml
     - assets/sendmail_template.txt
     - lib/NfcoreTemplate.groovy
     - .prettierignore

README.md

@@ -5,7 +5,7 @@
 [![GitHub Actions Linting Status](https://github.com/sanger-tol/blobtoolkit/workflows/nf-core%20linting/badge.svg)](https://github.com/sanger-tol/blobtoolkit/actions?query=workflow%3A%22nf-core+linting%22)
 [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.7949058-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.7949058)
 
-[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.1-23aa62.svg)](https://www.nextflow.io/)
+[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/)
 [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
 [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)

assets/test_full/full_samplesheet.csv

@@ -1,3 +1,3 @@
 sample,datatype,datafile
-gfLaeSulp1,hic,/lustre/scratch123/tol/projects/.sandbox/data/fungi/Laetiporus_sulphureus/analysis/gfLaeSulp1.1/read_mapping/hic/GCA_927399515.1.unmasked.hic.gfLaeSulp1.cram
-gfLaeSulp1,pacbio,/lustre/scratch123/tol/projects/.sandbox/data/fungi/Laetiporus_sulphureus/analysis/gfLaeSulp1.1/read_mapping/pacbio/GCA_927399515.1.unmasked.pacbio.gfLaeSulp1.cram
+gfLaeSulp1,hic,/lustre/scratch123/tol/resources/nextflow/test-data/Laetiporus_sulphureus/analysis/gfLaeSulp1.1/read_mapping/hic/GCA_927399515.1.unmasked.hic.gfLaeSulp1.cram
+gfLaeSulp1,pacbio,/lustre/scratch123/tol/resources/nextflow/test-data/Laetiporus_sulphureus/analysis/gfLaeSulp1.1/read_mapping/pacbio/GCA_927399515.1.unmasked.pacbio.gfLaeSulp1.cram

conf/modules.config

@@ -14,7 +14,7 @@ process {
 
     withName: "SAMPLESHEET_CHECK" {
         publishDir = [
-            path: { "${params.outdir}/blobtoolkit_info" },
+            path: { "${params.outdir}/pipeline_info/blobtoolkit" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals("versions.yml") ? null : filename }
         ]
@@ -54,6 +54,7 @@ process {
 
     withName: "BUSCO" {
         scratch = true
+        // Overridden in the test profile, see at the end of this file
         ext.args = "--mode genome --force"
     }
 
@@ -92,7 +93,7 @@ process {
 
     withName: "CUSTOM_DUMPSOFTWAREVERSIONS" {
         publishDir = [
-            path: { "${params.outdir}/blobtoolkit_info" },
+            path: { "${params.outdir}/pipeline_info/blobtoolkit" },
             mode: params.publish_dir_mode,
             pattern: "*_versions.yml"
         ]
@@ -108,3 +109,22 @@ process {
     }
 
 }
+
+
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Additional configuration to speed processes up during testing.
+
+----------------------------------------------------------------------------------------
+*/
+
+profiles {
+    test {
+        process {
+            withName: BUSCO {
+                // Note: BUSCO *must* see the double-quotes around the parameters
+                ext.args = '--mode genome --force --metaeuk_parameters \'"-s=2"\' --metaeuk_rerun_parameters \'"-s=2"\''
+            }
+        }
+    }
+}

conf/test.config

@@ -22,15 +22,15 @@ params {
     // Input test data
     // Specify the paths to your test data
     // Give any required params for the test so that command line flags are not needed
-    input     = "${projectDir}/assets/test/samplesheet.csv"
+    input     = "${projectDir}/assets/test/samplesheet_s3.csv"
 
     // Fasta references
-    fasta     = "/lustre/scratch123/tol/resources/nextflow/test-data/Meles_meles/assembly/release/mMelMel3.1_paternal_haplotype/GCA_922984935.2.subset.fasta.gz"
+    fasta     = "https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/assembly/release/mMelMel3.1_paternal_haplotype/GCA_922984935.2.subset.fasta.gz"
     accession = "GCA_922984935.2"
     taxon     = "Meles meles"
 
     // Databases
     taxdump   = "/lustre/scratch123/tol/teams/grit/geval_pipeline/btk_databases/taxdump"
-    busco     = "/lustre/scratch123/tol/resources/nextflow/busco_2021_06_reduced/"
-    uniprot   = "${projectDir}/assets/test/mCerEla1.1.buscogenes.dmnd"
+    busco     = "/lustre/scratch123/tol/resources/nextflow/busco/blobtoolkit.GCA_922984935.2.2023-08-03"
+    uniprot   = "https://tolit.cog.sanger.ac.uk/test-data/resources/diamond/mCerEla1.1.buscogenes.dmnd"
 }

conf/test_full.config

@@ -22,12 +22,12 @@ params {
     input     = "${projectDir}/assets/test_full/full_samplesheet.csv"
 
     // Fasta references
-    fasta     = "/lustre/scratch124/tol/projects/darwin/data/fungi/Laetiporus_sulphureus/assembly/release/gfLaeSulp1.1/insdc/GCA_927399515.1.fasta.gz"
+    fasta     = "/lustre/scratch123/tol/resources/nextflow/test-data/Laetiporus_sulphureus/assembly/release/gfLaeSulp1.1/insdc/GCA_927399515.1.fasta.gz"
     accession = "GCA_927399515.1"
     taxon     = "Laetiporus sulphureus"
 
     // Databases
     taxdump   = "/lustre/scratch123/tol/teams/grit/geval_pipeline/btk_databases/taxdump"
     busco     = "/lustre/scratch123/tol/resources/busco/v5/"
-    uniprot   = "${projectDir}/assets/test_full/gfLaeSulp1.1.buscogenes.dmnd"
+    uniprot   = "https://tolit.cog.sanger.ac.uk/test-data/resources/diamond/gfLaeSulp1.1.buscogenes.dmnd"
 }

docs/output.md

@@ -52,7 +52,7 @@ Results generated by MultiQC collate pipeline QC from supported tools. The pipel
 <details markdown="1">
 <summary>Output files</summary>
 
-- `blobtoolkit_info/`
+- `pipeline_info/blobtoolkit/`
   - Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`.
   - Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline.
   - Reformatted samplesheet files used as input to the pipeline: `samplesheet.valid.csv`.

lib/NfcoreTemplate.groovy

@@ -154,7 +154,7 @@ class NfcoreTemplate {
         }
 
         // Write summary e-mail HTML to a file
-        def output_d = new File("${params.outdir}/blobtoolkit_info/")
+        def output_d = new File("${params.outdir}/pipeline_info/blobtoolkit/")
         if (!output_d.exists()) {
             output_d.mkdirs()
         }

modules/local/blobtoolkit/blobdir.nf

@@ -5,7 +5,7 @@ process BLOBTOOLKIT_BLOBDIR {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         exit 1, "BLOBTOOLKIT_BLOBDIR module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
-    container "genomehubs/blobtoolkit:4.1.5"
+    container "docker.io/genomehubs/blobtoolkit:4.1.5"
 
     input:
     tuple val(meta), path(window, stageAs: 'windowstats/*')

modules/local/blobtoolkit/config.nf

@@ -5,7 +5,7 @@ process BLOBTOOLKIT_CONFIG {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         exit 1, "GENERATE_CONFIG module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
-    container "genomehubs/blobtoolkit:4.1.5"
+    container "docker.io/genomehubs/blobtoolkit:4.1.5"
 
     input:
     tuple val(meta), path(fasta)

modules/local/blobtoolkit/countbuscos.nf

@@ -5,7 +5,7 @@ process BLOBTOOLKIT_COUNTBUSCOS {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         exit 1, "BLOBTOOLKIT_COUNTBUSCOS module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
-    container "genomehubs/blobtoolkit:4.1.5"
+    container "docker.io/genomehubs/blobtoolkit:4.1.5"
 
     input:
     tuple val(meta), path(table, stageAs: 'dir??/*')

modules/local/blobtoolkit/extractbuscos.nf

@@ -5,7 +5,7 @@ process BLOBTOOLKIT_EXTRACTBUSCOS {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         exit 1, "BLOBTOOLKIT_EXTRACTBUSCOS module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
-    container "genomehubs/blobtoolkit:4.1.5"
+    container "docker.io/genomehubs/blobtoolkit:4.1.5"
 
     input:
     tuple val(meta), path(fasta)

modules/local/blobtoolkit/images.nf

@@ -5,7 +5,7 @@ process BLOBTOOLKIT_IMAGES {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         exit 1, "BLOBTOOLKIT_IMAGES module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
-    container "genomehubs/blobtk:0.3.3"
+    container "docker.io/genomehubs/blobtk:0.3.3"
 
     input:
     tuple val(meta), path(blobdir)

modules/local/blobtoolkit/metadata.nf

@@ -5,7 +5,7 @@ process BLOBTOOLKIT_METADATA {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         exit 1, "BLOBTOOLKIT_METADATA module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
-    container "genomehubs/blobtoolkit:4.1.5"
+    container "docker.io/genomehubs/blobtoolkit:4.1.5"
 
     input:
     tuple val(meta), path(yaml)

modules/local/blobtoolkit/summary.nf

@@ -5,7 +5,7 @@ process BLOBTOOLKIT_SUMMARY {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         exit 1, "BLOBTOOLKIT_SUMMARY module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
-    container "genomehubs/blobtoolkit:4.1.5"
+    container "docker.io/genomehubs/blobtoolkit:4.1.5"
 
     input:
     tuple val(meta), path(blobdir)

modules/local/blobtoolkit/windowstats.nf

@@ -5,7 +5,7 @@ process BLOBTOOLKIT_WINDOWSTATS {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         exit 1, "GET_WINDOW_STATS module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
-    container "genomehubs/blobtoolkit:4.1.5"
+    container "docker.io/genomehubs/blobtoolkit:4.1.5"
 
     input:
     tuple val(meta), path(tsv)

modules/local/create_bed.nf

@@ -5,7 +5,7 @@ process CREATE_BED {
     conda "conda-forge::gawk=5.1.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
     'https://depot.galaxyproject.org/singularity/gawk:5.1.0' :
-    'quay.io/biocontainers/gawk:5.1.0' }"
+    'biocontainers/gawk:5.1.0' }"
 
     input:
     tuple val(meta), path(tsv)      //path to tsv output from fasta windows

modules/local/samplesheet_check.nf

@@ -5,7 +5,7 @@ process SAMPLESHEET_CHECK {
     conda "conda-forge::python=3.9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/python:3.9--1' :
-        'quay.io/biocontainers/python:3.9--1' }"
+        'biocontainers/python:3.9--1' }"
 
     input:
     path samplesheet

modules/local/windowstats_input.nf

@@ -5,7 +5,7 @@ process WINDOWSTATS_INPUT {
     conda "conda-forge::pandas=1.5.2"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/pandas:1.5.2':
-        'quay.io/biocontainers/pandas:1.5.2' }"
+        'biocontainers/pandas:1.5.2' }"
 
     input:
     tuple val(meta), path(freq)