Skip to content

Commit

Permalink
Merge branch 'master' into update-panaroo/run
Browse files Browse the repository at this point in the history
* master: (120 commits)
  Use long form docker.io address for Ubuntu (nf-core#3358)
  Default registry to quay.io (nf-core#3344)
  Sbwf impute glimpse2 (nf-core#3349)
  added cellranger vdj/mkvdjref modules, with module test for vdj (nf-core#3033)
  Add space to fix singularity download error on shinyngs modules (nf-core#3351)
  Output channel modification star (nf-core#3350)
  fix corrected typo in output (nf-core#3337)
  give execution permissions to dumpsoftwareversions.py (nf-core#3347)
  fix meta.ymls for dumpsoftware and multiqc (nf-core#3345)
  Fixing failing subworkflow tests (nf-core#3343)
  remove imputeme module since it is evidently no longer supported (nf-core#3223)
  Update path for yaml-schema json (nf-core#3340)
  New module MetaPhlAn4 (nf-core#3298)
  Update ALL bcftools modules (nf-core#3339)
  update docker and singularity version of p7zip (nf-core#3338)
  update sratools/fasterqdump to handle 10X data (nf-core#3336)
  Make MAD plots optional in shinyngs (nf-core#3334)
  reset tools in CI to released version (nf-core#3335)
  new module wisecondorx/gender (nf-core#3333)
  Bump shinyngs modules (nf-core#3323)
  ...
  • Loading branch information
jvfe committed May 2, 2023
2 parents d4bd008 + 38b1670 commit 0e077da
Show file tree
Hide file tree
Showing 645 changed files with 12,128 additions and 1,639 deletions.
9 changes: 4 additions & 5 deletions .github/workflows/nf-core-linting.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ jobs:
with:
python-version: "3.x"

- uses: actions/cache@v2
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
Expand All @@ -59,9 +59,8 @@ jobs:
- name: Install pip
run: python -m pip install --upgrade pip

# FIXME: Remove this when nf-core modules lint stabilizes and install stable release
- name: Install nf-core tools development version
run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
- name: Install nf-core tools
run: python -m pip install nf-core

- name: Install Nextflow
env:
Expand All @@ -75,7 +74,7 @@ jobs:
# HACK
if: startsWith( matrix.tags, 'subworkflow' ) != true

- uses: actions/cache@v2
- uses: actions/cache@v3
with:
path: /usr/local/bin/nextflow
key: ${{ runner.os }}
Expand Down
182 changes: 182 additions & 0 deletions .github/workflows/nf-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
name: nf-test
on:
push:
branches: [master]
pull_request:
branches: [master]
merge_group:
types: [checks_requested]
branches: [master]

# Cancel if a newer run is started
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

jobs:
changes:
name: Check for changes
runs-on: ubuntu-latest
outputs:
# Expose matched filters as job 'modules' output variable
modules: ${{ steps.filter.outputs.changes }}
steps:
- uses: actions/checkout@v3

- uses: dorny/paths-filter@v2
id: filter
with:
filters: "tests/config/nftest_modules.yml"

test:
runs-on: ubuntu-20.04

name: ${{ matrix.tags }} ${{ matrix.profile }}
needs: changes
if: needs.changes.outputs.modules != '[]'
strategy:
fail-fast: false
matrix:
tags: ["${{ fromJson(needs.changes.outputs.modules) }}"]
profile: ["docker", "singularity", "conda"]
exclude:
- profile: "conda"
tags: annotsv
- profile: "conda"
tags: bases2fastq
- profile: "conda"
tags: bcl2fastq
- profile: "conda"
tags: bclconvert
- profile: "conda"
tags: cellranger/count
- profile: "conda"
tags: cellranger/mkfastq
- profile: "conda"
tags: cellranger/mkgtf
- profile: "conda"
tags: cellranger/mkref
- profile: "conda"
tags: deepvariant
- profile: "conda"
tags: ensemblvep/vep
- profile: "conda"
tags: fastk/fastk
- profile: "conda"
tags: fastk/histex
- profile: "conda"
tags: fastk/merge
- profile: "conda"
tags: fcs/fcsadaptor
- profile: "conda"
tags: fcs/fcsgx
- profile: "conda"
tags: gatk4/cnnscorevariants
- profile: "conda"
tags: gatk4/determinegermlinecontigploidy
- profile: "conda"
tags: genescopefk
- profile: "conda"
tags: ilastik/multicut
- profile: "conda"
tags: ilastik/pixelclassification
- profile: "conda"
tags: imputeme/vcftoprs
- profile: "conda"
tags: merquryfk/katcomp
- profile: "conda"
tags: merquryfk/katgc
- profile: "conda"
tags: merquryfk/merquryfk
- profile: "conda"
tags: merquryfk/ploidyplot
- profile: "conda"
tags: sentieon/bwaindex
- profile: "conda"
tags: sentieon/bwamem
- profile: "conda"
tags: universc
- profile: "singularity"
tags: universc
- profile: "conda"
tags: subworkflows/vcf_annotate_ensemblvep
env:
NXF_ANSI_LOG: false
SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
steps:
- uses: actions/checkout@v3

- uses: actions/cache@v3
with:
path: /usr/local/bin/nextflow
key: ${{ runner.os }}
restore-keys: |
${{ runner.os }}-nextflow-
- name: Install Nextflow
env:
CAPSULE_LOG: none
run: |
wget -qO- get.nextflow.io | bash
sudo mv nextflow /usr/local/bin/
- name: Cache nf-test installation
id: cache-software
uses: actions/cache@v3
with:
path: |
/usr/local/bin/nf-test
/home/runner/.nf-test/nf-test.jar
key: ${{ runner.os }}-nftest

- name: Install nf-test
if: steps.cache-software.outputs.cache-hit != 'true'
run: |
wget -qO- https://code.askimed.com/install/nf-test | bash
sudo mv nf-test /usr/local/bin/
- name: Set up Singularity
if: matrix.profile == 'singularity'
uses: eWaterCycle/setup-singularity@v5
with:
singularity-version: 3.7.1

- name: Set up miniconda
if: matrix.profile == 'conda'
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: true
channels: conda-forge,bioconda,defaults
python-version: ${{ matrix.python-version }}

- name: Conda setup
if: matrix.profile == 'conda'
run: |
conda clean -a
conda install -n base conda-libmamba-solver
conda config --set solver libmamba
# Set up secrets
- name: Set up nextflow secrets
if: env.SENTIEON_LICENSE_BASE64 != null
run: |
nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }}
nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }}
SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d)
SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d)
SENTIEON_AUTH_DATA=$(python tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0)
nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64
# Test the module
- name: Run nf-test
run: |
nf-test test \
--profile=${{ matrix.profile }} \
--tag ${{ matrix.tags }} \
--tap=test.tap
- uses: pcolby/tap-summary@v1
with:
path: >-
test.tap
53 changes: 50 additions & 3 deletions .github/workflows/pytest-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,18 @@ jobs:
tags: ["${{ fromJson(needs.changes.outputs.modules) }}"]
profile: ["docker", "singularity", "conda"]
exclude:
- profile: "conda"
tags: annotsv
- profile: "conda"
tags: cellpose
- profile: "conda"
tags: mcquant
- profile: "conda"
tags: bases2fastq
- profile: "conda"
tags: backsub
- profile: "conda"
tags: basicpy
- profile: "conda"
tags: bcl2fastq
- profile: "conda"
Expand All @@ -54,6 +64,14 @@ jobs:
tags: cellranger/mkgtf
- profile: "conda"
tags: cellranger/mkref
- profile: "conda"
tags: cellranger/mkvdjref
- profile: "conda"
tags: cellranger/vdj
- profile: "conda"
tags: coreograph
- profile: "conda"
tags: deepcell/mesmer
- profile: "conda"
tags: deepvariant
- profile: "conda"
Expand All @@ -68,10 +86,18 @@ jobs:
tags: fcs/fcsadaptor
- profile: "conda"
tags: fcs/fcsgx
- profile: "conda"
tags: gatk4/baserecalibratorspark
- profile: "conda"
tags: gatk4/cnnscorevariants
- profile: "conda"
tags: gatk4/determinegermlinecontigploidy
- profile: "conda"
tags: gatk4/germlinecnvcaller
- profile: "conda"
tags: gatk4/markduplicatesspark
- profile: "conda"
tags: gatk4/postprocessgermlinecnvcalls
- profile: "conda"
tags: genescopefk
- profile: "conda"
Expand All @@ -88,16 +114,29 @@ jobs:
tags: merquryfk/merquryfk
- profile: "conda"
tags: merquryfk/ploidyplot
- profile: "conda"
tags: mitohifi/findmitoreference
- profile: "conda"
tags: scimap/mcmicro
- profile: "conda"
tags: sentieon/bwaindex
- profile: "conda"
tags: sentieon/bwamem
- profile: "conda"
tags: sentieon/dedup
- profile: "conda"
tags: sentieon/haplotyper
- profile: "conda"
tags: universc
- profile: "singularity"
tags: universc
- profile: "singularity"
tags: gatk4/determinegermlinecontigploidy
- profile: "conda"
tags: subworkflows/bcl_demultiplex
- profile: "conda"
tags: subworkflows/vcf_annotate_ensemblvep

env:
NXF_ANSI_LOG: false
SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
Expand All @@ -109,7 +148,7 @@ jobs:
with:
python-version: "3.x"

- uses: actions/cache@v2
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
Expand All @@ -119,7 +158,7 @@ jobs:
- name: Install Python dependencies
run: python -m pip install --upgrade pip pytest-workflow cryptography

- uses: actions/cache@v2
- uses: actions/cache@v3
with:
path: /usr/local/bin/nextflow
key: ${{ runner.os }}
Expand Down Expand Up @@ -178,11 +217,19 @@ jobs:
sudo apt-get install bat > /dev/null
batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}
- name: Setting global variables
uses: actions/github-script@v6
id: parsed
with:
script: |
return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"')
result-encoding: string

- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v2
with:
name: logs-${{ matrix.profile }}
name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }}
path: |
/home/runner/pytest_workflow_*/*/.nextflow.log
/home/runner/pytest_workflow_*/*/log.out
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
work/
results/
test_output/
.nf-test/
output/
.DS_Store
*.code-workspace
Expand Down
8 changes: 6 additions & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,13 @@ repos:
hooks:
- id: prettier
- repo: https://github.com/python-jsonschema/check-jsonschema
rev: 0.21.0
rev: 0.22.0
hooks:
- id: check-jsonschema
# match meta.ymls in one of the subdirectories of modules/nf-core
files: ^modules/nf-core/.*/meta\.yml$
args: ["--schemafile", "yaml-schema.json"]
args: ["--schemafile", "modules/yaml-schema.json"]
- id: check-jsonschema
# match meta.ymls in one of the subdirectories of subworkflows/nf-core
files: ^subworkflows/nf-core/.*/meta\.yml$
args: ["--schemafile", "subworkflows/yaml-schema.json"]
Loading

0 comments on commit 0e077da

Please sign in to comment.