Release candidate 3.2.2

.github/workflows/awsfulltest.yml

@@ -23,6 +23,7 @@ jobs:
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}/somatic_test
           parameters: |
             {
+              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/somatic_test"
             }
           profiles: test_full,public_aws_ecr

.github/workflows/awsfulltest_germline.yml

@@ -23,6 +23,7 @@ jobs:
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}/germline_test
           parameters: |
             {
+              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/germline_test"
             }
           profiles: test_full_germline,public_aws_ecr

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
           - "latest-everything"
         test:
           - "default"
-        profile: ["docker", "singularity"]
+        profile: ["docker"]
         # profile: ["docker", "singularity", "conda"]
     env:
       NXF_ANSI_LOG: false

.github/workflows/pytest-workflow.yml

@@ -2,7 +2,7 @@ name: pytest-workflow
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
 on:
   pull_request:
-    branches: [dev, master]
+    branches: [dev]
 
 # Cancel if a newer run is started
 concurrency:
@@ -32,7 +32,7 @@ jobs:
       fail-fast: false
       matrix:
         tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
-        profile: ["docker", "singularity"]
+        profile: ["docker"]
         # profile: ["docker", "singularity", "conda"]
         TEST_DATA_BASE:
           - "test-datasets/data"

.github/workflows/pytest-workflow_release.yml

@@ -0,0 +1,154 @@
+name: pytest-workflow-release
+# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+on:
+  pull_request:
+    branches: [master]
+  release:
+    types: [published]
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    name: Check for changes
+    runs-on: ubuntu-latest
+    outputs:
+      # Expose matched filters as job 'tags' output variable
+      tags: ${{ steps.filter.outputs.changes }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: "tests/config/tags.yml"
+
+  test:
+    name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }}
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.tags != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
+        profile: ["docker", "singularity"]
+        TEST_DATA_BASE:
+          - "test-datasets/data"
+        NXF_VER:
+          - "23.04.0"
+          - "latest-everything"
+        exclude:
+          - profile: "singularity"
+            tags: concatenate_vcfs
+          - profile: "singularity"
+            tags: merge
+          - profile: "singularity"
+            tags: validation_checks
+    env:
+      NXF_ANSI_LOG: false
+      TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v3
+
+      - name: Hash Github Workspace
+        id: hash_workspace
+        run: |
+          echo "digest=$(echo sarek3_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT
+
+      - name: Cache test data
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: test-datasets/
+          key: ${{ steps.hash_workspace.outputs.digest }}
+
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
+        with:
+          repository: nf-core/test-datasets
+          ref: sarek3
+          path: test-datasets/
+
+      - name: Replace remote paths in samplesheets
+        run: |
+          for f in tests/csv/3.0/*csv; do
+            sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/modules/=${{ github.workspace }}/test-datasets/=g" $f
+            echo "========== $f ============"
+            cat $f
+            echo "========================================"
+          done;
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install Python dependencies
+        run: python -m pip install --upgrade pip pytest-workflow
+
+      - uses: actions/cache@v3
+        with:
+          path: /usr/local/bin/nextflow
+          key: ${{ runner.os }}
+          restore-keys: |
+            ${{ runner.os }}-nextflow-
+
+      - name: Install Nextflow ${{ matrix.NXF_VER }}
+        uses: nf-core/[email protected]
+        with:
+          version: "${{ matrix.NXF_VER }}"
+
+      - name: Set up Singularity
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-singularity@v5
+        with:
+          singularity-version: 3.7.1
+
+      - name: Set up miniconda
+        if: matrix.profile == 'conda'
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          channels: conda-forge,bioconda,defaults
+          python-version: ${{ matrix.python-version }}
+
+      - name: Conda clean
+        if: matrix.profile == 'conda'
+        run: conda clean -a
+
+      - name: Run pytest-workflow
+        uses: Wandalen/[email protected]
+        with:
+          command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes
+          attempt_limit: 3
+
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}
+
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: logs-${{ matrix.profile }}
+          path: |
+            /home/runner/pytest_workflow_*/*/.nextflow.log
+            /home/runner/pytest_workflow_*/*/log.out
+            /home/runner/pytest_workflow_*/*/log.err
+            /home/runner/pytest_workflow_*/*/work
+            !/home/runner/pytest_workflow_*/*/work/conda
+            !/home/runner/pytest_workflow_*/*/work/singularity

CHANGELOG.md

@@ -5,6 +5,35 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.2.2](https://github.com/nf-core/sarek/releases/tag/3.2.2) - Vuoinesluobbalah
+
+Vuoinesluobbalah is a lake close to Bierikjávrre.
+
+### Added
+
+- [#1106](https://github.com/nf-core/sarek/pull/1106) - Add Slack integration to Megatests
+- [#1107](https://github.com/nf-core/sarek/pull/1107) - Add `singularity.registry` to `public_aws_ecr`
+
+### Changed
+
+- [#1087](https://github.com/nf-core/sarek/pull/1087) - Back to dev
+- [#1087](https://github.com/nf-core/sarek/pull/1087) - Minor modules update
+- [#1088](https://github.com/nf-core/sarek/pull/1088) - Replace profile `test` by `test_cache` and add a `test` profile without hidden files
+- [#1095](https://github.com/nf-core/sarek/pull/1095) - Prepare release `3.2.2`
+
+### Fixed
+
+- [#1087](https://github.com/nf-core/sarek/pull/1087) - Fix wrong default memory in GATK4_CREATESEQUENCEDICTIONARY [#1085](https://github.com/nf-core/sarek/pull/1085)
+- [#1089](https://github.com/nf-core/sarek/pull/1089) - Remove duplicated code
+- [#1093](https://github.com/nf-core/sarek/pull/1093) - Fixing Ascat by reverting meta.id in channels allele_files, loci_files, gc_file and rt_file to baseName.
+- [#1098](https://github.com/nf-core/sarek/pull/1098) - Fix Channel issue in Mutect2 subworkflow [#1094](https://github.com/nf-core/sarek/pull/1094)
+- [#1100](https://github.com/nf-core/sarek/pull/1100) - Remove duplicate index with deepvariant when no_intervals [#1069](https://github.com/nf-core/sarek/pull/1069)
+- [#1101](https://github.com/nf-core/sarek/pull/1101) - Remove duplicate index computation for GATK4 Markduplicates & [#1065](https://github.com/nf-core/sarek/issues/1065)
+- [#1101](https://github.com/nf-core/sarek/pull/1101) - Fix GATK4 version for GATK4 MarkduplicatesSpark [#1068](https://github.com/nf-core/sarek/issues/1068)
+- [#1105](https://github.com/nf-core/sarek/pull/1105) - Remove `params.tracedir`
+- [#1108](https://github.com/nf-core/sarek/pull/1108) - Refactor bad prefix definition for vcf files [#938](https://github.com/nf-core/sarek/issues/938)
+- [#1109](https://github.com/nf-core/sarek/pull/1109) - Fix `mpileup` for variantcalling: only `bcftools` run and file publishing
+
 ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure
 
 Pierikjaure is a previous spelling of Bierikjávrre.

conf/modules/annotate.config

@@ -18,7 +18,7 @@ process {
     // SNPEFF
     if (params.tools && (params.tools.split(',').contains('snpeff') || params.tools.split(',').contains('merge'))) {
         withName: 'SNPEFF_SNPEFF' {
-            ext.prefix       = { "${vcf.baseName.minus(".vcf")}_snpEff" }
+            ext.prefix       = { vcf.baseName - ".vcf" + "_snpEff" }
             ext.args         = '-nodownload -canon -v'
             if (!params.snpeff_cache && !params.download_cache) container = { params.snpeff_genome ? "docker.io/nfcore/snpeff:${params.snpeff_version}.${params.snpeff_genome}" : "docker.io/nfcore/snpeff:${params.snpeff_version}.${params.genome}" }
             publishDir       = [
@@ -46,7 +46,7 @@ process {
                 (params.vep_custom_args)                                              ?: ''
             ].join(' ').trim() }
             // If just VEP: <vcf prefix>_VEP.ann.vcf
-            ext.prefix       = { "${vcf.baseName.minus(".vcf")}_VEP.ann" }
+            ext.prefix       = { vcf.baseName - ".vcf" + "_VEP.ann" }
             if (!params.vep_cache && !params.download_cache)    container = { params.vep_genome ? "docker.io/nfcore/vep:${params.vep_version}.${params.vep_genome}" : "docker.io/nfcore/vep:${params.vep_version}.${params.genome}" }
             publishDir       = [
                 [
@@ -67,14 +67,14 @@ process {
     if (params.tools && params.tools.split(',').contains('merge')) {
         withName: "NFCORE_SAREK:SAREK:VCF_ANNOTATE_ALL:VCF_ANNOTATE_MERGE:ENSEMBLVEP_VEP" {
             // If merge: Output file will have format *_snpEff_VEP.ann.vcf, *_snpEff_VEP.ann.json or *_snpEff_VEP.ann.tab
-            ext.prefix       = { "${vcf.baseName.minus(".ann.vcf")}_VEP.ann" }
+            ext.prefix       = { vcf.baseName - ".ann.vcf" + "_VEP.ann" }
         }
     }
 
     // ALL ANNOTATION TOOLS
     if (params.tools && (params.tools.split(',').contains('snpeff') || params.tools.split(',').contains('vep') || params.tools.split(',').contains('merge'))) {
         withName: "NFCORE_SAREK:SAREK:VCF_ANNOTATE_ALL:.*:(TABIX_BGZIPTABIX|TABIX_TABIX)" {
-            ext.prefix       = { input.baseName.minus(".vcf") }
+            ext.prefix       = { input.name - ".vcf" }
             publishDir       = [
                     mode: params.publish_dir_mode,
                     path: { "${params.outdir}/annotation/${meta.variantcaller}/${meta.id}/" },

conf/modules/deepvariant.config

@@ -22,7 +22,7 @@ process {
         publishDir       = [
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/" },
-            pattern: "*vcf.gz",
+            pattern: "*{vcf.gz,vcf.gz.tbi}",
             saveAs: { meta.num_intervals > 1 ? null : "deepvariant/${meta.id}/${it}" }
         ]
     }
@@ -40,12 +40,4 @@ process {
         ext.prefix       = {"${meta.id}.deepvariant.g"}
     }
 
-    withName : 'TABIX_VC_DEEPVARIANT_.*' {
-        ext.prefix       = {"${meta.id}.deepvariant"}
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/variant_calling/deepvariant/${meta.id}/" },
-            pattern: "*tbi"
-        ]
-    }
 }

conf/modules/freebayes.config

@@ -35,7 +35,7 @@ process {
     }
 
     withName: 'BCFTOOLS_SORT' {
-        ext.prefix       = { meta.num_intervals <= 1 ? "${meta.id}.freebayes" : "${vcf.minus("vcf")}.sort" }
+        ext.prefix       = { meta.num_intervals <= 1 ? meta.id + ".freebayes" : vcf.name - ".vcf" + ".sort" }
         publishDir       = [
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/" },

conf/modules/joint_germline.config

@@ -32,7 +32,7 @@ process {
 
     if (params.tools && params.tools.contains('haplotypecaller') && params.joint_germline) {
         withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_GATK:BCFTOOLS_SORT' {
-            ext.prefix       = { "${vcf.baseName.minus("vcf")}sort" }
+            ext.prefix       = { vcf.baseName - ".vcf" + ".sort" }
             publishDir       = [
                 enabled: false
             ]

conf/modules/modules.config

@@ -90,7 +90,7 @@ process {
     // VCF
     withName: 'BCFTOOLS_STATS' {
         ext.when         = { !(params.skip_tools && params.skip_tools.split(',').contains('bcftools')) }
-        ext.prefix       = { "${vcf.baseName.minus(".vcf")}" }
+        ext.prefix       = { vcf.baseName - ".vcf" }
         publishDir       = [
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/reports/bcftools/${meta.variantcaller}/${meta.id}/" },
@@ -100,7 +100,7 @@ process {
 
     withName: 'VCFTOOLS_.*' {
         ext.when         = { !(params.skip_tools && params.skip_tools.split(',').contains('vcftools')) }
-        ext.prefix       = { "${variant_file.baseName.minus(".vcf")}" }
+        ext.prefix       = { variant_file.baseName - ".vcf" }
         publishDir       = [
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/reports/vcftools/${meta.variantcaller}/${meta.id}/" },

conf/modules/mpileup.config

@@ -17,10 +17,7 @@ process {
 
     withName: 'CAT_MPILEUP' {
         publishDir       = [
-            enabled: true,
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/variant_calling/mpileup/${meta.id}/" },
-            pattern: "*{mpileup.gz}"
+            enabled: false
         ]
     }
 
@@ -31,25 +28,31 @@ process {
         ext.args3  = "-i 'count(GT==\"RR\")==0'" // only report non homozygous reference variants
         publishDir = [
             mode: params.publish_dir_mode,
-            path: { "${params.outdir}/variant_calling/mpileup/${meta.id}/" },
+            path: { "${params.outdir}/variant_calling/bcftools/${meta.id}/" },
             pattern: "*{vcf.gz,vcf.gz.tbi}",
             saveAs: { meta.num_intervals > 1 ? null : it }
         ]
     }
 
+    withName: 'MERGE_BCFTOOLS_MPILEUP' {
+        ext.prefix       = {"${meta.id}.bcftools"}
+        publishDir       = [
+            mode: params.publish_dir_mode,
+            path: { "${params.outdir}/variant_calling/bcftools/${meta.id}/" },
+            pattern: "*{vcf.gz,vcf.gz.tbi}"
+        ]
+    }
+
     withName: 'SAMTOOLS_MPILEUP' {
-        ext.when   = { params.tools && (params.tools.split(',').contains('controlfreec') || params.tools.split(',').contains('mpileup')) }
+        ext.when   = { params.tools && params.tools.split(',').contains('controlfreec') }
         publishDir = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/variant_calling/mpileup/${meta.id}/" },
-            pattern: "*mpileup.gz",
-            saveAs: { meta.num_intervals > 1 ? null : it }
+            enabled: false
         ]
 
     }
 
 // PAIR_VARIANT_CALLING
-    if (params.tools && (params.tools.split(',').contains('controlfreec') || params.tools.split(',').contains('mpileup'))) {
+    if (params.tools && params.tools.split(',').contains('controlfreec')) {
         withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_VARIANT_CALLING_MPILEUP:SAMTOOLS_MPILEUP' {
             ext.prefix = { meta.num_intervals <= 1 ? "${meta.id}.normal" : "${meta.id}_${intervals.simpleName}.normal" }
         }

conf/public_aws_ecr.config

@@ -8,8 +8,9 @@
 ----------------------------------------------------------------------------------------
 */
 
-docker.registry = 'public.ecr.aws'
-podman.registry = 'public.ecr.aws'
+docker.registry      = 'public.ecr.aws'
+podman.registry      = 'public.ecr.aws'
+singularity.registry = 'public.ecr.aws'
 
 process {
     withName: 'ADD_INFO_TO_VCF' {