Merge branch 'dev' into TEMPLATE

.circleci/config.yml

@@ -0,0 +1,109 @@
+version: 2.0
+jobs:
+  snpeffgrch37: &buildsnpeff
+    docker:
+      - image: circleci/buildpack-deps:stretch
+    environment:
+      GENOME: GRCh37
+      SNPEFF_CACHE_VERSION: "75"
+      SAREK_TAG: dev
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run:
+          command: docker build -t nfcore/sareksnpeff:${SAREK_TAG}.${GENOME} containers/snpeff/. --build-arg GENOME=${GENOME} --build-arg SNPEFF_CACHE_VERSION=${SNPEFF_CACHE_VERSION}
+      - run:
+          command: echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin ; docker push nfcore/sareksnpeff:${SAREK_TAG}.${GENOME}
+
+  snpeffgrch38:
+    << : *buildsnpeff
+    environment:
+      GENOME: GRCh38
+      SNPEFF_CACHE_VERSION: "86"
+      SAREK_TAG: dev
+
+  snpeffgrcm38:
+    << : *buildsnpeff
+    environment:
+      GENOME: GRCm38
+      SNPEFF_CACHE_VERSION: "86"
+      SAREK_TAG: dev
+
+  snpeffcanfam3_1:
+    << : *buildsnpeff
+    environment:
+      GENOME: CanFam3.1
+      SNPEFF_CACHE_VERSION: "86"
+      SAREK_TAG: dev
+
+  snpeffwbcel235:
+    << : *buildsnpeff
+    environment:
+      GENOME: WBcel235
+      SNPEFF_CACHE_VERSION: "86"
+      SAREK_TAG: dev
+
+  vepgrch37: &buildvep
+    docker:
+      - image: circleci/buildpack-deps:stretch
+    environment:
+      GENOME: GRCh37
+      SPECIES: homo_sapiens
+      VEP_VERSION: "99"
+      SAREK_TAG: dev
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run:
+          command: docker build -t nfcore/sarekvep:${SAREK_TAG}.${GENOME} containers/vep/. --build-arg GENOME=${GENOME} --build-arg SPECIES=${SPECIES} --build-arg VEP_VERSION=${VEP_VERSION}
+          no_output_timeout: 3h
+      - run:
+          command: echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin ; docker push nfcore/sarekvep:${SAREK_TAG}.${GENOME}
+          no_output_timeout: 30m
+
+  vepgrch38:
+    << : *buildvep
+    environment:
+      GENOME: GRCh38
+      SPECIES: homo_sapiens
+      VEP_VERSION: "99"
+      SAREK_TAG: dev
+
+  vepgrcm38:
+    << : *buildvep
+    environment:
+      GENOME: GRCm38
+      SPECIES: mus_musculus
+      VEP_VERSION: "99"
+      SAREK_TAG: dev
+
+  vepcanfam3_1:
+    << : *buildvep
+    environment:
+      GENOME: CanFam3.1
+      SPECIES: canis_familiaris
+      VEP_VERSION: "99"
+      SAREK_TAG: dev
+
+  vepwbcel235:
+    << : *buildvep
+    environment:
+      GENOME: WBcel235
+      SPECIES: caenorhabditis_elegans
+      VEP_VERSION: "99"
+      SAREK_TAG: dev
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - snpeffcanfam3_1
+      - snpeffgrch37
+      - snpeffgrch38
+      - snpeffgrcm38
+      - snpeffwbcel235
+      - vepcanfam3_1
+      - vepgrch37
+      - vepgrch38
+      - vepgrcm38
+      - vepwbcel235

.github/CODEOWNERS

@@ -0,0 +1 @@
+*   @MaxUlysse

.github/ISSUE_TEMPLATE/bug_report.md

@@ -38,7 +38,7 @@ Steps to reproduce the behaviour:
 
 - Engine: <!-- [e.g. Conda, Docker or Singularity] -->
 - version: <!-- [e.g. 1.0.0] -->
-- Image tag: <!-- [e.g. nfcore/sarek:1.0.0] -->
+- Image tag: <!-- [e.g. nfcore/sarek:2.6] -->
 
 ## Additional context
 

.github/RELEASE_CHECKLIST.md

@@ -0,0 +1,45 @@
+# Release checklist
+
+> This checklist is for our own reference, to help us prepare a new release
+
+1. Check that everything is ready to go
+   - Desired [PRs](https://github.com/nf-core/sarek/pulls) are merged
+   - [GitHub Actions](https://github.com/nf-core/sarek/actions?query=workflow%3A%22nf-core+CI%22) are passing on `dev`
+   - [nf-core linting](https://github.com/nf-core/sarek/actions?query=workflow%3A%22nf-core+linting%22) are passing on `dev`
+2. Increase version number following [semantic versioning](http://semver.org/spec/v2.0.0.html)
+3. Choose an appropriate codename for the release (if major or minor)
+   - i.e. Peaks in [Sarek National Park](https://en.wikipedia.org/wiki/Sarek_National_Park#Topography)
+4. Sync `dev` and checkout a new branch for the release
+5. Bump version:
+   - `nf-core bump-version . <VERSION>`
+   - edit `.circleci/config.yml`
+   - edit `.github/workflows/ci.yml`
+   - edit `conf/base.config`
+   - edit `conf/test.config`
+   - edit `containers/snpeff/Dockerfile`
+   - edit `containers/snpeff/environment.yml`
+   - edit `containers/vep/Dockerfile`
+   - edit `containers/vep/environment.yml`
+   - edit `docs/images/sarek_workflow.svg`
+   - generate a new `docs/images/sarek_workflow.png`
+   - edit `CHANGELOG`
+6. Make a PR to `master`
+7. Wait for reviews
+8. Merge said PR
+9. Make a [release](https://github.com/nf-core/sarek/releases) on GitHub
+10. Update [bio.tools](https://bio.tools/Sarek) with the new release details
+11. RT the nf-core automated tweet about the new released version
+12. Make a new branch from `dev`
+13. Checkout the `CHANGELOG.md` from `master`
+    - `git checkout upstream/master -- CHANGELOG.md`
+14. Add a new `Unreleased` section in `CHANGELOG.md` for the `dev` version
+15. Checkout `docs/images/sarek_workflow.svg` and `docs/images/sarek_workflow.pnh` from `master`
+    - `git checkout upstream/master -- docs/images/sarek_workflow.svg`
+    - `git checkout upstream/master -- docs/images/sarek_workflow.png`
+16. Make a PR to `dev`
+17. Wait for review
+18. Merge said PR
+19. Download all new containers to `/sw/data/uppnex/ToolBox/nf-core` on `rackham`
+20. Download newest `nf-core/sarek` to `/data1/containers` on `munin`
+21. Commit and push. Continue making more awesome :metal:
+22. Have fika :cake:

.github/markdownlint.yml

@@ -3,3 +3,8 @@ default: true,
 line-length: false
 no-duplicate-header:
     siblings_only: true
+no-inline-html: 
+    allowed_elements:
+        - a
+        - img
+        - p

.github/workflows/awstest.yml

@@ -21,9 +21,6 @@ jobs:
       - name: Install awscli
         run: conda install -c conda-forge awscli
       - name: Start AWS batch job
-        # TODO nf-core: You can customise CI pipeline run tests as required
-        # For example: adding multiple test runs with different parameters
-        # Remember that you can parallelise this by using strategy.matrix
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/branch.yml

@@ -1,5 +1,5 @@
 name: nf-core branch protection
-# This workflow is triggered on PRs to master branch on the repository
+# This workflow is triggered on PRs to `master` branch on the repository
 # It fails when someone tries to make a PR against the nf-core `master` branch instead of `dev`
 on:
   pull_request:

.github/workflows/ci.yml

@@ -41,15 +41,131 @@ jobs:
         run: |
           docker pull nfcore/sarek:dev
           docker tag nfcore/sarek:dev nfcore/sarek:dev
-
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
-
       - name: Run pipeline with test data
-        # TODO nf-core: You can customise CI pipeline run tests as required
-        # For example: adding multiple test runs with different parameters
-        # Remember that you can parallelise this by using strategy.matrix
         run: |
           nextflow run ${GITHUB_WORKSPACE} -profile test,docker
+
+  annotation:
+    env:
+      NXF_ANSI_LOG: false
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tools: [snpeff, vep]
+        species: [WBcel235]
+    steps:
+      - uses: actions/checkout@v2
+        env:
+          # Only check Nextflow pipeline minimum version
+          NXF_VER: '19.10.0'
+      - name: Pull docker image
+        run: |
+          docker pull nfcore/sarek:dev
+          docker pull nfcore/sarek${{ matrix.tools }}:dev.${{ matrix.species }}
+      - name: Run annotation test
+        run: nextflow run ${GITHUB_WORKSPACE} -profile test_annotation,docker --tools ${{ matrix.tools }}
+
+  germline:
+    env:
+      NXF_ANSI_LOG: false
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        markduplicates: [--skip_markduplicates, '']
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+        env:
+          # Only check Nextflow pipeline minimum version
+          NXF_VER: '19.10.0'
+      - name: Pull docker image
+        run: docker pull nfcore/sarek:dev
+      - name: Get test data
+        run: git clone --single-branch --branch sarek https://github.com/nf-core/test-datasets.git data
+      - name: Run germline test
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.markduplicates }} --input data/testdata/tiny/normal --saved_bam_mapped
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.markduplicates }} --input=false --step prepare_recalibration -resume
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.markduplicates }} --input=false --step recalibrate -resume
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.markduplicates }} --input=false --step variantCalling
+
+  minimal:
+    env:
+      NXF_ANSI_LOG: false
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        genome: [smallerGRCh37, minimalGRCh37]
+        intervals: [--no_intervals, '']
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+        env:
+          # Only check Nextflow pipeline minimum version
+          NXF_VER: '19.10.0'
+      - name: Pull docker image
+        run: docker pull nfcore/sarek:dev
+      - name: Run test for minimal genomes
+        run: nextflow run ${GITHUB_WORKSPACE} -profile test,docker --skipQC all --genome ${{ matrix.genome }} ${{ matrix.intervals }} --tools Manta,mpileup,Strelka,FreeBayes
+
+  profile:
+    env:
+      NXF_ANSI_LOG: false
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        profile: [test_split_fastq, test_targeted, test_trimming, test_no_gatk_spark, test_umi_tso, test_umi_qiaseq]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+        env:
+          # Only check Nextflow pipeline minimum version
+          NXF_VER: '19.10.0'
+      - name: Pull docker image
+        run: docker pull nfcore/sarek:dev
+      - name: Run ${{ matrix.profile }} test
+        run: nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.profile }},docker
+
+  tools:
+    env:
+      NXF_ANSI_LOG: false
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tool: [Haplotypecaller, Freebayes, Manta, mpileup, MSIsensor, Strelka, TIDDIT]
+        intervals: [--no_intervals, '']
+        exclude:
+          - tool: Manta
+            intervals: --no_intervals
+          - tool: MSIsensor
+            intervals: --no_intervals
+          - tool: Strelka
+            intervals: --no_intervals
+          - tool: TIDDIT
+            intervals: --no_intervals
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+        env:
+          # Only check Nextflow pipeline minimum version
+          NXF_VER: '19.10.0'
+      - name: Pull docker image
+        run: docker pull nfcore/sarek:dev
+      - name: Run ${{ matrix.tool }} test
+        run: nextflow run ${GITHUB_WORKSPACE} -profile test_tool,docker --tools ${{ matrix.tool }} ${{ matrix.intervals }}

.gitignore

@@ -1,6 +1,7 @@
 .nextflow*
 work/
 data/
+references/
 results/
 .DS_Store
 tests/