Skip to content

Commit

Permalink
Update-cellranger-modules (#5016)
Browse files Browse the repository at this point in the history
* Upgraded Dockerfile for CellRanger to 7.2.0

* Upgraded all modules to use 7.2.0 CellRanger

* Update dockerfile accordingly

* Update mkfastq

* Update to use docker hub for the images

* Update dockerfile for 8.0.0 release

* Improved readme

* Update all modules to use 8.0.0 container

* Make litner happier

* fix version info

* delete environment.yml files since it breaks linting and tool has no conda package

* remove pytest

* updated cellranger/mkref to nf-test

* remove mkref from pytest

* removed pytests

* update naming of tests

* updated cellranger/count tests for new v8.0.0 ( new version produces different results, thus new md5sums )

* remove cellranger/mkfastq from pytest

* port cellranger/mkfastq to nf-test

* remove cellranger/mkgtf from pytests

* Fix lint

* port cellranger/mkgtf to nf-tests

* Fix md5sum, so things can be ported over

* Fix untar tag for mkfastq

* Porting the tests over to nf-test

* Update tags

* remove from pytest

* port cellranger/vdj to nf-tests

* ported cellranger/mkvdjref to nf-tests

* update tag

* remove from pytest

* ported cellranger/multi to nf-test

* Fix vdj tests and linting

* Fix lint + tests for multi cellranger

* Skip conda tests for nf-test

* Fix multi tests, fix linting for multi tests

* Fix multi after all

* Fix conda in nf-test.yml

* You again, CR/multi!

* Fixed it

* main.nf.test aktualisieren

Co-authored-by: Maxime U Garcia <[email protected]>

* main.nf.test aktualisieren

Co-authored-by: Maxime U Garcia <[email protected]>

* Ditched configs that are not used at all

* Add vdj proper tests

* versions for multi

* mkref tests improvement(s)

* Adding in versions, too

* Adjusting time limits to 4 hours for cellranger modules (2hrs cause break)

* Pin to CPU= 2

* Fix to 5GB

* Add note

* remove process-specific nextflow configuration for cellranger count

* also for mkref

* Remove process resource configurations

* fix syntax error

* Add resource limits for mkref

---------

Co-authored-by: Gregor Sturm <[email protected]>
Co-authored-by: Felipe Marques de Almeida <[email protected]>
Co-authored-by: Maxime U Garcia <[email protected]>
Co-authored-by: Gregor Sturm <[email protected]>
  • Loading branch information
5 people authored Mar 26, 2024
1 parent 0735b6d commit e66183d
Show file tree
Hide file tree
Showing 63 changed files with 1,687 additions and 1,008 deletions.
20 changes: 6 additions & 14 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -207,20 +207,6 @@ jobs:
tags: bwa/samse
- profile: conda
tags: cellpose
- profile: conda
tags: cellranger/count
- profile: conda
tags: cellranger/mkfastq
- profile: conda
tags: cellranger/mkgtf
- profile: conda
tags: cellranger/mkref
- profile: conda
tags: cellranger/mkvdjref
- profile: conda
tags: cellranger/multi
- profile: conda
tags: cellranger/vdj
- profile: conda
tags: cellrangerarc/count
- profile: conda
Expand Down Expand Up @@ -517,6 +503,12 @@ jobs:
tags: cellranger/mkgtf
- profile: conda
tags: cellranger/mkref
- profile: conda
tags: cellranger/mkvdjref
- profile: conda
tags: cellranger/multi
- profile: conda
tags: cellranger/vdj
- profile: conda
tags: custom/dumpsoftwareversions
- profile: conda
Expand Down
4 changes: 2 additions & 2 deletions modules/nf-core/cellranger/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Dockerfile to create container with Cell Ranger v7.1.0
# Dockerfile to create container with Cell Ranger v8.0.0
# Push to nfcore/cellranger:<VER>

FROM docker.io/continuumio/miniconda3:22.11.1@sha256:10b38c9a8a51692838ce4517e8c74515499b68d58c8a2000d8a9df7f0f08fc5e
Expand All @@ -16,7 +16,7 @@ RUN apt-get update --allow-releaseinfo-change \
&& apt-get clean -y && rm -rf /var/lib/apt/lists/*

# Copy pre-downloaded cellranger file
ENV CELLRANGER_VER=7.1.0
ENV CELLRANGER_VER=8.0.0
COPY cellranger-$CELLRANGER_VER.tar.gz /opt/cellranger-$CELLRANGER_VER.tar.gz

# Install cellranger
Expand Down
4 changes: 2 additions & 2 deletions modules/nf-core/cellranger/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ ENV CELLRANGER_VER=<VERSION>
3. Create and test the container:

```bash
docker build . -t nfcore/cellranger:<VERSION>
docker build . -t quay.io/nf-core/cellranger:<VERSION>
```

4. Access rights are needed to push the container to the Dockerhub nfcore organization, please ask a core team member to do so.

```bash
docker push nfcore/cellranger:<VERSION>
docker push quay.io/nf-core/cellranger:<VERSION>
```
2 changes: 1 addition & 1 deletion modules/nf-core/cellranger/count/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ process CELLRANGER_COUNT {
tag "$meta.id"
label 'process_high'

container "nf-core/cellranger:7.1.0"
container "nf-core/cellranger:8.0.0"

input:
tuple val(meta), path(reads, stageAs: "fastq_???/*")
Expand Down
17 changes: 9 additions & 8 deletions modules/nf-core/cellranger/count/tests/main.nf.test
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ nextflow_process {

name "Test Process CELLRANGER_COUNT"
script "../main.nf"
config "./nextflow.config"
process "CELLRANGER_COUNT"

tag "modules"
Expand All @@ -28,7 +29,7 @@ nextflow_process {
"""
input[0] = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true)
input[1] = CELLRANGER_MKGTF.out.gtf
input[2] = "homo_sapiens_chr22_reference"
input[2] = "homo_sapiens_reference"
"""
}
}
Expand All @@ -55,13 +56,13 @@ nextflow_process {
assertAll(
{ assert process.success },
{
assert snapshot(
process.out.versions,
process.out.outs[0][1].findAll { file(it).name !in [
'web_summary.html', // unstable checksum
'barcodes.tsv.gz' // empty file in test data -> would raise linting error
]}
).match()
assert snapshot(
process.out.versions,
process.out.outs[0][1].findAll { file(it).name !in [
'web_summary.html', // unstable checksum
'barcodes.tsv.gz' // empty file in test data -> would raise linting error
]}
).match()
},
{ assert file(process.out.outs.get(0).get(1).find { file(it).name == 'web_summary.html' }).exists() },
{ assert file(process.out.outs.get(0).get(1).find { file(it).name == 'barcodes.tsv.gz' }).exists() }
Expand Down
24 changes: 12 additions & 12 deletions modules/nf-core/cellranger/count/tests/main.nf.test.snap

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion modules/nf-core/cellranger/count/tests/nextflow.config
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,6 @@ process {
--attribute=gene_biotype:TR_C_gene'
}
withName: CELLRANGER_COUNT {
ext.args = '--chemistry SC3Pv3'
ext.args = '--chemistry SC3Pv3 --create-bam true'
}
}
4 changes: 2 additions & 2 deletions modules/nf-core/cellranger/mkfastq/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Dockerfile to create container with Cell Ranger v7.1.0 and bcl2fastq v2.20.0
# Dockerfile to create container with Cell Ranger v8.0.0 and bcl2fastq v2.20.0
# Push to nfcore/cellrangermkfastq:<VER>

FROM continuumio/miniconda3:4.8.2
Expand All @@ -17,7 +17,7 @@ RUN apt-get update --allow-releaseinfo-change \

# Copy pre-downloaded bcl2fastq2 and cellranger file
ENV BCL2FASTQ2_VER=v2-20-0-linux-x86-64 \
CELLRANGER_VER=7.1.0
CELLRANGER_VER=8.0.0
COPY bcl2fastq2-$BCL2FASTQ2_VER.zip /tmp/bcl2fastq2-$BCL2FASTQ2_VER.zip
COPY cellranger-$CELLRANGER_VER.tar.gz /opt/cellranger-$CELLRANGER_VER.tar.gz

Expand Down
4 changes: 2 additions & 2 deletions modules/nf-core/cellranger/mkfastq/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ ENV BCL2FASTQ2_VER=<VERSION> \
3. Create and test the container:

```bash
docker build . -t nfcore/cellrangermkfastq:<CELLRANGER_VERSION>
docker build . -t quay.io/nf-core/cellrangermkfastq:<CELLRANGER_VERSION>
```

4. Access rights are needed to push the container to the Dockerhub nfcore organization, please ask a core team member to do so.

```bash
docker push nfcore/cellrangermkfastq:<CELLRANGER_VERSION>
docker push quay.io/nf-core/cellrangermkfastq:<CELLRANGER_VERSION>
```
5 changes: 0 additions & 5 deletions modules/nf-core/cellranger/mkfastq/environment.yml

This file was deleted.

11 changes: 9 additions & 2 deletions modules/nf-core/cellranger/mkfastq/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ process CELLRANGER_MKFASTQ {
tag "mkfastq"
label 'process_medium'

container "docker.io/nfcore/cellrangermkfastq:7.1.0"
container "nf-core/cellrangermkfastq:8.0.0"

input:
path bcl
Expand Down Expand Up @@ -44,7 +44,14 @@ process CELLRANGER_MKFASTQ {
def prefix = task.ext.prefix ?: "${bcl.getSimpleName()}"
"""
mkdir -p "${prefix}/outs/fastq_path/"
touch ${prefix}/outs/fastq_path/fake_file.fastq.gz
# data with something to avoid breaking nf-test java I/O stream
cat <<-FAKE_FQ > ${prefix}/outs/fastq_path/fake_file.fastq
@SEQ_ID
GATTTGGGGTTCAAAGCAGTATCGATCAAATAGTAAATCCATTTGTTCAACTCACAGTTT
+
!''*((((***+))%%%++)(%%%%).1***-+*''))**55CCF>>>>>>CCCCCCC65
FAKE_FQ
gzip -n ${prefix}/outs/fastq_path/fake_file.fastq
cat <<-END_VERSIONS > versions.yml
"${task.process}":
Expand Down
2 changes: 1 addition & 1 deletion modules/nf-core/cellranger/mkfastq/meta.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ tools:
homepage: https://support.10xgenomics.com/single-cell-gene-expression/software/pipelines/latest/what-is-cell-ranger
documentation: https://support.10xgenomics.com/single-cell-gene-expression/software/pipelines/latest/using/tutorial_ov
tool_dev_url: https://support.10xgenomics.com/single-cell-gene-expression/software/pipelines/latest/using/tutorial_ov
licence: 10x Genomics EULA
licence: [10X Genomics EULA]
input:
- bcl:
type: file
Expand Down
91 changes: 91 additions & 0 deletions modules/nf-core/cellranger/mkfastq/tests/main.nf.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// nf-core modules test cellranger/mkfastq
nextflow_process {

name "Test Process CELLRANGER_MKFASTQ"
script "../main.nf"
config "./nextflow.config"
process "CELLRANGER_MKFASTQ"

tag "modules"
tag "modules_nfcore"
tag "cellranger"
tag "cellranger/mkfastq"
tag "untar"

setup {
run("UNTAR") {
script "modules/nf-core/untar/main.nf"
process {
"""
input[0] = [
[],
file("https://cf.10xgenomics.com/supp/cell-exp/cellranger-tiny-bcl-1.2.0.tar.gz", checkIfExists: true)
]
"""
}
}
}

test("cellranger - tiny - simple") {

when {
process {
"""
input[0] = UNTAR.out.untar.map{ it[1] }
input[1] = file("https://cf.10xgenomics.com/supp/cell-exp/cellranger-tiny-bcl-simple-1.2.0.csv", checkIfExists: true)
"""
}
}

then {
assertAll(
{ assert process.success },
{ assert snapshot(process.out).match() }
)
}

}

test("cellranger - tiny - illumina") {

when {
process {
"""
input[0] = UNTAR.out.untar.map{ it[1] }
input[1] = file("https://cf.10xgenomics.com/supp/cell-exp/cellranger-tiny-bcl-samplesheet-1.2.0.csv", checkIfExists: true)
"""
}
}

then {
assertAll(
{ assert process.success },
{ assert snapshot(process.out).match() }
)
}

}

test("cellranger - tiny - simple - stub") {

options "-stub"

when {
process {
"""
input[0] = UNTAR.out.untar.map{ it[1] }
input[1] = file("https://cf.10xgenomics.com/supp/cell-exp/cellranger-tiny-bcl-simple-1.2.0.csv", checkIfExists: true)
"""
}
}

then {
assertAll(
{ assert process.success },
{ assert snapshot(process.out).match() }
)
}

}

}
Loading

0 comments on commit e66183d

Please sign in to comment.