diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efd086d6..7af44ea3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: fail-fast: false matrix: NXF_VER: - - "24.04.2" + - "24.10.2" - "latest-everything" profile: - "conda" diff --git a/README.md b/README.md index 4410c6e2..1f79805a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![GitHub Actions CI Status](https://github.com/nf-core/fetchngs/actions/workflows/ci.yml/badge.svg)](https://github.com/nf-core/fetchngs/actions/workflows/ci.yml) [![GitHub Actions Linting Status](https://github.com/nf-core/fetchngs/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/fetchngs/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/fetchngs/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.5070524-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.5070524)[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.2-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.10.2-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/conf/base.config b/conf/base.config index 13b3c3e7..1c4b53df 100644 --- a/conf/base.config +++ b/conf/base.config @@ -15,12 +15,6 @@ process { memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } - publishDir = [ - path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' diff --git a/main.nf b/main.nf index 6280a384..ee98b2bf 100644 --- a/main.nf +++ b/main.nf @@ -9,6 +9,9 @@ ---------------------------------------------------------------------------------------- */ +nextflow.preview.output = true +nextflow.preview.topic = true + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS @@ -18,6 +21,7 @@ include { SRA } from './workflows/sra' include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_fetchngs_pipeline' include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_fetchngs_pipeline' +include { softwareVersionsToYAML } from './subworkflows/nf-core/utils_nfcore_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -38,8 +42,16 @@ workflow NFCORE_FETCHNGS { // // WORKFLOW: Download FastQ files for SRA / ENA / GEO / DDBJ ids // - SRA ( ids ) + SRA ( + ids, + params.ena_metadata_fields ?: '', + params.download_method, + params.skip_fastq_download, + params.dbgap_key + ) + emit: + samples = SRA.out.samples } /* @@ -81,6 +93,36 @@ workflow { params.monochrome_logs, params.hook_url ) + + publish: + NFCORE_FETCHNGS.out.samples >> 'samples' + softwareVersionsToYAML() >> 'versions' +} + + +output { + samples { + path { _sample -> + def dirs = [ + 'fastq': 'fastq', + 'md5': 'fastq/md5' + ] + return { filename -> + def ext = filename.tokenize('.').last() + "${dirs[ext]}/${filename}" + } + } + index { + path 'samplesheet/samplesheet.json' + } + } + + versions { + path '.' + index { + path 'nf_core_fetchngs_software_mqc_versions.yml' + } + } } /* diff --git a/modules/local/aspera_cli/main.nf b/modules/local/aspera_cli/main.nf index b38d17c0..ceef844f 100644 --- a/modules/local/aspera_cli/main.nf +++ b/modules/local/aspera_cli/main.nf @@ -14,7 +14,7 @@ process ASPERA_CLI { output: tuple val(meta), path("*fastq.gz"), emit: fastq tuple val(meta), path("*md5") , emit: md5 - path "versions.yml" , emit: versions + tuple val("${task.process}"), val('aspera_cli'), eval('ascli --version'), topic: versions script: def args = task.ext.args ?: '' @@ -31,11 +31,6 @@ process ASPERA_CLI { echo "${meta.md5_1} ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5 md5sum -c ${meta.id}.fastq.gz.md5 - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - aspera_cli: \$(ascli --version) - END_VERSIONS """ } else { """ @@ -58,11 +53,22 @@ process ASPERA_CLI { echo "${meta.md5_2} ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5 md5sum -c ${meta.id}_2.fastq.gz.md5 - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - aspera_cli: \$(ascli --version) - END_VERSIONS """ } } + +workflow { + def input = [ + [ id:'SRX9626017_SRR13191702', single_end:false, md5_1: '89c5be920021a035084d8aeb74f32df7', md5_2: '56271be38a80db78ef3bdfc5d9909b98' ], + [ + 'fasp.sra.ebi.ac.uk:/vol1/fastq/SRR131/002/SRR13191702/SRR13191702_1.fastq.gz', + 'fasp.sra.ebi.ac.uk:/vol1/fastq/SRR131/002/SRR13191702/SRR13191702_2.fastq.gz' + ] + ] + def user = 'era-fasp' + + ASPERA_CLI ( + input, + user + ) +} diff --git a/modules/local/aspera_cli/nextflow.config b/modules/local/aspera_cli/nextflow.config index fa2dbd90..9a808242 100644 --- a/modules/local/aspera_cli/nextflow.config +++ b/modules/local/aspera_cli/nextflow.config @@ -1,17 +1,5 @@ process { withName: 'ASPERA_CLI' { ext.args = '-QT -l 300m -P33001' - publishDir = [ - [ - path: { "${params.outdir}/fastq" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz" - ], - [ - path: { "${params.outdir}/fastq/md5" }, - mode: params.publish_dir_mode, - pattern: "*.md5" - ] - ] } } diff --git a/modules/local/multiqc_mappings_config/main.nf b/modules/local/multiqc_mappings_config/main.nf index ac379065..b09f06da 100644 --- a/modules/local/multiqc_mappings_config/main.nf +++ b/modules/local/multiqc_mappings_config/main.nf @@ -11,17 +11,12 @@ process MULTIQC_MAPPINGS_CONFIG { output: path "*config.yml" , emit: yml - path "versions.yml", emit: versions + tuple val("${task.process}"), val('python'), eval("python --version | sed 's/Python //g'"), topic: versions script: """ multiqc_mappings_config.py \\ $csv \\ multiqc_config.yml - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - python: \$(python --version | sed 's/Python //g') - END_VERSIONS """ } diff --git a/modules/local/multiqc_mappings_config/nextflow.config b/modules/local/multiqc_mappings_config/nextflow.config deleted file mode 100644 index 11c58341..00000000 --- a/modules/local/multiqc_mappings_config/nextflow.config +++ /dev/null @@ -1,9 +0,0 @@ -process { - withName: 'MULTIQC_MAPPINGS_CONFIG' { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } -} diff --git a/modules/local/sra_fastq_ftp/main.nf b/modules/local/sra_fastq_ftp/main.nf index 017a08c9..ae47c3a6 100644 --- a/modules/local/sra_fastq_ftp/main.nf +++ b/modules/local/sra_fastq_ftp/main.nf @@ -15,7 +15,7 @@ process SRA_FASTQ_FTP { output: tuple val(meta), path("*fastq.gz"), emit: fastq tuple val(meta), path("*md5") , emit: md5 - path "versions.yml" , emit: versions + tuple val("${task.process}"), val('wget'), eval("echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//')"), topic: versions script: def args = task.ext.args ?: '' @@ -28,11 +28,6 @@ process SRA_FASTQ_FTP { echo "${meta.md5_1} ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5 md5sum -c ${meta.id}.fastq.gz.md5 - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - wget: \$(echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//')) - END_VERSIONS """ } else { """ @@ -51,11 +46,6 @@ process SRA_FASTQ_FTP { echo "${meta.md5_2} ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5 md5sum -c ${meta.id}_2.fastq.gz.md5 - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - wget: \$(echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//')) - END_VERSIONS """ } } diff --git a/modules/local/sra_fastq_ftp/nextflow.config b/modules/local/sra_fastq_ftp/nextflow.config index 56e43959..26261f26 100644 --- a/modules/local/sra_fastq_ftp/nextflow.config +++ b/modules/local/sra_fastq_ftp/nextflow.config @@ -1,17 +1,5 @@ process { withName: 'SRA_FASTQ_FTP' { ext.args = '-t 5 -nv -c -T 60' - publishDir = [ - [ - path: { "${params.outdir}/fastq" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz" - ], - [ - path: { "${params.outdir}/fastq/md5" }, - mode: params.publish_dir_mode, - pattern: "*.md5" - ] - ] } } diff --git a/modules/local/sra_ids_to_runinfo/main.nf b/modules/local/sra_ids_to_runinfo/main.nf index 7d47f5e3..4e7f9e87 100644 --- a/modules/local/sra_ids_to_runinfo/main.nf +++ b/modules/local/sra_ids_to_runinfo/main.nf @@ -14,7 +14,7 @@ process SRA_IDS_TO_RUNINFO { output: path "*.tsv" , emit: tsv - path "versions.yml", emit: versions + tuple val("${task.process}"), val('python'), eval("python --version | sed 's/Python //g'"), topic: versions script: def metadata_fields = fields ? "--ena_metadata_fields ${fields}" : '' @@ -24,10 +24,5 @@ process SRA_IDS_TO_RUNINFO { id.txt \\ ${id}.runinfo.tsv \\ $metadata_fields - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - python: \$(python --version | sed 's/Python //g') - END_VERSIONS """ } diff --git a/modules/local/sra_ids_to_runinfo/nextflow.config b/modules/local/sra_ids_to_runinfo/nextflow.config deleted file mode 100644 index 9b9d0b16..00000000 --- a/modules/local/sra_ids_to_runinfo/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - withName: 'SRA_IDS_TO_RUNINFO' { - publishDir = [ - path: { "${params.outdir}/metadata" }, - enabled: false - ] - } -} diff --git a/modules/local/sra_runinfo_to_ftp/main.nf b/modules/local/sra_runinfo_to_ftp/main.nf index 9c83cf53..40f6a03c 100644 --- a/modules/local/sra_runinfo_to_ftp/main.nf +++ b/modules/local/sra_runinfo_to_ftp/main.nf @@ -11,17 +11,12 @@ process SRA_RUNINFO_TO_FTP { output: path "*.tsv" , emit: tsv - path "versions.yml", emit: versions + tuple val("${task.process}"), val('python'), eval("python --version | sed 's/Python //g'"), topic: versions script: """ sra_runinfo_to_ftp.py \\ ${runinfo.join(',')} \\ ${runinfo.toString().tokenize(".")[0]}.runinfo_ftp.tsv - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - python: \$(python --version | sed 's/Python //g') - END_VERSIONS """ } diff --git a/modules/local/sra_runinfo_to_ftp/nextflow.config b/modules/local/sra_runinfo_to_ftp/nextflow.config deleted file mode 100644 index 43263648..00000000 --- a/modules/local/sra_runinfo_to_ftp/nextflow.config +++ /dev/null @@ -1,9 +0,0 @@ -process { - withName: 'SRA_RUNINFO_TO_FTP' { - publishDir = [ - path: { "${params.outdir}/metadata" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } -} diff --git a/modules/local/sra_to_samplesheet/nextflow.config b/modules/local/sra_to_samplesheet/nextflow.config deleted file mode 100644 index da241c1a..00000000 --- a/modules/local/sra_to_samplesheet/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - withName: SRA_TO_SAMPLESHEET { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - enabled: false - ] - } -} diff --git a/modules/nf-core/custom/sratoolsncbisettings/main.nf b/modules/nf-core/custom/sratoolsncbisettings/main.nf index 577117ed..7dcb66e9 100644 --- a/modules/nf-core/custom/sratoolsncbisettings/main.nf +++ b/modules/nf-core/custom/sratoolsncbisettings/main.nf @@ -12,7 +12,7 @@ process CUSTOM_SRATOOLSNCBISETTINGS { output: path('*.mkfg') , emit: ncbi_settings - path 'versions.yml', emit: versions + tuple val("${task.process}"), val('sratools'), eval("vdb-config --version 2>&1 | grep -Eo '[0-9.]+'"), topic: versions when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/custom/sratoolsncbisettings/templates/detect_ncbi_settings.sh b/modules/nf-core/custom/sratoolsncbisettings/templates/detect_ncbi_settings.sh index cfe3a324..b553659b 100644 --- a/modules/nf-core/custom/sratoolsncbisettings/templates/detect_ncbi_settings.sh +++ b/modules/nf-core/custom/sratoolsncbisettings/templates/detect_ncbi_settings.sh @@ -38,8 +38,3 @@ else fi cp "${NCBI_SETTINGS}" ./ fi - -cat <<-END_VERSIONS > versions.yml -"!{task.process}": - sratools: $(vdb-config --version 2>&1 | grep -Eo '[0-9.]+') -END_VERSIONS diff --git a/modules/nf-core/sratools/fasterqdump/main.nf b/modules/nf-core/sratools/fasterqdump/main.nf index e7cf157a..4fdd07fe 100644 --- a/modules/nf-core/sratools/fasterqdump/main.nf +++ b/modules/nf-core/sratools/fasterqdump/main.nf @@ -14,7 +14,8 @@ process SRATOOLS_FASTERQDUMP { output: tuple val(meta), path('*.fastq.gz'), emit: reads - path "versions.yml" , emit: versions + tuple val("${task.process}"), val('sratools'), eval("fasterq-dump --version 2>&1 | grep -Eo '[0-9.]+'"), topic: versions + tuple val("${task.process}"), val('pigz'), eval("pigz --version 2>&1 | sed 's/pigz //g'"), topic: versions when: task.ext.when == null || task.ext.when @@ -45,11 +46,5 @@ process SRATOOLS_FASTERQDUMP { --no-name \\ --processes $task.cpus \\ *.fastq - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - sratools: \$(fasterq-dump --version 2>&1 | grep -Eo '[0-9.]+') - pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) - END_VERSIONS """ } diff --git a/modules/nf-core/sratools/fasterqdump/nextflow.config b/modules/nf-core/sratools/fasterqdump/nextflow.config index f98b140d..7e1649d1 100644 --- a/modules/nf-core/sratools/fasterqdump/nextflow.config +++ b/modules/nf-core/sratools/fasterqdump/nextflow.config @@ -1,10 +1,5 @@ process { withName: SRATOOLS_FASTERQDUMP { ext.args = '--split-files --include-technical' - publishDir = [ - path: { "${params.outdir}/fastq" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz" - ] } } \ No newline at end of file diff --git a/modules/nf-core/sratools/prefetch/main.nf b/modules/nf-core/sratools/prefetch/main.nf index 74838d53..ac9d1e80 100644 --- a/modules/nf-core/sratools/prefetch/main.nf +++ b/modules/nf-core/sratools/prefetch/main.nf @@ -14,7 +14,7 @@ process SRATOOLS_PREFETCH { output: tuple val(meta), path(id, type: 'dir'), emit: sra - path 'versions.yml' , emit: versions + tuple val("${task.process}"), val('sratools'), eval("prefetch --version 2>&1 | grep -Eo '[0-9.]+'"), topic: versions when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/sratools/prefetch/nextflow.config b/modules/nf-core/sratools/prefetch/nextflow.config deleted file mode 100644 index a2ca8848..00000000 --- a/modules/nf-core/sratools/prefetch/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - withName: SRATOOLS_PREFETCH { - publishDir = [ - path: { "${params.outdir}/sra" }, - enabled: false - ] - } -} \ No newline at end of file diff --git a/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh b/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh index bfee6070..395593d6 100755 --- a/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh +++ b/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh @@ -57,9 +57,3 @@ if grep -q "checksums missing" vdb-validate_result.txt; then exit 1 fi fi - -cat <<-END_VERSIONS > versions.yml -"!{task.process}": - sratools: $(prefetch --version 2>&1 | grep -Eo '[0-9.]+') - curl: $(curl --version | head -n 1 | sed 's/^curl //; s/ .*$//') -END_VERSIONS diff --git a/modules/nf-core/untar/main.nf b/modules/nf-core/untar/main.nf index 8a75bb95..de3a38c2 100644 --- a/modules/nf-core/untar/main.nf +++ b/modules/nf-core/untar/main.nf @@ -12,7 +12,7 @@ process UNTAR { output: tuple val(meta), path("$prefix"), emit: untar - path "versions.yml" , emit: versions + tuple val("${task.process}"), val('untar'), eval("echo \$(tar --version 2>&1) | sed 's/^.*(GNU tar) //; s/ Copyright.*\$//'"), topic: versions when: task.ext.when == null || task.ext.when @@ -42,11 +42,6 @@ process UNTAR { $archive \\ $args2 fi - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - untar: \$(echo \$(tar --version 2>&1) | sed 's/^.*(GNU tar) //; s/ Copyright.*\$//') - END_VERSIONS """ stub: @@ -54,10 +49,5 @@ process UNTAR { """ mkdir $prefix touch ${prefix}/file.txt - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - untar: \$(echo \$(tar --version 2>&1) | sed 's/^.*(GNU tar) //; s/ Copyright.*\$//') - END_VERSIONS """ } diff --git a/nextflow.config b/nextflow.config index b69fc9b5..45283e3f 100644 --- a/nextflow.config +++ b/nextflow.config @@ -33,6 +33,7 @@ params { version = false modules_testdata_base_path = 's3://ngi-igenomes/testdata/nf-core/modules/' pipelines_testdata_base_path = 's3://ngi-igenomes/testdata/nf-core/pipelines/fetchngs/1.15.0/' + trace_report_suffix = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') // Config options config_profile_name = null @@ -197,22 +198,25 @@ set -C # No clobber - prevent output redirection from overwriting files. // Disable process selector warnings by default. Use debug profile to enable warnings. nextflow.enable.configProcessNamesValidation = false -def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') +// workflow outputs +outputDir = params.outdir +workflow.output.mode = params.publish_dir_mode + timeline { enabled = true - file = "${params.outdir}/pipeline_info/execution_timeline_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/execution_timeline_${params.trace_report_suffix}.html" } report { enabled = true - file = "${params.outdir}/pipeline_info/execution_report_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/execution_report_${params.trace_report_suffix}.html" } trace { enabled = true - file = "${params.outdir}/pipeline_info/execution_trace_${trace_timestamp}.txt" + file = "${params.outdir}/pipeline_info/execution_trace_${params.trace_report_suffix}.txt" } dag { enabled = true - file = "${params.outdir}/pipeline_info/pipeline_dag_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/pipeline_dag_${params.trace_report_suffix}.html" } manifest { @@ -221,7 +225,7 @@ manifest { homePage = 'https://github.com/nf-core/fetchngs' description = """Pipeline to fetch metadata and raw FastQ files from public databases""" mainScript = 'main.nf' - nextflowVersion = '!>=24.04.2' + nextflowVersion = '!>=24.10.2' version = '1.13.0dev' doi = '10.5281/zenodo.5070524' } diff --git a/nextflow_schema.json b/nextflow_schema.json index ba7c196b..f58a2358 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -199,6 +199,12 @@ "description": "Base URL or local path to location of pipeline test dataset files", "default": "s3://ngi-igenomes/testdata/nf-core/pipelines/fetchngs/1.15.0/", "hidden": true + }, + "trace_report_suffix": { + "type": "string", + "fa_icon": "far calendar", + "description": "Suffix to add to the trace report filename. Default is the date and time in the format yyyy-MM-dd_HH-mm-ss.", + "hidden": true } } }, diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/main.nf b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/main.nf index fbeacf4a..f29e99d5 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/main.nf +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/main.nf @@ -11,29 +11,22 @@ workflow FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS { ch_dbgap_key // channel: [ path(dbgap_key) ] main: - - ch_versions = Channel.empty() - // // Detect existing NCBI user settings or create new ones. // CUSTOM_SRATOOLSNCBISETTINGS ( ch_sra_ids.collect() ) ch_ncbi_settings = CUSTOM_SRATOOLSNCBISETTINGS.out.ncbi_settings - ch_versions = ch_versions.mix(CUSTOM_SRATOOLSNCBISETTINGS.out.versions) // // Prefetch sequencing reads in SRA format. // SRATOOLS_PREFETCH ( ch_sra_ids, ch_ncbi_settings, ch_dbgap_key ) - ch_versions = ch_versions.mix(SRATOOLS_PREFETCH.out.versions.first()) // // Convert the SRA format into one or more compressed FASTQ files. // SRATOOLS_FASTERQDUMP ( SRATOOLS_PREFETCH.out.sra, ch_ncbi_settings, ch_dbgap_key ) - ch_versions = ch_versions.mix(SRATOOLS_FASTERQDUMP.out.versions.first()) emit: reads = SRATOOLS_FASTERQDUMP.out.reads // channel: [ val(meta), [ reads ] ] - versions = ch_versions // channel: [ versions.yml ] } diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config index de803a38..187faf6d 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config @@ -1,2 +1 @@ -includeConfig '../../../modules/nf-core/sratools/prefetch/nextflow.config' includeConfig '../../../modules/nf-core/sratools/fasterqdump/nextflow.config' diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf index 0fcbf7b3..46de9941 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -77,8 +77,7 @@ def dumpParametersToJSON(outdir) { def jsonStr = groovy.json.JsonOutput.toJson(params) temp_pf.text = groovy.json.JsonOutput.prettyPrint(jsonStr) - nextflow.extension.FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") - temp_pf.delete() + temp_pf.toPath().copyTo("${outdir}/pipeline_info/params_${timestamp}.json") } // diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 5cb7bafe..58a03b7c 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -89,31 +89,30 @@ def getWorkflowVersion() { return version_string } -// -// Get software versions for pipeline -// -def processVersionsFromYAML(yaml_file) { - def yaml = new org.yaml.snakeyaml.Yaml() - def versions = yaml.load(yaml_file).collectEntries { k, v -> [k.tokenize(':')[-1], v] } - return yaml.dumpAsMap(versions).trim() -} - // // Get workflow version for pipeline // def workflowVersionToYAML() { - return """ - Workflow: - ${workflow.manifest.name}: ${getWorkflowVersion()} - Nextflow: ${workflow.nextflow.version} - """.stripIndent().trim() + return Channel.of( + [ 'Workflow', workflow.manifest.name, getWorkflowVersion() ], + [ 'Workflow', 'Nextflow', workflow.nextflow.version ] + ) } // // Get channel of software versions used in pipeline in YAML format // -def softwareVersionsToYAML(ch_versions) { - return ch_versions.unique().map { version -> processVersionsFromYAML(version) }.unique().mix(Channel.of(workflowVersionToYAML())) +def softwareVersionsToYAML() { + return Channel.topic('versions') + .unique() + .mix(workflowVersionToYAML()) + .map { process, name, version -> + [ + (process.tokenize(':').last()): [ + (name): version + ] + ] + } } // @@ -340,7 +339,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi def email_html = html_template.toString() // Render the sendmail template - def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as MemoryUnit def smail_fields = [email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes()] def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") def sendmail_template = engine.createTemplate(sf).make(smail_fields) @@ -350,8 +349,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi def colors = logColours(monochrome_logs) as Map if (email_address) { try { - if (plaintext_email) { -new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') } + if (plaintext_email) { throw new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') } // Try to send HTML e-mail using sendmail def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") sendmail_tf.withWriter { w -> w << sendmail_html } @@ -369,13 +367,13 @@ new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') // Write summary e-mail HTML to a file def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") output_hf.withWriter { w -> w << email_html } - nextflow.extension.FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html") + output_hf.toPath().copyTo("${outdir}/pipeline_info/pipeline_report.html") output_hf.delete() // Write summary e-mail TXT to a file def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } - nextflow.extension.FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt") + output_tf.toPath().copyTo("${outdir}/pipeline_info/pipeline_report.txt") output_tf.delete() } diff --git a/workflows/sra/main.nf b/workflows/sra/main.nf index 0c8cac0c..03273ab5 100644 --- a/workflows/sra/main.nf +++ b/workflows/sra/main.nf @@ -9,8 +9,6 @@ include { SRA_FASTQ_FTP } from '../../modules/local/sra_fastq_ftp' include { SRA_IDS_TO_RUNINFO } from '../../modules/local/sra_ids_to_runinfo' include { SRA_RUNINFO_TO_FTP } from '../../modules/local/sra_runinfo_to_ftp' include { ASPERA_CLI } from '../../modules/local/aspera_cli' -include { SRA_TO_SAMPLESHEET } from '../../modules/local/sra_to_samplesheet' -include { softwareVersionsToYAML } from '../../subworkflows/nf-core/utils_nfcore_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -29,19 +27,20 @@ include { FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS } from '../../subworkflow workflow SRA { take: - ids // channel: [ ids ] + ids // channel: [ ids ] + ena_metadata_fields // string + download_method // enum: 'aspera' | 'ftp' | 'sratools' + skip_fastq_download // boolean + dbgap_key // string main: - ch_versions = Channel.empty() - // // MODULE: Get SRA run information for public database ids // SRA_IDS_TO_RUNINFO ( ids, - params.ena_metadata_fields ?: '' + ena_metadata_fields ) - ch_versions = ch_versions.mix(SRA_IDS_TO_RUNINFO.out.versions.first()) // // MODULE: Parse SRA run information, create file containing FTP links and read into workflow as [ meta, [reads] ] @@ -49,7 +48,6 @@ workflow SRA { SRA_RUNINFO_TO_FTP ( SRA_IDS_TO_RUNINFO.out.tsv ) - ch_versions = ch_versions.mix(SRA_RUNINFO_TO_FTP.out.versions.first()) SRA_RUNINFO_TO_FTP .out @@ -64,27 +62,27 @@ workflow SRA { .unique() .set { ch_sra_metadata } - if (!params.skip_fastq_download) { + if (!skip_fastq_download) { ch_sra_metadata .branch { meta -> - def download_method = 'ftp' + def method = 'ftp' // meta.fastq_aspera is a metadata string with ENA fasp links supported by Aspera // For single-end: 'fasp.sra.ebi.ac.uk:/vol1/fastq/ERR116/006/ERR1160846/ERR1160846.fastq.gz' // For paired-end: 'fasp.sra.ebi.ac.uk:/vol1/fastq/SRR130/020/SRR13055520/SRR13055520_1.fastq.gz;fasp.sra.ebi.ac.uk:/vol1/fastq/SRR130/020/SRR13055520/SRR13055520_2.fastq.gz' - if (meta.fastq_aspera && params.download_method == 'aspera') { - download_method = 'aspera' + if (meta.fastq_aspera && download_method == 'aspera') { + method = 'aspera' } - if ((!meta.fastq_aspera && !meta.fastq_1) || params.download_method == 'sratools') { - download_method = 'sratools' + if ((!meta.fastq_aspera && !meta.fastq_1) || download_method == 'sratools') { + method = 'sratools' } - aspera: download_method == 'aspera' + aspera: method == 'aspera' return [ meta, meta.fastq_aspera.tokenize(';').take(2) ] - ftp: download_method == 'ftp' + ftp: method == 'ftp' return [ meta, [ meta.fastq_1, meta.fastq_2 ] ] - sratools: download_method == 'sratools' + sratools: method == 'sratools' return [ meta, meta.run_accession ] } .set { ch_sra_reads } @@ -95,16 +93,14 @@ workflow SRA { SRA_FASTQ_FTP ( ch_sra_reads.ftp ) - ch_versions = ch_versions.mix(SRA_FASTQ_FTP.out.versions.first()) // // SUBWORKFLOW: Download sequencing reads without FTP links using sra-tools. // FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS ( ch_sra_reads.sratools, - params.dbgap_key ? file(params.dbgap_key, checkIfExists: true) : [] + dbgap_key ? file(dbgap_key, checkIfExists: true) : [] ) - ch_versions = ch_versions.mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.versions.first()) // // MODULE: If Aspera link is provided in run information then download FastQ directly via Aspera CLI and validate with md5sums @@ -113,80 +109,37 @@ workflow SRA { ch_sra_reads.aspera, 'era-fasp' ) - ch_versions = ch_versions.mix(ASPERA_CLI.out.versions.first()) // Isolate FASTQ channel which will be added to emit block - SRA_FASTQ_FTP - .out - .fastq + ch_fastq = SRA_FASTQ_FTP.out.fastq .mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.reads) .mix(ASPERA_CLI.out.fastq) - .map { - meta, fastq -> - def reads = fastq instanceof List ? fastq.flatten() : [ fastq ] - def meta_clone = meta.clone() - meta_clone.fastq_1 = reads[0] ? "${params.outdir}/fastq/${reads[0].getName()}" : '' - meta_clone.fastq_2 = reads[1] && !meta.single_end ? "${params.outdir}/fastq/${reads[1].getName()}" : '' + ch_md5 = SRA_FASTQ_FTP.out.md5 + .mix(ASPERA_CLI.out.md5) - return meta_clone + ch_samples = ch_fastq + .join(ch_md5, remainder: true) + .map { + meta, fastq, md5 -> + def reads = fastq instanceof List ? fastq.flatten() : [ fastq ] + meta + [ + fastq_1: reads[0], + fastq_2: reads[1] && !meta.single_end ? reads[1] : null, + md5_1: md5[0], + md5_2: md5[1] && !meta.single_end ? md5[1] : null, + ] } - .set { ch_sra_metadata } } - - // - // MODULE: Stage FastQ files downloaded by SRA together and auto-create a samplesheet - // - SRA_TO_SAMPLESHEET ( - ch_sra_metadata, - params.nf_core_pipeline ?: '', - params.nf_core_rnaseq_strandedness ?: 'auto', - params.sample_mapping_fields - ) - - // Merge samplesheets and mapping files across all samples - SRA_TO_SAMPLESHEET - .out - .samplesheet - .map { it[1] } - .collectFile(name:'tmp_samplesheet.csv', newLine: true, keepHeader: true, sort: { it.baseName }) - .map { it.text.tokenize('\n').join('\n') } - .collectFile(name:'samplesheet.csv', storeDir: "${params.outdir}/samplesheet") - .set { ch_samplesheet } - - SRA_TO_SAMPLESHEET - .out - .mappings - .map { it[1] } - .collectFile(name:'tmp_id_mappings.csv', newLine: true, keepHeader: true, sort: { it.baseName }) - .map { it.text.tokenize('\n').join('\n') } - .collectFile(name:'id_mappings.csv', storeDir: "${params.outdir}/samplesheet") - .set { ch_mappings } - - // - // MODULE: Create a MutiQC config file with sample name mappings - // - ch_sample_mappings_yml = Channel.empty() - if (params.sample_mapping_fields) { - MULTIQC_MAPPINGS_CONFIG ( - ch_mappings - ) - ch_versions = ch_versions.mix(MULTIQC_MAPPINGS_CONFIG.out.versions) - ch_sample_mappings_yml = MULTIQC_MAPPINGS_CONFIG.out.yml + else { + ch_samples = Channel.empty() } - // - // Collate and save software versions - // - softwareVersionsToYAML(ch_versions) - .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_fetchngs_software_mqc_versions.yml', sort: true, newLine: true) - emit: - samplesheet = ch_samplesheet - mappings = ch_mappings - sample_mappings = ch_sample_mappings_yml - sra_metadata = ch_sra_metadata - versions = ch_versions.unique() + samples = ch_samples + + publish: + SRA_RUNINFO_TO_FTP.out.tsv >> 'metadata' } /* diff --git a/workflows/sra/nextflow.config b/workflows/sra/nextflow.config index d242c238..522b05b8 100644 --- a/workflows/sra/nextflow.config +++ b/workflows/sra/nextflow.config @@ -1,8 +1,3 @@ -includeConfig "../../modules/local/multiqc_mappings_config/nextflow.config" includeConfig "../../modules/local/aspera_cli/nextflow.config" includeConfig "../../modules/local/sra_fastq_ftp/nextflow.config" -includeConfig "../../modules/local/sra_ids_to_runinfo/nextflow.config" -includeConfig "../../modules/local/sra_runinfo_to_ftp/nextflow.config" -includeConfig "../../modules/local/sra_to_samplesheet/nextflow.config" -includeConfig "../../modules/nf-core/sratools/prefetch/nextflow.config" includeConfig "../../subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config"