From d0e747c0336d3c330b0e762a527cfe869a3c9da2 Mon Sep 17 00:00:00 2001 From: Felix Kummer Date: Mon, 4 Dec 2023 11:15:00 +0100 Subject: [PATCH] Fixed nf-core lint warnings --- .github/workflows/awsfulltest.yml | 1 - .github/workflows/ci.yml | 1 - .nf-core.yml | 5 ++++- conf/base.config | 1 - conf/modules.config | 2 +- modules/local/check_results.nf | 5 +++++ modules/local/force-generate_analysis_mask.nf | 5 +++++ modules/local/force-generate_tile_allow_list.nf | 5 +++++ modules/local/force-higher_level.nf | 9 +++++++-- modules/local/force-mosaic.nf | 11 ++++++++--- modules/local/force-preprocess.nf | 7 ++++++- modules/local/force-pyramid.nf | 9 +++++++-- modules/local/higher_level_force_config.nf | 11 ++++++++--- modules/local/merge.nf | 10 +++++++--- modules/local/preprocess_force_config.nf | 9 ++++++--- nextflow_schema.json | 9 ++------- 16 files changed, 71 insertions(+), 29 deletions(-) diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index e55fba7..280ca65 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -15,7 +15,6 @@ jobs: steps: - name: Launch workflow via tower uses: seqeralabs/action-tower-launch@v2 - # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b65ba9a..6266c1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,6 @@ jobs: version: "${{ matrix.NXF_VER }}" - name: Run pipeline with test data - # TODO nf-core: You can customise CI pipeline run tests as required # For example: adding multiple test runs with different parameters # Remember that you can parallelise this by using strategy.matrix run: | diff --git a/.nf-core.yml b/.nf-core.yml index 96b0d30..14eebe5 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -2,4 +2,7 @@ repository_type: pipeline template: prefix: nf-core skip: - - igenomes + - igenomes +lint: + files_exist: + - conf/igenomes.config diff --git a/conf/base.config b/conf/base.config index 748d140..2d99930 100644 --- a/conf/base.config +++ b/conf/base.config @@ -23,7 +23,6 @@ process { // These labels are used and recognised by default in DSL2 files hosted on nf-core/modules. // If possible, it would be nice to keep the same label naming convention when // adding in your local modules too. - // TODO nf-core: Customise requirements for specific processes. // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors withLabel:process_single { cpus = { check_max( 1 , 'cpus' ) } diff --git a/conf/modules.config b/conf/modules.config index 3758614..95b8962 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -124,7 +124,7 @@ process { publishDir = [ path: "${params.outdir}/trend/mosaic/", mode: params.publish_dir_mode, - saveAs: { it -> it.equals('versions.yml') ? null : "${params.outdir}/trend/mosaic/${task.tag}/${it.replaceAll("trend/","")}"} + saveAs: { it.equals('versions.yml') ? null : "${params.outdir}/trend/mosaic/${task.tag}/${it.replaceAll("trend/","")}"} ] } diff --git a/modules/local/check_results.nf b/modules/local/check_results.nf index 185f0db..cb57366 100644 --- a/modules/local/check_results.nf +++ b/modules/local/check_results.nf @@ -2,6 +2,8 @@ nextflow.enable.dsl = 2 process CHECK_RESULTS { + label 'process_low' + container 'docker.io/rocker/geospatial:4.3.1' input: @@ -16,6 +18,9 @@ process CHECK_RESULTS { output: path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: """ files=`find ./trend/ -maxdepth 1 -mindepth 1 -type d` diff --git a/modules/local/force-generate_analysis_mask.nf b/modules/local/force-generate_analysis_mask.nf index 90e5ba6..ba8ce54 100644 --- a/modules/local/force-generate_analysis_mask.nf +++ b/modules/local/force-generate_analysis_mask.nf @@ -2,6 +2,8 @@ nextflow.enable.dsl = 2 process FORCE_GENERATE_ANALYSIS_MASK{ + label 'process_single' + container "docker.io/davidfrantz/force:3.7.11" input: @@ -13,6 +15,9 @@ process FORCE_GENERATE_ANALYSIS_MASK{ path 'mask/*/*.tif', emit: masks path "versions.yml", emit: versions + when: + task.ext.when == null || task.ext.when + script: """ force-cube -o mask/ -s $params.resolution $aoi diff --git a/modules/local/force-generate_tile_allow_list.nf b/modules/local/force-generate_tile_allow_list.nf index 267c595..2a7bb70 100644 --- a/modules/local/force-generate_tile_allow_list.nf +++ b/modules/local/force-generate_tile_allow_list.nf @@ -2,6 +2,8 @@ nextflow.enable.dsl = 2 process FORCE_GENERATE_TILE_ALLOW_LIST{ + label 'process_single' + container "docker.io/davidfrantz/force:3.7.11" input: @@ -13,6 +15,9 @@ process FORCE_GENERATE_TILE_ALLOW_LIST{ path 'tile_allow.txt', emit: tile_allow path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: """ force-tile-extent $aoi tmp/ tile_allow.txt diff --git a/modules/local/force-higher_level.nf b/modules/local/force-higher_level.nf index f357535..8a2a644 100644 --- a/modules/local/force-higher_level.nf +++ b/modules/local/force-higher_level.nf @@ -2,16 +2,21 @@ nextflow.enable.dsl = 2 process FORCE_HIGHER_LEVEL { + label 'process_high' + container "docker.io/davidfrantz/force:3.7.11" - tag {tile} + tag { tile } input: - tuple val(tile), path( config ), path( ard ), path( aoi ), path ( datacube ), path ( endmember ) + tuple val(tile), path(config), path(ard), path(aoi), path (datacube), path (endmember) output: path 'trend/*.tif*', emit: trend_files path "versions.yml", emit: versions + when: + task.ext.when == null || task.ext.when + script: """ PARAM=$config diff --git a/modules/local/force-mosaic.nf b/modules/local/force-mosaic.nf index a8cbbae..519a5d0 100644 --- a/modules/local/force-mosaic.nf +++ b/modules/local/force-mosaic.nf @@ -2,16 +2,21 @@ nextflow.enable.dsl = 2 process FORCE_MOSAIC{ + label 'process_low' + tag { product } container "docker.io/davidfrantz/force:3.7.11" input: - tuple val( product ), path('trend/*') + tuple val(product), path('trend/*') path 'trend/datacube-definition.prj' output: - tuple val( product ), path( 'trend/*' ), emit: trend_files - path "versions.yml" , emit: versions + tuple val(product), path('trend/*'), emit: trend_files + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when script: """ diff --git a/modules/local/force-preprocess.nf b/modules/local/force-preprocess.nf index 86c02f5..999d143 100644 --- a/modules/local/force-preprocess.nf +++ b/modules/local/force-preprocess.nf @@ -1,6 +1,8 @@ nextflow.enable.dsl=2 process FORCE_PREPROCESS { + + label 'process_medium' tag { data.simpleName } container "docker.io/davidfrantz/force:3.7.11" @@ -11,9 +13,12 @@ process FORCE_PREPROCESS { output: path "**/*BOA.tif", optional:true, emit: boa_tiles path "**/*QAI.tif", optional:true, emit: qai_tiles - path "*.log" + path "*.log" , emit: log path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: """ PARAM=$conf diff --git a/modules/local/force-pyramid.nf b/modules/local/force-pyramid.nf index 143b060..98f6d5e 100644 --- a/modules/local/force-pyramid.nf +++ b/modules/local/force-pyramid.nf @@ -2,16 +2,21 @@ nextflow.enable.dsl = 2 process FORCE_PYRAMID { + label 'process_low' tag { tile } + container "docker.io/davidfrantz/force:3.7.11" input: - tuple val( tile ), path( image ) + tuple val(tile), path(image) output: - path( '**' ), emit: trends + path('**') , emit: trends path "versions.yml", emit: versions + when: + task.ext.when == null || task.ext.when + script: """ files="*.tif" diff --git a/modules/local/higher_level_force_config.nf b/modules/local/higher_level_force_config.nf index add93aa..5e9eac7 100644 --- a/modules/local/higher_level_force_config.nf +++ b/modules/local/higher_level_force_config.nf @@ -2,18 +2,23 @@ nextflow.enable.dsl = 2 process HIGHER_LEVEL_CONFIG { - container "docker.io/davidfrantz/force:3.7.11" + label 'process_single' tag { tile } + container "docker.io/davidfrantz/force:3.7.11" + input: - tuple val( tile ), path( "ard/${tile}/*" ), path( "ard/${tile}/*" ), path( "mask/${tile}/aoi.tif" ) + tuple val(tile), path("ard/${tile}/*"), path("ard/${tile}/*"), path("mask/${tile}/aoi.tif") path 'ard/datacube-definition.prj' path endmember output: - tuple val (tile), path( "trend_${tile}.prm" ), path( "ard/", includeInputs: true ), path( "mask/", includeInputs: true ), path( 'ard/datacube-definition.prj', includeInputs: true ), path( endmember, includeInputs: true ), emit: higher_level_configs_and_data + tuple val (tile), path("trend_${tile}.prm"), path("ard/", includeInputs: true), path("mask/", includeInputs: true), path('ard/datacube-definition.prj', includeInputs: true), path(endmember, includeInputs: true), emit: higher_level_configs_and_data path "versions.yml", emit: versions + when: + task.ext.when == null || task.ext.when + script: """ # generate parameterfile from scratch diff --git a/modules/local/merge.nf b/modules/local/merge.nf index fe00a5a..fb7a1b7 100644 --- a/modules/local/merge.nf +++ b/modules/local/merge.nf @@ -2,19 +2,23 @@ nextflow.enable.dsl = 2 process MERGE { + label 'process_low' tag { id } container 'docker.io/davidfrantz/force:dev' input: - val ( data_type ) // defines whether qai or boa is merged - tuple val( id ), path( 'input/?/*' ) + val (data_type) // defines whether qai or boa is merged + tuple val(id), path('input/?/*') path cube output: - tuple val( id ), path( "*.tif" ), emit: tiles_merged + tuple val(id), path("*.tif"), emit: tiles_merged path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: """ files=`find -L input/ -type f -printf "%f\\n" | sort | uniq` diff --git a/modules/local/preprocess_force_config.nf b/modules/local/preprocess_force_config.nf index 211d09a..2830ea7 100644 --- a/modules/local/preprocess_force_config.nf +++ b/modules/local/preprocess_force_config.nf @@ -2,6 +2,7 @@ nextflow.enable.dsl = 2 process PREPROCESS_CONFIG { + label 'process_single' tag { data.simpleName } container "docker.io/davidfrantz/force:3.7.11" @@ -14,9 +15,11 @@ process PREPROCESS_CONFIG { path wvdb output: - tuple path("*.prm"), path(data), path(cube), path(tile), path(dem), path(wvdb), emit: preprocess_config_and_data - path "*.prm" - path "versions.yml", emit: versions + tuple path("*.prm"), path(data), path(cube), path(tile), path(dem), path(wvdb), emit: preprocess_config_and_data + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when script: """ diff --git a/nextflow_schema.json b/nextflow_schema.json index 171f177..631007e 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -14,10 +14,10 @@ "properties": { "input": { "type": "string", - "default": "None", "fa_icon": "fas fa-satellite", "help_text": "Set this to the root directory of all satellite imagery. Directory structure should match format of data downloaded with '$ force-level1-csd\n' For concrete directory structure see ./docs/usage.md ", - "description": "Root directory of all sattelite imagery." + "description": "Root directory of all sattelite imagery.", + "mimetype": "application/x-tar" }, "input_tar": { "type": "boolean", @@ -27,7 +27,6 @@ }, "dem": { "type": "string", - "default": "None", "help_text": "Path to directory containing a subdirectory with tile-wise digital elevation files (.tif) and a virtual dataset file (.vrt file) comprising all tile-wise files from the subdirectory. \n\n\n", "fa_icon": "fas fa-mountain", "description": "Digital elevation model." @@ -40,7 +39,6 @@ }, "wvdb": { "type": "string", - "default": "None", "fa_icon": "fas fa-burn", "description": "Water vapor dataset.", "help_text": "Directory containg a number text files describing global water vapor data at different timestamps, and a coordinate order (.coo-)file containig the reference system of the water vapor data." @@ -53,21 +51,18 @@ }, "data_cube": { "type": "string", - "default": "None", "description": "Datacube definition.", "fa_icon": "fas fa-cube", "help_text": "A single .prj file describing the projection and reference grid of the generated datacube." }, "aoi": { "type": "string", - "default": "None", "fa_icon": "fas fa-chart-area", "description": "Area of interest.", "help_text": "A single vector file specifying the spacial extend of the targeted area." }, "endmember": { "type": "string", - "default": "None", "fa_icon": "fas fa-chart-line", "description": "Endmember definition.", "help_text": "A single text file specifying where lines correspond to satellite bands and columns correspond to endmembers. Values correspond to reflectance values and are separated by spaces."