From 5978db526e4feba05bf5cf0863cb7b5d12f34f7c Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Thu, 25 Jul 2024 09:14:42 -0500 Subject: [PATCH 01/16] Finalize workflow output definition Signed-off-by: Ben Sherman --- docs/cli.md | 3 + docs/config.md | 112 ++++++++++---- docs/workflow.md | 137 ++++-------------- .../src/main/groovy/nextflow/Session.groovy | 10 +- .../nextflow/ast/NextflowDSLImpl.groovy | 26 ---- .../main/groovy/nextflow/cli/CmdRun.groovy | 3 + .../nextflow/config/ConfigBuilder.groovy | 4 + .../nextflow/extension/PublishOp.groovy | 14 +- .../nextflow/processor/PublishDir.groovy | 2 +- .../groovy/nextflow/script/BaseScript.groovy | 2 - .../groovy/nextflow/script/OutputDsl.groovy | 85 +++-------- .../nextflow/script/ProcessConfig.groovy | 19 --- .../groovy/nextflow/script/ProcessDef.groovy | 9 -- .../nextflow/script/WorkflowMetadata.groovy | 6 + tests/output-dsl.nf | 17 +-- 15 files changed, 180 insertions(+), 269 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index df93409d7d..17e58e6c6b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1193,6 +1193,9 @@ The `run` command is used to execute a local pipeline script or remote pipeline `-offline` : Do not check for remote project updates. +`-o, -output-dir` (`results`) +: Directory where workflow outputs are stored. + `-params-file` : Load script parameters from a JSON/YAML file. diff --git a/docs/config.md b/docs/config.md index ba97797cdb..ff1431df15 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1274,32 +1274,6 @@ manifest { Read the {ref}`sharing-page` page to learn how to publish your pipeline to GitHub, BitBucket or GitLab. -(config-nextflow)= - -### Scope `nextflow` - -The `nextflow` scope provides configuration options for the Nextflow runtime. - -`nextflow.publish.retryPolicy.delay` -: :::{versionadded} 24.03.0-edge - ::: -: Delay when retrying a failed publish operation (default: `350ms`). - -`nextflow.publish.retryPolicy.jitter` -: :::{versionadded} 24.03.0-edge - ::: -: Jitter value when retrying a failed publish operation (default: `0.25`). - -`nextflow.publish.retryPolicy.maxAttempt` -: :::{versionadded} 24.03.0-edge - ::: -: Max attempts when retrying a failed publish operation (default: `5`). - -`nextflow.publish.retryPolicy.maxDelay` -: :::{versionadded} 24.03.0-edge - ::: -: Max delay when retrying a failed publish operation (default: `90s`). - (config-notification)= ### Scope `notification` @@ -1779,8 +1753,13 @@ The following settings are available: `wave.strategy` : The strategy to be used when resolving ambiguous Wave container requirements (default: `'container,dockerfile,conda,spack'`). +(config-workflow)= + ### Scope `workflow` +:::{versionadded} 24.10.0 +::: + The `workflow` scope provides workflow execution options. `workflow.failOnIgnore` @@ -1788,6 +1767,79 @@ The `workflow` scope provides workflow execution options. ::: : When `true`, the pipeline will exit with a non-zero exit code if any failed tasks are ignored using the `ignore` error strategy. +`workflow.output.contentType` +: *Currently only supported for S3.* +: Specify the media type a.k.a. [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_Types) of published files (default: `false`). Can be a string (e.g. `'text/html'`), or `true` to infer the content type from the file extension. + +`workflow.output.enabled` +: Enable or disable publishing (default: `true`). + +`workflow.output.ignoreErrors` +: When `true`, the workflow will not fail if a file can't be published for some reason (default: `false`). + +`workflow.output.mode` +: The file publishing method (default: `'symlink'`). The following options are available: + + `'copy'` + : Copy each file into the output directory. + + `'copyNoFollow'` + : Copy each file into the output directory without following symlinks, i.e. only the link is copied. + + `'link'` + : Create a hard link in the output directory for each file. + + `'move'` + : Move each file into the output directory. + : Should only be used for files which are not used by downstream processes in the workflow. + + `'rellink'` + : Create a relative symbolic link in the output directory for each file. + + `'symlink'` + : Create an absolute symbolic link in the output directory for each output file. + +`workflow.output.overwrite` +: When `true` any existing file in the specified folder will be overwritten (default: `'standard'`). The following options are available: + + `false` + : Never overwrite existing files. + + `true` + : Always overwrite existing files. + + `'deep'` + : Overwrite existing files when the file content is different. + + `'lenient'` + : Overwrite existing files when the file size is different. + + `'standard'` + : Overwrite existing files when the file size or last modified timestamp is different. + +`workflow.output.retryPolicy.delay` +: Delay when retrying a failed publish operation (default: `350ms`). + +`workflow.output.retryPolicy.jitter` +: Jitter value when retrying a failed publish operation (default: `0.25`). + +`workflow.output.retryPolicy.maxAttempt` +: Max attempts when retrying a failed publish operation (default: `5`). + +`workflow.output.retryPolicy.maxDelay` +: Max delay when retrying a failed publish operation (default: `90s`). + +`workflow.output.storageClass` +: *Currently only supported for S3.* +: Specify the storage class for published files. + +`workflow.output.tags` +: *Currently only supported for S3.* +: Specify arbitrary tags for published files. For example: + ```groovy + tags FOO: 'hello', BAR: 'world' + ``` + (config-miscellaneous)= ### Miscellaneous @@ -1804,6 +1856,9 @@ There are additional variables that can be defined within a configuration file t `dumpHashes` : If `true`, dump task hash keys in the log file, for debugging purposes. Equivalent to the `-dump-hashes` option of the `run` command. +`outputDir` +: Defines the pipeline output directory. Equivalent to the `-output-dir` option of the `run` command. + `resume` : If `true`, enable the use of previously cached task executions. Equivalent to the `-resume` option of the `run` command. @@ -2150,11 +2205,10 @@ Some features can be enabled using the `nextflow.enable` and `nextflow.preview` `nextflow.preview.output` -: :::{versionadded} 24.04.0 +: :::{deprecated} 24.10.0 + This feature was introduced as a preview in 24.04. It has now been finalized and no longer requires the preview flag. ::: -: *Experimental: may change in a future release.* - : When `true`, enables the use of the {ref}`workflow output definition `. `nextflow.preview.recursion` diff --git a/docs/workflow.md b/docs/workflow.md index bc87b38f80..cbbbe5cc17 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -41,12 +41,12 @@ The `main:` label can be omitted if there are no `take:` or `emit:` blocks. Workflows were introduced in DSL2. If you are still using DSL1, see the {ref}`dsl1-page` page to learn how to migrate your Nextflow pipelines to DSL2. ::: -## Implicit workflow +## Entry workflow -A script can define a single workflow without a name (also known as the *implicit workflow*), which is the default entrypoint of the script. The `-entry` command line option can be used to execute a different workflow as the entrypoint at runtime. +A script can define a single workflow without a name (also known as the *entry workflow*), which is the default entrypoint of the script. The `-entry` command line option can be used to execute a different workflow as the entrypoint at runtime. :::{note} -Implicit workflow definitions are ignored when a script is included as a module. This way, a script can be written such that it can be either imported as a module or executed as a pipeline. +Entry workflow definitions are ignored when a script is included as a module. This way, a script can be written such that it can be either imported as a module or executed as a pipeline. ::: ## Named workflows @@ -82,7 +82,7 @@ workflow { ``` :::{tip} -The use of global variables and params in named workflows is discouraged because it breaks the modularity of the workflow. As a best practice, every workflow input should be explicitly defined as such in the `take:` block, and params should only be used in the implicit workflow. +The use of global variables and params in named workflows is discouraged because it breaks the modularity of the workflow. As a best practice, every workflow input should be explicitly defined as such in the `take:` block, and params should only be used in the entry workflow. ::: ## Workflow inputs (`take`) @@ -397,30 +397,29 @@ In the above snippet, the initial channel is piped to the {ref}`operator-map` op ## Publishing outputs -:::{versionadded} 24.04.0 +:::{versionadded} 24.10.0 ::: :::{note} -This feature requires the `nextflow.preview.output` feature flag to be enabled. +This feature was introduced in 24.04.0 as a preview. It can be enabled in that version using the `nextflow.preview.output` feature flag. ::: -A script may define the set of outputs that should be published by the implicit workflow, known as the workflow output definition: +A workflow may define the set of outputs that should be published: ```groovy workflow { + main: foo(bar()) -} -output { - directory 'results' + publish: + foo.out >> 'foo' + bar.out >> 'bar' } ``` -The output definition must be defined after the implicit workflow. - ### Publishing channels -Processes and workflows can each define a `publish` section which maps channels to publish targets. For example: +Workflows can define a `publish` section which maps channels to publish targets. Any channel defined in the workflow, including process and subworkflow outputs, can be published. For example: ```groovy process foo { @@ -429,39 +428,31 @@ process foo { output: path 'result.txt', emit: results - publish: - results >> 'foo' - // ... } -workflow foobar { +workflow { main: foo(data) bar(foo.out) publish: - foo.out >> 'foobar/foo' - - emit: - bar.out + foo.out.results >> 'foo' + bar.out >> 'bar' } ``` -In the above example, the output `results` of process `foo` is published to the target `foo/` by default. However, when the workflow `foobar` invokes process `foo`, it publishes `foo.out` (i.e. `foo.out.results`) to the target `foobar/foo/`, overriding the default target defined by `foo`. - -In a process, any output with an `emit` name can be published. In a workflow, any channel defined in the workflow, including process and subworkflow outputs, can be published. +In the above example, the output `results` of process `foo` is published to the target `foo`, and all outputs of process `bar` are published to the target `bar`. -:::{note} -If the publish source is a process/workflow output (e.g. `foo.out`) with multiple channels, each channel will be published. Individual output channels can also be published by index or name (e.g. `foo.out[0]` or `foo.out.results`). +:::{tip} +A workflow can override the publish targets of a subworkflow by "re-publishing" the same channels to a different target. However, the best practice is to define all publish targets in the entry workflow, so that all publish targets are defined in one place at the top-level. ::: -As shown in the example, workflows can override the publish targets of process and subworkflow outputs. This way, each process and workflow can define some sensible defaults for publishing, which can be overridden by calling workflows as needed. - By default, all files emitted by the channel will be published into the specified directory. If a channel emits list values, any files in the list (including nested lists) will also be published. For example: ```groovy workflow { + main: ch_samples = Channel.of( [ [id: 'sample1'], file('sample1.txt') ] ) @@ -471,28 +462,25 @@ workflow { } ``` -### Publish directory +### Output directory -The `directory` statement is used to set the top-level publish directory of the workflow: +The top-level output directory of a workflow run can be set using the `-output-dir` command-line option or the `outputDir` config option: ```groovy -output { - directory 'results' - - // ... -} +outputDir = 'my-results' ``` -It is optional, and it defaults to the launch directory (`workflow.launchDir`). Published files will be saved within this directory. +It defaults to `results` in the launch directory. Published files will be saved within this directory. ### Publish targets -A publish target is a name with a specific publish configuration. By default, when a channel is published to a target in the `publish:` section of a process or workflow, the target name is used as the publish path. +A publish target is a name with a specific publish configuration. By default, when a channel is published to a target in the `publish:` section of a workflow, the target name is used as the publish path. For example, given the following output definition: ```groovy workflow { + main: ch_foo = foo() ch_bar = bar(ch_foo) @@ -500,10 +488,6 @@ workflow { ch_foo >> 'foo' ch_bar >> 'bar' } - -output { - directory 'results' -} ``` The following directory structure will be created: @@ -516,18 +500,15 @@ results/ └── ... ``` -:::{note} -The trailing slash in the target name is not required; it is only used to denote that the target name is intended to be used as the publish path. -::: - :::{warning} -The target name must not begin with a slash (`/`), it should be a relative path name. +The target name should not begin or end with a slash (`/`). ::: Workflows can also disable publishing for specific channels by redirecting them to `null`: ```groovy workflow { + main: ch_foo = foo() publish: @@ -541,82 +522,23 @@ For example: ```groovy output { - directory 'results' - mode 'copy' - 'foo' { mode 'link' } } ``` -In this example, all files will be copied by default, and files published to `foo/` will be hard-linked, overriding the default option. +In this example, all files will be copied by default, and files published to `foo` will be hard-linked, overriding the default option. Available options: -`contentType` -: *Currently only supported for S3.* -: Specify the media type a.k.a. [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_Types) of published files (default: `false`). Can be a string (e.g. `'text/html'`), or `true` to infer the content type from the file extension. - `enabled` : Enable or disable publishing (default: `true`). -`ignoreErrors` -: When `true`, the workflow will not fail if a file can't be published for some reason (default: `false`). - -`mode` -: The file publishing method (default: `'symlink'`). The following options are available: - - `'copy'` - : Copy each file into the output directory. - - `'copyNoFollow'` - : Copy each file into the output directory without following symlinks, i.e. only the link is copied. - - `'link'` - : Create a hard link in the output directory for each file. - - `'move'` - : Move each file into the output directory. - : Should only be used for files which are not used by downstream processes in the workflow. - - `'rellink'` - : Create a relative symbolic link in the output directory for each file. - - `'symlink'` - : Create an absolute symbolic link in the output directory for each output file. - -`overwrite` -: When `true` any existing file in the specified folder will be overwritten (default: `'standard'`). The following options are available: - - `false` - : Never overwrite existing files. - - `true` - : Always overwrite existing files. - - `'deep'` - : Overwrite existing files when the file content is different. - - `'lenient'` - : Overwrite existing files when the file size is different. - - `'standard'` - : Overwrite existing files when the file size or last modified timestamp is different. - `path` : Specify the publish path relative to the output directory (default: the target name). Can only be specified within a target definition. -`storageClass` -: *Currently only supported for S3.* -: Specify the storage class for published files. - -`tags` -: *Currently only supported for S3.* -: Specify arbitrary tags for published files. For example: - ```groovy - tags FOO: 'hello', BAR: 'world' - ``` +See also: {ref}`config-workflow` ### Index files @@ -626,6 +548,7 @@ For example: ```groovy workflow { + main: ch_foo = Channel.of( [id: 1, name: 'foo 1'], [id: 2, name: 'foo 2'], diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index 613a0246e1..797ac9aabe 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -122,6 +122,11 @@ class Session implements ISession { */ boolean resumeMode + /** + * The folder where workflow outputs are stored + */ + Path outputDir + /** * The folder where tasks temporary files are stored */ @@ -375,8 +380,11 @@ class Session implements ISession { // -- DAG object this.dag = new DAG() + // -- init output dir + this.outputDir = FileHelper.toCanonicalPath(config.outputDir ?: 'results') + // -- init work dir - this.workDir = ((config.workDir ?: 'work') as Path).complete() + this.workDir = FileHelper.toCanonicalPath(config.workDir ?: 'work') this.setLibDir( config.libDir as String ) // -- init cloud cache path diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy index b3c16b4af7..2a48ea4eb5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy @@ -633,11 +633,6 @@ class NextflowDSLImpl implements ASTTransformation { } break - case 'publish': - if( stm instanceof ExpressionStatement ) - convertPublishMethod( stm ) - break - case 'exec': bodyLabel = currentLabel iterator.remove() @@ -1299,27 +1294,6 @@ class NextflowDSLImpl implements ASTTransformation { return false } - protected void convertPublishMethod(ExpressionStatement stmt) { - if( stmt.expression !instanceof BinaryExpression ) { - syntaxError(stmt, "Invalid process publish statement") - return - } - - final binaryX = (BinaryExpression)stmt.expression - if( binaryX.operation.type != Types.RIGHT_SHIFT ) { - syntaxError(stmt, "Invalid process publish statement") - return - } - - final left = binaryX.leftExpression - if( left !instanceof VariableExpression ) { - syntaxError(stmt, "Invalid process publish statement") - return - } - - stmt.expression = callThisX('_publish_target', args(constX(((VariableExpression)left).name), binaryX.rightExpression)) - } - protected boolean isIllegalName(String name, ASTNode node) { if( name in RESERVED_NAMES ) { unit.addError( new SyntaxException("Identifier `$name` is reserved for internal use", node.lineNumber, node.columnNumber+8) ) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy index e5d18bdb87..bb1889c456 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy @@ -107,6 +107,9 @@ class CmdRun extends CmdBase implements HubOptions { @Parameter(names=['-test'], description = 'Test a script function with the name specified') String test + @Parameter(names=['-o', '-output-dir'], description = 'Directory where workflow outputs are stored') + String outputDir + @Parameter(names=['-w', '-work-dir'], description = 'Directory where intermediate result files are stored') String workDir diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy index 08d5f3f011..841a1c9bcc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy @@ -545,6 +545,10 @@ class ConfigBuilder { if( cmdRun.stubRun ) config.stubRun = cmdRun.stubRun + // -- set the output directory + if( cmdRun.outputDir ) + config.outputDir = cmdRun.outputDir + if( cmdRun.preview ) config.preview = cmdRun.preview diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy index 12f2e9d896..fb6f033753 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy @@ -36,10 +36,12 @@ class PublishOp { private DataflowReadChannel source - private PublishDir publisher + private Map opts private Path targetDir + private Closure pathAs + private IndexOpts indexOpts private List indexRecords = [] @@ -50,8 +52,10 @@ class PublishOp { PublishOp(DataflowReadChannel source, Map opts) { this.source = source - this.publisher = PublishDir.create(opts) + this.opts = opts this.targetDir = opts.path as Path + if( opts.pathAs instanceof Closure ) + this.pathAs = opts.pathAs as Closure if( opts.index ) this.indexOpts = new IndexOpts(targetDir, opts.index as Map) } @@ -68,6 +72,10 @@ class PublishOp { protected void onNext(value) { log.trace "Publish operator received: $value" + final publisher = PublishDir.create(opts) + if( pathAs ) + publisher.saveAs = { target -> pathAs.call(target, value) } + final result = collectFiles([:], value) for( final entry : result ) { final sourceDir = entry.key @@ -84,7 +92,7 @@ class PublishOp { } protected void onComplete(nope) { - if( indexOpts && indexRecords.size() > 0 && publisher.enabled ) { + if( opts.enabled && indexOpts && indexRecords.size() > 0 ) { log.trace "Saving records to index file: ${indexRecords}" new CsvWriter(header: indexOpts.header, sep: indexOpts.sep).apply(indexRecords, indexOpts.path) session.notifyFilePublish(indexOpts.path) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index ee2a5342ee..42d3010c2b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -222,7 +222,7 @@ class PublishDir { protected void apply0(Set files) { assert path - final retryOpts = session.config.navigate('nextflow.publish.retryPolicy') as Map ?: Collections.emptyMap() + final retryOpts = session.config.navigate('workflow.output.retryPolicy') as Map ?: Collections.emptyMap() this.retryConfig = new PublishRetryConfig(retryOpts) createPublishDir() diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy index 5ec823b5a8..f2cce2cd7f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy @@ -124,8 +124,6 @@ abstract class BaseScript extends Script implements ExecutionContext { } protected output(Closure closure) { - if( !NF.outputDefinitionEnabled ) - throw new IllegalStateException("Workflow output definition requires the `nextflow.preview.output` feature flag") if( !entryFlow ) throw new IllegalStateException("Workflow output definition must be defined after the anonymous workflow") if( ExecutionStack.withinWorkflow() ) diff --git a/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy index d7bb9c4f09..eb812a1d65 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy @@ -21,6 +21,8 @@ import java.nio.file.Path import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowWriteChannel +import nextflow.Global +import nextflow.Session import nextflow.exception.ScriptRuntimeException import nextflow.extension.CH import nextflow.extension.MixOp @@ -35,65 +37,19 @@ import nextflow.file.FileHelper @CompileStatic class OutputDsl { - private Map publishConfigs = [:] - - private Path directory + private Session session = Global.session as Session - private Map defaults = [:] + private Map publishConfigs = [:] private volatile List ops = [] void directory(String directory) { - if( this.directory ) - throw new ScriptRuntimeException("Publish directory cannot be defined more than once in the workflow publish definition") - this.directory = FileHelper.toCanonicalPath(directory) - } - - void contentType(String value) { - setDefault('contentType', value) - } - - void contentType(boolean value) { - setDefault('contentType', value) - } - - void ignoreErrors(boolean value) { - setDefault('ignoreErrors', value) - } - - void mode(String value) { - setDefault('mode', value) - } - - void overwrite(boolean value) { - setDefault('overwrite', value) - } - - void overwrite(String value) { - setDefault('overwrite', value) - } - - void storageClass(String value) { - setDefault('storageClass', value) - } - - void tags(Map value) { - setDefault('tags', value) - } - - void enabled( boolean value ) { - setDefault('enabled', value) - } - - private void setDefault(String name, Object value) { - if( defaults.containsKey(name) ) - throw new ScriptRuntimeException("Default `${name}` option cannot be defined more than once in the workflow publish definition") - defaults[name] = value + throw new ScriptRuntimeException('Output directory should be set using the `outputDir` config option or `-output-dir` command line option') } void target(String name, Closure closure) { if( publishConfigs.containsKey(name) ) - throw new ScriptRuntimeException("Target '${name}' is defined more than once in the workflow publish definition") + throw new ScriptRuntimeException("Target '${name}' is defined more than once in the workflow output definition") final dsl = new TargetDsl() final cl = (Closure)closure.clone() @@ -105,6 +61,8 @@ class OutputDsl { } void build(Map targets) { + final defaults = session.config.navigate('workflow.output', Collections.emptyMap()) as Map + // construct mapping of target name -> source channels final Map> publishSources = [:] for( final source : targets.keySet() ) { @@ -122,16 +80,14 @@ class OutputDsl { final mixed = sources.size() > 1 ? new MixOp(sources.collect( ch -> CH.getReadChannel(ch) )).apply() : sources.first() - final opts = publishOptions(name, publishConfigs[name] ?: [:]) + final overrides = publishConfigs[name] ?: Collections.emptyMap() + final opts = publishOptions(name, defaults, overrides) ops << new PublishOp(CH.getReadChannel(mixed), opts).apply() } } - private Map publishOptions(String name, Map overrides) { - if( !directory ) - directory = FileHelper.toCanonicalPath('.') - + private Map publishOptions(String name, Map defaults, Map overrides) { final opts = defaults + overrides if( opts.containsKey('ignoreErrors') ) opts.failOnError = !opts.remove('ignoreErrors') @@ -139,9 +95,9 @@ class OutputDsl { opts.overwrite = 'standard' final path = opts.path as String ?: name - if( path.startsWith('/') ) - throw new ScriptRuntimeException("Invalid publish target path '${path}' -- it should be a relative path") - opts.path = directory.resolve(path) + if( path.startsWith('/') || path.endsWith('/') ) + throw new ScriptRuntimeException("Invalid publish target path '${path}' -- it should not contain a leading or trailing slash") + opts.path = session.outputDir.resolve(path) if( opts.index && !(opts.index as Map).path ) throw new ScriptRuntimeException("Index file definition for publish target '${name}' is missing `path` option") @@ -168,6 +124,10 @@ class OutputDsl { setOption('contentType', value) } + void enabled(boolean value) { + setOption('enabled', value) + } + void ignoreErrors(boolean value) { setOption('ignoreErrors', value) } @@ -197,6 +157,11 @@ class OutputDsl { setOption('path', value) } + void path(Closure value) { + setOption('path', '.') + setOption('pathAs', value) + } + void storageClass(String value) { setOption('storageClass', value) } @@ -205,10 +170,6 @@ class OutputDsl { setOption('tags', value) } - void enabled( boolean value ) { - setOption('enabled', value) - } - private void setOption(String name, Object value) { if( opts.containsKey(name) ) throw new ScriptRuntimeException("Publish option `${name}` cannot be defined more than once for a given target") diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy index a17e80eb5a..df7ce27ec6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy @@ -164,11 +164,6 @@ class ProcessConfig implements Map, Cloneable { */ private outputs = new OutputsList() - /** - * Map of default publish targets - */ - private Map publishTargets = [:] - /** * Initialize the taskConfig object with the defaults values * @@ -519,13 +514,6 @@ class ProcessConfig implements Map, Cloneable { outputs } - /** - * Typed shortcut to {@code #publishTargets} - */ - Map getPublishTargets() { - publishTargets - } - /** * Implements the process {@code debug} directive. */ @@ -663,13 +651,6 @@ class ProcessConfig implements Map, Cloneable { result } - void _publish_target(String emit, String name) { - final emitNames = outputs.collect { param -> param.channelEmitName } - if( emit !in emitNames ) - throw new IllegalArgumentException("Invalid emit name '${emit}' in publish statement, valid emits are: ${emitNames.join(', ')}") - publishTargets[emit] = name - } - /** * Defines a special *dummy* input parameter, when no inputs are * provided by the user for the current task diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy index f7e59b371e..ae95b230e3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy @@ -18,7 +18,6 @@ package nextflow.script import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Const import nextflow.Global import nextflow.Session @@ -209,14 +208,6 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef { // make a copy of the output list because execution can change it output = new ChannelOut(declaredOutputs.clone()) - // register process publish targets - for( final entry : processConfig.getPublishTargets() ) { - final emit = entry.key - final name = entry.value - final source = (DataflowWriteChannel)output.getProperty(emit) - session.publishTargets[source] = name - } - // create the executor final executor = session .executorFactory diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy index 6f99b2f1a9..fbb73fc655 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy @@ -139,6 +139,11 @@ class WorkflowMetadata { */ Path launchDir + /** + * Workflow output directory + */ + Path outputDir + /** * Workflow working directory */ @@ -257,6 +262,7 @@ class WorkflowMetadata { this.container = session.fetchContainers() this.commandLine = session.commandLine this.nextflow = NextflowMeta.instance + this.outputDir = session.outputDir this.workDir = session.workDir this.launchDir = Paths.get('.').complete() this.profile = session.profile ?: ConfigBuilder.DEFAULT_PROFILE diff --git a/tests/output-dsl.nf b/tests/output-dsl.nf index 22d9cea365..1d04654c7a 100644 --- a/tests/output-dsl.nf +++ b/tests/output-dsl.nf @@ -14,8 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -nextflow.preview.output = true - params.save_foo = true process align { @@ -59,13 +57,14 @@ process foo { } workflow { - def input = Channel.of('alpha','beta','delta') + main: + input = Channel.of('alpha','beta','delta') align(input) - def bam = align.out[0].toSortedList { it.name } - def bai = align.out[1].toSortedList { it.name } - my_combine( bam, bai ) - my_combine.out.view{ it.text } + bams = align.out[0].toSortedList { bam -> bam.name } + bais = align.out[1].toSortedList { bai -> bai.name } + my_combine( bams, bais ) + my_combine.out.view { it -> it.text } foo() @@ -76,10 +75,8 @@ workflow { } output { - directory 'results' - mode 'copy' - 'data' { + path { filename, val -> filename } index { path 'index.csv' mapper { val -> [filename: val] } From b3d5f0d52360aec6ebdaa65e61cc21952707a871 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Sun, 22 Sep 2024 20:58:13 -0500 Subject: [PATCH 02/16] Restore preview flag Signed-off-by: Ben Sherman --- docs/workflow.md | 8 ++++++-- .../src/main/groovy/nextflow/script/BaseScript.groovy | 2 ++ tests/output-dsl.nf | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/workflow.md b/docs/workflow.md index cbbbe5cc17..09557a9e3c 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -397,11 +397,15 @@ In the above snippet, the initial channel is piped to the {ref}`operator-map` op ## Publishing outputs -:::{versionadded} 24.10.0 +:::{versionadded} 24.04.0 +::: + +:::{versionchanged} 24.10.0 +A second preview version has been introduced. Read below for details. ::: :::{note} -This feature was introduced in 24.04.0 as a preview. It can be enabled in that version using the `nextflow.preview.output` feature flag. +This feature requires the `nextflow.preview.output` feature flag to be enabled. ::: A workflow may define the set of outputs that should be published: diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy index f2cce2cd7f..5ec823b5a8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy @@ -124,6 +124,8 @@ abstract class BaseScript extends Script implements ExecutionContext { } protected output(Closure closure) { + if( !NF.outputDefinitionEnabled ) + throw new IllegalStateException("Workflow output definition requires the `nextflow.preview.output` feature flag") if( !entryFlow ) throw new IllegalStateException("Workflow output definition must be defined after the anonymous workflow") if( ExecutionStack.withinWorkflow() ) diff --git a/tests/output-dsl.nf b/tests/output-dsl.nf index 1d04654c7a..f6f120c13a 100644 --- a/tests/output-dsl.nf +++ b/tests/output-dsl.nf @@ -14,6 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +nextflow.preview.output = true + params.save_foo = true process align { From 9eb960ad55e0cca50d7fe534e6bfdb2937857eb5 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Sun, 22 Sep 2024 20:58:42 -0500 Subject: [PATCH 03/16] Add warning for unused targets in output block Signed-off-by: Ben Sherman --- .../main/groovy/nextflow/script/OutputDsl.groovy | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy index eb812a1d65..1a163dd104 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy @@ -39,7 +39,7 @@ class OutputDsl { private Session session = Global.session as Session - private Map publishConfigs = [:] + private Map targetConfigs = [:] private volatile List ops = [] @@ -48,7 +48,7 @@ class OutputDsl { } void target(String name, Closure closure) { - if( publishConfigs.containsKey(name) ) + if( targetConfigs.containsKey(name) ) throw new ScriptRuntimeException("Target '${name}' is defined more than once in the workflow output definition") final dsl = new TargetDsl() @@ -57,7 +57,7 @@ class OutputDsl { cl.setDelegate(dsl) cl.call() - publishConfigs[name] = dsl.getOptions() + targetConfigs[name] = dsl.getOptions() } void build(Map targets) { @@ -74,13 +74,19 @@ class OutputDsl { publishSources[name] << source } + // validate target configs + for( final name : targetConfigs.keySet() ) { + if( name !in publishSources ) + log.warn "Publish target '${name}' was defined in the output block but not used by the workflow" + } + // create publish op (and optional index op) for each target for( final name : publishSources.keySet() ) { final sources = publishSources[name] final mixed = sources.size() > 1 ? new MixOp(sources.collect( ch -> CH.getReadChannel(ch) )).apply() : sources.first() - final overrides = publishConfigs[name] ?: Collections.emptyMap() + final overrides = targetConfigs[name] ?: Collections.emptyMap() final opts = publishOptions(name, defaults, overrides) ops << new PublishOp(CH.getReadChannel(mixed), opts).apply() From 7a7a2dbe97dd11caafe2b45586010d8e053d4a80 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Sun, 22 Sep 2024 21:09:50 -0500 Subject: [PATCH 04/16] Fix bug with enabled and index file Signed-off-by: Ben Sherman --- .../src/main/groovy/nextflow/extension/PublishOp.groovy | 2 +- .../nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy index fb6f033753..05772b60a0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy @@ -92,7 +92,7 @@ class PublishOp { } protected void onComplete(nope) { - if( opts.enabled && indexOpts && indexRecords.size() > 0 ) { + if( indexOpts && indexRecords.size() > 0 ) { log.trace "Saving records to index file: ${indexRecords}" new CsvWriter(header: indexOpts.header, sep: indexOpts.sep).apply(indexRecords, indexOpts.path) session.notifyFilePublish(indexOpts.path) diff --git a/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy index 1a163dd104..cbc076c698 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy @@ -89,7 +89,8 @@ class OutputDsl { final overrides = targetConfigs[name] ?: Collections.emptyMap() final opts = publishOptions(name, defaults, overrides) - ops << new PublishOp(CH.getReadChannel(mixed), opts).apply() + if( opts.enabled == null || opts.enabled ) + ops << new PublishOp(CH.getReadChannel(mixed), opts).apply() } } From 1be3ca7c0a0ec789cb84912e64d4cf5aca79fb92 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Sun, 22 Sep 2024 21:10:29 -0500 Subject: [PATCH 05/16] Add support for json index file Signed-off-by: Ben Sherman --- .../src/main/groovy/nextflow/extension/PublishOp.groovy | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy index 05772b60a0..7f2dab5a8f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy @@ -18,6 +18,7 @@ package nextflow.extension import java.nio.file.Path +import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel @@ -94,7 +95,13 @@ class PublishOp { protected void onComplete(nope) { if( indexOpts && indexRecords.size() > 0 ) { log.trace "Saving records to index file: ${indexRecords}" - new CsvWriter(header: indexOpts.header, sep: indexOpts.sep).apply(indexRecords, indexOpts.path) + final ext = indexOpts.path.getExtension() + if( ext == 'csv' ) + new CsvWriter(header: indexOpts.header, sep: indexOpts.sep).apply(indexRecords, indexOpts.path) + else if( ext == 'json' ) + indexOpts.path.text = DumpHelper.prettyPrint(indexRecords) + else + log.warn "Invalid extension '${ext}' for index file '${indexOpts.path}' -- should be 'csv' or 'json'" session.notifyFilePublish(indexOpts.path) } From 29094695912a029975abb60ac3bf80694bed17f9 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Sun, 22 Sep 2024 21:11:45 -0500 Subject: [PATCH 06/16] Update dynamic path syntax Signed-off-by: Ben Sherman --- .../groovy/nextflow/extension/PublishOp.groovy | 18 +++++++++++++++--- tests/output-dsl.nf | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy index 7f2dab5a8f..125913bcbe 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy @@ -73,10 +73,21 @@ class PublishOp { protected void onNext(value) { log.trace "Publish operator received: $value" - final publisher = PublishDir.create(opts) - if( pathAs ) - publisher.saveAs = { target -> pathAs.call(target, value) } + // evaluate dynamic path + final path = pathAs != null + ? pathAs.call(value) + : targetDir + if( path == null ) + return + + // create publisher + final overrides = path instanceof Closure + ? [saveAs: path] + : [path: path] + final publisher = PublishDir.create(opts + overrides) + + // publish files final result = collectFiles([:], value) for( final entry : result ) { final sourceDir = entry.key @@ -84,6 +95,7 @@ class PublishOp { publisher.apply(files, sourceDir) } + // create record for index file if( indexOpts ) { final record = indexOpts.mapper != null ? indexOpts.mapper.call(value) : value final normalized = normalizePaths(record) diff --git a/tests/output-dsl.nf b/tests/output-dsl.nf index f6f120c13a..64d3bb1185 100644 --- a/tests/output-dsl.nf +++ b/tests/output-dsl.nf @@ -78,7 +78,7 @@ workflow { output { 'data' { - path { filename, val -> filename } + path { val -> { file -> file } } index { path 'index.csv' mapper { val -> [filename: val] } From 89beee4a32c3cc91fae9cbb3ba8d5ed7157011e2 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Sun, 22 Sep 2024 22:15:42 -0500 Subject: [PATCH 07/16] Improve syntax checking of output block Signed-off-by: Ben Sherman --- .../main/groovy/nextflow/ast/NextflowDSLImpl.groovy | 12 ++++++------ .../src/main/groovy/nextflow/script/OutputDsl.groovy | 4 ---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy index 2a48ea4eb5..aa9f09c204 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy @@ -550,23 +550,23 @@ class NextflowDSLImpl implements ASTTransformation { for( Statement stmt : block.statements ) { if( stmt !instanceof ExpressionStatement ) { syntaxError(stmt, "Invalid publish target definition") - return + return } final stmtExpr = (ExpressionStatement)stmt if( stmtExpr.expression !instanceof MethodCallExpression ) { syntaxError(stmt, "Invalid publish target definition") - return + return } final call = (MethodCallExpression)stmtExpr.expression assert call.arguments instanceof ArgumentListExpression - // HACK: target definition is a method call with single closure argument - // custom parser will be able to detect more elegantly final targetArgs = (ArgumentListExpression)call.arguments - if( targetArgs.size() != 1 || targetArgs[0] !instanceof ClosureExpression ) - continue + if( targetArgs.size() != 1 || targetArgs[0] !instanceof ClosureExpression ) { + syntaxError(stmt, "Invalid publish target definition") + return + } final targetName = call.method final targetBody = (ClosureExpression)targetArgs[0] diff --git a/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy index cbc076c698..79ed13cde0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy @@ -43,10 +43,6 @@ class OutputDsl { private volatile List ops = [] - void directory(String directory) { - throw new ScriptRuntimeException('Output directory should be set using the `outputDir` config option or `-output-dir` command line option') - } - void target(String name, Closure closure) { if( targetConfigs.containsKey(name) ) throw new ScriptRuntimeException("Target '${name}' is defined more than once in the workflow output definition") From 8e255b6e803fff796cea423a91a57cf43c31e5f3 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Sun, 22 Sep 2024 22:16:35 -0500 Subject: [PATCH 08/16] Update docs Signed-off-by: Ben Sherman --- docs/workflow.md | 228 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 156 insertions(+), 72 deletions(-) diff --git a/docs/workflow.md b/docs/workflow.md index 09557a9e3c..63e48bc814 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -401,29 +401,16 @@ In the above snippet, the initial channel is piped to the {ref}`operator-map` op ::: :::{versionchanged} 24.10.0 -A second preview version has been introduced. Read below for details. +A second preview version has been introduced. Read the [migration notes](#migrating-from-first-preview) for details. ::: :::{note} This feature requires the `nextflow.preview.output` feature flag to be enabled. ::: -A workflow may define the set of outputs that should be published: +A workflow can publish outputs by sending channels to "publish targets" in the workflow `publish` section. Any channel in the workflow can be published, including process and subworkflow outputs. This approach is intended to replace the {ref}`publishDir ` directive. -```groovy -workflow { - main: - foo(bar()) - - publish: - foo.out >> 'foo' - bar.out >> 'bar' -} -``` - -### Publishing channels - -Workflows can define a `publish` section which maps channels to publish targets. Any channel defined in the workflow, including process and subworkflow outputs, can be published. For example: +Here is a basic example: ```groovy process foo { @@ -435,6 +422,10 @@ process foo { // ... } +process bar { + // ... +} + workflow { main: foo(data) @@ -446,41 +437,32 @@ workflow { } ``` -In the above example, the output `results` of process `foo` is published to the target `foo`, and all outputs of process `bar` are published to the target `bar`. +In the above example, the `results` output of process `foo` is published to the target `foo`, and all outputs of process `bar` are published to the target `bar`. + +A "publish target" is simply a name that identifies a group of related outputs. How these targets are saved into a directory structure is described in the next section. :::{tip} A workflow can override the publish targets of a subworkflow by "re-publishing" the same channels to a different target. However, the best practice is to define all publish targets in the entry workflow, so that all publish targets are defined in one place at the top-level. ::: -By default, all files emitted by the channel will be published into the specified directory. If a channel emits list values, any files in the list (including nested lists) will also be published. For example: - -```groovy -workflow { - main: - ch_samples = Channel.of( - [ [id: 'sample1'], file('sample1.txt') ] - ) - - publish: - ch_samples >> 'samples' // sample1.txt will be published -} -``` - ### Output directory The top-level output directory of a workflow run can be set using the `-output-dir` command-line option or the `outputDir` config option: +```bash +nextflow run main.nf -output-dir 'my-results' +``` + ```groovy +// nextflow.config outputDir = 'my-results' ``` -It defaults to `results` in the launch directory. Published files will be saved within this directory. - -### Publish targets +It defaults to `results` in the launch directory. All published outputs will be saved into this directory. -A publish target is a name with a specific publish configuration. By default, when a channel is published to a target in the `publish:` section of a workflow, the target name is used as the publish path. +Each publish target is saved into a subdirectory of the output directory. By default, the target name is used as the directory name. -For example, given the following output definition: +For example, given the following publish targets: ```groovy workflow { @@ -505,10 +487,24 @@ results/ ``` :::{warning} -The target name should not begin or end with a slash (`/`). +Target names cannot begin or end with a slash (`/`). ::: -Workflows can also disable publishing for specific channels by redirecting them to `null`: +By default, all files emitted by a published channel will be published into the specified directory. If a channel emits list values, each file in the list (including nested lists) will be published. For example: + +```groovy +workflow { + main: + ch_samples = Channel.of( + [ [id: 'foo'], [ file('1.txt'), file('2.txt') ] ] + ) + + publish: + ch_samples >> 'samples' // 1.txt and 2.txt will be published +} +``` + +A workflow can also disable publishing for a specific channel by redirecting it to `null`: ```groovy workflow { @@ -520,53 +516,102 @@ workflow { } ``` -Publish targets can be customized in the output definition using a set of options similar to the {ref}`process-publishdir` directive. +### Customizing outputs + +The output directory structure can be customized further in the "output block", which can be defined alongside an entry workflow. The output block consists of "target" blocks, which can be used to customize specific targets. For example: ```groovy +workflow { + // ... +} + output { 'foo' { - mode 'link' + enabled params.save_foo + path 'intermediates/foo' + } + + 'bar' { + mode 'copy' } } ``` -In this example, all files will be copied by default, and files published to `foo` will be hard-linked, overriding the default option. +This output block has the following effect: -Available options: +- The target `foo` will be published only if `params.save_foo` is enabled, and it will be published to a different path within the output directory. -`enabled` -: Enable or disable publishing (default: `true`). +- The target `bar` will publish files via copy instead of symlink. -`path` -: Specify the publish path relative to the output directory (default: the target name). Can only be specified within a target definition. +See [Reference](#reference) for all available directives in the output block. -See also: {ref}`config-workflow` +:::{tip} +The output block is only needed if you want to customize the behavior of specific targets. If you are satisfied with the default behavior and don't need to customize anything, the output block can be omitted. +::: + +### Dynamic publish path + +The `path` directive in a target block can also be a closure which defines a custom publish path for each channel value: + +```groovy +workflow { + main: + ch_fastq = Channel.of( [ [id: 'SAMP1'], file('1.fastq'), file('2.fastq') ] ) + + publish: + ch_fastq >> 'fastq' +} + +output { + 'fastq' { + path { meta, fastq_1, fastq_2 -> "fastq/${meta.id}" } + } +} +``` + +The above example will publish each channel value to a different subdirectory. In this case, each pair of FASTQ files will be published to a subdirectory based on the sample ID. + +The closure can even define a different path for each individual file by returning an inner closure, similar to the `saveAs` option of the {ref}`publishDir ` directive: + +```groovy +output { + 'fastq' { + path { meta, fastq_1, fastq_2 -> + { file -> "fastq/${meta.id}/${file.baseName}" } + } + } +} +``` + +The inner closure will be applied to each file in the channel value, in this case `fastq_1` and `fastq_2`. + +:::{tip} +A mapping closure should usually have only one parameter. However, if the incoming values are tuples, the closure can specify a parameter for each tuple element for more convenient access, also known as "destructuring" or "unpacking". +::: ### Index files -A publish target can create an index file of the values that were published. An index file is a useful way to save the metadata associated with files, and is more flexible than encoding metadata in the file path. Currently only CSV files are supported. +A publish target can create an index file of the values that were published. An index file preserves the structure of channel values, including metadata, which is simpler than encoding this information with directories and file names. The index file can be CSV (`.csv`) or JSON (`.json`). For example: ```groovy workflow { main: - ch_foo = Channel.of( - [id: 1, name: 'foo 1'], - [id: 2, name: 'foo 2'], - [id: 3, name: 'foo 3'] + ch_fastq = Channel.of( + [ [id: 1, name: 'sample 1'], '1a.fastq', '1b.fastq' ], + [ [id: 2, name: 'sample 2'], '2a.fastq', '2b.fastq' ], + [ [id: 3, name: 'sample 3'], '3a.fastq', '3b.fastq' ] ) publish: - ch_foo >> 'foo' + ch_fastq >> 'fastq' } output { - directory 'results' - - 'foo' { + 'fastq' { index { path 'index.csv' } @@ -574,36 +619,75 @@ output { } ``` -The above example will write the following CSV file to `results/foo/index.csv`: +The above example will write the following CSV file to `results/fastq/index.csv`: ```csv -"id","name" -"1","foo 1" -"2","foo 2" -"3","foo 3" +"id","name","fastq_1","fastq_2" +"1","sample 1","results/fastq/1a.fastq","results/fastq/1b.fastq" +"2","sample 2","results/fastq/2a.fastq","results/fastq/2b.fastq" +"3","sample 3","results/fastq/3a.fastq","results/fastq/3b.fastq" ``` -You can customize the index file by specifying options in a block, for example: +You can customize the index file with additional directives, for example: ```groovy index { path 'index.csv' - header ['name', 'extra_option'] + header ['id', 'fastq_1', 'fastq_1'] sep '\t' - mapper { val -> val + [extra_option: 'bar'] } + mapper { meta, fq_1, fq_2 -> meta + [fastq_1: fq_1, fastq_2: fq_2] } } ``` -The following options are available: +This example will produce the same index file as above, but with the `name` column removed and with tabs instead of commas. + +See [Reference](#reference) for the list of available index directives. + +### Migrating from first preview + +The first preview of workflow publishing was introduced in 24.04. The second preview, introduced in 24.10, made the following breaking changes: + +- The process `publish:` section has been removed. Channels should be published only in workflows, ideally the entry workflow. -`header` -: When `true`, the keys of the first record are used as the column names (default: `false`). Can also be a list of column names. +- The `directory` output directive has been replaced with the `outputDir` config option and `-output-dir` command line option, which is `results` by default. The other directives such as `mode` have been replaced with config options under `workflow.output.*`. -`mapper` -: Closure which defines how to transform each published value into a CSV record. The closure should return a list or map. By default, no transformation is applied. + In other words, only target blocks can be specified in the output block, but target blocks can still specify directives such as `mode`. + +- Target names cannot begin or end with a slash (`/`); + +### Reference + +The following directives are available in a target block: + +`index` +: Create an index file which will contain a record of each published value. + + The following directives are available in an index definition: + + `header` + : When `true`, the keys of the first record are used as the column names (default: `false`). Can also be a list of column names. Only used for `csv` files. + + `mapper` + : Closure which defines how to transform each published value into a record. The closure should return a list or map. By default, no transformation is applied. + + `path` + : The name of the index file relative to the target path (required). Can be a `csv` or `json` file. + + `sep` + : The character used to separate values (default: `','`). Only used for `csv` files. `path` -: The name of the index file relative to the target path (required). +: Specify the publish path relative to the output directory (default: the target name). Can be a path, a closure that defines a custom directory for each published value, or a closure that defines a custom path for each individual file. -`sep` -: The character used to separate values (default: `','`). +Additionally, the following options from the {ref}`workflow ` config scope can be specified as directives: +- `contentType` +- `enabled` +- `ignoreErrors` +- `mode` +- `overwrite` +- `storageClass` +- `tags` + +:::{note} +Similarly to process directives vs {ref}`process ` config options, directives in the `output` block are specified without an equals sign (`=`). +::: From 9418992b9c656aac493126fa76001e074346cccb Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Sun, 22 Sep 2024 23:13:13 -0500 Subject: [PATCH 09/16] Update tests Signed-off-by: Ben Sherman --- .../nextflow/script/OutputDslTest.groovy | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/modules/nextflow/src/test/groovy/nextflow/script/OutputDslTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/OutputDslTest.groovy index 3828ae5fc4..265672fd3f 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/OutputDslTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/OutputDslTest.groovy @@ -18,15 +18,23 @@ class OutputDslTest extends Specification { def 'should publish workflow outputs'() { given: def root = Files.createTempDirectory('test') + def outputDir = root.resolve('results') def workDir = root.resolve('work') def work1 = workDir.resolve('ab/1234'); Files.createDirectories(work1) def work2 = workDir.resolve('cd/5678'); Files.createDirectories(work2) def file1 = work1.resolve('file1.txt'); file1.text = 'Hello' def file2 = work2.resolve('file2.txt'); file2.text = 'world' - def target = root.resolve('results') and: def session = Mock(Session) { - getConfig() >> [:] + getConfig() >> [ + workflow: [ + output: [ + mode: 'symlink', + overwrite: true + ] + ] + ] + getOutputDir() >> outputDir getWorkDir() >> workDir } Global.session = session @@ -48,9 +56,6 @@ class OutputDslTest extends Specification { SysEnv.push(NXF_FILE_ROOT: root.toString()) when: - dsl.directory('results') - dsl.mode('symlink') - dsl.overwrite(true) dsl.target('bar') { path('barbar') index { @@ -67,74 +72,69 @@ class OutputDslTest extends Specification { } then: - target.resolve('foo/file1.txt').text == 'Hello' - target.resolve('barbar/file2.txt').text == 'world' - target.resolve('barbar/index.csv').text == """\ - "file2","${target}/barbar/file2.txt" + outputDir.resolve('foo/file1.txt').text == 'Hello' + outputDir.resolve('barbar/file2.txt').text == 'world' + outputDir.resolve('barbar/index.csv').text == """\ + "file2","${outputDir}/barbar/file2.txt" """.stripIndent() and: - 1 * session.notifyFilePublish(target.resolve('foo/file1.txt'), file1) - 1 * session.notifyFilePublish(target.resolve('barbar/file2.txt'), file2) - 1 * session.notifyFilePublish(target.resolve('barbar/index.csv')) + 1 * session.notifyFilePublish(outputDir.resolve('foo/file1.txt'), file1) + 1 * session.notifyFilePublish(outputDir.resolve('barbar/file2.txt'), file2) + 1 * session.notifyFilePublish(outputDir.resolve('barbar/index.csv')) cleanup: SysEnv.pop() root?.deleteDir() } - def 'should set options' () { + def 'should set target dsl' () { when: - def dsl1 = new OutputDsl() + def dsl1 = new OutputDsl.TargetDsl() then: - dsl1.@defaults == [:] + dsl1.getOptions() == [:] when: - def dsl2 = new OutputDsl() + def dsl2 = new OutputDsl.TargetDsl() and: dsl2.contentType('simple/text') + dsl2.enabled(true) dsl2.ignoreErrors(true) dsl2.mode('someMode') dsl2.overwrite(true) dsl2.storageClass('someClass') dsl2.tags([foo:'1',bar:'2']) - dsl2.enabled(true) then: - dsl2.@defaults == [ + dsl2.getOptions() == [ contentType:'simple/text', + enabled: true, ignoreErrors: true, mode: 'someMode', overwrite: true, storageClass: 'someClass', - tags: [foo:'1',bar:'2'], - enabled: true + tags: [foo:'1',bar:'2'] ] } - def 'should set target dsl' () { + def 'should set index dsl' () { when: - def dsl1 = new OutputDsl.TargetDsl() + def dsl1 = new OutputDsl.IndexDsl() then: dsl1.getOptions() == [:] when: - def dsl2 = new OutputDsl.TargetDsl() + def dsl2 = new OutputDsl.IndexDsl() + def mapper = { v -> v } and: - dsl2.contentType('simple/text') - dsl2.ignoreErrors(true) - dsl2.mode('someMode') - dsl2.overwrite(true) - dsl2.storageClass('someClass') - dsl2.tags([foo:'1',bar:'2']) - dsl2.enabled(true) + dsl2.header(true) + dsl2.mapper(mapper) + dsl2.path('path') + dsl2.sep(',') then: dsl2.getOptions() == [ - contentType:'simple/text', - ignoreErrors: true, - mode: 'someMode', - overwrite: true, - storageClass: 'someClass', - tags: [foo:'1',bar:'2'], - enabled: true + header: true, + mapper: mapper, + path: 'path', + sep: ',' ] } From 99341a78f2ddb65f3c121d3c252d368263979235 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Sun, 22 Sep 2024 23:23:40 -0500 Subject: [PATCH 10/16] update docs [ci skip] Signed-off-by: Ben Sherman --- docs/cli.md | 2 ++ docs/config.md | 7 +++++-- docs/metadata.md | 5 +++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 17e58e6c6b..2f89f87edc 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1194,6 +1194,8 @@ The `run` command is used to execute a local pipeline script or remote pipeline : Do not check for remote project updates. `-o, -output-dir` (`results`) +: :::{versionadded} 24.10.0 + ::: : Directory where workflow outputs are stored. `-params-file` diff --git a/docs/config.md b/docs/config.md index ff1431df15..829003a46f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1857,6 +1857,8 @@ There are additional variables that can be defined within a configuration file t : If `true`, dump task hash keys in the log file, for debugging purposes. Equivalent to the `-dump-hashes` option of the `run` command. `outputDir` +: :::{versionadded} 24.10.0 + ::: : Defines the pipeline output directory. Equivalent to the `-output-dir` option of the `run` command. `resume` @@ -2205,10 +2207,11 @@ Some features can be enabled using the `nextflow.enable` and `nextflow.preview` `nextflow.preview.output` -: :::{deprecated} 24.10.0 - This feature was introduced as a preview in 24.04. It has now been finalized and no longer requires the preview flag. +: :::{versionadded} 24.04.0 ::: +: *Experimental: may change in a future release.* + : When `true`, enables the use of the {ref}`workflow output definition `. `nextflow.preview.recursion` diff --git a/docs/metadata.md b/docs/metadata.md index 2294e3cbe9..e297963a2c 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -80,6 +80,11 @@ The following table lists the properties that can be accessed on the `workflow` `workflow.manifest` : Entries of the workflow manifest. +`workflow.outputDir` +: :::{versionadded} 24.10.0 + ::: +: Workflow output directory. + `workflow.preview` : :::{versionadded} 24.04.0 ::: From 04a1e9825291f978f0e331edccde3f8de00bb9a5 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Mon, 23 Sep 2024 03:00:03 -0500 Subject: [PATCH 11/16] Add onWorkflowPublish event Signed-off-by: Ben Sherman --- .../nextflow/src/main/groovy/nextflow/Session.groovy | 11 +++++++++++ .../main/groovy/nextflow/extension/PublishOp.groovy | 3 +++ .../main/groovy/nextflow/trace/TraceObserver.groovy | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index 797ac9aabe..c06201d6c7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -1118,6 +1118,17 @@ class Session implements ISession { } } + void notifyWorkflowPublish(Object value) { + for( final observer : observers ) { + try { + observer.onWorkflowPublish(value) + } + catch( Exception e ) { + log.error "Failed to invoke observer on workflow publish: $observer", e + } + } + } + void notifyFilePublish(Path destination, Path source=null) { def copy = new ArrayList(observers) for( TraceObserver observer : copy ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy index 125913bcbe..a624757fcf 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy @@ -81,6 +81,9 @@ class PublishOp { if( path == null ) return + // emit workflow publish event + session.notifyWorkflowPublish(value) + // create publisher final overrides = path instanceof Closure ? [saveAs: path] diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/TraceObserver.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/TraceObserver.groovy index 636ad03287..fd7dedc321 100644 --- a/modules/nextflow/src/main/groovy/nextflow/trace/TraceObserver.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/trace/TraceObserver.groovy @@ -122,6 +122,13 @@ trait TraceObserver { */ void onFlowError(TaskHandler handler, TraceRecord trace){} + /** + * Method that is invoked when a value is published from a channel. + * + * @param value + */ + void onWorkflowPublish(Object value){} + /** * Method that is invoke when an output file is published * into a `publishDir` folder. From 5569eec6df629f375ca66254ee0c654e6681bdbb Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Mon, 7 Oct 2024 12:01:44 -0500 Subject: [PATCH 12/16] Normalize publish paths Signed-off-by: Ben Sherman --- .../src/main/groovy/nextflow/extension/PublishOp.groovy | 2 +- .../src/main/groovy/nextflow/processor/PublishDir.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy index a624757fcf..3f10b7c16c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy @@ -180,7 +180,7 @@ class PublishOp { private Path normalizePath(Path path) { final sourceDir = getTaskDir(path) - return targetDir.resolve(sourceDir.relativize(path)) + return targetDir.resolve(sourceDir.relativize(path)).normalize() } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index 42d3010c2b..339161c23f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -317,7 +317,7 @@ class PublishDir { return } - final destination = resolveDestination(target) + final destination = resolveDestination(target).normalize() // apply tags if( this.tags!=null && destination instanceof TagAwareFile ) { From a27c6f9460af9f7cd8787b9ffa0d2d653df6e6d1 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 21 Oct 2024 14:42:50 +0200 Subject: [PATCH 13/16] Update docs/reference/config.md [ci skip] Co-authored-by: Christopher Hakkaart Signed-off-by: Paolo Di Tommaso --- docs/reference/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/config.md b/docs/reference/config.md index cd7deb5d71..6c80ce6bed 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -1623,7 +1623,7 @@ The `workflow` scope provides workflow execution options. `workflow.output.contentType` : *Currently only supported for S3.* -: Specify the media type a.k.a. [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_Types) of published files (default: `false`). Can be a string (e.g. `'text/html'`), or `true` to infer the content type from the file extension. +: Specify the media type, also known as [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/MIME_types), of published files (default: `false`). Can be a string (e.g. `'text/html'`), or `true` to infer the content type from the file extension. `workflow.output.enabled` : Enable or disable publishing (default: `true`). From b31e1c6839fdf34078a701ff7bbe6a4b8c71f77f Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Mon, 21 Oct 2024 10:13:52 -0500 Subject: [PATCH 14/16] Make e2e test comply with strict syntax Signed-off-by: Ben Sherman --- tests/output-dsl.nf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/output-dsl.nf b/tests/output-dsl.nf index 64d3bb1185..f3ed165185 100644 --- a/tests/output-dsl.nf +++ b/tests/output-dsl.nf @@ -26,6 +26,7 @@ process align { path("*.bam") path("${x}.bai") + script: """ echo ${x} > ${x}.bam echo ${x} | rev > ${x}.bai @@ -40,6 +41,7 @@ process my_combine { output: path 'result.txt' + script: """ cat $bamfile > result.txt cat $baifile >> result.txt @@ -50,6 +52,7 @@ process foo { output: path 'xxx' + script: ''' mkdir xxx touch xxx/A From 56f6afae9a29d52a7e52925930755496712b709a Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Tue, 22 Oct 2024 08:35:55 -0500 Subject: [PATCH 15/16] Add deprecation warning for `nextflow.publish` config scope Signed-off-by: Ben Sherman --- docs/reference/config.md | 24 +++---------------- .../nextflow/processor/PublishDir.groovy | 6 ++++- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/docs/reference/config.md b/docs/reference/config.md index 6c80ce6bed..ca305ddcf7 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -1231,27 +1231,9 @@ Read the {ref}`sharing-page` page to learn how to publish your pipeline to GitHu ## `nextflow` -The `nextflow` scope provides configuration options for the Nextflow runtime. - -`nextflow.publish.retryPolicy.delay` -: :::{versionadded} 24.03.0-edge - ::: -: Delay when retrying a failed publish operation (default: `350ms`). - -`nextflow.publish.retryPolicy.jitter` -: :::{versionadded} 24.03.0-edge - ::: -: Jitter value when retrying a failed publish operation (default: `0.25`). - -`nextflow.publish.retryPolicy.maxAttempt` -: :::{versionadded} 24.03.0-edge - ::: -: Max attempts when retrying a failed publish operation (default: `5`). - -`nextflow.publish.retryPolicy.maxDelay` -: :::{versionadded} 24.03.0-edge - ::: -: Max delay when retrying a failed publish operation (default: `90s`). +:::{deprecated} 24.10.0 +The `nextflow.publish` scope has been renamed to `workflow.output`. See {ref}`config-workflow` for more information. +::: (config-notification)= diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index 339161c23f..331860236f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -222,7 +222,11 @@ class PublishDir { protected void apply0(Set files) { assert path - final retryOpts = session.config.navigate('workflow.output.retryPolicy') as Map ?: Collections.emptyMap() + def retryOpts = session.config.navigate('nextflow.publish.retryPolicy') as Map + if( retryOpts != null ) + log.warn 'The `nextflow.publish` config scope has been renamed to `workflow.output`' + else + retryOpts = session.config.navigate('workflow.output.retryPolicy') as Map ?: Collections.emptyMap() this.retryConfig = new PublishRetryConfig(retryOpts) createPublishDir() From 7912bd45b68c3fae087447241d6e62385c51dfd3 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 22 Oct 2024 16:05:41 +0200 Subject: [PATCH 16/16] Minor change for sake of readability [ci fast] Signed-off-by: Paolo Di Tommaso --- .../nextflow/processor/PublishDir.groovy | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index 331860236f..4ca5764519 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -219,15 +219,19 @@ class PublishDir { return result } - protected void apply0(Set files) { - assert path - - def retryOpts = session.config.navigate('nextflow.publish.retryPolicy') as Map - if( retryOpts != null ) + protected Map getRetryOpts() { + def result = session.config.navigate('nextflow.publish.retryPolicy') as Map + if( result != null ) log.warn 'The `nextflow.publish` config scope has been renamed to `workflow.output`' else - retryOpts = session.config.navigate('workflow.output.retryPolicy') as Map ?: Collections.emptyMap() - this.retryConfig = new PublishRetryConfig(retryOpts) + result = session.config.navigate('workflow.output.retryPolicy') as Map ?: Collections.emptyMap() + return result + } + + protected void apply0(Set files) { + assert path + // setup the retry policy config to be used + this.retryConfig = new PublishRetryConfig(getRetryOpts()) createPublishDir() validatePublishMode()